---

SimpleKLRetMethod.hpp

Go to the documentation of this file.

/*==========================================================================
 *
 *  Original source copyright (c) 2001, Carnegie Mellon University.
 *  See copyright.cmu for details.
 *  Modifications copyright (c) 2002, University of Massachusetts.
 *  See copyright.umass for details.
 *
 *==========================================================================
*/


#ifndef _SIMPLEKLRETMETHOD_HPP
#define _SIMPLEKLRETMETHOD_HPP

#include <cmath>
#include "UnigramLM.hpp"
#include "ScoreFunction.hpp"
#include "SimpleKLDocModel.hpp"
#include "TextQueryRep.hpp"
#include "TextQueryRetMethod.hpp"
#include "Counter.hpp"
#include "DocUnigramCounter.hpp"


class SimpleKLQueryModel : public ArrayQueryRep {
public:
  SimpleKLQueryModel(const TermQuery &qry, const Index &dbIndex) : 
    ArrayQueryRep(dbIndex.termCountUnique()+1, qry, dbIndex), qm(NULL), 
    ind(dbIndex), colKLComputed(false) {
    colQLikelihood = 0;
    colQueryLikelihood();    
  }

  SimpleKLQueryModel(const Index &dbIndex) : 
    ArrayQueryRep(dbIndex.termCountUnique()+1), qm(NULL), ind(dbIndex), 
    colKLComputed(false) {
    colQLikelihood = 0;
    startIteration();
    while (hasMore()) {
      QueryTerm *qt = nextTerm();
      setCount(qt->id(), 0);
      delete qt;
    }
  }


  virtual ~SimpleKLQueryModel(){ if (qm) delete qm;}



  virtual void interpolateWith(const UnigramLM &qModel, double origModCoeff, 
                               int howManyWord, double prSumThresh=1, 
                               double prThresh=0);
  virtual double scoreConstant() const {
    return totalCount();
  }
  
  virtual void load(istream &is);

  virtual void save(ostream &os);

  virtual void clarity(ostream &os);
  virtual double clarity() const;

  double colDivergence() const {
    if (colKLComputed) {
      return colKL;
    } else {
      colKLComputed = true;
      double d=0;
      startIteration();
      while (hasMore()) {
        QueryTerm *qt=nextTerm();
        double pr = qt->weight()/(double)totalCount();
        double colPr = ((double)ind.termCount(qt->id()) /
                        (double)(ind.termCount())); // ML smoothing
        d += pr*log(pr/colPr);
        delete qt;
      }
      colKL=d;
      return d;
    }
  }

  double KLDivergence(const UnigramLM &refMod) {
    double d=0;
    startIteration();
    while (hasMore()) {
      QueryTerm *qt=nextTerm();
      double pr = qt->weight()/(double)totalCount();
      d += pr*log(pr/refMod.prob(qt->id()));
      delete qt;
    }
    return d;
  }

  double colQueryLikelihood() const {
    if (colQLikelihood == 0) {
      //Sum w in Q qtf * log(qtcf/termcount);
      COUNT_T tc = ind.termCount();
      startIteration();
      while (hasMore()) {
        QueryTerm *qt = nextTerm();
        TERMID_T id = qt->id();
        double qtf = qt->weight();
        COUNT_T qtcf = ind.termCount(id);
        double s = qtf * log((double)qtcf/(double)tc);
        colQLikelihood += s;
        delete qt;
      }
    }
    return colQLikelihood;
  }
  

protected:
  // For Query likelihood adjusted score
  mutable double colQLikelihood;
  mutable double colKL;
  mutable bool colKLComputed;

  IndexedRealVector *qm;
  const Index &ind;
};




class SimpleKLScoreFunc : public ScoreFunction {
public:
  enum SimpleKLParameter::adjustedScoreMethods adjScoreMethod;
  void setScoreMethod(enum SimpleKLParameter::adjustedScoreMethods adj) {
    adjScoreMethod = adj;
  }  
  virtual double matchedTermWeight(const QueryTerm *qTerm, 
                                   const TextQueryRep *qRep, 
                                   const DocInfo *info,
                                   const DocumentRep *dRep) const { 
    double w = qTerm->weight();
    double d = dRep->termWeight(qTerm->id(),info);
    double l = log(d);
    double score = w*l;
    /*
    cerr << "M:" << qTerm->id() <<" d:" << info->docID() << " w:" << w 
         << " d:" << d << " l:" << l << " s:" << score << endl;
    */
    return score;
    //    return (qTerm->weight()*log(dRep->termWeight(qTerm->id(),info)));
  }
  virtual double adjustedScore(double origScore, 
                               const TextQueryRep *qRep, 
                               const DocumentRep *dRep) const {
    const SimpleKLQueryModel *qm = dynamic_cast<const SimpleKLQueryModel *>(qRep);
    // this cast is unnecessary
    //    SimpleKLDocModel *dm = (SimpleKLDocModel *)dRep;
      // dynamic_cast<SimpleKLDocModel *>dRep;

    double qsc = qm->scoreConstant();
    double dsc = log(dRep->scoreConstant());
    double cql = qm->colQueryLikelihood();
    // real query likelihood
    double s = dsc * qsc + origScore + cql;
    double qsNorm = origScore/qsc;
    double qmD = qm->colDivergence();
    /*
      cerr << "A:"<< origScore << " dsc:" << dsc  << " qsc:" << qsc  
           << " cql:" << cql << " s:"  << s << endl;
    */
    switch (adjScoreMethod) {
    case SimpleKLParameter::QUERYLIKELIHOOD:
      // this is the original query likelihood scoring formula


      return s;
      //      return (origScore+log(dm->scoreConstant())*qm->scoreConstant());
    case SimpleKLParameter::CROSSENTROPY:
      // This is the normalized query-likelihood, i.e., cross-entropy
      assert(qm->scoreConstant()!=0);
      // return (origScore/qm->scoreConstant() + log(dm->scoreConstant()));
      // add the term colQueryLikelihood/qm->scoreConstant
      s = qsNorm + dsc + cql/qsc;
      return (s);
    case SimpleKLParameter::NEGATIVEKLD:
      // This is the exact (negative) KL-divergence value, i.e., -D(Mq||Md)
      assert(qm->scoreConstant()!=0);
      s = qsNorm + dsc - qmD;
      /*
      cerr << origScore << ":" << qsNorm << ":" << dsc  << ":" << qmD  << ":" << s << endl;
      */
      return s;
      //      return (origScore/qm->scoreConstant() + log(dm->scoreConstant())
      //              - qm->colDivergence());
    default:
      cerr << "unknown adjusted score method" << endl;
      return origScore;
    }
  }
  
};

class SimpleKLRetMethod : public TextQueryRetMethod {
public:

  SimpleKLRetMethod(const Index &dbIndex, const string &supportFileName, 
                    ScoreAccumulator &accumulator);
  virtual ~SimpleKLRetMethod();
  
  virtual TextQueryRep *computeTextQueryRep(const TermQuery &qry) {
    return (new SimpleKLQueryModel(qry, ind));
  }
  
  virtual DocumentRep *computeDocRep(DOCID_T docID);
  

  virtual ScoreFunction *scoreFunc() {
    return (scFunc);
  }

  virtual void updateTextQuery(TextQueryRep &origRep, 
                               const DocIDSet &relDocs);

  void setDocSmoothParam(SimpleKLParameter::DocSmoothParam &docSmthParam);
  void setQueryModelParam(SimpleKLParameter::QueryModelParam &queryModParam);

protected:

  double *mcNorm; 
  
  double *docProbMass; 
  COUNT_T *uniqueTermCount; 
  UnigramLM *collectLM; 
  DocUnigramCounter *collectLMCounter; 
  SimpleKLScoreFunc *scFunc; 



  void computeMixtureFBModel(SimpleKLQueryModel &origRep, 
                             const DocIDSet & relDocs);
  void computeDivMinFBModel(SimpleKLQueryModel &origRep, 
                            const DocIDSet &relDocs);
  void computeMarkovChainFBModel(SimpleKLQueryModel &origRep, 
                                 const DocIDSet &relDocs) ;
  void computeRM1FBModel(SimpleKLQueryModel &origRep, 
                         const DocIDSet & relDocs);
  void computeRM2FBModel(SimpleKLQueryModel &origRep, 
                         const DocIDSet & relDocs);

  SimpleKLParameter::DocSmoothParam docParam;
  SimpleKLParameter::QueryModelParam qryParam;

  void loadSupportFile();
  const string supportFile;
};


inline  void SimpleKLRetMethod::setDocSmoothParam(SimpleKLParameter::DocSmoothParam &docSmthParam)
{
  docParam = docSmthParam;
  loadSupportFile();
}

inline  void SimpleKLRetMethod::setQueryModelParam(SimpleKLParameter::QueryModelParam &queryModParam)
{
  qryParam = queryModParam;
  // add a parameter to the score function.
  // isn't available in the constructor.
  scFunc->setScoreMethod(qryParam.adjScoreMethod);
  loadSupportFile();
}

#endif /* _SIMPLEKLRETMETHOD_HPP */

---