---

XLingRetMethod.hpp

Go to the documentation of this file.

/*==========================================================================
 *
 *  Original source copyright (c) 2001, Carnegie Mellon University.
 *  See copyright.cmu for details.
 *  Modifications copyright (c) 2002, University of Massachusetts.
 *  See copyright.umass for details.
 *
 *==========================================================================
*/


#ifndef _XLINGRETMETHOD_HPP
#define _XLINGRETMETHOD_HPP

#include "common_headers.hpp"
#include <cmath>
#include <vector>
#include <algorithm>
#include "IndexTypes.hpp"
#include "FreqVector.hpp"
#include "UnigramLM.hpp"
#include "ScoreFunction.hpp"
#include "XLingDocModel.hpp"
#include "TextQueryRep.hpp"
#include "TextQueryRetMethod.hpp"
#include "Counter.hpp"
#include "DocUnigramCounter.hpp"
#include "PDict.hpp"
#include "TextHandlerManager.hpp"

class XLQueryTerm : public QueryTerm {
public:
  XLQueryTerm(TERMID_T tid, double  wt, const char *term, double pge,
              PDict &dic, Stemmer *stm = NULL) :
    QueryTerm(tid, wt), source(term), p_s_ge(pge), dict(dic),
    stemmer(stm) {
  }

  XLQueryTerm(const char *term, PDict &dic, Stemmer *stm = NULL) : 
    QueryTerm(0, 0), source(term), p_s_ge(0), dict(dic), stemmer(stm) {
  }

  XLQueryTerm(const XLQueryTerm &other) : QueryTerm(0,0), dict(other.dict) {
    ti = other.ti;
    w = other.w;
    p_s_ge = other.p_s_ge;
    source = other.source;
    stemmer = other.stemmer;
  }


  virtual ~XLQueryTerm() { }

  const string &getSource() const {return source;}

  const double getP_s_GE() const {
    return p_s_ge;
  }
  
  void setWeight(double wt) {
    w = wt;
  }

  void incWeight(double wt) {
    w += wt;
  }
  virtual bool operator==(const XLQueryTerm& other) const {
    return (other.source == source);
  }

  virtual XLQueryTerm& operator=(const XLQueryTerm& other)  {
    ti = other.ti;
    w = other.w;
    p_s_ge = other.p_s_ge;
    source = other.source;
    dict = other.dict;
    stemmer = other.stemmer;
    return (*this);
  }
  DictEntryVector *getTranslations() const {
    DictEntryVector *xlates = dict.getTranslations(source);
    // If no xlates, Leah's version stems the term and tries again.
    if (xlates == NULL && stemmer != NULL) {
      // porter stemmer is destructive
      char tmpTerm[512];
      strcpy(tmpTerm, source.c_str());
      string stem = stemmer->stemWord(tmpTerm);
      cerr << "getTranslations: stemming " << source << " to " << stem 
           << endl;
      xlates = dict.getTranslations(stem);
    }
    return xlates;
  }
private:
  string source;
  double p_s_ge;
  PDict &dict;  
  Stemmer *stemmer;
};


class XLingQueryModel : public QueryRep {
public:
  XLingQueryModel(const TermQuery &qry, const Index &source, 
                  bool dbS, double numSource,
                  PDict &dict, const Stopper *stp = NULL, 
                  Stemmer *stm = NULL) {
    // fill in weighted terms
    // P(e|GE)
    double pge;
    numTerms = 0;
    
    qry.startTermIteration();
    while (qry.hasMore()) {
      const Term *t = qry.nextTerm();
      // if Stopper is not NULL, test for stopwords.
      if (stp == NULL || !(stp->stopWord(t->spelling()))) {
        numTerms++;
        XLQueryTerm st(t->spelling(), dict, stm);
        iter = find(qTerms.begin(), qTerms.end(), st);
        if (iter != qTerms.end()) {
          // found it, bump count
          (*iter).incWeight(1);
        } else {
          // new term
          TERMID_T ti = source.term(t->spelling());
          if (ti>0) {
            // pge
            if (dbS) {
              pge = source.docCount(ti)/numSource;
            } else {
              pge = (source.termCount(ti)/numSource);      
            }
          } else {
            // OOV, use default pge
            // perhaps this would be better estimated with:
            //    pge = 1/(numSource + 1);
            pge = (0.000001*0.000001);
          }
          XLQueryTerm newTerm(ti, 1, t->spelling(), pge, dict, stm);
          qTerms.push_back(newTerm);
        }
      } else {
        cerr << "XLingQueryModel: " << t->spelling() 
             << " on stoplist, ignoring" << endl;
      }
      
    }
  }
  
  virtual ~XLingQueryModel() {
  }

  virtual void startIteration() const {
    iter = qTerms.begin();
  }
  virtual bool hasMore() const {
    return (iter != qTerms.end());
  }
  //  virtual XLQueryTerm &nextTerm() {
  virtual XLQueryTerm &nextTerm() const {
    return (*iter++);
  }
  virtual int getNumTerms() const {return numTerms;}
  
private:
  mutable vector<XLQueryTerm> qTerms;
  mutable vector<XLQueryTerm>::iterator iter;
  int numTerms;
};

// Should not really be a TextQueryRetMethod, as it does not score
// in a like fashion. but does take advantage of the cached doc reps.
//
//class XLingRetMethod : public TextQueryRetMethod {
class XLingRetMethod : public RetrievalMethod {
public:

  XLingRetMethod(const Index &dbIndex, const Index &background, 
                 PDict &dict, ScoreAccumulator &accumulator, 
                 double l, double b, bool cacheDR,
                 string &sBM, string &tBM, 
                 const Stopper *stp = NULL, Stemmer *stm = NULL);
  virtual ~XLingRetMethod();
  
  virtual DocumentRep *computeDocRep(DOCID_T docID);

  virtual double matchedTermWeight(TERMID_T id, double weight,
                                   const DocInfo *info, 
                                   const DocumentRep *dRep) const { 
    double d = dRep->termWeight(id,info); //P(a|D)
    double score = d * weight; //P(a|D) * P(e|a)
    return score;
  }

  virtual double adjustedScore(double origScore, double pge) const {
    return (log((lambda * origScore) + ((1 - lambda) * pge)));
  }

  virtual void scoreCollection(const QueryRep &qry, 
                               IndexedRealVector &results){
    scoreInvertedIndex(qry, results);
  }
  // Override (have to do individual doc ones too.
  virtual void scoreInvertedIndex(const QueryRep &qryRep, 
                                  IndexedRealVector &scores, 
                                  bool scoreAll = false);

  virtual QueryRep *computeQueryRep(const Query &qry) {
    if (const TermQuery *q = dynamic_cast<const TermQuery *>(&qry))
      return (new XLingQueryModel(*q, source, docBasedSourceSmooth, numSource,
                                  dictionary, stopper, stemmer));
    else LEMUR_THROW(LEMUR_RUNTIME_ERROR, "XLingRetMethod expects query of type TermQuery");
  } 

  virtual QueryRep *computeTargetKLRep(const QueryRep *qry);

  virtual double scoreDoc(const QueryRep &qry, DOCID_T docID);

  virtual void updateQuery(QueryRep &qryRep, const DocIDSet &relDocs) {}

protected:
  virtual double scoreDocVector(const XLingQueryModel &qRep, DOCID_T docID, 
                                FreqVector &docVector);

  double lambda;
  double beta;
  double numSource;
  double numTarget;
  bool docBasedSourceSmooth;
  bool docBasedTargetSmooth;
  ScoreAccumulator &scAcc; // this does not need to be passed in. Bleah.
  PDict &dictionary;
  Stemmer *stemmer; // source language
  const Stopper *stopper; // source language
  const Index &source;
  DocumentRep **docReps;
  bool cacheDocReps;
  int docRepsSize;
  ScoreAccumulator *termScores;
};

#endif /* _XLINGRETMETHOD_HPP */

---