---

UnigramLM.hpp

Go to the documentation of this file.

/*==========================================================================
 * Copyright (c) 2001 Carnegie Mellon University.  All Rights Reserved.
 *
 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval
 * is subject to the terms of the software license set forth in the LICENSE
 * file included with this software, and also available at
 * http://www.lemurproject.org/license.html
 *
 *==========================================================================
*/


#ifndef _UNIGRAMLM_HPP
#define _UNIGRAMLM_HPP

#include "Counter.hpp"
#include "Exception.hpp"
#include "IndexTypes.hpp"
#include <cstring>

class UnigramLM {
public:
  virtual double prob(TERMID_T wordIndex) const = 0;
  virtual const string lexiconID() const= 0;

  virtual void startIteration() const = 0;
  virtual bool hasMore() const = 0;
  virtual void nextWordProb(TERMID_T &wordIndex, double &prob) const = 0;
};



class SmoothedMLEstimator : public UnigramLM {
public:
  SmoothedMLEstimator(const Counter &counter, const string &lexiconID) : ct(counter), lexID(lexiconID) {}
  virtual ~SmoothedMLEstimator() {}

  virtual double prob(TERMID_T wordIndex) const {
    return (probEstimate(wordIndex, ct.count(wordIndex),ct.sum()));
  }

  virtual void startIteration() const {
    ct.startIteration();
  }

  virtual bool hasMore() const {
    return ct.hasMore();
  }

  virtual void nextWordProb(TERMID_T &wordIndex, double &prob) const{
    double count;
    //dmf FIXME
    ct.nextCount((int&)wordIndex, count);
    prob = probEstimate(wordIndex, count, ct.sum());
  }
  
  virtual const string lexiconID() const { return lexID;}

  virtual double probEstimate(TERMID_T wordIndex, double wdCount, double sumCount) const=0;

protected:
  const Counter &ct;
  const string lexID;
};
  

class MLUnigramLM : public SmoothedMLEstimator { 
public:
  MLUnigramLM(const Counter & counter, const string &lexiconID) : SmoothedMLEstimator(counter, lexiconID) {};
  virtual ~MLUnigramLM() {}
  
  virtual double probEstimate(TERMID_T wordIndex, double count, double sum) const{
    return (count/sum);
  }
};

class LaplaceUnigramLM : public SmoothedMLEstimator { 
public:
  LaplaceUnigramLM(const Counter & counter, const string &lexiconID, double vocabSize) : SmoothedMLEstimator(counter, lexiconID), vocSz(vocabSize) {};
  virtual ~LaplaceUnigramLM() {}
  
  virtual double probEstimate(TERMID_T wordIndex, double count, double sum) const {
    return ((count+1)/(sum+vocSz));
  }
private:
  double vocSz;
};



class DirichletUnigramLM : public SmoothedMLEstimator { 
public:
  DirichletUnigramLM(const Counter & counter, const string &lexiconID, 
                     const UnigramLM &refLM, double priorSampleSize) 
    : SmoothedMLEstimator(counter, lexiconID), ref(&refLM), 
    s(priorSampleSize) {}

  virtual ~DirichletUnigramLM() {}
  
  virtual double probEstimate(TERMID_T wordIndex, double count, double sum) const {
    return ((count+s*ref->prob(wordIndex))/(sum+s));
  }

private:
  const UnigramLM *ref;
  double s;  
};







class InterpUnigramLM : public SmoothedMLEstimator { 
public:
  InterpUnigramLM(const Counter & counter, const string &lexiconID, 
                     const UnigramLM &refLM, double refCoeff) 
    : SmoothedMLEstimator(counter, lexiconID), ref(&refLM), 
    refC(refCoeff) {}

  virtual ~InterpUnigramLM() {}
  
  virtual double probEstimate(TERMID_T wordIndex, double count, double sum) const {
    return ((1-refC)*count/sum + refC*ref->prob(wordIndex));
  }

private:
  const UnigramLM *ref;
  double refC;  
};





#endif /* _UNIGRAMLM_HPP */

---