---

PDict.hpp

Go to the documentation of this file.

/*==========================================================================
 * Copyright (c) 2004 University of Massachusetts.  All Rights Reserved.
 *
 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval
 * is subject to the terms of the software license set forth in the LICENSE
 * file included with this software, and also available at
 * http://www.lemurproject.org/license.html
 *
 *==========================================================================
 */


/* 
 * dmf 02/2004
 */
#ifndef _LEMUR_PDICT_HPP
#define _LEMUR_PDICT_HPP

#include "common_headers.hpp"
#include "algorithm"
#include "Keyfile.hpp"
#include "File.hpp"
#include "TextHandlerManager.hpp"

class DictEntry {
public:
  DictEntry();

  DictEntry(const string &targ, const string &typ, double pr) : target(targ), type(typ), 
                                                    prob(pr) { }
  
  char *toBytes(int &numBytes) const;

  int toBytes(char *buffer) const;

  int numBytes() const;
  int fromBytes(char *buffer);
  bool operator==(const DictEntry & a) const { 
    return (target == a.target && type == a.type); 
  }
  string toString(string delim = ";") const ;
    
  string target;
  string type;
  double prob;
};

class DictEntryFilter {
public:
  virtual bool accept(const DictEntry &entry) const = 0;
  virtual ~DictEntryFilter() {}
} ;


class AllDictEntryFilter : public DictEntryFilter {
public:
  bool accept(const DictEntry &entry) const { return true; }
};


class ProbDictEntryFilter : public DictEntryFilter {
public:
  ProbDictEntryFilter(double thresh = 0.0) : threshold(thresh) {
  }
  bool accept(const DictEntry &entry) const { return entry.prob > threshold; }
private:
  double threshold;
};


class TypeDictEntryFilter : public DictEntryFilter {
public:
  TypeDictEntryFilter(const string &filtType) : type(filtType) {
  }
  bool accept(const DictEntry &entry) const { return entry.type == type; }
private:
  string type;
};

class StopwordDictEntryFilter : public DictEntryFilter {
public:
  StopwordDictEntryFilter(const string &stopwords) {
    stopper = TextHandlerManager::createStopper(stopwords);
  }
  bool accept(const DictEntry &entry) const { 
    return !(stopper->stopWord(entry.target.c_str())); 
  }
private:
  Stopper *stopper;
};

class DictEntryVector : public vector<DictEntry> {
public:
  DictEntryVector() : vector<DictEntry>() {
  }
  DictEntryVector(char *buffer, DictEntryFilter *filter);
  void sortScores() {
    sort(this->begin(), this->end(), cmpFn);
  }
  bool addEntry(DictEntry &entry, double (*compose)(double, double) = NULL);

  bool removeEntry(DictEntry &entry);

  char *toBytes(int &numBytes) const;

  void toBytes(char *buffer) const;

  void fromBytes(char *buffer, DictEntryFilter *filter);

  int numEntries() const;

  void normalize();
  
private:
  class DictEntryProbDescending { 
  public: 
    bool operator()(const DictEntry & a, const DictEntry & b) {
      return a.prob > b.prob;
    }
  };
  static DictEntryProbDescending cmpFn;
};

struct dictStats {
  int dictSize;
  int sourceSize;
  int targetSize;
};

class PDict {
public:
  PDict();
  
  ~PDict();

  DictEntryVector *getTranslations(const string &term, 
                                   DictEntryFilter *filter=NULL) const ;
  int numTranslations(const string &term, 
                      DictEntryFilter *filter=NULL) const;
  int getNumPairs() const;

  int getSourceCount() const;

  int getTargetCount() const ;

  const string &getName() const {return name;}

  bool isUsingCounts() const {return usingCounts;}
  
  void setUsingCounts(bool val) {usingCounts = val;}
  
  void add(const string &source, DictEntry &value, 
           double (*compose)(double, double) = NULL);

  void remove(const string &source, DictEntry &value);

  void remove(const string &source);

  void write(const string &outputName, const string &delim);

  bool read(const string &dictName, const string &delim, bool counts = false);

  bool open(const string &dictName);

  bool create(const string &dictName);

  void close();
  
  void normalize();

  void startIteration() {dict.setFirst();}

  DictEntryVector *nextTranslations(string &term, 
                                   DictEntryFilter *filter=NULL) const;

private:
  void writeTOC() const;  
  bool contains(const string &term, Keyfile &keyfile) const;
  void flush();
  dictStats stats;
  DictEntryVector* currentVec;
  bool usingCounts;
  string currentTerm;
  string name;
  mutable Keyfile dict;
  mutable Keyfile targetIDs;
  mutable File dictEntries;
};

#endif // _LEMUR_PDICT_HPP

---