---

FreqCounter.hpp

Go to the documentation of this file.

/*==========================================================================
 * Copyright (c) 2001 Carnegie Mellon University.  All Rights Reserved.
 *
 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval
 * is subject to the terms of the software license set forth in the LICENSE
 * file included with this software, and also available at
 * http://www.lemurproject.org/license.html
 *
 *==========================================================================
 */

#ifndef _FREQCOUNTER_HPP
#define _FREQCOUNTER_HPP

#include <map>
#include <set>
#include "InvFPTypes.hpp"
#include "TextHandler.hpp"
#include "Stopper.hpp"

#define R_CTF 0

#define R_DF 1

#define R_AVE_TF 2

#define R_UNIFORM 3

typedef struct freqinfo_tt {
  char * word;
  int ctf;
  int df;
} freqinfo_t;

typedef map<char *, freqinfo_t, ltstr> freqmap;
typedef set<char *, ltstr> stringset;


class FreqCounter : public TextHandler {

public:
  FreqCounter(const Stopper * stopWords = NULL);
  FreqCounter(const string &filename, const Stopper * stopWords = NULL);
  
  ~FreqCounter();

  void clear();

  void output(const string &filename) const;

  char * randomWord();
  void setRandomMode(int mode);
  int getRandomMode() const;

  char * randomCtf() const;
  char * randomDf() const;
  char * randomAveTf() const;
  char * randomUniform() const;


  int numWords() const;
  int totWords() const;

  const freqmap * getFreqInfo() const;

  int getCtf(const char * word) const;
  int getDf(const char * word) const;
  double getAveTf(const char * word) const;

  double ctfRatio(FreqCounter & lm1) const;

  char * handleDoc(char * docno);
  char * handleWord(char * word);

  void endDoc();

  void setName(const string &freqCounterName);
  const string & getName() const;

  void pruneBottomWords(int topWords);
  

protected:
  /* Loads a language model from file. */
  void input(const string &filename);

  /* Collection term frequencies. */
  mutable freqmap freqInfo;

  /* Words in a doc. */
  stringset doc;
  /* Random words returned so far. */
  stringset randdone;

  /* The frequency counter's name. */
  string name;

  /* Stopword list */
  const Stopper * stopper;


  /* used for calculating probabilities when
   * selecting a random word
   */
  /* Sum over words of ctf. */
  long ctfTot;
  /* Sum over words of df. */
  int dfTot;  
  /* Sum over words of average tf. */
  mutable long double avetfTot;
  /* Indicates whether avetfTot is valid (true)
   * or needs to be recalculated (false). */
  mutable bool atfValid;
  /* Random selection mode. */
  int randomMode;
  /* Number of unique words. */
  int nWords;


};



#endif

---