---

KeyfileIncIndex.hpp

Go to the documentation of this file.

/*==========================================================================
 * Copyright (c) 2001 Carnegie Mellon University.  All Rights Reserved.
 *
 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval
 * is subject to the terms of the software license set forth in the LICENSE
 * file included with this software, and also available at
 * http://www.lemurproject.org/license.html
 *
 *==========================================================================
*/


#ifndef _LEMUR_KEYFILE_INCINDEX_HPP
#define _LEMUR_KEYFILE_INCINDEX_HPP

/*
 * NAME DATE - COMMENTS
 * tnt 01/02 - created
 * dmf 07/03 - converted to incremental berkeley db btree indexer with
 * integrated document manager.
 * tds 09/03 - modified from BTIncIndex to use keyfile
 * dmf 12/03 - update to 2.1 API and remove parser/docmgr components.
*/
#include "common_headers.hpp"
#include "Index.hpp"
#include "InvFPDocList.hpp"
#include "InvFPTermList.hpp"
#include "InvFPTypes.hpp"
#include "Param.hpp"
#include "PushIndex.hpp"
#include "MemCache.hpp"
#include "Keyfile.hpp"
#include "KeyfileDocMgr.hpp"
#include "ReadBuffer.hpp"
#include "WriteBuffer.hpp"
#include "TermCache.hpp"
#include <cstring>
#include <queue>

// for counts array
#define UNIQUE_TERMS 0
#define TOTAL_TERMS  1
#define DOCS         2
#define DT_FILES     3
#define INV_FILES    4
// keyref.h -- 512
#define MAX_DOCID_LENGTH 512
#define MAX_TERM_LENGTH 512

#define KEYFILE_MAX_SEGMENTS (16)

// we love multiple inheritance

class KeyfileIncIndex : public PushIndex, public Index {
public:
  class record {
  public:
    File::offset_type offset;
    int len;
    int totalLen;
    int num;     
  };
  struct SegmentOffset {
    unsigned int segment;
    unsigned int length;
    File::offset_type offset;
  };
  struct TermData {
    COUNT_T totalCount;
    COUNT_T documentCount;
    SegmentOffset segments[ KEYFILE_MAX_SEGMENTS ];
  };
  KeyfileIncIndex(const string &prefix, int cachesize=128000000, 
                  DOCID_T startdocid=1);
  KeyfileIncIndex();
  ~KeyfileIncIndex();

  void setName(const string &prefix);

  bool beginDoc(const DocumentProps* dp);

  bool addTerm(const Term& t);

  void endDoc(const DocumentProps* dp);

  virtual void endDoc(const DocumentProps* dp, const string &mgr);

  void endCollection(const CollectionProps* cp);

  void setDocManager(const string &mgrID);
    
protected:
  bool tryOpen();
  void writeTOC();
  void writeCache( bool lastRun = false );
  void lastWriteCache();

  void mergeCacheSegments();
  void writeCacheSegment();
  void writeDocMgrIDs();
  int docMgrID(const string &mgr);
  virtual void doendDoc(const DocumentProps* dp, int mgrid);
  int listlengths;
  
public:


  bool open(const string &indexName);



  TERMID_T term(const TERM_T &word) const;

  const TERM_T term(TERMID_T termID) const;

  DOCID_T document(const EXDOCID_T &docIDStr) const;

  const EXDOCID_T document(DOCID_T docID) const; 

  const DocumentManager *docManager(DOCID_T docID) const;




  COUNT_T docCount() const { return counts[DOCS]; };

  COUNT_T termCountUnique() const { return counts[UNIQUE_TERMS]; };

  COUNT_T termCount(TERMID_T termID) const;

  COUNT_T termCount() const { return counts[TOTAL_TERMS]; };

  float docLengthAvg() const;

  COUNT_T docCount(TERMID_T termID) const;

  COUNT_T docLength(DOCID_T docID) const;

  virtual COUNT_T totaldocLength (DOCID_T docID) const;

  COUNT_T docLengthCounted(DOCID_T docID) const;




  DocInfoList* docInfoList(TERMID_T termID) const;

  TermInfoList* termInfoList(DOCID_T docID) const;
  TermInfoList* termInfoListSeq(DOCID_T docID) const;


  void setMesgStream(ostream * lemStream);
  void addKnownTerm( TERMID_T termID, LOC_T position );
  TERMID_T addUnknownTerm( const InvFPTerm* term );
  TERMID_T addUncachedTerm( const InvFPTerm* term );

protected:
  void openDBs();
  void openSegments();
  void createDBs();

  void fullToc();
  bool docMgrIDs();
  record fetchDocumentRecord( DOCID_T key ) const;
  void addDocumentLookup( DOCID_T documentKey, const char* documentName );
  void addTermLookup( TERMID_T termKey, const char* termSpelling );
  void addGeneralLookup( Keyfile& numberNameIndex, Keyfile& nameNumberIndex, 
                         TERMID_T number, const char* name );
  InvFPDocList* internalDocInfoList(TERMID_T termID) const;
  void _updateTermlist( InvFPDocList* curlist, LOC_T position );
  int _cacheSize();
  void _computeMemoryBounds( int memorySize );
  void _resetEstimatePoint();
  COUNT_T* counts;    
  std::vector<std::string> names;
  float aveDocLen; 
  vector<std::string> docmgrs;
  ostream* msgstream;

  // All database handles are marked mutable since they sometimes
  // must be used to fetch values during const methods
  mutable Keyfile invlookup;
  
  // int <-> string mappings for documents and terms
  mutable Keyfile dIDs;
  mutable Keyfile dSTRs;
  mutable Keyfile tIDs;
  mutable Keyfile tSTRs;
  mutable File dtlookup; 
  ReadBuffer* dtlookupReadBuffer; 
  mutable File writetlist; 

  mutable char termKey[MAX_TERM_LENGTH];
  mutable char docKey[MAX_DOCID_LENGTH];
  int _listsSize;
  int _memorySize;
  std::string name;
  vector<InvFPDocList*> invertlists; 
  vector<LocatedTerm> termlist; 
  int curdocmgr; 
  vector<DocumentManager*> docMgrs; 
  TermCache _cache;

  std::vector<File*> _segments;
  TERMID_T _largestFlushedTermID;
  int _estimatePoint; 
  bool ignoreDoc;  
};


#endif //_LEMUR_KEYFILE_INCINDEX_HPP

---