---

InvIndexMerge.hpp

Go to the documentation of this file.

/*==========================================================================
 * Copyright (c) 2001 Carnegie Mellon University.  All Rights Reserved.
 *
 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval
 * is subject to the terms of the software license set forth in the LICENSE
 * file included with this software, and also available at
 * http://www.lemurproject.org/license.html
 *
 *==========================================================================
*/


#ifndef _INVINDEXMERGE_HPP
#define _INVINDEXMERGE_HPP

#include "common_headers.hpp"
#include "InvDocList.hpp"
#include "InvFPTypes.hpp"
#include "Exception.hpp"

#define READBUFSIZE 2000000
#define NUM_FH_OPEN 32

struct IndexReader {
  InvDocList* list;
  ifstream* reader;
};

// this class could actually be static
class InvIndexMerge {
public:
  InvIndexMerge(char* buffer, long size, long maxfilesize=2100000000);
  InvIndexMerge(long buffersize=64000000, long maxfilesize=2100000000);
  virtual ~InvIndexMerge();

  int merge(vector<string>* tf, const string &prefix);

  void setMaxFileSize(long size);
  char* setBuffer(char* buffer, long size);

  int hierMerge(vector<string>* files, int level);

  virtual int mergeFiles(vector<string>* files, vector<string>* intmed, int level);

  virtual int finalMerge(vector<string>* files);

protected:
  virtual void writeInvFIDs();
  virtual void least(vector<IndexReader*>* r, vector<TERMID_T>* ret);
  void setbuf(ifstream* fs, char* bp, int bytes);

  string name;
  vector<string> invfiles; // list of files that we've written to
  long maxfile; // maximum file size for each index
  long bufsize;
  char* readbuffer;
};

#endif

---