---

RawTextParser.hpp

Go to the documentation of this file.

/*==========================================================================
 * Copyright (c) 2004 University of Massachusetts.  All Rights Reserved.
 *
 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval
 * is subject to the terms of the software license set forth in the LICENSE
 * file included with this software, and also available at
 * http://www.lemurproject.org/license.html
 *
 *==========================================================================
*/


//
// RawTextParser
//
// 10 February 2004 -- tds
//

#ifndef INDRI_RAWTEXTPARSER_HPP
#define INDRI_RAWTEXTPARSER_HPP

#define PARSER_MAX_WORD_LENGTH (30)

class RawTextParser {
private:
  std::ifstream _in;
  char* _buffer;
  char* _current;
  int _bufferSize;

public:
  RawTextParser( int memorySize = 1024*1024 ) {
    _bufferSize = memorySize;
    _buffer = new char[_bufferSize]; // let's hope that 1mb is enough for any docs we're gonna see
  }

  ~RawTextParser() {
    delete _buffer;
  }

  bool open( const std::string& fileName ) {
    _in.open( fileName.c_str(), std::ifstream::in );
    return _in.good();
  }

  void close() {
    _in.close();
  }


  bool parseDocument( std::string& docName, greedy_vector<char*>& words ) { 
    static const char docPrefix[] = "<DOC>";
    static const char endDocPrefix[] = "</DOC>";
    static const char docnoPrefix[] = "<DOCNO ";
    static const char urlPrefix[] = "<URL";
    bool gotDocID = false;
    int bufferPos = 0;

    while( 1 ) {
      int remainingSpace = _bufferSize - bufferPos;
      _in.getline( _buffer + bufferPos, remainingSpace );
      int length = _in.gcount();

      if( _in.rdstate() & (std::ifstream::failbit|std::ifstream::eofbit) ) {
        if( _in.rdstate() & std::ifstream::eofbit ) {
          return false; // at end of file, we're done
        }
        
        if( length == remainingSpace - 1 ) {
          throw Exception( "RawTextParser", "Buffer size is too small to handle some document in the corpus, use -parserMemory to change." );
        }

        if( _in.rdstate() & std::ifstream::failbit ) {
          throw Exception( "RawTextParser", "Unable to recover from failed read" );
        }
      }

      _buffer[bufferPos+length] = 0;
      char* line = _buffer + bufferPos;

      if( length && _buffer[bufferPos] == '<' ) {
        if( length > sizeof docnoPrefix-1 && !strncmp( docnoPrefix, line, sizeof docnoPrefix-1 ) ) {
          docName.assign( line+sizeof docnoPrefix-1, line + length - 2 );
          gotDocID = true;
        } else if ( length > sizeof endDocPrefix-1 && !strncmp( endDocPrefix, line, sizeof endDocPrefix-1 ) ) {
          // handle end doc -- return
          if( gotDocID )
            return true;
        }
      } else {
        if( !gotDocID )
          continue;

        int i = 0;

        while(1) {
          for( ; isspace(line[i]) && i<length && line[i]; i++ )
            ;

          if( i>= length || !line[i] )
            break;

          char* begin = &line[i];

          for( ; !isspace(line[i]) && i<length && line[i]; i++ )
            ;

          line[i] = 0;
          i++;

          if( &line[i] - begin > PARSER_MAX_WORD_LENGTH )
            begin[PARSER_MAX_WORD_LENGTH-1] = 0;

          words.push_back(begin);
        }
      }

      bufferPos += length + 1; // have to skip trailing \0
    }
  }
};

#endif // INDRI_RAWTEXTPARSER_HPP

---