00001 /*========================================================================== 00002 * Copyright (c) 2003-2004 University of Massachusetts. All Rights Reserved. 00003 * 00004 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval 00005 * is subject to the terms of the software license set forth in the LICENSE 00006 * file included with this software, and also available at 00007 * http://www.lemurproject.org/license.html 00008 * 00009 *========================================================================== 00010 */ 00011 00012 00013 // 00014 // TextParser 00015 // 00016 // 16 August 2004 -- tds 00017 // 00018 00019 #ifndef INDRI_TEXTPARSER_HPP 00020 #define INDRI_TEXTPARSER_HPP 00021 00022 #include <stdio.h> 00023 #include <ctype.h> 00024 #include <string.h> 00025 #include <string> 00026 #include <vector> 00027 #include "indri/IndriParser.hpp" 00028 #include "indri/Buffer.hpp" 00029 #include "string-set.h" 00030 00031 class TextParser : public indri::Parser { 00032 public: 00033 TextParser(); 00034 ~TextParser(); 00035 00036 ParsedDocument* parse( UnparsedDocument* document ); 00037 00038 void handle( UnparsedDocument* document ); 00039 void setHandler( ObjectHandler<ParsedDocument>& h ); 00040 00041 void setTags( const std::vector<std::string>& include, 00042 const std::vector<std::string>& exclude, 00043 const std::vector<std::string>& index, 00044 const std::vector<std::string>& metadata, 00045 const std::map<std::string, std::string>& conflations ); 00046 00047 protected: 00048 void writeToken(char* token); 00049 void writeToken(char *token, int start, int end); 00050 Buffer _termBuffer; 00051 00052 private: 00053 ObjectHandler<ParsedDocument>* _handler; 00054 ParsedDocument _document; 00055 }; 00056 00057 #endif // INDRI_TEXTPARSER_HPP 00058