---

text2idngram.c File Reference

Convert a text stream to an id n-gram stream. More...

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/utsname.h>
#include <unistd.h>
#include "toolkit.h"
#include "rr_libs/general.h"
#include "pc_libs/pc_general.h"
#include "idngram.h"

Go to the source code of this file.

Defines
#define	DEFAULT_HASH_SIZE   200000
#define	DEFAULT_MAX_FILES   20
#define	MAX_N   20
#define	TEMP_FILE_ROOT   "text2idngram.temp."
Functions
void	add_to_buffer (unsigned short word_index, int ypos, int xpos, unsigned short *buffer)
unsigned short	buffer_contents (int ypos, int xpos, unsigned short *buffer)
void	main (int argc, char *argv[])

---

Detailed Description

Convert a text stream to an id n-gram stream.

Definition in file text2idngram.c.

---

Define Documentation

#define DEFAULT_HASH_SIZE   200000

Definition at line 20 of file text2idngram.c. Referenced by main().

#define DEFAULT_MAX_FILES   20

Definition at line 21 of file text2idngram.c. Referenced by main().

#define MAX_N   20

Definition at line 22 of file text2idngram.c.

#define TEMP_FILE_ROOT   "text2idngram.temp."

Definition at line 23 of file text2idngram.c. Referenced by main().

---

Function Documentation

void add_to_buffer (  unsigned short    word_index, int    ypos, int    xpos, unsigned short *    buffer )

Definition at line 41 of file text2idngram.c. Referenced by main().

unsigned short buffer_contents (  int    ypos, int    xpos, unsigned short *    buffer )

Definition at line 51 of file text2idngram.c. Referenced by main().

void main (  int    argc, char *    argv[] )

Definition at line 65 of file text2idngram.c. References add_to_buffer(), add_to_hashtable(), buffer_contents(), compare_ngrams(), DEFAULT_HASH_SIZE, DEFAULT_MAX_FILES, DEFAULT_N, DEFAULT_TEMP, DEFAULT_VERBOSITY, flag, fprintf(), get_word(), hash(), i, index2(), j, MAX_VOCAB_SIZE, MAX_WORD_LENGTH, merge_tempfiles(), nearest_prime(), new_hashtable(), pc_flagarg(), pc_intarg(), pc_message(), pc_report_unk_args(), pc_stringarg(), quit(), report_version(), rr_feof(), rr_fwrite(), rr_iopen(), rr_malloc(), rr_oclose(), rr_oopen(), salloc(), STD_MEM, TEMP_FILE_ROOT, and verbosity.

---