idngram2lm.h Source File

00001 /*=====================================================================
00002                 =======   COPYRIGHT NOTICE   =======
00003 Copyright (C) 1996, Carnegie Mellon University, Cambridge University,
00004 Ronald Rosenfeld and Philip Clarkson.
00005 
00006 All rights reserved.
00007 
00008 This software is made available for research purposes only.  It may be
00009 redistributed freely for this purpose, in full or in part, provided
00010 that this entire copyright notice is included on any copies of this
00011 software and applications and derivations thereof.
00012 
00013 This software is provided on an "as is" basis, without warranty of any
00014 kind, either expressed or implied, as to any matter including, but not
00015 limited to warranty of fitness of purpose, or merchantability, or
00016 results obtained from use of this software.
00017 ======================================================================*/
00018 
00019 
00020 /* Function prototypes */
00021 
00022 #ifndef _IDNGRAM2LM_H_
00023 #define _IDNGRAM2LM_H_
00024 
00025 #include "ngram.h"
00026 
00031 unsigned short num_of_types(int k,
00032                             int ind,
00033                             ng_t *ng);
00034 int get_ngram(FILE *id_ngram_fp,ngram *ng,flag is_ascii);
00035 void calc_mem_req(ng_t *ng,flag is_ascii);
00036 void write_arpa_lm(ng_t *ng,int verbosity);
00037 void write_bin_lm(ng_t *ng,int verbosity);
00038 unsigned short new_index(int full_index,
00039                          int *ind_table,
00040                          unsigned short *ind_table_size,
00041                          int position_in_list);
00042 int get_full_index(unsigned short short_index,
00043                    int *ind_table,
00044                    int ind_table_size,
00045                    int position_in_list);
00046 void compute_gt_discount(int            n,
00047                          int            *freq_of_freq,
00048                          int            fof_size,
00049                          unsigned short *disc_range,
00050                          int cutoff,
00051                          int verbosity,
00052                          disc_val_t **discounted_values);
00053 int lookup_index_of(int *lookup_table, 
00054                     int lookup_table_size, 
00055                     int intintval);
00056 void compute_unigram(ng_t *ng,int verbosity);
00057 void compute_back_off(ng_t *ng,int n,int verbosity);
00058 void bo_ng_prob(int context_length,
00059                 id__t *sought_ngram,
00060                 ng_t *ng,
00061                 int verbosity,
00062                 double *p_prob,
00063                 int *bo_case);
00064 void increment_context(ng_t *ng, int k, int verbosity);
00065 unsigned short short_alpha(double long_alpha,
00066                            double *alpha_array,
00067                            unsigned short *size_of_alpha_array,
00068                            int elements_in_range,
00069                            double min_range,
00070                            double max_range);
00071 
00072 double double_alpha(unsigned short short_alpha,
00073                     double *alpha_array,
00074                     int size_of_alpha_array,
00075                     int elements_in_range,
00076                     double min_range,
00077                     double max_range);
00078 
00079 void guess_mem(int total_mem,
00080                int middle_size,
00081                int end_size,
00082                int n,
00083                table_size_t *table_sizes,
00084                int verbosity);
00085 
00086 void read_voc(char *filename, int verbosity,   
00087               sih_t *p_vocab_ht, char ***p_vocab, 
00088               unsigned short *p_vocab_size);
00089 
00090 void store_count(flag four_byte_counts,
00091                  int *count_table,
00092                  int count_table_size,
00093                  unsigned short *short_counts,
00094                  int *long_counts,
00095                  int position,
00096                  int count);
00097 
00098 int return_count(flag four_byte_counts,
00099                  int *count_table,
00100                  unsigned short *short_counts,
00101                  int *long_counts,
00102                  int position);
00103 
00104 #endif