---

evallm.h File Reference

Declaration of Function prototypes used by evallm. More...

#include "pc_libs/pc_general.h"
#include "rr_libs/general.h"
#include "ngram.h"
#include "toolkit.h"

Go to the source code of this file.

Compounds
struct	arpa_lm_t
	Type specification for ARPA LM type. More...
struct	fb_info
	Type specification for forced back-off list. More...
Typedefs
typedef float	bo_t
typedef float	prob_t
Functions
unsigned short	num_of_types (int k, int ind, ng_t *ng)
void	decode_bo_case (int bo_case, int context_length, FILE *annotation_fp)
void	display_stats (ng_t *ng)
void	display_arpa_stats (arpa_lm_t *arpa_ng)
void	load_lm (ng_t *ng, char *lm_filename)
void	load_arpa_lm (arpa_lm_t *arpa_lm, char *lm_filename)
void	parse_comline (char *input_line, int *num_of_args, char **args)
void	compute_perplexity (ng_t *ng, arpa_lm_t *arpa_ng, char *text_stream_filename, char *probs_stream_filename, char *annotation_filename, char *oov_filename, char *fb_list_filename, flag backoff_from_unk_inc, flag backoff_from_unk_exc, flag backoff_from_ccs_inc, flag backoff_from_ccs_exc, flag arpa_lm, flag include_unks, double log_base)
fb_info *	gen_fb_list (sih_t *vocab_ht, int vocab_size, char **vocab, flag *context_cue, flag backoff_from_unk_inc, flag backoff_from_unk_exc, flag backoff_from_ccs_inc, flag backoff_from_ccs_exc, char *fb_list_filename)
void	validate (ng_t *ng, arpa_lm_t *arpa_ng, char **words, flag backoff_from_unk_inc, flag backoff_from_unk_exc, flag backoff_from_ccs_inc, flag backoff_from_ccs_exc, flag arpa_lm, char *fb_list_filename)
double	calc_prob_of (id__t sought_word, id__t *context, int context_length, ng_t *ng, arpa_lm_t *arpa_ng, fb_info *fb_list, int *bo_case, int *actual_context_length, flag arpa_lm)
void	arpa_bo_ng_prob (int context_length, id__t *sought_ngram, arpa_lm_t *arpa_ng, int verbosity, double *p_prob, int *bo_case)

---

Detailed Description

Declaration of Function prototypes used by evallm.

Definition in file evallm.h.

---

Typedef Documentation

typedef float bo_t

Definition at line 44 of file evallm.h. Referenced by load_arpa_lm().

typedef float prob_t

Definition at line 45 of file evallm.h. Referenced by load_arpa_lm().

---

Function Documentation

void arpa_bo_ng_prob (  int    context_length, id__t *    sought_ngram, arpa_lm_t *    arpa_ng, int    verbosity, double *    p_prob, int *    bo_case )

Definition at line 25 of file arpa_bo_ng_prob.c. References arpa_lm_t::bo_weight, flag, fprintf(), free(), get_full_index(), i, arpa_lm_t::ind, arpa_lm_t::num_kgrams, arpa_lm_t::probs, arpa_lm_t::ptr_table, arpa_lm_t::ptr_table_size, rr_malloc(), verbosity, arpa_lm_t::vocab_size, and arpa_lm_t::word_id. Referenced by calc_prob_of().

double calc_prob_of (  id__t    sought_word, id__t *    context, int    context_length, ng_t *    ng, arpa_lm_t *    arpa_ng, fb_info *    fb_list, int *    bo_case, int *    actual_context_length, flag    arpa_lm )

Definition at line 29 of file calc_prob_of.c. References arpa_bo_ng_prob(), bo_ng_prob(), CLOSED_VOCAB, flag, free(), i, quit(), rr_malloc(), ng_t::vocab_type, and arpa_lm_t::vocab_type. Referenced by compute_perplexity(), and validate().

void compute_perplexity (  ng_t *    ng, arpa_lm_t *    arpa_ng, char *    text_stream_filename, char *    probs_stream_filename, char *    annotation_filename, char *    oov_filename, char *    fb_list_filename, flag    backoff_from_unk_inc, flag    backoff_from_unk_exc, flag    backoff_from_ccs_inc, flag    backoff_from_ccs_exc, flag    arpa_lm, flag    include_unks, double    log_base )

Definition at line 28 of file perplexity.c. References calc_prob_of(), CLOSED_VOCAB, arpa_lm_t::context_cue, ng_t::context_cue, decode_bo_case(), flag, fprintf(), free(), gen_fb_list(), i, arpa_lm_t::n, ng_t::n, quit(), rr_calloc(), rr_feof(), rr_fexists(), rr_iclose(), rr_iopen(), rr_malloc(), rr_oclose(), rr_oopen(), sih_lookup(), arpa_lm_t::vocab, ng_t::vocab, arpa_lm_t::vocab_ht, ng_t::vocab_ht, arpa_lm_t::vocab_size, ng_t::vocab_size, arpa_lm_t::vocab_type, and ng_t::vocab_type. Referenced by main().

void decode_bo_case (  int    bo_case, int    context_length, FILE *    annotation_fp )

Definition at line 28 of file decode_bo_case.c. References fprintf(), and i. Referenced by compute_perplexity().

void display_arpa_stats (  arpa_lm_t *    arpa_ng )

Display statistics of an ARPA n-gram Definition at line 122 of file stats.c. References CLOSED_VOCAB, arpa_lm_t::context_cue, arpa_lm_t::first_id, fprintf(), i, arpa_lm_t::n, arpa_lm_t::no_of_ccs, arpa_lm_t::num_kgrams, OPEN_VOCAB_1, OPEN_VOCAB_2, arpa_lm_t::vocab, arpa_lm_t::vocab_size, and arpa_lm_t::vocab_type. Referenced by main().

void display_stats (  ng_t *    ng )

Display statistics of an n-gram Definition at line 27 of file stats.c. References ng_t::abs_disc_const, ABSOLUTE, CLOSED_VOCAB, ng_t::context_cue, ng_t::disc_range, ng_t::discounting_method, ng_t::first_id, ng_t::fof_size, ng_t::four_byte_alphas, fprintf(), ng_t::freq_of_freq, GOOD_TURING, ng_t::gt_disc_ratio, i, j, ng_t::lin_disc_ratio, LINEAR, ng_t::n, ng_t::no_of_ccs, ng_t::num_kgrams, ng_t::oov_fraction, OPEN_VOCAB_1, OPEN_VOCAB_2, ng_t::vocab, ng_t::vocab_size, ng_t::vocab_type, and WITTEN_BELL. Referenced by main().

fb_info* gen_fb_list (  sih_t *    vocab_ht, int    vocab_size, char **    vocab, flag *    context_cue, flag    backoff_from_unk_inc, flag    backoff_from_unk_exc, flag    backoff_from_ccs_inc, flag    backoff_from_ccs_exc, char *    fb_list_filename )

Definition at line 31 of file gen_fb_list.c. References fb_info::backed_off, flag, fprintf(), i, fb_info::inclusive, rr_calloc(), rr_iclose(), rr_iopen(), and sih_lookup(). Referenced by compute_perplexity(), and validate().

void load_arpa_lm (  arpa_lm_t *    arpa_lm, char *    lm_filename )

Read an ARPA LM file from a text format ARPA LM file Definition at line 316 of file load_lm.c. References bo_t, arpa_lm_t::bo_weight, CLOSED_VOCAB, arpa_lm_t::first_id, flag, fprintf(), free(), get_full_index(), i, arpa_lm_t::ind, index__t, j, arpa_lm_t::n, new_index(), arpa_lm_t::num_kgrams, OPEN_VOCAB_1, prob_t, arpa_lm_t::probs, arpa_lm_t::ptr_table, arpa_lm_t::ptr_table_size, quit(), rr_calloc(), rr_feof(), rr_iclose(), rr_iopen(), rr_malloc(), salloc(), sih_add(), sih_create(), sih_lookup(), arpa_lm_t::table_sizes, arpa_lm_t::vocab, arpa_lm_t::vocab_ht, arpa_lm_t::vocab_size, arpa_lm_t::vocab_type, and arpa_lm_t::word_id. Referenced by main().

void load_lm (  ng_t *    ng, char *    lm_filename )

Definition at line 39 of file load_lm.c. References ng_t::abs_disc_const, ABSOLUTE, ng_t::alpha_array, BBO_FILE_VERSION, ng_t::bin_fp, ng_t::bo_weight, ng_t::bo_weight4, bo_weight_t, ng_t::context_cue, ng_t::count, ng_t::count4, count_ind_t, count_t, ng_t::count_table, ng_t::count_table_size, cutoff_t, ng_t::cutoffs, ng_t::disc_range, disc_val_t, ng_t::discounting_method, ng_t::first_id, flag, ng_t::fof_size, ng_t::four_byte_alphas, ng_t::four_byte_counts, four_byte_t, ng_t::freq_of_freq, get_vocab_from_vocab_ht(), GOOD_TURING, ng_t::gt_disc_ratio, i, ng_t::ind, index__t, ng_t::lin_disc_ratio, LINEAR, ng_t::marg_counts, ng_t::marg_counts4, ng_t::max_alpha, ng_t::min_alpha, ng_t::n, ng_t::n_unigrams, ng_t::no_of_ccs, ng_t::num_kgrams, ng_t::oov_fraction, ng_t::out_of_range_alphas, ptr_tab_t, ng_t::ptr_table, ng_t::ptr_table_size, quit(), rr_calloc(), rr_fread(), rr_iclose(), rr_iopen(), rr_malloc(), salloc(), sih_val_read_from_file(), ng_t::size_of_alpha_array, ng_t::uni_log_probs, ng_t::uni_probs, uni_probs_t, ng_t::version, ng_t::vocab, ng_t::vocab_ht, ng_t::vocab_size, ng_t::vocab_type, WITTEN_BELL, ng_t::word_id, and ng_t::zeroton_fraction. Referenced by main().

unsigned short num_of_types (  int    k, int    ind, ng_t *    ng )

Definition at line 23 of file num_of_types.c.

void parse_comline (  char *    input_line, int *    num_of_args, char **    args )

Definition at line 31 of file parse_comline.c. References salloc(). Referenced by main().

void validate (  ng_t *    ng, arpa_lm_t *    arpa_ng, char **    words, flag    backoff_from_unk_inc, flag    backoff_from_unk_exc, flag    backoff_from_ccs_inc, flag    backoff_from_ccs_exc, flag    arpa_lm, char *    fb_list_filename )

Definition at line 26 of file validate.c. References calc_prob_of(), CLOSED_VOCAB, arpa_lm_t::context_cue, ng_t::context_cue, arpa_lm_t::first_id, ng_t::first_id, flag, fprintf(), free(), gen_fb_list(), i, arpa_lm_t::n, ng_t::n, quit(), rr_malloc(), sih_lookup(), arpa_lm_t::vocab, ng_t::vocab, arpa_lm_t::vocab_ht, ng_t::vocab_ht, arpa_lm_t::vocab_size, ng_t::vocab_size, arpa_lm_t::vocab_type, and ng_t::vocab_type. Referenced by main().

---