#include <dict.h>

Public Member Functions

Dict (Image *image_ptr)

~Dict ()

const Image * getImage () const

Image * getImage ()

const UNICHARSET & getUnicharset () const

UNICHARSET & getUnicharset ()

const UnicharAmbigs & getUnicharAmbigs ()

bool compound_marker (UNICHAR_ID unichar_id)

bool hyphenated () const

Returns true if we've recorded the beginning of a hyphenated word.

int hyphen_base_size () const

Size of the base word (the part on the line before) of a hyphenated word.

void copy_hyphen_info (WERD_CHOICE *word) const

void remove_hyphen_head (WERD_CHOICE *word) const

bool has_hyphen_end (UNICHAR_ID unichar_id, bool first_pos) const

Check whether the word has a hyphen at the end.

bool has_hyphen_end (const WERD_CHOICE &word) const

Same as above, but check the unichar at the end of the word.

void reset_hyphen_vars (bool last_word_on_line)

void set_hyphen_word (const WERD_CHOICE &word, const DawgInfoVector &active_dawgs, const DawgInfoVector &constraints)

void update_best_choice (const WERD_CHOICE &word, WERD_CHOICE *best_choice)

void init_active_dawgs (int sought_word_length, DawgInfoVector *active_dawgs, bool ambigs_mode) const

void init_constraints (DawgInfoVector *constraints) const

bool ambigs_mode (float rating_limit)

Returns true if we are operating in ambigs mode.

WERD_CHOICE * dawg_permute_and_select (const BLOB_CHOICE_LIST_VECTOR &char_choices, float rating_limit)

WERD_CHOICE * get_top_choice_word (const BLOB_CHOICE_LIST_VECTOR &char_choices)

WERD_CHOICE * permute_top_choice (const BLOB_CHOICE_LIST_VECTOR &char_choices, float *rating_limit, WERD_CHOICE *raw_choice, BOOL8 *any_alpha)

WERD_CHOICE * permute_all (const BLOB_CHOICE_LIST_VECTOR &char_choices, const WERD_CHOICE *best_choice, WERD_CHOICE *raw_choice)

void end_permute ()

void permute_subword (const BLOB_CHOICE_LIST_VECTOR &char_choices, float rating_limit, int start, int end, WERD_CHOICE *current_word)

bool permute_characters (const BLOB_CHOICE_LIST_VECTOR &char_choices, WERD_CHOICE *best_choice, WERD_CHOICE *raw_choice)

WERD_CHOICE * permute_compound_words (const BLOB_CHOICE_LIST_VECTOR &char_choices, float rating_limit)

WERD_CHOICE * permute_fixed_length_words (const BLOB_CHOICE_LIST_VECTOR &char_choices, PermuterState *permuter_state)

void incorporate_segcost (WERD_CHOICE *word)

Incoporate segmentation cost into word rating.

WERD_CHOICE * permute_script_words (const BLOB_CHOICE_LIST_VECTOR &char_choices, PermuterState *permuter_state)

WERD_CHOICE * permute_chartype_words (const BLOB_CHOICE_LIST_VECTOR &char_choices, PermuterState *permuter_state)

checks for consistency in character property (eg. alpah, digit, punct)

char top_word_chartype (const BLOB_CHOICE_LIST_VECTOR &char_choices, char *pos_chartypes)

bool NoDangerousAmbig (WERD_CHOICE *BestChoice, DANGERR *fixpt, bool fix_replaceable, BLOB_CHOICE_LIST_VECTOR *Choices, bool *modified_blobs)

double StopperAmbigThreshold (double f1, double f2)

int FreeBadChoice (void *item1, void *item2)

void ReplaceAmbig (int wrong_ngram_begin_index, int wrong_ngram_size, UNICHAR_ID correct_ngram_id, WERD_CHOICE *werd_choice, BLOB_CHOICE_LIST_VECTOR *blob_choices, bool *modified_blobs)

void DisableChoiceAccum ()

void EnableChoiceAccum ()

bool ChoiceAccumEnabled ()

int LengthOfShortestAlphaRun (const WERD_CHOICE &WordChoice)

Returns the length of the shortest alpha run in WordChoice.

VIABLE_CHOICE NewViableChoice (const WERD_CHOICE &WordChoice, FLOAT32 AdjustFactor, const float Certainties[])

void PrintViableChoice (FILE *File, const char *Label, VIABLE_CHOICE Choice)

Dumps a text representation of the specified Choice to File.

bool StringSameAs (const WERD_CHOICE &WordChoice, VIABLE_CHOICE ViableChoice)

bool StringSameAs (const char *String, const char *String_lengths, VIABLE_CHOICE ViableChoice)

Compares String to ViableChoice and returns true if they are the same.

int UniformCertainties (const BLOB_CHOICE_LIST_VECTOR &Choices, const WERD_CHOICE &BestChoice)

bool AcceptableChoice (BLOB_CHOICE_LIST_VECTOR *Choices, WERD_CHOICE *BestChoice, DANGERR *fixpt, ACCEPTABLE_CHOICE_CALLER caller, bool *modified_blobs)

Returns true if the given best_choice is good enough to stop.

bool AcceptableResult (const WERD_CHOICE &BestChoice)

int ChoiceSameAs (const WERD_CHOICE &WordChoice, VIABLE_CHOICE ViableChoice)

void LogNewChoice (FLOAT32 AdjustFactor, const float Certainties[], bool raw_choice, WERD_CHOICE *WordChoice)

void EndDangerousAmbigs ()

bool CurrentBestChoiceIs (const WERD_CHOICE &WordChoice)

Returns true if WordChoice is the same as the current best choice.

FLOAT32 CurrentBestChoiceAdjustFactor ()

Returns the adjustment factor for the best choice for the current word.

bool CurrentWordAmbig ()

Returns true if there are multiple good choices for the current word.

void DebugWordChoices ()

Prints the current choices for this word to stdout.

void PrintAmbigAlternatives (FILE *file, const char *label, int label_num_unichars)

Print all the choices in raw_choices_ list for non 1-1 ambiguities.

void FillViableChoice (const WERD_CHOICE &WordChoice, FLOAT32 AdjustFactor, const float Certainties[], VIABLE_CHOICE ViableChoice)

bool AlternativeChoicesWorseThan (FLOAT32 Threshold)

void FilterWordChoices ()

void FindClassifierErrors (FLOAT32 MinRating, FLOAT32 MaxRating, FLOAT32 RatingMargin, FLOAT32 Thresholds[])

void InitChoiceAccum ()

void ClearBestChoiceAccum ()

Clears best_choices_ list accumulated by the stopper.

void LogNewSegmentation (PIECES_STATE BlobWidth)

void LogNewSplit (int Blob)

void AddNewChunk (VIABLE_CHOICE Choice, int Blob)

void SettupStopperPass1 ()

Sets up stopper variables in preparation for the first pass.

void SettupStopperPass2 ()

Sets up stopper variables in preparation for the second pass.

int case_ok (const WERD_CHOICE &word, const UNICHARSET &unicharset)

Check a string to see if it matches a set of lexical rules.

bool absolute_garbage (const WERD_CHOICE &word, const UNICHARSET &unicharset)

void Load ()

void End ()

void ResetDocumentDictionary ()

void LoadEquivalenceList (const char *unichar_strings[])

UNICHAR_ID NormalizeUnicharIdForMatch (UNICHAR_ID unichar_id) const

int def_letter_is_okay (void *void_dawg_args, UNICHAR_ID unichar_id, bool word_end) const

int LetterIsOkay (void *void_dawg_args, UNICHAR_ID unichar_id, bool word_end) const

Calls letter_is_okay_ member function.

double ProbabilityInContext (const char *context, int context_bytes, const char *character, int character_bytes)

Calls probability_in_context_ member function.

double def_probability_in_context (const char *lang, const char *context, int context_bytes, const char *character, int character_bytes)

Default (no-op) implementation of probability in context function.

double ngram_probability_in_context (const char *lang, const char *context, int context_bytes, const char *character, int character_bytes)

const int NumDawgs () const

Return the number of dawgs in the dawgs_ vector.

const Dawg * GetDawg (int index) const

Return i-th dawg pointer recorded in the dawgs_ vector.

const Dawg * GetPuncDawg () const

Return the points to the punctuation dawg.

const Dawg * GetUnambigDawg () const

Return the points to the unambiguous words dawg.

const Dawg * GetFixedLengthDawg (int word_length) const

Return the pointer to the Dawg that contains words of length word_length.

const int GetMaxFixedLengthDawgIndex () const

bool ConstraintsOk (const DawgInfoVector &constraints, int word_end, DawgType current_dawg_type) const

void ProcessPatternEdges (const Dawg *dawg, const DawgInfo &info, UNICHAR_ID unichar_id, bool word_end, DawgArgs *dawg_args, PermuterType *current_permuter) const

int valid_word (const WERD_CHOICE &word, bool numbers_ok) const

int valid_word (const WERD_CHOICE &word) const

int valid_word_or_number (const WERD_CHOICE &word) const

int valid_word (const char *string) const

This function is used by api/tesseract_cube_combiner.cpp.

bool valid_bigram (const WERD_CHOICE &word1, const WERD_CHOICE &word2) const

bool valid_punctuation (const WERD_CHOICE &word)

int good_choice (const WERD_CHOICE &choice)

Returns true if a good answer is found for the unknown blob rating.

void add_document_word (const WERD_CHOICE &best_choice)

Adds a word found on this document to the document specific dictionary.

int get_top_word_script (const BLOB_CHOICE_LIST_VECTOR &char_choices, const UNICHARSET &unicharset)

void adjust_word (WERD_CHOICE *word, float *certainty_array, const BLOB_CHOICE_LIST_VECTOR *char_choices, bool nonword, float additional_adjust, bool debug)

Adjusts the rating of the given word.

void adjust_word (WERD_CHOICE *word, float *certainty_array, bool debug)

void adjust_non_word (WERD_CHOICE *word, float *certainty_array, bool debug)

void SetWordsegRatingAdjustFactor (float f)

Set wordseg_rating_adjust_factor_ to the given value.

const LIST & getBestChoices ()

go_deeper_dawg_fxn

If the choice being composed so far could be a dictionary word keep exploring choices.

There are two modes for deciding whether to go deeper: regular dawg permuter mode and the special ambigs mode. If *limit is <= 0.0 the function switches to the ambigs mode (this is the case when dawg_permute_and_select() function is called from NoDangerousAmbigs()) and only searches for the first choice that has a rating better than *limit (in this case ratings are fake, since the real ratings can not be < 0). Modification of the hyphen state is turned off in the ambigs mode. When in the regular dawg permuter mode, the function explores all the possible words and chooses the one with the best rating. The letters with ratings that are far worse than the ones seen so far are pruned out.

WERD_CHOICE * dawg_permute_and_select (const BLOB_CHOICE_LIST_VECTOR &char_choices, float rating_limit, int sought_word_length, int end_char_choice_index)

void go_deeper_dawg_fxn (const char *debug, const BLOB_CHOICE_LIST_VECTOR &char_choices, int char_choice_index, const CHAR_FRAGMENT_INFO *prev_char_frag_info, bool word_ending, WERD_CHOICE *word, float certainties[], float *limit, WERD_CHOICE *best_choice, int *attempts_left, void *void_more_args)

choose_il1

Choose between the candidate il1 chars.

Parameters:

first_char	first choice
second_char	second choice
third_char	third choice
prev_char	prev in word
next_char	next in word
next_next_char	after next next in word

const char * choose_il1 (const char *first_char, const char *second_char, const char *third_char, const char *prev_char, const char *next_char, const char *next_next_char)

fragment_state

Given the current char choice and information about previously seen fragments, determines whether adjacent character fragments are present and whether they can be concatenated.

The given prev_char_frag_info contains:

fragment: if not NULL contains information about immediately preceeding fragmented character choice
num_fragments: number of fragments that have been used so far to construct a character
certainty: certainty of the current choice or minimum certainty of all fragments concatenated so far
rating: rating of the current choice or sum of fragment ratings concatenated so far

The output char_frag_info is filled in as follows:

character: is set to be NULL if the choice is a non-matching or non-ending fragment piece; is set to unichar of the given choice if it represents a regular character or a matching ending fragment
fragment,num_fragments,certainty,rating are set as described above

Returns:: false if a non-matching fragment is discovered, true otherwise.

WERD_CHOICE * top_fragments_permute_and_select (const BLOB_CHOICE_LIST_VECTOR &char_choices, float rating_limit)

void go_deeper_top_fragments_fxn (const char *debug, const BLOB_CHOICE_LIST_VECTOR &char_choices, int char_choice_index, const CHAR_FRAGMENT_INFO *prev_char_frag_info, bool word_ending, WERD_CHOICE *word, float certainties[], float *limit, WERD_CHOICE *best_choice, int *attempts_left, void *more_args)

bool fragment_state_okay (UNICHAR_ID curr_unichar_id, float curr_rating, float curr_certainty, const CHAR_FRAGMENT_INFO *prev_char_frag_info, const char *debug, int word_ending, CHAR_FRAGMENT_INFO *char_frag_info)

Semi-generic functions used by multiple permuters.

void permute_choices (const char *debug, const BLOB_CHOICE_LIST_VECTOR &char_choices, int char_choice_index, const CHAR_FRAGMENT_INFO *prev_char_frag_info, WERD_CHOICE *word, float certainties[], float *limit, WERD_CHOICE *best_choice, int *attempts_left, void *more_args)

void append_choices (const char *debug, const BLOB_CHOICE_LIST_VECTOR &char_choices, const BLOB_CHOICE &blob_choice, int char_choice_index, const CHAR_FRAGMENT_INFO *prev_char_frag_info, WERD_CHOICE *word, float certainties[], float *limit, WERD_CHOICE *best_choice, int *attempts_left, void *more_args)

Static Public Member Functions

static NODE_REF GetStartingNode (const Dawg *dawg, EDGE_REF edge_ref)

Returns the appropriate next node given the EDGE_REF.

static void ReadFixedLengthDawgs (DawgType type, const STRING &lang, PermuterType perm, int debug_level, FILE *file, DawgVector *dawg_vec, int *max_wdlen)

static void WriteFixedLengthDawgs (const GenericVector< SquishedDawg * > &dawg_vec, int num_dawgs, int debug_level, FILE *output_file)

static bool valid_word_permuter (uinT8 perm, bool numbers_ok)

Check all the DAWGs to see if this word is in any of them.

Public Attributes

void(Dict::* go_deeper_fxn_ )(const char *debug, const BLOB_CHOICE_LIST_VECTOR &char_choices, int char_choice_index, const CHAR_FRAGMENT_INFO *prev_char_frag_info, bool word_ending, WERD_CHOICE *word, float certainties[], float *limit, WERD_CHOICE *best_choice, int *attempts_left, void *void_more_args)

Pointer to go_deeper function that will be modified by various permuters.

int(Dict::* letter_is_okay_ )(void *void_dawg_args, UNICHAR_ID unichar_id, bool word_end) const

double(Dict::* probability_in_context_ )(const char *lang, const char *context, int context_bytes, const char *character, int character_bytes)

Probability in context function used by the ngram permuter.

char * user_words_suffix = ""

char * user_patterns_suffix = ""

bool load_system_dawg = true

bool load_freq_dawg = true

bool load_unambig_dawg = true

bool load_punc_dawg = true

bool load_number_dawg = true

bool load_fixed_length_dawgs = true

bool load_bigram_dawg = false

double segment_penalty_dict_frequent_word = 1.0

double segment_penalty_dict_case_ok = 1.1

double segment_penalty_dict_case_bad = 1.3125

double segment_penalty_ngram_best_choice = 1.24

double segment_penalty_dict_nonword = 1.25

double segment_penalty_garbage = 1.50

char * output_ambig_words_file = ""

int dawg_debug_level = 0

int hyphen_debug_level = 0

int max_viterbi_list_size = 10

bool use_only_first_uft8_step = false

double certainty_scale = 20.0

double stopper_nondict_certainty_base = -2.50

double stopper_phase2_certainty_rejection_offset = 1.0

int stopper_smallword_size = 2

double stopper_certainty_per_char = -0.50

double stopper_allowable_character_badness = 3.0

int stopper_debug_level = 0

bool stopper_no_acceptable_choices = false

double stopper_ambiguity_threshold_gain = 8.0

double stopper_ambiguity_threshold_offset = 1.5

bool save_raw_choices = false

int tessedit_truncate_wordchoice_log = 10

char * word_to_debug = ""

char * word_to_debug_lengths = ""

int fragments_debug = 0

int segment_debug = 0

bool permute_debug = 0

double bestrate_pruning_factor = 2.0

bool permute_script_word = 0

bool segment_segcost_rating = 0

bool segment_nonalphabetic_script = false

double segment_reward_script = 0.95

bool permute_fixed_length_dawg = 0

bool permute_chartype_word = 0

double segment_reward_chartype = 0.97

double segment_reward_ngram_best_choice = 0.99

bool save_doc_words = 0

bool doc_dict_enable = 1

double doc_dict_pending_threshold = 0.0

double doc_dict_certainty_threshold = -2.25

bool ngram_permuter_activated = false

int max_permuter_attempts = 10000

bool permute_only_top = false

Detailed Description

Definition at line 90 of file dict.h.

Constructor & Destructor Documentation

tesseract::Dict::Dict ( Image * image_ptr )

Definition at line 33 of file dict.cpp.

    : letter_is_okay_(&tesseract::Dict::def_letter_is_okay),
      probability_in_context_(&tesseract::Dict::def_probability_in_context),
      image_ptr_(image_ptr),
      STRING_INIT_MEMBER(user_words_suffix, "",
                         "A list of user-provided words.",
                         getImage()->getCCUtil()->params()),
      STRING_INIT_MEMBER(user_patterns_suffix, "",
                         "A list of user-provided patterns.",
                         getImage()->getCCUtil()->params()),
      BOOL_INIT_MEMBER(load_system_dawg, true, "Load system word dawg.",
                       getImage()->getCCUtil()->params()),
      BOOL_INIT_MEMBER(load_freq_dawg, true, "Load frequent word dawg.",
                       getImage()->getCCUtil()->params()),
      BOOL_INIT_MEMBER(load_unambig_dawg, true, "Load unambiguous word dawg.",
                       getImage()->getCCUtil()->params()),
      BOOL_INIT_MEMBER(load_punc_dawg, true, "Load dawg with punctuation"
                       " patterns.", getImage()->getCCUtil()->params()),
      BOOL_INIT_MEMBER(load_number_dawg, true, "Load dawg with number"
                       " patterns.", getImage()->getCCUtil()->params()),
      BOOL_INIT_MEMBER(load_fixed_length_dawgs, true, "Load fixed length dawgs"
                       " (e.g. for non-space delimited languages)",
                       getImage()->getCCUtil()->params()),
      BOOL_INIT_MEMBER(load_bigram_dawg, false, "Load dawg with special word "
                       "bigrams.", getImage()->getCCUtil()->params()),
      double_MEMBER(segment_penalty_dict_frequent_word, 1.0,
                    "Score multiplier for word matches which have good case and"
                    "are frequent in the given language (lower is better).",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(segment_penalty_dict_case_ok, 1.1,
                    "Score multiplier for word matches that have good case "
                    "(lower is better).", getImage()->getCCUtil()->params()),
      double_MEMBER(segment_penalty_dict_case_bad, 1.3125,
                    "Default score multiplier for word matches, which may have "
                    "case issues (lower is better).",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(segment_penalty_ngram_best_choice, 1.24,
                   "Multipler to for the best choice from the ngram model.",
                   getImage()->getCCUtil()->params()),
      double_MEMBER(segment_penalty_dict_nonword, 1.25,
                    "Score multiplier for glyph fragment segmentations which "
                    "do not match a dictionary word (lower is better).",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(segment_penalty_garbage, 1.50,
                    "Score multiplier for poorly cased strings that are not in"
                    " the dictionary and generally look like garbage (lower is"
                    " better).", getImage()->getCCUtil()->params()),
      STRING_MEMBER(output_ambig_words_file, "",
                    "Output file for ambiguities found in the dictionary",
                    getImage()->getCCUtil()->params()),
      INT_MEMBER(dawg_debug_level, 0, "Set to 1 for general debug info"
                 ", to 2 for more details, to 3 to see all the debug messages",
                 getImage()->getCCUtil()->params()),
      INT_MEMBER(hyphen_debug_level, 0, "Debug level for hyphenated words.",
                 getImage()->getCCUtil()->params()),
      INT_MEMBER(max_viterbi_list_size, 10, "Maximum size of viterbi list.",
                 getImage()->getCCUtil()->params()),
      BOOL_MEMBER(use_only_first_uft8_step, false,
                  "Use only the first UTF8 step of the given string"
                  " when computing log probabilities.",
                  getImage()->getCCUtil()->params()),
      double_MEMBER(certainty_scale, 20.0, "Certainty scaling factor",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(stopper_nondict_certainty_base, -2.50,
                    "Certainty threshold for non-dict words",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(stopper_phase2_certainty_rejection_offset, 1.0,
                    "Reject certainty offset",
                    getImage()->getCCUtil()->params()),
      INT_MEMBER(stopper_smallword_size, 2,
                 "Size of dict word to be treated as non-dict word",
                 getImage()->getCCUtil()->params()),
      double_MEMBER(stopper_certainty_per_char, -0.50, "Certainty to add"
                    " for each dict char above small word size.",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(stopper_allowable_character_badness, 3.0,
                    "Max certaintly variation allowed in a word (in sigma)",
                    getImage()->getCCUtil()->params()),
      INT_MEMBER(stopper_debug_level, 0, "Stopper debug level",
                 getImage()->getCCUtil()->params()),
      BOOL_MEMBER(stopper_no_acceptable_choices, false,
                  "Make AcceptableChoice() always return false. Useful"
                  " when there is a need to explore all segmentations",
                  getImage()->getCCUtil()->params()),
      double_MEMBER(stopper_ambiguity_threshold_gain, 8.0,
                    "Gain factor for ambiguity threshold.",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(stopper_ambiguity_threshold_offset, 1.5,
                    "Certainty offset for ambiguity threshold.",
                    getImage()->getCCUtil()->params()),
      BOOL_MEMBER(save_raw_choices, false, "Save all explored raw choices",
                  getImage()->getCCUtil()->params()),
      INT_MEMBER(tessedit_truncate_wordchoice_log, 10,
                 "Max words to keep in list",
                 getImage()->getCCUtil()->params()),
      STRING_MEMBER(word_to_debug, "", "Word for which stopper debug"
                    " information should be printed to stdout",
                    getImage()->getCCUtil()->params()),
      STRING_MEMBER(word_to_debug_lengths, "",
                    "Lengths of unichars in word_to_debug",
                    getImage()->getCCUtil()->params()),
      INT_MEMBER(fragments_debug, 0, "Debug character fragments",
                 getImage()->getCCUtil()->params()),
      INT_MEMBER(segment_debug, 0, "Debug the whole segmentation process",
                 getImage()->getCCUtil()->params()),
      BOOL_MEMBER(permute_debug, 0, "Debug char permutation process",
                  getImage()->getCCUtil()->params()),
      double_MEMBER(bestrate_pruning_factor, 2.0, "Multiplying factor of"
                    " current best rate to prune other hypotheses",
                    getImage()->getCCUtil()->params()),
      BOOL_MEMBER(permute_script_word, 0,
                  "Turn on word script consistency permuter",
                  getImage()->getCCUtil()->params()),
      BOOL_MEMBER(segment_segcost_rating, 0,
                  "incorporate segmentation cost in word rating?",
                  getImage()->getCCUtil()->params()),
      BOOL_MEMBER(segment_nonalphabetic_script, false,
                 "Don't use any alphabetic-specific tricks."
                 "Set to true in the traineddata config file for"
                 " scripts that are cursive or inherently fixed-pitch",
                 getImage()->getCCUtil()->params()),
      double_MEMBER(segment_reward_script, 0.95,
                    "Score multipler for script consistency within a word. "
                    "Being a 'reward' factor, it should be <= 1. "
                    "Smaller value implies bigger reward.",
                    getImage()->getCCUtil()->params()),
      BOOL_MEMBER(permute_fixed_length_dawg, 0,
                  "Turn on fixed-length phrasebook search permuter",
                  getImage()->getCCUtil()->params()),
      BOOL_MEMBER(permute_chartype_word, 0,
                  "Turn on character type (property) consistency permuter",
                  getImage()->getCCUtil()->params()),
      double_MEMBER(segment_reward_chartype, 0.97,
                    "Score multipler for char type consistency within a word. ",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(segment_reward_ngram_best_choice, 0.99,
                    "Score multipler for ngram permuter's best choice"
                    " (only used in the Han script path).",
                    getImage()->getCCUtil()->params()),
      BOOL_MEMBER(save_doc_words, 0, "Save Document Words",
                  getImage()->getCCUtil()->params()),
      BOOL_MEMBER(doc_dict_enable, 1, "Enable Document Dictionary ",
                  getImage()->getCCUtil()->params()),
      double_MEMBER(doc_dict_pending_threshold, 0.0,
                    "Worst certainty for using pending dictionary",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(doc_dict_certainty_threshold, -2.25,
                    "Worst certainty for words that can be inserted into the"
                    "document dictionary", getImage()->getCCUtil()->params()),
      BOOL_MEMBER(ngram_permuter_activated, false,
                  "Activate character-level n-gram-based permuter",
                  getImage()->getCCUtil()->params()),
      INT_MEMBER(max_permuter_attempts, 10000, "Maximum number of different"
                 " character choices to consider during permutation."
                 " This limit is especially useful when user patterns"
                 " are specified, since overly generic patterns can result in"
                 " dawg search exploring an overly large number of options.",
                 getImage()->getCCUtil()->params()),
      BOOL_MEMBER(permute_only_top, false, "Run only the top choice permuter",
                  getImage()->getCCUtil()->params()) {
  dang_ambigs_table_ = NULL;
  replace_ambigs_table_ = NULL;
  keep_word_choices_ = false;
  reject_offset_ = 0.0;
  best_raw_choice_ = NULL;
  best_choices_ = NIL_LIST;
  raw_choices_ = NIL_LIST;
  go_deeper_fxn_ = NULL;
  hyphen_word_ = NULL;
  last_word_on_line_ = false;
  hyphen_unichar_id_ = INVALID_UNICHAR_ID;
  document_words_ = NULL;
  pending_words_ = NULL;
  bigram_dawg_ = NULL;
  freq_dawg_ = NULL;
  punc_dawg_ = NULL;
  max_fixed_length_dawgs_wdlen_ = -1;
  wordseg_rating_adjust_factor_ = -1.0f;
  output_ambig_words_file_ = NULL;
}

tesseract::Dict::~Dict ( )

Definition at line 214 of file dict.cpp.

            {
  if (hyphen_word_ != NULL) delete hyphen_word_;
  if (output_ambig_words_file_ != NULL) fclose(output_ambig_words_file_);
}

Member Function Documentation

bool tesseract::Dict::absolute_garbage	(	const WERD_CHOICE &	word,
		const UNICHARSET &	unicharset
	)

Returns true if the word looks like an absolute garbage (e.g. image mistakenly recognized as text).

Definition at line 78 of file context.cpp.

                                                          {
  if (word.length() < kMinAbsoluteGarbageWordLength) return false;
  int num_alphanum = 0;
  for (int x = 0; x < word.length(); ++x) {
    num_alphanum += (unicharset.get_isalpha(word.unichar_id(x)) ||
                     unicharset.get_isdigit(word.unichar_id(x)));
  }
  return (static_cast<float>(num_alphanum) /
          static_cast<float>(word.length()) < kMinAbsoluteGarbageAlphanumFrac);
}

bool tesseract::Dict::AcceptableChoice	(	BLOB_CHOICE_LIST_VECTOR *	Choices,
		WERD_CHOICE *	BestChoice,
		DANGERR *	fixpt,
		ACCEPTABLE_CHOICE_CALLER	caller,
		bool *	modified_blobs
	)

Returns true if the given best_choice is good enough to stop.

Definition at line 170 of file stopper.cpp.

                                                  {
  float CertaintyThreshold = stopper_nondict_certainty_base;
  int WordSize;
  if (modified_blobs != NULL) *modified_blobs = false;

  if (stopper_no_acceptable_choices) return false;

  if (fixpt != NULL) fixpt->clear();
  if (BestChoice->length() == 0)
    return false;
  if (caller == CHOPPER_CALLER && BestChoice->fragment_mark()) {
    if (stopper_debug_level >= 1) {
      cprintf("AcceptableChoice(): a choice with fragments beats BestChoice");
    }
    return false;
  }

  bool no_dang_ambigs = (GetMaxFixedLengthDawgIndex() >= 0 ||
                         NoDangerousAmbig(BestChoice, fixpt, true,
                                          Choices, modified_blobs));
  bool is_valid_word = valid_word_permuter(BestChoice->permuter(), false);
  bool is_case_ok = case_ok(*BestChoice, getUnicharset());

  if (stopper_debug_level >= 1)
    tprintf("\nStopper:  %s (word=%c, case=%c)\n",
            BestChoice->debug_string().string(),
            (is_valid_word ? 'y' : 'n'),
            (is_case_ok ? 'y' : 'n'));

  // Do not accept invalid words in PASS1.
  if (reject_offset_ <= 0.0f && !is_valid_word) return false;
  if (is_valid_word && is_case_ok) {
    WordSize = LengthOfShortestAlphaRun(*BestChoice);
    WordSize -= stopper_smallword_size;
    if (WordSize < 0)
      WordSize = 0;
    CertaintyThreshold += WordSize * stopper_certainty_per_char;
  }

  if (stopper_debug_level >= 1)
    tprintf("Stopper:  Certainty = %4.1f, Threshold = %4.1f\n",
            BestChoice->certainty(), CertaintyThreshold);

  if (no_dang_ambigs &&
      BestChoice->certainty() > CertaintyThreshold &&
      UniformCertainties(*Choices, *BestChoice)) {
    return true;
  } else {
    if (stopper_debug_level >= 2) {
      tprintf("AcceptableChoice() returned false"
              " (no_dang_ambig:%d cert:%g thresh:%g uniform:%d)\n",
              no_dang_ambigs, BestChoice->certainty(),
              CertaintyThreshold,
              UniformCertainties(*Choices, *BestChoice));
    }
    return false;
  }
}

bool tesseract::Dict::AcceptableResult ( const WERD_CHOICE & BestChoice )

Returns false if the best choice for the current word is questionable and should be tried again on the second pass or should be flagged to the user.

Definition at line 233 of file stopper.cpp.

                                                         {
  float CertaintyThreshold = stopper_nondict_certainty_base - reject_offset_;
  int WordSize;

  if (stopper_debug_level >= 1) {
    tprintf("\nRejecter: %s (word=%c, case=%c, unambig=%c)\n",
            BestChoice.debug_string().string(),
            (valid_word(BestChoice) ? 'y' : 'n'),
            (case_ok(BestChoice, getUnicharset()) ? 'y' : 'n'),
            ((list_rest (best_choices_) != NIL_LIST) ? 'n' : 'y'));
  }

  if (BestChoice.length() == 0 || CurrentWordAmbig())
    return false;
  if (BestChoice.fragment_mark()) {
    if (stopper_debug_level >= 1) {
      cprintf("AcceptableResult(): a choice with fragments beats BestChoice\n");
    }
    return false;
  }
  if (valid_word(BestChoice) && case_ok(BestChoice, getUnicharset())) {
    WordSize = LengthOfShortestAlphaRun(BestChoice);
    WordSize -= stopper_smallword_size;
    if (WordSize < 0)
      WordSize = 0;
    CertaintyThreshold += WordSize * stopper_certainty_per_char;
  }

  if (stopper_debug_level >= 1)
    cprintf ("Rejecter: Certainty = %4.1f, Threshold = %4.1f   ",
      BestChoice.certainty(), CertaintyThreshold);

  if (BestChoice.certainty() > CertaintyThreshold &&
      !stopper_no_acceptable_choices) {
    if (stopper_debug_level >= 1)
      cprintf("ACCEPTED\n");
    return true;
  }
  else {
    if (stopper_debug_level >= 1)
      cprintf("REJECTED\n");
    return false;
  }
}

void tesseract::Dict::add_document_word ( const WERD_CHOICE & best_choice )

Adds a word found on this document to the document specific dictionary.

Definition at line 690 of file dict.cpp.

                                                           {
  // Do not add hyphenated word parts to the document dawg.
  // hyphen_word_ will be non-NULL after the set_hyphen_word() is
  // called when the first part of the hyphenated word is
  // discovered and while the second part of the word is recognized.
  // hyphen_word_ is cleared in cc_recg() before the next word on
  // the line is recognized.
  if (hyphen_word_) return;

  char filename[CHARS_PER_LINE];
  FILE *doc_word_file;
  int stringlen = best_choice.length();

  if (!doc_dict_enable || valid_word(best_choice) ||
      CurrentWordAmbig() || stringlen < 2)
    return;

  // Discard words that contain >= kDocDictMaxRepChars repeating unichars.
  if (best_choice.length() >= kDocDictMaxRepChars) {
    int num_rep_chars = 1;
    UNICHAR_ID uch_id = best_choice.unichar_id(0);
    for (int i = 1; i < best_choice.length(); ++i) {
      if (best_choice.unichar_id(i) != uch_id) {
        num_rep_chars = 1;
        uch_id = best_choice.unichar_id(i);
      } else {
        ++num_rep_chars;
        if (num_rep_chars == kDocDictMaxRepChars) return;
      }
    }
  }

  if (best_choice.certainty() < doc_dict_certainty_threshold ||
      stringlen == 2) {
    if (best_choice.certainty() < doc_dict_pending_threshold)
      return;

    if (!pending_words_->word_in_dawg(best_choice)) {
      if (stringlen > 2 ||
          (stringlen == 2 &&
           getUnicharset().get_isupper(best_choice.unichar_id(0)) &&
           getUnicharset().get_isupper(best_choice.unichar_id(1)))) {
        pending_words_->add_word_to_dawg(best_choice);
      }
      return;
    }
  }

  if (save_doc_words) {
    strcpy(filename, getImage()->getCCUtil()->imagefile.string());
    strcat(filename, ".doc");
    doc_word_file = open_file (filename, "a");
    fprintf(doc_word_file, "%s\n",
            best_choice.debug_string().string());
    fclose(doc_word_file);
  }
  document_words_->add_word_to_dawg(best_choice);
}

void tesseract::Dict::AddNewChunk	(	VIABLE_CHOICE	Choice,
		int	Blob
	)

Increments the chunk count of the character in Choice which corresponds to Blob (index of the blob being split).

Definition at line 763 of file stopper.cpp.

                                                     {
  int i, LastChunk;
  for (i = 0, LastChunk = 0; i < Choice->Length; i++) {
    LastChunk += Choice->Blob[i].NumChunks;
    if (Blob < LastChunk) {
      (Choice->Blob[i].NumChunks)++;
      return;
    }
  }
  cprintf ("AddNewChunk failed:Choice->Length=%d, LastChunk=%d, Blob=%d\n",
           Choice->Length, LastChunk, Blob);
  assert(false);  // this should never get executed
}

void tesseract::Dict::adjust_non_word	(	WERD_CHOICE *	word,
		float *	certainty_array,
		bool	debug
	)		`[inline]`

Definition at line 713 of file dict.h.

                                                                              {
    adjust_word(word, certainty_array, NULL, true, 0.0f, debug);
  }

void tesseract::Dict::adjust_word	(	WERD_CHOICE *	word,
		float *	certainty_array,
		const BLOB_CHOICE_LIST_VECTOR *	char_choices,
		bool	nonword,
		float	additional_adjust,
		bool	debug
	)

Adjusts the rating of the given word.

Definition at line 749 of file dict.cpp.

                                   {
  bool is_han = (char_choices != NULL &&
                 getUnicharset().han_sid() != getUnicharset().null_sid() &&
                 get_top_word_script(*char_choices, getUnicharset()) ==
                 getUnicharset().han_sid());
  bool case_is_ok = (is_han || case_ok(*word, getUnicharset()));
  bool punc_is_ok = (is_han || !nonword || valid_punctuation(*word));

  float adjust_factor = additional_adjust;
  float new_rating = word->rating();
  if (debug) {
    tprintf("%sWord: %s %4.2f ", nonword ? "Non-" : "",
            word->debug_string().string(), word->rating());
  }
  new_rating += kRatingPad;
  if (nonword) {  // non-dictionary word
    if (case_is_ok && punc_is_ok) {
      adjust_factor += segment_penalty_dict_nonword;
      new_rating *= adjust_factor;
      if (debug) tprintf(", W");
    } else {
      adjust_factor += segment_penalty_garbage;
      new_rating *= adjust_factor;
      if (debug) {
        if (!case_is_ok) tprintf(", C");
        if (!punc_is_ok) tprintf(", P");
      }
    }
  } else {  // dictionary word
    if (case_is_ok) {
      if (!is_han && freq_dawg_ != NULL && freq_dawg_->word_in_dawg(*word)) {
        word->set_permuter(FREQ_DAWG_PERM);
        adjust_factor += segment_penalty_dict_frequent_word;
        new_rating *= adjust_factor;
        if (debug) tprintf(", F");
      } else {
        adjust_factor += segment_penalty_dict_case_ok;
        new_rating *= adjust_factor;
        if (debug) tprintf(", ");
      }
    } else {
      adjust_factor += segment_penalty_dict_case_bad;
      new_rating *= adjust_factor;
      if (debug) tprintf(", C");
    }
  }
  new_rating -= kRatingPad;
  word->set_rating(new_rating);
  if (debug) tprintf(" %4.2f --> %4.2f\n", adjust_factor, new_rating);
  LogNewChoice(adjust_factor, certainty_array, false, word);
}

void tesseract::Dict::adjust_word	(	WERD_CHOICE *	word,
		float *	certainty_array,
		bool	debug
	)		`[inline]`

Definition at line 710 of file dict.h.

                                                                          {
    adjust_word(word, certainty_array, NULL, false, 0.0f, debug);
  }

bool tesseract::Dict::AlternativeChoicesWorseThan ( FLOAT32 Threshold )

Returns true if there are no alternative choices for the current word or if all alternatives have an adjust factor worse than Threshold.

Definition at line 278 of file stopper.cpp.

                                                        {
  LIST Alternatives;
  VIABLE_CHOICE Choice;
  Alternatives = list_rest (best_choices_);
  iterate(Alternatives) {
    Choice = (VIABLE_CHOICE) first_node (Alternatives);
    if (Choice->AdjustFactor <= Threshold)
      return false;
  }
  return true;
}

bool tesseract::Dict::ambigs_mode ( float rating_limit ) [inline]

Returns true if we are operating in ambigs mode.

Definition at line 180 of file dict.h.

                                              {
    return rating_limit <= 0.0;
  }

void tesseract::Dict::append_choices	(	const char *	debug,
		const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		const BLOB_CHOICE &	blob_choice,
		int	char_choice_index,
		const CHAR_FRAGMENT_INFO *	prev_char_frag_info,
		WERD_CHOICE *	word,
		float	certainties[],
		float *	limit,
		WERD_CHOICE *	best_choice,
		int *	attempts_left,
		void *	more_args
	)

append_choices

Checks to see whether or not the next choice is worth appending to the word being generated. If so then keeps going deeper into the word.

This function assumes that Dict::go_deeper_fxn_ is set.

Definition at line 1477 of file permute.cpp.

                     {
  int word_ending =
    (char_choice_index == char_choices.length() - 1) ? true : false;

  // Deal with fragments.
  CHAR_FRAGMENT_INFO char_frag_info;
  if (!fragment_state_okay(blob_choice.unichar_id(), blob_choice.rating(),
                           blob_choice.certainty(), prev_char_frag_info, debug,
                           word_ending, &char_frag_info)) {
    return;  // blob_choice must be an invalid fragment
  }
  // Search the next letter if this character is a fragment.
  if (char_frag_info.unichar_id == INVALID_UNICHAR_ID) {
    permute_choices(debug, char_choices, char_choice_index + 1,
                    &char_frag_info, word, certainties, limit,
                    best_choice, attempts_left, more_args);
    return;
  }

  // Add the next unichar.
  float old_rating = word->rating();
  float old_certainty = word->certainty();
  uinT8 old_permuter = word->permuter();
  certainties[word->length()] = char_frag_info.certainty;
  word->append_unichar_id_space_allocated(
      char_frag_info.unichar_id, char_frag_info.num_fragments,
      char_frag_info.rating, char_frag_info.certainty);

  // Explore the next unichar.
  (this->*go_deeper_fxn_)(debug, char_choices, char_choice_index,
                          &char_frag_info, word_ending, word, certainties,
                          limit, best_choice, attempts_left, more_args);

  // Remove the unichar we added to explore other choices in it's place.
  word->remove_last_unichar_id();
  word->set_rating(old_rating);
  word->set_certainty(old_certainty);
  word->set_permuter(old_permuter);
}

int tesseract::Dict::case_ok	(	const WERD_CHOICE &	word,
		const UNICHARSET &	unicharset
	)

Check a string to see if it matches a set of lexical rules.

Definition at line 58 of file context.cpp.

                                                                       {
  int last_state = 0;
  int state = 0;
  int x;
  for (x = 0; x < word.length(); ++x) {
    UNICHAR_ID ch_id = word.unichar_id(x);
    if (unicharset.get_isupper(ch_id))
      state = case_state_table[state][1];
    else if (unicharset.get_islower(ch_id))
      state = case_state_table[state][2];
    else if (unicharset.get_isdigit(ch_id))
      state = case_state_table[state][3];
    else
      state = case_state_table[state][0];
    if (state == -1) return false;
    last_state = state;
  }
  return state != 5; // single lower is bad
}

bool tesseract::Dict::ChoiceAccumEnabled ( ) [inline]

Definition at line 346 of file dict.h.

{ return keep_word_choices_; }

int tesseract::Dict::ChoiceSameAs	(	const WERD_CHOICE &	WordChoice,
		VIABLE_CHOICE	ViableChoice
	)

Compares the corresponding strings of WordChoice and ViableChoice and returns true if they are the same.

Definition at line 853 of file stopper.cpp.

                                                   {
  return (StringSameAs(WordChoice, ViableChoice));
}

const char * tesseract::Dict::choose_il1	(	const char *	first_char,
		const char *	second_char,
		const char *	third_char,
		const char *	prev_char,
		const char *	next_char,
		const char *	next_next_char
	)

Definition at line 1161 of file permute.cpp.

                                                         {
  inT32 type1;                   //1/I/l type of first choice
  inT32 type2;                   //1/I/l type of second choice
  inT32 type3;                   //1/I/l type of third choice

  int first_char_length = strlen(first_char);
  int prev_char_length = strlen(prev_char);
  int next_char_length = strlen(next_char);
  int next_next_char_length = strlen(next_next_char);

  if (*first_char == 'l' && *second_char != '\0') {
    if (*second_char == 'I'
        && (((prev_char_length != 0 &&
            getUnicharset().get_isupper (prev_char, prev_char_length)) &&
            (next_char_length == 0 ||
             !getUnicharset().get_islower (next_char, next_char_length)) &&
            (next_char_length == 0 ||
             !getUnicharset().get_isdigit (next_char, next_char_length))) ||
            ((next_char_length != 0 &&
             getUnicharset().get_isupper (next_char, next_char_length)) &&
            (prev_char_length == 0 ||
             !getUnicharset().get_islower (prev_char, prev_char_length)) &&
            (prev_char_length == 0 ||
             !getUnicharset().get_isdigit (prev_char, prev_char_length)))))
      first_char = second_char;  //override
    else if (*second_char == '1' || *third_char == '1') {
      if ((next_char_length != 0 &&
           getUnicharset().get_isdigit (next_char, next_char_length)) ||
          (prev_char_length != 0 &&
           getUnicharset().get_isdigit (prev_char, prev_char_length))
          || (*next_char == 'l' &&
          (next_next_char_length != 0 &&
           getUnicharset().get_isdigit (next_next_char,
                                        next_next_char_length)))) {
        first_char = "1";
        first_char_length = 1;
      }
      else if ((prev_char_length == 0 ||
                !getUnicharset().get_islower (prev_char, prev_char_length)) &&
               ((next_char_length == 0 ||
                 !getUnicharset().get_islower (next_char, next_char_length)) ||
                (*next_char == 's' &&
                *next_next_char == 't'))) {
        if (((*prev_char != '\'' && *prev_char != '`') || *next_char != '\0')
            && ((*next_char != '\'' && *next_char != '`')
                || *prev_char != '\0')) {
          first_char = "1";
          first_char_length = 1;
        }
      }
    }
    if (*first_char == 'l' && *next_char != '\0' &&
        (prev_char_length == 0 ||
         !getUnicharset().get_isalpha (prev_char, prev_char_length))) {
      type1 = 2;

      if (*second_char == '1')
        type2 = 0;
      else if (*second_char == 'I')
        type2 = 1;
      else if (*second_char == 'l')
        type2 = 2;
      else
        type2 = type1;

      if (*third_char == '1')
        type3 = 0;
      else if (*third_char == 'I')
        type3 = 1;
      else if (*third_char == 'l')
        type3 = 2;
      else
        type3 = type1;

#if 0
      if (bigram_counts[*next_char][type2] >
      bigram_counts[*next_char][type1]) {
        first_char = second_char;
        type1 = type2;
      }
      if (bigram_counts[*next_char][type3] >
      bigram_counts[*next_char][type1]) {
        first_char = third_char;
      }
#endif
    }
  }
  return first_char;
}

void tesseract::Dict::ClearBestChoiceAccum ( )

Clears best_choices_ list accumulated by the stopper.

Definition at line 437 of file stopper.cpp.

                                {
  if (best_choices_) destroy_nodes(best_choices_, DeleteViableChoiceStruct);
  best_choices_ = NIL_LIST;
}

bool tesseract::Dict::compound_marker ( UNICHAR_ID unichar_id ) [inline]

Definition at line 110 of file dict.h.

                                                     {
    return (unichar_id == getUnicharset().unichar_to_id("-") ||
            unichar_id == getUnicharset().unichar_to_id("/"));
  }

bool tesseract::Dict::ConstraintsOk	(	const DawgInfoVector &	constraints,
		int	word_end,
		DawgType	current_dawg_type
	)		const `[inline]`

At word ending make sure all the recorded constraints are satisfied. Each constraint signifies that we found a beginning pattern in a pattern dawg. Check that this pattern can end here (e.g. if some leading punctuation is found this would ensure that we are not expecting any particular trailing punctuation after the word).

Definition at line 628 of file dict.h.

                                                                            {
    if (!word_end) return true;
    if (current_dawg_type == DAWG_TYPE_PUNCTUATION) return true;
    for (int c = 0; c < constraints.length(); ++c) {
      const DawgInfo &cinfo = constraints[c];
      Dawg *cdawg = dawgs_[cinfo.dawg_index];
      if (!cdawg->end_of_word(cinfo.ref)) {
        if (dawg_debug_level >= 3) {
          tprintf("Constraint [%d, " REFFORMAT "] is not satisfied\n",
                  cinfo.dawg_index, cinfo.ref);
        }
        return false;
      }
    }
    return true;
  }

void tesseract::Dict::copy_hyphen_info ( WERD_CHOICE * word ) const [inline]

If this word is hyphenated copy the base word (the part on the line before) of a hyphenated word into the given word. This function assumes that word is not NULL.

Definition at line 128 of file dict.h.

                                                        {
    if (this->hyphenated()) {
      *word = *hyphen_word_;
      if (hyphen_debug_level) word->print("copy_hyphen_info: ");
    }
  }

FLOAT32 tesseract::Dict::CurrentBestChoiceAdjustFactor ( )

Returns the adjustment factor for the best choice for the current word.

Definition at line 295 of file stopper.cpp.

                                            {
  VIABLE_CHOICE BestChoice;
  if (best_choices_ == NIL_LIST)
    return (MAX_FLOAT32);
  BestChoice = (VIABLE_CHOICE) first_node (best_choices_);
  return (BestChoice->AdjustFactor);
}

bool tesseract::Dict::CurrentBestChoiceIs ( const WERD_CHOICE & WordChoice )

Returns true if WordChoice is the same as the current best choice.

Definition at line 290 of file stopper.cpp.

                                                            {
  return (best_choices_ != NIL_LIST &&
          StringSameAs(WordChoice, (VIABLE_CHOICE)first_node(best_choices_)));
}

bool tesseract::Dict::CurrentWordAmbig ( )

Returns true if there are multiple good choices for the current word.

Definition at line 304 of file stopper.cpp.

                            {
  return (list_rest (best_choices_) != NIL_LIST);
}

WERD_CHOICE * tesseract::Dict::dawg_permute_and_select	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		float	rating_limit,
		int	sought_word_length,
		int	start_char_choice_index
	)

Recursively explore all the possible character combinations in the given char_choices. Use go_deeper_dawg_fxn() to explore all the dawgs in the dawgs_ vector in parallel and discard invalid words.

Allocate and return a WERD_CHOICE with the best valid word found.

dawg_permute_and_select

Recursively explore all the possible character combinations in the given char_choices. Use go_deeper_dawg_fxn() to search all the dawgs in the dawgs_ vector in parallel and discard invalid words.

If sought_word_length is not kAnyWordLength, the function only searches for a valid word formed by the given char_choices in one fixed length dawg (that contains words of length sought_word_length) starting at the start_char_choice_index.

Allocate and return a WERD_CHOICE with the best valid word found.

Definition at line 263 of file permdawg.cpp.

                                                         {
  WERD_CHOICE *best_choice = new WERD_CHOICE(&getUnicharset());
  best_choice->make_bad();
  best_choice->set_rating(rating_limit);
  if (char_choices.length() == 0) return best_choice;
  DawgInfoVector *active_dawgs = new DawgInfoVector[char_choices.length() + 1];
  DawgInfoVector *constraints =  new DawgInfoVector[char_choices.length() + 1];
  init_active_dawgs(sought_word_length, &(active_dawgs[0]),
                    ambigs_mode(rating_limit));
  init_constraints(&(constraints[0]));
  int end_char_choice_index = (sought_word_length == kAnyWordLength) ?
    char_choices.length()-1 : start_char_choice_index+sought_word_length-1;
  // Need to skip accumulating word choices if we are only searching a part of
  // the word (e.g. for the phrase search in non-space delimited languages).
  // Also need to skip accumulating choices if char_choices are expanded
  // with ambiguities.
  bool re_enable_choice_accum = ChoiceAccumEnabled();
  if (sought_word_length != kAnyWordLength ||
      ambigs_mode(rating_limit)) DisableChoiceAccum();
  DawgArgs dawg_args(&(active_dawgs[0]), &(constraints[0]),
                     &(active_dawgs[1]), &(constraints[1]),
                     (segment_penalty_dict_case_bad /
                      segment_penalty_dict_case_ok),
                     NO_PERM, sought_word_length, end_char_choice_index);
  WERD_CHOICE word(&getUnicharset(), MAX_WERD_LENGTH);
  copy_hyphen_info(&word);
  // Discard rating and certainty of the hyphen base (if any).
  word.set_rating(0.0);
  word.set_certainty(0.0);
  if (word.length() + char_choices.length() > MAX_WERD_LENGTH) {
    delete[] active_dawgs;
    delete[] constraints;
    return best_choice;  // the word is too long to permute
  }
  float certainties[MAX_WERD_LENGTH];
  this->go_deeper_fxn_ = &tesseract::Dict::go_deeper_dawg_fxn;
  int attempts_left = max_permuter_attempts;
  permute_choices((permute_debug && dawg_debug_level) ?
                  "permute_dawg_debug" : NULL,
                  char_choices, start_char_choice_index, NULL, &word,
                  certainties, &rating_limit, best_choice, &attempts_left,
                  &dawg_args);
  delete[] active_dawgs;
  delete[] constraints;
  if (re_enable_choice_accum) EnableChoiceAccum();
  return best_choice;
}

WERD_CHOICE* tesseract::Dict::dawg_permute_and_select	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		float	rating_limit
	)		`[inline]`

Definition at line 191 of file dict.h.

                                                                       {
    return dawg_permute_and_select(char_choices, rating_limit,
                                   kAnyWordLength, 0);
  }

void tesseract::Dict::DebugWordChoices ( )

Prints the current choices for this word to stdout.

Definition at line 309 of file stopper.cpp.

                            {
  LIST Choices;
  int i;
  char LabelString[80];
  VIABLE_CHOICE VChoice = (VIABLE_CHOICE)first_node(best_choices_);
  bool force_debug =
    fragments_debug && VChoice != NULL && VChoice->ComposedFromCharFragments;

  if (stopper_debug_level >= 1 || force_debug ||
  (((STRING)word_to_debug).length() > 0 && best_choices_ &&
       StringSameAs(word_to_debug.string(), word_to_debug_lengths.string(),
                    (VIABLE_CHOICE)first_node(best_choices_)))) {
    if (best_raw_choice_)
      PrintViableChoice(stderr, "\nBest Raw Choice:   ", best_raw_choice_);

    i = 1;
    Choices = best_choices_;
    if (Choices)
      cprintf("\nBest Cooked Choices:\n");
    iterate(Choices) {
      sprintf(LabelString, "Cooked Choice #%d:  ", i);
      PrintViableChoice(stderr, LabelString,
                        (VIABLE_CHOICE)first_node(Choices));
      i++;
    }
  }
}

int tesseract::Dict::def_letter_is_okay	(	void *	void_dawg_args,
		UNICHAR_ID	unichar_id,
		bool	word_end
	)		const

Returns the maximal permuter code (from ccstruct/ratngs.h) if in light of the current state the letter at word_index in the given word is allowed according to at least one of the dawgs in dawgs_, otherwise returns NO_PERM.

The state is described by void_dawg_args, which are interpreted as DawgArgs and contain two relevant input vectors: active_dawgs and constraints. Each entry in the active_dawgs vector contains an index into the dawgs_ vector and an EDGE_REF that indicates the last edge followed in the dawg. Each entry in the constraints vector contains an index into the dawgs_ vector and an EDGE_REF that indicates an edge in a pattern dawg followed to match a pattern. Currently constraints are used to save the state of punctuation dawgs after leading punctuation was found.

Input: At word_index 0 dawg_args->active_dawgs should contain an entry for each dawg whose type has a bit set in kBeginningDawgsType, dawg_args->constraints should be empty. EDGE_REFs in active_dawgs and constraints vectors should be initialized to NO_EDGE. If hyphen state needs to be applied, initial dawg_args->active_dawgs and dawg_args->constrains can be copied from the saved hyphen state (maintained by Dict). For word_index > 0 the corresponding state (active_dawgs and constraints) can be obtained from dawg_args->updated_* passed to def_letter_is_okay for word_index-1. Note: the function assumes that active_dags, constraints and updated_* member variables of dawg_args are not NULL.

Output: The function fills in dawg_args->updated_active_dawgs vector with the entries for dawgs that contain the word up to the letter at word_index. The new constraints (if any) are added to dawg_args->updated_constraints, the constraints from dawg_args->constraints are also copied into it.

Detailed description: In order to determine whether the word is still valid after considering all the letters up to the one at word_index the following is done for each entry in dawg_args->active_dawgs:

next starting node is obtained from entry.ref and edge_char_of() is called to obtain the next edge
if a valid edge is found, the function returns the updated permuter code true and an entry [entry.dawg_index, edge] is inserted in dawg_args->updated_active_dawgs otherwise:
- if we are dealing with dawg of type DAWG_TYPE_PUNCTUATION, edge_char_of() is called again, but now with kPatternUnicharID as unichar_id; if a valid edge is found it is recorded in dawg_args->updated_constraints
- the function checks whether the word can end with the previous letter
- each successor of the dawg (e.g. dawgs with type DAWG_TYPE_WORD could be successors to dawgs with type DAWG_TYPE_PUNCTUATION; the successors are defined by successors_ vector) is explored and if a letter is found in the successor dawg, a new entry is inserted into dawg_args->updated_active_dawgs with EDGE_REF being either NO_EDGE or an EDGE_REF recorded in constraints vector for the corresponding dawg index

Definition at line 380 of file dict.cpp.

                                                  {
  DawgArgs *dawg_args = reinterpret_cast<DawgArgs*>(void_dawg_args);

  if (dawg_debug_level >= 3) {
    tprintf("def_letter_is_okay: current unichar=%s word_end=%d"
            " num active dawgs=%d num constraints=%d\n",
            getUnicharset().debug_str(unichar_id).string(), word_end,
            dawg_args->active_dawgs->length(),
            dawg_args->constraints->length());
  }

  // Do not accept words that contain kPatternUnicharID.
  // (otherwise pattern dawgs would not function correctly).
  // Do not accept words containing INVALID_UNICHAR_IDs.
  if (unichar_id == Dawg::kPatternUnicharID ||
      unichar_id == INVALID_UNICHAR_ID) {
    dawg_args->permuter = NO_PERM;
    return NO_PERM;
  }

  // Initialization.
  PermuterType curr_perm = NO_PERM;
  dawg_args->updated_active_dawgs->clear();
  const DawgInfoVector &constraints = *(dawg_args->constraints);
  *dawg_args->updated_constraints = constraints;

  // Go over the active_dawgs vector and insert DawgInfo records with the
  // updated ref (an edge with the corresponding unichar id) into
  // dawg_args->updated_active_dawgs.
  for (int a = 0; a < dawg_args->active_dawgs->length(); ++a) {
    const DawgInfo &info = (*dawg_args->active_dawgs)[a];
    const Dawg *dawg = dawgs_[info.dawg_index];
    // dawg_unichar_id will contain the literal unichar_id to be found in the
    // dawgs (e.g. didgit pattern if unichar_id is a digit and dawg contains
    // number patterns, word pattern if dawg is a puncutation dawg and we
    // reached an end of beginning puntuation pattern, etc).
    UNICHAR_ID dawg_unichar_id = unichar_id;

    // If we are dealing with the pattern dawg, look up all the
    // possible edges, not only for the exact unichar_id, but also
    // for all its character classes (alpha, digit, etc).
    if (dawg->type() == DAWG_TYPE_PATTERN) {
      ProcessPatternEdges(dawg, info, dawg_unichar_id, word_end,
                          dawg_args, &curr_perm);
      // There can't be any successors to dawg that is of type
      // DAWG_TYPE_PATTERN, so we are done examining this DawgInfo.
      continue;
    }

    // The number dawg generalizes all digits to be kPatternUnicharID,
    // so try to match kPatternUnicharID if the current unichar is a digit.
    if (dawg->type() == DAWG_TYPE_NUMBER &&
        getUnicharset().get_isdigit(dawg_unichar_id)) {
      dawg_unichar_id = Dawg::kPatternUnicharID;
    }

    // Find the edge out of the node for the dawg_unichar_id.
    NODE_REF node = GetStartingNode(dawg, info.ref);
    EDGE_REF edge = (node != NO_EDGE) ?
      dawg->edge_char_of(node, dawg_unichar_id, word_end) : NO_EDGE;

    if (dawg_debug_level >= 3) {
      tprintf("Active dawg: [%d, " REFFORMAT "] edge=" REFFORMAT "\n",
              info.dawg_index, node, edge);
    }

    if (edge != NO_EDGE) {  // the unichar was found in the current dawg
      if (ConstraintsOk(*(dawg_args->updated_constraints),
                        word_end, dawg->type())) {
        if (dawg_debug_level >=3) {
          tprintf("Letter found in dawg %d\n", info.dawg_index);
        }
        if (dawg->permuter() > curr_perm) curr_perm = dawg->permuter();
        dawg_args->updated_active_dawgs->add_unique(
            DawgInfo(info.dawg_index, edge), dawg_debug_level > 0,
            "Append current dawg to updated active dawgs: ");
      }
    } else if (dawg_args->sought_word_length == kAnyWordLength) {
      // The unichar was not found in the current dawg.
      // Explore the successor dawgs (but only if we are not
      // just searching one dawg with a fixed word length).

      // Handle leading/trailing punctuation dawgs that denote a word pattern
      // as an edge with kPatternUnicharID. If such an edge is found we add a
      // constraint denoting the state of the dawg before the word pattern.
      // This constraint will be applied later when this dawg is found among
      // successor dawgs as well potentially at the end of the word.
      if (dawg->type() == DAWG_TYPE_PUNCTUATION) {
        edge = dawg->edge_char_of(node, Dawg::kPatternUnicharID, word_end);
        if (edge != NO_EDGE) {
          dawg_args->updated_constraints->add_unique(
              DawgInfo(info.dawg_index, edge), dawg_debug_level > 0,
              "Recording constraint: ");
        } else {
          // Do not explore successors of this dawg, since this
          // must be invalid leading or trailing punctuation.
          if (dawg_debug_level >= 3) {
            tprintf("Invalid punctuation from dawg %d\n", info.dawg_index);
          }
          continue;
        }
      }

      if (info.ref == NO_EDGE) {
        if (dawg_debug_level >= 3) {
          tprintf("No letters matched in dawg %d\n", info.dawg_index);
        }
        continue;
      }

      // Discard the dawg if the pattern can not end at previous letter.
      if (edge == NO_EDGE &&  // previous part is not leading punctuation
          !dawg->end_of_word(info.ref)) {
        if (dawg_debug_level >= 3) {
          tprintf("No valid pattern end in dawg %d\n", info.dawg_index);
        }
        continue;
      }

      // Look for the unichar in each of this dawg's successors
      // and append those in which it is found to active_dawgs.
      const SuccessorList &slist = *(successors_[info.dawg_index]);
      for (int s = 0; s < slist.length(); ++s) {
        int sdawg_index = slist[s];
        const Dawg *sdawg = dawgs_[sdawg_index];
        NODE_REF snode = 0;
        // Apply constraints to the successor dawg.
        for (int c = 0; c < constraints.length(); ++c) {
          // If the successor dawg is described in the constraints change
          // the start ref from 0 to the one recorded as the constraint.
          const DawgInfo &cinfo = constraints[c];
          if (cinfo.dawg_index == sdawg_index) {
            snode = sdawg->next_node(cinfo.ref);
            // Make sure we do not search the successor dawg if after
            // applying the saved constraint we are at the end of the word.
            if (snode == 0) snode = NO_EDGE;
            if (dawg_debug_level >= 3) {
               tprintf("Applying constraint [%d, " REFFORMAT "]\n",
                       sdawg_index, snode);
            }
          }
        }
        // Look for the letter in this successor dawg.
        EDGE_REF sedge = sdawg->edge_char_of(snode, unichar_id, word_end);
        // If we found the letter append sdawg to the active_dawgs list.
        if (sedge != NO_EDGE &&
            ConstraintsOk(*(dawg_args->updated_constraints), word_end,
                          dawgs_[sdawg_index]->type())) {
          if (dawg_debug_level >= 3) {
            tprintf("Letter found in the successor dawg %d\n", sdawg_index);
          }
          if (sdawg->permuter() > curr_perm) curr_perm = sdawg->permuter();
          if (sdawg->next_node(sedge) != 0) {  // if not word end
            dawg_args->updated_active_dawgs->add_unique(
              DawgInfo(sdawg_index, sedge), dawg_debug_level > 0,
              "Append successor to updated active dawgs: ");
          }
        }
      }  // end successors loop
    }  // end if/else
  }  // end for
  // Update dawg_args->permuter if it used to be NO_PERM or became NO_PERM
  // or if we found the current letter in a non-punctuation dawg. This
  // allows preserving information on which dawg the "core" word came from.
  // Keep the old value of dawg_args->permuter if it is COMPOUND_PERM.
  if (dawg_args->permuter == NO_PERM || curr_perm == NO_PERM ||
      (curr_perm != PUNC_PERM && dawg_args->permuter != COMPOUND_PERM)) {
    dawg_args->permuter = curr_perm;
  }
  return dawg_args->permuter;
}

double tesseract::Dict::def_probability_in_context	(	const char *	lang,
		const char *	context,
		int	context_bytes,
		const char *	character,
		int	character_bytes
	)		`[inline]`

Default (no-op) implementation of probability in context function.

Definition at line 584 of file dict.h.

                                                  {
    (void) context;
    (void) context_bytes;
    (void) character;
    (void) character_bytes;
    return 0.0;
  }

void tesseract::Dict::DisableChoiceAccum ( ) [inline]

Definition at line 344 of file dict.h.

{ keep_word_choices_ = false; }

void tesseract::Dict::EnableChoiceAccum ( ) [inline]

Definition at line 345 of file dict.h.

{ keep_word_choices_ = true; }

void tesseract::Dict::End ( )

Definition at line 335 of file dict.cpp.

               {
  if (dawgs_.length() == 0)
    return;  // Not safe to call twice.
  dawgs_.delete_data_pointers();
  successors_.delete_data_pointers();
  dawgs_.clear();
  delete bigram_dawg_;
  successors_.clear();
  document_words_ = NULL;
  max_fixed_length_dawgs_wdlen_ = -1;
  if (pending_words_ != NULL) {
    delete pending_words_;
    pending_words_ = NULL;
  }
}

void tesseract::Dict::end_permute ( )

void tesseract::Dict::EndDangerousAmbigs ( )

Definition at line 753 of file stopper.cpp.

{}

void tesseract::Dict::FillViableChoice	(	const WERD_CHOICE &	WordChoice,
		FLOAT32	AdjustFactor,
		const float	Certainties[],
		VIABLE_CHOICE	ViableChoice
	)

Fill ViableChoice with information from WordChoice, AChoice, AdjustFactor, and Certainties.

Definition at line 918 of file stopper.cpp.

                                                        {
  ViableChoice->Init(WordChoice, current_segmentation_, Certainties,
                     AdjustFactor);

}

void tesseract::Dict::FilterWordChoices ( )

Removes from best_choices_ all choices which are not within a reasonable range of the best choice.

Definition at line 354 of file stopper.cpp.

                             {
  EXPANDED_CHOICE BestChoice;

  if (best_choices_ == NIL_LIST || second_node (best_choices_) == NIL_LIST)
    return;

  // Compute certainties and class for each chunk in best choice.
  VIABLE_CHOICE_STRUCT *best_choice =
      (VIABLE_CHOICE_STRUCT *)first_node(best_choices_);
  ExpandChoice(best_choice, &BestChoice);
  if (stopper_debug_level >= 2)
    PrintViableChoice(stderr, "\nFiltering against best choice: ", best_choice);
  TessResultCallback2<int, void*, void*>* is_bad =
      NewPermanentTessCallback(this, &Dict::FreeBadChoice);
  set_rest(best_choices_, delete_d(list_rest(best_choices_),
                                   &BestChoice, is_bad));
  delete is_bad;
}

void tesseract::Dict::FindClassifierErrors	(	FLOAT32	MinRating,
		FLOAT32	MaxRating,
		FLOAT32	RatingMargin,
		FLOAT32	Thresholds[]
	)

Compares the best choice for the current word to the best raw choice to determine which characters were classified incorrectly by the classifier. Then places a separate threshold into Thresholds for each character in the word. If the classifier was correct, MaxRating is placed into Thresholds. If the classifier was incorrect, the avg. match rating (error percentage) of the classifier's incorrect choice minus some margin is placed into thresholds.This can then be used by the caller to try to create a new template for the desired class that will classify the character with a rating better than the threshold value. The match rating placed into Thresholds is never allowed to be below MinRating in order to prevent trying to make overly tight templates. MinRating limits how tight to make a template. MaxRating limits how loose to make a template. RatingMargin denotes the amount of margin to put in template.

Definition at line 373 of file stopper.cpp.

                                                      {
  EXPANDED_CHOICE BestRaw;
  VIABLE_CHOICE Choice;
  int i, j, Chunk;
  FLOAT32 AvgRating;
  int NumErrorChunks;

  assert (best_choices_ != NIL_LIST);
  assert (best_raw_choice_ != NULL);

  ExpandChoice(best_raw_choice_, &BestRaw);
  Choice = (VIABLE_CHOICE) first_node (best_choices_);

  for (i = 0, Chunk = 0; i < Choice->Length; i++, Thresholds++) {
    AvgRating = 0.0;
    NumErrorChunks = 0;

    for (j = 0; j < Choice->Blob[i].NumChunks; j++, Chunk++) {
      if (Choice->Blob[i].Class != BestRaw.ChunkClass[Chunk]) {
        AvgRating += BestRaw.ChunkCertainty[Chunk];
        NumErrorChunks++;
      }
    }

    if (NumErrorChunks > 0) {
      AvgRating /= NumErrorChunks;
      *Thresholds = (AvgRating / -certainty_scale) * (1.0 - RatingMargin);
    }
    else
      *Thresholds = MaxRating;

    if (*Thresholds > MaxRating)
      *Thresholds = MaxRating;
    if (*Thresholds < MinRating)
      *Thresholds = MinRating;
  }
}

bool tesseract::Dict::fragment_state_okay	(	UNICHAR_ID	curr_unichar_id,
		float	curr_rating,
		float	curr_certainty,
		const CHAR_FRAGMENT_INFO *	prev_char_frag_info,
		const char *	debug,
		int	word_ending,
		CHAR_FRAGMENT_INFO *	char_frag_info
	)

Semi-generic functions used by multiple permuters.

Definition at line 1281 of file permute.cpp.

                                                                   {
  const CHAR_FRAGMENT *this_fragment =
    getUnicharset().get_fragment(curr_unichar_id);
  const CHAR_FRAGMENT *prev_fragment =
    prev_char_frag_info != NULL ? prev_char_frag_info->fragment : NULL;

  // Print debug info for fragments.
  if (debug && (prev_fragment || this_fragment)) {
    cprintf("%s check fragments: choice=%s word_ending=%d\n", debug,
            getUnicharset().debug_str(curr_unichar_id).string(),
            word_ending);
    if (prev_fragment) {
      cprintf("prev_fragment %s\n", prev_fragment->to_string().string());
    }
    if (this_fragment) {
      cprintf("this_fragment %s\n", this_fragment->to_string().string());
    }
  }

  char_frag_info->unichar_id = curr_unichar_id;
  char_frag_info->fragment = this_fragment;
  char_frag_info->rating = curr_rating;
  char_frag_info->certainty = curr_certainty;
  char_frag_info->num_fragments = 1;
  if (prev_fragment && !this_fragment) {
    if (debug) tprintf("Skip choice with incomplete fragment\n");
    return false;
  }
  if (this_fragment) {
    // We are dealing with a fragment.
    char_frag_info->unichar_id = INVALID_UNICHAR_ID;
    if (prev_fragment) {
      if (!this_fragment->is_continuation_of(prev_fragment)) {
        if (debug) tprintf("Non-matching fragment piece\n");
        return false;
      }
      if (this_fragment->is_ending()) {
        char_frag_info->unichar_id =
          getUnicharset().unichar_to_id(this_fragment->get_unichar());
        char_frag_info->fragment = NULL;
        if (debug) {
          tprintf("Built character %s from fragments\n",
                  getUnicharset().debug_str(
                      char_frag_info->unichar_id).string());
        }
      } else {
        if (debug) tprintf("Record fragment continuation\n");
        char_frag_info->fragment = this_fragment;
      }
      // Update certainty and rating.
      char_frag_info->rating =
        prev_char_frag_info->rating + curr_rating;
      char_frag_info->num_fragments = prev_char_frag_info->num_fragments + 1;
      char_frag_info->certainty =
        MIN(curr_certainty, prev_char_frag_info->certainty);
    } else {
      if (this_fragment->is_beginning()) {
        if (debug) cprintf("Record fragment beginning\n");
      } else {
        if (debug) {
          tprintf("Non-starting fragment piece with no prev_fragment\n");
        }
        return false;
      }
    }
  }
  if (word_ending && char_frag_info->fragment) {
    if (debug) tprintf("Word can not end with a fragment\n");
    return false;
  }
  return true;
}

int tesseract::Dict::FreeBadChoice	(	void *	item1,
		void *	item2
	)

Definition at line 146 of file stopper.cpp.

                 {  // EXPANDED_CHOICE *BestChoice
  int i, j, Chunk;
  FLOAT32 Threshold;
  VIABLE_CHOICE Choice = reinterpret_cast<VIABLE_CHOICE>(item1);
  EXPANDED_CHOICE *BestChoice = reinterpret_cast<EXPANDED_CHOICE *>(item2);
  Threshold = StopperAmbigThreshold(BestChoice->Choice->AdjustFactor,
                                    Choice->AdjustFactor);
  for (i = 0, Chunk = 0; i < Choice->Length; i++) {
    for (j = 0; j < Choice->Blob[i].NumChunks; j++, Chunk++) {
      if (Choice->Blob[i].Class != BestChoice->ChunkClass[Chunk] &&
          Choice->Blob[i].Certainty - BestChoice->ChunkCertainty[Chunk] <
          Threshold) {
        if (stopper_debug_level >= 2)
          PrintViableChoice(stderr, "\nDiscarding bad choice:  ", Choice);
        delete Choice;
        return true;
      }
    }
  }
  return false;
}

WERD_CHOICE * tesseract::Dict::get_top_choice_word ( const BLOB_CHOICE_LIST_VECTOR & char_choices )

Return the top choice for each character as the choice for the word.

Definition at line 908 of file permute.cpp.

                                                 {
  WERD_CHOICE *top_word = new WERD_CHOICE(&getUnicharset(), MAX_PERM_LENGTH);
  float certainties[MAX_PERM_LENGTH];
  top_word->set_permuter(TOP_CHOICE_PERM);
  for (int x = 0; x < char_choices.length(); x++) {
    BLOB_CHOICE_IT blob_choice_it;
    blob_choice_it.set_to_list(char_choices.get(x));
    BLOB_CHOICE *top_choice = blob_choice_it.data();
    top_word->append_unichar_id_space_allocated(top_choice->unichar_id(), 1,
                                                top_choice->rating(),
                                                top_choice->certainty());
    certainties[x] = top_choice->certainty();
  }
  LogNewChoice(1.0, certainties, true, top_word);
  return top_word;
}

int tesseract::Dict::get_top_word_script	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		const UNICHARSET &	unicharset
	)

Definition at line 907 of file dict.cpp.

                                                            {
  int max_script = unicharset.get_script_table_size();
  int *sid = new int[max_script];
  int x;
  for (x = 0; x < max_script; x++) sid[x] = 0;
  for (x = 0; x < char_choices.length(); ++x) {
    BLOB_CHOICE_IT blob_choice_it(char_choices.get(x));
    sid[blob_choice_it.data()->script_id()]++;
  }
  if (unicharset.han_sid() != unicharset.null_sid()) {
    // Add the Hiragana & Katakana counts to Han and zero them out.
    if (unicharset.hiragana_sid() != unicharset.null_sid()) {
      sid[unicharset.han_sid()] += sid[unicharset.hiragana_sid()];
      sid[unicharset.hiragana_sid()] = 0;
    }
    if (unicharset.katakana_sid() != unicharset.null_sid()) {
      sid[unicharset.han_sid()] += sid[unicharset.katakana_sid()];
      sid[unicharset.katakana_sid()] = 0;
    }
  }
  // Note that high script ID overrides lower one on a tie, thus biasing
  // towards non-Common script (if sorted that way in unicharset file).
  int max_sid = 0;
  for (x = 1; x < max_script; x++)
    if (sid[x] >= sid[max_sid]) max_sid = x;
  if (sid[max_sid] < char_choices.length() / 2)
    max_sid = unicharset.null_sid();
  delete[] sid;
  return max_sid;
}

const LIST& tesseract::Dict::getBestChoices ( ) [inline]

Definition at line 721 of file dict.h.

{ return best_choices_; }

const Dawg* tesseract::Dict::GetDawg ( int index ) const [inline]

Return i-th dawg pointer recorded in the dawgs_ vector.

Definition at line 602 of file dict.h.

{ return dawgs_[index]; }

const Dawg* tesseract::Dict::GetFixedLengthDawg ( int word_length ) const [inline]

Return the pointer to the Dawg that contains words of length word_length.

Definition at line 608 of file dict.h.

                                                               {
    if (word_length > max_fixed_length_dawgs_wdlen_) return NULL;
    assert(dawgs_.size() > word_length);
    return dawgs_[word_length];
  }

const Image* tesseract::Dict::getImage ( ) const [inline]

Definition at line 94 of file dict.h.

                                {
    return image_ptr_;
  }

Image* tesseract::Dict::getImage ( ) [inline]

Definition at line 97 of file dict.h.

                    {
    return image_ptr_;
  }

const int tesseract::Dict::GetMaxFixedLengthDawgIndex ( ) const [inline]

Definition at line 613 of file dict.h.

                                                      {
    return max_fixed_length_dawgs_wdlen_;
  }

const Dawg* tesseract::Dict::GetPuncDawg ( ) const [inline]

Return the points to the punctuation dawg.

Definition at line 604 of file dict.h.

{ return punc_dawg_; }

static NODE_REF tesseract::Dict::GetStartingNode	(	const Dawg *	dawg,
		EDGE_REF	edge_ref
	)		`[inline, static]`

Returns the appropriate next node given the EDGE_REF.

Definition at line 617 of file dict.h.

                                                                              {
    if (edge_ref == NO_EDGE) return 0;  // beginning to explore the dawg
    NODE_REF node = dawg->next_node(edge_ref);
    if (node == 0) node = NO_EDGE;  // end of word
    return node;
  }

const Dawg* tesseract::Dict::GetUnambigDawg ( ) const [inline]

Return the points to the unambiguous words dawg.

Definition at line 606 of file dict.h.

{ return unambig_dawg_; }

const UnicharAmbigs& tesseract::Dict::getUnicharAmbigs ( ) [inline]

Definition at line 106 of file dict.h.

                                          {
    return getImage()->getCCUtil()->unichar_ambigs;
  }

const UNICHARSET& tesseract::Dict::getUnicharset ( ) const [inline]

Definition at line 100 of file dict.h.

                                          {
    return getImage()->getCCUtil()->unicharset;
  }

UNICHARSET& tesseract::Dict::getUnicharset ( ) [inline]

Definition at line 103 of file dict.h.

                              {
    return getImage()->getCCUtil()->unicharset;
  }

void tesseract::Dict::go_deeper_dawg_fxn	(	const char *	debug,
		const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		int	char_choice_index,
		const CHAR_FRAGMENT_INFO *	prev_char_frag_info,
		bool	word_ending,
		WERD_CHOICE *	word,
		float	certainties[],
		float *	limit,
		WERD_CHOICE *	best_choice,
		int *	attempts_left,
		void *	void_more_args
	)

If the choice being composed so far could be a dictionary word and we have not reached the end of the word keep exploring the char_choices further. Also: -- sets hyphen word if needed -- if word_ending is true and the word is better than best_choice, copies word to best_choice and logs new word choice

Definition at line 67 of file permdawg.cpp.

                                                                        {
  DawgArgs *more_args = reinterpret_cast<DawgArgs*>(void_more_args);
  word_ending = (char_choice_index == more_args->end_char_choice_index);
  int word_index = word->length() - 1;

  if (ambigs_mode(*limit)) {
    if (best_choice->rating() < *limit) return;
  } else {
    // Prune bad subwords
    if (more_args->rating_array[word_index] == NO_RATING) {
      more_args->rating_array[word_index] = word->rating();
    } else {
      float permdawg_limit = more_args->rating_array[word_index] *
        more_args->rating_margin + kPermDawgRatingPad;
      if (permdawg_limit < word->rating()) {
        if (permute_debug && dawg_debug_level) {
          tprintf("early pruned word rating=%4.2f,"
                  " permdawg_limit=%4.2f, word=%s\n", word->rating(),
                  permdawg_limit, word->debug_string().string());
        }
        return;
      }
    }
  }
  // Deal with hyphens
  if (word_ending && more_args->sought_word_length == kAnyWordLength &&
      has_hyphen_end(*word) && !ambigs_mode(*limit)) {
    // Copy more_args->active_dawgs to clean_active_dawgs removing
    // dawgs of type DAWG_TYPE_PATTERN.
    DawgInfoVector clean_active_dawgs;
    const DawgInfoVector &active_dawgs = *(more_args->active_dawgs);
    for (int i = 0; i < active_dawgs.size(); ++i) {
      if (dawgs_[active_dawgs[i].dawg_index]->type() != DAWG_TYPE_PATTERN) {
        clean_active_dawgs += active_dawgs[i];
      }
    }
    if (clean_active_dawgs.size() > 0) {
      if (permute_debug && dawg_debug_level)
        tprintf("new hyphen choice = %s\n", word->debug_string().string());
      word->set_permuter(more_args->permuter);
      adjust_word(word, certainties, permute_debug);
      set_hyphen_word(*word, *(more_args->active_dawgs),
                      *(more_args->constraints));
      update_best_choice(*word, best_choice);
    }
  } else {  // Look up char in DAWG
    // TODO(daria): update the rest of the code that specifies alternative
    // letter_is_okay_ functions (e.g. TessCharNgram class) to work with
    // multi-byte unichars and/or unichar ids.

    // If the current unichar is an ngram first try calling
    // letter_is_okay() for each unigram it contains separately.
    UNICHAR_ID orig_uch_id = word->unichar_id(word_index);
    bool checked_unigrams = false;
    if (getUnicharset().get_isngram(orig_uch_id)) {
      if (permute_debug && dawg_debug_level) {
        tprintf("checking unigrams in an ngram %s\n",
                getUnicharset().debug_str(orig_uch_id).string());
      }
      int orig_num_fragments = word->fragment_length(word_index);
      int num_unigrams = 0;
      word->remove_last_unichar_id();
      const char *ngram_str = getUnicharset().id_to_unichar(orig_uch_id);
      const char *ngram_str_end = ngram_str + strlen(ngram_str);
      const char *ngram_ptr = ngram_str;
      bool unigrams_ok = true;
      // Construct DawgArgs that reflect the current state.
      DawgInfoVector unigram_active_dawgs = *(more_args->active_dawgs);
      DawgInfoVector unigram_constraints = *(more_args->constraints);
      DawgInfoVector unigram_updated_active_dawgs;
      DawgInfoVector unigram_updated_constraints;
      DawgArgs unigram_dawg_args(&unigram_active_dawgs,
                                 &unigram_constraints,
                                 &unigram_updated_active_dawgs,
                                 &unigram_updated_constraints, 0.0,
                                 more_args->permuter,
                                 more_args->sought_word_length,
                                 more_args->end_char_choice_index);
      // Check unigrams in the ngram with letter_is_okay().
      while (unigrams_ok && ngram_ptr < ngram_str_end) {
        int step = getUnicharset().step(ngram_ptr);
        UNICHAR_ID uch_id = (step <= 0) ? INVALID_UNICHAR_ID :
            getUnicharset().unichar_to_id(ngram_ptr, step);
        ngram_ptr += step;
        ++num_unigrams;
        word->append_unichar_id(uch_id, 1, 0.0, 0.0);
        unigrams_ok = unigrams_ok && (this->*letter_is_okay_)(
            &unigram_dawg_args,
            word->unichar_id(word_index+num_unigrams-1),
            word_ending && (ngram_ptr == ngram_str_end));
        (*unigram_dawg_args.active_dawgs) =
          *(unigram_dawg_args.updated_active_dawgs);
        (*unigram_dawg_args.constraints) =
          *(unigram_dawg_args.updated_constraints);
        if (permute_debug && dawg_debug_level) {
          tprintf("unigram %s is %s\n",
                  getUnicharset().debug_str(uch_id).string(),
                  unigrams_ok ? "OK" : "not OK");
        }
      }
      // Restore the word and copy the updated dawg state if needed.
      while (num_unigrams-- > 0) word->remove_last_unichar_id();
      word->append_unichar_id_space_allocated(
          orig_uch_id, orig_num_fragments, 0.0, 0.0);
      if (unigrams_ok) {
        checked_unigrams = true;
        more_args->permuter = unigram_dawg_args.permuter;
        *(more_args->updated_active_dawgs) =
          *(unigram_dawg_args.updated_active_dawgs);
        *(more_args->updated_constraints) =
          *(unigram_dawg_args.updated_constraints);
      }
    }

    // Check which dawgs from the dawgs_ vector contain the word
    // up to and including the current unichar.
    if (checked_unigrams || (this->*letter_is_okay_)(
        more_args, word->unichar_id(word_index), word_ending)) {
      // Add a new word choice
      if (word_ending) {
        if (permute_debug && dawg_debug_level) {
          tprintf("found word = %s\n", word->debug_string().string());
        }
        if (ambigs_mode(*limit) &&
            strcmp(output_ambig_words_file.string(), "") != 0) {
          if (output_ambig_words_file_ == NULL) {
            output_ambig_words_file_ =
                fopen(output_ambig_words_file.string(), "wb+");
            if (output_ambig_words_file_ == NULL) {
              tprintf("Failed to open output_ambig_words_file %s\n",
                      output_ambig_words_file.string());
              exit(1);
            }
          }
          STRING word_str;
          word->string_and_lengths(&word_str, NULL);
          word_str += " ";
          fprintf(output_ambig_words_file_, word_str.string());
        }
        WERD_CHOICE *adjusted_word = word;
        WERD_CHOICE hyphen_tail_word(&getUnicharset());
        if (hyphen_base_size() > 0) {
          hyphen_tail_word = *word;
          remove_hyphen_head(&hyphen_tail_word);
          adjusted_word = &hyphen_tail_word;
        }
        adjusted_word->set_permuter(more_args->permuter);
        if (!ambigs_mode(*limit)) {
          adjust_word(adjusted_word, &certainties[hyphen_base_size()],
                      permute_debug);
        }
        update_best_choice(*adjusted_word, best_choice);
      } else {  // search the next letter
        // Make updated_* point to the next entries in the DawgInfoVector
        // arrays (that were originally created in dawg_permute_and_select)
        ++(more_args->updated_active_dawgs);
        ++(more_args->updated_constraints);
        // Make active_dawgs and constraints point to the updated ones.
        ++(more_args->active_dawgs);
        ++(more_args->constraints);
        permute_choices(debug, char_choices, char_choice_index + 1,
                        prev_char_frag_info, word, certainties, limit,
                        best_choice, attempts_left, more_args);
        // Restore previous state to explore another letter in this position.
        --(more_args->updated_active_dawgs);
        --(more_args->updated_constraints);
        --(more_args->active_dawgs);
        --(more_args->constraints);
      }
    } else {
      if (permute_debug && dawg_debug_level) {
        tprintf("last unichar not OK at index %d in %s\n",
                word_index, word->debug_string().string());
      }
    }
  }
}

void tesseract::Dict::go_deeper_top_fragments_fxn	(	const char *	debug,
		const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		int	char_choice_index,
		const CHAR_FRAGMENT_INFO *	prev_char_frag_info,
		bool	word_ending,
		WERD_CHOICE *	word,
		float	certainties[],
		float *	limit,
		WERD_CHOICE *	best_choice,
		int *	attempts_left,
		void *	more_args
	)

While the choice being composed so far could be better than best_choice keeps exploring char_choices. If the end of the word is reached and the word is better than best_choice, copies word to best_choice and logs the new word choice.

go_deeper_top_fragments_fxn

While the choice being composed so far could be better than best_choice keeps exploring char_choices. If the end of the word is reached and the word is better than best_choice, copies word to best_choice and logs the new word choice.

Definition at line 1536 of file permute.cpp.

                                                                   {
  if (word->rating() < *limit) {
    if (word_ending) {
      if (fragments_debug > 1) {
        tprintf("fragments_debug new choice = %s\n",
                word->debug_string().string());
      }
      *limit = word->rating();
      adjust_non_word(word, certainties, permute_debug);
      update_best_choice(*word, best_choice);
    } else {  // search the next letter
      permute_choices(debug, char_choices, char_choice_index + 1,
                      prev_char_frag_info, word, certainties, limit,
                      best_choice, attempts_left, more_args);
    }
  } else {
    if (fragments_debug > 1) {
      tprintf("fragments_debug pruned word (%s, rating=%4.2f, limit=%4.2f)\n",
              word->debug_string().string(), word->rating(), *limit);
    }
  }
}

int tesseract::Dict::good_choice ( const WERD_CHOICE & choice )

Returns true if a good answer is found for the unknown blob rating.

bool tesseract::Dict::has_hyphen_end	(	UNICHAR_ID	unichar_id,
		bool	first_pos
	)		const `[inline]`

Check whether the word has a hyphen at the end.

Definition at line 144 of file dict.h.

                                                                          {
    return (last_word_on_line_ && !first_pos &&
            unichar_id == hyphen_unichar_id_);
  }

bool tesseract::Dict::has_hyphen_end ( const WERD_CHOICE & word ) const [inline]

Same as above, but check the unichar at the end of the word.

Definition at line 149 of file dict.h.

                                                            {
    int word_index = word.length() - 1;
    return has_hyphen_end(word.unichar_id(word_index), word_index == 0);
  }

int tesseract::Dict::hyphen_base_size ( ) const [inline]

Size of the base word (the part on the line before) of a hyphenated word.

Definition at line 122 of file dict.h.

                                      {
    return this->hyphenated() ? hyphen_word_->length() : 0;
  }

bool tesseract::Dict::hyphenated ( ) const [inline]

Returns true if we've recorded the beginning of a hyphenated word.

Definition at line 118 of file dict.h.

                                 { return
    !last_word_on_line_ && hyphen_word_ && GetMaxFixedLengthDawgIndex() < 0;
  }

void tesseract::Dict::incorporate_segcost ( WERD_CHOICE * word )

Incoporate segmentation cost into word rating.

Incorporate segmentation cost into the word rating. This is done through a multiplier wordseg_rating_adjust_factor_ which is determined in bestfirst.cpp during state evaluation. This is not the cleanest way to do this. It would be better to reorganize the SEARCH_STATE to keep track of associated states, or do the rating adjustment outside the permuter in evalaute_state.

Definition at line 409 of file permute.cpp.

                                                {
  if (!word || wordseg_rating_adjust_factor_ <= 0) return;

  float old_rating = word->rating();
  float new_rating = old_rating * wordseg_rating_adjust_factor_;
  word->set_rating(new_rating);
  if (permute_debug)
    tprintf("Permute segadjust %f * %f --> %f\n",
            old_rating, wordseg_rating_adjust_factor_, new_rating);
}

void tesseract::Dict::init_active_dawgs	(	int	sought_word_length,
		DawgInfoVector *	active_dawgs,
		bool	ambigs_mode
	)		const

Fill the given active_dawgs vector with dawgs that could contain the beginning of the word. If hyphenated() returns true, copy the entries from hyphen_active_dawgs_ instead.

Definition at line 643 of file dict.cpp.

                                                     {
  int i;
  if (sought_word_length != kAnyWordLength) {
    // Only search one fixed word length dawg.
    if (sought_word_length <= max_fixed_length_dawgs_wdlen_ &&
        dawgs_[sought_word_length] != NULL) {
      *active_dawgs += DawgInfo(sought_word_length, NO_EDGE);
    }
  } else if (hyphenated()) {
    *active_dawgs = hyphen_active_dawgs_;
    if (dawg_debug_level >= 3) {
      for (i = 0; i < hyphen_active_dawgs_.size(); ++i) {
        tprintf("Adding hyphen beginning dawg [%d, " REFFORMAT "]\n",
                hyphen_active_dawgs_[i].dawg_index,
                hyphen_active_dawgs_[i].ref);
      }
    }
  } else {
    for (i = 0; i < dawgs_.length(); ++i) {
      if (dawgs_[i] != NULL && kBeginningDawgsType[(dawgs_[i])->type()] &&
          !(ambigs_mode && (dawgs_[i])->type() == DAWG_TYPE_PATTERN)) {
        *active_dawgs += DawgInfo(i, NO_EDGE);
        if (dawg_debug_level >= 3) {
          tprintf("Adding beginning dawg [%d, " REFFORMAT "]\n", i, NO_EDGE);
        }
      }
    }
  }
}

void tesseract::Dict::init_constraints ( DawgInfoVector * constraints ) const

If hyphenated() returns true, copy the entries from hyphen_constraints_ into the given constraints vector.

Definition at line 677 of file dict.cpp.

                                                             {
  if (hyphenated()) {
    *constraints = hyphen_constraints_;
    if (dawg_debug_level >= 3) {
      for (int i = 0; i < hyphen_constraints_.size(); ++i) {
        tprintf("Adding hyphen constraint [%d, " REFFORMAT "]\n",
                hyphen_constraints_[i].dawg_index,
                hyphen_constraints_[i].ref);
      }
    }
  }
}

void tesseract::Dict::InitChoiceAccum ( )

Initializes the data structures used to keep track the good word choices found for a word.

Definition at line 414 of file stopper.cpp.

                           {
  BLOB_WIDTH *BlobWidth, *End;

  if (best_raw_choice_)
    delete best_raw_choice_;
  best_raw_choice_ = NULL;

  if (best_choices_)
    destroy_nodes(best_choices_, DeleteViableChoiceStruct);
  best_choices_ = NIL_LIST;

  if (raw_choices_)
    destroy_nodes(raw_choices_, DeleteViableChoiceStruct);
  raw_choices_ = NIL_LIST;

  EnableChoiceAccum();

  for (BlobWidth = current_segmentation_,
    End = current_segmentation_ + MAX_NUM_CHUNKS;
    BlobWidth < End; *BlobWidth++ = 1);

}

int tesseract::Dict::LengthOfShortestAlphaRun ( const WERD_CHOICE & WordChoice )

Returns the length of the shortest alpha run in WordChoice.

Definition at line 858 of file stopper.cpp.

                                                                {
  int shortest = MAX_INT32;
  int curr_len = 0;
  for (int w = 0; w < WordChoice.length(); ++w) {
    if (getUnicharset().get_isalpha(WordChoice.unichar_id(w))) {
      curr_len++;
    } else if (curr_len > 0) {
      if (curr_len < shortest) shortest = curr_len;
      curr_len = 0;
    }
  }
  if (curr_len > 0 && curr_len < shortest) {
    shortest = curr_len;
  } else if (shortest == MAX_INT32) {
    shortest = 0;
  }
  return shortest;
}

int tesseract::Dict::LetterIsOkay	(	void *	void_dawg_args,
		UNICHAR_ID	unichar_id,
		bool	word_end
	)		const `[inline]`

Calls letter_is_okay_ member function.

Definition at line 560 of file dict.h.

                                                               {
    return (this->*letter_is_okay_)(void_dawg_args, unichar_id, word_end);
  }

void tesseract::Dict::Load ( )

Initialize Dict class - load dawgs from [lang].traineddata and user-specified wordlist and parttern list.

Definition at line 219 of file dict.cpp.

                {
  STRING name;
  STRING &lang = getImage()->getCCUtil()->lang;

  if (dawgs_.length() != 0) this->End();

  hyphen_unichar_id_ = getUnicharset().unichar_to_id(kHyphenSymbol);

  LoadEquivalenceList(kHyphenLikeUTF8);
  LoadEquivalenceList(kApostropheLikeUTF8);

  TessdataManager &tessdata_manager =
    getImage()->getCCUtil()->tessdata_manager;

  // Load dawgs_.
  if (load_punc_dawg && tessdata_manager.SeekToStart(TESSDATA_PUNC_DAWG)) {
    punc_dawg_ = new SquishedDawg(tessdata_manager.GetDataFilePtr(),
                                  DAWG_TYPE_PUNCTUATION, lang, PUNC_PERM,
                                  dawg_debug_level);
    dawgs_ += punc_dawg_;
  }
  if (load_system_dawg && tessdata_manager.SeekToStart(TESSDATA_SYSTEM_DAWG)) {
    dawgs_ += new SquishedDawg(tessdata_manager.GetDataFilePtr(),
                               DAWG_TYPE_WORD, lang, SYSTEM_DAWG_PERM,
                               dawg_debug_level);
  }
  if (load_number_dawg && tessdata_manager.SeekToStart(TESSDATA_NUMBER_DAWG)) {
    dawgs_ +=
      new SquishedDawg(tessdata_manager.GetDataFilePtr(),
                       DAWG_TYPE_NUMBER, lang, NUMBER_PERM, dawg_debug_level);
  }
  if (load_bigram_dawg && tessdata_manager.SeekToStart(TESSDATA_BIGRAM_DAWG)) {
    bigram_dawg_ = new SquishedDawg(tessdata_manager.GetDataFilePtr(),
                                    DAWG_TYPE_WORD, // doesn't actually matter.
                                    lang,
                                    COMPOUND_PERM,  // doesn't actually matter.
                                    dawg_debug_level);
  }
  if (load_freq_dawg && tessdata_manager.SeekToStart(TESSDATA_FREQ_DAWG)) {
    freq_dawg_ = new SquishedDawg(tessdata_manager.GetDataFilePtr(),
                                  DAWG_TYPE_WORD, lang, FREQ_DAWG_PERM,
                                  dawg_debug_level);
    dawgs_ += freq_dawg_;
  }
  if (load_unambig_dawg &&
      tessdata_manager.SeekToStart(TESSDATA_UNAMBIG_DAWG)) {
    unambig_dawg_ = new SquishedDawg(tessdata_manager.GetDataFilePtr(),
                                     DAWG_TYPE_WORD, lang, SYSTEM_DAWG_PERM,
                                     dawg_debug_level);
    dawgs_ += unambig_dawg_;
  }

  if (((STRING &)user_words_suffix).length() > 0) {
    Trie *trie_ptr = new Trie(DAWG_TYPE_WORD, lang, USER_DAWG_PERM,
                              kMaxUserDawgEdges, getUnicharset().size(),
                              dawg_debug_level);
    name = getImage()->getCCUtil()->language_data_path_prefix;
    name += user_words_suffix;
    if (!trie_ptr->read_word_list(name.string(), getUnicharset(),
                                  Trie::RRP_REVERSE_IF_HAS_RTL)) {
      tprintf("Error: failed to load %s\n", name.string());
      exit(1);
    }
    dawgs_ += trie_ptr;
  }

  if (((STRING &)user_patterns_suffix).length() > 0) {
    Trie *trie_ptr = new Trie(DAWG_TYPE_PATTERN, lang, USER_PATTERN_PERM,
                              kMaxUserDawgEdges, getUnicharset().size(),
                              dawg_debug_level);
    trie_ptr->initialize_patterns(&(getUnicharset()));
    name = getImage()->getCCUtil()->language_data_path_prefix;
    name += user_patterns_suffix;
    if (!trie_ptr->read_pattern_list(name.string(), getUnicharset())) {
      tprintf("Error: failed to load %s\n", name.string());
      exit(1);
    }
    dawgs_ += trie_ptr;
  }

  document_words_ = new Trie(DAWG_TYPE_WORD, lang, DOC_DAWG_PERM,
                             kMaxDocDawgEdges, getUnicharset().size(),
                             dawg_debug_level);
  dawgs_ += document_words_;

  // This dawg is temporary and should not be searched by letter_is_ok.
  pending_words_ = new Trie(DAWG_TYPE_WORD, lang, NO_PERM,
                            kMaxDocDawgEdges, getUnicharset().size(),
                            dawg_debug_level);

  // Load fixed length dawgs if necessary (used for phrase search
  // for non-space delimited languages).
  if (load_fixed_length_dawgs &&
      tessdata_manager.SeekToStart(TESSDATA_FIXED_LENGTH_DAWGS)) {
    ReadFixedLengthDawgs(DAWG_TYPE_WORD, lang, SYSTEM_DAWG_PERM,
                         dawg_debug_level, tessdata_manager.GetDataFilePtr(),
                         &dawgs_, &max_fixed_length_dawgs_wdlen_);
  }

  // Construct a list of corresponding successors for each dawg. Each entry i
  // in the successors_ vector is a vector of integers that represent the
  // indices into the dawgs_ vector of the successors for dawg i.
  successors_.reserve(dawgs_.length());
  for (int i = 0; i < dawgs_.length(); ++i) {
    const Dawg *dawg = dawgs_[i];
    SuccessorList *lst = new SuccessorList();
    for (int j = 0; j < dawgs_.length(); ++j) {
      const Dawg *other = dawgs_[j];
      if (dawg != NULL && other != NULL &&
          (dawg->lang() == other->lang()) &&
          kDawgSuccessors[dawg->type()][other->type()]) *lst += j;
    }
    successors_ += lst;
  }
}

void tesseract::Dict::LoadEquivalenceList ( const char * unichar_strings[] )

Definition at line 354 of file dict.cpp.

                                                            {
  equivalent_symbols_.push_back(GenericVectorEqEq<UNICHAR_ID>());
  const UNICHARSET &unicharset = getUnicharset();
  GenericVectorEqEq<UNICHAR_ID> *equiv_list = &equivalent_symbols_.back();
  for (int i = 0; unichar_strings[i] != 0; i++) {
    UNICHAR_ID unichar_id = unicharset.unichar_to_id(unichar_strings[i]);
    if (unichar_id != INVALID_UNICHAR_ID) {
      equiv_list->push_back(unichar_id);
    }
  }
}

void tesseract::Dict::LogNewChoice	(	FLOAT32	AdjustFactor,
		const float	Certainties[],
		bool	raw_choice,
		WERD_CHOICE *	WordChoice
	)

Adds Choice to ChoicesList if the adjusted certainty for Choice is within a reasonable range of the best choice in ChoicesList. The ChoicesList list is kept in sorted order by rating. Duplicates are removed. WordChoice is the new choice for current word. AdjustFactor is an adjustment factor which was applied to choice. Certainties are certainties for each char in new choice. raw_choice indicates whether WordChoice is a raw or best choice.

Definition at line 463 of file stopper.cpp.

                                                 {
  LIST ChoicesList;
  LIST Choices;
  FLOAT32 Threshold;

  if (!keep_word_choices_)
    return;

  if (raw_choice) {
    if (!best_raw_choice_) {
      best_raw_choice_ =
          NewViableChoice(*WordChoice, AdjustFactor, Certainties);
    } else if (WordChoice->rating() < best_raw_choice_->Rating) {
      if (ChoiceSameAs(*WordChoice, best_raw_choice_)) {
        FillViableChoice(*WordChoice, AdjustFactor, Certainties,
                         best_raw_choice_);
      } else {
        delete best_raw_choice_;
        best_raw_choice_ =
          NewViableChoice(*WordChoice, AdjustFactor, Certainties);
      }
    }
    if (!save_raw_choices) return;
    ChoicesList = raw_choices_;
  } else {
    ChoicesList = best_choices_;
  }

  // Throw out obviously bad choices to save some work.
  if (ChoicesList != NIL_LIST) {
    Threshold = StopperAmbigThreshold(BestFactor(ChoicesList), AdjustFactor);
    if (Threshold > -stopper_ambiguity_threshold_offset)
      Threshold = -stopper_ambiguity_threshold_offset;
    if (WordChoice->certainty() - BestCertainty (ChoicesList) < Threshold) {
      // Set the rating of the word to be terrible, so that it does not
      // get chosen as the best choice.
      if (stopper_debug_level >= 2) {
        STRING bad_string;
        WordChoice->string_and_lengths(&bad_string, NULL);
        tprintf("Discarding choice \"%s\" with an overly low certainty"
                " %.4f vs best choice certainty %.4f (Threshold: %.4f)\n",
                bad_string.string(), WordChoice->certainty(),
                BestCertainty(ChoicesList),
                Threshold + BestCertainty(ChoicesList));
      }
      WordChoice->set_rating(WERD_CHOICE::kBadRating);
      return;
    }
  }

  // See if a choice with the same text string has already been found.
  VIABLE_CHOICE NewChoice = NULL;
  Choices = ChoicesList;

  iterate(Choices) {
    if (ChoiceSameAs (*WordChoice, (VIABLE_CHOICE) first_node (Choices))) {
      if (WordChoice->rating() < BestRating (Choices)) {
        NewChoice = (VIABLE_CHOICE) first_node (Choices);
      } else {
        return;
      }
    }
  }

  if (NewChoice) {
    FillViableChoice(*WordChoice, AdjustFactor, Certainties, NewChoice);
    ChoicesList = delete_d(ChoicesList, NewChoice, is_same_node);
  } else {
    NewChoice = NewViableChoice(*WordChoice, AdjustFactor, Certainties);
  }

  ChoicesList = s_adjoin (ChoicesList, NewChoice, CmpChoiceRatings);
  if (stopper_debug_level >= 2)
    raw_choice ? PrintViableChoice (stderr, "New Raw Choice:  ", NewChoice) :
      PrintViableChoice (stderr, "New Word Choice:  ", NewChoice);
  if (count (ChoicesList) > tessedit_truncate_wordchoice_log) {
    Choices =
      (LIST) nth_cell (ChoicesList, tessedit_truncate_wordchoice_log);
    destroy_nodes(list_rest (Choices), DeleteViableChoiceStruct);
    set_rest(Choices, NIL_LIST);
  }

  // Update raw_choices_/best_choices_ pointer.
  if (raw_choice) {
    raw_choices_ = ChoicesList;
  } else {
    best_choices_ = ChoicesList;
  }
}

void tesseract::Dict::LogNewSegmentation ( PIECES_STATE BlobWidth )

Updates the blob widths in current_segmentation_ to be the same as provided in BlobWidth. BlobWidth[] contains the number of chunks in each blob in the current segmentation.

Definition at line 442 of file stopper.cpp.

                                                    {
  BLOB_WIDTH *Segmentation;
  for (Segmentation = current_segmentation_; *BlobWidth != 0;
    BlobWidth++, Segmentation++)
  *Segmentation = *BlobWidth;
  *Segmentation = 0;
}

void tesseract::Dict::LogNewSplit ( int Blob )

Given Blob (the index of the blob that was split), adds 1 chunk to the specified blob for each choice in best_choices_ and for best_raw_choice_.

Definition at line 450 of file stopper.cpp.

                               {
  LIST Choices;
  if (best_raw_choice_) AddNewChunk(best_raw_choice_, Blob);
  Choices = best_choices_;
  iterate(Choices) {
    AddNewChunk ((VIABLE_CHOICE) first_node (Choices), Blob);
  }
  Choices = raw_choices_;
  iterate(Choices) {
    AddNewChunk ((VIABLE_CHOICE) first_node (Choices), Blob);
  }
}

VIABLE_CHOICE tesseract::Dict::NewViableChoice	(	const WERD_CHOICE &	WordChoice,
		FLOAT32	AdjustFactor,
		const float	Certainties[]
	)

Allocates a new viable choice data structure, copies WordChoice, Certainties, and current_segmentation_ into it, returns a pointer to the newly created VIABLE_CHOICE. WordChoice is a choice to be converted to a viable choice. AdjustFactor is a factor used to adjust ratings for WordChoice. Certainties contain certainty for each character in WordChoice.

Definition at line 877 of file stopper.cpp.

                                                               {
  int Length = WordChoice.length();
  assert (Length <= MAX_NUM_CHUNKS && Length > 0);
  VIABLE_CHOICE NewChoice = new VIABLE_CHOICE_STRUCT(Length);
  FillViableChoice(WordChoice, AdjustFactor, Certainties, NewChoice);
  return NewChoice;
}

double tesseract::Dict::ngram_probability_in_context	(	const char *	lang,
		const char *	context,
		int	context_bytes,
		const char *	character,
		int	character_bytes
	)

bool tesseract::Dict::NoDangerousAmbig	(	WERD_CHOICE *	BestChoice,
		DANGERR *	fixpt,
		bool	fix_replaceable,
		BLOB_CHOICE_LIST_VECTOR *	Choices,
		bool *	modified_blobs
	)

Definition at line 556 of file stopper.cpp.

                                                  {
  if (stopper_debug_level > 2) {
    tprintf("\nRunning NoDangerousAmbig() for %s\n",
            best_choice->debug_string().string());
  }

  // Construct BLOB_CHOICE_LIST_VECTOR with ambiguities
  // for each unichar id in BestChoice.
  BLOB_CHOICE_LIST_VECTOR ambig_blob_choices;
  int i;
  bool modified_best_choice = false;
  bool ambigs_found = false;
  // For each position in best_choice:
  // -- choose AMBIG_SPEC_LIST that corresponds to unichar_id at best_choice[i]
  // -- initialize wrong_ngram with a single unichar_id at best_choice[i]
  // -- look for ambiguities corresponding to wrong_ngram in the list while
  //    adding the following unichar_ids from best_choice to wrong_ngram
  //
  // Repeat the above procedure twice: first time look through
  // ambigs to be replaced and replace all the ambiguities found;
  // second time look through dangerous ambiguities and construct
  // ambig_blob_choices with fake a blob choice for each ambiguity
  // and pass them to dawg_permute_and_select() to search for
  // ambiguous words in the dictionaries.
  //
  // Note that during the execution of the for loop (on the first pass)
  // if replacements are made the length of best_choice might change.
  for (int pass = 0; pass < (fix_replaceable ? 2 : 1); ++pass) {
    bool replace = (fix_replaceable && pass == 0);
    const UnicharAmbigsVector &table = replace ?
      getUnicharAmbigs().replace_ambigs() : getUnicharAmbigs().dang_ambigs();
    if (!replace) {
      // Initialize ambig_blob_choices with lists containing a single
      // unichar id for the correspoding position in best_choice.
      // best_choice consisting from only the original letters will
      // have a rating of 0.0.
      for (i = 0; i < best_choice->length(); ++i) {
        BLOB_CHOICE_LIST *lst = new BLOB_CHOICE_LIST();
        BLOB_CHOICE_IT lst_it(lst);
        // TODO(rays/antonova) Should these BLOB_CHOICEs use real xheights
        // or are these fake ones good enough?
        lst_it.add_to_end(new BLOB_CHOICE(best_choice->unichar_id(i),
                                          0.0, 0.0, -1, -1, -1, 0, 1, false));
        ambig_blob_choices.push_back(lst);
      }
    }
    UNICHAR_ID wrong_ngram[MAX_AMBIG_SIZE + 1];
    int wrong_ngram_index;
    int next_index;
    int blob_index = 0;
    for (i = 0; i < best_choice->length(); ++i) {
      if (i > 0) blob_index += best_choice->fragment_length(i-1);
      UNICHAR_ID curr_unichar_id = best_choice->unichar_id(i);
      if (stopper_debug_level > 2) {
        tprintf("Looking for %s ngrams starting with %s:\n",
                replace ? "replaceable" : "ambiguous",
                getUnicharset().debug_str(curr_unichar_id).string());
      }
      wrong_ngram_index = 0;
      wrong_ngram[wrong_ngram_index] = curr_unichar_id;
      if (curr_unichar_id == INVALID_UNICHAR_ID ||
          curr_unichar_id >= table.size() ||
          table[curr_unichar_id] == NULL) {
        continue;  // there is no ambig spec for this unichar id
      }
      AmbigSpec_IT spec_it(table[curr_unichar_id]);
      for (spec_it.mark_cycle_pt(); !spec_it.cycled_list();) {
        const AmbigSpec *ambig_spec = spec_it.data();
        wrong_ngram[wrong_ngram_index+1] = INVALID_UNICHAR_ID;
        int compare = UnicharIdArrayUtils::compare(wrong_ngram,
                                                   ambig_spec->wrong_ngram);
        if (stopper_debug_level > 2) {
          tprintf("candidate ngram: ");
          UnicharIdArrayUtils::print(wrong_ngram, getUnicharset());
          tprintf("current ngram from spec: ");
          UnicharIdArrayUtils::print(ambig_spec->wrong_ngram, getUnicharset());
          tprintf("comparison result: %d\n", compare);
        }
        if (compare == 0) {
          // Record the place where we found an ambiguity.
          if (fixpt != NULL) {
            fixpt->push_back(DANGERR_INFO(
                blob_index, blob_index+wrong_ngram_index, replace,
                getUnicharset().get_isngram(ambig_spec->correct_ngram_id)));
            if (stopper_debug_level > 1) {
              tprintf("fixpt+=(%d %d %d %d)\n", blob_index,
                      blob_index+wrong_ngram_index, false,
                      getUnicharset().get_isngram(
                          ambig_spec->correct_ngram_id));
            }
          }

          if (replace) {
            if (stopper_debug_level > 2) {
              tprintf("replace ambiguity with: ");
              UnicharIdArrayUtils::print(
                  ambig_spec->correct_fragments, getUnicharset());
            }
            ReplaceAmbig(i, ambig_spec->wrong_ngram_size,
                         ambig_spec->correct_ngram_id,
                         best_choice, blob_choices, modified_blobs);
            modified_best_choice = true;
          } else if (i > 0 || ambig_spec->type != CASE_AMBIG) {
            // We found dang ambig - update ambig_blob_choices.
            if (stopper_debug_level > 2) {
              tprintf("found ambiguity: ");
              UnicharIdArrayUtils::print(
                  ambig_spec->correct_fragments, getUnicharset());
            }
            ambigs_found = true;
            for (int tmp_index = 0; tmp_index <= wrong_ngram_index;
                 ++tmp_index) {
              // Add a blob choice for the corresponding fragment of the
              // ambiguity. These fake blob choices are initialized with
              // negative ratings (which are not possible for real blob
              // choices), so that dawg_permute_and_select() considers any
              // word not consisting of only the original letters a better
              // choice and stops searching for alternatives once such a
              // choice is found.
              BLOB_CHOICE_IT bc_it(ambig_blob_choices[i+tmp_index]);
              bc_it.add_to_end(new BLOB_CHOICE(
                  ambig_spec->correct_fragments[tmp_index], -1.0, 0.0,
                  -1, -1, -1, 0, 1, false));
            }
          }
          spec_it.forward();
        } else if (compare == -1) {
          if (wrong_ngram_index+1 < ambig_spec->wrong_ngram_size &&
              ((next_index = wrong_ngram_index+1+i) < best_choice->length())) {
            // Add the next unichar id to wrong_ngram and keep looking for
            // more ambigs starting with curr_unichar_id in AMBIG_SPEC_LIST.
            wrong_ngram[++wrong_ngram_index] =
              best_choice->unichar_id(next_index);
          } else {
            break;  // no more matching ambigs in this AMBIG_SPEC_LIST
          }
        } else {
          spec_it.forward();
        }
      }  // end searching AmbigSpec_LIST
    }  // end searching best_choice
  }  // end searching replace and dangerous ambigs

  // If any ambiguities were found permute the constructed ambig_blob_choices
  // to see if an alternative dictionary word can be found.
  if (ambigs_found) {
    if (stopper_debug_level > 2) {
      tprintf("\nResulting ambig_blob_choices:\n");
      for (i = 0; i < ambig_blob_choices.length(); ++i) {
        print_ratings_list("", ambig_blob_choices.get(i), getUnicharset());
        tprintf("\n");
      }
    }
    WERD_CHOICE *alt_word = dawg_permute_and_select(ambig_blob_choices, 0.0);
    ambigs_found = (alt_word->rating() < 0.0);
    if (ambigs_found) {
      if (stopper_debug_level >= 1) {
        tprintf ("Stopper: Possible ambiguous word = %s\n",
                 alt_word->debug_string().string());
      }
      if (fixpt != NULL) {
        // Note: Currently character choices combined from fragments can only
        // be generated by NoDangrousAmbigs(). This code should be updated if
        // the capability to produce classifications combined from character
        // fragments is added to other functions.
        int orig_i = 0;
        for (i = 0; i < alt_word->length(); ++i) {
          bool replacement_is_ngram =
              getUnicharset().get_isngram(alt_word->unichar_id(i));
          int end_i = orig_i + alt_word->fragment_length(i) - 1;
          if (alt_word->fragment_length(i) > 1 ||
              (orig_i == end_i && replacement_is_ngram)) {
            fixpt->push_back(DANGERR_INFO(orig_i, end_i, true,
                                          replacement_is_ngram));
             if (stopper_debug_level > 1) {
               tprintf("fixpt->dangerous+=(%d %d %d %d)\n", orig_i, end_i,
                       true, replacement_is_ngram);
             }
          }
          orig_i += alt_word->fragment_length(i);
        }
      }
    }
    delete alt_word;
  }
  if (output_ambig_words_file_ != NULL) {
    fprintf(output_ambig_words_file_, "\n");
  }

  ambig_blob_choices.delete_data_pointers();
  return !ambigs_found;
}

UNICHAR_ID tesseract::Dict::NormalizeUnicharIdForMatch ( UNICHAR_ID unichar_id ) const

Definition at line 368 of file dict.cpp.

                                                                       {
  for (int i = 0; i < equivalent_symbols_.size(); i++) {
    if (equivalent_symbols_[i].contains(unichar_id)) {
      return equivalent_symbols_[i][0];
    }
  }
  return unichar_id;
}

const int tesseract::Dict::NumDawgs ( ) const [inline]

Return the number of dawgs in the dawgs_ vector.

Definition at line 600 of file dict.h.

{ return dawgs_.size(); }

WERD_CHOICE * tesseract::Dict::permute_all	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		const WERD_CHOICE *	best_choice,
		WERD_CHOICE *	raw_choice
	)

Definition at line 331 of file permute.cpp.

                                                        {
  WERD_CHOICE *result1 = NULL;
  WERD_CHOICE *result2 = NULL;
  BOOL8 any_alpha;
  float top_choice_rating_limit = best_choice->rating();
  int word_script_id = get_top_word_script(char_choices, getUnicharset());

  PermuterState permuter_state;
  if (getUnicharset().han_sid() != getUnicharset().null_sid() &&
      word_script_id == getUnicharset().han_sid()) {
    permuter_state.Init(char_choices, getUnicharset(), 1.0f, permute_debug);

    result1 = get_top_choice_word(char_choices);

    // Note that we no longer need the returned word from these permuters,
    // except to delete the memory.  The word choice from all permutations
    // is returned by permuter_state.GetpermutedWord() at the end.
    if (permute_fixed_length_dawg) {
      result2 = permute_fixed_length_words(char_choices, &permuter_state);
      delete result2;
    }
    if (permute_chartype_word) {
      result2 = permute_chartype_words(char_choices, &permuter_state);
      delete result2;
    }
    if (permute_script_word) {
      result2 = permute_script_words(char_choices, &permuter_state);
      delete result2;
    }

    float certainties[MAX_PERM_LENGTH];
    float adjust_factor;
    result2 = permuter_state.GetPermutedWord(certainties, &adjust_factor);
    LogNewChoice(adjust_factor, certainties, false, result2);
    result1 = get_best_delete_other(result1, result2);

    if (segment_segcost_rating) incorporate_segcost(result1);
  } else {
    result1 = permute_top_choice(char_choices, &top_choice_rating_limit,
                                 raw_choice, &any_alpha);
    if (result1 == NULL)
      return (NULL);
    if (permute_only_top)
      return result1;

    if (permute_chartype_word) {
      permuter_state.Init(char_choices, getUnicharset(),
                          segment_penalty_garbage, permute_debug);
      result2 = permute_chartype_words(char_choices, &permuter_state);
      result1 = get_best_delete_other(result1, result2);
    }

    // Permute character fragments if necessary.
    if (result1 == NULL || result1->fragment_mark()) {
      result2 = top_fragments_permute_and_select(char_choices,
                                                 top_choice_rating_limit);
      result1 = get_best_delete_other(result1, result2);
    }

    result2 = dawg_permute_and_select(char_choices, best_choice->rating());
    result1 = get_best_delete_other(result1, result2);

    result2 = permute_compound_words(char_choices, best_choice->rating());
    result1 = get_best_delete_other(result1, result2);
  }
  return result1;
}

bool tesseract::Dict::permute_characters	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		WERD_CHOICE *	best_choice,
		WERD_CHOICE *	raw_choice
	)

permute_characters

Permute these characters together according to each of the different permuters that are enabled. Returns true if best_choice was updated.

Definition at line 765 of file permute.cpp.

                                                       {
  if (permute_debug) {
    tprintf("\n\n\n##### Permute_Characters #######\n");
    print_char_choices_list("\n==> Input CharChoices", char_choices,
                            getUnicharset(), segment_debug > 1);
    tprintf("\n");
  }

  if (char_choices.length() == 1 &&
      get_top_choice_uid(char_choices.get(0)) == 0) return false;
  WERD_CHOICE *this_choice = permute_all(char_choices, best_choice, raw_choice);

  if (this_choice && this_choice->rating() < best_choice->rating()) {
    *best_choice = *this_choice;

    if (permute_debug) {
      best_choice->print("\n**** Populate BestChoice");
      cprintf("populate best_choice\n\t%s\n",
              best_choice->debug_string().string());
    }
    delete this_choice;
    return true;
  }
  delete this_choice;
  return false;
}

WERD_CHOICE * tesseract::Dict::permute_chartype_words	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		PermuterState *	permuter_state
	)

checks for consistency in character property (eg. alpah, digit, punct)

Definition at line 542 of file permute.cpp.

                                   {

  if (char_choices.length() >= MAX_PERM_LENGTH)
    return NULL;
  // Store main character property of top choice at every position
  char pos_chartypes[MAX_PERM_LENGTH];
  char word_type = top_word_chartype(char_choices, pos_chartypes);
  if (word_type == 0 || word_type == 'p')
    return NULL;     // skip if word type is punctuation or unknown
  if (permute_debug) {
    tprintf("\n\nPermuteCharType[%c]\n", word_type);
    print_char_choices_list("", char_choices, getUnicharset(), true);
  }

  WERD_CHOICE *current_word = new WERD_CHOICE(&getUnicharset());
  BLOB_CHOICE_IT blob_choice_it;
  const UNICHARSET& unicharset = getUnicharset();
  bool replaced = false;        // has any character choice been replaced
  int prev_unambig_type = 0;    // the last chartype of an unambiguous char
  float certainties[MAX_PERM_LENGTH + 1];
  for (int x = 0; x < char_choices.length(); ++x) {
    BLOB_CHOICE_LIST* pos_choice = char_choices.get(x);
    UNICHAR_ID unichar_id = get_top_choice_uid(pos_choice);
    if (unichar_id == 0) {
      delete current_word;
      return NULL;
    }
    blob_choice_it.set_to_list(pos_choice);
    BLOB_CHOICE *first_choice = blob_choice_it.data();
    ASSERT_HOST(first_choice != NULL);

    const UnicharIdVector* ambig_uids =
        getUnicharAmbigs().OneToOneDefiniteAmbigs(unichar_id);
    bool is_ambiguous = (ambig_uids != NULL);
    bool is_punct = unicharset.get_ispunctuation(unichar_id);
    bool is_consistent = is_punct ||
        unicharset.get_chartype(unichar_id) == prev_unambig_type ||
        unicharset.get_chartype(unichar_id) == word_type;
    bool is_fragment = getUnicharset().get_fragment(unichar_id) != NULL;
    if (permute_debug)
      tprintf("char[%d]:%s is_ambig %c   is_punct %c  is_consistent %c\n",
              x, unicharset.id_to_unichar(unichar_id),
              is_ambiguous?'T':'F', is_punct?'T':'F', is_consistent?'T':'F');

    if (is_fragment) {
      // Ignore any fragmented characters by skipping them to next choice
      // (originally first choice).
      first_choice = get_nth_choice(pos_choice, 1);
      ASSERT_HOST(first_choice != NULL);
    } else if (is_ambiguous && !is_consistent) {
      // Check every confusable blob choice where the top choice is inconsistent
      // with the character type of the previous unambiguous character.
      if (permute_debug) {
        tprintf("Checking %s r%g  PrevCharType %c\n",
                unicharset.id_to_unichar(unichar_id),
                first_choice->rating(), prev_unambig_type);
        print_ratings_list("\t", pos_choice, getUnicharset());
      }
      BLOB_CHOICE* c_it = NULL;
      if (c_it == NULL) {
        c_it = find_choice_by_type(pos_choice, word_type, unicharset);
      }

      // Prefer a character choice whose type is the same as the previous
      // unambiguous character and the confusion appears in the ambig list.
      if (c_it == NULL && prev_unambig_type > 0) {
        c_it = find_choice_by_type(pos_choice, prev_unambig_type, unicharset);
        if (c_it &&
            UnicharIdArrayUtils::find_in(*ambig_uids, c_it->unichar_id()) < 0)
          c_it = NULL;
      }

      // Otherwise, perfer a punctuation
      if (c_it == NULL) {
        c_it = find_choice_by_type(pos_choice, 'p', unicharset);
        if (c_it &&
            UnicharIdArrayUtils::find_in(*ambig_uids, c_it->unichar_id()) < 0)
          c_it = NULL;
      }

      // save any preference other than the top choice
      if (c_it != NULL) {
        if (permute_debug) {
          tprintf("Replacing %s r%g ==> %s r%g\n",
                  unicharset.id_to_unichar(unichar_id), first_choice->rating(),
                  unicharset.id_to_unichar(c_it->unichar_id()), c_it->rating());
          tprintf("\n\nPermuteCharType[%c]\n", word_type);
          print_char_choices_list("", char_choices, getUnicharset(), false);
        }
        if (permuter_state)
          permuter_state->AddPreference(x, c_it, segment_reward_chartype);
        first_choice = c_it;
        replaced = true;
      }
    } else if (!is_ambiguous && !is_punct) {
      // keep the last unambiguous character type
      prev_unambig_type = pos_chartypes[x];
    }
    current_word->append_unichar_id(first_choice->unichar_id(), 1,
                                    first_choice->rating(),
                                    first_choice->certainty());
    certainties[x] = first_choice->certainty();
  }
  // All permuter choices should go through adjust_non_word so the choice
  // rating would be adjusted on the same scale.
  adjust_non_word(current_word, certainties, permute_debug);
  if (replaced) {
    // Apply a reward multiplier on rating if an chartype permutation is made.
    float rating = current_word->rating();
    current_word->set_rating(rating * segment_reward_chartype);
    if (permute_debug)
      current_word->print("<== permute_chartype_word **");
  }
  return current_word;
}

void tesseract::Dict::permute_choices	(	const char *	debug,
		const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		int	char_choice_index,
		const CHAR_FRAGMENT_INFO *	prev_char_frag_info,
		WERD_CHOICE *	word,
		float	certainties[],
		float *	limit,
		WERD_CHOICE *	best_choice,
		int *	attempts_left,
		void *	more_args
	)

permute_choices

Call append_choices() for each BLOB_CHOICE in BLOB_CHOICE_LIST with the given char_choice_index in char_choices.

Definition at line 1435 of file permute.cpp.

                     {
  if (debug) {
    tprintf("%s permute_choices: char_choice_index=%d"
            " limit=%g rating=%g, certainty=%g word=%s\n",
            debug, char_choice_index, *limit, word->rating(),
            word->certainty(), word->debug_string().string());
  }
  if (char_choice_index < char_choices.length()) {
    BLOB_CHOICE_IT blob_choice_it;
    blob_choice_it.set_to_list(char_choices.get(char_choice_index));
    for (blob_choice_it.mark_cycle_pt(); !blob_choice_it.cycled_list();
         blob_choice_it.forward()) {
      (*attempts_left)--;
      append_choices(debug, char_choices, *(blob_choice_it.data()),
                     char_choice_index, prev_char_frag_info, word,
                     certainties, limit, best_choice, attempts_left, more_args);
      if (*attempts_left <= 0) {
        if (debug) tprintf("permute_choices(): attempts_left is 0\n");
        break;
      }
    }
  }
}

WERD_CHOICE * tesseract::Dict::permute_compound_words	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		float	rating_limit
	)

permute_compound_words

Return the top choice for each character as the choice for the word.

Definition at line 799 of file permute.cpp.

                        {
  BLOB_CHOICE *first_choice;
  WERD_CHOICE *best_choice = NULL;
  WERD_CHOICE current_word(&getUnicharset(), MAX_WERD_LENGTH);
  int first_index = 0;
  int x;
  BLOB_CHOICE_IT blob_choice_it;

  if (char_choices.length() > MAX_WERD_LENGTH) {
    WERD_CHOICE *bad_word_choice = new WERD_CHOICE(&getUnicharset());
    bad_word_choice->make_bad();
    return bad_word_choice;
  }

  UNICHAR_ID slash = getUnicharset().unichar_to_id("/");
  UNICHAR_ID dash = getUnicharset().unichar_to_id("-");
  for (x = 0; x < char_choices.length(); ++x) {
    blob_choice_it.set_to_list(char_choices.get(x));
    first_choice = blob_choice_it.data();
    if (first_choice->unichar_id() == slash ||
        first_choice->unichar_id() == dash) {
      if (x > first_index) {
        if (segment_debug)
          cprintf ("Hyphenated word found\n");
        permute_subword(char_choices, rating_limit, first_index,
                        x - 1, &current_word);
        if (current_word.rating() > rating_limit)
          break;
      }
      // Append hyphen/slash separator to current_word.
      current_word.append_unichar_id_space_allocated(
          first_choice->unichar_id(), 1,
          first_choice->rating(), first_choice->certainty());

      first_index = x + 1;  // update first_index
    }
  }

  if (first_index > 0 && first_index < x &&
      current_word.rating() <= rating_limit) {
    permute_subword(char_choices, rating_limit, first_index,
                    x - 1, &current_word);
    best_choice = new WERD_CHOICE(current_word);
    best_choice->set_permuter(COMPOUND_PERM);
  }
  return (best_choice);
}

WERD_CHOICE * tesseract::Dict::permute_fixed_length_words	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		PermuterState *	permuter_state
	)

Find permutations matching a list of fixed-char-length dawgs The bestchoice based on this permuter alone is returned. Alternatively, non-conflicting changes can be combined through permuter_state.

Perform search on fixed-length dictionaries within a word. This is used for non-space delimited languages like CJK when a "word" corresponds to a "phrase" consisted of multiple short words. It iterates over every character position looking for longest matches against a set of fixed-length dawgs. Each dictionary hit is rewarded with a rating bonus. Note: this is very slow as it is performed on every segmentation state.

Definition at line 430 of file permute.cpp.

                                   {
  if (permute_debug)
    print_char_choices_list("\n\nPermute FixedLength Word",
                            char_choices, getUnicharset(), false);
  WERD_CHOICE* best_choice =
      new WERD_CHOICE(&getUnicharset(), char_choices.length());
  const int max_dict_len = max_fixed_length_dawgs_wdlen_;
  const int min_dict_len = 2;
  char posstr[256];
  int match_score = 0;
  int anchor_pos = 0;
  while (anchor_pos < char_choices.length()) {
     // search from longest phrase to shortest, stop when we find a match
     WERD_CHOICE* part_choice = NULL;
     int step = max_dict_len;
     while (step >= min_dict_len) {
       int end_pos = anchor_pos + step - 1;
       if (end_pos < char_choices.length()) {
         part_choice = dawg_permute_and_select(char_choices,
                                               200.0,   // rate limit
                                               step,
                                               anchor_pos);
         if (part_choice->length() == step) {
           if (permute_debug)
             tprintf("match found at pos=%d len=%d\n%s\n", anchor_pos, step,
                     part_choice->unichar_string().string());
           break;
         }
         delete part_choice;
         part_choice = NULL;
       }
       step--;
     }

     if (part_choice && step > 1) {   // found lexicon match
       get_posstr_from_choice(char_choices, part_choice, anchor_pos, posstr);
       float adjust_factor = pow(0.95, 1.0 + step*2.0/char_choices.length());
       if (permuter_state)
         permuter_state->AddPreference(anchor_pos, posstr, adjust_factor);
       match_score += step - 1;   // single chars don't count
       if (permute_debug)
         tprintf("Promote word rating %d-len%d\n%s\n", anchor_pos, step,
                 part_choice->unichar_string().string());
     } else {     // no lexicon match
       step = 1;
       part_choice = get_choice_from_posstr(&getUnicharset(), char_choices,
                                            anchor_pos, "0", NULL);
       if (permute_debug)
         tprintf("Single char %d %s\n", anchor_pos,
                 part_choice->unichar_string().string());
     }
     if (part_choice && part_choice->length() > 0)
       (*best_choice) += (*part_choice);
     if (part_choice) delete part_choice;
     anchor_pos += step;
  }

  if (match_score > 0) {
    float adjust_factor = pow(0.8,    // 1.0/segment_penalty_dict_nonword,
                              match_score * 2.0 / char_choices.length());
    float adjusted_score = best_choice->rating() * adjust_factor;
    if (permute_debug)
      tprintf("Adjusting score %f @ %d -> %f\n",
              best_choice->rating(), match_score, adjusted_score);
    best_choice->set_rating(adjusted_score);
  }
  if (permute_debug)
    tprintf("Found Best CJK word %f: %s\n",
            best_choice->rating(), best_choice->unichar_string().string());
  return best_choice;
}

WERD_CHOICE * tesseract::Dict::permute_script_words	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		PermuterState *	permuter_state
	)

Checks for script-consistent permutations. Similar to fixed-length permuter, the best choice is returned by the function, but the combined changes are also recorded into permuter_state.

Try flipping characters in a word to get better script consistency. Similar to how upper/lower case checking is done in top_choice_permuter, this permuter tries to suggest a more script-consistent choice AND modifies the rating. So it combines both the case_ok check and adjust_non_word functionality. However, instead of penalizing an inconsistent word with a > 1 multiplier, we reward the script-consistent choice with a < 1 multiplier.

Definition at line 669 of file permute.cpp.

                                   {
  if (char_choices.length() >= MAX_WERD_LENGTH)
    return NULL;

  int word_sid = get_top_word_script(char_choices, getUnicharset());
  if (word_sid == getUnicharset().null_sid())
    return NULL;

  if (permute_debug) {
    tprintf("\n\nPermuteScript %s\n",
            getUnicharset().get_script_from_script_id(word_sid));
    print_char_choices_list("", char_choices, getUnicharset(),
                            permute_debug > 1);
  }

  WERD_CHOICE *current_word = new WERD_CHOICE(&getUnicharset());
  BLOB_CHOICE_IT blob_choice_it;
  bool replaced = false;
  bool prev_is_consistent = false;
  float certainties[MAX_PERM_LENGTH + 1];
  for (int x = 0; x < char_choices.length(); ++x) {
    blob_choice_it.set_to_list(char_choices.get(x));
    BLOB_CHOICE *first_choice = blob_choice_it.data();
    if (!first_choice) {
      delete current_word;
      return NULL;
    }
    UNICHAR_ID unichar_id = first_choice->unichar_id();
    if (unichar_id == 0) {
      delete current_word;
      return NULL;
    }
    bool sid_consistent = (getUnicharset().get_script(unichar_id) == word_sid);
    bool this_is_punct = getUnicharset().get_chartype(unichar_id) == 'p';
    bool is_fragment = getUnicharset().get_fragment(unichar_id) != NULL;

    if (is_fragment) {
      // Ignore any fragmented characters by skipping them to next choice
      // (originally first choice).
      first_choice = get_nth_choice(char_choices.get(x), 1);
      ASSERT_HOST(first_choice != NULL);
    } else if (!sid_consistent && !this_is_punct && prev_is_consistent) {
      // If the previous char is CJK, we prefer a cjk over non-cjk char
      if (permute_debug) {
        tprintf("Checking %s r%g\n", getUnicharset().id_to_unichar(unichar_id),
                                     first_choice->rating());
        print_ratings_list("\t", char_choices.get(x), getUnicharset());
      }
      // prefer a script consistent choice
      BLOB_CHOICE* c_it = find_choice_by_script(char_choices.get(x),
                                                word_sid, 0, 0);
      // otherwise, prefer a punctuation
      if (c_it == NULL)
        c_it = find_choice_by_type(char_choices.get(x), 'p', getUnicharset());

      if (c_it != NULL) {
        if (permute_debug)
          tprintf("Replacing %s r%g ==> %s r%g\n",
                  getUnicharset().id_to_unichar(unichar_id),
                  first_choice->rating(),
                  getUnicharset().id_to_unichar(c_it->unichar_id()),
                  c_it->rating());
        if (permuter_state)
          permuter_state->AddPreference(x, c_it, segment_reward_script);
        first_choice = c_it;
        replaced = true;
      }
    }
    current_word->append_unichar_id(first_choice->unichar_id(), 1,
                                    first_choice->rating(),
                                    first_choice->certainty());
    certainties[x] = first_choice->certainty();
    prev_is_consistent = sid_consistent;
  }
  // All permuter choices should go through adjust_non_word so the choice
  // rating would be adjusted on the same scale.
  adjust_non_word(current_word, certainties, permute_debug);
  if (replaced) {
    // Apply a reward multiplier on rating if an script permutation is made.
    float rating = current_word->rating();
    current_word->set_rating(rating * segment_reward_script);
    if (permute_debug)
      current_word->print("<== permute_script_word **");
  }
  return current_word;
}

void tesseract::Dict::permute_subword	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		float	rating_limit,
		int	start,
		int	end,
		WERD_CHOICE *	current_word
	)

permute_subword

Permute a part of a compound word this subword is bounded by hyphens and the start and end of the word. Call the standard word permute function on a set of choices covering only part of the original word. When it is done reclaim the memory that was used in the exercise.

Definition at line 859 of file permute.cpp.

                                                      {
  int x;
  BLOB_CHOICE_LIST_VECTOR subchoices;
  WERD_CHOICE *best_choice = NULL;
  WERD_CHOICE raw_choice(&getUnicharset());
  raw_choice.make_bad();

  DisableChoiceAccum();

  for (x = start; x <= end; x++) {
    if (char_choices.get(x) != NULL) {
      subchoices += char_choices.get(x);
    }
  }

  if (!subchoices.empty()) {
    WERD_CHOICE initial_choice(&getUnicharset());
    initial_choice.make_bad();
    initial_choice.set_rating(rating_limit);

    best_choice = permute_all(subchoices, &initial_choice, &raw_choice);

    if (best_choice && best_choice->length() > 0) {
      *current_word += *best_choice;
    } else {
      current_word->set_rating(MAX_FLOAT32);
    }
  } else {
    current_word->set_rating(MAX_FLOAT32);
  }

  if (best_choice)
    delete best_choice;

  if (segment_debug && current_word->rating() < MAX_FLOAT32) {
    cprintf ("Subword permuted = %s, %5.2f, %5.2f\n\n",
             current_word->debug_string().string(),
             current_word->rating(), current_word->certainty());
  }
  EnableChoiceAccum();
}

WERD_CHOICE * tesseract::Dict::permute_top_choice	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		float *	rating_limit,
		WERD_CHOICE *	raw_choice,
		BOOL8 *	any_alpha
	)

permute_top_choice

Return the top choice for each character as the choice for the word. In addition a choice is created for the best lower and upper case non-words. In each character position the best lower (or upper) case character is substituted for the best overall character.

Definition at line 934 of file permute.cpp.

                      {
  BLOB_CHOICE *first_choice;
  const char *first_char;             //first choice
  const char *second_char;            //second choice
  const char *third_char;             //third choice
  char prev_char[UNICHAR_LEN + 1];    //prev in word
  const char *next_char = "";         //next in word
  const char *next_next_char = "";    //after next next in word

  WERD_CHOICE word(&getUnicharset(), MAX_PERM_LENGTH);
  word.set_permuter(TOP_CHOICE_PERM);
  WERD_CHOICE capital_word(&getUnicharset(), MAX_PERM_LENGTH);
  capital_word.set_permuter(UPPER_CASE_PERM);
  WERD_CHOICE lower_word(&getUnicharset(), MAX_PERM_LENGTH);
  lower_word.set_permuter(LOWER_CASE_PERM);

  int x;
  BOOL8 char_alpha;
  float first_rating = 0;

  float certainties[MAX_PERM_LENGTH + 1];
  float lower_certainties[MAX_PERM_LENGTH + 1];
  float upper_certainties[MAX_PERM_LENGTH + 1];

  BLOB_CHOICE_IT blob_choice_it;
  UNICHAR_ID temp_id;
  UNICHAR_ID unichar_id;
  UNICHAR_ID space = getUnicharset().unichar_to_id(" ");
  register const char* ch;
  register inT8 lower_done;
  register inT8 upper_done;

  prev_char[0] = '\0';

  if (any_alpha != NULL)
    *any_alpha = FALSE;

  if (char_choices.length() > MAX_PERM_LENGTH) {
    return (NULL);
  }

  for (x = 0; x < char_choices.length(); ++x) {
    if (x + 1 < char_choices.length()) {
      unichar_id = get_top_choice_uid(char_choices.get(x+1));
      next_char = unichar_id != INVALID_UNICHAR_ID ?
        getUnicharset().id_to_unichar(unichar_id) : "";
    } else {
      next_char = "";
    }

    if (x + 2 < char_choices.length()) {
      unichar_id = get_top_choice_uid(char_choices.get(x+2));
      next_next_char = unichar_id != INVALID_UNICHAR_ID ?
        getUnicharset().id_to_unichar(unichar_id) : "";
    } else {
      next_next_char = "";
    }

    blob_choice_it.set_to_list(char_choices.get(x));
    ASSERT_HOST(!blob_choice_it.empty());
    first_choice = NULL;
    for (blob_choice_it.mark_cycle_pt(); !blob_choice_it.cycled_list();
         blob_choice_it.forward()) {  // find the best non-fragment char choice
      temp_id = blob_choice_it.data()->unichar_id();
      if (!(getUnicharset().get_fragment(temp_id))) {
        first_choice = blob_choice_it.data();
        break;
      } else if (char_choices.length() > 1) {
        word.set_fragment_mark(true);
        capital_word.set_fragment_mark(true);
        lower_word.set_fragment_mark(true);
      }
    }
    if (first_choice == NULL) {
      cprintf("Permuter found only fragments for"
              " character at position %d; word=%s\n",
              x, word.debug_string().string());
    }
    ASSERT_HOST(first_choice != NULL);

    unichar_id = first_choice->unichar_id() != INVALID_UNICHAR_ID ?
      first_choice->unichar_id() : space;
    first_char = getUnicharset().id_to_unichar(unichar_id);
    first_rating = first_choice->rating();
    word.append_unichar_id_space_allocated(
        unichar_id, 1, first_choice->rating(), first_choice->certainty());
    capital_word.append_unichar_id_space_allocated(
        unichar_id, 1, first_choice->rating(), first_choice->certainty());
    lower_word.append_unichar_id_space_allocated(
        unichar_id, 1, first_choice->rating(), first_choice->certainty());

    certainties[x] = first_choice->certainty();
    lower_certainties[x] = first_choice->certainty();
    upper_certainties[x] = first_choice->certainty();

    lower_done = FALSE;
    upper_done = FALSE;
    char_alpha = FALSE;
    second_char = "";
    third_char = "";
    for (; !blob_choice_it.cycled_list(); blob_choice_it.forward()) {
      unichar_id = blob_choice_it.data()->unichar_id();
      if (getUnicharset().eq(unichar_id, "l") && !blob_choice_it.at_last() &&
          blob_choice_it.data_relative(1)->rating() == first_rating) {
        temp_id = blob_choice_it.data_relative(1)->unichar_id();
        if (getUnicharset().eq(temp_id, "1") ||
            getUnicharset().eq(temp_id, "I")) {
          second_char = getUnicharset().id_to_unichar(temp_id);
          blob_choice_it.forward();
          if (!blob_choice_it.at_last() &&
              blob_choice_it.data_relative(1)->rating() == first_rating) {
            temp_id = blob_choice_it.data_relative(1)->unichar_id();
            if (getUnicharset().eq(temp_id, "1") ||
                getUnicharset().eq(temp_id, "I")) {
              third_char = getUnicharset().id_to_unichar(temp_id);
              blob_choice_it.forward();
            }
          }
          ch = choose_il1 (first_char, second_char, third_char,
            prev_char, next_char, next_next_char);
          unichar_id = (ch != NULL && *ch != '\0') ?
            getUnicharset().unichar_to_id(ch) : INVALID_UNICHAR_ID;
          if (strcmp(ch, "l") != 0 &&
              getUnicharset().eq(word.unichar_id(x), "l")) {
            word.set_unichar_id(unichar_id, x);
            lower_word.set_unichar_id(unichar_id, x);
            capital_word.set_unichar_id(unichar_id, x);
          }
        }
      }
      if (unichar_id != INVALID_UNICHAR_ID) {
        /* Find lower case */
        if (!lower_done &&
            (getUnicharset().get_islower(unichar_id) ||
             (getUnicharset().get_isupper(unichar_id) && x == 0))) {
          lower_word.set_unichar_id(unichar_id, x);
          lower_word.set_rating(lower_word.rating() -
            first_choice->rating() + blob_choice_it.data()->rating());
          if (blob_choice_it.data()->certainty() < lower_word.certainty()) {
            lower_word.set_certainty(blob_choice_it.data()->certainty());
          }
          lower_certainties[x] = blob_choice_it.data()->certainty();
          lower_done = TRUE;
        }
        /* Find upper case */
        if (!upper_done && getUnicharset().get_isupper(unichar_id)) {
          capital_word.set_unichar_id(unichar_id, x);
          capital_word.set_rating(capital_word.rating() -
            first_choice->rating() + blob_choice_it.data()->rating());
          if (blob_choice_it.data()->certainty() < capital_word.certainty()) {
            capital_word.set_certainty(blob_choice_it.data()->certainty());
          }
          upper_certainties[x] = blob_choice_it.data()->certainty();
          upper_done = TRUE;
        }
        if (!char_alpha) {
          const CHAR_FRAGMENT *fragment =
            getUnicharset().get_fragment(unichar_id);
          temp_id = !fragment ? unichar_id :
            getUnicharset().unichar_to_id(fragment->get_unichar());
          if (getUnicharset().get_isalpha(temp_id)) {
            char_alpha = TRUE;
          }
        }
        if (lower_done && upper_done)
          break;
      }
    }
    if (char_alpha && any_alpha != NULL)
      *any_alpha = TRUE;

    if (word.rating() > bestrate_pruning_factor * *rating_limit) {
      if (permute_debug)
        tprintf("\n***** Aborting high-cost word: %g > limit %g\n",
                word.rating(), bestrate_pruning_factor * *rating_limit);
      return (NULL);
    }

    *prev_char = '\0';
    temp_id = word.unichar_id(word.length()-1);
    if (temp_id != INVALID_UNICHAR_ID) {
      strcpy(prev_char, getUnicharset().id_to_unichar(temp_id));
    }
  }

  if (raw_choice != NULL && word.rating() < raw_choice->rating()) {
    *raw_choice = word;
    LogNewChoice(1.0, certainties, true, raw_choice);
  }
  float rating = word.rating();
  adjust_non_word(&word, certainties, permute_debug);

  float lower_rating = lower_word.rating();
  adjust_non_word(&lower_word, lower_certainties, permute_debug);

  float upper_rating = capital_word.rating();
  adjust_non_word(&capital_word, upper_certainties, permute_debug);

  WERD_CHOICE *best_choice = &word;
  *rating_limit = rating;
  if (lower_word.rating() < best_choice->rating()) {
    best_choice = &lower_word;
    *rating_limit = lower_rating;
  }
  if (capital_word.rating() < best_choice->rating()) {
    best_choice = &capital_word;
    *rating_limit = upper_rating;
  }
  return new WERD_CHOICE(*best_choice);
}

void tesseract::Dict::PrintAmbigAlternatives	(	FILE *	file,
		const char *	label,
		int	label_num_unichars
	)

Print all the choices in raw_choices_ list for non 1-1 ambiguities.

Definition at line 337 of file stopper.cpp.

                                                          {
  iterate(raw_choices_) {
    VIABLE_CHOICE Choice = (VIABLE_CHOICE)first_node(raw_choices_);
    if (Choice->Length > 0 &&
        (label_num_unichars > 1 || Choice->Length > 1)) {
      for (int i = 0; i < Choice->Length; i++) {
        fprintf(file, "%s",
                getUnicharset().id_to_unichar(Choice->Blob[i].Class));
      }
      fflush(file);
      fprintf(file, "\t%s\t%.4f\t%.4f\n", label,
              Choice->Rating, Choice->Certainty);
    }
  }
}

void tesseract::Dict::PrintViableChoice	(	FILE *	File,
		const char *	Label,
		VIABLE_CHOICE	Choice
	)

Dumps a text representation of the specified Choice to File.

Definition at line 887 of file stopper.cpp.

                                                                                {
  int i, j;
  fprintf (File, "%s", Label);
  fprintf(File, "(R=%5.1f, C=%4.1f, F=%4.2f, Frag=%d)  ",
    Choice->Rating, Choice->Certainty,
    Choice->AdjustFactor, Choice->ComposedFromCharFragments);

  for (i = 0; i < Choice->Length; i++)
    fprintf(File, "%s", getUnicharset().id_to_unichar(Choice->Blob[i].Class));
  fprintf(File, "\n");

  for (i = 0; i < Choice->Length; i++) {
    fprintf(File, "  %s", getUnicharset().id_to_unichar(Choice->Blob[i].Class));
    for (j = 0; j < Choice->Blob[i].NumChunks - 1; j++)
      fprintf(File, "    ");
  }
  fprintf(File, "\n");

  for (i = 0; i < Choice->Length; i++) {
    for (j = 0; j < Choice->Blob[i].NumChunks; j++)
      fprintf(File, "%3d ", (int) (Choice->Blob[i].Certainty * -10.0));
  }
  fprintf(File, "\n");

  for (i = 0; i < Choice->Length; i++) {
    for (j = 0; j < Choice->Blob[i].NumChunks; j++)
      fprintf(File, "%3d ", Choice->Blob[i].NumChunks);
  }
  fprintf(File, "\n");
}

double tesseract::Dict::ProbabilityInContext	(	const char *	context,
		int	context_bytes,
		const char *	character,
		int	character_bytes
	)		`[inline]`

Calls probability_in_context_ member function.

Definition at line 573 of file dict.h.

                                                   {
    return (this->*probability_in_context_)(
        getImage()->getCCUtil()->lang.string(),
        context, context_bytes,
        character, character_bytes);
  }

void tesseract::Dict::ProcessPatternEdges	(	const Dawg *	dawg,
		const DawgInfo &	info,
		UNICHAR_ID	unichar_id,
		bool	word_end,
		DawgArgs *	dawg_args,
		PermuterType *	current_permuter
	)		const

For each of the character classes of the given unichar_id (and the unichar_id itself) finds the corresponding outgoing node or self-loop in the given dawg and (after checking that it is valid) records it in dawg_args->updated_ative_dawgs. Updates current_permuter if any valid edges were found.

Definition at line 554 of file dict.cpp.

                                                              {
  NODE_REF node = GetStartingNode(dawg, info.ref);
  // Try to find the edge corresponding to the exact unichar_id and to all the
  // edges corresponding to the character class of unichar_id.
  GenericVector<UNICHAR_ID> unichar_id_patterns;
  unichar_id_patterns.push_back(unichar_id);
  dawg->unichar_id_to_patterns(unichar_id, getUnicharset(),
                               &unichar_id_patterns);
  for (int i = 0; i < unichar_id_patterns.size(); ++i) {
    // On the first iteration check all the outgoing edges.
    // On the second iteration check all self-loops.
    for (int k = 0; k < 2; ++k) {
    EDGE_REF edge = (k == 0) ?
      dawg->edge_char_of(node, unichar_id_patterns[i], word_end)
      : dawg->pattern_loop_edge(info.ref, unichar_id_patterns[i], word_end);
      if (edge != NO_EDGE) {
        if (dawg_debug_level >= 3) {
          tprintf("Pattern dawg: [%d, " REFFORMAT "] edge=" REFFORMAT "\n",
                  info.dawg_index, node, edge);
        }
        if (ConstraintsOk(*(dawg_args->updated_constraints),
                          word_end, dawg->type())) {
          if (dawg_debug_level >=3) {
            tprintf("Letter found in pattern dawg %d\n", info.dawg_index);
          }
          if (dawg->permuter() > *curr_perm) *curr_perm = dawg->permuter();
          dawg_args->updated_active_dawgs->add_unique(
              DawgInfo(info.dawg_index, edge), dawg_debug_level > 0,
              "Append current dawg to updated active dawgs: ");
        }
      }
    }
  }
}

void tesseract::Dict::ReadFixedLengthDawgs	(	DawgType	type,
		const STRING &	lang,
		PermuterType	perm,
		int	debug_level,
		FILE *	file,
		DawgVector *	dawg_vec,
		int *	max_wdlen
	)		`[static]`

Read/Write/Access special purpose dawgs which contain words only of a certain length (used for phrase search for non-space-delimited languages). Reads a sequence of dawgs from the given file. Appends the constructed dawgs to the given dawg_vec. Fills the given table with indices of the dawgs in the dawg_vec corresponding to the dawgs with words of a particular length.

Definition at line 592 of file dict.cpp.

                                                {
  int i;
  DawgVector dawg_vec_copy;
  dawg_vec_copy.move(dawg_vec); // save the input dawg_vec.
  inT32 num_dawgs;
  fread(&num_dawgs, sizeof(inT32), 1, file);
  bool swap = (num_dawgs > MAX_WERD_LENGTH);
  if (swap) num_dawgs = reverse32(num_dawgs);
  inT32 word_length;
  int max_word_length = 0;
  // Read and record pointers to fixed-length dawgs such that:
  // dawg_vec[word_length] = pointer to dawg with word length of word_length,
  //                         NULL if such fixed-length dawg does not exist.
  for (i = 0; i < num_dawgs; ++i) {
    fread(&word_length, sizeof(inT32), 1, file);
    if (swap) word_length = reverse32(word_length);
    ASSERT_HOST(word_length >  0 && word_length <= MAX_WERD_LENGTH);
    while (word_length >= dawg_vec->size()) dawg_vec->push_back(NULL);
    (*dawg_vec)[word_length] =
      new SquishedDawg(file, type, lang, perm, debug_level);
    if (word_length > max_word_length) max_word_length = word_length;
  }
  *max_wdlen = max_word_length;
  // Entries dawg_vec[0] to dawg_vec[max_word_length] now hold pointers
  // to fixed-length dawgs. The rest of the vector will contain the dawg
  // pointers from the original input dawg_vec.
  for (i = 0; i < dawg_vec_copy.size(); ++i) {
    dawg_vec->push_back(dawg_vec_copy[i]);
  }
}

void tesseract::Dict::remove_hyphen_head ( WERD_CHOICE * word ) const [inline]

Erase the unichar ids corresponding to the portion of the word from the previous line. The word is not changed if it is not split between lines and hyphenated.

Definition at line 137 of file dict.h.

                                                          {
    if (this->hyphenated()) {
      word->remove_unichar_ids(0, hyphen_word_->length());
      if (hyphen_debug_level) hyphen_word_->print("remove_hyphen_head: ");
    }
  }

void tesseract::Dict::ReplaceAmbig	(	int	wrong_ngram_begin_index,
		int	wrong_ngram_size,
		UNICHAR_ID	correct_ngram_id,
		WERD_CHOICE *	werd_choice,
		BLOB_CHOICE_LIST_VECTOR *	blob_choices,
		bool *	modified_blobs
	)

Replaces the corresponding wrong ngram in werd_choice with the correct one. We indicate that this newly inserted ngram unichar is composed from several fragments and modify the corresponding entries in blob_choices to contain fragments of the correct ngram unichar instead of the original unichars. Ratings and certainties of entries in blob_choices and werd_choice are unichaged. E.g. for werd_choice mystring'' and ambiguity ''->": werd_choice becomes mystring", first ' in blob_choices becomes |"|0|2, second one is set to |"|1|2.

Definition at line 777 of file stopper.cpp.

                                              {
  int num_blobs_to_replace = 0;
  int begin_blob_index = 0;
  int i;
  for (i = 0; i < wrong_ngram_begin_index + wrong_ngram_size; ++i) {
    if (i >= wrong_ngram_begin_index) {
      num_blobs_to_replace +=  werd_choice->fragment_length(i);
    } else {
      begin_blob_index += werd_choice->fragment_length(i);
    }
  }
  BLOB_CHOICE_IT bit;
  int temp_blob_index = begin_blob_index;
  const char *temp_uch = NULL;
  const char *correct_ngram_str =
    getUnicharset().id_to_unichar(correct_ngram_id);
  for (int replaced_count = 0; replaced_count < wrong_ngram_size;
       ++replaced_count) {
    if (blob_choices != NULL) {
      UNICHAR_ID uch_id = werd_choice->unichar_id(wrong_ngram_begin_index);
      int fraglen = werd_choice->fragment_length(wrong_ngram_begin_index);
      if (fraglen > 1) temp_uch = getUnicharset().id_to_unichar(uch_id);
      for (i = 0; i < fraglen; ++i) {
        if (fraglen > 1) {
          STRING frag_str =
            CHAR_FRAGMENT::to_string(temp_uch, i, fraglen, false);
          getUnicharset().unichar_insert(frag_str.string());
          uch_id = getUnicharset().unichar_to_id(frag_str.string());
        }
        bit.set_to_list(blob_choices->get(temp_blob_index));
        STRING correct_frag_uch =
          CHAR_FRAGMENT::to_string(correct_ngram_str,
                                   temp_blob_index - begin_blob_index,
                                   num_blobs_to_replace, false);
        getUnicharset().unichar_insert(correct_frag_uch.string());
        UNICHAR_ID correct_frag_uch_id =
          getUnicharset().unichar_to_id(correct_frag_uch.string());
        // Find the WERD_CHOICE corresponding to the original unichar in
        // the list of blob choices, add the derived character fragment
        // before it with the same rating and certainty.
        for (bit.mark_cycle_pt(); !bit.cycled_list(); bit.forward()) {
          if (bit.data()->unichar_id() == correct_frag_uch_id) {
            break;  // the unichar we want to insert is already there
          }
          if (bit.data()->unichar_id() == uch_id) {
            bit.add_before_then_move(new BLOB_CHOICE(*(bit.data())));
            bit.data()->set_unichar_id(correct_frag_uch_id);
            if (modified_blobs != NULL) *modified_blobs = true;
            break;
          }
        }
        temp_blob_index++;
      }
    }
    // Remove current unichar from werd_choice. On the last iteration
    // set the correct replacement unichar instead of removing a unichar.
    if (replaced_count + 1 == wrong_ngram_size) {
      werd_choice->set_unichar_id(correct_ngram_id,
          num_blobs_to_replace, 0.0, 0.0, wrong_ngram_begin_index);
    } else {
      werd_choice->remove_unichar_id(wrong_ngram_begin_index);
    }
  }
  if (stopper_debug_level >= 1 && modified_blobs != NULL &&
      *modified_blobs && blob_choices != NULL) {
      werd_choice->print("ReplaceAmbig() ");
      tprintf("Modified blob_choices: ");
      for (int i = 0; i < blob_choices->size(); ++i) {
        print_ratings_list("\n", blob_choices->get(i), getUnicharset());
    }
  }
}

void tesseract::Dict::reset_hyphen_vars ( bool last_word_on_line )

Unless the previous word was the last one on the line, and the current one is not (thus it is the first one on the line), erase hyphen_word_, clear hyphen_active_dawgs_, hyphen_constraints_ update last_word_on_line_.

Definition at line 32 of file hyphen.cpp.

                                                   {
  if (!(last_word_on_line_ == true && last_word_on_line == false)) {
    if (hyphen_word_ != NULL) {
      delete hyphen_word_;
      hyphen_word_ = NULL;
      hyphen_active_dawgs_.clear();
      hyphen_constraints_.clear();
    }
  }
  if (hyphen_debug_level) {
    tprintf("reset_hyphen_vars: last_word_on_line %d -> %d\n",
            last_word_on_line_, last_word_on_line);
  }
  last_word_on_line_ = last_word_on_line;
}

void tesseract::Dict::ResetDocumentDictionary ( ) [inline]

Definition at line 475 of file dict.h.

                                 {
    if (pending_words_ != NULL)
      pending_words_->clear();
    if (document_words_ != NULL)
      document_words_->clear();
  }

void tesseract::Dict::set_hyphen_word	(	const WERD_CHOICE &	word,
		const DawgInfoVector &	active_dawgs,
		const DawgInfoVector &	constraints
	)

Update hyphen_word_, and copy the given DawgInfoVectors into hyphen_active_dawgs_ and hyphen_constraints_.

Definition at line 50 of file hyphen.cpp.

                                                              {
  if (hyphen_word_ == NULL) {
    hyphen_word_ = new WERD_CHOICE(word.unicharset());
    hyphen_word_->make_bad();
  }
  if (hyphen_word_->rating() > word.rating()) {
    *hyphen_word_ = word;
    // Remove the last unichar id as it is a hyphen, and remove
    // any unichar_string/lengths that are present.
    hyphen_word_->remove_last_unichar_id();
    hyphen_active_dawgs_ = active_dawgs;
    hyphen_constraints_ = constraints;
  }
  if (hyphen_debug_level) {
    hyphen_word_->print("set_hyphen_word: ");
  }
}

void tesseract::Dict::SettupStopperPass1 ( )

Sets up stopper variables in preparation for the first pass.

Definition at line 755 of file stopper.cpp.

                              {
  reject_offset_ = 0.0;
}

void tesseract::Dict::SettupStopperPass2 ( )

Sets up stopper variables in preparation for the second pass.

Definition at line 759 of file stopper.cpp.

                              {
  reject_offset_ = stopper_phase2_certainty_rejection_offset;
}

void tesseract::Dict::SetWordsegRatingAdjustFactor ( float f ) [inline]

Set wordseg_rating_adjust_factor_ to the given value.

Definition at line 717 of file dict.h.

                                                    {
    wordseg_rating_adjust_factor_ = f;
  }

double tesseract::Dict::StopperAmbigThreshold	(	double	f1,
		double	f2
	)		`[inline]`

Definition at line 322 of file dict.h.

                                                     {
    return (f2 - f1) * stopper_ambiguity_threshold_gain -
        stopper_ambiguity_threshold_offset;
  }

bool tesseract::Dict::StringSameAs	(	const WERD_CHOICE &	WordChoice,
		VIABLE_CHOICE	ViableChoice
	)

Compares unichar ids in word_choice to those in viable_choice, returns true if they are the same.

Definition at line 926 of file stopper.cpp.

                                                    {
  if (WordChoice.length() != ViableChoice->Length) {
    return false;
  }
  int i;
  CHAR_CHOICE *CharChoice;
  for (i = 0, CharChoice = &(ViableChoice->Blob[0]);
       i < ViableChoice->Length; CharChoice++, i++) {
    if (CharChoice->Class != WordChoice.unichar_id(i)) {
      return false;
    }
  }
  return true;
}

bool tesseract::Dict::StringSameAs	(	const char *	String,
		const char *	String_lengths,
		VIABLE_CHOICE	ViableChoice
	)

Compares String to ViableChoice and returns true if they are the same.

Definition at line 942 of file stopper.cpp.

                                                    {
  CHAR_CHOICE *Char;
  int i;
  int current_unichar_length;

  for (Char = &(ViableChoice->Blob[0]), i = 0;
    i < ViableChoice->Length;
       String += *(String_lengths++), Char++, i++) {
    current_unichar_length = strlen(getUnicharset().id_to_unichar(Char->Class));
  if (current_unichar_length != *String_lengths ||
      strncmp(String, getUnicharset().id_to_unichar(Char->Class),
              current_unichar_length) != 0)
    return false;
  }
  return (*String == 0) ? true : false;
}

WERD_CHOICE * tesseract::Dict::top_fragments_permute_and_select	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		float	rating_limit
	)

top_fragments_permute_and_select

Creates a copy of character choices list that contain only fragments and the best non-fragmented character choice. Permutes character in this shortened list, builds characters from fragments if possible and returns a better choice if found.

Definition at line 1365 of file permute.cpp.

                        {
  if (char_choices.length() <= 1 ||
      char_choices.length() > MAX_PERM_LENGTH) {
    return NULL;
  }
  // See it would be possible to benefit from permuting fragments.
  int x;
  float min_rating = 0.0;
  BLOB_CHOICE_IT blob_choice_it;
  for (x = 0; x < char_choices.length(); ++x) {
    blob_choice_it.set_to_list(char_choices.get(x));
    if (blob_choice_it.data()) {
      min_rating += blob_choice_it.data()->rating();
    }
    if (min_rating >= rating_limit) {
      return NULL;
    }
  }
  if (fragments_debug > 1) {
    tprintf("A choice with fragment beats top choice\n");
    tprintf("Running fragment permuter...\n");
  }

  // Construct a modified choices list that contains (for each position):
  // the best choice, all fragments and at least one choice for
  // a non-fragmented character.
  BLOB_CHOICE_LIST_VECTOR frag_char_choices(char_choices.length());
  for (x = 0; x < char_choices.length(); ++x) {
    bool need_nonfrag_char = true;
    BLOB_CHOICE_LIST *frag_choices = new BLOB_CHOICE_LIST();
    BLOB_CHOICE_IT frag_choices_it;
    frag_choices_it.set_to_list(frag_choices);
    blob_choice_it.set_to_list(char_choices.get(x));
    for (blob_choice_it.mark_cycle_pt(); !blob_choice_it.cycled_list();
         blob_choice_it.forward()) {
      if (getUnicharset().get_fragment(blob_choice_it.data()->unichar_id())) {
        frag_choices_it.add_after_then_move(
            new BLOB_CHOICE(*(blob_choice_it.data())));
      } else if (need_nonfrag_char) {
        frag_choices_it.add_after_then_move(
            new BLOB_CHOICE(*(blob_choice_it.data())));
        need_nonfrag_char = false;
      }
    }
    frag_char_choices += frag_choices;
  }

  WERD_CHOICE *best_choice = new WERD_CHOICE(&getUnicharset());
  best_choice->make_bad();
  WERD_CHOICE word(&getUnicharset(), MAX_PERM_LENGTH);
  word.set_permuter(TOP_CHOICE_PERM);
  float certainties[MAX_PERM_LENGTH];
  this->go_deeper_fxn_ = &tesseract::Dict::go_deeper_top_fragments_fxn;
  int attempts_left = max_permuter_attempts;
  permute_choices((fragments_debug > 1) ? "fragments_debug" : NULL,
                  frag_char_choices, 0, NULL, &word, certainties,
                  &rating_limit, best_choice, &attempts_left, NULL);

  frag_char_choices.delete_data_pointers();
  return best_choice;
}

char tesseract::Dict::top_word_chartype	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		char *	pos_chartypes
	)

Look up the main chartype for each character position and store it in the given array. Also returns the dominant type from unambiguous top choices.

Definition at line 512 of file permute.cpp.

                                                  {
  const UNICHARSET &unicharset = getUnicharset();
  const int hist_size = 128;   // to contain 'A','a','0','x','p'
  int chprop[hist_size];
  int x;
  for (x = 0; x < hist_size; x++) chprop[x] = 0;
  for (x = 0; x < char_choices.length(); ++x) {
    UNICHAR_ID unichar_id = get_top_choice_uid(char_choices.get(x));
    char ctype = unicharset.get_chartype(unichar_id);
    if (pos_chartypes) pos_chartypes[x] = ctype;
    if (ctype == 0 || ctype == 'p') continue;
    if (getUnicharAmbigs().OneToOneDefiniteAmbigs(unichar_id) != NULL) continue;
    chprop[ctype]++;
    if (x == 0 && ctype == 'A')  // first-cap also counts as lower
      chprop['a']++;
  }
  int max_prop = 0;
  for (x = 1; x < hist_size; x++)
    if (chprop[x] >= chprop[max_prop]) max_prop = x;
  return (chprop[max_prop] > 0) ? max_prop : 0;
}

int tesseract::Dict::UniformCertainties	(	const BLOB_CHOICE_LIST_VECTOR &	Choices,
		const WERD_CHOICE &	BestChoice
	)

Returns true if the certainty of the BestChoice word is within a reasonable range of the average certainties for the best choices for each character in the segmentation. This test is used to catch words in which one character is much worse than the other characters in the word (i.e. false will be returned in that case). The algorithm computes the mean and std deviation of the certainties in the word with the worst certainty thrown out.

Definition at line 961 of file stopper.cpp.

                                                            {
  float Certainty;
  float WorstCertainty = MAX_FLOAT32;
  float CertaintyThreshold;
  FLOAT64 TotalCertainty;
  FLOAT64 TotalCertaintySquared;
  FLOAT64 Variance;
  FLOAT32 Mean, StdDev;
  int WordLength;

  WordLength = Choices.length();
  if (WordLength < 3)
    return true;

  TotalCertainty = TotalCertaintySquared = 0.0;
  BLOB_CHOICE_IT BlobChoiceIt;
  for (int i = 0; i < Choices.length(); ++i) {
    BlobChoiceIt.set_to_list(Choices.get(i));
    Certainty = BlobChoiceIt.data()->certainty();
    TotalCertainty += Certainty;
    TotalCertaintySquared += Certainty * Certainty;
    if (Certainty < WorstCertainty)
      WorstCertainty = Certainty;
  }

  // Subtract off worst certainty from statistics.
  WordLength--;
  TotalCertainty -= WorstCertainty;
  TotalCertaintySquared -= WorstCertainty * WorstCertainty;

  Mean = TotalCertainty / WordLength;
  Variance = ((WordLength * TotalCertaintySquared -
    TotalCertainty * TotalCertainty) /
    (WordLength * (WordLength - 1)));
  if (Variance < 0.0)
    Variance = 0.0;
  StdDev = sqrt (Variance);

  CertaintyThreshold = Mean - stopper_allowable_character_badness * StdDev;
  if (CertaintyThreshold > stopper_nondict_certainty_base)
    CertaintyThreshold = stopper_nondict_certainty_base;

  if (BestChoice.certainty() < CertaintyThreshold) {
    if (stopper_debug_level >= 1)
      cprintf("Stopper: Non-uniform certainty = %4.1f"
              " (m=%4.1f, s=%4.1f, t=%4.1f)\n",
              BestChoice.certainty(), Mean, StdDev, CertaintyThreshold);
    return false;
  } else {
    return true;
  }
}

void tesseract::Dict::update_best_choice	(	const WERD_CHOICE &	word,
		WERD_CHOICE *	best_choice
	)		`[inline]`

Copies word into best_choice if its rating is smaller than that of best_choice.

Definition at line 166 of file dict.h.

                                                           {
    if (word.rating() < best_choice->rating()) *best_choice = word;
  }

bool tesseract::Dict::valid_bigram	(	const WERD_CHOICE &	word1,
		const WERD_CHOICE &	word2
	)		const

Definition at line 848 of file dict.cpp.

                                                        {
  if (bigram_dawg_ == NULL) return false;

  // Extract the core word from the middle of each word with any digits
  //         replaced with question marks.
  int w1start, w1end, w2start, w2end;
  word1.punct_stripped(&w1start, &w1end);
  word2.punct_stripped(&w2start, &w2end);

  // We don't want to penalize a single guillemet, hyphen, etc.
  // But our bigram list doesn't have any information about punctuation.
  if (w1start >= w1end) return word1.length() < 3;
  if (w2start >= w2end) return word2.length() < 3;

  const UNICHARSET& uchset = getUnicharset();
  STRING bigram_string;
  for (int i = w1start; i < w1end; i++) {
    UNICHAR_ID ch = NormalizeUnicharIdForMatch(word1.unichar_id(i));
    bigram_string += uchset.get_isdigit(ch) ? "?" : uchset.id_to_unichar(ch);
  }
  bigram_string += " ";
  for (int i = w2start; i < w2end; i++) {
    UNICHAR_ID ch = NormalizeUnicharIdForMatch(word2.unichar_id(i));
    bigram_string += uchset.get_isdigit(ch) ? "?" : uchset.id_to_unichar(ch);
  }
  WERD_CHOICE normalized_word(bigram_string.string(), uchset);
  return bigram_dawg_->word_in_dawg(normalized_word);
}

bool tesseract::Dict::valid_punctuation ( const WERD_CHOICE & word )

Returns true if the word contains a valid punctuation pattern. Note: Since the domains of punctuation symbols and symblos used in numbers are not disjoint, a valid number might contain an invalid punctuation pattern (e.g. .99).

Definition at line 878 of file dict.cpp.

                                                    {
  if (word.length() == 0) return NO_PERM;
  int i;
  WERD_CHOICE new_word(word.unicharset());
  int last_index = word.length() - 1;
  int new_len = 0;
  for (i = 0; i <= last_index; ++i) {
    UNICHAR_ID unichar_id = (word.unichar_id(i));
    if (getUnicharset().get_ispunctuation(unichar_id)) {
      new_word.append_unichar_id(unichar_id, 1, 0.0, 0.0);
    } else if (!getUnicharset().get_isalpha(unichar_id) &&
               !getUnicharset().get_isdigit(unichar_id)) {
      return false;  // neither punc, nor alpha, nor digit
    } else if ((new_len = new_word.length()) == 0 ||
               new_word.unichar_id(new_len-1) != Dawg::kPatternUnicharID) {
      new_word.append_unichar_id(Dawg::kPatternUnicharID, 1, 0.0, 0.0);
    }
  }
  for (i = 0; i < dawgs_.size(); ++i) {
    if (dawgs_[i] != NULL &&
        dawgs_[i]->type() == DAWG_TYPE_PUNCTUATION &&
        dawgs_[i]->word_in_dawg(new_word)) return true;
  }
  return false;
}

int tesseract::Dict::valid_word	(	const WERD_CHOICE &	word,
		bool	numbers_ok
	)		const

Definition at line 806 of file dict.cpp.

                                                                   {
  const WERD_CHOICE *word_ptr = &word;
  WERD_CHOICE temp_word(word.unicharset());
  if (hyphenated()) {
    copy_hyphen_info(&temp_word);
    temp_word += word;
    word_ptr = &temp_word;
  }
  if (word_ptr->length() == 0) return NO_PERM;
  // Allocate vectors for holding current and updated
  // active_dawgs and constraints and initialize them.
  DawgInfoVector *active_dawgs = new DawgInfoVector[2];
  DawgInfoVector *constraints = new DawgInfoVector[2];
  init_active_dawgs(kAnyWordLength, &(active_dawgs[0]), false);
  init_constraints(&(constraints[0]));
  DawgArgs dawg_args(&(active_dawgs[0]), &(constraints[0]),
                     &(active_dawgs[1]), &(constraints[1]),
                     0.0, NO_PERM, kAnyWordLength, 0);
  int last_index = word_ptr->length() - 1;
  // Call leter_is_okay for each letter in the word.
  for (int i = hyphen_base_size(); i <= last_index; ++i) {
    if (!((this->*letter_is_okay_)(&dawg_args, word_ptr->unichar_id(i),
                                   i == last_index))) break;
    // Swap active_dawgs, constraints with the corresponding updated vector.
    if (dawg_args.updated_active_dawgs == &(active_dawgs[1])) {
      dawg_args.updated_active_dawgs = &(active_dawgs[0]);
      dawg_args.updated_constraints = &(constraints[0]);
      ++(dawg_args.active_dawgs);
      ++(dawg_args.constraints);
    } else {
      ++(dawg_args.updated_active_dawgs);
      ++(dawg_args.updated_constraints);
      dawg_args.active_dawgs = &(active_dawgs[0]);
      dawg_args.constraints = &(constraints[0]);
    }
  }
  delete[] active_dawgs;
  delete[] constraints;
  return valid_word_permuter(dawg_args.permuter, numbers_ok) ?
    dawg_args.permuter : NO_PERM;
}

int tesseract::Dict::valid_word ( const WERD_CHOICE & word ) const [inline]

Definition at line 682 of file dict.h.

                                                {
    return valid_word(word, false);  // return NO_PERM for words with digits
  }

int tesseract::Dict::valid_word ( const char * string ) const [inline]

This function is used by api/tesseract_cube_combiner.cpp.

Definition at line 689 of file dict.h.

                                           {
    WERD_CHOICE word(string, getUnicharset());
    return valid_word(word);
  }

int tesseract::Dict::valid_word_or_number ( const WERD_CHOICE & word ) const [inline]

Definition at line 685 of file dict.h.

                                                          {
    return valid_word(word, true);  // return NUMBER_PERM for valid numbers
  }

static bool tesseract::Dict::valid_word_permuter	(	uinT8	perm,
		bool	numbers_ok
	)		`[inline, static]`

Check all the DAWGs to see if this word is in any of them.

Definition at line 676 of file dict.h.

                                                                      {
    return (perm == SYSTEM_DAWG_PERM || perm == FREQ_DAWG_PERM ||
            perm == DOC_DAWG_PERM || perm == USER_DAWG_PERM ||
            perm == USER_PATTERN_PERM || (numbers_ok && perm == NUMBER_PERM));
  }

void tesseract::Dict::WriteFixedLengthDawgs	(	const GenericVector< SquishedDawg * > &	dawg_vec,
		int	num_dawgs,
		int	debug_level,
		FILE *	output_file
	)		`[static]`

Writes the dawgs in the dawgs_vec to a file. Updates the given table with the indices of dawgs in the dawg_vec for the corresponding word lengths.

Definition at line 626 of file dict.cpp.

                                                       {
  fwrite(&num_dawgs, sizeof(inT32), 1, output_file);
  if (debug_level) tprintf("Writing %d split length dawgs\n", num_dawgs);
  for (int i = 1; i < dawg_vec.size(); ++i) {
    if ((dawg_vec)[i] != NULL) {
      fwrite(&i, sizeof(inT32), 1, output_file);
      dawg_vec[i]->write_squished_dawg(output_file);
      if (debug_level) tprintf("Wrote Dawg with word length %d\n", i);
    }
  }
}

Member Data Documentation

double tesseract::Dict::bestrate_pruning_factor = 2.0

"Multiplying factor of" " current best rate to prune other hypotheses"

Definition at line 865 of file dict.h.

double tesseract::Dict::certainty_scale = 20.0

"Certainty scaling factor"

Definition at line 836 of file dict.h.

int tesseract::Dict::dawg_debug_level = 0

"Set to 1 for general debug info" ", to 2 for more details, to 3 to see all the debug messages"

Definition at line 830 of file dict.h.

double tesseract::Dict::doc_dict_certainty_threshold = -2.25

"Worst certainty" " for words that can be inserted into the document dictionary"

Definition at line 893 of file dict.h.

bool tesseract::Dict::doc_dict_enable = 1

"Enable Document Dictionary "

Definition at line 889 of file dict.h.

double tesseract::Dict::doc_dict_pending_threshold = 0.0

"Worst certainty for using pending dictionary"

Definition at line 891 of file dict.h.

int tesseract::Dict::fragments_debug = 0

"Debug character fragments"

Definition at line 861 of file dict.h.

void(Dict::* tesseract::Dict::go_deeper_fxn_)(const char *debug, const BLOB_CHOICE_LIST_VECTOR &char_choices, int char_choice_index, const CHAR_FRAGMENT_INFO *prev_char_frag_info, bool word_ending, WERD_CHOICE *word, float certainties[], float *limit, WERD_CHOICE *best_choice, int *attempts_left, void *void_more_args)

Pointer to go_deeper function that will be modified by various permuters.

Definition at line 308 of file dict.h.

int tesseract::Dict::hyphen_debug_level = 0

"Debug level for hyphenated words."

Definition at line 831 of file dict.h.

int(Dict::* tesseract::Dict::letter_is_okay_)(void *void_dawg_args, UNICHAR_ID unichar_id, bool word_end) const

Definition at line 557 of file dict.h.

bool tesseract::Dict::load_bigram_dawg = false

"Load dawg with special word bigrams."

Definition at line 802 of file dict.h.

bool tesseract::Dict::load_fixed_length_dawgs = true

"Load fixed length" " dawgs (e.g. for non-space delimited languages)"

Definition at line 800 of file dict.h.

bool tesseract::Dict::load_freq_dawg = true

"Load frequent word dawg."

Definition at line 794 of file dict.h.

bool tesseract::Dict::load_number_dawg = true

"Load dawg with number patterns."

Definition at line 798 of file dict.h.

bool tesseract::Dict::load_punc_dawg = true

"Load dawg with punctuation patterns."

Definition at line 797 of file dict.h.

bool tesseract::Dict::load_system_dawg = true

"Load system word dawg."

Definition at line 793 of file dict.h.

bool tesseract::Dict::load_unambig_dawg = true

"Load unambiguous word dawg."

Definition at line 795 of file dict.h.

int tesseract::Dict::max_permuter_attempts = 10000

"Maximum number of different" " character choices to consider during permutation." " This limit is especially useful when user patterns" " are specified, since overly generic patterns can result in" " dawg search exploring an overly large number of options."

Definition at line 900 of file dict.h.

int tesseract::Dict::max_viterbi_list_size = 10

"Maximum size of viterbi list."

Definition at line 832 of file dict.h.

bool tesseract::Dict::ngram_permuter_activated = false

"Activate character-level n-gram-based permuter"

Definition at line 895 of file dict.h.

char* tesseract::Dict::output_ambig_words_file = ""

"Output file for ambiguities found in the dictionary"

Definition at line 828 of file dict.h.

bool tesseract::Dict::permute_chartype_word = 0

"Turn on character type (property) consistency permuter"

Definition at line 881 of file dict.h.

bool tesseract::Dict::permute_debug = 0

"Debug char permutation process"

Definition at line 863 of file dict.h.

bool tesseract::Dict::permute_fixed_length_dawg = 0

"Turn on fixed-length phrasebook search permuter"

Definition at line 879 of file dict.h.

bool tesseract::Dict::permute_only_top = false

"Run only the top choice permuter"

Definition at line 901 of file dict.h.

bool tesseract::Dict::permute_script_word = 0

"Turn on word script consistency permuter"

Definition at line 867 of file dict.h.

double(Dict::* tesseract::Dict::probability_in_context_)(const char *lang, const char *context, int context_bytes, const char *character, int character_bytes)

Probability in context function used by the ngram permuter.

Definition at line 567 of file dict.h.

bool tesseract::Dict::save_doc_words = 0

"Save Document Words"

Definition at line 888 of file dict.h.

bool tesseract::Dict::save_raw_choices = false

"Save all explored raw choices"

Definition at line 855 of file dict.h.

int tesseract::Dict::segment_debug = 0

"Debug the whole segmentation process"

Definition at line 862 of file dict.h.

bool tesseract::Dict::segment_nonalphabetic_script = false

"Don't use any alphabetic-specific tricks." "Set to true in the traineddata config file for" " scripts that are cursive or inherently fixed-pitch"

Definition at line 873 of file dict.h.

double tesseract::Dict::segment_penalty_dict_case_bad = 1.3125

"Default score multiplier for word matches, which may have " "case issues (lower is better)."

Definition at line 813 of file dict.h.

double tesseract::Dict::segment_penalty_dict_case_ok = 1.1

"Score multiplier for word matches that have good case " "(lower is better)."

Definition at line 809 of file dict.h.

double tesseract::Dict::segment_penalty_dict_frequent_word = 1.0

"Score multiplier for word matches which have good case and" "are frequent in the given language (lower is better)."

Definition at line 805 of file dict.h.

double tesseract::Dict::segment_penalty_dict_nonword = 1.25

"Score multiplier for glyph fragment segmentations which " "do not match a dictionary word (lower is better)."

Definition at line 821 of file dict.h.

double tesseract::Dict::segment_penalty_garbage = 1.50

"Score multiplier for poorly cased strings that are not in" " the dictionary and generally look like garbage (lower is" " better)."

Definition at line 826 of file dict.h.

double tesseract::Dict::segment_penalty_ngram_best_choice = 1.24

"Multipler to for the best choice from the ngram model."

Definition at line 817 of file dict.h.

double tesseract::Dict::segment_reward_chartype = 0.97

"Score multipler for char type consistency within a word. "

Definition at line 883 of file dict.h.

double tesseract::Dict::segment_reward_ngram_best_choice = 0.99

"Score multipler for ngram permuter's best choice" " (only used in the Han script path)."

Definition at line 887 of file dict.h.

double tesseract::Dict::segment_reward_script = 0.95

"Score multipler for script consistency within a word. " "Being a 'reward' factor, it should be <= 1. " "Smaller value implies bigger reward."

Definition at line 877 of file dict.h.

bool tesseract::Dict::segment_segcost_rating = 0

"incorporate segmentation cost in word rating?"

Definition at line 869 of file dict.h.

double tesseract::Dict::stopper_allowable_character_badness = 3.0

"Max certaintly variation allowed in a word (in sigma)"

Definition at line 846 of file dict.h.

double tesseract::Dict::stopper_ambiguity_threshold_gain = 8.0

"Gain factor for ambiguity threshold."

Definition at line 852 of file dict.h.

double tesseract::Dict::stopper_ambiguity_threshold_offset = 1.5

"Certainty offset for ambiguity threshold."

Definition at line 854 of file dict.h.

double tesseract::Dict::stopper_certainty_per_char = -0.50

"Certainty to add for each dict char above small word size."

Definition at line 844 of file dict.h.

int tesseract::Dict::stopper_debug_level = 0

"Stopper debug level"

Definition at line 847 of file dict.h.

bool tesseract::Dict::stopper_no_acceptable_choices = false

"Make AcceptableChoice() always return false. Useful" " when there is a need to explore all segmentations"

Definition at line 850 of file dict.h.

double tesseract::Dict::stopper_nondict_certainty_base = -2.50

"Certainty threshold for non-dict words"

Definition at line 838 of file dict.h.

double tesseract::Dict::stopper_phase2_certainty_rejection_offset = 1.0

"Reject certainty offset"

Definition at line 840 of file dict.h.

int tesseract::Dict::stopper_smallword_size = 2

"Size of dict word to be treated as non-dict word"

Definition at line 842 of file dict.h.

int tesseract::Dict::tessedit_truncate_wordchoice_log = 10

"Max words to keep in list"

Definition at line 856 of file dict.h.

bool tesseract::Dict::use_only_first_uft8_step = false

"Use only the first UTF8 step of the given string" " when computing log probabilities."

Definition at line 835 of file dict.h.

char* tesseract::Dict::user_patterns_suffix = ""

"A list of user-provided patterns."

Definition at line 792 of file dict.h.

char* tesseract::Dict::user_words_suffix = ""

Variable members. These have to be declared and initialized after image_ptr_, which contains the pointer to the params vector - the member of its base CCUtil class. "A list of user-provided words."

Definition at line 790 of file dict.h.

char* tesseract::Dict::word_to_debug = ""

"Word for which stopper debug information" " should be printed to stdout"

Definition at line 858 of file dict.h.

char* tesseract::Dict::word_to_debug_lengths = ""

"Lengths of unichars in word_to_debug"

Definition at line 860 of file dict.h.

The documentation for this class was generated from the following files:

tesseract-ocr/dict/dict.h
tesseract-ocr/dict/context.cpp
tesseract-ocr/dict/dict.cpp
tesseract-ocr/dict/hyphen.cpp
tesseract-ocr/dict/permdawg.cpp
tesseract-ocr/dict/permute.cpp
tesseract-ocr/dict/stopper.cpp