<commit_before>///////////////////////////////////////////////////////////////////////
// File:        mastertrainer.cpp
// Description: Trainer to build the MasterClassifier.
// Author:      <NAME>
// Created:     Wed Nov 03 18:10:01 PDT 2010
//
// (C) Copyright 2010, Google Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
///////////////////////////////////////////////////////////////////////

// Include automatically generated configuration file if running autoconf.
#ifdef HAVE_CONFIG_H
#include "config_auto.h"
#endif

#include "mastertrainer.h"
#include <cmath>
#include <ctime>
#include "allheaders.h"
#include "boxread.h"
#include "classify.h"
#include "errorcounter.h"
#include "featdefs.h"
#include "sampleiterator.h"
#include "shapeclassifier.h"
#include "shapetable.h"
#include "svmnode.h"

#include "scanutils.h"

namespace tesseract {

// Constants controlling clustering. With a low kMinClusteredShapes and a high
// kMaxUnicharsPerCluster, then kFontMergeDistance is the only limiting factor.
// Min number of shapes in the output.
const int kMinClusteredShapes = 1;
// Max number of unichars in any individual cluster.
const int kMaxUnicharsPerCluster = 2000;
// Mean font distance below which to merge fonts and unichars.
const float kFontMergeDistance = 0.025;

MasterTrainer::MasterTrainer(NormalizationMode norm_mode,
                             bool shape_analysis,
                             bool replicate_samples,
                             int debug_level)
  : norm_mode_(norm_mode), samples_(fontinfo_table_),
    junk_samples_(fontinfo_table_), verify_samples_(fontinfo_table_),
    charsetsize_(0),
    enable_shape_anaylsis_(shape_analysis),
    enable_replication_(replicate_samples),
    fragments_(nullptr), prev_unichar_id_(-1), debug_level_(debug_level) {
}

MasterTrainer::~MasterTrainer() {
  delete [] fragments_;
  for (int p = 0; p < page_images_.size(); ++p)
    pixDestroy(&page_images_[p]);
}

// WARNING! Serialize/DeSerialize are only partial, providing
// enough data to get the samples back and display them.
// Writes to the given file. Returns false in case of error.
bool MasterTrainer::Serialize(FILE* fp) const {
  if (fwrite(&norm_mode_, sizeof(norm_mode_), 1, fp) != 1) return false;
  if (!unicharset_.save_to_file(fp)) return false;
  if (!feature_space_.Serialize(fp)) return false;
  if (!samples_.Serialize(fp)) return false;
  if (!junk_samples_.Serialize(fp)) return false;
  if (!verify_samples_.Serialize(fp)) return false;
  if (!master_shapes_.Serialize(fp)) return false;<commit_msg>MasterTrainer: Use new serialization API<commit_after>///////////////////////////////////////////////////////////////////////
// File:        mastertrainer.cpp
// Description: Trainer to build the MasterClassifier.
// Author:      <NAME>
// Created:     Wed Nov 03 18:10:01 PDT 2010
//
// (C) Copyright 2010, Google Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
///////////////////////////////////////////////////////////////////////

// Include automatically generated configuration file if running autoconf.
#ifdef HAVE_CONFIG_H
#include "config_auto.h"
#endif

#include "mastertrainer.h"
#include <cmath>
#include <ctime>
#include "allheaders.h"
#include "boxread.h"
#include "classify.h"
#include "errorcounter.h"
#include "featdefs.h"
#include "sampleiterator.h"
#include "shapeclassifier.h"
#include "shapetable.h"
#include "svmnode.h"

#include "scanutils.h"

namespace tesseract {

// Constants controlling clustering. With a low kMinClusteredShapes and a high
// kMaxUnicharsPerCluster, then kFontMergeDistance is the only limiting factor.
// Min number of shapes in the output.
const int kMinClusteredShapes = 1;
// Max number of unichars in any individual cluster.
const int kMaxUnicharsPerCluster = 2000;
// Mean font distance below which to merge fonts and unichars.
const float kFontMergeDistance = 0.025;

MasterTrainer::MasterTrainer(NormalizationMode norm_mode,
                             bool shape_analysis,
                             bool replicate_samples,
                             int debug_level)
  : norm_mode_(norm_mode), samples_(fontinfo_table_),
    junk_samples_(fontinfo_table_), verify_samples_(fontinfo_table_),
    charsetsize_(0),
    enable_shape_anaylsis_(shape_analysis),
    enable_replication_(replicate_samples),
    fragments_(nullptr), prev_unichar_id_(-1), debug_level_(debug_level) {
}

MasterTrainer::~MasterTrainer() {
  delete [] fragments_;
  for (int p = 0; p < page_images_.size(); ++p)
    pixDestroy(&page_images_[p]);
}

// WARNING! Serialize/DeSerialize are only partial, providing
// enough data to get the samples back and display them.
// Writes to the given file. Returns false in case of error.
bool MasterTrainer::Serialize(FILE* fp) const {
  uint32_t value = norm_mode_;
  if (!tesseract::Serialize(fp, &value)) return false;
  if (!unicharset_.save_to_file(fp)) return false;
  if (!feature_space_.Serialize(fp)) return false;
  if (!samples_.Serialize(fp)) return false;
  if (!junk_samples_.Serialize(fp)) return false;
  if (!verify_samples_.Serialize(fp)) return false;
  if (!master_shapes_.Serialize(fp)) return false;
{'fraction_non_alphanumeric': 0.14211550940947437, 'fraction_numerical': 0.011031797534068787, 'mean_word_length': 3.2546583850931676, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '220540', 'n_tokens_mistral': 1782, 'n_tokens_neox': 1739, 'n_words': 581}