<commit_before>///////////////////////////////////////////////////////////////////////
// File:        tessdatamanager.cpp
// Description: Functions to handle loading/combining tesseract data files.
// Author:      <NAME>
// Created:     Wed Jun 03 11:26:43 PST 2009
//
// (C) Copyright 2009, Google Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
///////////////////////////////////////////////////////////////////////

#ifdef _MSC_VER
#pragma warning(disable:4244)  // Conversion warnings
#endif

#include "tessdatamanager.h"

#include <stdio.h>

#include "serialis.h"
#include "strngs.h"
#include "tprintf.h"
#include "params.h"

namespace tesseract {

bool TessdataManager::Init(const char *data_file_name, int debug_level) {
  int i;
  debug_level_ = debug_level;
  data_file_ = fopen(data_file_name, "rb");
  if (data_file_ == NULL) {
    tprintf("Error opening data file %s\n", data_file_name);
    tprintf("Please make sure the TESSDATA_PREFIX environment variable is set "
            "to the parent directory of your \"tessdata\" directory.\n");
    return false;
  }
  fread(&actual_tessdata_num_entries_, sizeof(inT32), 1, data_file_);
  swap_ = (actual_tessdata_num_entries_ > kMaxNumTessdataEntries);
  if (swap_) {
    actual_tessdata_num_entries_ = reverse32(actual_tessdata_num_entries_);
  }
  ASSERT_HOST(actual_tessdata_num_entries_ <= TESSDATA_NUM_ENTRIES);
  fread(offset_table_, sizeof(inT64),
        actual_tessdata_num_entries_, data_file_);
  if (swap_) {
    for (i = 0 ; i < actual_tessdata_num_entries_; ++i) {
      offset_table_[i] = reverse64(offset_table_[i]);
    }
  }
  if (debug_level_) {
    tprintf("TessdataManager loaded %d types of tesseract data files.\n",
            actual_tessdata_num_entries_);
    for (i = 0; i < actual_tessdata_num_entries_; ++i) {
      tprintf("Offset for type %d is %lld\n", i, offset_table_[i]);
    }
  }
  return true;
}

void TessdataManager::CopyFile(FILE *input_file, FILE *output_file,
                               bool newline_end, inT64 num_bytes_to_copy) {
  if (num_bytes_to_copy == 0) return;
  int buffer_size = 1024;
  if (num_bytes_to_copy > 0 && buffer_size > num_bytes_to_copy) {
    buffer_size = num_bytes_to_copy;
  }
  inT64 num_bytes_copied = 0;
  char *chunk = new char[buffer_size];
  int bytes_read;
  char last_char = 0x0;
  while ((bytes_read = fread(chunk, sizeof(char),
                             buffer_size, input_file))) {
    fwrite(chunk, sizeof(char), bytes_read, output_file);
    last_char = chunk[bytes_read-1];
    if (num_bytes_to_copy > 0) {
      num_bytes_copied += bytes_read;
      if (num_bytes_copied == num_bytes_to_copy) break;
      if (num_bytes_copied + buffer_size > num_bytes_to_copy) {
        buffer_size = num_bytes_to_copy - num_bytes_copied;
      }
    }
  }
  if (newline_end) ASSERT_HOST(last_char == '\n');
  delete[] chunk;
}

void TessdataManager::WriteMetadata(inT64 *offset_table, FILE *output_file) {
  fseek(output_file, 0, SEEK_SET);
  inT32 num_entries = TESSDATA_NUM_ENTRIES;
  fwrite(&num_entries, sizeof(inT32), 1, output_file);
  fwrite(offset_table, sizeof(inT64), TESSDATA_NUM_ENTRIES, output_file);
  fclose(output_file);

  tprintf("TessdataManager combined tesseract data files.\n");
  for (int i = 0; i < TESSDATA_NUM_ENTRIES; ++i) {
    tprintf("Offset for type %d is %lld\n", i, offset_table[i]);
  }
}

bool TessdataManager::CombineDataFiles(
    const char *language_data_path_prefix,
    const char *output_filename) {
  int i;
  inT64 offset_table[TESSDATA_NUM_ENTRIES];<commit_msg>More info for combine_tessdata files (thanks to gmvbif - issue 917)<commit_after>///////////////////////////////////////////////////////////////////////
// File:        tessdatamanager.cpp
// Description: Functions to handle loading/combining tesseract data files.
// Author:      <NAME>
// Created:     Wed Jun 03 11:26:43 PST 2009
//
// (C) Copyright 2009, Google Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
///////////////////////////////////////////////////////////////////////

#ifdef _MSC_VER
#pragma warning(disable:4244)  // Conversion warnings
#endif

#include "tessdatamanager.h"

#include <stdio.h>

#include "serialis.h"
#include "strngs.h"
#include "tprintf.h"
#include "params.h"

namespace tesseract {

bool TessdataManager::Init(const char *data_file_name, int debug_level) {
  int i;
  debug_level_ = debug_level;
  data_file_ = fopen(data_file_name, "rb");
  if (data_file_ == NULL) {
    tprintf("Error opening data file %s\n", data_file_name);
    tprintf("Please make sure the TESSDATA_PREFIX environment variable is set "
            "to the parent directory of your \"tessdata\" directory.\n");
    return false;
  }
  fread(&actual_tessdata_num_entries_, sizeof(inT32), 1, data_file_);
  swap_ = (actual_tessdata_num_entries_ > kMaxNumTessdataEntries);
  if (swap_) {
    actual_tessdata_num_entries_ = reverse32(actual_tessdata_num_entries_);
  }
  ASSERT_HOST(actual_tessdata_num_entries_ <= TESSDATA_NUM_ENTRIES);
  fread(offset_table_, sizeof(inT64),
        actual_tessdata_num_entries_, data_file_);
  if (swap_) {
    for (i = 0 ; i < actual_tessdata_num_entries_; ++i) {
      offset_table_[i] = reverse64(offset_table_[i]);
    }
  }
  if (debug_level_) {
    tprintf("TessdataManager loaded %d types of tesseract data files.\n",
            actual_tessdata_num_entries_);
    for (i = 0; i < actual_tessdata_num_entries_; ++i) {
      tprintf("Offset for type %d is %lld\n", i, offset_table_[i]);
    }
  }
  return true;
}

void TessdataManager::CopyFile(FILE *input_file, FILE *output_file,
                               bool newline_end, inT64 num_bytes_to_copy) {
  if (num_bytes_to_copy == 0) return;
  int buffer_size = 1024;
  if (num_bytes_to_copy > 0 && buffer_size > num_bytes_to_copy) {
    buffer_size = num_bytes_to_copy;
  }
  inT64 num_bytes_copied = 0;
  char *chunk = new char[buffer_size];
  int bytes_read;
  char last_char = 0x0;
  while ((bytes_read = fread(chunk, sizeof(char),
                             buffer_size, input_file))) {
    fwrite(chunk, sizeof(char), bytes_read, output_file);
    last_char = chunk[bytes_read-1];
    if (num_bytes_to_copy > 0) {
      num_bytes_copied += bytes_read;
      if (num_bytes_copied == num_bytes_to_copy) break;
      if (num_bytes_copied + buffer_size > num_bytes_to_copy) {
        buffer_size = num_bytes_to_copy - num_bytes_copied;
      }
    }
  }
  if (newline_end) ASSERT_HOST(last_char == '\n');
  delete[] chunk;
}

void TessdataManager::WriteMetadata(inT64 *offset_table, FILE *output_file) {
  fseek(output_file, 0, SEEK_SET);
  inT32 num_entries = TESSDATA_NUM_ENTRIES;
  fwrite(&num_entries, sizeof(inT32), 1, output_file);
  fwrite(offset_table, sizeof(inT64), TESSDATA_NUM_ENTRIES, output_file);
  fclose(output_file);

  tprintf("TessdataManager combined tesseract data files.\n");
  for (int i = 0; i < TESSDATA_NUM_ENTRIES; ++i) {
    tprintf("Offset for type %2d (lang.%-22s) is %lld\n", i, kTessdataFileSuffixes[i], offset_table[i]);
  }
}

bool TessdataManager::CombineDataFiles(
    const char *language_data_path_prefix,
    const char *output_filename) {
  int i;
  inT64 offset_table[TESSDATA_NUM_ENTRIES];
{'fraction_non_alphanumeric': 0.13924823041249695, 'fraction_numerical': 0.016109348303636806, 'mean_word_length': 3.3131578947368423, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1329577', 'n_tokens_mistral': 2823, 'n_tokens_neox': 2728, 'n_words': 690}