<commit_before>// Copyright 2010 Google Inc. All Rights Reserved.
// Author: <EMAIL> (<NAME>)
///////////////////////////////////////////////////////////////////////
// File:        shapetable.cpp
// Description: Class to map a classifier shape index to unicharset
//              indices and font indices.
// Author:      <NAME>
// Created:     Tue Nov 02 15:31:32 PDT 2010
//
// (C) Copyright 2010, Google Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
///////////////////////////////////////////////////////////////////////

#include "shapetable.h"

#include "intfeaturespace.h"
#include "strngs.h"
#include "unicharset.h"

namespace tesseract {

// Writes to the given file. Returns false in case of error.
bool UnicharAndFonts::Serialize(FILE* fp) {
  inT32 uni_id = unichar_id;
  if (fwrite(&uni_id, sizeof(uni_id), 1, fp) != 1) return false;
  if (!font_ids.Serialize(fp)) return false;
  return true;
}
// Reads from the given file. Returns false in case of error.
// If swap is true, assumes a big/little-endian swap is needed.
bool UnicharAndFonts::DeSerialize(bool swap, FILE* fp) {
  inT32 uni_id;
  if (fread(&uni_id, sizeof(uni_id), 1, fp) != 1) return false;
  if (swap)
    ReverseN(&uni_id, sizeof(uni_id));
  unichar_id = uni_id;
  if (!font_ids.DeSerialize(swap, fp)) return false;
  return true;
}

// Sort function to sort a pair of UnicharAndFonts by unichar_id.
int UnicharAndFonts::SortByUnicharId(const void* v1, const void* v2) {
  const UnicharAndFonts* p1 = reinterpret_cast<const UnicharAndFonts*>(v1);
  const UnicharAndFonts* p2 = reinterpret_cast<const UnicharAndFonts*>(v2);
  return p1->unichar_id - p2->unichar_id;
}

// Writes to the given file. Returns false in case of error.
bool Shape::Serialize(FILE* fp) {
  if (fwrite(&unichars_sorted_, sizeof(unichars_sorted_), 1, fp) != 1)
    return false;
  if (!unichars_.SerializeClasses(fp)) return false;
  return true;
}
// Reads from the given file. Returns false in case of error.
// If swap is true, assumes a big/little-endian swap is needed.
bool Shape::DeSerialize(bool swap, FILE* fp) {
  if (fread(&unichars_sorted_, sizeof(unichars_sorted_), 1, fp) != 1)
    return false;
  if (!unichars_.DeSerializeClasses(swap, fp)) return false;
  return true;
}

// Adds a font_id for the given unichar_id. If the unichar_id is not
// in the shape, it is added.
void Shape::AddToShape(int unichar_id, int font_id) {
  for (int c = 0; c < unichars_.size(); ++c) {
    if (unichars_[c].unichar_id == unichar_id) {
      // Found the unichar in the shape table.
      GenericVector<int>& font_list = unichars_[c].font_ids;
      for (int f = 0; f < font_list.size(); ++f) {
        if (font_list[f] == font_id)
          return;  // Font is already there.
      }
      font_list.push_back(font_id);
      return;
    }
  }
  // Unichar_id is not in shape, so add it to shape.
  unichars_.push_back(UnicharAndFonts(unichar_id, font_id));
  unichars_sorted_ =  unichars_.size() <= 1;
}<commit_msg>Fix a shapetable serialization issue -- sizeof(bool) is not portable.<commit_after>// Copyright 2010 Google Inc. All Rights Reserved.
// Author: <EMAIL> (<NAME>)
///////////////////////////////////////////////////////////////////////
// File:        shapetable.cpp
// Description: Class to map a classifier shape index to unicharset
//              indices and font indices.
// Author:      <NAME>
// Created:     Tue Nov 02 15:31:32 PDT 2010
//
// (C) Copyright 2010, Google Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
///////////////////////////////////////////////////////////////////////

#include "shapetable.h"

#include "intfeaturespace.h"
#include "strngs.h"
#include "unicharset.h"

namespace tesseract {

// Writes to the given file. Returns false in case of error.
bool UnicharAndFonts::Serialize(FILE* fp) {
  inT32 uni_id = unichar_id;
  if (fwrite(&uni_id, sizeof(uni_id), 1, fp) != 1) return false;
  if (!font_ids.Serialize(fp)) return false;
  return true;
}
// Reads from the given file. Returns false in case of error.
// If swap is true, assumes a big/little-endian swap is needed.
bool UnicharAndFonts::DeSerialize(bool swap, FILE* fp) {
  inT32 uni_id;
  if (fread(&uni_id, sizeof(uni_id), 1, fp) != 1) return false;
  if (swap)
    ReverseN(&uni_id, sizeof(uni_id));
  unichar_id = uni_id;
  if (!font_ids.DeSerialize(swap, fp)) return false;
  return true;
}

// Sort function to sort a pair of UnicharAndFonts by unichar_id.
int UnicharAndFonts::SortByUnicharId(const void* v1, const void* v2) {
  const UnicharAndFonts* p1 = reinterpret_cast<const UnicharAndFonts*>(v1);
  const UnicharAndFonts* p2 = reinterpret_cast<const UnicharAndFonts*>(v2);
  return p1->unichar_id - p2->unichar_id;
}

// Writes to the given file. Returns false in case of error.
bool Shape::Serialize(FILE* fp) {
  if (fwrite(&unichars_sorted_, 1, 1, fp) != 1)
    return false;
  if (!unichars_.SerializeClasses(fp)) return false;
  return true;
}
// Reads from the given file. Returns false in case of error.
// If swap is true, assumes a big/little-endian swap is needed.
bool Shape::DeSerialize(bool swap, FILE* fp) {
  if (fread(&unichars_sorted_, 1, 1, fp) != 1)
    return false;
  if (!unichars_.DeSerializeClasses(swap, fp)) return false;
  return true;
}

// Adds a font_id for the given unichar_id. If the unichar_id is not
// in the shape, it is added.
void Shape::AddToShape(int unichar_id, int font_id) {
  for (int c = 0; c < unichars_.size(); ++c) {
    if (unichars_[c].unichar_id == unichar_id) {
      // Found the unichar in the shape table.
      GenericVector<int>& font_list = unichars_[c].font_ids;
      for (int f = 0; f < font_list.size(); ++f) {
        if (font_list[f] == font_id)
          return;  // Font is already there.
      }
      font_list.push_back(font_id);
      return;
    }
  }
  // Unichar_id is not in shape, so add it to shape.
  unichars_.push_back(UnicharAndFonts(unichar_id, font_id));
  unichars_sorted_ =  unichars_.size() <= 1;
}
{'fraction_non_alphanumeric': 0.15266628440366972, 'fraction_numerical': 0.013761467889908258, 'mean_word_length': 2.980034227039361, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3706856', 'n_tokens_mistral': 2367, 'n_tokens_neox': 2288, 'n_words': 774}