<commit_before>﻿/* -*- coding: utf-8; mode: c++; tab-width: 3; indent-tabs-mode: nil -*-

Copyright 2010, 2011, 2012, 2013, 2014
<NAME>

This file is part of Application-Building Components (henceforth referred to as ABC).

ABC is free software: you can redistribute it and/or modify it under the terms of the GNU General
Public License as published by the Free Software Foundation, either version 3 of the License, or (at
your option) any later version.

ABC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
License for more details.

You should have received a copy of the GNU General Public License along with ABC. If not, see
<http://www.gnu.org/licenses/>.
--------------------------------------------------------------------------------------------------*/

#include <abc/core.hxx>



////////////////////////////////////////////////////////////////////////////////////////////////////
// abc::text globals

namespace abc {

namespace text {

ABCAPI size_t estimate_transcoded_size(
   encoding encSrc, void const * pSrc, size_t cbSrc, encoding encDst
) {
   ABC_TRACE_FN((encSrc, pSrc, cbSrc, encDst));

   // Little helper to map abc::text::encoding values with byte sizes (see below).
   struct enc_cb_map_t {
      uint8_t enc;
      uint8_t cb;
   };

   // Average size, in bytes, of 10 characters in each supported encoding.
   static enc_cb_map_t const sc_aecmAvg10Chars[] = {
      { encoding::utf8,         15 }, // Some languages require 3 bytes per character.
      { encoding::utf16le,      20 }, // Consider surrogates extremely unlikely, as they are.
      { encoding::utf16be,      20 }, // Same as encoding::utf16le.
      { encoding::utf32le,      40 }, // Exact constant.
      { encoding::utf32be,      40 }, // Same as encoding::utf32le.
      { encoding::iso_8859_1,   10 }, // Exact constant.
      { encoding::windows_1252, 10 }, // Exact constant.
      { encoding::ebcdic,       10 }, // Exact constant.
   };

   if (encSrc < encoding::_charsets_offset) {
      ABC_THROW(argument_error, ());
   }
   if (encDst < encoding::_charsets_offset) {
      ABC_THROW(argument_error, ());
   }
   // TODO: use this to give a more accurate estimate for UTF-8, by evaluating which language block
   // seems to be dominant in the source.
   ABC_UNUSED_ARG(pSrc);

   size_t cbSrcAvg, cbDstAvg;
   // Estimate the number of code points in the source.
   for (size_t i(0); i < ABC_COUNTOF(sc_aecmAvg10Chars); ++i) {
      encoding::enum_type enc(encoding::enum_type(sc_aecmAvg10Chars[i].enc));
      if (encSrc == enc) {
         cbSrcAvg = sc_aecmAvg10Chars[i].cb;
      }
      if (encDst == enc) {
         cbDstAvg = sc_aecmAvg10Chars[i].cb;
      }
   }
   // If we were using floating-point math, this would be the return statement’s expression:
   //    ceil(cbSrc / cbSrcAvg) * cbDstAvg
   // We need to emulate ceil() on integers with:
   //    (cbSrc + cbSrcAvg - 1) / cbSrcAvg
   // Also, we have to multiply first, to avoid underflow, but this would expose to overflow, in
   // which case we’ll divide first, as in the original expression.
   size_t ccp(cbSrc * cbDstAvg);
   if (ccp >= cbSrc) {
      // No integer overflow.
      return (ccp + cbSrcAvg - 1) / cbSrcAvg;
   } else {
      // Integer overflow occurred: evaluate the expression in the origianal order.
      return ((cbSrc + cbSrcAvg - 1) / cbSrcAvg) * cbDstAvg;
   }
}


ABCAPI size_t get_encoding_size(encoding enc) {
   // Character size, in bytes, for each recognized encoding.
   static uint8_t const sc_acbEncChar[] = {
      0, // encoding::unknown
      0, // encoding::identity
      1, // encoding::utf8
      2, // encoding::utf16le
      2, // encoding::utf16be
      4, // encoding::utf32le
      4, // encoding::utf32be
      1, // encoding::iso_8859_1
      1, // encoding::windows_1252
      1, // encoding::ebcdic
   };
   return sc_acbEncChar[enc.base()];
}


ABCAPI void const * get_line_terminator_bytes(
   encoding enc, line_terminator lterm, size_t * pcb
) {<commit_msg>Make data structures in abc::text::get_encoding_size() more secure<commit_after>﻿/* -*- coding: utf-8; mode: c++; tab-width: 3; indent-tabs-mode: nil -*-

Copyright 2010, 2011, 2012, 2013, 2014
<NAME>

This file is part of Application-Building Components (henceforth referred to as ABC).

ABC is free software: you can redistribute it and/or modify it under the terms of the GNU General
Public License as published by the Free Software Foundation, either version 3 of the License, or (at
your option) any later version.

ABC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
License for more details.

You should have received a copy of the GNU General Public License along with ABC. If not, see
<http://www.gnu.org/licenses/>.
--------------------------------------------------------------------------------------------------*/

#include <abc/core.hxx>



////////////////////////////////////////////////////////////////////////////////////////////////////
// abc::text globals

namespace abc {

namespace text {

ABCAPI size_t estimate_transcoded_size(
   encoding encSrc, void const * pSrc, size_t cbSrc, encoding encDst
) {
   ABC_TRACE_FN((encSrc, pSrc, cbSrc, encDst));

   // Little helper to map abc::text::encoding values with byte sizes (see below).
   struct enc_cb_t {
      uint8_t enc;
      uint8_t cb;
   };

   // Average size, in bytes, of 10 characters in each supported encoding.
   static enc_cb_t const sc_aecAvg10Chars[] = {
      { encoding::utf8,         15 }, // Some languages require 3 bytes per character.
      { encoding::utf16le,      20 }, // Consider surrogates extremely unlikely, as they are.
      { encoding::utf16be,      20 }, // Same as encoding::utf16le.
      { encoding::utf32le,      40 }, // Exact constant.
      { encoding::utf32be,      40 }, // Same as encoding::utf32le.
      { encoding::iso_8859_1,   10 }, // Exact constant.
      { encoding::windows_1252, 10 }, // Exact constant.
      { encoding::ebcdic,       10 }, // Exact constant.
   };

   if (encSrc < encoding::_charsets_offset) {
      ABC_THROW(argument_error, ());
   }
   if (encDst < encoding::_charsets_offset) {
      ABC_THROW(argument_error, ());
   }
   // TODO: use this to give a more accurate estimate for UTF-8, by evaluating which language block
   // seems to be dominant in the source.
   ABC_UNUSED_ARG(pSrc);

   size_t cbSrcAvg, cbDstAvg;
   // Estimate the number of code points in the source.
   // TODO: improve search algorithm, or maybe use a real map.
   for (size_t i(0); i < ABC_COUNTOF(sc_aecAvg10Chars); ++i) {
      encoding::enum_type enc(encoding::enum_type(sc_aecAvg10Chars[i].enc));
      if (encSrc == enc) {
         cbSrcAvg = sc_aecAvg10Chars[i].cb;
      }
      if (encDst == enc) {
         cbDstAvg = sc_aecAvg10Chars[i].cb;
      }
   }
   // If we were using floating-point math, this would be the return statement’s expression:
   //    ceil(cbSrc / cbSrcAvg) * cbDstAvg
   // We need to emulate ceil() on integers with:
   //    (cbSrc + cbSrcAvg - 1) / cbSrcAvg
   // Also, we have to multiply first, to avoid underflow, but this would expose to overflow, in
   // which case we’ll divide first, as in the original expression.
   size_t ccp(cbSrc * cbDstAvg);
   if (ccp >= cbSrc) {
      // No integer overflow.
      return (ccp + cbSrcAvg - 1) / cbSrcAvg;
   } else {
      // Integer overflow occurred: evaluate the expression in the origianal order.
      return ((cbSrc + cbSrcAvg - 1) / cbSrcAvg) * cbDstAvg;
   }
}


ABCAPI size_t get_encoding_size(encoding enc) {
   ABC_TRACE_FN((enc));

   // Little helper to map abc::text::encoding values with byte sizes (see below).
   struct enc_cb_t {
      uint8_t enc;
      uint8_t cb;
   };

   // Character size, in bytes, for each recognized encoding.
   static enc_cb_t const sc_aecEncChar[] = {
      { encoding::utf8,         1 },
      { encoding::utf16le,      2 },
      { encoding::utf16be,      2 },
      { encoding::utf32le,      4 },
      { encoding::utf32be,      4 },
      { encoding::iso_8859_1,   1 },
      { encoding::windows_1252, 1 },
      { encoding::ebcdic,       1 },
   };
   // TODO: improve search algorithm, or maybe use a real map.
   for (size_t i(0); i < ABC_COUNTOF(sc_aecEncChar); ++i) {
      if (enc == encoding::enum_type(sc_aecEncChar[i].enc)) {
         return sc_aecEncChar[i].cb;
      }
   }
   return 0;
}


ABCAPI void const * get_line_terminator_bytes(
   encoding enc, line_terminator lterm, size_t * pcb
) {
{'fraction_non_alphanumeric': 0.14894346087949742, 'fraction_numerical': 0.02524271844660194, 'mean_word_length': 2.404354587869362, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3551510', 'n_tokens_mistral': 2848, 'n_tokens_neox': 2669, 'n_words': 889}