<commit_before># encoding: utf-8

# Encoding::Windows_1252
# Encoding::MacRoman
# Encoding::UTF_8
# Encoding::ISO8859_15

require 'tabledata/exceptions'

module TableData
  module Detection
    UnlikelyCharsWin1252    = "\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA\xAB\xAC\xAD" \
                              "\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB" \
                              "\xBC\xBD\xBE\xBF\xD7\xF7"
    UnlikelyCharsIso8859_1  = ""
    UnlikelyCharsMacRoman   = ""

    UmlautsMac  = "äöü".encode(Encoding::MacRoman).force_encoding(Encoding::BINARY)
    UmlautsWin  = "äöü".encode(Encoding::Windows_1252).force_encoding(Encoding::BINARY)

    DiacritsMac = "âàéèô".encode(Encoding::MacRoman).force_encoding(Encoding::BINARY)
    DiacritsWin = "âàéèô".encode(Encoding::Windows_1252).force_encoding(Encoding::BINARY)

  module_function
    def force_guessed_encoding!(string)
      return string if string.force_encoding(Encoding::UTF_8).valid_encoding?
      string.force_encoding(Encoding::BINARY)<commit_msg>Added minimal documentation to TableData::Detection.<commit_after># encoding: utf-8

# Encoding::Windows_1252
# Encoding::MacRoman
# Encoding::UTF_8
# Encoding::ISO8859_15

require 'tabledata/exceptions'

module TableData

  # This module provides methods to detect the encoding of a text file.
  # Currently the detection is limited to utf-8, windows-1252 and macroman in
  # western european languages.
  module Detection
    UnlikelyCharsWin1252    = "\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA\xAB\xAC\xAD" \
                              "\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB" \
                              "\xBC\xBD\xBE\xBF\xD7\xF7"
    UnlikelyCharsIso8859_1  = ""
    UnlikelyCharsMacRoman   = ""

    UmlautsMac  = "äöü".encode(Encoding::MacRoman).force_encoding(Encoding::BINARY)
    UmlautsWin  = "äöü".encode(Encoding::Windows_1252).force_encoding(Encoding::BINARY)

    DiacritsMac = "âàéèô".encode(Encoding::MacRoman).force_encoding(Encoding::BINARY)
    DiacritsWin = "âàéèô".encode(Encoding::Windows_1252).force_encoding(Encoding::BINARY)

  module_function
    def force_guessed_encoding!(string)
      return string if string.force_encoding(Encoding::UTF_8).valid_encoding?
      string.force_encoding(Encoding::BINARY)
{'fraction_non_alphanumeric': 0.12391304347826088, 'fraction_numerical': 0.04739130434782609, 'mean_word_length': 3.734567901234568, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5688095', 'n_tokens_mistral': 892, 'n_tokens_neox': 834, 'n_words': 99}