<commit_before>require_relative 'lib/text'

def banner text
  puts "\e[32m", '*', text.gsub(/^/, '* '),  "*\e[0m"
end
def wee_banner text
  puts "\e[36m", text.gsub(/^/, '* '), "\e[0m"
end

[
  ['simple ASCII string', "ABC 123".force_encoding(Encoding::UTF_8)],
  ['ASCII string with controls', "ABC 123\n".force_encoding(Encoding::UTF_8)],
  ['8-bit string', "\u{00E6}".force_encoding(Encoding::UTF_8)],
  ['astral emoji', "\xF0\x9F\xA4\x94".force_encoding(Encoding::UTF_8)],
].each do |label, str|
  banner label

  wee_banner 'ruby string (utf-8)'
  p [str, str.encoding]

  wee_banner 'Text object'
  text = Text.from_ruby_string(str)
  p text

  [
    [CES::UTF8],
    [CES::UTF16],
    [CES::UTF32],
    [CES::SBCS, CCS::ASCII],
    [CES::SBCS, CCS::ISO8859_1],
    [CES::SBCS, CCS::WINDOWS1252],
    [CES::SBCS, CCS::CP437],
  ].each do |args|
    begin
      wee_banner "to #{args.first}+#{(args[1] || args.first.default_ccs)}"
      p text.serialize(*args)

      srl = text.transcode_and_serialize(*args)
      p srl

      srls = srl.to_s
      p [srls, srls.encoding]
    rescue RuntimeError => e
      puts "\e[91m#{e.inspect}\e[0m"
    end
  end
end
<commit_msg>make test thingy pass rubocop<commit_after>require_relative 'lib/text'

def banner text
  puts "\e[32m", '*', text.gsub(/^/, '* '),  "*\e[0m"
end

def wee_banner text
  puts "\e[36m", text.gsub(/^/, '* '), "\e[0m"
end

[
  ['simple ASCII string', 'ABC 123'.force_encoding(Encoding::UTF_8)],
  ['ASCII string with controls', "ABC 123\n".force_encoding(Encoding::UTF_8)],
  ['8-bit string', "\u{00E6}".force_encoding(Encoding::UTF_8)],
  ['astral emoji', "\xF0\x9F\xA4\x94".force_encoding(Encoding::UTF_8)],
].each do |label, str|
  banner label

  wee_banner 'ruby string (utf-8)'
  p [str, str.encoding]

  wee_banner 'Text object'
  text = Text.from_ruby_string(str)
  p text

  [
    [CES::UTF8],
    [CES::UTF16],
    [CES::UTF32],
    [CES::SBCS, CCS::ASCII],
    [CES::SBCS, CCS::ISO8859_1],
    [CES::SBCS, CCS::WINDOWS1252],
    [CES::SBCS, CCS::CP437],
  ].each do |args|
    begin
      wee_banner "to #{args.first}+#{(args[1] || args.first.default_ccs)}"
      p text.serialize(*args)

      srl = text.transcode_and_serialize(*args)
      p srl

      srls = srl.to_s
      p [srls, srls.encoding]
    rescue RuntimeError => e
      puts "\e[91m#{e.inspect}\e[0m"
    end
  end
end

# vim: ts=2:sts=2:sw=2:expandtab
{'fraction_non_alphanumeric': 0.1948376353039134, 'fraction_numerical': 0.040383014154870944, 'mean_word_length': 3.5084427767354596, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2701177', 'n_tokens_mistral': 1092, 'n_tokens_neox': 1005, 'n_words': 192}