<commit_before>require_relative "../lib/unibits/kernel_method"
require "minitest/autorun"

describe Unibits do
  it "will not work for unsupported encodings" do
    proc{
      Unibits.of("string".encode("UTF-32"))
    }.must_raise ArgumentError
  end

  describe 'Encodings' do
    let(:string){ "🌫 Idio﻿syncrätic ℜսᖯʏ" }

    it "works with UTF-8" do
      result = Paint.unpaint(Unibits.visualize(string))
      result.must_match "ℜ"
      result.must_match "U+211C"
      result.must_match /E2.*84.*9C/m
      result.must_match /11100010.*10000100.*10011100/m
    end

    it "works with UTF-16LE" do
      result = Paint.unpaint(Unibits.visualize(string.encode('UTF-16LE')))
      result.must_match "ℜ"
      result.must_match "U+211C"
      result.must_match /1C.*21/m
      result.must_match /00011100.*00100001/m
    end

    it "works with UTF-16BE" do
      result = Paint.unpaint(Unibits.visualize(string.encode('UTF-16BE')))
      result.must_match "ℜ"
      result.must_match "U+211C"
      result.must_match /21.*1C/m
      result.must_match /00100001.*00011100/m
    end

    it "works with UTF-32LE" do
      result = Paint.unpaint(Unibits.visualize(string.encode('UTF-32LE')))
      result.must_match "ℜ"
      result.must_match "U+211C"
      result.must_match /1C.*21.*00.*00/m
      result.must_match /00011100.*00100001.*00000000.*00000000/m
    end

    it "works with UTF-32BE" do
      result = Paint.unpaint(Unibits.visualize(string.encode('UTF-32BE')))
      result.must_match "ℜ"
      result.must_match "U+211C"
      result.must_match /00.*00.*21.*1C/m
      result.must_match /00000000.*00000000.*00100001.*00011100/m
    end

    it "works with BINARY" do
      result = Paint.unpaint(Unibits.visualize(string.dup.force_encoding('BINARY')))
      # testing for the UTF-8 encoded "ℜ"
      result.must_match "\\xE2"
      result.must_match "\\x84"
      result.must_match "\\x9C"
      result.must_match /11100010.*10000100.*10011100/m
    end

    it "works with ASCII" do
      result = Paint.unpaint(Unibits.visualize("ASCII string".force_encoding('ASCII')))
      result.must_match "C"
      result.must_match "43"
      result.must_match "01000011"
    end

    it "works with 'ISO-8859-' encodings" do
      string = "\xBC Idiosyncr\xE4tic\n\x91".force_encoding("ISO-8859-1")
      result = Paint.unpaint(Unibits.visualize(string))
      result.must_match "BC"  # ¼
      result.must_match "E4"  # ä
      result.must_match "␊"   # \n
      result.must_match "PU1" # C1 name for \x91
    end

    it "works with 'Windows-125' encodings" do
      string = "\xBC Idiosyncr\xE4tic\n\x81".force_encoding("Windows-1252")
      result = Paint.unpaint(Unibits.visualize(string))
      result.must_match "BC"  # ¼
      result.must_match "E4"  # ä
      result.must_match "␊"   # \n
      result.must_match "n/a" # \x81 is not assigned
    end

    describe "invalid UTF-8 encodings" do
      it "- unexpected continuation byte (1/2)" do
        string = "abc\x80efg"
        result = Paint.unpaint(Unibits.visualize(string))
        result.must_match "�"
        result.must_match "unexp.c."
        result.must_match /e.*f.*g/m
      end

      it "- unexpected continuation byte (2/2)" do
        string = "🌫\x81efg"
        result = Paint.unpaint(Unibits.visualize(string))
        result.must_match "�"
        result.must_match "unexp.c."
        result.must_match /e.*f.*g/m
      end

      it "- not enough continuation bytes" do
        string = "\xF0\x9F\x8CABC"
        result = Paint.unpaint(Unibits.visualize(string))
        result.must_match "�"
        result.must_match "n.e.con."
        result.must_match /A.*B.*C/m
      end

      it "- overlong padding (1/2)" do
        string = "\xE0\x81\x81ABC"
        result = Paint.unpaint(Unibits.visualize(string))<commit_msg>Specs for IBMX and CP85X<commit_after>require_relative "../lib/unibits/kernel_method"
require "minitest/autorun"

describe Unibits do
  it "will not work for unsupported encodings" do
    proc{
      Unibits.of("string".encode("UTF-32"))
    }.must_raise ArgumentError
  end

  describe 'Encodings' do
    let(:string){ "🌫 Idio﻿syncrätic ℜսᖯʏ" }

    it "works with UTF-8" do
      result = Paint.unpaint(Unibits.visualize(string))
      result.must_match "ℜ"
      result.must_match "U+211C"
      result.must_match /E2.*84.*9C/m
      result.must_match /11100010.*10000100.*10011100/m
    end

    it "works with UTF-16LE" do
      result = Paint.unpaint(Unibits.visualize(string.encode('UTF-16LE')))
      result.must_match "ℜ"
      result.must_match "U+211C"
      result.must_match /1C.*21/m
      result.must_match /00011100.*00100001/m
    end

    it "works with UTF-16BE" do
      result = Paint.unpaint(Unibits.visualize(string.encode('UTF-16BE')))
      result.must_match "ℜ"
      result.must_match "U+211C"
      result.must_match /21.*1C/m
      result.must_match /00100001.*00011100/m
    end

    it "works with UTF-32LE" do
      result = Paint.unpaint(Unibits.visualize(string.encode('UTF-32LE')))
      result.must_match "ℜ"
      result.must_match "U+211C"
      result.must_match /1C.*21.*00.*00/m
      result.must_match /00011100.*00100001.*00000000.*00000000/m
    end

    it "works with UTF-32BE" do
      result = Paint.unpaint(Unibits.visualize(string.encode('UTF-32BE')))
      result.must_match "ℜ"
      result.must_match "U+211C"
      result.must_match /00.*00.*21.*1C/m
      result.must_match /00000000.*00000000.*00100001.*00011100/m
    end

    it "works with BINARY" do
      result = Paint.unpaint(Unibits.visualize(string.dup.force_encoding('BINARY')))
      # testing for the UTF-8 encoded "ℜ"
      result.must_match "\\xE2"
      result.must_match "\\x84"
      result.must_match "\\x9C"
      result.must_match /11100010.*10000100.*10011100/m
    end

    it "works with ASCII" do
      result = Paint.unpaint(Unibits.visualize("ASCII string".force_encoding('ASCII')))
      result.must_match "C"
      result.must_match "43"
      result.must_match "01000011"
    end

    it "works with 'ISO-8859-X' encodings" do
      string = "\xBC Idiosyncr\xE4tic\n\x91".force_encoding("ISO-8859-1")
      result = Paint.unpaint(Unibits.visualize(string))
      result.must_match "BC"  # ¼
      result.must_match "E4"  # ä
      result.must_match "␊"   # \n
      result.must_match "PU1" # C1 name for \x91
    end

    it "works with 'Windows-125X' encodings" do
      string = "\xBC Idiosyncr\xE4tic\n\x81".force_encoding("Windows-1252")
      result = Paint.unpaint(Unibits.visualize(string))
      result.must_match "BC"  # ¼
      result.must_match "E4"  # ä
      result.must_match "␊"   # \n
      result.must_match "n/a" # \x81 is not assigned
    end

    it "works with 'IBMX' encodings" do
      string = "\xFE Idiosyncr\x84tic\n".force_encoding("IBM437")
      result = Paint.unpaint(Unibits.visualize(string))
      result.must_match "FE"  # ■
      result.must_match "84"  # ä
      result.must_match "␊"   # \n
    end

    it "works with 'CP85X' encodings" do
      string = "\xFE Idiosyncr\x84tic\n".force_encoding("CP850")
      result = Paint.unpaint(Unibits.visualize(string))
      result.must_match "FE"  # ■
      result.must_match "84"  # ä
      result.must_match "␊"   # \n
    end

    describe "invalid UTF-8 encodings" do
      it "- unexpected continuation byte (1/2)" do
        string = "abc\x80efg"
        result = Paint.unpaint(Unibits.visualize(string))
        result.must_match "�"
        result.must_match "unexp.c."
        result.must_match /e.*f.*g/m
      end

      it "- unexpected continuation byte (2/2)" do
        string = "🌫\x81efg"
        result = Paint.unpaint(Unibits.visualize(string))
        result.must_match "�"
        result.must_match "unexp.c."
        result.must_match /e.*f.*g/m
      end

      it "- not enough continuation bytes" do
        string = "\xF0\x9F\x8CABC"
        result = Paint.unpaint(Unibits.visualize(string))
        result.must_match "�"
        result.must_match "n.e.con."
        result.must_match /A.*B.*C/m
      end

      it "- overlong padding (1/2)" do
        string = "\xE0\x81\x81ABC"
        result = Paint.unpaint(Unibits.visualize(string))
{'fraction_non_alphanumeric': 0.13053556179089912, 'fraction_numerical': 0.06709771867756496, 'mean_word_length': 2.526021505376344, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 8, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5176416', 'n_tokens_mistral': 3459, 'n_tokens_neox': 2974, 'n_words': 571}