<commit_before># encoding: utf-8
require_relative "test_helper"

describe "Simple" do
  def assert_chardet_spec_detect(file, expected)
    content = File.open("test/simple_assets/#{file}.txt", 'rb'){|io| io.read }
    assert_equal expected, CharDet.detect(content)
  end

  def pending
    yield
  rescue StandardError
    skip
  else
    raise "Fixed"
  end

  it "detects EUC_JP" do
    assert_chardet_spec_detect 'EUC-JP', {
      "encoding" => 'EUC-JP', "confidence" => 0.99
    }
  end

  it "detects Shift_JIS" do
    assert_chardet_spec_detect 'Shift_JIS', {
      "encoding" => 'SHIFT_JIS', "confidence" => (RUBY_VERSION > "1.9.3" ? 0.99 : 1) # TODO the 1.9 value might be wrong but I cannot find any bug
    }
  end

  it "detects UTF_8" do
    assert_chardet_spec_detect 'UTF-8' , {
      "encoding" => 'utf-8', "confidence" => 0.99
    }
  end

  it "detects eucJP_ms" do
    assert_chardet_spec_detect 'eucJP-ms' , {
      "encoding" => 'EUC-JP', "confidence" => 0.99
    }
  end

  it "detects UTF_16BE" do
    assert_chardet_spec_detect 'UTF-16BE' , {
      "encoding" => 'UTF-16BE', "confidence" => 1
    }
  end

  it "detects UTF_16LE" do
    assert_chardet_spec_detect 'UTF-16LE' , {
      "encoding" => 'UTF-16LE', "confidence" => 1
    }
  end

  it "detects ISO_2022_JP" do
    assert_chardet_spec_detect  'ISO-2022-JP'  , {
      "encoding" => 'ISO-2022-JP', "confidence" => 0.99
    }
  end

  it "detects big5" do
    assert_chardet_spec_detect  'big5'  , {
      "encoding" => 'Big5', "confidence" => 0.99
    }
  end

  it "detects russian" do
    # this failed when using $KCODE='u' on 1.8 ... just making sure it stays put
    CharDet.detect("Toto je zpr\xE1va ve form\xE1tu MIME s n\xECkolika \xE8\xE1stmi.\n")["encoding"].must_equal "windows-1251"
  end

  it "detects what is likely to be ISO-8859-2 w/ garbage chars" do
    bad = "Colegio Nuestra Se\xcc\xb1ora Del Rosario"
    CharDet.detect(bad)["encoding"].must_equal "ISO-8859-2"
  end

  it "does not blow up on invalid encoding" do
    bad = "bad encoding: \xc3\x28"
    CharDet.detect(bad)["encoding"].must_equal "ISO-8859-2"
    bad.encoding.must_equal Encoding::UTF_8
  end

  it "does not blow up on multibyte UTF-8 chars" do
    accented = "<NAME>"
    CharDet.detect(accented)["encoding"].must_equal "ISO-8859-2"
    accented.encoding.must_equal Encoding::UTF_8
  end
end
<commit_msg>Add test cases for short Shift_JIS strings<commit_after># encoding: utf-8
require_relative "test_helper"

describe "Simple" do
  def assert_chardet_spec_detect(file, expected)
    content = File.open("test/simple_assets/#{file}.txt", 'rb'){|io| io.read }
    assert_equal expected, CharDet.detect(content)
  end

  def pending
    yield
  rescue StandardError
    skip
  else
    raise "Fixed"
  end

  it "detects EUC_JP" do
    assert_chardet_spec_detect 'EUC-JP', {
      "encoding" => 'EUC-JP', "confidence" => 0.99
    }
  end

  it "detects Shift_JIS" do
    assert_chardet_spec_detect 'Shift_JIS', {
      "encoding" => 'SHIFT_JIS', "confidence" => (RUBY_VERSION > "1.9.3" ? 0.99 : 1) # TODO the 1.9 value might be wrong but I cannot find any bug
    }
  end

  it "detects Shift_JIS from short string" do
    CharDet.detect("日本語".encode("Shift_JIS"))["encoding"].must_equal "SHIFT_JIS"
  end

  it "detects Shift_JIS from more than four characters" do
    CharDet.detect("四文字以上の日本語".encode("Shift_JIS"))["encoding"].must_equal "SHIFT_JIS"
  end

  it "detects Shift_JIS from Japanese and ASCII characters" do
    CharDet.detect("日本語 and ASCII characters".encode("Shift_JIS"))["encoding"].must_equal "SHIFT_JIS"
  end

  it "detects UTF_8" do
    assert_chardet_spec_detect 'UTF-8' , {
      "encoding" => 'utf-8', "confidence" => 0.99
    }
  end

  it "detects eucJP_ms" do
    assert_chardet_spec_detect 'eucJP-ms' , {
      "encoding" => 'EUC-JP', "confidence" => 0.99
    }
  end

  it "detects UTF_16BE" do
    assert_chardet_spec_detect 'UTF-16BE' , {
      "encoding" => 'UTF-16BE', "confidence" => 1
    }
  end

  it "detects UTF_16LE" do
    assert_chardet_spec_detect 'UTF-16LE' , {
      "encoding" => 'UTF-16LE', "confidence" => 1
    }
  end

  it "detects ISO_2022_JP" do
    assert_chardet_spec_detect  'ISO-2022-JP'  , {
      "encoding" => 'ISO-2022-JP', "confidence" => 0.99
    }
  end

  it "detects big5" do
    assert_chardet_spec_detect  'big5'  , {
      "encoding" => 'Big5', "confidence" => 0.99
    }
  end

  it "detects russian" do
    # this failed when using $KCODE='u' on 1.8 ... just making sure it stays put
    CharDet.detect("Toto je zpr\xE1va ve form\xE1tu MIME s n\xECkolika \xE8\xE1stmi.\n")["encoding"].must_equal "windows-1251"
  end

  it "detects what is likely to be ISO-8859-2 w/ garbage chars" do
    bad = "Colegio Nuestra Se\xcc\xb1ora Del Rosario"
    CharDet.detect(bad)["encoding"].must_equal "ISO-8859-2"
  end

  it "does not blow up on invalid encoding" do
    bad = "bad encoding: \xc3\x28"
    CharDet.detect(bad)["encoding"].must_equal "ISO-8859-2"
    bad.encoding.must_equal Encoding::UTF_8
  end

  it "does not blow up on multibyte UTF-8 chars" do
    accented = "<NAME>"
    CharDet.detect(accented)["encoding"].must_equal "ISO-8859-2"
    accented.encoding.must_equal Encoding::UTF_8
  end
end

{'fraction_non_alphanumeric': 0.13100686498855835, 'fraction_numerical': 0.03585049580472922, 'mean_word_length': 3.196, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1735373', 'n_tokens_mistral': 2144, 'n_tokens_neox': 1959, 'n_words': 481}