<commit_before>require File.dirname(__FILE__) + '/../../spec_helper'

# These specs are not intended to be a complete example of the
# strings recognized by the String#count algorithm. See also
# the String#count specs.
describe "String#count_table" do
  it "returns a String of 256 1's if not passed a string" do
    t = "".count_table
    t.size.should == 256
    t.each_byte { |b| b.should == 1 }
  end

  it "returns a String of 256 0's if passed an empty string" do
    t = "".count_table ""
    t.each_byte { |b| b.should == 0 }
  end

  it "returns a Tuple with entries corresponding to characters set to 1" do
    t = "".count_table "-\\|*\030 abcde"
    256.times { |i| t[i].should == ("-|*\030 abcde".include?(i.chr) ? 1 : 0) }
  end

  it "returns a Tuple with entries corresponding to sequences set to 1" do
    t = "".count_table "-ab-g\\d-abhi--A\\"
    256.times { |i|
      t[i].should == ("-abcdefgbhA\\".include?(i.chr) ? 1 : 0)
    }
  end

  it "returns a Tuple with entries corresponding included and excluded characters" do
    t = "".count_table "ab", "^bc"
    256.times { |i| t[i].should == (i == ?a ? 1 : 0) }
  end

  it "returns a Tuple with entries set to false for '^abc-e' inverse match strings" do
    t = "".count_table "^ab-g\\d-abhi--A\\"
    256.times { |i|
      t[i].should == ("abcdefgbhA\\".include?(i.chr) ? 0 : 1)
    }
  end
end<commit_msg>Fix errors in String#count_table spec descriptions.<commit_after>require File.dirname(__FILE__) + '/../../spec_helper'

# These specs are not intended to be a complete example of the
# strings recognized by the String#count algorithm. See also
# the String#count specs.
describe "String#count_table" do
  it "returns a String of 256 1's if not passed a string" do
    t = "".count_table
    t.size.should == 256
    t.each_byte { |b| b.should == 1 }
  end

  it "returns a String of 256 0's if passed an empty string" do
    t = "".count_table ""
    t.each_byte { |b| b.should == 0 }
  end

  it "returns a String with entries corresponding to characters set to 1" do
    t = "".count_table "-\\|*\030 abcde"
    256.times { |i| t[i].should == ("-|*\030 abcde".include?(i.chr) ? 1 : 0) }
  end

  it "returns a String with entries corresponding to sequences set to 1" do
    t = "".count_table "-ab-g\\d-abhi--A\\"
    256.times { |i|
      t[i].should == ("-abcdefgbhA\\".include?(i.chr) ? 1 : 0)
    }
  end

  it "returns a String with entries corresponding included and excluded characters" do
    t = "".count_table "ab", "^bc"
    256.times { |i| t[i].should == (i == ?a ? 1 : 0) }
  end

  it "returns a String with entries set to false for '^abc-e' inverse match strings" do
    t = "".count_table "^ab-g\\d-abhi--A\\"
    256.times { |i|
      t[i].should == ("abcdefgbhA\\".include?(i.chr) ? 0 : 1)
    }
  end
end
{'fraction_non_alphanumeric': 0.15384615384615385, 'fraction_numerical': 0.0292022792022792, 'mean_word_length': 2.720529801324503, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7232125', 'n_tokens_mistral': 1060, 'n_tokens_neox': 975, 'n_words': 342}