<commit_before>require "spec_helper"

class NormalizerTest
  include Reckless::Normalizer
end

describe Reckless::Normalizer do
  let(:normalizer) { NormalizerTest.new }

  describe "#normalize_text" do
    let(:examples) do
      {
        "ALL CAPS TEXT"        => "All Caps Text",
        "all low text"         => "All Low Text",
        "13TH FLOOR ELEVATORS" => "13th Floor Elevators"
      }
    end

    it "convert text into capitalized words" do
      examples.each_pair do |before, after|
        expect(normalizer.normalize_text(before)).to eq after
      end
    end
  end

  describe "#normalize_artist" do
    let(:examples) do
      {
        "<NAME>"                => "<NAME>",
        "Beck,Jeff"                 => "<NAME>",
        "Astatke, Mulatu & Band"    => "Mulatu Astatke & Band",
        "10,000 Maniacs"            => "10,000 Maniacs",
        "Allman Brothers Band, The" => "The Allman Brothers Band"
      }
    end

    it "converts weird artist name into correct form" do
      examples.each_pair do |before, after|
        expect(normalizer.normalize_artist(before)).to eq after
      end<commit_msg>Add extra test case<commit_after>require "spec_helper"

class NormalizerTest
  include Reckless::Normalizer
end

describe Reckless::Normalizer do
  let(:normalizer) { NormalizerTest.new }

  describe "#normalize_text" do
    let(:examples) do
      {
        "ALL CAPS TEXT"        => "All Caps Text",
        "all low text"         => "All Low Text",
        "13TH FLOOR ELEVATORS" => "13th Floor Elevators"
      }
    end

    it "convert text into capitalized words" do
      examples.each_pair do |before, after|
        expect(normalizer.normalize_text(before)).to eq after
      end
    end
  end

  describe "#normalize_artist" do
    let(:examples) do
      {
        "<NAME>"                => "<NAME>",
        "Beck,Jeff"                 => "<NAME>",
        "Astatke, Mulatu & Band"    => "Mulatu Astatke & Band",
        "10,000 Maniacs"            => "10,000 Maniacs",
        "Allman Brothers Band, The" => "The Allman Brothers Band",
        "Grateful Dead, The"        => "The Grateful Dead"
      }
    end

    it "converts weird artist name into correct form" do
      examples.each_pair do |before, after|
        expect(normalizer.normalize_artist(before)).to eq after
      end
{'fraction_non_alphanumeric': 0.10787671232876712, 'fraction_numerical': 0.011986301369863013, 'mean_word_length': 2.2013698630136984, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '906473', 'n_tokens_mistral': 748, 'n_tokens_neox': 710, 'n_words': 204}