<commit_before>class ContentHelperTestHelper
  include Licensee::ContentHelper
  attr_accessor :content

  def initialize(content = nil)
    @content = content
  end
end

RSpec.describe Licensee::ContentHelper do
  let(:content) do
    <<-EOS.freeze.gsub(/^\s*/, '')
  # The MIT License
  =================

  Copyright 2016 <NAME>

  The made
  up  license.
  -----------
EOS
  end
  subject { ContentHelperTestHelper.new(content) }
  let(:mit) { Licensee::License.find('mit') }

  it 'creates the wordset' do
    expect(subject.wordset).to eql(Set.new(%w[the made up license]))
  end

  it 'knows the length' do
    expect(subject.length).to eql(20)
  end

  it 'knows the max delta' do
    expect(subject.max_delta).to eql(1)
  end

  it 'knows the length delta' do
    expect(subject.length_delta(mit)).to eql(1000)
    expect(subject.length_delta(subject)).to eql(0)
  end

  it 'knows the similarity' do
    expect(subject.similarity(mit)).to be_within(1).of(2)
    expect(subject.similarity(subject)).to eql(100.0)
  end

  it 'calculates the hash' do
    expect(subject.hash).to eql('3c59634b9fae4396a76a978f3f6aa718ed790a9a')
  end

  context 'normalizing' do
    let(:normalized_content) { subject.content_normalized }

    it 'strips copyright' do
      expect(normalized_content).to_not match 'Copyright'
      expect(normalized_content).to_not match '<NAME>'
    end<commit_msg>add test to ensure that deltas cannot exceed 150<commit_after>class ContentHelperTestHelper
  include Licensee::ContentHelper
  attr_accessor :content

  def initialize(content = nil)
    @content = content
  end
end

RSpec.describe Licensee::ContentHelper do
  let(:content) do
    <<-EOS.freeze.gsub(/^\s*/, '')
  # The MIT License
  =================

  Copyright 2016 <NAME>

  The made
  up  license.
  -----------
EOS
  end
  subject { ContentHelperTestHelper.new(content) }
  let(:mit) { Licensee::License.find('mit') }

  it 'creates the wordset' do
    expect(subject.wordset).to eql(Set.new(%w[the made up license]))
  end

  it 'knows the length' do
    expect(subject.length).to eql(20)
  end

  it 'knows the max delta' do
    expect(subject.max_delta).to eql(1)
  end

  context "a very long license" do
    let(:content) { "license" * 1000 }

    it 'does not return a max delta larger than 150' do
      expect(subject.max_delta).to eql(150)
    end
  end

  it 'knows the length delta' do
    expect(subject.length_delta(mit)).to eql(1000)
    expect(subject.length_delta(subject)).to eql(0)
  end

  it 'knows the similarity' do
    expect(subject.similarity(mit)).to be_within(1).of(2)
    expect(subject.similarity(subject)).to eql(100.0)
  end

  it 'calculates the hash' do
    expect(subject.hash).to eql('3c59634b9fae4396a76a978f3f6aa718ed790a9a')
  end

  context 'normalizing' do
    let(:normalized_content) { subject.content_normalized }

    it 'strips copyright' do
      expect(normalized_content).to_not match 'Copyright'
      expect(normalized_content).to_not match '<NAME>'
    end
{'fraction_non_alphanumeric': 0.1277020285999335, 'fraction_numerical': 0.03292317924842035, 'mean_word_length': 3.6419753086419755, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5814512', 'n_tokens_mistral': 1107, 'n_tokens_neox': 1014, 'n_words': 248}