<commit_before>require 'spec_helper'

RSpec.describe ConfidentialInfoRedactorLite::Redactor do
  let(:corpus) { ['i', 'in', 'you', 'top', 'so', 'are', 'december', 'please', 'prizes'] }
  let(:en_dow) { %w(monday tuesday wednesday thursday friday saturday sunday) }
  let(:en_dow_abbr) { %w(mon tu tue tues wed th thu thur thurs fri sat sun) }
  let(:en_months) { %w(january february march april may june july august september october november december) }
  let(:en_month_abbr) { %w(jan feb mar apr jun jul aug sep sept oct nov dec) }
  let(:de_dow) { %w(montag dienstag mittwoch donnerstag freitag samstag sonntag sonnabend) }
  let(:de_dow_abbr) { %w(mo di mi do fr sa so) }
  let(:de_months) { %w(januar februar märz april mai juni juli august september oktober november dezember) }
  let(:de_month_abbr) { %w(jan jän feb märz apr mai juni juli aug sep sept okt nov dez) }

  describe '#dates' do
    it 'handles nil as a text argument' do
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).dates(nil)).to eq('')
    end

    it 'redacts dates from a text #001' do
      text = 'Coca-Cola announced a merger with Pepsi that will happen on December 15th, 2020 for $200,000,000,000.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).dates(text)).to eq('Coca-Cola announced a merger with Pepsi that will happen on <redacted date> for $200,000,000,000.')
    end

    it 'redacts dates from a text #002' do
      text = 'Coca-Cola announced a merger with Pepsi that will happen on December 15th, 2020.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).dates(text)).to eq('Coca-Cola announced a merger with Pepsi that will happen on <redacted date>.')
    end

    it 'redacts dates from a text #003' do
      text = 'December 5, 2010 - Coca-Cola announced a merger with Pepsi.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).dates(text)).to eq('<redacted date> - Coca-Cola announced a merger with Pepsi.')
    end

    it 'redacts dates from a text #004' do
      text = 'The scavenger hunt ends on Dec. 31st, 2011.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).dates(text)).to eq('The scavenger hunt ends on <redacted date>.')
    end

    it 'redacts dates from a text #004' do
      text = '18/01/2013'
      expect(described_class.new(language: 'de', dow: de_dow, dow_abbr: de_dow_abbr, months: de_months, months_abbr: de_month_abbr).dates(text)).to eq('<redacted date>')
    end

    it 'handles nil date objects' do
      text = 'The scavenger hunt ends on Dec. 31st, 2011.'
      expect(described_class.new(language: 'en', dow: nil, dow_abbr: nil, months: nil, months_abbr: nil).dates(text)).to eq('The scavenger hunt ends on Dec. 31st, 2011.')
    end

    it 'handles empty string date objects' do
      text = 'The scavenger hunt ends on Dec. 31st, 2011.'
      expect(described_class.new(language: 'en', dow: '', dow_abbr: '', months: '', months_abbr: '').dates(text)).to eq('The scavenger hunt ends on Dec. 31st, 2011.')
    end

    it 'handles empty array date objects' do
      text = 'The scavenger hunt ends on Dec. 31st, 2011.'
      expect(described_class.new(language: 'en', dow: [], dow_abbr: [], months: [], months_abbr: []).dates(text)).to eq('The scavenger hunt ends on Dec. 31st, 2011.')
    end
  end

  describe '#dates_html' do
    it 'handles nil as a text argument' do
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, date_text: "*****").dates_html(nil)).to eq([])
    end

    it 'surrounds the redacted dates in spans and return the redacted dates from a text #001' do
      text = 'On May 1st, 2000 Coca-Cola announced a merger with Pepsi that will happen on December 15th, 2020.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, date_text: "*****").dates_html(text)).to eq(["On <span class='confidentialDate'>*****</span> Coca-Cola announced a merger with Pepsi that will happen on <span class='confidentialDate'>*****</span>.", ['May 1st, 2000', 'December 15th, 2020']])
    end

    it 'surrounds the redacted dates in spans and return the redacted dates from a text #002' do
      text = '２０１１年１２月３１日です。'
      expect(described_class.new(language: 'ja', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, date_text: "*****").dates_html(text)).to eq(["<span class='confidentialDate'>*****</span> です。", ["２０１１年１２月３１日"]])
    end

    it 'surrounds the redacted dates in spans and return the redacted dates from a text #003' do
      text = '18/01/2013'
      expect(described_class.new(language: 'de', dow: de_dow, dow_abbr: de_dow_abbr, months: de_months, months_abbr: de_month_abbr, date_text: "*****").dates_html(text)[1]).to eq(['18/01/2013'])
    end

    it 'surrounds the redacted dates in spans and return the redacted dates from a text #004' do
      text = '16/01/2013'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, date_text: "*****").dates_html(text)[1]).to eq(["16/01/2013"])
    end

    it 'surrounds the redacted dates in spans and return the redacted dates from a text #005' do
      text = '88966-5.0-ENG'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, date_text: "*****").dates_html(text)[1]).to eq([])
    end
  end

  describe '#numbers' do
    it 'handles nil as a text argument' do
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers(nil)).to eq('')
    end

    it 'redacts numbers from a text #001' do
      text = 'Coca-Cola announced a merger with Pepsi that will happen on <redacted date> for $200,000,000,000.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers(text)).to eq('Coca-Cola announced a merger with Pepsi that will happen on <redacted date> for <redacted number>.')
    end

    it 'redacts numbers from a text #002' do
      text = '200 years ago.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers(text)).to eq('<redacted number> years ago.')
    end

    it 'redacts numbers from a text #003' do
      text = 'It was his 1st time, not yet his 10th, not even his 2nd. The wood was 3/4" thick.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers(text)).to eq('It was his <redacted number> time, not yet his <redacted number>, not even his <redacted number>. The wood was <redacted number> thick.')
    end

    it 'redacts numbers from a text #004' do
      text = 'Checking file of %2'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers(text)).to eq('Checking file of <redacted number>')
    end

    it 'redacts numbers from a text #005' do
      text = 'zawiera pliki skompresowane (%2).'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers(text)).to eq('zawiera pliki skompresowane (<redacted number>).')
    end

    it 'redacts numbers from a text #006' do
      text = '２１３４か２４か０'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers(text)).to eq("<redacted number> か <redacted number> か <redacted number>")
    end

    it 'redacts numbers from a text #007' do
      text = '100'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers(text)).to eq('<redacted number>')
    end

    it 'redacts numbers from a text #008' do
      text = "Test '0' hello."
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers_html(text)[1]).to eq(["0"])
    end

    it 'redacts numbers from a text #009' do
      text = "Test ‘0’ hello."
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers_html(text)[1]).to eq(["0"])
    end

    it 'redacts numbers from a text #010' do
      text = "1.4,5.3,7.3"
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers_html(text)[1]).to eq(["1.4,5.3,7.3"])
    end

    it 'redacts numbers from a text #011' do
      text = "machine width – 3900mm (overall), 3500mm (cross members)"
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers_html(text)[1]).to eq(["3900mm", "3500mm"])
    end

    it 'redacts numbers from a text #012' do
      text = "Maximale Maschinenbreite – 3.900 mm (Gesamtlänge), 3.500 mm (Traversen)"
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers_html(text)[1]).to eq(["3.900", "3.500"])
    end

    it 'redacts numbers from a text #013' do
      text = "Page 4"
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers_html(text)[1]).to eq(["4"])
    end
  end

  describe '#numbers_html' do
    it 'surrounds the redacted numbers in spans and return the redacted numbers from a text #001' do
      text = 'It was his 1st) time, not yet his 10th, not even his 2nd. The wood was 3/4" thick. It cost $200,000.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, number_text: "*****").numbers_html(text)).to eq(["It was his <span class='confidentialNumber'>*****</span>) time, not yet his <span class='confidentialNumber'>*****</span>, not even his <span class='confidentialNumber'>*****</span>. The wood was <span class='confidentialNumber'>*****</span> thick. It cost <span class='confidentialNumber'>*****</span>.", ["1st", "10th,", "2nd", "3/4\"", "$200,000"]])
    end

    it 'surrounds the redacted numbers in spans and return the redacted numbers from a text #002' do
      text = 'プロのミニチュアゴルファー２人のサイン。２人の出身国は別であること。（４５ポイント；それぞれが別の大陸出身だった場合、５ボーナスポイント。）'
      expect(described_class.new(language: 'ja', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, number_text: "*****").numbers_html(text)).to eq(["プロのミニチュアゴルファー <span class='confidentialNumber'>*****</span> 人のサイン。 <span class='confidentialNumber'>*****</span> 人の出身国は別であること。（ <span class='confidentialNumber'>*****</span> ポイント；それぞれが別の大陸出身だった場合、 <span class='confidentialNumber'>*****</span> ボーナスポイント。）", ["２", "２", "４５", "５"]])
    end

    it 'surrounds the redacted numbers in spans and return the redacted numbers from a text #003' do
      text = "machine width – 3900mm (overall), 3500mm (cross members)"
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, number_text: "*****").numbers_html(text)).to eq(["machine width – <span class='confidentialNumber'>*****</span> (overall), <span class='confidentialNumber'>*****</span> (cross members)", ["3900mm", "3500mm"]])
    end

    it 'surrounds the redacted numbers in spans and return the redacted numbers from a text #004' do
      text = "Page 1500"
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, number_text: "*****").numbers_html(text)[1]).to eq(["1500"])
    end
  end

  describe '#emails' do
    it 'redacts email addresses from a text #001' do
      text = 'His email is <EMAIL> or you can try <EMAIL>.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).emails(text)).to eq('His email is <redacted email> or you can try <redacted email>.')
    end

    it 'redacts email addresses from a text #002' do
      text = 'His email is (<EMAIL>) or you can try (<EMAIL>).'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).emails(text)).to eq('His email is (<redacted email>) or you can try (<redacted email>).')
    end
  end

  describe '#emails_html' do
    it 'surrounds the redacted emails in spans and return the redacted emails from a text #001' do
      text = 'His email is (<EMAIL>) or you can try (<EMAIL>).'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, token_text: "*****").emails_html(text)).to eq(["His email is (<span class='confidentialEmail'><redacted email></span>) or you can try (<span class='confidentialEmail'><redacted email></span>).", ["<EMAIL>", "<EMAIL>"]])
    end
  end

  describe '#hyperlinks' do
    it 'redacts hyperlinks from a text #001' do
      text = 'Visit https://www.tm-town.com for more info.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).hyperlinks(text)).to eq('Visit <redacted hyperlink> for more info.')
    end

    it 'redacts hyperlinks from a text #002' do
      text = 'Visit www.tm-town.com for more info.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).hyperlinks(text)).to eq('Visit <redacted hyperlink> for more info.')<commit_msg>Add number specs<commit_after>require 'spec_helper'

RSpec.describe ConfidentialInfoRedactorLite::Redactor do
  let(:corpus) { ['i', 'in', 'you', 'top', 'so', 'are', 'december', 'please', 'prizes'] }
  let(:en_dow) { %w(monday tuesday wednesday thursday friday saturday sunday) }
  let(:en_dow_abbr) { %w(mon tu tue tues wed th thu thur thurs fri sat sun) }
  let(:en_months) { %w(january february march april may june july august september october november december) }
  let(:en_month_abbr) { %w(jan feb mar apr jun jul aug sep sept oct nov dec) }
  let(:de_dow) { %w(montag dienstag mittwoch donnerstag freitag samstag sonntag sonnabend) }
  let(:de_dow_abbr) { %w(mo di mi do fr sa so) }
  let(:de_months) { %w(januar februar märz april mai juni juli august september oktober november dezember) }
  let(:de_month_abbr) { %w(jan jän feb märz apr mai juni juli aug sep sept okt nov dez) }

  describe '#dates' do
    it 'handles nil as a text argument' do
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).dates(nil)).to eq('')
    end

    it 'redacts dates from a text #001' do
      text = 'Coca-Cola announced a merger with Pepsi that will happen on December 15th, 2020 for $200,000,000,000.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).dates(text)).to eq('Coca-Cola announced a merger with Pepsi that will happen on <redacted date> for $200,000,000,000.')
    end

    it 'redacts dates from a text #002' do
      text = 'Coca-Cola announced a merger with Pepsi that will happen on December 15th, 2020.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).dates(text)).to eq('Coca-Cola announced a merger with Pepsi that will happen on <redacted date>.')
    end

    it 'redacts dates from a text #003' do
      text = 'December 5, 2010 - Coca-Cola announced a merger with Pepsi.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).dates(text)).to eq('<redacted date> - Coca-Cola announced a merger with Pepsi.')
    end

    it 'redacts dates from a text #004' do
      text = 'The scavenger hunt ends on Dec. 31st, 2011.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).dates(text)).to eq('The scavenger hunt ends on <redacted date>.')
    end

    it 'redacts dates from a text #004' do
      text = '18/01/2013'
      expect(described_class.new(language: 'de', dow: de_dow, dow_abbr: de_dow_abbr, months: de_months, months_abbr: de_month_abbr).dates(text)).to eq('<redacted date>')
    end

    it 'handles nil date objects' do
      text = 'The scavenger hunt ends on Dec. 31st, 2011.'
      expect(described_class.new(language: 'en', dow: nil, dow_abbr: nil, months: nil, months_abbr: nil).dates(text)).to eq('The scavenger hunt ends on Dec. 31st, 2011.')
    end

    it 'handles empty string date objects' do
      text = 'The scavenger hunt ends on Dec. 31st, 2011.'
      expect(described_class.new(language: 'en', dow: '', dow_abbr: '', months: '', months_abbr: '').dates(text)).to eq('The scavenger hunt ends on Dec. 31st, 2011.')
    end

    it 'handles empty array date objects' do
      text = 'The scavenger hunt ends on Dec. 31st, 2011.'
      expect(described_class.new(language: 'en', dow: [], dow_abbr: [], months: [], months_abbr: []).dates(text)).to eq('The scavenger hunt ends on Dec. 31st, 2011.')
    end
  end

  describe '#dates_html' do
    it 'handles nil as a text argument' do
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, date_text: "*****").dates_html(nil)).to eq([])
    end

    it 'surrounds the redacted dates in spans and return the redacted dates from a text #001' do
      text = 'On May 1st, 2000 Coca-Cola announced a merger with Pepsi that will happen on December 15th, 2020.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, date_text: "*****").dates_html(text)).to eq(["On <span class='confidentialDate'>*****</span> Coca-Cola announced a merger with Pepsi that will happen on <span class='confidentialDate'>*****</span>.", ['May 1st, 2000', 'December 15th, 2020']])
    end

    it 'surrounds the redacted dates in spans and return the redacted dates from a text #002' do
      text = '２０１１年１２月３１日です。'
      expect(described_class.new(language: 'ja', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, date_text: "*****").dates_html(text)).to eq(["<span class='confidentialDate'>*****</span> です。", ["２０１１年１２月３１日"]])
    end

    it 'surrounds the redacted dates in spans and return the redacted dates from a text #003' do
      text = '18/01/2013'
      expect(described_class.new(language: 'de', dow: de_dow, dow_abbr: de_dow_abbr, months: de_months, months_abbr: de_month_abbr, date_text: "*****").dates_html(text)[1]).to eq(['18/01/2013'])
    end

    it 'surrounds the redacted dates in spans and return the redacted dates from a text #004' do
      text = '16/01/2013'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, date_text: "*****").dates_html(text)[1]).to eq(["16/01/2013"])
    end

    it 'surrounds the redacted dates in spans and return the redacted dates from a text #005' do
      text = '88966-5.0-ENG'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, date_text: "*****").dates_html(text)[1]).to eq([])
    end
  end

  describe '#numbers' do
    it 'handles nil as a text argument' do
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers(nil)).to eq('')
    end

    it 'redacts numbers from a text #001' do
      text = 'Coca-Cola announced a merger with Pepsi that will happen on <redacted date> for $200,000,000,000.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers(text)).to eq('Coca-Cola announced a merger with Pepsi that will happen on <redacted date> for <redacted number>.')
    end

    it 'redacts numbers from a text #002' do
      text = '200 years ago.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers(text)).to eq('<redacted number> years ago.')
    end

    it 'redacts numbers from a text #003' do
      text = 'It was his 1st time, not yet his 10th, not even his 2nd. The wood was 3/4" thick.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers(text)).to eq('It was his <redacted number> time, not yet his <redacted number>, not even his <redacted number>. The wood was <redacted number> thick.')
    end

    it 'redacts numbers from a text #004' do
      text = 'Checking file of %2'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers(text)).to eq('Checking file of <redacted number>')
    end

    it 'redacts numbers from a text #005' do
      text = 'zawiera pliki skompresowane (%2).'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers(text)).to eq('zawiera pliki skompresowane (<redacted number>).')
    end

    it 'redacts numbers from a text #006' do
      text = '２１３４か２４か０'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers(text)).to eq("<redacted number> か <redacted number> か <redacted number>")
    end

    it 'redacts numbers from a text #007' do
      text = '100'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers(text)).to eq('<redacted number>')
    end

    it 'redacts numbers from a text #008' do
      text = "Test '0' hello."
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers_html(text)[1]).to eq(["0"])
    end

    it 'redacts numbers from a text #009' do
      text = "Test ‘0’ hello."
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers_html(text)[1]).to eq(["0"])
    end

    it 'redacts numbers from a text #010' do
      text = "1.4,5.3,7.3"
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers_html(text)[1]).to eq(["1.4,5.3,7.3"])
    end

    it 'redacts numbers from a text #011' do
      text = "machine width – 3900mm (overall), 3500mm (cross members)"
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers_html(text)[1]).to eq(["3900mm", "3500mm"])
    end

    it 'redacts numbers from a text #012' do
      text = "Maximale Maschinenbreite – 3.900 mm (Gesamtlänge), 3.500 mm (Traversen)"
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers_html(text)[1]).to eq(["3.900", "3.500"])
    end

    it 'redacts numbers from a text #013' do
      text = "Page 4"
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).numbers_html(text)[1]).to eq(["4"])
    end
  end

  describe '#numbers_html' do
    it 'surrounds the redacted numbers in spans and return the redacted numbers from a text #001' do
      text = 'It was his 1st) time, not yet his 10th, not even his 2nd. The wood was 3/4" thick. It cost $200,000.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, number_text: "*****").numbers_html(text)).to eq(["It was his <span class='confidentialNumber'>*****</span>) time, not yet his <span class='confidentialNumber'>*****</span>, not even his <span class='confidentialNumber'>*****</span>. The wood was <span class='confidentialNumber'>*****</span> thick. It cost <span class='confidentialNumber'>*****</span>.", ["1st", "10th,", "2nd", "3/4\"", "$200,000"]])
    end

    it 'surrounds the redacted numbers in spans and return the redacted numbers from a text #002' do
      text = 'プロのミニチュアゴルファー２人のサイン。２人の出身国は別であること。（４５ポイント；それぞれが別の大陸出身だった場合、５ボーナスポイント。）'
      expect(described_class.new(language: 'ja', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, number_text: "*****").numbers_html(text)).to eq(["プロのミニチュアゴルファー <span class='confidentialNumber'>*****</span> 人のサイン。 <span class='confidentialNumber'>*****</span> 人の出身国は別であること。（ <span class='confidentialNumber'>*****</span> ポイント；それぞれが別の大陸出身だった場合、 <span class='confidentialNumber'>*****</span> ボーナスポイント。）", ["２", "２", "４５", "５"]])
    end

    it 'surrounds the redacted numbers in spans and return the redacted numbers from a text #003' do
      text = "machine width – 3900mm (overall), 3500mm (cross members)"
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, number_text: "*****").numbers_html(text)).to eq(["machine width – <span class='confidentialNumber'>*****</span> (overall), <span class='confidentialNumber'>*****</span> (cross members)", ["3900mm", "3500mm"]])
    end

    it 'surrounds the redacted numbers in spans and return the redacted numbers from a text #004' do
      text = "Page 1500"
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, number_text: "*****").numbers_html(text)[1]).to eq(["1500"])
    end

    it 'surrounds the redacted numbers in spans and return the redacted numbers from a text #005' do
      text = "Maximale Maschinenbreite – 3.900 mm (Gesamtlänge), 3.500 mm (Traversen)"
      expect(described_class.new(language: 'de', dow: de_dow, dow_abbr: de_dow_abbr, months: de_months, months_abbr: de_month_abbr, number_text: "*****").numbers_html(text)[1]).to eq(["3.900", "3.500"])
    end

    it 'surrounds the redacted numbers in spans and return the redacted numbers from a text #006' do
      text = "Maschinenlänge – 1.225 mm"
      expect(described_class.new(language: 'de', dow: de_dow, dow_abbr: de_dow_abbr, months: de_months, months_abbr: de_month_abbr, number_text: "*****").numbers_html(text)[1]).to eq(["1.225"])
    end

    it 'surrounds the redacted numbers in spans and return the redacted numbers from a text #007' do
      text = "600 V 60 Hz 3 Ph zzgl. Erdung"
      expect(described_class.new(language: 'de', dow: de_dow, dow_abbr: de_dow_abbr, months: de_months, months_abbr: de_month_abbr, number_text: "*****").numbers_html(text)[1]).to eq(["600", "60", "3"])
    end

    it 'surrounds the redacted numbers in spans and return the redacted numbers from a text #008' do
      text = "380/415V 50/60Hz 3Ph plus neutral plus earth"
      expect(described_class.new(language: 'de', dow: de_dow, dow_abbr: de_dow_abbr, months: de_months, months_abbr: de_month_abbr, number_text: "*****").numbers_html(text)[1]).to eq(["380/415V", "50/60Hz", "3Ph"])
    end

    it 'surrounds the redacted numbers in spans and return the redacted numbers from a text #009' do
      text = "380/415 V 50/60 Hz 3Ph zzgl. neutral zzgl. Erdung"
      expect(described_class.new(language: 'de', dow: de_dow, dow_abbr: de_dow_abbr, months: de_months, months_abbr: de_month_abbr, number_text: "*****").numbers_html(text)[1]).to eq(["380/415V", "50/60Hz", "3Ph"])
    end
  end

  describe '#emails' do
    it 'redacts email addresses from a text #001' do
      text = 'His email is <EMAIL> or you can try <EMAIL>.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).emails(text)).to eq('His email is <redacted email> or you can try <redacted email>.')
    end

    it 'redacts email addresses from a text #002' do
      text = 'His email is (<EMAIL>) or you can try (<EMAIL>).'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).emails(text)).to eq('His email is (<redacted email>) or you can try (<redacted email>).')
    end
  end

  describe '#emails_html' do
    it 'surrounds the redacted emails in spans and return the redacted emails from a text #001' do
      text = 'His email is (<EMAIL>) or you can try (<EMAIL>).'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr, token_text: "*****").emails_html(text)).to eq(["His email is (<span class='confidentialEmail'><redacted email></span>) or you can try (<span class='confidentialEmail'><redacted email></span>).", ["<EMAIL>", "<EMAIL>"]])
    end
  end

  describe '#hyperlinks' do
    it 'redacts hyperlinks from a text #001' do
      text = 'Visit https://www.tm-town.com for more info.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).hyperlinks(text)).to eq('Visit <redacted hyperlink> for more info.')
    end

    it 'redacts hyperlinks from a text #002' do
      text = 'Visit www.tm-town.com for more info.'
      expect(described_class.new(language: 'en', dow: en_dow, dow_abbr: en_dow_abbr, months: en_months, months_abbr: en_month_abbr).hyperlinks(text)).to eq('Visit <redacted hyperlink> for more info.')
{'fraction_non_alphanumeric': 0.13786516483878405, 'fraction_numerical': 0.030629384448177057, 'mean_word_length': 4.448411986362821, 'pattern_counts': {'":': 0, '<': 137, '<?xml version=': 0, '>': 137, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4735641', 'n_tokens_mistral': 12097, 'n_tokens_neox': 10897, 'n_words': 3209}