<commit_before># encoding: utf-8

describe Gnrd::Text do
  include_context "shared_context"
  subject { Gnrd::Text }

  describe ".new" do
    it "creates an instance from dossier" do
      ts = subject.new(utf_dossier)
      expect(ts).to be_kind_of(subject)
      expect(ts.dossier).to be_kind_of(Gnrd::Dossier)
    end

    it "fails creating an instance with string" do
      expect { subject.new("hello") }.to raise_error TypeError
    end
  end

  describe "#text_orig" do
    it "returns the text from utf sources verbatim" do
      expect(subject.new(utf_txt_dossier).text_orig).to eq utf_txt
      expect(subject.new(utf_dossier).text_orig).to eq utf_txt
    end

    it "returns the text from ascii sources verbatim" do
      expect(subject.new(ascii_dossier).text_orig).to eq ascii_txt
    end

    it "returns text from image" do
      expect(subject.new(img_dossier).text_orig)
        .to match(/Baccha el.ngata 7-10/)
    end

    it "returns empty text from no text image" do
      expect(subject.new(img_no_names_dossier).text_orig.strip).to eq ""
    end

    it "returns text from pdf" do
      expect(subject.new(pdf_dossier).text_orig).to be_kind_of(String)
    end

    it "returns text from image pdf" do
      expect(subject.new(pdf_img_dossier).text_orig)
        .to match(/Baccha el.ngata 7-10/)
    end

    it "returns nil from unknown file" do
      expect(subject.new(binary_dossier).text_orig).to eq ""
    end
  end

  describe "#text_norm" do
    it "returns normalized text" do
      pdf = subject.new(pdf_dossier)
      expect(pdf.text_orig.encoding.to_s).to eq "US-ASCII"
      expect(pdf.text_norm.encoding.to_s).to eq "UTF-8"
    end
  end
end<commit_msg>change one test for travis<commit_after># encoding: utf-8

describe Gnrd::Text do
  include_context "shared_context"
  subject { Gnrd::Text }

  describe ".new" do
    it "creates an instance from dossier" do
      ts = subject.new(utf_dossier)
      expect(ts).to be_kind_of(subject)
      expect(ts.dossier).to be_kind_of(Gnrd::Dossier)
    end

    it "fails creating an instance with string" do
      expect { subject.new("hello") }.to raise_error TypeError
    end
  end

  describe "#text_orig" do
    it "returns the text from utf sources verbatim" do
      expect(subject.new(utf_txt_dossier).text_orig).to eq utf_txt
      expect(subject.new(utf_dossier).text_orig).to eq utf_txt
    end

    it "returns the text from ascii sources verbatim" do
      expect(subject.new(ascii_dossier).text_orig).to eq ascii_txt
    end

    it "returns text from image" do
      expect(subject.new(img_dossier).text_orig)
        .to match(/Baccha el.ngata 7-10/)
    end

    it "returns empty text from no text image" do
      expect(subject.new(img_no_names_dossier).text_orig.strip).to eq ""
    end

    it "returns text from pdf" do
      expect(subject.new(pdf_dossier).text_orig).to be_kind_of(String)
    end

    it "returns text from image pdf" do
      expect(subject.new(pdf_img_dossier).text_orig)
        .to match(/Baccha el.ngata 7-10/)
    end

    it "returns nil from unknown file" do
      expect(subject.new(binary_dossier).text_orig).to eq ""
    end
  end

  describe "#text_norm" do
    it "returns normalized text" do
      pdf = subject.new(pdf_dossier)
      expect(pdf.text_norm.encoding.to_s).to eq "UTF-8"
      expect(pdf.text_orig).to_not eq  pdf.text_norm
    end
  end
end
{'fraction_non_alphanumeric': 0.09602356406480118, 'fraction_numerical': 0.00471281296023564, 'mean_word_length': 3.0141843971631204, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7422491', 'n_tokens_mistral': 1204, 'n_tokens_neox': 1133, 'n_words': 303}