<commit_before># coding: utf-8
require_relative '../wisdom'

RSpec.describe Wisdom do
  before(:each) do
    @wisdom = Wisdom.new("mikolov.pdf")
    @wisdom.process
  end
  
  it "extracts the title from a pdf paper" do
    
    expect(@wisdom.title).to eq("Bag of Tricks for Efﬁcient Text Classiﬁcation")
  end

  it "extracts the authors from a pdf paper" do
    expect(@wisdom.authors).to eq("<NAME>, <NAME>, <NAME>, <NAME>")
  end

  it "extracts the abstract from a pdf paper" do
    expect(@wisdom.abstract[0..28]).to eq("In this work, we explore ways")

  end
end
<commit_msg>add some test about info<commit_after># coding: utf-8
require_relative '../wisdom'

RSpec.describe Wisdom do
  before(:each) do
    @wisdom = Wisdom.new("mikolov.pdf")
    @wisdom.process
  end
  
  it "extracts the title from a pdf paper" do    
    expect(@wisdom.title).to eq("Bag of Tricks for Efﬁcient Text Classiﬁcation")
  end

  it "extracts the authors from a pdf paper" do
    expect(@wisdom.authors).to eq("<NAME>, <NAME>, <NAME>, <NAME>")
  end

  it "extracts the abstract from a pdf paper" do
    expect(@wisdom.abstract[0..28]).to eq("In this work, we explore ways")
  end

  it "extracts the title even if it's not the first sentence"

  it "gets the first 3000 characters if cannot find the abstract"

  it "adds a log to each pdf to remember how the extraction went"
end

{'fraction_non_alphanumeric': 0.1129737609329446, 'fraction_numerical': 0.008746355685131196, 'mean_word_length': 3.472312703583062, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2534992', 'n_tokens_mistral': 486, 'n_tokens_neox': 467, 'n_words': 170}