<commit_before>require "nokogiri"
require "open-uri"

class String
  def clean
    # removes leading and trailing whitespace, commas
    self.gsub!(/(^[\s,]+)|([\s,]+$)/, '')
    return self
  end
end

class RubyScholar
  class Paper < Struct.new(:title, :url, :authors, :journalName, :journalDetails, :year, :citationCount, :citingPapers, :doi)
  end  
  
  class Parser
    attr_accessor :parsedPapers, :crossRefEmail
    
    def initialize(url, crossRefEmail = "")
      @parsedPapers  = []
      @crossRefEmail = crossRefEmail # if nil doesn't return any DOI
      parse(url)
    end

    def parse(url)
      papers = Nokogiri::HTML(open(url)).css(".cit-table .item")
      STDOUT << "Found #{papers.length} papers.\n"
      papers.each do |paper|
        paperDetails   = paper.css("#col-title")
        title          = paperDetails[0].children[0].content.clean
        googleUrl      = paperDetails[0].children[0].attribute('href')
        authors        = paperDetails[0].children[2].content.clean
        authors.gsub!("...", "et al")

        journal        = paperDetails[0].children[4].content
        journalName    = journal.split(/,|\d/).first.clean
        journalDetails = journal.gsub(journalName, '').clean

        year           = paper.css("#col-year").text # is the last thing we get

        #citations
        citeInfo      = paper.css(".cit-dark-link")
        citationCount = citeInfo.text
        citationUrl   = citationCount.empty?  ? nil : citeInfo.attribute('href').to_s 

        # get DOI: needs last name of first author, no funny chars
        lastNameFirstAuthor = ((authors.split(',').first ).split(' ').last ).gsub(/[^A-Za-z\-]/, '')
        doi                 = getDoi( lastNameFirstAuthor, title, @crossRefEmail)

        @parsedPapers.push(Paper.new( title, googleUrl, authors, journalName, journalDetails, year, citationCount, citationUrl, doi))
      end
      STDOUT << "Scraped #{parsedPapers.length} from Google Scholar.\n"
    end
<commit_msg>changed info from stdout to stderr<commit_after>require "nokogiri"
require "open-uri"

class String
  def clean
    # removes leading and trailing whitespace, commas
    self.gsub!(/(^[\s,]+)|([\s,]+$)/, '')
    return self
  end
end

class RubyScholar
  class Paper < Struct.new(:title, :url, :authors, :journalName, :journalDetails, :year, :citationCount, :citingPapers, :doi)
  end  
  
  class Parser
    attr_accessor :parsedPapers, :crossRefEmail
    
    def initialize(url, crossRefEmail = "")
      @parsedPapers  = []
      @crossRefEmail = crossRefEmail # if nil doesn't return any DOI
      parse(url)
    end

    def parse(url)
      papers = Nokogiri::HTML(open(url)).css(".cit-table .item")
      STDERR << "Found #{papers.length} papers.\n"
      papers.each do |paper|
        paperDetails   = paper.css("#col-title")
        title          = paperDetails[0].children[0].content.clean
        googleUrl      = paperDetails[0].children[0].attribute('href')
        authors        = paperDetails[0].children[2].content.clean
        authors.gsub!("...", "et al")

        journal        = paperDetails[0].children[4].content
        journalName    = journal.split(/,|\d/).first.clean
        journalDetails = journal.gsub(journalName, '').clean

        year           = paper.css("#col-year").text # is the last thing we get

        #citations
        citeInfo      = paper.css(".cit-dark-link")
        citationCount = citeInfo.text
        citationUrl   = citationCount.empty?  ? nil : citeInfo.attribute('href').to_s 

        # get DOI: needs last name of first author, no funny chars
        lastNameFirstAuthor = ((authors.split(',').first ).split(' ').last ).gsub(/[^A-Za-z\-]/, '')
        doi                 = getDoi( lastNameFirstAuthor, title, @crossRefEmail)

        @parsedPapers.push(Paper.new( title, googleUrl, authors, journalName, journalDetails, year, citationCount, citationUrl, doi))
      end
      STDERR << "Scraped #{parsedPapers.length} from Google Scholar.\n"
    end

{'fraction_non_alphanumeric': 0.13918682963332502, 'fraction_numerical': 0.003991020204539786, 'mean_word_length': 2.6454545454545455, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6299520', 'n_tokens_mistral': 1262, 'n_tokens_neox': 1201, 'n_words': 281}