<commit_before>require "rubygems"
require "net/http"
require "json"

class MendeleyDetails
  include Sidekiq::Worker
  sidekiq_options :queue => :mendeley_details, :retry => 5, :unique => true

  def perform(mendeley_paper_id)
     # Logging stuff
     Rails.logger.level = 0
     Rails.logger = Logger.new("#{Rails.root}/log/mendeleydetails_#{Rails.env}.log")
     @mendeley_paper = MendeleyPaper.find_by_id(mendeley_paper_id.to_i)

     key_handle = File.open(::Rails.root.to_s+"/key_mendeley.txt")
     api_key = key_handle.readline.rstrip

     detail_url = "http://api.mendeley.com/oapi/documents/details/" + @mendeley_paper.uuid + "/?consumer_key="+api_key
     begin
        detail_resp = Net::HTTP.get_response(URI.parse(detail_url))
     rescue
        retry
     end

     detail_data = detail_resp.body
     detail_result = JSON.parse(detail_data)

     if detail_result["oa_journal"] != false
        @mendeley_paper.open_access = true
     else
        @mendeley_paper.open_access = false
     end

     log "mendeley details: updated oa- and reader-status\n"
     if detail_result["stats"]
       @mendeley_paper.reader = detail_result["stats"]["readers"]
     elsif detail_result["reader"]
       @mendeley_paper.reader = detail_result["reader"]
     else
       @mendeley_paper.reader = "Unknown"
     end

     @mendeley_paper.save
     log "-> sleep for 5 secs\n"
     sleep(5)
  end

  def log msg
    Rails.logger.info "#{DateTime.now}: #{msg}"
  end
end<commit_msg>fixing undefined method in mendeley details job<commit_after>require "rubygems"
require "net/http"
require "json"

class MendeleyDetails
  include Sidekiq::Worker
  sidekiq_options :queue => :mendeley_details, :retry => 5, :unique => true

  def perform(mendeley_paper_id)
     # Logging stuff
     Rails.logger.level = 0
     Rails.logger = Logger.new("#{Rails.root}/log/mendeleydetails_#{Rails.env}.log")

     mendeley_paper = MendeleyPaper.find_by_id(mendeley_paper_id.to_i)
     return if mendeley_paper.nil?

     key_handle = File.open(::Rails.root.to_s+"/key_mendeley.txt")
     api_key = key_handle.readline.rstrip

     detail_url = "http://api.mendeley.com/oapi/documents/details/" + mendeley_paper.uuid + "/?consumer_key="+api_key
     begin
        detail_resp = Net::HTTP.get_response(URI.parse(detail_url))
     rescue
        retry
     end

     detail_data = detail_resp.body
     detail_result = JSON.parse(detail_data)

     if detail_result["oa_journal"] != false
        mendeley_paper.open_access = true
     else
        mendeley_paper.open_access = false
     end

     log "mendeley details: updated oa- and reader-status\n"
     if detail_result["stats"]
       mendeley_paper.reader = detail_result["stats"]["readers"]
     elsif detail_result["reader"]
       mendeley_paper.reader = detail_result["reader"]
     else
       mendeley_paper.reader = "Unknown"
     end

     mendeley_paper.save
     log "-> sleep for 5 secs\n"
     sleep(5)
  end

  def log msg
    Rails.logger.info "#{DateTime.now}: #{msg}"
  end
end
{'fraction_non_alphanumeric': 0.11470006591957811, 'fraction_numerical': 0.0026367831245880024, 'mean_word_length': 3.232914923291492, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1052940', 'n_tokens_mistral': 1093, 'n_tokens_neox': 1043, 'n_words': 186}