<commit_before>require 'resque'
require 'net/http'
require 'rexml/document'
require 'media_wiki'

class Snpedia
   include Resque::Plugins::UniqueJob
   @queue = :snpedia

   def self.perform(snp_id)
      @snp = Snp.find(snp_id)
      if @snp.snpedia_updated < 31.days.ago
        mw = MediaWiki::Gateway.new("http://www.snpedia.com/api.php")
        # return an array of page-titles
        pages = mw.list(@snp.name + "(")
        if pages != nil
          pages.each do |p|
            if p.index("(") != nil
              puts "snpedia: Got page\n"
              url = "http://www.snpedia.com/index.php/" + p.to_s
              if SnpediaPaper.find_all_by_url(url)  == []
                puts "-> Parsing new site\n"
                toparse = mw.get(p)
                summary = toparse[toparse.index("summary=")+8..toparse.length()-4]
                @snpedia_link = SnpediaPaper.new(:url => url, :snp_id => @snp.id, :summary => summary)
                @snpedia_link.save
								if @snp.mendeley_paper.count == nil:
									mendeley_count = 0
								else
									mendeley_count = @snp.mendeley_paper.count
								end
								if @snp.plos_paper.count == nil:
									plos_count = 0
								else
									plos_count = @snp.plos_paper.count
								end
								if @snp.snpedia_paper.count == nil:
									snpedia_count = 0
								else
									snpedia_count = @snp.snpedia_paper.count

                @snp.ranking = mendeley_count + 2*plos_count + 5*snpedia_count
              else
                puts "-> old site\n"
              end
            else
              puts "snpedia: No pages\n"
            end
          end
        end
        @snp.snpedia_updated = Time.zone.now
        @snp.save
        print "snpedia: sleep for 5 seconds\n"
        sleep(5)
      else
        print "snpedia: time threshold not met\n"
      end
   end
end<commit_msg>Had some Python-syntax in the Ruby-code (cough, cough)<commit_after>require 'resque'
require 'net/http'
require 'rexml/document'
require 'media_wiki'

class Snpedia
   include Resque::Plugins::UniqueJob
   @queue = :snpedia

   def self.perform(snp_id)
      @snp = Snp.find(snp_id)
      if @snp.snpedia_updated < 31.days.ago
        mw = MediaWiki::Gateway.new("http://www.snpedia.com/api.php")
        # return an array of page-titles
        pages = mw.list(@snp.name + "(")
        if pages != nil
          pages.each do |p|
            if p.index("(") != nil
              puts "snpedia: Got page\n"
              url = "http://www.snpedia.com/index.php/" + p.to_s
              if SnpediaPaper.find_all_by_url(url)  == []
                puts "-> Parsing new site\n"
                toparse = mw.get(p)
                summary = toparse[toparse.index("summary=")+8..toparse.length()-4]
                @snpedia_link = SnpediaPaper.new(:url => url, :snp_id => @snp.id, :summary => summary)
                @snpedia_link.save
								if @snp.mendeley_paper.count == nil
									mendeley_count = 0
								else
									mendeley_count = @snp.mendeley_paper.count
								end
								if @snp.plos_paper.count == nil
									plos_count = 0
								else
									plos_count = @snp.plos_paper.count
								end
								if @snp.snpedia_paper.count == nil
									snpedia_count = 0
								else
									snpedia_count = @snp.snpedia_paper.count

                @snp.ranking = mendeley_count + 2*plos_count + 5*snpedia_count
              else
                puts "-> old site\n"
              end
            else
              puts "snpedia: No pages\n"
            end
          end
        end
        @snp.snpedia_updated = Time.zone.now
        @snp.save
        print "snpedia: sleep for 5 seconds\n"
        sleep(5)
      else
        print "snpedia: time threshold not met\n"
      end
   end
end
{'fraction_non_alphanumeric': 0.11058201058201059, 'fraction_numerical': 0.00582010582010582, 'mean_word_length': 1.633008356545961, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '466674', 'n_tokens_mistral': 1531, 'n_tokens_neox': 1277, 'n_words': 260}