<commit_before>require "fileutils"
require "cran"
require "dcf"

module Crantastic

  class UpdatePackages
    # To limit the number of updates per hour to avoid long-running processes,
    # set =max= to a positive digit.
    def start(max=-1)
      Log.log!("Starting cron task: UpdatePackages")
      `curl -s http://cran.r-project.org/src/contrib/PACKAGES.gz -o tmp/PACKAGES.gz`
      `gunzip -f tmp/PACKAGES.gz`
      latest_versions = Dcf.parse((File.read("tmp/PACKAGES"))).sort_by { |i| i["Package"].downcase }

      i = 0
      latest_versions.each do |new|
        next if new["Package"] == "orientlib" && new["Version"] == "0.10.1" # known bad entry
        break if i == max
        cur = Package.find_by_name(new["Package"])
        if cur
          if cur.latest.version != new["Version"]
            Log.log!("Updating package: #{new['Package']} (#{new['Version']})")
            add_version_to_db(CRAN::CranPackage.new(new["Package"], new["Version"]))
            i += 1
          end
        else
          Log.log!("New package: #{new['Package']} (#{new['Version']})")
          Package.transaction do
            Package.create!(:name => new["Package"]) # Start by creating the package entry
            add_version_to_db(CRAN::CranPackage.new(new["Package"], new["Version"]))
          end
          i += 1
        end
      end
      Log.log!("Cron task finished.")
      return true
    end

    def add_version_to_db(pkg)
      filename = "#{pkg.name}_#{pkg.version}.tar.gz"
      `curl -s "http://cran.r-project.org/src/contrib/#{pkg.name}_#{pkg.version}.tar.gz" -o tmp/#{filename}`
      `tar -C tmp -zxvf tmp/#{filename}; rm tmp/#{filename}`

      pkgdir = File.join(RAILS_ROOT, "/tmp/#{pkg.name}/")

      description = Dcf.parse(File.read(pkgdir + "DESCRIPTION"))
      throw Exception.new("Couldn't parse DESCRIPTION for #{pkg.name}. " +
                          "Look at http://cran.r-project.org/web/packages/#{pkg.name}/DESCRIPTION " +
                          "for clues.") if description.nil?
      data = description.first
      data = data.downcase_keys.symbolize_keys

      fields = [:title, :license, :description, :author, :enhances, :priority,
                :maintainer, :date, :url, :depends, :suggests, :imports]
      data.delete_if { |k,v| !fields.include?(k) } # Remove unwanted fields<commit_msg>crantastic.rb experiment<commit_after>require "fileutils"
require "cran"
require "dcf"

module Crantastic

  class UpdatePackages
    # To limit the number of updates per hour to avoid long-running processes,
    # set =max= to a positive digit.
    def start(max=-1)
      Log.log!("Starting cron task: UpdatePackages")
      `curl -s http://cran.r-project.org/src/contrib/PACKAGES.gz -o tmp/PACKAGES.gz`
      `gunzip -f tmp/PACKAGES.gz`
      packages = File.read("tmp/PACKAGES")
      packages.split("\n\n").each do |entry|
        package = entry.scan(/Package: (.+)/)[0][0]
        version = entry.scan(/Version: (.+)/)[0][0]

        next if package == "orientlib" && version == "0.10.1" # known bad entry

        cur = Package.find_by_name(package)
        if cur
          if cur.latest.version != version
            Log.log!("Updating package: #{package} (#{version})")
            add_version_to_db(CRAN::CranPackage.new(package, version))
            i += 1
          end
        else
          Log.log!("New package: #{package} (#{version})")
          Package.transaction do
            Package.create!(:name => package) # Start by creating the package entry
            add_version_to_db(CRAN::CranPackage.new(package, version))
          end
          i += 1
        end
      end
      Log.log!("Cron task finished.")
      return true
    end

    def add_version_to_db(pkg)
      filename = "#{pkg.name}_#{pkg.version}.tar.gz"
      `curl -s "http://cran.r-project.org/src/contrib/#{pkg.name}_#{pkg.version}.tar.gz" -o tmp/#{filename}`
      `tar -C tmp -zxvf tmp/#{filename}; rm tmp/#{filename}`

      pkgdir = File.join(RAILS_ROOT, "/tmp/#{pkg.name}/")

      description = Dcf.parse(File.read(pkgdir + "DESCRIPTION"))
      throw Exception.new("Couldn't parse DESCRIPTION for #{pkg.name}. " +
                          "Look at http://cran.r-project.org/web/packages/#{pkg.name}/DESCRIPTION " +
                          "for clues.") if description.nil?
      data = description.first
      data = data.downcase_keys.symbolize_keys

      fields = [:title, :license, :description, :author, :enhances, :priority,
                :maintainer, :date, :url, :depends, :suggests, :imports]
      data.delete_if { |k,v| !fields.include?(k) } # Remove unwanted fields
{'fraction_non_alphanumeric': 0.1540785498489426, 'fraction_numerical': 0.004100129477772983, 'mean_word_length': 2.4641255605381165, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3779929', 'n_tokens_mistral': 1493, 'n_tokens_neox': 1454, 'n_words': 330}