<commit_before>class Repositories
  class Wordpress < Base
    HAS_VERSIONS = true
    HAS_DEPENDENCIES = false
    URL = 'https://wordpress.org/plugins'

    def self.recent_names
      page = 1
      r = Typhoeus.post 'http://api.wordpress.org/plugins/info/1.0/', body: "action=query_plugins&request=O%3A8%3A%22stdClass%22%3A3%3A%7Bs%3A6%3A%22browse%22%3Bs%3A3%3A%22new%22%3Bs%3A8%3A%22per_page%22%3Bi%3A100%3Bs%3A4%3A%22page%22%3Bi%3A#{page}%3B%7D"
      r.body.scan(/"slug";s:[0-9]+:"([^"]+)";/).flatten.compact.uniq
    end

    def self.project_names
      REDIS.smembers 'wordpress-names'
    end

    def self.load_names(limit = nil)
      num = REDIS.get('wordpress-page')
      if limit
        REDIS.set 'wordpress-page', limit
        num = limit
      elsif num.nil?
        REDIS.set 'wordpress-page', 362
        num = 362
      else
        num = num.to_i
      end

      (1..num).to_a.reverse.each do |number|
        r = Typhoeus.post 'http://api.wordpress.org/plugins/info/1.0/', body: "action=query_plugins&request=O%3A8%3A%22stdClass%22%3A3%3A%7Bs%3A6%3A%22browse%22%3Bs%3A3%3A%22new%22%3Bs%3A8%3A%22per_page%22%3Bi%3A100%3Bs%3A4%3A%22page%22%3Bi%3A#{number}%3B%7D"
        r.body.scan(/"slug";s:[0-9]+:"([^"]+)";/).flatten.compact.uniq.each do |name|
          REDIS.sadd 'wordpress-names', name
        end
        REDIS.set 'wordpress-page', number
      end
    end

    def self.project(name)
      get("https://api.wordpress.org/plugins/info/1.0/#{name}.json")
    end

    def self.mapping(project)
      {
        :name => project["slug"],
        :description => project["short_description"],
        :homepage => project["homepage"],
        :keywords => project.fetch("tags", {}).values.join(',')
      }
    end

    def self.versions(project)
      [
        {
          :number => project['version'],
          :published_at => project['last_updated']
        }
      ]
    end
  end
end
<commit_msg>Look for github urls in homepage for wordpress<commit_after>class Repositories
  class Wordpress < Base
    HAS_VERSIONS = true
    HAS_DEPENDENCIES = false
    URL = 'https://wordpress.org/plugins'

    def self.recent_names
      page = 1
      r = Typhoeus.post 'http://api.wordpress.org/plugins/info/1.0/', body: "action=query_plugins&request=O%3A8%3A%22stdClass%22%3A3%3A%7Bs%3A6%3A%22browse%22%3Bs%3A3%3A%22new%22%3Bs%3A8%3A%22per_page%22%3Bi%3A100%3Bs%3A4%3A%22page%22%3Bi%3A#{page}%3B%7D"
      r.body.scan(/"slug";s:[0-9]+:"([^"]+)";/).flatten.compact.uniq
    end

    def self.project_names
      REDIS.smembers 'wordpress-names'
    end

    def self.load_names(limit = nil)
      num = REDIS.get('wordpress-page')
      if limit
        REDIS.set 'wordpress-page', limit
        num = limit
      elsif num.nil?
        REDIS.set 'wordpress-page', 362
        num = 362
      else
        num = num.to_i
      end

      (1..num).to_a.reverse.each do |number|
        r = Typhoeus.post 'http://api.wordpress.org/plugins/info/1.0/', body: "action=query_plugins&request=O%3A8%3A%22stdClass%22%3A3%3A%7Bs%3A6%3A%22browse%22%3Bs%3A3%3A%22new%22%3Bs%3A8%3A%22per_page%22%3Bi%3A100%3Bs%3A4%3A%22page%22%3Bi%3A#{number}%3B%7D"
        r.body.scan(/"slug";s:[0-9]+:"([^"]+)";/).flatten.compact.uniq.each do |name|
          REDIS.sadd 'wordpress-names', name
        end
        REDIS.set 'wordpress-page', number
      end
    end

    def self.project(name)
      get("https://api.wordpress.org/plugins/info/1.0/#{name}.json")
    end

    def self.mapping(project)
      {
        :name => project["slug"],
        :description => project["short_description"],
        :homepage => project["homepage"],
        :keywords => project.fetch("tags", {}).values.join(','),
        :repository_url => repo_fallback('', project["homepage"])
      }
    end

    def self.versions(project)
      [
        {
          :number => project['version'],
          :published_at => project['last_updated']
        }
      ]
    end
  end
end

{'fraction_non_alphanumeric': 0.16842900302114805, 'fraction_numerical': 0.06042296072507553, 'mean_word_length': 2.783809523809524, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 16, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7123650', 'n_tokens_mistral': 1676, 'n_tokens_neox': 1530, 'n_words': 200}