<commit_before>require 'rubygems'
require 'net/http'
require 'net/https'
require 'open-uri'
require 'nokogiri'
require 'csv'
require 'ruby-debug'

module AndroidMarketScraper
  class AppInfo
    ATTRIBUTES = [:market_rank, :title, :developer, :price_usd, :market_id, :market_url, :stars,
                  :updated, :app_version, :minimum_android_version, :category, :installs,
                  :installs_min, :installs_max, :size]

    def initialize(options={})
      ATTRIBUTES.each do |attrib|
        instance_variable_set("@#{attrib}", options[attrib])
      end
    end

    def method_missing(symbol)
      if ATTRIBUTES.include?(symbol)
        return instance_variable_get("@#{symbol}")
      else
        raise NoMethodError
      end
    end
  end

  class AppInfoSet
    def initialize(app_infos=[])
      @app_infos = app_infos
    end

    def <<(app_info)
      @app_infos << app_info

      return self
    end

    def output_report
      CSV::Writer.generate(STDOUT) do |csv|
        csv << AppInfo::ATTRIBUTES
        @app_infos.each do |app|
          csv << AppInfo::ATTRIBUTES.map{ |attrib| app.send(attrib).to_s }
        end
      end
    end
  end

  class Scraper
    def initialize
      @items_per_page = 24
      @default_max_pages = 35
    end

    def fetch(url)
      url = URI.parse(url)
      http = Net::HTTP.new(url.host, url.port)
      http.use_ssl = true if url.port == 443
      http.verify_mode = OpenSSL::SSL::VERIFY_NONE if url.port == 443
      path = url.path + "?" + url.query
      res, data = http.get(path)
      return data
    end

    def scrape(options={})
      max_pages = options[:max_pages] || @default_max_pages
      category = options[:category] || 'GAME'
      purchase_type = options[:purchase_type] || 'free'

      market_rank = 0
      app_info_set = AppInfoSet.new

      # Loop through every rankings page.
      (0..max_pages).each do |page|
        start_offset = 24 * page

        base_url = "https://market.android.com/details"
        url_params = "?id=apps_topselling_#{purchase_type}&cat=#{category}&start=#{start_offset}&num=24"
        doc = Nokogiri::HTML(fetch(base_url + url_params))

        # Loop through every app on a specific rankings page.
        doc.css('.snippet').each do |snippet_node|
          details_node = snippet_node.css('.details')
<commit_msg>clean up<commit_after>require 'rubygems'
require 'net/http'
require 'net/https'
require 'nokogiri'
require 'csv'
require 'ruby-debug'

module AndroidMarketScraper
  class AppInfo
    ATTRIBUTES = [:market_rank, :title, :developer, :price_usd, :market_id, :market_url, :stars,
                  :updated, :app_version, :minimum_android_version, :category, :installs,
                  :installs_min, :installs_max, :size]

    def initialize(options={})
      ATTRIBUTES.each do |attrib|
        instance_variable_set("@#{attrib}", options[attrib])
      end
    end

    def method_missing(symbol)
      if ATTRIBUTES.include?(symbol)
        return instance_variable_get("@#{symbol}")
      else
        raise NoMethodError
      end
    end
  end

  class AppInfoSet
    def initialize(app_infos=[])
      @app_infos = app_infos
    end

    def <<(app_info)
      @app_infos << app_info

      return self
    end

    def output_report
      CSV::Writer.generate(STDOUT) do |csv|
        csv << AppInfo::ATTRIBUTES
        @app_infos.each do |app|
          csv << AppInfo::ATTRIBUTES.map{ |attrib| app.send(attrib).to_s }
        end
      end
    end
  end

  class Scraper
    def initialize
      @items_per_page = 24
      @default_max_pages = 35
    end

    def fetch(url)
      url = URI.parse(url)
      http = Net::HTTP.new(url.host, url.port)
      http.use_ssl = true if url.port == 443
      http.verify_mode = OpenSSL::SSL::VERIFY_NONE if url.port == 443
      path = url.path + "?" + url.query
      res, data = http.get(path)

      return data
    end

    def scrape(options={})
      max_pages = options[:max_pages] || @default_max_pages
      category = options[:category] || 'GAME'
      purchase_type = options[:purchase_type] || 'free'

      market_rank = 0
      app_info_set = AppInfoSet.new

      # Loop through every rankings page.
      (0..max_pages).each do |page|
        start_offset = 24 * page

        base_url = "https://market.android.com/details"
        url_params = "?id=apps_topselling_#{purchase_type}&cat=#{category}&start=#{start_offset}&num=24"
        doc = Nokogiri::HTML(fetch(base_url + url_params))

        # Loop through every app on a specific rankings page.
        doc.css('.snippet').each do |snippet_node|
          details_node = snippet_node.css('.details')

{'fraction_non_alphanumeric': 0.11544227886056972, 'fraction_numerical': 0.006853715999143285, 'mean_word_length': 2.6541471048513303, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '163916', 'n_tokens_mistral': 1596, 'n_tokens_neox': 1516, 'n_words': 337}