<commit_before># encoding: utf-8
require 'mechanize'
require 'proxynova'
module Spidey
  class AbstractSpider
    attr_accessor :urls, :handlers, :results, :request_interval, :errors, :proxy_addr, :proxy_port, :agent
    attr_accessor :proxies, :random_proxy, :total_proxies, :current_proxy
    
    DEFAULT_REQUEST_INTERVAL = 3  # seconds
    
    def self.handle(url, handler, default_data = {})
      start_urls << url
      handlers[url] = [handler, default_data]
    end
    
    # Accepts:
    #   request_interval: number of seconds to wait between requests (default: 3)
    def initialize(attrs = {})
      agent
      @urls = []
      @handlers = {}
      @results = []
      @proxies = []
      @current_proxy = 0
      self.class.start_urls.each { |url| handle url, *self.class.handlers[url] }
      @request_interval = attrs[:request_interval] || DEFAULT_REQUEST_INTERVAL
      @proxy_port = attrs[:proxy_port]
      @proxy_addr = attrs[:proxy_addr]
      @random_proxy = attrs[:random_proxy]
      if @random_proxy == true
        @proxies = Proxynova.get_list
        @total_proxies = @proxies.count
      end
    end
    
    # Make is accessible
    def agent
      @agent = Mechanize.new { |agent|
        agent.user_agent_alias = 'Windows Mozilla'
        agent.open_timeout = 10
        agent.read_timeout = 10
        agent.keep_alive = false
        agent.max_history = 0
      }
    end

    # Cycle true proxy server IPs
    # Could be improved later if we pushed this randomization into Proxynova gem
    def ip_cycle
      if @total_proxies == 0
        @total_proxies = @current_proxy
        @current_proxy = 0
      end
      proxy_server = @proxies[@current_proxy]
      @current_proxy += 1
      @total_proxies -= 1
      proxy_server
    end
  
    # Iterates through URLs queued for handling, including any that are added in the course of crawling. Accepts:
    #   max_urls: maximum number of URLs to crawl before returning (optional)
    def crawl(options = {})
      unless @proxy_addr.nil? || @proxy_port.nil?
        @agent.set_proxy @proxy_addr, @proxy_port
      end

      if @random_proxy == true
        proxy = ip_cycle
        @proxy_addr = proxy[:ip]
        @proxy_port = proxy[:port]
        @agent.set_proxy @proxy_addr, @proxy_port
      end

      @errors = []
      i = 0
      each_url do |url, handler, default_data|
        break if options[:max_urls] && i >= options[:max_urls]
        begin
          page = @agent.get(url)
          Spidey.logger.info "Handling #{url.inspect}"
          send handler, page, default_data
          rescue => ex
          add_error url: url, handler: handler, error: ex
        end
        sleep request_interval if request_interval > 0
        i += 1<commit_msg>Move Proxy Cycle Into Loop<commit_after># encoding: utf-8
require 'mechanize'
require 'proxynova'
module Spidey
  class AbstractSpider
    attr_accessor :urls, :handlers, :results, :request_interval, :errors, :proxy_addr, :proxy_port, :agent
    attr_accessor :proxies, :random_proxy, :total_proxies, :current_proxy
    
    DEFAULT_REQUEST_INTERVAL = 3  # seconds
    
    def self.handle(url, handler, default_data = {})
      start_urls << url
      handlers[url] = [handler, default_data]
    end
    
    # Accepts:
    #   request_interval: number of seconds to wait between requests (default: 3)
    def initialize(attrs = {})
      agent
      @urls = []
      @handlers = {}
      @results = []
      @proxies = []
      @current_proxy = 0
      self.class.start_urls.each { |url| handle url, *self.class.handlers[url] }
      @request_interval = attrs[:request_interval] || DEFAULT_REQUEST_INTERVAL
      @proxy_port = attrs[:proxy_port]
      @proxy_addr = attrs[:proxy_addr]
      @random_proxy = attrs[:random_proxy]
      if @random_proxy == true
        @proxies = Proxynova.get_list
        @total_proxies = @proxies.count
      end
    end
    
    # Make is accessible
    def agent
      @agent = Mechanize.new { |agent|
        agent.user_agent_alias = 'Windows Mozilla'
        agent.open_timeout = 10
        agent.read_timeout = 10
        agent.keep_alive = false
        agent.max_history = 0
      }
    end

    # Cycle true proxy server IPs
    # Could be improved later if we pushed this randomization into Proxynova gem
    def ip_cycle
      if @total_proxies == 0
        @total_proxies = @current_proxy
        @current_proxy = 0
      end
      proxy_server = @proxies[@current_proxy]
      @current_proxy += 1
      @total_proxies -= 1
      proxy_server
    end
  
    # Iterates through URLs queued for handling, including any that are added in the course of crawling. Accepts:
    #   max_urls: maximum number of URLs to crawl before returning (optional)
    def crawl(options = {})
      unless @proxy_addr.nil? || @proxy_port.nil?
        @agent.set_proxy @proxy_addr, @proxy_port
      end

      @errors = []
      i = 0
      each_url do |url, handler, default_data|
        break if options[:max_urls] && i >= options[:max_urls]
        begin
          if @random_proxy == true
            proxy = ip_cycle
            @proxy_addr = proxy[:ip]
            @proxy_port = proxy[:port]
            @agent.set_proxy @proxy_addr, @proxy_port
          end
          page = @agent.get(url)
          Spidey.logger.info "Handling #{url.inspect}"
          send handler, page, default_data
          rescue => ex
          add_error url: url, handler: handler, error: ex
        end
        sleep request_interval if request_interval > 0
        i += 1
{'fraction_non_alphanumeric': 0.09205776173285199, 'fraction_numerical': 0.005776173285198556, 'mean_word_length': 2.386919315403423, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '750315', 'n_tokens_mistral': 1735, 'n_tokens_neox': 1711, 'n_words': 453}