<commit_before>require 'spidercrawl/request'
require 'spidercrawl/page'
require 'colorize'

module Spidercrawl
  # Start working hard
  class SpiderWorker

    attr_reader :page

    def initialize(url, options = {})
      @url = url
      #@headers = options[:headers]
      @delay = options[:delay] ? options[:delay] : 0 # default 0 seconds
      @threads = options[:threads] ? options[:threads] : 10 # default 10 threads
      @timeout = options[:timeout] ? options[:timeout] : 20 # default 20 seconds
      @allow_redirections = options[:allow_redirections]
      @max_pages = options[:max_pages]
      @pattern = options[:pattern]
      @setup = nil
      @teardown = nil
      @redirect = nil
      @success = nil
      @failure = nil
    end

    def crawl
      link_queue = Queue.new
      pages, visited_links = [], []
      link_queue << @url

      spider_worker = Request.new(@url, :threads => @threads, :timeout => @timeout)

      begin
        url = link_queue.pop
        next if visited_links.include?(url) 
        next if @pattern && url !~ @pattern
        visited_links << url
        spider_worker.uri = URI.parse(url)

        start_time = Time.now
        response = @setup.yield url unless @setup.nil?
        end_time = Time.now

        page = (response ? setup_page(URI(url), response, ((end_time - start_time).to_f*1000).to_i) : spider_worker.curl)

        if page.success? || page.redirect? then
          while page.redirect?
            puts ("### redirect to #{page.location}" + (visited_links.include?(page.location) ? " which we have already visited!" : "")).white.on_black
            break if visited_links.include?(page.location)
            
            start_time = Time.now
            response = @redirect.yield page.location unless @redirect.nil?
            end_time = Time.now

            spider_worker.uri = (URI(page.location)).to_s
            page = (response ? setup_page(URI(page.location), response, ((end_time - start_time).to_f*1000).to_i) : spider_worker.curl
            visited_links << page
          end
          unless visited_links.include?(page.location)
            pages << page unless page.content == ""
            page.internal_links.each do |link| 
              if !visited_links.include?(link) 
                if @pattern
                  link_queue << link if link =~ @pattern
                else
                  link_queue << link
                end
              end
            end rescue nil
            @teardown.yield page unless @teardown.nil?
            sleep @delay
          end
        elsif page.not_found? then
          puts "page not found"
        end
      end until link_queue.empty?
      puts "Total pages crawled: #{visited_links.size}"
      pages
    end

    def parallel_crawl
      link_queue = Queue.new
      pages, visited_links = [], []
      link_queue << @url<commit_msg>syntax error<commit_after>require 'spidercrawl/request'
require 'spidercrawl/page'
require 'colorize'

module Spidercrawl
  # Start working hard
  class SpiderWorker

    attr_reader :page

    def initialize(url, options = {})
      @url = url
      #@headers = options[:headers]
      @delay = options[:delay] ? options[:delay] : 0 # default 0 seconds
      @threads = options[:threads] ? options[:threads] : 10 # default 10 threads
      @timeout = options[:timeout] ? options[:timeout] : 20 # default 20 seconds
      @allow_redirections = options[:allow_redirections]
      @max_pages = options[:max_pages]
      @pattern = options[:pattern]
      @setup = nil
      @teardown = nil
      @redirect = nil
      @success = nil
      @failure = nil
    end

    def crawl
      link_queue = Queue.new
      pages, visited_links = [], []
      link_queue << @url

      spider_worker = Request.new(@url, :threads => @threads, :timeout => @timeout)

      begin
        url = link_queue.pop
        next if visited_links.include?(url) 
        next if @pattern && url !~ @pattern
        visited_links << url
        spider_worker.uri = URI.parse(url)

        start_time = Time.now
        response = @setup.yield url unless @setup.nil?
        end_time = Time.now

        page = (response ? setup_page(URI(url), response, ((end_time - start_time).to_f*1000).to_i) : spider_worker.curl)

        if page.success? || page.redirect? then
          while page.redirect?
            puts ("### redirect to #{page.location}" + (visited_links.include?(page.location) ? " which we have already visited!" : "")).white.on_black
            break if visited_links.include?(page.location)
            
            start_time = Time.now
            response = @redirect.yield page.location unless @redirect.nil?
            end_time = Time.now

            spider_worker.uri = (URI(page.location)).to_s
            page = (response ? setup_page(URI(page.location), response, ((end_time - start_time).to_f*1000).to_i) : spider_worker.curl)
            visited_links << page
          end
          unless visited_links.include?(page.location)
            pages << page unless page.content == ""
            page.internal_links.each do |link| 
              if !visited_links.include?(link) 
                if @pattern
                  link_queue << link if link =~ @pattern
                else
                  link_queue << link
                end
              end
            end rescue nil
            @teardown.yield page unless @teardown.nil?
            sleep @delay
          end
        elsif page.not_found? then
          puts "page not found"
        end
      end until link_queue.empty?
      puts "Total pages crawled: #{visited_links.size}"
      pages
    end

    def parallel_crawl
      link_queue = Queue.new
      pages, visited_links = [], []
      link_queue << @url
{'fraction_non_alphanumeric': 0.10519930675909879, 'fraction_numerical': 0.006239168110918544, 'mean_word_length': 2.0696808510638296, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2247703', 'n_tokens_mistral': 1694, 'n_tokens_neox': 1612, 'n_words': 410}