<commit_before>require 'virtus'
require 'nokogiri'
require 'faraday'
require 'typhoeus'
require 'typhoeus/adapters/faraday'

class ProductFetcher
  HOST = 'http://intertop.ua'

  class Product
    include Virtus.value_object

    values do
      attribute :external_id, String
      attribute :url, String
      attribute :thumb_url, String
      attribute :price, Integer
    end
  end

  def fetch_products
    connection = Faraday.new(url: HOST) do |faraday|
      faraday.adapter :typhoeus
    end

    connection.headers[:user_agent] = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0"

    response = connection.get('/master')
    first_page = Nokogiri::HTML(response.body)
    last_paging_anchor = first_page.css('.paging a:not(.arr)').last
    last_paging_href = last_paging_anchor.attributes['href'].value
    page_count = last_paging_href.match(/&page=(\d+)/)[1].to_i

    products = parse_products(first_page)

    manager = Typhoeus::Hydra.new(max_concurrency: 10)

    # responses = nil
    # connection.in_parallel(manager) do
      responses = (2..page_count).map do |page|<commit_msg>Fetch 200(max allowed) products per page<commit_after>require 'virtus'
require 'nokogiri'
require 'faraday'
require 'typhoeus'
require 'typhoeus/adapters/faraday'

class ProductFetcher
  HOST = 'http://intertop.ua'

  class Product
    include Virtus.value_object

    values do
      attribute :external_id, String
      attribute :url, String
      attribute :thumb_url, String
      attribute :price, Integer
    end
  end

  def fetch_products
    connection = Faraday.new(url: HOST) do |faraday|
      faraday.adapter :typhoeus
    end

    connection.headers[:user_agent] = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0"
    connection.headers[:cookie] = "ON_PAGE=200"

    response = connection.get('/master')
    first_page = Nokogiri::HTML(response.body)
    last_paging_anchor = first_page.css('.paging a:not(.arr)').last
    last_paging_href = last_paging_anchor.attributes['href'].value
    page_count = last_paging_href.match(/&page=(\d+)/)[1].to_i

    products = parse_products(first_page)

    manager = Typhoeus::Hydra.new(max_concurrency: 10)

    # responses = nil
    # connection.in_parallel(manager) do
      responses = (2..page_count).map do |page|
{'fraction_non_alphanumeric': 0.11648822269807281, 'fraction_numerical': 0.024839400428265525, 'mean_word_length': 3.747967479674797, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7423779', 'n_tokens_mistral': 852, 'n_tokens_neox': 785, 'n_words': 162}