<commit_before>require_relative './page_retriever.rb'
require 'parallel'

# Class that scrapes a games actual page
class GamePageScraper
  def initialize(*_args)
    @page_retriever = PageRetriever.new
  end

  def scrape(games_hash)
    result = Parallel.map(games_hash,
                          progress: 'Scraping additional per game data',
                          in_processes: 8) do |game|
      url = game[:url]
      scrape_game(game, url) unless url.nil?
    end
    games_hash.push(result).flatten!
  end

  def scrape_game(game, url)
    page_contents = get_page_contents(url)
    game = scrape_game_with_valid_contents(game, page_contents) unless page_contents.nil?

    game
  end

  def scrape_game_with_valid_contents(game, page_contents)
    game[:metacritic] = scrape_metacritic(page_contents)
    game[:tags] = scrape_tags(page_contents)
    game[:genres] = scrape_genres(page_contents)
    game[:developer] = scrape_developer(page_contents)
    game[:publisher] = scrape_publisher(page_contents)
    game[:min_spec] = scrape_min_spec(page_contents)
    game[:recommended_spec] = scrape_recommended_spec(page_contents)

    game
  end

  def get_page_contents(url)
    @page_retriever.retrieve(url)
  end

  def scrape_metacritic(page_contents)
    score_element = page_contents.xpath("//div[@id='game_area_metascore']/span").first
    score_element.text.to_i unless score_element.nil?
  end

  def scrape_tags(page_contents)
    tags = []
    page_contents.xpath("//div[contains(@class, 'popular_tags')]/a").each do |tag|
      tags.push(tag.text.strip)
    end

    tags
  end

  def scrape_genres(page_contents)
    genres = []
    details = page_contents.xpath("//div[@class='details_block']")
    details.xpath(".//a[contains(@href, 'genre')]").each do |genre|
      genres.push(genre.text.strip)
    end

    genres
  end

  def scrape_developer(page_contents)
    details = page_contents.xpath("//div[@class='details_block']")
    details.xpath(".//a[contains(@href, 'developer')]").text.strip
  end

  def scrape_publisher(page_contents)
    details = page_contents.xpath("//div[@class='details_block']")
    details.xpath(".//a[contains(@href, 'publisher')]").text.strip
  end

  def scrape_min_spec(page_contents)
    spec_block = page_contents.xpath("//div[@data-os='win']/div[@class='game_area_sys_req_leftCol']/ul/ul")
    if spec_block.empty?
      spec_block = page_contents.xpath("//div[@data-os='win']/div[@class='game_area_sys_req_full']/ul/ul")
    end
    scrape_spec(spec_block)
  end

  def scrape_recommended_spec(page_contents)
    spec_block = page_contents.xpath("//div[@data-os='win']/div[@class='game_area_sys_req_rightCol']/ul/ul")
    scrape_spec(spec_block)
  end

  def scrape_spec(node)
    spec_array = node.text.split "\r"
    spec_hash = {}
    spec_array.each do |entry|
      value_pair = entry.split(':')
      next if value_pair.first.nil?
      key = value_pair.first.to_sym
      value = value_pair.last.strip
      spec_hash[key] = value
    end

    spec_hash
  end
end<commit_msg>Fixed bug where specs that had lines ending in \n instead of \r didn't get specs stored<commit_after>require_relative './page_retriever.rb'
require 'parallel'

# Class that scrapes a games actual page
class GamePageScraper
  def initialize(*_args)
    @page_retriever = PageRetriever.new
  end

  def scrape(games_hash)
    result = Parallel.map(games_hash,
                          progress: 'Scraping additional per game data',
                          in_processes: 8) do |game|
      url = game[:url]
      scrape_game(game, url) unless url.nil?
    end
    games_hash.push(result).flatten!
  end

  def scrape_game(game, url)
    page_contents = get_page_contents(url)
    game = scrape_game_with_valid_contents(game, page_contents) unless page_contents.nil?

    game
  end

  def scrape_game_with_valid_contents(game, page_contents)
    game[:metacritic] = scrape_metacritic(page_contents)
    game[:tags] = scrape_tags(page_contents)
    game[:genres] = scrape_genres(page_contents)
    game[:developer] = scrape_developer(page_contents)
    game[:publisher] = scrape_publisher(page_contents)
    game[:min_spec] = scrape_min_spec(page_contents)
    game[:recommended_spec] = scrape_recommended_spec(page_contents)

    game
  end

  def get_page_contents(url)
    @page_retriever.retrieve(url)
  end

  def scrape_metacritic(page_contents)
    score_element = page_contents.xpath("//div[@id='game_area_metascore']/span").first
    score_element.text.to_i unless score_element.nil?
  end

  def scrape_tags(page_contents)
    tags = []
    page_contents.xpath("//div[contains(@class, 'popular_tags')]/a").each do |tag|
      tags.push(tag.text.strip)
    end

    tags
  end

  def scrape_genres(page_contents)
    genres = []
    details = page_contents.xpath("//div[@class='details_block']")
    details.xpath(".//a[contains(@href, 'genre')]").each do |genre|
      genres.push(genre.text.strip)
    end

    genres
  end

  def scrape_developer(page_contents)
    details = page_contents.xpath("//div[@class='details_block']")
    details.xpath(".//a[contains(@href, 'developer')]").text.strip
  end

  def scrape_publisher(page_contents)
    details = page_contents.xpath("//div[@class='details_block']")
    details.xpath(".//a[contains(@href, 'publisher')]").text.strip
  end

  def scrape_min_spec(page_contents)
    spec_block = page_contents.xpath("//div[@data-os='win']/div[@class='game_area_sys_req_leftCol']/ul/ul")
    if spec_block.empty?
      spec_block = page_contents.xpath("//div[@data-os='win']/div[@class='game_area_sys_req_full']/ul/ul")
    end
    scrape_spec(spec_block)
  end

  def scrape_recommended_spec(page_contents)
    spec_block = page_contents.xpath("//div[@data-os='win']/div[@class='game_area_sys_req_rightCol']/ul/ul")
    scrape_spec(spec_block)
  end

  def scrape_spec(node)
    spec_array = node.text.split "\r"
    spec_array = node.text.split "\n" if spec_array.length == 1
    spec_hash = {}
    spec_array.each do |entry|
      value_pair = entry.split(':')
      next if value_pair.first.nil?
      key = value_pair.first.strip.to_sym
      value = value_pair.last.strip
      spec_hash[key] = value
    end

    spec_hash
  end
end
{'fraction_non_alphanumeric': 0.12560153994225218, 'fraction_numerical': 0.00048123195380173246, 'mean_word_length': 3.8035439137134053, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3732439', 'n_tokens_mistral': 2224, 'n_tokens_neox': 2119, 'n_words': 347}