<commit_before>require "movies_report/version"
require 'nokogiri'
require "net/http"
require "uri"

module MoviesReport

  TO_REMOVE = %w{ .BRRiP MX DVDRiP DVDRip XViD PSiG
    lektor Lektor lekyor .napisy
    -orgonalny --orgonalny --orgoinalny .oryginalny oryginalny --oryginalny --orginalny orginalny
    .pl PL ivo
    chomikuj Chomikuj.avi .avi dubbing.pl.avi
  }

  MATCHERS = {
    'chomikuj.pl'     => '#FilesListContainer .fileItemContainer .filename',
    'www.filmweb.pl'  => '.searchResult a.searchResultTitle'
  }

  CHECK_MOVIE_URL = "http://www.filmweb.pl/search?q=%s"

  class DSL

    def self.parse_html(url)
      uri = URI(url)
      doc = Nokogiri::HTML(Net::HTTP.get_response(uri).body)

      doc.css(MATCHERS[uri.host]).map do |el|
        title = parse_title(el)
        links = search_movies(title)
        puts "GOT #{title}, #{links}"

        { title: title, links: links}
      end
    end

    def self.parse_title(el)
      el.content.strip.gsub(/#{TO_REMOVE.join('|')}/, '').strip.gsub(/[-\s\.]+$/, '')
    end

    def self.search_movies(title)
      uri = URI(CHECK_MOVIE_URL % CGI::escape(title))
      doc = Nokogiri::HTML(Net::HTTP.get_response(uri).body)

      doc.css(MATCHERS[uri.host]).map do |el|
        [el.content.strip, el.attr('href')]
      end
    end
  end
end
<commit_msg>comments<commit_after>require "movies_report/version"
require 'nokogiri'
require "net/http"
require "uri"

module MoviesReport

  TO_REMOVE = %w{ .BRRiP MX DVDRiP DVDRip XViD PSiG
    lektor Lektor lekyor .napisy
    -orgonalny --orgonalny --orgoinalny .oryginalny oryginalny --oryginalny --orginalny orginalny
    .pl PL ivo
    chomikuj Chomikuj.avi .avi dubbing.pl.avi
  }

  MATCHERS = {
    'chomikuj.pl'     => '#FilesListContainer .fileItemContainer .filename',
    'www.filmweb.pl'  => '.searchResult a.searchResultTitle'
  }

  CHECK_MOVIE_URL = "http://www.filmweb.pl/search?q=%s"

  class DSL

    def self.parse_html(url)
      uri = URI(url)
      doc = Nokogiri::HTML(Net::HTTP.get_response(uri).body)

      doc.css(MATCHERS[uri.host]).map do |el|
        title = parse_title(el)
        links = search_movies(title)
        #puts "GOT #{title}, #{links}"

        { title: title, links: links}
      end
    end

    def self.parse_title(el)
      el.content.strip.gsub(/#{TO_REMOVE.join('|')}/, '').strip.gsub(/[-\s\.]+$/, '')
    end

    def self.search_movies(title)
      uri = URI(CHECK_MOVIE_URL % CGI::escape(title))
      doc = Nokogiri::HTML(Net::HTTP.get_response(uri).body)

      doc.css(MATCHERS[uri.host]).map do |el|
        [el.content.strip, el.attr('href')]
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.14852607709750568, 'fraction_numerical': 0.0, 'mean_word_length': 2.8529839883551675, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1161932', 'n_tokens_mistral': 1037, 'n_tokens_neox': 999, 'n_words': 186}