<commit_before>require "nokogiri"
require "hashr"

module Trawler
  class Parser
    attr_reader :url
    attr_reader :meta_data

    def initialize(options)
      @page           = options[:page]
      @url            = options[:url]
      @min_image_size = options[:image_size] 
      @meta_data      = Hashr.new
    end

    def title
      meta_title.nil? ? html_title : meta_title
    end

    def description
      meta_description.nil? ? html_description : meta_description
    end

    def images
      images = []
      images << meta_image 
      images << find_images
      images.flatten.compact.map { |i| i.strip }.uniq
    end

    def video
      meta_video
    end

    def document 
      @document ||= Nokogiri::HTML(@page) 
    end

    private

    def html_title
      document.css("title").inner_text rescue nil
    end

    def html_description<commit_msg>Fix all urls to be absolute for images.<commit_after>require "nokogiri"
require "hashr"

module Trawler
  class Parser
    attr_reader :url
    attr_reader :meta_data

    def initialize(options)
      @page           = options[:page]
      @url            = options[:url]
      @min_image_size = options[:image_size] 
      @meta_data      = Hashr.new
    end

    def title
      meta_title.nil? ? html_title : meta_title
    end

    def description
      meta_description.nil? ? html_description : meta_description
    end

    def images
      images = [meta_image]
      images << find_images
      images.flatten!
      images = images.select { |img| !img.nil? }.select { |s| !s.empty? }
      images.flatten.map! { |img| img.strip }.uniq
      images.map { |img| normalize_url(img) }
    end

    def normalize_url(uri)
      if uri =~ /^\w*\:/i
        return uri
      else
        Addressable::URI.join(url, uri).normalize.to_s
      end
    rescue URI::InvalidURIError, Addressable::URI::InvalidURIError => e
      add_fatal_error "Link parsing exception: #{e.message}" and nil
    end

    def video
      meta_video
    end

    def document 
      @document ||= Nokogiri::HTML(@page) 
    end

    private

    def html_title
      document.css("title").inner_text rescue nil
    end

    def html_description
{'fraction_non_alphanumeric': 0.08896958692691784, 'fraction_numerical': 0.0, 'mean_word_length': 2.2175182481751823, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2810152', 'n_tokens_mistral': 706, 'n_tokens_neox': 674, 'n_words': 169}