<commit_before>require 'rubygems'
require 'nokogiri'
require 'guess_html_encoding'

module Readability
  class Document
    DEFAULT_OPTIONS = {
      :retry_length               => 250,
      :min_text_length            => 25,
      :remove_unlikely_candidates => true,
      :weight_classes             => true,
      :clean_conditionally        => true,
      :remove_empty_nodes         => true,
      :min_image_width            => 130,
      :min_image_height           => 80,
      :ignore_image_format        => []
    }.freeze

    attr_accessor :options, :html, :best_candidate, :candidates, :best_candidate_has_image

    def initialize(input, options = {})
      @options = DEFAULT_OPTIONS.merge(options)
      @input = input

      if RUBY_VERSION =~ /^1\.9\./ && !@options[:encoding]
        @input = GuessHtmlEncoding.encode(@input, @options[:html_headers]) unless @options[:do_not_guess_encoding]
        @options[:encoding] = @input.encoding.to_s
      end

      @input = @input.gsub(REGEXES[:replaceBrsRe], '</p><p>').gsub(REGEXES[:replaceFontsRe], '<\1span>')
      @remove_unlikely_candidates = @options[:remove_unlikely_candidates]
      @weight_classes = @options[:weight_classes]
      @clean_conditionally = @options[:clean_conditionally]
      @best_candidate_has_image = true
      make_html
    end

    def prepare_candidates
      @html.css("script, style").each { |i| i.remove }
      remove_unlikely_candidates! if @remove_unlikely_candidates
      transform_misused_divs_into_paragraphs!
      
      @candidates     = score_paragraphs(options[:min_text_length])
      @best_candidate = select_best_candidate(@candidates)
    end

    def make_html
      @html = Nokogiri::HTML(@input, nil, @options[:encoding])
      # In case document has no body, such as from empty string or redirect
      @html = Nokogiri::HTML('<body />', nil, @options[:encoding]) if @html.css('body').length == 0

      # Remove html comment tags
      @html.xpath('//comment()').each { |i| i.remove }
    end

    def images(content=nil, reload=false)
      begin
        require 'fastimage'
      rescue LoadError
        raise "Please install fastimage in order to use the #images feature."
      end

      @best_candidate_has_image = false if reload

      prepare_candidates
      list_images   = []
      tested_images = []
      content       = @best_candidate[:elem] unless reload

      return list_images if content.nil?
      elements = content.css("img").map(&:attributes)

        elements.each do |element|
          next unless element["src"]

          url     = element["src"].value
          height  = element["height"].nil?  ? 0 : element["height"].value.to_i
          width   = element["width"].nil?   ? 0 : element["width"].value.to_i
          
          if url =~ /\Ahttps?:\/\//i && (height.zero? || width.zero?)
            image   = get_image_size(url) 
            next unless image
          else
            image = {:width => width, :height => height}
          end
          
          image[:format] = File.extname(url).gsub(".", "")
          
          if tested_images.include?(url)
            debug("Image was tested: #{url}")
            next
          end

          tested_images.push(url)
          if image_meets_criteria?(image)
            list_images << url
          else
            debug("Image discarded: #{url} - height: #{image[:height]} - width: #{image[:width]} - format: #{image[:format]}")
          end
        end

      (list_images.empty? and content != @html) ? images(@html, true) : list_images
    end

    def get_image_size(url)
      begin
        w, h = FastImage.size(url)
        raise "Couldn't get size." if w.nil? || h.nil?
        {width: w, height: h}
      rescue => e
        debug("Image error: #{e}")
        nil
      end
    end<commit_msg>Removed instances of Ruby 1.9 hash syntax<commit_after>require 'rubygems'
require 'nokogiri'
require 'guess_html_encoding'

module Readability
  class Document
    DEFAULT_OPTIONS = {
      :retry_length               => 250,
      :min_text_length            => 25,
      :remove_unlikely_candidates => true,
      :weight_classes             => true,
      :clean_conditionally        => true,
      :remove_empty_nodes         => true,
      :min_image_width            => 130,
      :min_image_height           => 80,
      :ignore_image_format        => []
    }.freeze

    attr_accessor :options, :html, :best_candidate, :candidates, :best_candidate_has_image

    def initialize(input, options = {})
      @options = DEFAULT_OPTIONS.merge(options)
      @input = input

      if RUBY_VERSION =~ /^1\.9\./ && !@options[:encoding]
        @input = GuessHtmlEncoding.encode(@input, @options[:html_headers]) unless @options[:do_not_guess_encoding]
        @options[:encoding] = @input.encoding.to_s
      end

      @input = @input.gsub(REGEXES[:replaceBrsRe], '</p><p>').gsub(REGEXES[:replaceFontsRe], '<\1span>')
      @remove_unlikely_candidates = @options[:remove_unlikely_candidates]
      @weight_classes = @options[:weight_classes]
      @clean_conditionally = @options[:clean_conditionally]
      @best_candidate_has_image = true
      make_html
    end

    def prepare_candidates
      @html.css("script, style").each { |i| i.remove }
      remove_unlikely_candidates! if @remove_unlikely_candidates
      transform_misused_divs_into_paragraphs!
      
      @candidates     = score_paragraphs(options[:min_text_length])
      @best_candidate = select_best_candidate(@candidates)
    end

    def make_html
      @html = Nokogiri::HTML(@input, nil, @options[:encoding])
      # In case document has no body, such as from empty string or redirect
      @html = Nokogiri::HTML('<body />', nil, @options[:encoding]) if @html.css('body').length == 0

      # Remove html comment tags
      @html.xpath('//comment()').each { |i| i.remove }
    end

    def images(content=nil, reload=false)
      begin
        require 'fastimage'
      rescue LoadError
        raise "Please install fastimage in order to use the #images feature."
      end

      @best_candidate_has_image = false if reload

      prepare_candidates
      list_images   = []
      tested_images = []
      content       = @best_candidate[:elem] unless reload

      return list_images if content.nil?
      elements = content.css("img").map(&:attributes)

        elements.each do |element|
          next unless element["src"]

          url     = element["src"].value
          height  = element["height"].nil?  ? 0 : element["height"].value.to_i
          width   = element["width"].nil?   ? 0 : element["width"].value.to_i
          
          if url =~ /\Ahttps?:\/\//i && (height.zero? || width.zero?)
            image   = get_image_size(url) 
            next unless image
          else
            image = {:width => width, :height => height}
          end
          
          image[:format] = File.extname(url).gsub(".", "")
          
          if tested_images.include?(url)
            debug("Image was tested: #{url}")
            next
          end

          tested_images.push(url)
          if image_meets_criteria?(image)
            list_images << url
          else
            debug("Image discarded: #{url} - height: #{image[:height]} - width: #{image[:width]} - format: #{image[:format]}")
          end
        end

      (list_images.empty? and content != @html) ? images(@html, true) : list_images
    end

    def get_image_size(url)
      begin
        w, h = FastImage.size(url)
        raise "Couldn't get size." if w.nil? || h.nil?
        {:width => w, :height => h}
      rescue => e
        debug("Image error: #{e}")
        nil
      end
    end
{'fraction_non_alphanumeric': 0.12366710013003901, 'fraction_numerical': 0.004421326397919376, 'mean_word_length': 2.2952013710368466, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5172545', 'n_tokens_mistral': 2353, 'n_tokens_neox': 2275, 'n_words': 499}