<commit_before>module ImageScraper
  class Client
    attr_accessor :url, :convert_to_absolute_url, :include_css_images, :include_css_data_images, :doc

    def initialize(url,options={})
      options.reverse_merge!(:convert_to_absolute_url=>true,:include_css_images=>true, :include_css_data_images=>false)
      @url = URI.escape(url)
      @convert_to_absolute_url = options[:convert_to_absolute_url]
      @include_css_images = options[:include_css_images]
      @include_css_data_images = options[:include_css_data_images]
      html = open(@url).read rescue nil
      @doc = html ? Nokogiri::HTML(html) : nil
    end

    def image_urls
      images = page_images
      images += stylesheet_images if include_css_images
      images
    end

    def page_images
      urls = []
      return urls if doc.blank?
      doc.xpath("//img").each do |img|
        next if img["src"].blank?
        image = URI.escape(img["src"].strip)
        image = ImageScraper::Util.absolute_url(url,image) if convert_to_absolute_url
        urls << image
      end
      urls
    end

    def stylesheet_images
      images = []
      stylesheets.each do |stylesheet|
        file = open(stylesheet)
        css = file.string rescue IO.read(file)

        images += css.scan(/url\((.*?)\)/).collect do |image_url|
          image_url = URI.escape image_url[0]
          if image_url.include?("data:image") and @include_css_data_images
            image_url
          else
            image_url = ImageScraper::Util.strip_quotes(image_url)
            @convert_to_absolute_url ? ImageScraper::Util.absolute_url(stylesheet, image_url) : image_url
          end
        end
      end
      images
    end

    def stylesheets
      return [] if doc.blank?
      doc.xpath('//link[@rel="stylesheet"]').collect do |stylesheet|
        ImageScraper::Util.absolute_url url, URI.escape(stylesheet['href'])
      end
    end
  end
end<commit_msg>don't crash if we get a 404 when trying to fetch a stylesheet.<commit_after>module ImageScraper
  class Client
    attr_accessor :url, :convert_to_absolute_url, :include_css_images, :include_css_data_images, :doc

    def initialize(url,options={})
      options.reverse_merge!(:convert_to_absolute_url=>true,:include_css_images=>true, :include_css_data_images=>false)
      @url = URI.escape(url)
      @convert_to_absolute_url = options[:convert_to_absolute_url]
      @include_css_images = options[:include_css_images]
      @include_css_data_images = options[:include_css_data_images]
      html = open(@url).read rescue nil
      @doc = html ? Nokogiri::HTML(html) : nil
    end

    def image_urls
      images = page_images
      images += stylesheet_images if include_css_images
      images
    end

    def page_images
      urls = []
      return urls if doc.blank?
      doc.xpath("//img").each do |img|
        next if img["src"].blank?
        image = URI.escape(img["src"].strip)
        image = ImageScraper::Util.absolute_url(url,image) if convert_to_absolute_url
        urls << image
      end
      urls
    end

    def stylesheet_images
      images = []
      stylesheets.each do |stylesheet|
        file = open(stylesheet) rescue next
        css = file.string rescue IO.read(file) rescue next

        images += css.scan(/url\((.*?)\)/).collect do |image_url|
          image_url = URI.escape image_url[0]
          if image_url.include?("data:image") and @include_css_data_images
            image_url
          else
            image_url = ImageScraper::Util.strip_quotes(image_url)
            @convert_to_absolute_url ? ImageScraper::Util.absolute_url(stylesheet, image_url) : image_url
          end
        end
      end
      images
    end

    def stylesheets
      return [] if doc.blank?
      doc.xpath('//link[@rel="stylesheet"]').collect do |stylesheet|
        ImageScraper::Util.absolute_url url, URI.escape(stylesheet['href'])
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.102014792144861, 'fraction_numerical': 0.0012751849018107625, 'mean_word_length': 2.800387596899225, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3191655', 'n_tokens_mistral': 1219, 'n_tokens_neox': 1200, 'n_words': 247}