<commit_before>require 'open-uri'
require 'css_parser'

module CSSInliner
  class Extractor
    attr_reader :document

    def initialize(document, directory)
      @document, @basedir = document, directory
    end

    def extract(remove_link_and_style = true)
      sources = []
      sources.concat extract_from_link(remove_link_and_style)
      sources.concat extract_from_style(remove_link_and_style)
      integrate! sources
    end

    def extract_from_link(remove_link_element = true)
      @document.css('link[rel="stylesheet"]').inject([]) do |sources, link|
        begin
          open(File.join(basedir, link['href'])) {|f| sources << f.read}
        rescue Errno::ENOENT
          warn File.join(basedir, link['href']) + 'not found'
        end
        link.remove if remove_link_element
        sources
      end
    end

    def extract_from_style(remove_style_element = true)
      @document.css('style').inject([]) do |sources, style|
        sources << style.content
        style.remove if remove_style_element
        sources
      end
    end

    def integrate!(sources)
      parser = CssParser::Parser.new
      parser.add_block!(sources * $/)
      rule_sets = parser.enum_for :each_rule_set
      i = 0
      rule_sets = rule_sets.sort_by {|rs| [rs.specificity, i += 1]}
      blank_rule_set = CssParser::RuleSet.new(nil, nil)
      rule_sets.inject(Hash.new(blank_rule_set)) do |rules, rs|
        sel = rs.selectors * ','
        rules[sel] = CssParser.merge rules[sel], rs
        rules
      end
    end

    def basedir
      base = @document.css('base')
      base.empty? ? @basedir : base[0]['href']
    end
  end
end<commit_msg>Extractor#integrate! -> integrate, it's not destructive any more<commit_after>require 'open-uri'
require 'css_parser'

module CSSInliner
  class Extractor
    attr_reader :document

    def initialize(document, directory)
      @document, @basedir = document, directory
    end

    def extract(remove_link_and_style = true)
      sources = []
      sources.concat extract_from_link(remove_link_and_style)
      sources.concat extract_from_style(remove_link_and_style)
      integrate sources
    end

    def extract_from_link(remove_link_element = true)
      @document.css('link[rel="stylesheet"]').inject([]) do |sources, link|
        begin
          open(File.join(basedir, link['href'])) {|f| sources << f.read}
        rescue Errno::ENOENT
          warn File.join(basedir, link['href']) + 'not found'
        end
        link.remove if remove_link_element
        sources
      end
    end

    def extract_from_style(remove_style_element = true)
      @document.css('style').inject([]) do |sources, style|
        sources << style.content
        style.remove if remove_style_element
        sources
      end
    end

    def integrate(sources)
      parser = CssParser::Parser.new
      parser.add_block!(sources * $/)
      rule_sets = parser.enum_for :each_rule_set
      i = 0
      rule_sets = rule_sets.sort_by {|rs| [rs.specificity, i += 1]}
      blank_rule_set = CssParser::RuleSet.new(nil, nil)
      rule_sets.inject(Hash.new(blank_rule_set)) do |rules, rs|
        sel = rs.selectors * ','
        rules[sel] = CssParser.merge rules[sel], rs
        rules
      end
    end

    def basedir
      base = @document.css('base')
      base.empty? ? @basedir : base[0]['href']
    end
  end
end
{'fraction_non_alphanumeric': 0.10838021912940479, 'fraction_numerical': 0.0017767249037607344, 'mean_word_length': 2.740863787375415, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '638097', 'n_tokens_mistral': 1058, 'n_tokens_neox': 1039, 'n_words': 231}