<commit_before>require 'nokogiri'
require 'tidy'

module RTF::Converters
  class HTML

    def initialize(html, options = {})
      html  = options[:noclean] ? html : clean(html, options[:tidy_options] || {})
      puts html
      @html = Nokogiri::HTML::Document.parse(html)
    end

    def to_rtf(options = {})
      to_rtf_document(options).to_rtf
    end

    def to_rtf_document(options = {})
      font  = Helpers.font(options[:font] || :default)
      nodes = NodeSet.new @html.css('body').children

      RTF::Document.new(font).tap do |rtf|
        nodes.to_rtf(rtf)
      end
    end

    protected
      def clean(html, options = {})
        defaults = {
          :doctype          => 'omit',
          :bare             => true,
          :clean            => true,
          :drop_empty_paras => true,<commit_msg>Whoops - remove debug puts - sorry :)<commit_after>require 'nokogiri'
require 'tidy'

module RTF::Converters
  class HTML

    def initialize(html, options = {})
      html  = options[:noclean] ? html : clean(html, options[:tidy_options] || {})
      @html = Nokogiri::HTML::Document.parse(html)
    end

    def to_rtf(options = {})
      to_rtf_document(options).to_rtf
    end

    def to_rtf_document(options = {})
      font  = Helpers.font(options[:font] || :default)
      nodes = NodeSet.new @html.css('body').children

      RTF::Document.new(font).tap do |rtf|
        nodes.to_rtf(rtf)
      end
    end

    protected
      def clean(html, options = {})
        defaults = {
          :doctype          => 'omit',
          :bare             => true,
          :clean            => true,
          :drop_empty_paras => true,
{'fraction_non_alphanumeric': 0.13341346153846154, 'fraction_numerical': 0.0, 'mean_word_length': 2.153409090909091, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1974120', 'n_tokens_mistral': 547, 'n_tokens_neox': 543, 'n_words': 106}