<commit_before>
require 'friendly_format/adapter/abstract'
require 'nokogiri'

module FriendlyFormat
  class NokogiriAdapter
    extend Abstract

    def self.parse html
      # root is html, children is [body], first is body
      # same as libxml
      # drop zzz with .children.first since it would wrap a tag p for the article
      Nokogiri::HTML.parse("<zzz>#{html}</zzz>").root.children.first.children.first
    end

  end # of NokogiriAdapter
end # of FriendlyFormat
<commit_msg>fixed nokogiri encoding error in ruby 1.9.1<commit_after>
require 'friendly_format/adapter/abstract'
require 'nokogiri'

module FriendlyFormat
  class NokogiriAdapter
    extend Abstract

    def self.parse html
      # root is html, children is [body], first is body
      # same as libxml
      # drop zzz with .children.first since it would wrap a tag p for the article
      Nokogiri::HTML.parse(
        "<zzz>#{html}</zzz>",
        nil, # url?
        html.respond_to?(:encoding) ? html.encoding.name : nil
        ).root.children.first.children.first
    end

  end # of NokogiriAdapter
end # of FriendlyFormat

{'fraction_non_alphanumeric': 0.08770343580470162, 'fraction_numerical': 0.0027124773960217, 'mean_word_length': 3.1, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1208064', 'n_tokens_mistral': 343, 'n_tokens_neox': 339, 'n_words': 109}