<commit_before>require 'reverse_markdown'
require 'descriptive_statistics'

class WordToMarkdown

  HEADING_DEPTH = 6 # Number of headings to guess, e.g., h6
  HEADING_STEP = 100/HEADING_DEPTH
  LI_SELECTORS = %w[
    MsoListParagraphCxSpFirst
    MsoListParagraphCxSpMiddle
    MsoListParagraphCxSpLast
  ]

  attr_reader :path, :doc, :html

  # Create a new WordToMarkdown object
  #
  # input - a HTML string or path to an HTML file
  #
  # Returns the WordToMarkdown object
  def initialize(input)
    path = File.expand_path input, Dir.pwd
    if File.exist?(path)
      @html = File.open(path).read
      @path = path
    else
      @path = String
      @html = input
    end
    @html = @html.force_encoding('iso8859-1').encode("UTF-8", :invalid => :replace, :replace => "")
    @doc = Nokogiri::HTML @html
    semanticize!
  end

  def inspect
    "<WordToMarkdown path=\"#{@path}\">"
  end

  def to_s
    @markdown ||= scrub_whitespace(ReverseMarkdown.parse(@doc.to_html))
  end

  def scrub_whitespace(string)
    string.sub!(/\A[[:space:]]+/,'')                # leading whitespace
    string.sub!(/[[:space:]]+\z/,'')                # trailing whitespace
    string.gsub!(/\n\n \n\n/,"\n\n")                # Quadruple line breaks
    string.gsub!(/^([0-9]+)\.[[:space:]]*/,"\\1. ") # Numbered lists
    string.gsub!(/^-[[:space:]á]*/,"- ")             # Unnumbered lists
    string
  end

  # Returns an array of Nokogiri nodes that are implicit headings
  def implicit_headings
    @implicit_headings ||= begin
      headings = []
      @doc.css("[style]").each do |element|
        headings.push element unless element.font_size.nil?
      end
      headings
    end
  end

  # Returns an array of font-sizes for implicit headings in the document
  def font_sizes
    @font_sizes ||= begin
      sizes = []
      implicit_headings.each { |element| sizes.push element.font_size }
      sizes
    end<commit_msg>smarter encoding support, fixes #1<commit_after>require 'reverse_markdown'
require 'descriptive_statistics'

class WordToMarkdown

  HEADING_DEPTH = 6 # Number of headings to guess, e.g., h6
  HEADING_STEP = 100/HEADING_DEPTH
  LI_SELECTORS = %w[
    MsoListParagraphCxSpFirst
    MsoListParagraphCxSpMiddle
    MsoListParagraphCxSpLast
  ]

  attr_reader :path, :doc, :html

  # Create a new WordToMarkdown object
  #
  # input - a HTML string or path to an HTML file
  #
  # Returns the WordToMarkdown object
  def initialize(input)
    path = File.expand_path input, Dir.pwd
    if File.exist?(path)
      @html = File.open(path).read
      @path = path
    else
      @path = String
      @html = input.to_s
    end
    @html = @html.force_encoding(encoding).encode("UTF-8", :invalid => :replace, :replace => "")
    @doc = Nokogiri::HTML @html
    semanticize!
  end

  def inspect
    "<WordToMarkdown path=\"#{@path}\">"
  end

  def to_s
    @markdown ||= scrub_whitespace(ReverseMarkdown.parse(@doc.to_html))
  end

  def encoding
    match = @html.encode("UTF-8", :invalid => :replace, :replace => "").match(/charset=([^\"]+)/)
    if match
      match[1].sub("macintosh", "MacRoman")
    else
      "UTF-8"
    end
  end

  def scrub_whitespace(string)
    string.sub!(/\A[[:space:]]+/,'')                # leading whitespace
    string.sub!(/[[:space:]]+\z/,'')                # trailing whitespace
    string.gsub!(/\n\n \n\n/,"\n\n")                # Quadruple line breaks
    string.gsub!(/^([0-9]+)\.[[:space:]]*/,"\\1. ") # Numbered lists
    string.gsub!(/^-[[:space:]·]*/,"- ")            # Unnumbered lists
    string
  end

  # Returns an array of Nokogiri nodes that are implicit headings
  def implicit_headings
    @implicit_headings ||= begin
      headings = []
      @doc.css("[style]").each do |element|
        headings.push element unless element.font_size.nil?
      end
      headings
    end
  end

  # Returns an array of font-sizes for implicit headings in the document
  def font_sizes
    @font_sizes ||= begin
      sizes = []
      implicit_headings.each { |element| sizes.push element.font_size }
      sizes
    end
{'fraction_non_alphanumeric': 0.13400293973542382, 'fraction_numerical': 0.006614404703576678, 'mean_word_length': 2.6916817359855334, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5250977', 'n_tokens_mistral': 1430, 'n_tokens_neox': 1372, 'n_words': 323}