<commit_before>require 'nokogiri'

module Markdevn
  class << self
    def to_md(doc)
      @doc = Nokogiri::XML(doc)
      to_md_single(@doc.css("en-note"))
    end

    def sanitize(node, opts = { :keep => [] })
      node.attributes.each{|k,_|node.remove_attribute(k) unless opts[:keep].include?(k)}
    end

    def to_md_single(fragment, list_type = :ul, list_depth = -1)
      def preserve(node)
        node.inner_html = to_md_single(node.children)
        node.to_html
      end

      list_index = 1

      fragment.map.with_index do |node, i|
        case node.name
        when "en-note", "div"
          to_md_single(node.children) + "\n\n"
        when "ol"
          to_md_single(node.children, :ol, list_depth + 1) + "\n\n"
        when "ul"
          to_md_single(node.children, :ul, list_depth + 1) + "\n\n"
        when "li"
          spaces = list_index == 1 ? "" : " " * (list_depth * 2)
          prefix = list_type == :ul ? "* " : "#{list_index}. "
          list_index += 1
          spaces + prefix + to_md_single(node.children, list_type, list_depth) + "\n"
        when "span"
          if node.attr("style").nil?
            to_md_single(node.children)
          else
            preserve(node)
          end
        when "br"
          ""
        when "en-todo"
          if node.attr("checked") == "true"
            "[x]"
          else
            "[ ]"
          end
        when "sup", "sub"
          preserve(node)
        when "strong"
          "**" + to_md_single(node.children) + "**"<commit_msg>handle paragraphs<commit_after>require 'nokogiri'

module Markdevn
  class << self
    def to_md(doc)
      @doc = Nokogiri::XML(doc)
      to_md_single(@doc.css("en-note"))
    end

    def sanitize(node, opts = { :keep => [] })
      node.attributes.each{|k,_|node.remove_attribute(k) unless opts[:keep].include?(k)}
    end

    def to_md_single(fragment, list_type = :ul, list_depth = -1)
      def preserve(node)
        node.inner_html = to_md_single(node.children)
        node.to_html
      end

      list_index = 1

      fragment.map.with_index do |node, i|
        case node.name
        when "en-note", "div", "p"
          to_md_single(node.children) + "\n\n"
        when "ol"
          to_md_single(node.children, :ol, list_depth + 1) + "\n\n"
        when "ul"
          to_md_single(node.children, :ul, list_depth + 1) + "\n\n"
        when "li"
          spaces = list_index == 1 ? "" : " " * (list_depth * 2)
          prefix = list_type == :ul ? "* " : "#{list_index}. "
          list_index += 1
          spaces + prefix + to_md_single(node.children, list_type, list_depth) + "\n"
        when "span"
          if node.attr("style").nil?
            to_md_single(node.children)
          else
            preserve(node)
          end
        when "br"
          ""
        when "en-todo"
          if node.attr("checked") == "true"
            "[x]"
          else
            "[ ]"
          end
        when "sup", "sub"
          preserve(node)
        when "strong"
          "**" + to_md_single(node.children) + "**"
{'fraction_non_alphanumeric': 0.14387788242936017, 'fraction_numerical': 0.004546930821695355, 'mean_word_length': 1.7873303167420815, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7019267', 'n_tokens_mistral': 1052, 'n_tokens_neox': 1026, 'n_words': 208}