<commit_before>require 'github/markdown'

module GitHub::HTML
  # HTML Filter that converts Markdown text into HTML and converts into a
  # DocumentFragment. This is different from most filters in that it can take a
  # non-HTML as input. It must be used as the first filter in a pipeline.
  #
  # Context options:
  #   :gfm      => false    Disable GFM line-end processing
  #   :autolink => false    Disable autolinking URLs
  #
  # This filter does not write any additional information to the context hash.
  class MarkdownFilter < Filter
    def initialize(text, context={})
      raise TypeError, "text cannot be HTML" if text.is_a?(DocumentFragment)
      @text = text.to_s.gsub("\r", '')
      @context = context
      @doc = nil
    end

    # Convert Markdown to HTML using the best available implementation
    # and convert into a DocumentFragment.
    def call
      flags = [
        :fenced_code, :tables,
        :strikethrough, :lax_htmlblock,
        :gh_blockcode, :no_intraemphasis,
        :spaced_header
      ]
      flags << :autolink if context[:autolink] != false
      flags << :hard_wrap if context[:gfm] != false
      html = GitHub::Markdown.new(@text, *flags).to_html
      @doc = parse_html(html)
    end
  end
end<commit_msg>Off by one<commit_after>require 'github/markdown'

module GitHub::HTML
  # HTML Filter that converts Markdown text into HTML and converts into a
  # DocumentFragment. This is different from most filters in that it can take a
  # non-HTML as input. It must be used as the first filter in a pipeline.
  #
  # Context options:
  #   :gfm      => false    Disable GFM line-end processing
  #   :autolink => false    Disable autolinking URLs
  #
  # This filter does not write any additional information to the context hash.
  class MarkdownFilter < Filter
    def initialize(text, context={})
      raise TypeError, "text cannot be HTML" if text.is_a?(DocumentFragment)
      @text = text.to_s.gsub("\r", '')
      @context = context
      @doc = nil
    end

    # Convert Markdown to HTML using the best available implementation
    # and convert into a DocumentFragment.
    def call
      flags = [
        :fenced_code, :tables,
        :strikethrough, :lax_htmlblock,
        :gh_blockcode, :no_intraemphasis,
        :space_header
      ]
      flags << :autolink if context[:autolink] != false
      flags << :hard_wrap if context[:gfm] != false
      html = GitHub::Markdown.new(@text, *flags).to_html
      @doc = parse_html(html)
    end
  end
end
{'fraction_non_alphanumeric': 0.08678343949044585, 'fraction_numerical': 0.0, 'mean_word_length': 2.836641221374046, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5544656', 'n_tokens_mistral': 748, 'n_tokens_neox': 729, 'n_words': 265}