<commit_before># Taken and modified from the Jekyll project under the MIT License
# https://github.com/jekyll/jekyll/blob/6849d6a/lib/jekyll/converters/markdown/redcarpet_parser.rb
# https://github.com/jekyll/jekyll/blob/6849d6a/LICENSE
require 'redcarpet'
require 'rouge'
require 'rouge/plugins/redcarpet'

module DocRepo
  module Converters
    class MarkdownParser
      class RougeRenderer < Redcarpet::Render::HTML
        include Rouge::Plugins::Redcarpet

        def add_code_tags(code, lang)
          code = code.to_s
          code = code.sub(
            /<pre>/,
            "<pre><code class=\"language-#{lang}\" data-lang=\"#{lang}\">"
          )
          code = code.sub(/<\/pre>/, "</code></pre>")
        end

        def block_code(code, lang)
          code = "<pre>#{super}</pre>"
          "<div class=\"highlight\">#{add_code_tags(code, lang)}</div>"
        end

      protected

        def rouge_formatter(opts = {})
          Rouge::Formatters::HTML.new(opts.merge(wrap: false))
        end
      end

      def initialize(config)
        @config = config
        @extensions = config.fetch(:extensions, [])
          .each_with_object({}){ |e, h| h[e.to_sym] = true }
        @extensions[:fenced_code_blocks] ||= !@extensions.fetch(:no_fenced_code_blocks, false)
        @extensions[:fenced_code_blocks] ||= !@extensions[:no_fenced_code_blocks]

        render_class = config.fetch(:render_with, RougeRenderer)
        @renderer = render_class.new(@extensions)
        if extensions.fetch(:smart, false)
          @renderer.extend Redcarpet::Render::SmartyPants
        end
      end

      def convert(content)
        markdown = Redcarpet::Markdown.new(renderer, extensions)
        markdown.render(content)
      end

      attr_reader :config, :extensions, :renderer
      private :config, :extensions, :renderer
    end
  end
end<commit_msg>Fix rouge's formatter options<commit_after># Taken and modified from the Jekyll project under the MIT License
# https://github.com/jekyll/jekyll/blob/6849d6a/lib/jekyll/converters/markdown/redcarpet_parser.rb
# https://github.com/jekyll/jekyll/blob/6849d6a/LICENSE
require 'redcarpet'
require 'rouge'
require 'rouge/plugins/redcarpet'

module DocRepo
  module Converters
    class MarkdownParser
      class RougeRenderer < Redcarpet::Render::HTML
        include Rouge::Plugins::Redcarpet

        def add_code_tags(code, lang)
          code = code.to_s
          code = code.sub(
            /<pre>/,
            "<pre><code class=\"language-#{lang}\" data-lang=\"#{lang}\">"
          )
          code = code.sub(/<\/pre>/, "</code></pre>")
        end

        def block_code(code, lang)
          code = "<pre>#{super}</pre>"
          "<div class=\"highlight\">#{add_code_tags(code, lang)}</div>"
        end

      protected

        def rouge_formatter(lexer)
          Rouge::Formatters::HTML.new(wrap: false)
        end

      end

      def initialize(config)
        @config = config
        @extensions = config.fetch(:extensions, [])
          .each_with_object({}){ |e, h| h[e.to_sym] = true }
        @extensions[:fenced_code_blocks] ||= !@extensions.fetch(:no_fenced_code_blocks, false)
        @extensions[:fenced_code_blocks] ||= !@extensions[:no_fenced_code_blocks]

        render_class = config.fetch(:render_with, RougeRenderer)
        @renderer = render_class.new(@extensions)
        if extensions.fetch(:smart, false)
          @renderer.extend Redcarpet::Render::SmartyPants
        end
      end

      def convert(content)
        markdown = Redcarpet::Markdown.new(renderer, extensions)
        markdown.render(content)
      end

      attr_reader :config, :extensions, :renderer
      private :config, :extensions, :renderer
    end
  end
end
{'fraction_non_alphanumeric': 0.13136157740474289, 'fraction_numerical': 0.005329070077271516, 'mean_word_length': 2.68762278978389, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 23, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '240742', 'n_tokens_mistral': 1191, 'n_tokens_neox': 1159, 'n_words': 218}