<commit_before># encoding: utf-8

module Nanoc3::Filters
  class ColorizeSyntax < Nanoc3::Filter

    DEFAULT_COLORIZER = :coderay

    def run(content, params={})
      require 'nokogiri'

      # Take colorizers from parameters
      @colorizers = Hash.new(DEFAULT_COLORIZER)
      (params[:colorizers] || {}).each_pair do |language, colorizer|
        @colorizers[language] = colorizer
      end

      # Colorize
      doc = Nokogiri::HTML.fragment(content)
      doc.css('pre > code[class^="lang"]').each do |element|
        language = element['class'][/lang-([^ ]+)/].sub(/^lang-/, '')
        element.parent.replace(
          Nokogiri.make(highlight(element.inner_text, language, params))
        )
      end

      doc.to_s
    end

  private

    KNOWN_COLORIZERS = [ :coderay, :dummy, :pygmentize ]

    def highlight(code, language, params={})
      colorizer = @colorizers[language]
      if KNOWN_COLORIZERS.include?(colorizer)
        send(colorizer, code, language, params[colorizer])<commit_msg>made syntax coloring filter detect code languages more reliably<commit_after># encoding: utf-8

module Nanoc3::Filters
  class ColorizeSyntax < Nanoc3::Filter

    DEFAULT_COLORIZER = :coderay

    def run(content, params={})
      require 'nokogiri'

      # Take colorizers from parameters
      @colorizers = Hash.new(DEFAULT_COLORIZER)
      (params[:colorizers] || {}).each_pair do |language, colorizer|
        @colorizers[language] = colorizer
      end

      # Colorize
      doc = Nokogiri::HTML.fragment(content)
      doc.css('pre > code[class*="lang-"]').each do |element|
        # Get language
        match = element['class'].match(/(^| )lang-([^ ]+)/)
        next if match.nil?
        language = match[2]

        # Highlight
        element.parent.replace(
          Nokogiri.make(highlight(element.inner_text, language, params))
        )
      end

      doc.to_s
    end

  private

    KNOWN_COLORIZERS = [ :coderay, :dummy, :pygmentize ]

    def highlight(code, language, params={})
      colorizer = @colorizers[language]
      if KNOWN_COLORIZERS.include?(colorizer)
        send(colorizer, code, language, params[colorizer])
{'fraction_non_alphanumeric': 0.1260387811634349, 'fraction_numerical': 0.003231763619575254, 'mean_word_length': 2.897482014388489, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '233535', 'n_tokens_mistral': 689, 'n_tokens_neox': 663, 'n_words': 141}