<commit_before>module MdInc
  module Commands
    public :instance_eval

    class << self
      def root(path)
        @root = path
      end

      def full_path(path)
        @root ? File.join(@root, path) : path
      end

      def x(*args)
        []
      end

      def inc(path)
        lines = File.readlines(full_path(path))
        lines.map &:rstrip!
      end

      def code_inc(path, language='', re1=nil, re2=nil)
        if re1
          code(language, between(re1, re2, inc(path)))
        else
          code(language, inc(path))
        end
      end

      def code(language, lines)
        ["```#{language}"] + lines + ["```"]
      end

      def between(re1, re2, lines)
        state = :outside
        output = []
        lines.each do |l|
          if state == :outside && re1 =~ l
            state = :inside
          elsif state == :inside && re2 =~ l
            state = :outside
          else
            output << l if state==:inside
          end
        end
        STDERR.puts "Warning: no output from included file" if output.empty?
        output
      end

      def skip(re, lines)
        output = []
        lines.each do |l|
          output << l unless l =~ re
        end<commit_msg>Bring back straight markdown code blocks<commit_after>module MdInc
  module Commands
    public :instance_eval

    class << self
      def root(path)
        @root = path
      end

      def full_path(path)
        @root ? File.join(@root, path) : path
      end

      def x(*args)
        []
      end

      def inc(path)
        lines = File.readlines(full_path(path))
        lines.map &:rstrip!
      end

      def code_inc(path, language=nil, re1=nil, re2=nil)
        if re1
          code(language, between(re1, re2, inc(path)))
        else
          code(language, inc(path))
        end
      end

      def code(language, lines)
        unless language.nil?
          ["```#{language}"] + lines + ["```"]
        else
          lines.map {|l| l.rstrip.prepend('    ')}
        end
      end

      def between(re1, re2, lines)
        state = :outside
        output = []
        lines.each do |l|
          if state == :outside && re1 =~ l
            state = :inside
          elsif state == :inside && re2 =~ l
            state = :outside
          else
            output << l if state==:inside
          end
        end
        STDERR.puts "Warning: no output from included file" if output.empty?
        output
      end

      def skip(re, lines)
        output = []
        lines.each do |l|
          output << l unless l =~ re
        end
{'fraction_non_alphanumeric': 0.10982658959537572, 'fraction_numerical': 0.006936416184971098, 'mean_word_length': 1.4747378455672069, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '493653', 'n_tokens_mistral': 806, 'n_tokens_neox': 768, 'n_words': 225}