<commit_before>module Kor
  module Input
    class Markdown < Base
      def parse(opt)
        opt.on("--key=KEY", "filter key set (e.g. foo,bar,baz)") do |arg|
          @filter_key = arg
        end
      end

      def head
        line = io.gets
        raise ReadError unless line
        line.strip!
        first_index = line[0] == '|' ? 1 : 0
        keys = line.split('|')[first_index..-1].map(&:strip)
        # skip separate line
        io.gets
        if @filter_key
          filter_keys = @filter_key.split(",")
          filter_keys = keys.select { |key|
            filter_keys.include?(key)
          }
          keys = filter_keys
          @filter_key = filter_keys.map { |key| keys.index(key) }
        end
        keys
      end

      def gets
        if line = io.gets
          first_index = line[0] == '|' ? 1 : 0
          body = line.strip.split('|')[first_index..-1].map(&:strip)
          if @filter_key
            return @filter_key.map{ |index| body[index] }
          end
          body
        else
          nil
        end
      end
    end
  end
end
<commit_msg>Add message<commit_after>module Kor
  module Input
    class Markdown < Base
      def parse(opt)
        opt.on("--key=KEY", "filter key set (e.g. foo,bar,baz)") do |arg|
          @filter_key = arg
        end
      end

      def head
        line = io.gets
        raise ReadError, "cannot get markdown header" unless line
        line.strip!
        first_index = line[0] == '|' ? 1 : 0
        keys = line.split('|')[first_index..-1].map(&:strip)
        # skip separate line
        io.gets
        if @filter_key
          filter_keys = @filter_key.split(",")
          filter_keys = keys.select { |key|
            filter_keys.include?(key)
          }
          keys = filter_keys
          @filter_key = filter_keys.map { |key| keys.index(key) }
        end
        keys
      end

      def gets
        if line = io.gets
          first_index = line[0] == '|' ? 1 : 0
          body = line.strip.split('|')[first_index..-1].map(&:strip)
          if @filter_key
            return @filter_key.map{ |index| body[index] }
          end
          body
        else
          nil
        end
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.11704035874439461, 'fraction_numerical': 0.007174887892376682, 'mean_word_length': 1.467920353982301, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2327937', 'n_tokens_mistral': 709, 'n_tokens_neox': 697, 'n_words': 172}