<commit_before># coding: utf-8

module Erudite
  # Parses IRB output into examples.
  class Parser
    def self.parse(string)
      group(string).map { |lines| exemplify(lines) }
    end

    def self.group(string)
      buffer = []

      groups = string.each_line.each_with_object([]) do |line, array|
        if line.start_with?('>> ') && !buffer.empty?
          array.push(buffer)
          buffer = []
        end

        buffer.push(line)
      end

      buffer.empty? ? groups : groups.push(buffer)
    end

    def self.exemplify(lines)
      source = extract_source(lines)
      result = extract_result(lines)
      output = extract_output(lines)

      Example.new(source, result, output)
    end

    def self.extract_source(lines)
      source = lines
        .select { |line| line.start_with?('>> ', '.. ') }
        .map { |line| line[3..-1].chomp }
      source.join("\n") unless source.empty?
    end

    def self.extract_result(lines)
      result = lines
        .select { |line| line.start_with?('=> ') }
        .map { |line| line[3..-1].chomp }
      result.join("\n") unless result.empty?
    end

    def self.extract_output(lines)
      output = lines
        .reject { |line| line.start_with?('>> ', '.. ', '=> ') }
        .map { |line| line.chomp }
      output.join("\n") unless output.empty?
    end
  end
end<commit_msg>Fix RuboCop warning<commit_after># coding: utf-8

module Erudite
  # Parses IRB output into examples.
  class Parser
    def self.parse(string)
      group(string).map { |lines| exemplify(lines) }
    end

    def self.group(string)
      buffer = []

      groups = string.each_line.each_with_object([]) do |line, array|
        if line.start_with?('>> ') && !buffer.empty?
          array.push(buffer)
          buffer = []
        end

        buffer.push(line)
      end

      buffer.empty? ? groups : groups.push(buffer)
    end

    def self.exemplify(lines)
      source = extract_source(lines)
      result = extract_result(lines)
      output = extract_output(lines)

      Example.new(source, result, output)
    end

    def self.extract_source(lines)
      source = lines
        .select { |line| line.start_with?('>> ', '.. ') }
        .map { |line| line[3..-1].chomp }
      source.join("\n") unless source.empty?
    end

    def self.extract_result(lines)
      result = lines
        .select { |line| line.start_with?('=> ') }
        .map { |line| line[3..-1].chomp }
      result.join("\n") unless result.empty?
    end

    def self.extract_output(lines)
      output = lines
        .reject { |line| line.start_with?('>> ', '.. ', '=> ') }
        .map(&:chomp)
      output.join("\n") unless output.empty?
    end
  end
end
{'fraction_non_alphanumeric': 0.1477230655312847, 'fraction_numerical': 0.0037023324694557573, 'mean_word_length': 2.171361502347418, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2843075', 'n_tokens_mistral': 887, 'n_tokens_neox': 868, 'n_words': 187}