<commit_before>require "parser_combinator/version"

class ParserCombinator
  ParserCombinatorError = Class.new(RuntimeError)

  class Ok
    attr_reader :parsed, :rest
    def initialize(parsed, rest)
      @parsed, @rest = parsed, rest
    end
  end

  class Fail
    attr_reader :status
    def initialize(status=nil)
      @status = status
    end
  end

  class StandardFailStatus
    attr_reader :message, :rest
    def initialize(message, rest)
      @message, @rest = message, rest
    end
  end

  class Item
    attr_reader :item, :tag

    def initialize(item, tag)
      @item, @tag = item, tag
    end

    def to_s
      item.to_s
    end

    def inspect
      "Item {item = #{item}, tag = #{tag}}"
    end
  end

  class Items < Array
    def head
      self.first
    end

    def rest
      self.drop(1)
    end

    def to_s
      self.map(&:to_s).join
    end

    def inspect
      "Items [#{self.map(&:inspect).join(",\n")}]"
    end
  end

  class ParsedSeq
    def initialize(seq)
      @seq = seq
    end

    def to_a
      @seq.map{|e| e[:entity]}
    end

    def to_h
      @seq.select{|e| e[:name]}.map{|e| [e[:name], e[:entity]]}.to_h
    end

    def self.empty
      new([])
    end

    def cons(entity, name)
      self.class.new([{:entity => entity, :name => name}] + @seq)
    end

    def [](key)
      case key
      when Integer
        if 0 <= key && key < @seq.length
          @seq[key][:entity]
        else
          raise "out of bounds for ParsedSeq"
        end
      else
        if e = @seq.find{|e| e[:name] == key}
          e[:entity]
        else
          raise "key #{key} is not found in ParsedSeq"<commit_msg>modified Array#to_h -> Hash[Array] to allow ruby2.0<commit_after>require "parser_combinator/version"

class ParserCombinator
  ParserCombinatorError = Class.new(RuntimeError)

  class Ok
    attr_reader :parsed, :rest
    def initialize(parsed, rest)
      @parsed, @rest = parsed, rest
    end
  end

  class Fail
    attr_reader :status
    def initialize(status=nil)
      @status = status
    end
  end

  class StandardFailStatus
    attr_reader :message, :rest
    def initialize(message, rest)
      @message, @rest = message, rest
    end
  end

  class Item
    attr_reader :item, :tag

    def initialize(item, tag)
      @item, @tag = item, tag
    end

    def to_s
      item.to_s
    end

    def inspect
      "Item {item = #{item}, tag = #{tag}}"
    end
  end

  class Items < Array
    def head
      self.first
    end

    def rest
      self.drop(1)
    end

    def to_s
      self.map(&:to_s).join
    end

    def inspect
      "Items [#{self.map(&:inspect).join(",\n")}]"
    end
  end

  class ParsedSeq
    def initialize(seq)
      @seq = seq
    end

    def to_a
      @seq.map{|e| e[:entity]}
    end

    def to_h
      Hash[@seq.select{|e| e[:name]}.map{|e| [e[:name], e[:entity]]}]
    end

    def self.empty
      new([])
    end

    def cons(entity, name)
      self.class.new([{:entity => entity, :name => name}] + @seq)
    end

    def [](key)
      case key
      when Integer
        if 0 <= key && key < @seq.length
          @seq[key][:entity]
        else
          raise "out of bounds for ParsedSeq"
        end
      else
        if e = @seq.find{|e| e[:name] == key}
          e[:entity]
        else
          raise "key #{key} is not found in ParsedSeq"
{'fraction_non_alphanumeric': 0.11717591219222782, 'fraction_numerical': 0.0017798872738059924, 'mean_word_length': 1.9683098591549295, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4705334', 'n_tokens_mistral': 1174, 'n_tokens_neox': 1132, 'n_words': 322}