<commit_before>require "peg/version"

module Peg
  class ParseError < StandardError; end

  class Rule
    attr_accessor :action, :name
    attr_reader :value

    def initialize(name: nil, &action)
      @name = name
      @action = action
    end

    def bindings
      if @name && @value
        {@name => @value}
      else
        {}
      end
    end

    def =~(other)
      match(other)
    end

    def ===(other)
      match(other)
    end
  end

  class Literal < Rule
    def initialize(s, **options, &action)
      super(**options, &action)
      @s = s
      @value = nil
    end

    def match(g, lookup_context = NullContext.new)
      return nil unless g.input.start_with?(@s)

      g.advance(@s.size)
      @value = @s

      if @action
        @value = @action.call(**bindings)
      end

      @value
    end
  end

  class VariableLookup
    def initialize(name)
      @name = name
    end

    def lookup(context)
      context.lookup(@name)
    end
  end

  class NullContext
    def lookup(name)
      nil
    end

    def merge(seq)
      SequenceContext.new(seq, self)
    end
  end

  class SequenceContext
    def initialize(seq, parent)
      @seq = seq
      @parent = parent
    end

    def merge(seq)
      SequenceContext.new(seq, self)
    end<commit_msg>SequenceContext => LookupContext<commit_after>require "peg/version"

module Peg
  class ParseError < StandardError; end

  class Rule
    attr_accessor :action, :name
    attr_reader :value

    def initialize(name: nil, &action)
      @name = name
      @action = action
    end

    def bindings
      if @name && @value
        {@name => @value}
      else
        {}
      end
    end

    def =~(other)
      match(other)
    end

    def ===(other)
      match(other)
    end
  end

  class Literal < Rule
    def initialize(s, **options, &action)
      super(**options, &action)
      @s = s
      @value = nil
    end

    def match(g, lookup_context = NullContext.new)
      return nil unless g.input.start_with?(@s)

      g.advance(@s.size)
      @value = @s

      if @action
        @value = @action.call(**bindings)
      end

      @value
    end
  end

  class VariableLookup
    def initialize(name)
      @name = name
    end

    def lookup(context)
      context.lookup(@name)
    end
  end

  class NullContext
    def lookup(name)
      nil
    end

    def merge(seq)
      LookupContext.new(seq, self)
    end
  end

  class LookupContext
    def initialize(seq, parent)
      @seq = seq
      @parent = parent
    end

    def merge(seq)
      LookupContext.new(seq, self)
    end
{'fraction_non_alphanumeric': 0.09626492106276473, 'fraction_numerical': 0.0, 'mean_word_length': 1.9289740698985345, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4836894', 'n_tokens_mistral': 860, 'n_tokens_neox': 814, 'n_words': 230}