<commit_before>class State
  attr_reader :library, :text, :length, :weight

  def initialize(path, lib)
    @library = lib
    @path = path
    @text = ''
    @length = 0
    @weight = 1

    path.each do |rule|
     # puts item.inspect
      @text += rule.symbol
      @length += rule.length
      @weight *= rule.weight
    end
  end

  def append(rule)
    State.new @path + [rule], State.get_library(rule ? rule.symbol : '', @library)
  end

  def failure?
    not @library
  end

  def success?
    @library.value
  end

  def ==(sec)
    self.text == sec.text && self.length == sec.length && self.weight == sec.weight && self.library == sec.library
  end

  class << self<commit_msg>Update state.rb<commit_after>class State
  attr_reader :library, :text, :length, :weight

  def initialize(path, lib)
    @library = lib
    @path = path
    @text = ''
    @length = 0
    @weight = 1

    path.each_index do |index|
     # puts item.inspect
      rule = path[index]
      @text += rule.symbol
      @length += rule.length
      weight_multiplier = if index != 0 and path[index - 1].symbol == rule.symbol
          1
      else
        rule.weight
      end
      @weight *= weight_multitplier
    end
  end

  def append(rule)
    State.new @path + [rule], State.get_library(rule ? rule.symbol : '', @library)
  end

  def failure?
    not @library
  end

  def success?
    @library.value
  end

  def ==(sec)
    self.text == sec.text && self.length == sec.length && self.weight == sec.weight && self.library == sec.library
  end

  class << self
{'fraction_non_alphanumeric': 0.1287001287001287, 'fraction_numerical': 0.0045045045045045045, 'mean_word_length': 2.3440860215053765, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6862877', 'n_tokens_mistral': 521, 'n_tokens_neox': 502, 'n_words': 140}