<commit_before>class LocalRecords
  def initialize(dir, limit, marker)
    @dir = dir
    @limit = limit
    @marker = marker
  end

  def each
    i = 1
    Dir.glob("#{@dir}/*.json") do |filepath|
      @marked = reached_marker?(@marker, id(filepath)) if !@marked
      file = File.open(filepath, 'r')
      yield(id(filepath), process_record(file.read)) if @marked
      break if @limit == i
      i = i + 1 if @marked
    end
  end

  def reached_marker?(marker, filename)
    (filename == marker || marker.nil?) ? true : false
  end

  def id(path)
    File.basename(path, ".*")
  end

  def process_record(record)
    escape_original_record(JSON.parse(record))
  end

  def escape_original_record(record)
    record['originalRecord'] = record['originalRecord'].to_json
    record
  end
end<commit_msg>don't start counting until the marker has been reached.<commit_after>class LocalRecords
  def initialize(dir, limit, marker)
    @dir = dir
    @limit = limit
    @marker = marker
  end

  def each
    i = 0
    Dir.glob("#{@dir}/*.json") do |filepath|
      @marked = reached_marker?(@marker, id(filepath)) if !@marked
      file = File.open(filepath, 'r')
      yield(id(filepath), process_record(file.read)) if @marked
      i = i + 1 if @marked
      break if @limit == i
    end
  end

  def reached_marker?(marker, filename)
    (filename == marker || marker.nil?) ? true : false
  end

  def id(path)
    File.basename(path, ".*")
  end

  def process_record(record)
    escape_original_record(JSON.parse(record))
  end

  def escape_original_record(record)
    record['originalRecord'] = record['originalRecord'].to_json
    record
  end
end
{'fraction_non_alphanumeric': 0.12681159420289856, 'fraction_numerical': 0.0024154589371980675, 'mean_word_length': 3.163316582914573, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4852012', 'n_tokens_mistral': 557, 'n_tokens_neox': 535, 'n_words': 141}