<commit_before>#!/usr/bin/ruby

require 'rubygems'
require 'treetop'
require 'polyglot'
require 'ris'

blarg = "AR  - blarg\nAR  - foo\nER  - \n"
parser = RisParser.new

puts parser.parse(blarg).eval
<commit_msg>Collates multiple AR's into an array<commit_after>#!/usr/bin/ruby

require 'rubygems'
require 'treetop'
require 'polyglot'
require 'ris'
require 'pp'

blarg = "AR  - blarg\nAR  - foo\nER  - \nAR  - test\nAR  - quux\nER  - \n"
parser = RisParser.new

a = parser.parse(blarg).eval
records = []
a.each do |record|
  entry = {}
  record.each do |key_value|
    if key_value[0] == 'AR'
      if entry[key_value[0]] == nil
        entry[key_value[0]] = [key_value[1]]
      else
        entry[key_value[0]].push(key_value[1])
      end
    end
  end
  records << entry
end

pp records


{'fraction_non_alphanumeric': 0.14646464646464646, 'fraction_numerical': 0.007575757575757576, 'mean_word_length': 3.066666666666667, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4382256', 'n_tokens_mistral': 325, 'n_tokens_neox': 320, 'n_words': 72}