<commit_before>require "irs527/version"

module Irs527

  class TextParser

    def self.parse(path)
      file_8871 = File.open('8871.txt', 'w')
      file_8872 = File.open('8872.txt', 'w')
      file_skeda = File.open('skeda.txt', 'w')
      file_skedb = File.open('skedb.txt', 'w')
      f = File.open(path).readlines
      f.each do |line|
        if line[0..1] == "1|"
          file_8871.write(line)
        elsif line[0..1] == "2|"
          file_8872.write(line)
        elsif line[0..1] == "A|"
          file_skeda.write(line)
        elsif line[0..1] == "B|"
          file_skedb.write(line)
        end
      end
      file_8871.close<commit_msg>read input file line-by-line instead of slurping into memory<commit_after>require "irs527/version"

module Irs527

  class TextParser

    def self.parse(path)
      file_8871 = File.open('8871.txt', 'w')
      file_8872 = File.open('8872.txt', 'w')
      file_skeda = File.open('skeda.txt', 'w')
      file_skedb = File.open('skedb.txt', 'w')
      File.open(path) do |f|
        while line = f.gets
          if line[0..1] == "1|"
            file_8871.write(line)
          elsif line[0..1] == "2|"
            file_8872.write(line)
          elsif line[0..1] == "A|"
            file_skeda.write(line)
          elsif line[0..1] == "B|"
            file_skedb.write(line)
          end
        end
      end
      file_8871.close
{'fraction_non_alphanumeric': 0.1531272465851905, 'fraction_numerical': 0.06326383896477354, 'mean_word_length': 1.9617021276595745, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5645061', 'n_tokens_mistral': 561, 'n_tokens_neox': 520, 'n_words': 97}