<commit_before># #######################################################################
#
# Author: <NAME>
# Name: Crystal bindings for MonetDB
# Codename: Dagobert I
# Description:
#   MonetDB RAW Data processor
#
#   For clarity this is just a test for parsing the RAW output
#
# #######################################################################

require "json"

monetdb_hdr_data = ""
monetdb_raw_data = ""
hdrinc = 0
skipfirst = 0
fields = Hash(Int32, String).new
types = Hash(Int32, String).new
valiter = 0
File.open("monetdb_raw_result.txt", "r") { |n|
  n.each_line { |l|
    unless hdrinc == 4
      monetdb_hdr_data = "#{monetdb_hdr_data}#{l}"
      if hdrinc == 1
        field_data = l.gsub("% ", "").split("#")[0].strip
        fields.merge!({valiter => field_data})
      elsif hdrinc == 2
        type_data = l.gsub("% ", "").split("#")[0].strip
        types.merge!({valiter => type_data})
      end
      hdrinc += 1
    end
    if hdrinc == 4
      if skipfirst > 0
        monetdb_raw_data = "#{monetdb_raw_data}#{l}"
      end
      skipfirst += 1
    end
    valiter += 1
  }
}

b_fields = Array(String).new
fields.each { |n,x| b_fields << x.strip }
b_types = Array(String).new
types.each { |n,x| b_types << x.strip }

puts "Header Data:\n"
puts "#{monetdb_hdr_data}"

puts "Fields"
p fields
puts "Types"
p types

puts "\nRaw Data:\n"
monetdb_raw_data.each_line { |n|
  puts n
}<commit_msg>Added formatting<commit_after># #######################################################################
#
# Author: <NAME>
# Name: Crystal bindings for MonetDB
# Codename: Dagobert I
# Description:
#   MonetDB RAW Data processor
#
#   For clarity this is just a test for parsing the RAW output
#
# #######################################################################

require "json"

monetdb_hdr_data = ""
monetdb_raw_data = ""
hdrinc = 0
skipfirst = 0
fields = Hash(Int32, String).new
types = Hash(Int32, String).new
valiter = 0
File.open("monetdb_raw_result.txt", "r") { |n|
  n.each_line { |l|
    unless hdrinc == 4
      monetdb_hdr_data = "#{monetdb_hdr_data}#{l}"
      if hdrinc == 1
        field_data = l.gsub("% ", "").split("#")[0].strip
        fields.merge!({valiter => field_data})
      elsif hdrinc == 2
        type_data = l.gsub("% ", "").split("#")[0].strip
        types.merge!({valiter => type_data})
      end
      hdrinc += 1
    end
    if hdrinc == 4
      if skipfirst > 0
        monetdb_raw_data = "#{monetdb_raw_data}#{l}"
      end
      skipfirst += 1
    end
    valiter += 1
  }
}

b_fields = Array(String).new
fields.each { |n,x| b_fields << x.strip }
b_types = Array(String).new
types.each { |n,x| b_types << x.strip }

puts "Header Data:\n"
puts "#{monetdb_hdr_data}"

puts "Fields"
fields.each {|n,x| puts x.split("\t"); }
puts "Types"
types.each {|n,x| puts x.split("\t") }

puts "\nRaw Data:\n"
monetdb_raw_data.each_line { |n|
  puts n
}
{'fraction_non_alphanumeric': 0.2412008281573499, 'fraction_numerical': 0.011732229123533472, 'mean_word_length': 3.4737654320987654, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '92418', 'n_tokens_mistral': 1081, 'n_tokens_neox': 1038, 'n_words': 245}