<commit_before># this small test script reads 
# integer-ID-lized feature-vector per-line format 
# and outputs feature x feature inner-product matrix. 
# (rows of so-called B matrix. See memo: ) 

require 'time' 

arg_check
##
## first, load the feature vectors. 
t1 = Time.new 

fvectors_arr = Array.new
File.open(ARGV[0]).each_line do |line|
  id_string, vector_string = line.split("\t\t")
  id = id_string.to_i
  if (fvectors_arr[id]) # sanity check, this can't exist yet. 
    $stderr.puts("Something is wrong. duplicated feature id #{i}") 
  end
  
  tmp = Array.new 
  vector_string.split.each_with_index do |s, i|
    if (i.even?)
      tmp.push(s.to_i)
    else
      tmp.push(s.to_f)
    end
  end
  fvectors_arr[id]= Hash[*tmp] # as hash 
  #fvectors_arr[id]= tmp # as array 
end

# dcode, walk over the fvector_arr, where each member is a hash 
#fvectors_arr.each_with_index do |h, i|
#  print "#{i}\t"
#  h.keys.sort.each do |k|
#    print("#{k}:#{h[k]}\t")
#  end
#  print "\n" 
#end 

t2 = Time.new 
tdiff_1_2 = t2 - t1
$stderr.puts("Loading complete: time took #{tdiff_1_2}") 

# Using hash, required 7 Gb, with 20 min (1260 secs), on 17inch-macbook  
# Using array, required x.x Gb, with MM min. 

##
## second, (with the loaded vectors), calculate from - to 
from = ARGV[1].to_i
to = ARGV[2].to_i

# sanity chcek 
if (from < 0)
  from = 0
end 

if (to > fvectors_arr.size)
  to = fvectors_arr.size - 1 
end 

# actual run 
(from..to).each do |i|
  # calculate row i of B matrix. 
  target_h = fvectors_arr[i]
  result_B_vec = Hash.new 

  fvectors_arr.each_with_index do |h, i|
    result_B_vec[i] = inner_product_hh(target_h, h)
  end

  # print result directly, without storing. <commit_msg>- fixed a bug (no 0 values should be there)<commit_after># this small test script reads 
# integer-ID-lized feature-vector per-line format 
# and outputs feature x feature inner-product matrix. 
# (rows of so-called B matrix. See memo: ) 

require 'time' 

arg_check
##
## first, load the feature vectors. 
t1 = Time.new 

fvectors_arr = Array.new
File.open(ARGV[0]).each_line do |line|
  id_string, vector_string = line.split("\t\t")
  id = id_string.to_i
  if (fvectors_arr[id]) # sanity check, this can't exist yet. 
    $stderr.puts("Something is wrong. duplicated feature id #{i}") 
  end
  
  tmp = Array.new 
  vector_string.split.each_with_index do |s, i|
    if (i.even?)
      tmp.push(s.to_i)
    else
      tmp.push(s.to_f)
    end
  end
  fvectors_arr[id]= Hash[*tmp] # as hash 
  #fvectors_arr[id]= tmp # as array 
end

# dcode, walk over the fvector_arr, where each member is a hash 
#fvectors_arr.each_with_index do |h, i|
#  print "#{i}\t"
#  h.keys.sort.each do |k|
#    print("#{k}:#{h[k]}\t")
#  end
#  print "\n" 
#end 

t2 = Time.new 
tdiff_1_2 = t2 - t1
$stderr.puts("Loading complete: time took #{tdiff_1_2}") 

# Using hash, required 7 Gb, with 20 min (1260 secs), on 17inch-macbook  
# Using array, required x.x Gb, with MM min. 

##
## second, (with the loaded vectors), calculate from - to 
from = ARGV[1].to_i
to = ARGV[2].to_i

# sanity chcek 
if (from < 0)
  from = 0
end 

if (to > fvectors_arr.size)
  to = fvectors_arr.size - 1 
end 

# actual run 
(from..to).each do |i|
  # calculate row i of B matrix. 
  target_h = fvectors_arr[i]
  result_B_vec = Hash.new 

  fvectors_arr.each_with_index do |h, i|
    val = inner_product_hh(target_h, h)
    if (val > 0.0)
      result_B_vec[i] = val
    end
  end

  # print result directly, without storing. 
{'fraction_non_alphanumeric': 0.12989985693848355, 'fraction_numerical': 0.01402002861230329, 'mean_word_length': 3.152019002375297, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4238891', 'n_tokens_mistral': 1447, 'n_tokens_neox': 1358, 'n_words': 399}