<commit_before># vim: ft=awk:tabstop=2:softtabstop=2:shiftwidth=2:expandtab:cindent:formatoptions+=cro


# I lie tradidd lilights
# 
# - when they are green
# - adas
# 
@include "lib"

END { 
  print ""
  for(w in W)
    printf("%10s ",Name[w])
  print "\n"report(Data); tree(Data,NF-2,NF) 
}

function tree(rows,max,k,   pre,ranges,r,c,use,sorted) {
  array(ranges)
  for(r in rows)
    for(c in Data[r])
      if (c < max) 
        kept(r,c, Data[r][c], Data[r][k], ranges)
  use = best(ranges,sorted)
  for(r=length(sorted); r>=1; r--)
    if (sorted[r].column == use)
        kids(use, sorted[r],max,k,pre) 
}
# Somethings to note:
#
# - asda
# - sdsd
function kids(use,range,max,k,pre,    txt) {
   if (length( range.rows ) > length(Data)^0.5) {
     print  report(range.rows) pre Name[use] " = " range.value 
     tree(range.rows,max,k,"|__ " pre)  }
}
function range(i,c,x) {
  num(i)
  i.column = c
  i.value  = x
  has(i,"rows")
}
function kept(r,c,x,y,ranges,    key) {
  if (x=="?") return 0
  key = c "=" x
  if (! (key in ranges))
    hAS(ranges, key, "range",c,x)
  ninc(ranges[key],y)
  ranges[key].rows[r]
}<commit_msg>pushing<commit_after># vim: ft=awk:tabstop=2:softtabstop=2:shiftwidth=2:expandtab:cindent:formatoptions+=cro


# I lie tradidd lilights
# 
# - when they are green
# - adas
# 
@include "lib"

END { 
  print ""
  for(w in W)
    printf("%10s ",Name[w])
  print "\n"report(Data); tree(Data,NF-2,NF) 
}

function tree(rows,max,k,   pre,ranges,r,c,use,sorted) {
  array(ranges)
  for(r in rows)
    for(c in Data[r])
      if (c < max) 
        kept(r,c, Data[r][c], Data[r][k], ranges)
  use = best(ranges,sorted)
  for(r=length(sorted); r>=1; r--)
    if (sorted[r].column == use)
        kids(use, sorted[r],max,k,pre)  
}
# Somethings to note:
#
# - asda
# - sdsd
function kids(use,range,max,k,pre,    txt) {
   if (length( range.rows ) > length(Data)^0.5) {
     print  report(range.rows) pre Name[use] " = " range.value 
     tree(range.rows,max,k,"|__ " pre)  }
}
function range(i,c,x) {
  num(i)
  i.column = c
  i.value  = x
  has(i,"rows")
}
function kept(r,c,x,y,ranges,    key) {
  if (x=="?") return 0
  key = c "=" x
  if (! (key in ranges))
    hAS(ranges, key, "range",c,x)
  ninc(ranges[key],y)
  ranges[key].rows[r]
}
{'fraction_non_alphanumeric': 0.1986754966887417, 'fraction_numerical': 0.008830022075055188, 'mean_word_length': 3.3745173745173744, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3656449', 'n_tokens_mistral': 1014, 'n_tokens_neox': 1004, 'n_words': 194}