<commit_before>from __future__ import print_function

import csv
import sys
import os
import os.path

def inc(d, k):
  if k not in d:
      d[k] = 0
  d[k] += 1
    
acc_tab = {}

for arg in sys.argv[1:]:
    with open(arg, 'rb') as csvfile:
        csvr = csv.reader(csvfile)
        for stuff in csvr:
            #print(stuff, file=sys.stderr)
            (path,
            mutation,
            window_rank,
            window_entropy,
            token_type,
            token_line,
            token_string,
            parse_error,
            parse_error_same_line,
            parse_error_file_name,
            parse_error_line,
            parse_error_function,
            window_start_line,
            token_start_line,
            line_rank,
            token_rank,
            fix_notruevalid,
            fix_valid,
            fix_operation) = stuff
            if mutation not in acc_tab:
                acc_tab[mutation] = {
                  'total': 0,
                  'i': 0,
                  'd': 0,
                  's': 0,
                  'no': 0,
                  'valid': 0,
                  'true': 0
                }
            mut_tab = acc_tab[mutation]
            inc(mut_tab, 'total')
            inc(mut_tab, fix_operation)
            inc(mut_tab, fix_notruevalid)
        
rows = [
  ('insertRandom', 'Insertion'),
  ('deleteRandom', 'Deletion'),        
  ('replaceRandom', 'Substitution'),        
]

cols = [
  'Insert',
  'Delete',
  'Replace',
  'NoFix',
  'ValidFix',
  'TrueFix'
]

hilite = set([
  ('insertRandom', 'Delete'),
  ('deleteRandom', 'Insert'),        
  ('replaceRandom', 'Replace'),        
  ])

print(r"""
% Requires this in the preamble:
%
%  \usepackage(xcolor,colortbl}
%  \definecolor{highlightcolor}{rgb}{0.8,0.8,0.8}
%  \newcommand{\highlight}[1]{\cellcolor{highlightcolor} #1}
%
\begin{tabular}{@{}lrrrrrrrrrrrr@{}}
\toprule
             &                          \multicolumn{6}{c}{Fix created by 10-gram model}                               &                               \multicolumn{6}{c}{Summary}                                 \\
\cmidrule(r){2-7} \cmidrule{8-13}
             & \multicolumn{2}{c}{Insertion} & \multicolumn{2}{c}{Deletion} & \multicolumn{2}{c}{Substitution} & \multicolumn{2}{c}{No fix} & \multicolumn{2}{c}{Valid fix} & \multicolumn{2}{c}{True fix} \\
 Mutation    & Total        & \%             & Total        & \%            & Total          & \%              & Total        & \%          & Total          & \%           & Total        & \%            \\
\cmidrule(r){2-7} \cmidrule{8-13}
""".strip())
# Print each row (mutation) in the table
for row, name in rows:
    rowdata = [name]
    for col in cols:
        t = ""
        if (row, col) in hilite:
          t += "\\highlight{"
        t += str(acc_tab[row][col])
        if (row, col) in hilite:
          t += "}"
        rowdata.append(t)
        t = ""
        if (row, col) in hilite:
          t += "\\highlight{"
        t += "%.2f" % (acc_tab[row][col]/acc_tab[row]['total'])
        if (row, col) in hilite:
          t += "}"
        rowdata.append(t)
    print(" & ".join(rowdata) + r"\\")

print(r"""
\bottomrule
\end{tabular}
""".strip())
<commit_msg>results table generator<commit_after>from __future__ import print_function, division

import csv
import sys
import os
import os.path

def inc(d, k):
  if k not in d:
      d[k] = 0
  d[k] += 1
    
acc_tab = {}

for arg in sys.argv[1:]:
    with open(arg, 'rb') as csvfile:
        csvr = csv.reader(csvfile)
        for stuff in csvr:
            #print(stuff, file=sys.stderr)
            (path,
            mutation,
            window_rank,
            window_entropy,
            token_type,
            token_line,
            token_string,
            parse_error,
            parse_error_same_line,
            parse_error_file_name,
            parse_error_line,
            parse_error_function,
            window_start_line,
            token_start_line,
            line_rank,
            token_rank,
            fix_notruevalid,
            fix_valid,
            fix_operation) = stuff
            if mutation not in acc_tab:
                acc_tab[mutation] = {
                }
            mut_tab = acc_tab[mutation]
            inc(mut_tab, 'total')
            inc(mut_tab, fix_operation)
            inc(mut_tab, fix_notruevalid)
        
rows = [
  ('insertRandom', 'Insertion'),
  ('deleteRandom', 'Deletion'),        
  ('replaceRandom', 'Substitution'),        
]

cols = [
  'Insert',
  'Delete',
  'Replace',
  'NoFix',
  'ValidFix',
  'TrueFix'
]

hilite = set([
  ('insertRandom', 'Delete'),
  ('deleteRandom', 'Insert'),        
  ('replaceRandom', 'Replace'),        
  ])

print(r"""
% Requires this in the preamble:
%
%  \usepackage(xcolor,colortbl}
%  \definecolor{highlightcolor}{rgb}{0.8,0.8,0.8}
%  \newcommand{\highlight}[1]{\cellcolor{highlightcolor} #1}
%
\begin{tabular}{@{}lrrrrrrrrrrrr@{}}
\toprule
             &                          \multicolumn{6}{c}{Fix created by 10-gram model}                               &                               \multicolumn{6}{c}{Summary}                                 \\
\cmidrule(r){2-7} \cmidrule{8-13}
             & \multicolumn{2}{c}{Insertion} & \multicolumn{2}{c}{Deletion} & \multicolumn{2}{c}{Substitution} & \multicolumn{2}{c}{No fix} & \multicolumn{2}{c}{Valid fix} & \multicolumn{2}{c}{True fix} \\
 Mutation    & Total        & \%             & Total        & \%            & Total          & \%              & Total        & \%          & Total          & \%           & Total        & \%            \\
\cmidrule(r){2-7} \cmidrule{8-13}
""".strip())
# Print each row (mutation) in the table
for row, name in rows:
    rowdata = [name]
    for col in cols:
        t = ""
        if (row, col) in hilite:
          t += "\\highlight{"
        t += str(acc_tab[row][col])
        if (row, col) in hilite:
          t += "}"
        rowdata.append(t)
        t = ""
        if (row, col) in hilite:
          t += "\\highlight{"
        t += "%.2f" % (acc_tab[row][col]/acc_tab[row]['total'])
        if (row, col) in hilite:
          t += "}"
        rowdata.append(t)
    print(" & ".join(rowdata) + r"\\")

print(r"""
\bottomrule
\end{tabular}
""".strip())

{'fraction_non_alphanumeric': 0.1468764902241297, 'fraction_numerical': 0.011285964075663646, 'mean_word_length': 1.5535714285714286, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4408677', 'n_tokens_mistral': 2079, 'n_tokens_neox': 2026, 'n_words': 411}