<commit_before>#add_labels_self.py
#
# Adds labels as a columns in files

import argparse

parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('infile', help='input filename')
parser.add_argument('outfile', help='output filename')
parser.add_argument('--label', default=None,\
            help='optional label file (see specification)')
parser.add_argument('--csv', default=False, action='store_true',\
                    help='use commas instead of tabs to separate data\
                            (default: False)')
args = parser.parse_args()

#print args.infile
#print args.outfile
#print args.csv

if args.csv is True:
    separator = ", "
else:
    separator = "\t"

# Open optional label file
if args.label is not None:
    labeldict = {}
    with open(args.label, 'rb') as f:
        for line in f:
            p = line.replace('\n', '').split(' ')
            labeldict[p[0]] = p[1]


fout = open(args.outfile, 'wb')
title = "ID"
if args.label is not None:
    title += separator
    title += "LABEL"

first_line = True
attN = 1
with open(args.infile, 'rb') as f:
    for line in f:
        p = line.replace('\n', '').split(' ')
        lineout = ""
        ID = p[0].split('/')[-1]
        lineout += ID
        lineout += separator
        lineout += labeldict[ID]
        for i in range(1, len(p)):
            lineout += separator
            lineout += p[i]

    if first_line is True:
        for i in range(1, len(p)):
            title += separator
            title += i
        title += "\n"
        fout.write(title)
        first_line = False

    lineout += "\n"
    fout.write(lineout)

fout.close()

#print(len(itemlist))
#print(itemlist[0].attributes['name'].value)
#for s in itemlist:
#        print(s.attributes['name'].value)

<commit_msg>[midi] Corrected errors in add_labels_self.py and now it is tested<commit_after>#add_labels_self.py
#
# Adds labels as a columns in files

import argparse

parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('infile', help='input filename')
parser.add_argument('outfile', help='output filename')
parser.add_argument('--label', default=None,\
            help='optional label file (see specification)')
parser.add_argument('--csv', default=False, action='store_true',\
                    help='use commas instead of tabs to separate data\
                            (default: False)')
args = parser.parse_args()

#print args.infile
#print args.outfile
#print args.csv

if args.csv is True:
    separator = ", "
else:
    separator = "\t"

# Open optional label file
if args.label is not None:
    labeldict = {}
    with open(args.label, 'rb') as f:
        for line in f:
            p = line.replace('\n', '').split(' ')
            labeldict[p[0]] = p[1]


fout = open(args.outfile, 'wb')
title = "ID"
if args.label is not None:
    title += separator
    title += "LABEL"

first_line = True
attN = 1
with open(args.infile, 'rb') as f:


    for line in f:
        print line
        p = line.replace('\n', '').split(' ')
        lineout = ""
        ID = p[0].split('/')[-1]
        lineout += ID
        lineout += separator
        lineout += labeldict[ID]
        for i in range(1, len(p)):
            lineout += separator
            lineout += p[i]

        if first_line is True:
            for i in range(1, len(p)):
                title += separator
                title += str(i)
            title += "\n"
            fout.write(title)
            first_line = False

        lineout += "\n"
        fout.write(lineout)

fout.close()

#print(len(itemlist))
#print(itemlist[0].attributes['name'].value)
#for s in itemlist:
#        print(s.attributes['name'].value)


{'fraction_non_alphanumeric': 0.14285714285714285, 'fraction_numerical': 0.004288394532296971, 'mean_word_length': 2.3712737127371275, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5808429', 'n_tokens_mistral': 1194, 'n_tokens_neox': 1162, 'n_words': 325}