<commit_before>import os
import re
from collections import defaultdict

w = defaultdict(int)
concepts = []
with open("1383") as f:
    for line in f:
        if not line.startswith("%"):
            concepts.append(line.split("\t")[0].strip())
            if len(line.split("\t")) > 8:
                line = line.split("\t")[7].split("%")[0]
                line = re.sub(",", " ", line)
                line = re.sub("\{", " ", line)
                line = re.sub("\}", " ", line)
                line = re.sub("\(", " ", line)
                line = re.sub("\)", " ", line)
                line = re.sub("\[", " ", line)
                line = re.sub("\]", " ", line)
                line = re.sub("[0-9]*", "", line)
                line = re.sub("/", "", line)
                words = line.split()
                for wo in words:
                    wo = wo.strip()
                    if wo and not wo.startswith("=") and not wo.startswith('"') and not wo.isupper() and not ">" in wo and not "<" in wo:
                        w[wo.strip()] += 1

w_keys = w.keys()
inter = set(concepts) & set(w_keys)
not_in = {}

for key in w:
    if key not in inter:
        not_in[key] = w[key]

t = sorted(not_in.items(), key=lambda x: x[1])

for elem in t[::-1]:
    print(str(elem[0]) + "\t" + str(elem[1]))<commit_msg>Added inputfile<commit_after>import os
import re
import sys, getopt
from collections import defaultdict

def main(argv):
    inputfile = ''
    try:
        opts, args = getopt.getopt(argv,"hi:o:",["ifile=","odir="])
    except getopt.GetoptError:
        print('def_ply_parser.py -i <inputfile> -o <outputdir>')
        sys.exit(2)
    for opt, arg in opts:
        if opt == '-h':
            print('get_undefined.py -i <inputfile>')
            sys.exit()
        elif opt in ("-i", "--ifile"):
            inputfile = arg

    w = defaultdict(int)
    concepts = []
    with open(inputfile) as f:
        for line in f:
            if not line.startswith("%"):
                concepts.append(line.split("\t")[0].strip())
                if len(line.split("\t")) > 8:
                    line = line.split("\t")[7].split("%")[0]
                    line = re.sub(",", " ", line)
                    line = re.sub("\{", " ", line)
                    line = re.sub("\}", " ", line)
                    line = re.sub("\(", " ", line)
                    line = re.sub("\)", " ", line)
                    line = re.sub("\[", " ", line)
                    line = re.sub("\]", " ", line)
                    line = re.sub("[0-9]*", "", line)
                    line = re.sub("/", "", line)
                    words = line.split()
                    for wo in words:
                        wo = wo.strip()
                        if wo and not wo.startswith("=") and not wo.startswith('"') and not wo.isupper() and not ">" in wo and not "<" in wo:
                            w[wo.strip()] += 1

    w_keys = w.keys()
    inter = set(concepts) & set(w_keys)
    not_in = {}

    for key in w:
        if key not in inter:
            not_in[key] = w[key]

    t = sorted(not_in.items(), key=lambda x: x[1])

    for elem in t[::-1]:
        print(str(elem[0]) + "\t" + str(elem[1]))


if __name__ == "__main__":
    main(sys.argv[1:])
{'fraction_non_alphanumeric': 0.18623232944068838, 'fraction_numerical': 0.008604794099569761, 'mean_word_length': 1.507704160246533, 'pattern_counts': {'":': 1, '<': 8, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7021983', 'n_tokens_mistral': 1067, 'n_tokens_neox': 1041, 'n_words': 267}