<commit_before>import re
import sys

def readList(filename):
    f = open(filename)
    res = list(map(lambda w: w.replace("\n", ""), f.readlines()))
    f.close()
    return res

def countKeywords(kws, filename):
    count = dict()
    for kw in kws:
        count[kw] = 0
    with open(filename, 'r') as file:
        contents = file.read()
        for kw in kws:
            count[kw] = len(re.findall(kw, contents))
        total = sum(count.values()) * 1.0
        for kw in kws:
            count[kw] /= total

    return count



def countKeywordsAndSymbols(kws, filename):
    count = dict()
    symbols = dict()
    for kw in kws:
        count[kw] = 0
    with open(filename, 'r') as file:
        contents = file.read()
        for kw in kws:
            count[kw] = len(re.findall(kw, contents))
        sanitizedContents = contents.replace(" ", "").replace("\t", "").replace("\n", "")
        totalKeywords = sum(count.values()) * 1.0
        if totalKeywords != 0:
            for kw in kws:
                count[kw] /= totalKeywords

        for sym in sanitizedContents:
           symbols[sym] = symbols.get(sym, 0) + 1

        totalLength = len(sanitizedContents) * 1.
        if totalLength != 0:
            for sym in symbols:
                symbols[sym] /= totalLength

    return count
<commit_msg>counting keywords<commit_after>import re
import sys

def readList(filename):
    f = open(filename)
    res = list(map(lambda w: w.replace("\n", ""), f.readlines()))
    f.close()
    return res

def countKeywords(kws, filename):
    count = dict()
    for kw in kws:
        count[kw] = 0
    with open(filename, 'r') as file:
        contents = file.read()
        for kw in kws:
            count[kw] = len(re.findall(kw, contents))
        total = sum(count.values()) * 1.0
        for kw in kws:
            count[kw] /= total

    return count



def countKeywordsAndSymbols(kws, filename):
    kwCount = dict()
    symCount = dict()
    for kw in kws:
        kwCount[kw] = 0
    with open(filename, 'r') as file:
        contents = file.read()
        for kw in kws:
            kwCount[kw] = len(re.findall(kw, contents))
        sanitizedContents = contents.replace(" ", "").replace("\t", "").replace("\n", "")
        totalKeywords = sum(kwCount.values()) * 1.0
        if totalKeywords != 0:
            for kw in kws:
                kwCount[kw] /= totalKeywords

        for sym in sanitizedContents:
           symCount[sym] = symCount.get(sym, 0) + 1

        totalLength = len(sanitizedContents) * 1.
        if totalLength != 0:
            for sym in symCount:
                symCount[sym] /= totalLength

    return dict(symCount, **kwCount)

{'fraction_non_alphanumeric': 0.1313732787495348, 'fraction_numerical': 0.008187569780424266, 'mean_word_length': 2.0236220472440944, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6870772', 'n_tokens_mistral': 875, 'n_tokens_neox': 841, 'n_words': 236}