<commit_before>import math, pdb

def load_counts(ifile):
    counts = {}
    total_kmers = 0.0
    with open(ifile) as iinput:
        for line in iinput:
            word, count = line.strip().split('\t')
            word = word[1:-1]
            count = int(count)
            counts[word] = float(count + 1) # the +1 implements add-one smoothing
            total_kmers += float(count + 1)
    return counts, total_kmers

def score(counts_pt, total_trimers_pt, counts_en, total_trimers_en, test_sentence):
    val = 1.
    for i in xrange(len(test_sentence)-3):
        tri = test_sentence[i:i+3]
        tri_pt = counts_pt.get(tri, 1.0) # this will attempt to get counts from the dictionary; if it fails, it will return 1.0
        log_prob_tri_pt = math.log10(tri_pt/total_trimers_pt)
        tri_en = counts_en.get(tri, 1.0)
        log_prob_tri_en = math.log10(tri_en/total_trimers_en)

        val += log_prob_tri_pt-log_prob_tri_en
        
    if val >= 0:
        language = "PT"
    else:
        language = "EN"
    if abs(val) >= 5:
        print "This is a", language, "sentence."
    else:
        print "This seems to be a", language, "sentence, but I'm not sure."
    print "Log-ratio:", abs(val)



counts_pt, total_trimers_pt = load_counts('output.pt.txt')
counts_en, total_trimers_en = load_counts('output.en.txt')
<commit_msg>Changed EN and PT count file names in postprocess.py to match the guide.<commit_after>import math, pdb

def load_counts(ifile):
    counts = {}
    total_kmers = 0.0
    with open(ifile) as iinput:
        for line in iinput:
            word, count = line.strip().split('\t')
            word = word[1:-1]
            count = int(count)
            counts[word] = float(count + 1) # the +1 implements add-one smoothing
            total_kmers += float(count + 1)
    return counts, total_kmers

def score(counts_pt, total_trimers_pt, counts_en, total_trimers_en, test_sentence):
    val = 1.
    for i in xrange(len(test_sentence)-3):
        tri = test_sentence[i:i+3]
        tri_pt = counts_pt.get(tri, 1.0) # this will attempt to get counts from the dictionary; if it fails, it will return 1.0
        log_prob_tri_pt = math.log10(tri_pt/total_trimers_pt)
        tri_en = counts_en.get(tri, 1.0)
        log_prob_tri_en = math.log10(tri_en/total_trimers_en)

        val += log_prob_tri_pt-log_prob_tri_en
        
    if val >= 0:
        language = "PT"
    else:
        language = "EN"
    if abs(val) >= 5:
        print "This is a", language, "sentence."
    else:
        print "This seems to be a", language, "sentence, but I'm not sure."
    print "Log-ratio:", abs(val)



counts_pt, total_trimers_pt = load_counts('pt.counts.txt')
counts_en, total_trimers_en = load_counts('en.counts.txt')

{'fraction_non_alphanumeric': 0.11252268602540835, 'fraction_numerical': 0.015970961887477313, 'mean_word_length': 2.6026143790849674, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7123706', 'n_tokens_mistral': 998, 'n_tokens_neox': 959, 'n_words': 248}