<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

from sys import stdin
from os import linesep
from math import log

def get_count(words):
    tfs = {}
    for key in set(words):
        tfs[key] = 0
    for w in words:
        tfs[w] += 1
    return tfs

texts = ([l.strip().split(' ') for l in stdin])
names = map(lambda text: text[0], texts)
texts = map(lambda text: text[1:], texts)

tfs = map(get_count, texts)

idf = {}
for word in set(sum(texts, [])):
    idf[word] = 0

for tf in tfs:
    for word in tf:
        idf[word] += 1

logN = log(len(texts))
for word in idf:
    idf[word] = logN - log(idf[word])


tf_idfs = []
for i, tf in enumerate(tfs):
    tf_idfs.append({})
    for word in tf:
        tf_idfs[i][word] = float(tf[word] * idf[word]) / len(tf)

#result = sorted(counts.iteritems(),key=lambda x: x[1],
#                reverse=True)

print linesep.join('%s,%s,%f'%(names[i], word, tf_idf[word])
                    for i, tf_idf in enumerate(tf_idfs) for word in tf_idf)
<commit_msg>Removed redundant comments<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

from sys import stdin
from os import linesep
from math import log


def get_count(words):
    tfs = {}
    for key in set(words):
        tfs[key] = 0
    for w in words:
        tfs[w] += 1
    return tfs


texts = ([l.strip().split(' ') for l in stdin])
names = map(lambda text: text[0], texts)
texts = map(lambda text: text[1:], texts)


tfs = map(get_count, texts)


idf = {}
for word in set(sum(texts, [])):
    idf[word] = 0


for tf in tfs:
    for word in tf:
        idf[word] += 1


logN = log(len(texts))
for word in idf:
    idf[word] = logN - log(idf[word])


tf_idfs = []
for i, tf in enumerate(tfs):
    tf_idfs.append({})
    for word in tf:
        tf_idfs[i][word] = float(tf[word] * idf[word]) / len(tf)


print linesep.join('%s,%s,%f'%(names[i], word, tf_idf[word])
                    for i, tf_idf in enumerate(tf_idfs) for word in tf_idf)

{'fraction_non_alphanumeric': 0.16087844739530133, 'fraction_numerical': 0.007660878447395302, 'mean_word_length': 2.781853281853282, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1596658', 'n_tokens_mistral': 803, 'n_tokens_neox': 766, 'n_words': 217}