<commit_before>#
# usage: python k71.py {file name}
#

import sys
from collections import Counter
from bisect import bisect_left

def stopwords(sl, fr):
    c, st = Counter([w for s in sl for w in list(set(s))]), len(sl) * fr / 100
    return sorted([k for k, v in c.items() if v > st])

def is_stopword(w, ss):
    i, b = bisect_left(ss, w), False
    if i != len(ss) and ss[i] == w:
        b = True
    return b

if __name__ == '__main__':
    fn = sys.argv[1]
    
    sl = [l.split()[1:] for l in open(fn)]
    ss = stopwords(sl, 10)

    for w in ['is', 'a', 'the', 'happy', 'sad', 'cry']:
        print(w, is_stopword(w, ss))<commit_msg>clean k71<commit_after>#
# usage: python k71.py {file name}
#

import sys
from collections import Counter

def stopwords(sl, fr):
    c, st = Counter([w for s in sl for w in list(set(s))]), len(sl) * fr / 100
    return [k for k, v in c.items() if v > st]

def is_stopword(w, ss):
    return w in ss

if __name__ == '__main__':
    fn = sys.argv[1]
    
    sl = [l.split()[1:] for l in open(fn)]
    ss = stopwords(sl, 10)

    for w in ['is', 'a', 'the', 'happy', 'sad', 'cry']:
        print(w, is_stopword(w, ss))
{'fraction_non_alphanumeric': 0.17054263565891473, 'fraction_numerical': 0.017226528854435832, 'mean_word_length': 2.86046511627907, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5848542', 'n_tokens_mistral': 482, 'n_tokens_neox': 456, 'n_words': 159}