<commit_before>
import tserver, sys, discoapi, disco

def data_gen(path):
        #import time, random
        #time.sleep(random.random())
        return "Gutta cavat capidem\n" * 500

def fun_map(e, params):
        return [(w, 1) for w in re.sub("\W", " ", e).lower().split()]

def fun_reduce(iter, out, params):
        s = {}
        for k, v in iter:
                if k in s:
                        s[k] += int(v)
                else:
                        s[k] = int(v)
        for k, v in s.iteritems():
                out.add(k, v)

tserver.run_server(data_gen)
name = disco.job(sys.argv[1], "test_50k", tserver.makeurl([""] * int(5e4)),
                       fun_map, reduce = fun_reduce, nr_reduces = 320)

for key, value in disco.result_iterator(results):
	print key, value<commit_msg>minor changes in test_50k test<commit_after>
import tserver, sys, discoapi, disco

def data_gen(path):
        import random
        return "Gutta cavat capidem\n" * random.randint(100,500)

def fun_map(e, params):
        return [(w, 1) for w in re.sub("\W", " ", e).lower().split()]

def fun_reduce(iter, out, params):
        s = {}
        for k, v in iter:
                if k in s:
                        s[k] += int(v)
                else:
                        s[k] = int(v)
        for k, v in s.iteritems():
                out.add(k, v)

tserver.run_server(data_gen)
name = disco.job(sys.argv[1], "test_50k", tserver.makeurl([""] * int(5e4)),
                       fun_map, reduce = fun_reduce, nr_reduces = 300)

for key, value in disco.result_iterator(results):
	print key, value
{'fraction_non_alphanumeric': 0.13911416094822207, 'fraction_numerical': 0.018091079226450407, 'mean_word_length': 1.9163636363636363, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2528618', 'n_tokens_mistral': 571, 'n_tokens_neox': 545, 'n_words': 147}