<commit_before>#!/usr/bin/env python3

import os
import random
import multiprocessing

def random_file(para):
    random.seed(50)
    filename, size_mb = para
    with open(filename,'wb') as fl:
        for _ in range(size_mb):
            mb = map(lambda x: random.getrandbits(8),
                    range(1024*1024))
            fl.write(bytes(mb))
    print("created {0}".format(filename))
    return


if __name__ == "__main__":
    DIR = "test_data"
    N_FILE = 10
    SIZE_MB = 50

    try:
        os.mkdir(DIR)
    except:
        pass

    para = []
    for n in range(N_FILE):
        fl_name = os.path.join(DIR, "file_öäüß_{0}.rnd".format(n))
        para.append( (fl_name, SIZE_MB) )

    p = multiprocessing.Pool()
    list(p.imap_unordered(random_file, para))<commit_msg>test_data using numpy<commit_after>#!/usr/bin/env python3

import os
import multiprocessing
import numpy as np

def random_file(para):
    np.random.seed(50)
    filename, size_mb = para
    with open(filename,'wb') as fl:
        for _ in range(size_mb):
            mb = np.random.bytes(1024*1024)
            fl.write(mb)
    print("created {0}".format(filename))
    return

if __name__ == "__main__":
    DIR = "test_data"
    N_FILE = 10
    SIZE_MB = 100

    try:
        os.mkdir(DIR)
    except:
        pass

    para = []
    for n in range(N_FILE):
        fl_name = os.path.join(DIR, "file_öäüß_{0}.rnd".format(n))
        para.append( (fl_name, SIZE_MB) )

    p = multiprocessing.Pool()
    list(p.imap_unordered(random_file, para))

{'fraction_non_alphanumeric': 0.12760416666666666, 'fraction_numerical': 0.0234375, 'mean_word_length': 2.4461883408071747, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1504833', 'n_tokens_mistral': 588, 'n_tokens_neox': 548, 'n_words': 116}