<commit_before>import os.path
from os import unlink
from utils.launcher import Launcher
from utils.entries import Entries
from utils.loop import BooleanLoop, CounterLoop
from utils.files import generate, checksum
from utils.tempdirs import TempDirs

launcher = None
loop = None
dirs = TempDirs()
rep1, rep2 = dirs.create(), dirs.create()
json_file = 'test_copy.json'


def setup_module(module):
    global launcher, loop
    entries = Entries()
    entries.add('local_storage', 'rep1', {'root': rep1})
    entries.add('local_storage', 'rep2', {'root': rep2})
    entries.save(json_file)
    loop = CounterLoop(2)
    launcher = Launcher(json_file)
    launcher.on_driver_started(loop.check, 'rep1')
    launcher.on_driver_started(loop.check, 'rep2')
    launcher()


def teardown_module(module):
    launcher.kill()
    unlink(json_file)


def test_simple_copy():
    global loop
    loop.run(timeout=2)
    loop = BooleanLoop()
    launcher.on_end_transfer(loop.stop, 'rep1', 'rep2', 1)
    generate(os.path.join(rep1, 'foo'), 100)
    loop.run(timeout=5)
    launcher.quit()
    launcher.wait()
    assert(checksum(os.path.join(rep1, 'foo')) ==
           checksum(os.path.join(rep2, 'foo')))
<commit_msg>Copy tests set (with smaller and bigger files)<commit_after>import os.path
from os import unlink
from utils.launcher import Launcher
from utils.entries import Entries
from utils.loop import BooleanLoop, CounterLoop
from utils.files import generate, checksum
from utils.tempdirs import TempDirs

launcher = None
loop = None
dirs = TempDirs()
rep1, rep2 = dirs.create(), dirs.create()
json_file = 'test_copy.json'


def setup_module(module):
    global launcher, loop
    entries = Entries()
    entries.add('local_storage', 'rep1', {'root': rep1})
    entries.add('local_storage', 'rep2', {'root': rep2})
    entries.save(json_file)
    loop = CounterLoop(2)
    launcher = Launcher(json_file)
    launcher.on_driver_started(loop.check, 'rep1')
    launcher.on_driver_started(loop.check, 'rep2')
    launcher()


def teardown_module(module):
    launcher.kill()
    unlink(json_file)


def test_startup():
    global loop
    loop.run(timeout=2)


def gen_test_copy(filename, size, fileids=[]):
    if not filename in fileids:
        fileids.append(filename)
    fileid = fileids.index(filename) + 1
    def test():
        loop = BooleanLoop()
        launcher.on_end_transfer(loop.stop, 'rep1', 'rep2', fileid)
        generate(os.path.join(rep1, filename), size)
        loop.run(timeout=5)
        assert(checksum(os.path.join(rep1, filename)) ==
               checksum(os.path.join(rep2, filename)))
    return test


test_simple_copy = gen_test_copy('foo', 100)
test_other_copy = gen_test_copy('bar', 100)
test_bigger_copy = gen_test_copy('foo', 1000)
test_smaller_copy = gen_test_copy('foo', 10)


def test_end():
    launcher.quit()
    launcher.wait()

{'fraction_non_alphanumeric': 0.12652492157546183, 'fraction_numerical': 0.017079121645172533, 'mean_word_length': 4.097690941385435, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3719523', 'n_tokens_mistral': 1015, 'n_tokens_neox': 979, 'n_words': 218}