<commit_before>#!/usr/bin/env python


import click
from datetime import datetime
from ._transfer import parallel_rsync


@click.group()
def ekpy():
    pass


@ekpy.command()
@click.argument('sources', type=click.File(), nargs=1)
@click.option('--dest', '-d', required=True, help='destination')
@click.option('--processes', '-p', default=10)
def par_sync(sources, dest, processes):
    """
    Parallel rsync to copy large amount of data
    """
    files = [source.strip() for source in sources.read().split('\n')]

    print 'Copy {} files'.format(len(files))

    start = datetime.now()
    parallel_rsync(files, dest, n=processes)
    end = datetime.now()

    duration = end - start

    print 'Copied {} files in {}'.format(len(files), duration)
<commit_msg>works now with python3, added documentation.<commit_after>#!/usr/bin/env python


import click
from datetime import datetime
from ._transfer import parallel_rsync


@click.group()
def ekpy():
    pass


@ekpy.command()
@click.argument('sources', type=click.File(), nargs=1, help="File with all (ssh-)paths to the files to be copied: "
                                                            "'<EMAIL>:/path/to/file.root' \n "
                                                            "Note: ssh forwarding must be enabled.")
@click.option('--dest', '-d', type=str, required=True, help='Target directory')
@click.option('--processes', '-p', default=10, help='Number of parallel processes.')
def par_sync(sources, dest, processes):
    """
    Parallel rsync to copy large amount of data
    """
    files = [source.strip() for source in sources.read().split('\n')]

    print('Copy {} files'.format(len(files)))

    start = datetime.now()
    parallel_rsync(files, dest, n=processes)
    end = datetime.now()

    duration = end - start

    print('Copied {} files in {}'.format(len(files), duration))

{'fraction_non_alphanumeric': 0.1553657234383342, 'fraction_numerical': 0.0037373198077949813, 'mean_word_length': 3.0475161987041037, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3777691', 'n_tokens_mistral': 563, 'n_tokens_neox': 543, 'n_words': 155}