<commit_before>from __future__ import absolute_import

import collections
from os import listdir, walk
from os.path import basename, isdir, join

from pickle_blosc import pickle, unpickle
from tqdm import tqdm

from ._elapsed import BeginEnd
from ._path import cp, folder_hash, make_sure_path_exists, temp_folder


def pickle_merge(folder):
    """Merges pickle files and save it to `all.pkl`."""
    file_list = _get_file_list(folder)

    if len(file_list) == 0:
        print('There is nothing to merge because no file' +
              ' has been found in %s.' % folder)
        return

    ha = folder_hash(folder, ['all.pkl', '.folder_hash'])

    subfolders = [d for d in listdir(folder) if isdir(join(folder, d))]

    with temp_folder() as tf:
        desc = 'Copying files from %s' % _summarize(folder, 16)
        for sf in tqdm(subfolders, desc=desc):
            make_sure_path_exists(join(tf, sf))
            cp(join(folder, sf), join(tf, sf))
<commit_msg>copying files<commit_after>from __future__ import absolute_import

import collections
from os import listdir, walk
from os.path import basename, isdir, join

from pickle_blosc import pickle, unpickle
from tqdm import tqdm

from ._elapsed import BeginEnd
from ._path import cp, folder_hash, make_sure_path_exists, temp_folder


def pickle_merge(folder):
    """Merges pickle files and save it to `all.pkl`."""
    file_list = _get_file_list(folder)

    if len(file_list) == 0:
        print('There is nothing to merge because no file' +
              ' has been found in %s.' % folder)
        return

    ha = folder_hash(folder, ['all.pkl', '.folder_hash'])

    subfolders = [d for d in listdir(folder) if isdir(join(folder, d))]

    with temp_folder() as tf:
        desc = 'Copying files' % _summarize(folder, 16)
        for sf in tqdm(subfolders, desc=desc):
            make_sure_path_exists(join(tf, sf))
            cp(join(folder, sf), join(tf, sf))

{'fraction_non_alphanumeric': 0.09989648033126294, 'fraction_numerical': 0.003105590062111801, 'mean_word_length': 3.3536036036036037, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3919724', 'n_tokens_mistral': 644, 'n_tokens_neox': 629, 'n_words': 203}