<commit_before>"""
File/directory helper functions
Copying of file trees, creating directories, reading files and more
"""
import os
import datetime
import shutil
import yaml
from collections import OrderedDict


def copytree(src, dst, symlinks = False, ignore = None):
    """
    Copy a tree of files and dirs and merge into existing dir if needed
    Source: http://stackoverflow.com/a/22331852
    """
    if not os.path.exists(dst):
        os.makedirs(dst)
        shutil.copystat(src, dst)
    lst = os.listdir(src)
    if ignore:
        excl = ignore(src, lst)
        lst = [x for x in lst if x not in excl]
    for item in lst:
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if symlinks and os.path.islink(s):
            if os.path.lexists(d):
                os.remove(d)
            os.symlink(os.readlink(s), d)
            try:
                st = os.lstat(s)
                mode = stat.S_IMODE(st.st_mode)
                os.lchmod(d, mode)
            except:
                pass # lchmod not available
        elif os.path.isdir(s):
            copytree(s, d, symlinks, ignore)
        else:
            shutil.copy2(s, d)


def archive_if_exists(filename):
    """
    Move `filename` out of the way, archiving it by appending the current datetime
    Can be a file or a directory<commit_msg>Support for renaming files when copying, through a dict {'orig': 'newfile'}<commit_after>"""
File/directory helper functions
Copying of file trees, creating directories, reading files and more
"""
import os
import datetime
import shutil
import yaml
from collections import OrderedDict


def copytree(src, dst, symlinks=False, ignore=None, rename=None):
    """
    Copy a tree of files and dirs and merge into existing dir if needed
    Optionally rename items with names in the `rename` dict
    Source: http://stackoverflow.com/a/22331852
    """
    if not os.path.exists(dst):
        os.makedirs(dst)
        shutil.copystat(src, dst)
    lst = os.listdir(src)
    if ignore:
        excl = ignore(src, lst)
        lst = [x for x in lst if x not in excl]
    for item in lst:
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if rename and item in rename:
            d = os.path.join(dst, rename[item])
        if symlinks and os.path.islink(s):
            if os.path.lexists(d):
                os.remove(d)
            os.symlink(os.readlink(s), d)
            try:
                st = os.lstat(s)
                mode = stat.S_IMODE(st.st_mode)
                os.lchmod(d, mode)
            except:
                pass # lchmod not available
        elif os.path.isdir(s):
            copytree(s, d, symlinks, ignore, rename)
        else:
            shutil.copy2(s, d)


def archive_if_exists(filename):
    """
    Move `filename` out of the way, archiving it by appending the current datetime
    Can be a file or a directory
{'fraction_non_alphanumeric': 0.09654228004108183, 'fraction_numerical': 0.0061622731941116055, 'mean_word_length': 2.069327731092437, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6112487', 'n_tokens_mistral': 938, 'n_tokens_neox': 911, 'n_words': 297}