<commit_before># -*- coding: utf-8 -*-

import os
import os.path
import fnmatch
import shutil
import time

from nimp.utilities.logging import *
from nimp.utilities.paths   import *

def list_files(context):
    return _FileListSet(vars(context))

def copy_files(context):
    return _CopyFileSet(vars(context))

def copy_newer(context, source, destination):
    files = copy_files(context).frm(source).newer().to(destination)
    files = files.recursive().add('*')
    files.process(shutil.copy)

def add_to_p4(context, transaction, path):
    files = list_files(context).frm(path).recursive().add('*').process(transaction.add)

def copy_mkdest_dir(source, destination):
    target_dir = os.path.dirname(destination)
    if not os.path.exists(target_dir):
        os.makedirs(target_dir)
    shutil.copy2(source, destination)

class _FileSet(object):
    """ This class is used to enumerate and copy files with a 'fluent' syntax,
        allowing quite complex operations on paths, like including, excluding
        globs, capitalizing file names, relocationg subdirectories in target
        directory, aggregate modification dates, etc...
        For example, to copy all pdbs recreating the source arborescence in
        'source_directory' in the target directory:
        copy_set().from('/a/path/to/source/directory).to('/target_directory').recursive().add('*')(shutils.copy)
        To submit all'.bnk' except one matching 'E1*' files in a dir :
        with p4_transaction("Update wwise banks) as trans:
            copy_list().exclude('E1*').add('WwiseDir/**/*.bnk')(trans.add)
    """
    def __init__(self, add_callback, source_path = '', format_args = {}, next = None):
        super(_FileSet, self).__init__()
        self._format_args   = format_args
        self._source_path   = os.path.normpath(source_path)
        self._add_callback  = add_callback
        self._next          = next

    def add(self, *globs):
        base_path_len = len(split_path(self._source_path))
        for glob_path in globs:
            glob_path = os.path.join(self._source_path, glob_path)<commit_msg>Trying removing destination file when shutil.copy2 fails<commit_after># -*- coding: utf-8 -*-

import os
import os.path
import fnmatch
import shutil
import time

from nimp.utilities.logging import *
from nimp.utilities.paths   import *

def list_files(context):
    return _FileListSet(vars(context))

def copy_files(context):
    return _CopyFileSet(vars(context))

def copy_newer(context, source, destination):
    files = copy_files(context).frm(source).newer().to(destination)
    files = files.recursive().add('*')
    files.process(shutil.copy)

def add_to_p4(context, transaction, path):
    files = list_files(context).frm(path).recursive().add('*').process(transaction.add)

def copy_mkdest_dir(source, destination):
    target_dir = os.path.dirname(destination)
    if not os.path.exists(target_dir):
        os.makedirs(target_dir)
    try:
        shutil.copy2(source, destination)
    except:
        log_verbose("Error running shutil.copy2 {0} {1}, trying by deleting destination file first", source, destination)
        os.remove(destination)
        shutil.copy2(source, destination)

class _FileSet(object):
    """ This class is used to enumerate and copy files with a 'fluent' syntax,
        allowing quite complex operations on paths, like including, excluding
        globs, capitalizing file names, relocationg subdirectories in target
        directory, aggregate modification dates, etc...
        For example, to copy all pdbs recreating the source arborescence in
        'source_directory' in the target directory:
        copy_set().from('/a/path/to/source/directory).to('/target_directory').recursive().add('*')(shutils.copy)
        To submit all'.bnk' except one matching 'E1*' files in a dir :
        with p4_transaction("Update wwise banks) as trans:
            copy_list().exclude('E1*').add('WwiseDir/**/*.bnk')(trans.add)
    """
    def __init__(self, add_callback, source_path = '', format_args = {}, next = None):
        super(_FileSet, self).__init__()
        self._format_args   = format_args
        self._source_path   = os.path.normpath(source_path)
        self._add_callback  = add_callback
        self._next          = next

    def add(self, *globs):
        base_path_len = len(split_path(self._source_path))
        for glob_path in globs:
            glob_path = os.path.join(self._source_path, glob_path)
{'fraction_non_alphanumeric': 0.11561729778176115, 'fraction_numerical': 0.003809097019941743, 'mean_word_length': 3.495468277945619, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5255910', 'n_tokens_mistral': 1353, 'n_tokens_neox': 1332, 'n_words': 331}