<commit_before>import shutil

def copyfile(src_fs, src_path, dst_fs, dst_path, chunk_size=1024*16):

    def getsyspath(_fs, path):
        try:
            return _fs.getsyspath(path)
        except NoSysPathError:
            return ""

    src_syspath = src_fs.getsyspath(src_path)
    dst_syspath = dst_fs.getsyspath(dst_path)

    # System copy if there are two sys paths
    if src_syspath and dst_syspath:
        shutil.copyfile(src_syspath, dst_syspath)
        return

    src = src_fs.open(src_path, 'rb')
    dst = dst_fs.open(dst_path, 'wb')

    # Chunk copy
    while True:
        chunk = src.read(chunk_size)
        if not chunk:
            break
        dst.write(chunk)
<commit_msg>Beginning a utils module to keep various fs related functions in...<commit_after>import shutil

def copyfile(src_fs, src_path, dst_fs, dst_path, chunk_size=1024*16):

    def getsyspath(_fs, path):
        try:
            return _fs.getsyspath(path)
        except NoSysPathError:
            return ""

    src_syspath = src_fs.getsyspath(src_path)
    dst_syspath = dst_fs.getsyspath(dst_path)

    # System copy if there are two sys paths
    if src_syspath and dst_syspath:
        shutil.copyfile(src_syspath, dst_syspath)
        return

    src, dst = None

    try:
        # Chunk copy
        src = src_fs.open(src_path, 'rb')
        dst = dst_fs.open(dst_path, 'wb')

        while True:
            chunk = src.read(chunk_size)
            if not chunk:
                break
            dst.write(chunk)
            
    finally:
        if src is not None:
            src.close()
        if dst is not None:
            dst.close()
{'fraction_non_alphanumeric': 0.08298001211387038, 'fraction_numerical': 0.0072683222289521504, 'mean_word_length': 2.0763500931098697, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1241438', 'n_tokens_mistral': 553, 'n_tokens_neox': 534, 'n_words': 133}