<commit_before># -*- coding: utf-8 -*-
# @author: vuolter

from __future__ import absolute_import, unicode_literals

import io
import os
import shutil
from builtins import dict, int, next

import portalocker
import psutil
import send2trash
from future import standard_library

from pyload.utils.layer.legacy import hashlib

standard_library.install_aliases()

try:
    import magic
except ImportError:
    magic = None
    from filetype import guess_mime
try:
    import zlib
except ImportError:
    zlib = None


def availspace(path):
    if os.name != 'nt':
        stat = os.statvfs(path)
        res = stat.f_frsize * stat.f_bavail
    else:
        import ctypes
        free_bytes = ctypes.c_ulonglong(0)
        ctypes.windll.kernel32.GetDiskFreeSpaceExW(
            ctypes.c_wchar_p(path), None, None, ctypes.pointer(free_bytes))
        res = int(free_bytes.value)
    return res


def _shdo(func, src, dst, overwrite=None, ref=None):
    mtime = os.path.getmtime
    try:
        if os.path.isfile(dst):
            if overwrite is None and mtime(src) <= mtime(dst):
                return
            elif not overwrite:
                return
            if os.name == 'nt':
                os.remove(dst)
        func(src, dst)
        if isinstance(ref, list):
            del ref[:]
    except Exception:
        pass


def _shdorc(func, filenames, src_dir, dst_dir, overwrite=None):
    join = os.path.join
    for fname in filenames:
        src_file = join(src_dir, fname)
        dst_file = join(dst_dir, fname)
        _shdo(func, src_file, dst_file, overwrite)


def _copyrc(src, dst, overwrite, preserve_metadata):
    copy = shutil.copy2 if preserve_metadata else shutil.copy
    copytree = shutil.copytree
    exists = os.path.exists
    for src_dir, dirnames, filenames in os.walk(src):
        dst_dir = src_dir.replace(src, dst, 1)
        if exists(dst_dir):
            _shdorc(copy, filenames, src_dir, dst_dir, overwrite)
        else:
            _shdo(copytree, src_dir, dst_dir, overwrite, dirnames)


def copy(src, dst, overwrite=None, preserve_metadata=True):
    if not os.path.isdir(dst) or not os.path.isdir(src):
        return _shdo(shutil.copytree, src, dst, overwrite)
    return _copyrc(src, dst, overwrite, preserve_metadata)


def exists(path, strict=False):
    """Case-sensitive os.path.exists."""
    if not strict:
        return os.path.exists(path)
    if os.path.exists(path):
        dirpath, name = os.path.split(path)
        return name in os.listdir(dirpath)
    return False


def filesize(filename):
    return os.stat(filename).st_size


def filetype(filename):
    try:
        return magic.from_file(filename, mime=True)
    except AttributeError:
        pass
    return guess_mime(filename)


def decode(path):
    try:
        upath = unicode(path)
    except NameError:
        upath = os.fsdecode(path)
    return upath


def fullpath(path):
    return os.path.realpath(os.path.expanduser(path))

<commit_msg>Rewrite fs encode/decode<commit_after># -*- coding: utf-8 -*-
# @author: vuolter

from __future__ import absolute_import, unicode_literals

import io
import os
import shutil
from builtins import dict, int, next

import portalocker
import psutil
import send2trash
from future import standard_library

from pyload.utils.convert import to_bytes, to_str
from pyload.utils.layer.legacy import hashlib

standard_library.install_aliases()

try:
    import magic
except ImportError:
    magic = None
    from filetype import guess_mime
try:
    import zlib
except ImportError:
    zlib = None


def availspace(path):
    if os.name != 'nt':
        stat = os.statvfs(path)
        res = stat.f_frsize * stat.f_bavail
    else:
        import ctypes
        free_bytes = ctypes.c_ulonglong(0)
        ctypes.windll.kernel32.GetDiskFreeSpaceExW(
            ctypes.c_wchar_p(path), None, None, ctypes.pointer(free_bytes))
        res = int(free_bytes.value)
    return res


def _shdo(func, src, dst, overwrite=None, ref=None):
    mtime = os.path.getmtime
    try:
        if os.path.isfile(dst):
            if overwrite is None and mtime(src) <= mtime(dst):
                return
            elif not overwrite:
                return
            if os.name == 'nt':
                os.remove(dst)
        func(src, dst)
        if isinstance(ref, list):
            del ref[:]
    except Exception:
        pass


def _shdorc(func, filenames, src_dir, dst_dir, overwrite=None):
    join = os.path.join
    for fname in filenames:
        src_file = join(src_dir, fname)
        dst_file = join(dst_dir, fname)
        _shdo(func, src_file, dst_file, overwrite)


def _copyrc(src, dst, overwrite, preserve_metadata):
    copy = shutil.copy2 if preserve_metadata else shutil.copy
    copytree = shutil.copytree
    exists = os.path.exists
    for src_dir, dirnames, filenames in os.walk(src):
        dst_dir = src_dir.replace(src, dst, 1)
        if exists(dst_dir):
            _shdorc(copy, filenames, src_dir, dst_dir, overwrite)
        else:
            _shdo(copytree, src_dir, dst_dir, overwrite, dirnames)


def copy(src, dst, overwrite=None, preserve_metadata=True):
    if not os.path.isdir(dst) or not os.path.isdir(src):
        return _shdo(shutil.copytree, src, dst, overwrite)
    return _copyrc(src, dst, overwrite, preserve_metadata)


def exists(path, strict=False):
    """Case-sensitive os.path.exists."""
    if not strict:
        return os.path.exists(path)
    if os.path.exists(path):
        dirpath, name = os.path.split(path)
        return name in os.listdir(dirpath)
    return False


def filesize(filename):
    return os.stat(filename).st_size


def filetype(filename):
    try:
        return magic.from_file(filename, mime=True)
    except AttributeError:
        pass
    return guess_mime(filename)

    
def encode(path):
    try:
        return os.fsencode(path)
    except AttributeError:
        return to_bytes(path)
    
    
def decode(path):
    try:
        return os.fsdecode(path)
    except AttributeError:
        return to_str(path)


def fullpath(path):
    return os.path.realpath(os.path.expanduser(path))


{'fraction_non_alphanumeric': 0.09554867112343061, 'fraction_numerical': 0.0022827327572150662, 'mean_word_length': 2.688514732411305, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7276066', 'n_tokens_mistral': 2043, 'n_tokens_neox': 1984, 'n_words': 510}