<commit_before>#!/usr/bin/env python
#
# Auxiliary functions library
# Copyright (C) 2015 <NAME>
#

import os
import argparse
from pathlib2 import PurePath, PureWindowsPath, PurePosixPath # opposite operation of os.path.join (split a path into parts)
import posixpath # to generate unix paths
import shutil

try:
    from scandir import walk # use the faster scandir module if available (Python >= 3.5), see https://github.com/benhoyt/scandir
except ImportError:
    from os import walk # else, default to os.walk()


def is_file(dirname):
    '''Checks if a path is an actual file that exists'''
    if not os.path.isfile(dirname):
        msg = "{0} is not an existing file".format(dirname)
        raise argparse.ArgumentTypeError(msg)
    else:
        return dirname

def is_dir(dirname):
    '''Checks if a path is an actual directory that exists'''
    if not os.path.isdir(dirname):
        msg = "{0} is not a directory".format(dirname)
        raise argparse.ArgumentTypeError(msg)
    else:
        return dirname

def is_dir_or_file(dirname):
    '''Checks if a path is an actual directory that exists or a file'''
    if not os.path.isdir(dirname) and not os.path.isfile(dirname):
        msg = "{0} is not a directory nor a file".format(dirname)
        raise argparse.ArgumentTypeError(msg)
    else:
        return dirname

def fullpath(relpath):
    '''Relative path to absolute'''
    if (type(relpath) is object or type(relpath) is file):
        relpath = relpath.name
    return os.path.abspath(os.path.expanduser(relpath))

def recwalk(inputpath, sorting=True):
    '''Recursively walk through a folder. This provides a mean to flatten out the files restitution (necessary to show a progress bar). This is a generator.'''
    # If it's only a single file, return this single file
    if os.path.isfile(inputpath):
        abs_path = fullpath(inputpath)
        yield os.path.dirname(abs_path), os.path.basename(abs_path)
    # Else if it's a folder, walk recursively and return every files
    else:
        for dirpath, dirs, files in walk(inputpath):	
            if sorting:
                files.sort()
                dirs.sort() # sort directories in-place for ordered recursive walking
            for filename in files:
                yield (dirpath, filename) # return directory (full path) and filename

def sizeof_fmt(num, suffix='B', mod=1024.0):
    '''Readable size format, courtesy of <NAME>'''
    for unit in ['','K','M','G','T','P','E','Z']:
        if abs(num) < mod:
            return "%3.1f%s%s" % (num, unit, suffix)
        num /= mod
    return "%.1f%s%s" % (num, 'Y', suffix)

def path2unix(path, fromwinpath=False):
    '''From a path given in any format, converts to posix path format
    fromwinpath=True forces the input path to be recognized as a Windows path (useful on Unix machines to unit test Windows paths)'''
    if fromwinpath:
        return posixpath.join(*list(PureWindowsPath(path).parts))
    else:
        return posixpath.join(*list(PurePath(path).parts))

def create_dir_if_not_exist(path):
    '''Create a directory if it does not already exist, else nothing is done and no error is return'''
    if not os.path.exists(path):
        os.makedirs(path)<commit_msg>delete redundant code<commit_after>#!/usr/bin/env python
#
# Auxiliary functions library
# Copyright (C) 2015 <NAME>
#

import os
import argparse
from pathlib2 import PurePath, PureWindowsPath, PurePosixPath # opposite operation of os.path.join (split a path into parts)
import posixpath # to generate unix paths
import shutil

try:
    from scandir import walk # use the faster scandir module if available (Python >= 3.5), see https://github.com/benhoyt/scandir
except ImportError:
    from os import walk # else, default to os.walk()


def is_file(dirname):
    '''Checks if a path is an actual file that exists'''
    if not os.path.isfile(dirname):
        msg = "{0} is not an existing file".format(dirname)
        raise argparse.ArgumentTypeError(msg)
    else:
        return dirname

def is_dir(dirname):
    '''Checks if a path is an actual directory that exists'''
    if not os.path.isdir(dirname):
        msg = "{0} is not a directory".format(dirname)
        raise argparse.ArgumentTypeError(msg)
    else:
        return dirname

def is_dir_or_file(dirname):
    '''Checks if a path is an actual directory that exists or a file'''
    if not os.path.isdir(dirname) and not os.path.isfile(dirname):
        msg = "{0} is not a directory nor a file".format(dirname)
        raise argparse.ArgumentTypeError(msg)
    else:
        return dirname

def fullpath(relpath):
    '''Relative path to absolute'''
    if (type(relpath) is object or type(relpath) is file):
        relpath = relpath.name
    return os.path.abspath(os.path.expanduser(relpath))

def recwalk(inputpath, sorting=True):
    '''Recursively walk through a folder. This provides a mean to flatten out the files restitution (necessary to show a progress bar). This is a generator.'''
    # If it's only a single file, return this single file
    if os.path.isfile(inputpath):
        abs_path = fullpath(inputpath)
        yield os.path.dirname(abs_path), os.path.basename(abs_path)
    # Else if it's a folder, walk recursively and return every files
    else:
        for dirpath, dirs, files in walk(inputpath):	
            if sorting:
                files.sort()
                dirs.sort() # sort directories in-place for ordered recursive walking
            for filename in files:
                yield (dirpath, filename) # return directory (full path) and filename

def sizeof_fmt(num, suffix='B', mod=1024.0):
    '''Readable size format, courtesy of <NAME>'''
    for unit in ['','K','M','G','T','P','E','Z']:
        if abs(num) < mod:
            return "%3.1f%s%s" % (num, unit, suffix)
        num /= mod
    return "%.1f%s%s" % (num, 'Y', suffix)

def path2unix(path, fromwinpath=False):
    '''From a path given in any format, converts to posix path format
    fromwinpath=True forces the input path to be recognized as a Windows path (useful on Unix machines to unit test Windows paths)'''
    if fromwinpath:
        return posixpath.join(*list(PureWindowsPath(path).parts))
    else:
        return posixpath.join(*list(PurePath(path).parts))
{'fraction_non_alphanumeric': 0.10576616756928958, 'fraction_numerical': 0.006052883083784644, 'mean_word_length': 3.1445544554455447, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3344141', 'n_tokens_mistral': 1917, 'n_tokens_neox': 1859, 'n_words': 679}