<commit_before># standard library modules, , ,
import os
import errno
import shutil

def mkDirP(path):
    try:
        os.makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

def rmRf(path):
    try:
        shutil.rmtree(path)
    except OSError as exception:
        if 'cannot call rmtree on a symbolic link' in str(exception).lower():
            os.unlink(path)
        elif exception.errno != errno.ENOENT:
            raise

def rmF(path):
    try:
        os.remove(path)
    except OSError as exception: 
        if exception.errno != errno.ENOENT:
            raise

def fullySplitPath(path):
    components = []
    while True:
        path, component = os.path.split(path)
        if component != '':
            components.append(component)
        else:
            if path != '':
                components.append(path)
            break
    components.reverse()
    return components

def isLink(path):
    # !!! FIXME: when windows symlinks are supported this check needs to
    # support them too
    return os.path.islink(path)

def tryReadLink(path):
    try:
        return os.readlink(path)
    except OSError as e:
        return None

def symlink(src, dst):
    # !!! FIXME: recent windowses do support symlinks, but os.symlink doesn't
    # work on windows, so use something else
    try:
        os.symlink(src, dst)
    except OSError as exception:
        if exception.errno != errno.EEXIST and (tryReadLink(dst) != src):
            raise<commit_msg>rm before symlinking, fixes #29<commit_after># standard library modules, , ,
import os
import errno
import shutil

def mkDirP(path):
    try:
        os.makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

def rmRf(path):
    try:
        shutil.rmtree(path)
    except OSError as exception:
        if 'cannot call rmtree on a symbolic link' in str(exception).lower():
            os.unlink(path)
        elif exception.errno != errno.ENOENT:
            raise

def rmF(path):
    try:
        os.remove(path)
    except OSError as exception: 
        if exception.errno != errno.ENOENT:
            raise

def fullySplitPath(path):
    components = []
    while True:
        path, component = os.path.split(path)
        if component != '':
            components.append(component)
        else:
            if path != '':
                components.append(path)
            break
    components.reverse()
    return components

def isLink(path):
    # !!! FIXME: when windows symlinks are supported this check needs to
    # support them too
    return os.path.islink(path)

def tryReadLink(path):
    try:
        return os.readlink(path)
    except OSError as e:
        return None

def symlink(source, link_name):
    # !!! FIXME: recent windowses do support symlinks, but os.symlink doesn't
    # work on windows, so use something else
    try:
        # os.symlink doesn't update existing links, so need to rm first
        rmF(link_name)
        os.symlink(source, link_name)
    except OSError as exception:
        if exception.errno != errno.EEXIST and (tryReadLink(link_name) != source):
            raise
{'fraction_non_alphanumeric': 0.09079601990049752, 'fraction_numerical': 0.0006218905472636816, 'mean_word_length': 2.191468253968254, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7351453', 'n_tokens_mistral': 967, 'n_tokens_neox': 943, 'n_words': 290}