<commit_before>import os      as _os
import re      as _re
import glob    as _glob
import natsort as _natsort

def _getStrBetween(prefix, s, suffix):
    """Return the value inside s between prefix and suffix."""
    preIdx = s.index(prefix)
    sufIdx = s.index(suffix)

    return s[preIdx + len(prefix):sufIdx]

def _doAtomicFileCreation(filePath):
    """Atomically tries to create the file."""
    try:
        _os.close(_os.open(filePath, _os.O_CREAT | _os.O_EXCL))
        return True
    except OSError:
        return False


def _findNextFile(folder, prefix, suffix, fnameGen, base, maxattempts, loop):
    if loop >= maxattempts:
        raise OSError('Unable to create file after ' + str(maxattempts) + ' attempts.')

    if prefix is None and fnameGen is None:
        raise ValueError("Need at least one of prefix or fnameGen to proceed.")

    if (prefix is not None or suffix is not None) and fnameGen is not None:
        raise ValueError("Cannot provide prefix/suffix as well as an fnameGen.")

    nextFileIdx = base

    if fnameGen is not None:
        while _os.path.exists(_os.path.join(folder, fnameGen(nextFileIdx))):
            nextFileIdx += 1

    if prefix is not None or suffix is not None:
        prefix = prefix if prefix is not None else ''
        suffix = suffix if suffix is not None else ''

        globPattern = lambda x : _os.path.join(folder, prefix + x + suffix)
        allFiles = _natsort.natsorted(_glob.glob(globPattern('*')), alg=_natsort.ns.INT)
        selectFilesRegEx = _re.compile(globPattern('[0-9]+'))
        numberedFiles = [ f for f in allFiles if _re.match(selectFilesRegEx, f) ]

        if len(numberedFiles) > 0:
            nextFileIdx = int(_getStrBetween(prefix, numberedFiles[-1], suffix)) + 1


    if fnameGen is None:
        retFileName = _os.path.join(folder, prefix + str(nextFileIdx) + suffix)
    else:
        retFileName = _os.path.join(folder, fnameGen(nextFileIdx))

    if _doAtomicFileCreation(retFileName):
        return retFileName
    else:
        return _findNextFile(folder, prefix, suffix, fnameGen, base, maxattempts, loop + 1)



###########################################################################################
## Public interface
###########################################################################################


def findNextFile(folder='.', prefix=None, suffix=None, fnameGen=None, base=0, maxattempts=10):
    """Finds the next available file-name in a sequence.

    This function will create a file of zero size and will return a path to it
    to the caller. No files which exist will be harmed in this operation and
    concurrent executions of this function will return separate files. In case
    of conflict, the function will attempt to generate a new file name up to
    maxattempts number of times before failing.

    The sequence will start from the base argument (default: 0).

    If used with the prefix/suffix form, it will look for the next file in the
    sequence ignoring any gaps in the sequence. Hence, if the files "a.0.txt"
    and "a.3.txt" exist, then the next file returned will be "a.4.txt" when
    called with prefix="a." and suffix=".txt".

    In case fnameGen is provided, the first generated filename which does not
    exist will be created and its path will be returned. Hence, if the files
    "a.0.txt" and "a.3.txt" exist, then the next file returned will be
    "a.1.txt" if called with fnameGen = lambda x : "a." + str(x) + ".txt"


    Args:
        folder      - string which has path to the folder where the file should be created (default: '.')
        prefix      - prefix of the file to be generated (default: '')
        suffix      - suffix of the file to be generated (default: '')
        fnameGen    - function which generates the filenames given a number as input (default: None)
        base        - the first index to count (default: 0)
        maxattempts - number of attempts to create the file before failing with OSError (default: 10)

    Returns:
        Path of the file which follows the provided pattern and can be written to.

    Raises:
        RuntimeError - If an incorrect combination of arguments is provided.
        OSError      - If is unable to create a file on the disk.
    """
    return _findNextFile(folder, prefix, suffix, fnameGen, base, maxattempts, 0)


<commit_msg>Don't retry if file creation fails for uninteresting reasons.<commit_after>import os      as _os
import re      as _re
import glob    as _glob
import errno   as _errno
import natsort as _natsort

def _getStrBetween(prefix, s, suffix):
    """Return the value inside s between prefix and suffix."""
    preIdx = s.index(prefix)
    sufIdx = s.index(suffix)

    return s[preIdx + len(prefix):sufIdx]

def _doAtomicFileCreation(filePath):
    """Atomically tries to create the file."""
    try:
        _os.close(_os.open(filePath, _os.O_CREAT | _os.O_EXCL))
        return True
    except OSError as e:
        if e.errno == _errno.EEXIST:
            return False
        else:
            raise e


def _findNextFile(folder, prefix, suffix, fnameGen, base, maxattempts, loop):
    if loop >= maxattempts:
        raise OSError('Unable to create file after ' + str(maxattempts) + ' attempts.')

    if prefix is None and fnameGen is None:
        raise ValueError("Need at least one of prefix or fnameGen to proceed.")

    if (prefix is not None or suffix is not None) and fnameGen is not None:
        raise ValueError("Cannot provide prefix/suffix as well as an fnameGen.")

    nextFileIdx = base

    if fnameGen is not None:
        while _os.path.exists(_os.path.join(folder, fnameGen(nextFileIdx))):
            nextFileIdx += 1

    if prefix is not None or suffix is not None:
        prefix = prefix if prefix is not None else ''
        suffix = suffix if suffix is not None else ''

        globPattern = lambda x : _os.path.join(folder, prefix + x + suffix)
        allFiles = _natsort.natsorted(_glob.glob(globPattern('*')), alg=_natsort.ns.INT)
        selectFilesRegEx = _re.compile(globPattern('[0-9]+'))
        numberedFiles = [ f for f in allFiles if _re.match(selectFilesRegEx, f) ]

        if len(numberedFiles) > 0:
            nextFileIdx = int(_getStrBetween(prefix, numberedFiles[-1], suffix)) + 1


    if fnameGen is None:
        retFileName = _os.path.join(folder, prefix + str(nextFileIdx) + suffix)
    else:
        retFileName = _os.path.join(folder, fnameGen(nextFileIdx))

    if _doAtomicFileCreation(retFileName):
        return retFileName
    else:
        return _findNextFile(folder, prefix, suffix, fnameGen, base, maxattempts, loop + 1)



###########################################################################################
## Public interface
###########################################################################################


def findNextFile(folder='.', prefix=None, suffix=None, fnameGen=None, base=0, maxattempts=10):
    """Finds the next available file-name in a sequence.

    This function will create a file of zero size and will return a path to it
    to the caller. No files which exist will be harmed in this operation and
    concurrent executions of this function will return separate files. In case
    of conflict, the function will attempt to generate a new file name up to
    maxattempts number of times before failing.

    The sequence will start from the base argument (default: 0).

    If used with the prefix/suffix form, it will look for the next file in the
    sequence ignoring any gaps in the sequence. Hence, if the files "a.0.txt"
    and "a.3.txt" exist, then the next file returned will be "a.4.txt" when
    called with prefix="a." and suffix=".txt".

    In case fnameGen is provided, the first generated filename which does not
    exist will be created and its path will be returned. Hence, if the files
    "a.0.txt" and "a.3.txt" exist, then the next file returned will be
    "a.1.txt" if called with fnameGen = lambda x : "a." + str(x) + ".txt"


    Args:
        folder      - string which has path to the folder where the file should be created (default: '.')
        prefix      - prefix of the file to be generated (default: '')
        suffix      - suffix of the file to be generated (default: '')
        fnameGen    - function which generates the filenames given a number as input (default: None)
        base        - the first index to count (default: 0)
        maxattempts - number of attempts to create the file before failing with OSError (default: 10)

    Returns:
        Path of the file which follows the provided pattern and can be written to.

    Raises:
        RuntimeError - If an incorrect combination of arguments is provided.
        OSError      - If is unable to create a file on the disk.
    """
    return _findNextFile(folder, prefix, suffix, fnameGen, base, maxattempts, 0)



{'fraction_non_alphanumeric': 0.11837515472037809, 'fraction_numerical': 0.004726004275908631, 'mean_word_length': 2.9946067415730337, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '306622', 'n_tokens_mistral': 2545, 'n_tokens_neox': 2424, 'n_words': 1024}