<commit_before>import itertools
from pathlib import Path
import re
import urllib.request
import random
import string


BOX_PATH = [Path('../Box Sync/EDGAR Team'), Path('~/Box Sync/EDGAR Team').expanduser()]
RESULTS = Path('results')


def sanitize_fname(fname):
    fname = re.sub(r'[^a-zA-Z0-9_-]', '_', fname)
    fname = re.sub(r'_{2,}', '_', fname)
    fname = re.sub(r'_$', '', fname)
    return fname


def unused_fname(dire, fname):
    # TODO: make this more general, eliminate duplicated code

    def candidates():
        yield dire / fname  # try just dire/fname
        for n in itertools.count(1):
            yield dire / f'{fname}_{n}'  # otherwise add n

    for try_fname in candidates():
        if not try_fname.exists():<commit_msg>fixed bugs for compatibility with python3.5<commit_after>import itertools
from pathlib import Path
import re
import urllib.request
import random
import string


BOX_PATH = [Path('../Box Sync/EDGAR Team'), Path('~/Box Sync/EDGAR Team').expanduser(), Path('~/box/EDGAR Team').expanduser()]
RESULTS = Path('results')


def sanitize_fname(fname):
    fname = re.sub(r'[^a-zA-Z0-9_-]', '_', fname)
    fname = re.sub(r'_{2,}', '_', fname)
    fname = re.sub(r'_$', '', fname)
    return fname


def unused_fname(dire, fname):
    # TODO: make this more general, eliminate duplicated code

    def candidates():
        yield dire / fname  # try just dire/fname
        for n in itertools.count(1):
            yield dire / '{fname}_{n}'.format(**locals)  # otherwise add n

    for try_fname in candidates():
        if not try_fname.exists():
{'fraction_non_alphanumeric': 0.1451917033312382, 'fraction_numerical': 0.006285355122564425, 'mean_word_length': 3.3143631436314362, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7526781', 'n_tokens_mistral': 554, 'n_tokens_neox': 537, 'n_words': 150}