<commit_before>from __future__ import absolute_import

import os
import shutil
import locket
from toolz import memoize
from contextlib import contextmanager
from threading import Lock


locks = dict()
locks = dict()


@contextmanager
def lock(path):
    if path not in locks:
        locks[path] = Lock(), locket.lock_file(os.path.join(path, '.lock'))
    thread_lock, file_lock = locks[path]
    with thread_lock:
        with file_lock:
            yield


def escape_filename(fn):
    """ Escape text so that it is a valid filename

    >>> escape_filename('Foo!/bar')
    'Foobar'

    http://stackoverflow.com/questions/295135/turn-a-string-into-a-valid-filename-in-python
    """
    import string
    valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
    return ''.join(filter(valid_chars.__contains__, fn))

def filename(path, key):
    return os.path.join(path, escape_filename(str(key)))


def create(path):
    os.mkdir(path)
    with lock(path): pass

<commit_msg>revert to lock without threading.lock<commit_after>from __future__ import absolute_import

import os
import shutil
import locket
from toolz import memoize
from contextlib import contextmanager


locks = dict()


def lock(path):
    if path not in locks:
        locks[path] = locket.lock_file(os.path.join(path, '.lock'))
    return locks[path]


def escape_filename(fn):
    """ Escape text so that it is a valid filename

    >>> escape_filename('Foo!/bar')
    'Foobar'

    http://stackoverflow.com/questions/295135/turn-a-string-into-a-valid-filename-in-python
    """
    import string
    valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
    return ''.join(filter(valid_chars.__contains__, fn))

def filename(path, key):
    return os.path.join(path, escape_filename(str(key)))


def create(path):
    os.mkdir(path)
    with lock(path): pass


{'fraction_non_alphanumeric': 0.12137486573576799, 'fraction_numerical': 0.00644468313641246, 'mean_word_length': 3.680904522613065, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3570116', 'n_tokens_mistral': 627, 'n_tokens_neox': 586, 'n_words': 152}