<commit_before>"""A thread-safe file operation package for python"""
import os
import shutil
import tempfile
from hashlib import md5
from contextlib import contextmanager
from threading import Lock
from filelock import FileLock
import cmdy

MULTILOCK = Lock()
TMPDIR    = os.path.join(tempfile.gettempdir(), 'fsutil.locks')
if not os.path.exists(TMPDIR): # pragma: no cover
	os.makedirs(TMPDIR)

class TargetExistsError(OSError):
	"""Raise when target exists and not able to overwrite"""

class TargetNotExistsError(OSError):
	"""Raise when target does not exist and not able to ignore"""

def _getLockFile(path):
	basename = md5(str(path).encode()).hexdigest()
	return os.path.join(TMPDIR, basename + '.lock')

@contextmanager
def lock(*files, resolve = False):
	"""Lock file context"""
	files = [os.path.realpath(path) if resolve else os.path.abspath(path) for path in files]
	locks = [FileLock(_getLockFile(path)) for path in files]
	if len(files) == 1:
		with locks[0]:
			yield locks[0]
	else:
		with MULTILOCK:<commit_msg>Use TEMPDIR instead for _fsutil to avoid creating a subdirectory non-threading-safely.<commit_after>"""A thread-safe file operation package for python"""
import os
import shutil
import tempfile
from hashlib import md5
from contextlib import contextmanager
from threading import Lock
from filelock import FileLock
import cmdy

MULTILOCK = Lock()
TMPDIR    = tempfile.gettempdir()
#TMPDIR   = os.path.join(tempfile.gettempdir(), 'fsutil.locks')
## not threading-safe!!
# if not os.path.exists(TMPDIR): # pragma: no cover
# 	os.makedirs(TMPDIR)

class TargetExistsError(OSError):
	"""Raise when target exists and not able to overwrite"""

class TargetNotExistsError(OSError):
	"""Raise when target does not exist and not able to ignore"""

def _getLockFile(path):
	basename = md5(str(path).encode()).hexdigest()
	return os.path.join(TMPDIR, basename + '.lock')

@contextmanager
def lock(*files, resolve = False):
	"""Lock file context"""
	files = [os.path.realpath(path) if resolve else os.path.abspath(path) for path in files]
	locks = [FileLock(_getLockFile(path)) for path in files]
	if len(files) == 1:
		with locks[0]:
			yield locks[0]
	else:
		with MULTILOCK:
{'fraction_non_alphanumeric': 0.11547238703788225, 'fraction_numerical': 0.004564125969876769, 'mean_word_length': 5.372093023255814, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1951036', 'n_tokens_mistral': 729, 'n_tokens_neox': 709, 'n_words': 231}