<commit_before>from .logger import flogger
flog = flogger(label="BUSYDIR")

import os
import time
from contextlib import contextmanager


@contextmanager
def locked_directory(dirname, retry_after=5, timeout=600, busy_flag="__BUSY__"):
	busy_file = os.path.join(dirname, busy_flag)

	# check if there's another lock
	while os.path.exists(busy_file) and timeout>0:
		flog.info(f"waiting on {dirname}")
		timeout -= retry_after
		time.sleep(retry_after)

	if timeout <= 0:
		raise BlockingIOError(f'directory {dirname} is locked')

	# ready to lock
	flog.info(f"locking {dirname}")
	with open(busy_file, 'w') as f1:
		f1.write('busy')

	yield dirname

	flog.info(f"releasing {dirname}")
	os.remove(busy_file)




def lock_for_time(dirname, delay=10, retry_after=5, timeout=600, busy_flag="__BUSY__"):
	from threading import Timer

	busy_file = os.path.join(dirname, busy_flag)

	# check if there's another lock
	while os.path.exists(busy_file) and timeout>0:
		flog.info(f"waiting on {dirname}")
		timeout -= retry_after
		time.sleep(retry_after)

	if timeout <= 0:
		raise BlockingIOError(f'directory {dirname} is locked')

	# ready to lock
	flog.info(f"locking {dirname} for {delay}")
	with open(busy_file, 'w') as f1:
		f1.write('busy')

	def release():
		os.remove(busy_file)

	t = Timer(delay, release)
	t.start()



<commit_msg>catch exceptions<commit_after>from .logger import flogger
flog = flogger(label="BUSYDIR")

import os
import time
from contextlib import contextmanager


@contextmanager
def locked_directory(dirname, retry_after=5, timeout=600, busy_flag="__BUSY__"):
	busy_file = os.path.join(dirname, busy_flag)

	# check if there's another lock
	while os.path.exists(busy_file) and timeout>0:
		flog.info(f"waiting on {dirname}")
		timeout -= retry_after
		time.sleep(retry_after)

	if timeout <= 0:
		raise BlockingIOError(f'directory {dirname} is locked')

	# ready to lock
	flog.info(f"locking {dirname}")
	with open(busy_file, 'w') as f1:
		f1.write('busy')

	try:
		yield dirname
	except:
		raise
	finally:
		flog.info(f"releasing {dirname}")
		os.remove(busy_file)




def lock_for_time(dirname, delay=10, retry_after=5, timeout=600, busy_flag="__BUSY__"):
	from threading import Timer

	busy_file = os.path.join(dirname, busy_flag)

	# check if there's another lock
	while os.path.exists(busy_file) and timeout>0:
		flog.info(f"waiting on {dirname}")
		timeout -= retry_after
		time.sleep(retry_after)

	if timeout <= 0:
		raise BlockingIOError(f'directory {dirname} is locked')

	# ready to lock
	flog.info(f"locking {dirname} for {delay}")
	with open(busy_file, 'w') as f1:
		f1.write('busy')

	def release():
		os.remove(busy_file)

	t = Timer(delay, release)
	t.start()




{'fraction_non_alphanumeric': 0.12555555555555556, 'fraction_numerical': 0.013333333333333334, 'mean_word_length': 5.042505592841163, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3577450', 'n_tokens_mistral': 1071, 'n_tokens_neox': 998, 'n_words': 241}