<commit_before>__all__ = ['assureDirectoryExists', 'numbits', 'replaceGlobals']

import os.path
import sys
import math
import types

def replaceGlobals(f, g):
	# HACK closure is lost
	assert isinstance(f, types.FunctionType), type(f)
	return types.FunctionType(f.func_code, g, f.func_name, f.func_defaults)

def numbits(size):
	if size <= 1:
		return 0
	else:
		return int(math.ceil(math.log(size, 2)))

def assureDirectoryExists(dirname):
	if not os.path.exists(dirname): os.makedirs(dirname)

def moduleForGlobalDict(glbls):
	assert '__file__' in glbls, "Global dictionary does not come from a module?"

	for name, module in sys.modules.iteritems():
		if module and module.__dict__ is glbls:
			assert module.__file__ == glbls['__file__']
			return (name, module)
	assert False

def itergroupings(iterable, key, value):
	grouping = {}
	for i in iterable:
		group = key(i)
		data  = value(i)
		if not group in grouping:
			grouping[group] = [data]
		else:
			grouping[group].append(data)
	return grouping.iteritems()

def elapsedTimeString(t):
	if t < 1.0:
		return "%.1f ms" % (t*1000.0)
	elif t < 60.0:
		return "%.2f s" % (t)
	elif t < 3600.0:
		return "%.2f m" % (t/60.0)
	else:
		return "%.2f h" % (t/3600.0)

def memorySizeString(sz):
	sz = float(sz)
	if sz < 1024:
		return "%f B" % sz
	elif sz < 1024**2:
		return "%.1f kB" % (sz/(1024))
	elif sz < 1024**3:
		return "%.1f MB" % (sz/(1024**2))
	else:
		return "%.1f GB" % (sz/(1024**3))

# Note that the unique name may change between runs, as it takes the id of a type.
def uniqueSlotName(descriptor):
	# HACK GetSetDescriptors are not really slots?
	assert isinstance(descriptor, (types.MemberDescriptorType, types.GetSetDescriptorType)), (descriptor, type(descriptor), dir(descriptor))
	name     = descriptor.__name__
	objClass = descriptor.__objclass__
	return "%s#%s#%d" % (name, objClass.__name__, id(objClass))<commit_msg>Util: added code for computing bijections.  Currently unused, but may have future application.<commit_after>__all__ = ['assureDirectoryExists', 'numbits', 'replaceGlobals']

import os.path
import sys
import math
import types

def replaceGlobals(f, g):
	# HACK closure is lost
	assert isinstance(f, types.FunctionType), type(f)
	return types.FunctionType(f.func_code, g, f.func_name, f.func_defaults)

def numbits(size):
	if size <= 1:
		return 0
	else:
		return int(math.ceil(math.log(size, 2)))

def assureDirectoryExists(dirname):
	if not os.path.exists(dirname): os.makedirs(dirname)

def moduleForGlobalDict(glbls):
	assert '__file__' in glbls, "Global dictionary does not come from a module?"

	for name, module in sys.modules.iteritems():
		if module and module.__dict__ is glbls:
			assert module.__file__ == glbls['__file__']
			return (name, module)
	assert False

def itergroupings(iterable, key, value):
	grouping = {}
	for i in iterable:
		group = key(i)
		data  = value(i)
		if not group in grouping:
			grouping[group] = [data]
		else:
			grouping[group].append(data)
	return grouping.iteritems()

def elapsedTimeString(t):
	if t < 1.0:
		return "%.1f ms" % (t*1000.0)
	elif t < 60.0:
		return "%.2f s" % (t)
	elif t < 3600.0:
		return "%.2f m" % (t/60.0)
	else:
		return "%.2f h" % (t/3600.0)

def memorySizeString(sz):
	sz = float(sz)
	if sz < 1024:
		return "%f B" % sz
	elif sz < 1024**2:
		return "%.1f kB" % (sz/(1024))
	elif sz < 1024**3:
		return "%.1f MB" % (sz/(1024**2))
	else:
		return "%.1f GB" % (sz/(1024**3))

# Note that the unique name may change between runs, as it takes the id of a type.
def uniqueSlotName(descriptor):
	# HACK GetSetDescriptors are not really slots?
	assert isinstance(descriptor, (types.MemberDescriptorType, types.GetSetDescriptorType)), (descriptor, type(descriptor), dir(descriptor))
	name     = descriptor.__name__
	objClass = descriptor.__objclass__
	return "%s#%s#%d" % (name, objClass.__name__, id(objClass))

def _bijection(a, b):
	c = a+b
	return (c*(c+1))//2+a

def bijection(a, b, *others):
	result = _bijection(a, b)
	for o in others:
		result = _bijection(result, o)
	return result
{'fraction_non_alphanumeric': 0.1402378592666006, 'fraction_numerical': 0.030723488602576808, 'mean_word_length': 4.318840579710145, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7445534', 'n_tokens_mistral': 1601, 'n_tokens_neox': 1475, 'n_words': 436}