<commit_before>#!/usr/bin/env python

try:
	assert isinstance(Taxonomy, dict)
except NameError:
	Taxonomy = {}
#
class TaxonBaseObject:
	def __init__(self, name, namespace=Taxonomy, **params):
		if name in namespace:
			namespace[name].update(params)
		else:
			params['id'] = len(Taxonomy)
			params['name'] = name
			namespace[name] = params
		self.__dict__ = namespace[name]
	def update(self, *args):
		self.__dict__.update(*args)
	def __str__(self):
		return self.name
	def __repr__(self):
		#return "<{}({})>".format(self.name, len(self.__dict__))
		return "<{}>".format(self.name)
#
try:
	assert(isinstance(lookup_table, dict))
except NameError:
	lookup_table = {}
#
def name_cleaner(text):
	text = text.strip('+_')
	if ' ' in text:
		text = text.replace(' ', '_')
	return text
def tag(item, table=lookup_table, synonyms=[]):
	if isinstance(item, TaxonBaseObject):
		n = item.name
		t = table.get(n, item)
	else:
		s = name_cleaner(item)
		if s in table:
			t = table[s]
		else: # won't get called until later, when TaxonObject has been declared
			t = TaxonObject(s)
	if isinstance(synonyms, str):
		synonyms = synonyms.split()
	for linkname in synonyms:
		table[linkname] = t
	return t
#
class TaxonObject(TaxonBaseObject):
	def __eq__(self, other):
		if isinstance(other, TaxonBaseObject):
			return self.id == other.id
		else:
			assert isinstance(self, TaxonBaseObject)
			return self.name == name_cleaner(other)
	# if defining __eq__ then also define __hash__
	def __hash__(self):
		return self.id<commit_msg>OO stuff, NULL first introduced<commit_after>#!/usr/bin/env python

try:
	assert isinstance(Taxonomy, dict)
except NameError:
	Taxonomy = {'NULL': {'id': 0, 'name': None }}
#
class TaxonBaseObject:
	def __init__(self, name, namespace=Taxonomy, **params):
		if name in namespace:
			namespace[name].update(params)
		else:
			params['id'] = len(Taxonomy)
			params['name'] = name
			namespace[name] = params
		self.__dict__ = namespace[name]
	def update(self, *args):
		self.__dict__.update(*args)
	def __str__(self):
		return self.name
	def __repr__(self):
		#return "<{}({})>".format(self.name, len(self.__dict__))
		return "<{}>".format(self.name)
	def __int__(self):
		return self.id
	def __bool__(self):
		return 0 != int(self) 
#
try:
	assert(isinstance(lookup_table, dict))
except NameError:
	lookup_table = {}
#
# customize:
def name_cleaner(text):
	text = text.strip('+_')
	if ' ' in text:
		text = text.replace(' ', '_')
	return text
#
def tag(item, table=lookup_table, synonyms=[]):
	N = len(Taxonomy)
	if isinstance(item, TaxonBaseObject):
		t = item
		#n = item.name
		#t = table.get(n, item)
	elif isinstance(item, int):
		id = item % N
		for el, kv in Taxonomy.items():
			if id == kv['id']:
				t = tag(el)
				break
	elif isinstance(item, str):
		s = name_cleaner(item)
		if s in table:
			t = table[s]
		else: # won't get called until later, when TaxonObject (below) has been declared
			t = TaxonObject(s)
	elif item: # hope it's hashable
		t = TaxonObject(item)
	else:
		t = tag(0)
	if isinstance(synonyms, str):
		synonyms = synonyms.split()
	for linkname in synonyms:
		table[linkname] = t
	return t
#
class TaxonObject(TaxonBaseObject):
	def __eq__(self, other):
		if isinstance(other, TaxonBaseObject):
			return int(self) == int(other)
		else:
			assert isinstance(self, TaxonBaseObject)
			return self.name == name_cleaner(other)
	# if defining __eq__ then also define __hash__
	def __hash__(self):
		return int(self)
{'fraction_non_alphanumeric': 0.14211438474870017, 'fraction_numerical': 0.0008665511265164644, 'mean_word_length': 4.419405320813771, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '428912', 'n_tokens_mistral': 1335, 'n_tokens_neox': 1242, 'n_words': 328}