<commit_before># Describes the program "image" in memory.

import collections
import inspect

import types

# HACK types are considered constant
# This allows issubclass(type, type) to be folded
# Types are almost constant, however, by fiat.
constantTypes = set((float, int, long, str, bool, type(None), type, types.CodeType))

class ProgramDecl(object):
	__slots__ = ()

class TypeInfo(object):
	__slots__ = 'abstractInstance'
	def __init__(self):
		self.abstractInstance = None

class AbstractObject(ProgramDecl):
	__slots__ = 'type', '__weakref__'

	def isType(self):
		return False

	def isAbstract(self):
		return self.type.typeinfo.abstractInstance == self

	def isConcrete(self):
		return not self.isAbstract()

	def isConstant(self):
		return False

def isConstant(pyobj):
	if isinstance(pyobj, (tuple, frozenset)):
		for item in pyobj:
			if not isConstant(item):
				return False
		return True
	else:
		return type(pyobj) in constantTypes

class Object(AbstractObject):
	__slots__ = 'pyobj', 'typeinfo', 'slot', 'array', 'dictionary', 'lowlevel'

	def __init__(self, pyobj):
		assert not isinstance(pyobj, ProgramDecl), "Tried to wrap a wrapper."
		self.pyobj 	= pyobj

	def allocateDatastructures(self, type_):
		# Even the simple ones are set lazily,
		# so early accesses become hard errors.
		self.type       = type_
		self.typeinfo   = None
		self.slot 	= {}
		self.array 	= {}
		self.dictionary = {}
		self.lowlevel	= {}

	def isPreexisting(self):
		return True

	def isConstant(self):
		return isConstant(self.pyobj)

	def isType(self):
		return isinstance(self.pyobj, type)

	def abstractInstance(self):
		assert self.isType()
		return self.typeinfo.abstractInstance

	def addSlot(self, name, obj):
		assert isinstance(name, Object), name
		assert isinstance(name.pyobj, str), name
		assert isinstance(obj, AbstractObject), obj
		self.slot[name] = obj

	def addDictionaryItem(self, key, value):
		assert isinstance(key, Object), key
		assert isinstance(value, AbstractObject), value
		self.dictionary[key] = value

	def addArrayItem(self, index, value):
		assert isinstance(index, Object), index
		assert isinstance(index.pyobj, int), index
		assert isinstance(value, AbstractObject), value
		self.array[index] = value

	def addLowLevel(self, name, obj):
		assert isinstance(name, Object), name
		assert isinstance(name.pyobj, str), name
		assert isinstance(obj, AbstractObject), obj
		self.lowlevel[name] = obj


	def __repr__(self):
		if isinstance(self.pyobj, dict):
			# Simplifies large, global dictionaries.
			r = "dict"+repr(tuple(self.pyobj.iterkeys()))
		else:
			r = repr(self.pyobj)


		return "%s(%s)" % (type(self).__name__, r)


class ImaginaryObject(AbstractObject):
	__slots__ = 'name', 'preexisting'
	def __init__(self, name, t, preexisting):
		assert t.isType()
		self.name = name
		self.type = t
		self.preexisting = preexisting
<commit_msg>IR Objects now truncate long python object names.<commit_after># Describes the program "image" in memory.

import collections
import inspect

import types

# HACK types are considered constant
# This allows issubclass(type, type) to be folded
# Types are almost constant, however, by fiat.
constantTypes = set((float, int, long, str, bool, type(None), type, types.CodeType))

class ProgramDecl(object):
	__slots__ = ()

class TypeInfo(object):
	__slots__ = 'abstractInstance'
	def __init__(self):
		self.abstractInstance = None

class AbstractObject(ProgramDecl):
	__slots__ = 'type', '__weakref__'

	def isType(self):
		return False

	def isAbstract(self):
		return self.type.typeinfo.abstractInstance == self

	def isConcrete(self):
		return not self.isAbstract()

	def isConstant(self):
		return False

def isConstant(pyobj):
	if isinstance(pyobj, (tuple, frozenset)):
		for item in pyobj:
			if not isConstant(item):
				return False
		return True
	else:
		return type(pyobj) in constantTypes

class Object(AbstractObject):
	__slots__ = 'pyobj', 'typeinfo', 'slot', 'array', 'dictionary', 'lowlevel'

	def __init__(self, pyobj):
		assert not isinstance(pyobj, ProgramDecl), "Tried to wrap a wrapper."
		self.pyobj 	= pyobj

	def allocateDatastructures(self, type_):
		# Even the simple ones are set lazily,
		# so early accesses become hard errors.
		self.type       = type_
		self.typeinfo   = None
		self.slot 	= {}
		self.array 	= {}
		self.dictionary = {}
		self.lowlevel	= {}

	def isPreexisting(self):
		return True

	def isConstant(self):
		return isConstant(self.pyobj)

	def isType(self):
		return isinstance(self.pyobj, type)

	def abstractInstance(self):
		assert self.isType()
		return self.typeinfo.abstractInstance

	def addSlot(self, name, obj):
		assert isinstance(name, Object), name
		assert isinstance(name.pyobj, str), name
		assert isinstance(obj, AbstractObject), obj
		self.slot[name] = obj

	def addDictionaryItem(self, key, value):
		assert isinstance(key, Object), key
		assert isinstance(value, AbstractObject), value
		self.dictionary[key] = value

	def addArrayItem(self, index, value):
		assert isinstance(index, Object), index
		assert isinstance(index.pyobj, int), index
		assert isinstance(value, AbstractObject), value
		self.array[index] = value

	def addLowLevel(self, name, obj):
		assert isinstance(name, Object), name
		assert isinstance(name.pyobj, str), name
		assert isinstance(obj, AbstractObject), obj
		self.lowlevel[name] = obj


	def __repr__(self):
		if isinstance(self.pyobj, dict):
			# Simplifies large, global dictionaries.
			r = "dict"+repr(tuple(self.pyobj.iterkeys()))
		else:
			r = repr(self.pyobj)

		if len(r) > 30:
			return "%s(%s...)" % (type(self).__name__, r[:27])
		else:
			return "%s(%s)" % (type(self).__name__, r)


class ImaginaryObject(AbstractObject):
	__slots__ = 'name', 'preexisting'
	def __init__(self, name, t, preexisting):
		assert t.isType()
		self.name = name
		self.type = t
		self.preexisting = preexisting

{'fraction_non_alphanumeric': 0.11593462717058223, 'fraction_numerical': 0.0006809669731018045, 'mean_word_length': 4.584600760456274, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6177655', 'n_tokens_mistral': 2071, 'n_tokens_neox': 1893, 'n_words': 549}