<commit_before># encoding: utf-8
from django.utils.translation import ugettext_lazy as _
from django.contrib.auth.models import User, Group
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.contrib.sites.models import Site
from utils import fattr
from django.template import add_to_builtins as register_templatetags
from django.template import Template as DjangoTemplate
from django.template import TemplateDoesNotExist
from django.template import Context, RequestContext
from django.core.exceptions import ObjectDoesNotExist
try:
	import json
except ImportError:
	import simplejson as json
from UserDict import DictMixin
from templatetags.containers import ContainerNode
from django.template.loader_tags import ExtendsNode, ConstantIncludeNode, IncludeNode
from django.template.loader import get_template
from django.http import Http404, HttpResponse, HttpResponseServerError, HttpResponseRedirect
from django.core.servers.basehttp import FileWrapper


def _ct_model_name(model):
	opts = model._meta
	while opts.proxy:
		model = opts.proxy_for_model
		opts = model._meta
	return opts.object_name.lower()


class Attribute(models.Model):
	entity_content_type = models.ForeignKey(ContentType, verbose_name='Entity type')
	entity_object_id = models.PositiveIntegerField(verbose_name='Entity ID')
	entity = generic.GenericForeignKey('entity_content_type', 'entity_object_id')
	key = models.CharField(max_length=255)
	json_value = models.TextField(verbose_name='Value (JSON)', help_text='This value must be valid JSON.')
	
	@property
	def value(self):
		return json.loads(self.json_value)
	
	def __unicode__(self):
		return u'"%s": %s' % (self.key, self.value)


class Relationship(models.Model):
	_value_models = []
	
	@staticmethod
	def register_value_model(model):
		if issubclass(model, models.Model):
			model_name = _ct_model_name(model)
			if model_name not in Relationship._value_models:
				Relationship._value_models.append(model_name)
		else:
			raise TypeError('Relationship.register_value_model only accepts subclasses of django.db.models.Model')
	
	@staticmethod
	def unregister_value_model(model):
		if issubclass(model, models.Model):
			model_name = _ct_model_name(model)
			if model_name in Relationship._value_models:
				Relationship._value_models.remove(model_name)
		else:
			raise TypeError('Relationship.unregister_value_model only accepts subclasses of django.db.models.Model')
	
	entity_content_type = models.ForeignKey(ContentType, related_name='relationship_entity_set', verbose_name='Entity type')
	entity_object_id = models.PositiveIntegerField(verbose_name='Entity ID')
	entity = generic.GenericForeignKey('entity_content_type', 'entity_object_id')
	key = models.CharField(max_length=255)
	value_content_type = models.ForeignKey(ContentType, related_name='relationship_value_set', limit_choices_to={'model__in':_value_models}, verbose_name='Value type')
	value_object_id = models.PositiveIntegerField(verbose_name='Value ID')
	value = generic.GenericForeignKey('value_content_type', 'value_object_id')
	
	def __unicode__(self):
		return u'"%s": %s' % (self.key, self.value)


class QuerySetMapper(object, DictMixin):
	def __init__(self, queryset, passthrough=None):
		self.queryset = queryset
		self.passthrough = passthrough
	def __getitem__(self, key):
		try:
			return self.queryset.get(key__exact=key).value
		except ObjectDoesNotExist:
			if self.passthrough:
				return self.passthrough.__getitem__(key)
			raise KeyError
	def keys(self):
		keys = set(self.queryset.values_list('key', flat=True).distinct())
		if self.passthrough:
			keys += set(self.passthrough.keys())
		return list(keys)


class Entity(models.Model):
	attribute_set = generic.GenericRelation(Attribute, content_type_field='entity_content_type', object_id_field='entity_object_id')
	relationship_set = generic.GenericRelation(Relationship, content_type_field='entity_content_type', object_id_field='entity_object_id')
	
	@property
	def attributes(self):
		return QuerySetMapper(self.attribute_set)
	
	@property
	def relationships(self):
		return QuerySetMapper(self.relationship_set)
	
	class Meta:
		abstract = True


class Collection(models.Model):
	name = models.CharField(max_length=255)
	description = models.TextField(blank=True, null=True)


class CollectionMember(models.Model):
	_value_models = []
	
	@staticmethod
	def register_value_model(model):
		if issubclass(model, models.Model):
			model_name = _ct_model_name(model)
			if model_name not in CollectionMember._value_models:
				CollectionMember._value_models.append(model_name)
		else:
			raise TypeError('CollectionMember.register_value_model only accepts subclasses of django.db.models.Model')
	
	@staticmethod
	def unregister_value_model(model):
		if issubclass(model, models.Model):
			model_name = _ct_model_name(model)
			if model_name in CollectionMember._value_models:
				CollectionMember._value_models.remove(model_name)
		else:
			raise TypeError('CollectionMember.unregister_value_model only accepts subclasses of django.db.models.Model')
	
	collection = models.ForeignKey(Collection, related_name='members')
	index = models.PositiveIntegerField(verbose_name='Index', help_text='This will determine the ordering of the item within the collection. (Optional)', null=True, blank=True)
	member_content_type = models.ForeignKey(ContentType, limit_choices_to={'model__in':_value_models}, verbose_name='Member type')
	member_object_id = models.PositiveIntegerField(verbose_name='Member ID')
	member = generic.GenericForeignKey('member_content_type', 'member_object_id')


def register_value_model(model):
	Relationship.register_value_model(model)
	CollectionMember.register_value_model(model)


def unregister_value_model(model):
	Relationship.unregister_value_model(model)
	CollectionMember.unregister_value_model(model)


class TreeManager(models.Manager):
	use_for_related_fields = True
	
	def roots(self):
		return self.filter(parent__isnull=True)
	
	def get_with_path(self, path, root=None, absolute_result=True, pathsep='/'):
		"""
		Returns the object with the path, or None if there is no object with that path,
		unless absolute_result is set to False, in which case it returns a tuple containing
		the deepest object found along the path, and the remainder of the path after that
		object as a string (or None in the case that there is no remaining path).
		"""
		slugs = path.split(pathsep)
		obj = root
		remaining_slugs = list(slugs)
		remainder = None
		for slug in slugs:
			remaining_slugs.remove(slug)
			if slug: # ignore blank slugs, handles for multiple consecutive pathseps
				try:
					obj = self.get(slug__exact=slug, parent__exact=obj)
				except self.model.DoesNotExist:
					if absolute_result:
						obj = None
					remaining_slugs.insert(0, slug)
					remainder = pathsep.join(remaining_slugs)
					break
		if obj:
			if absolute_result:
				return obj
			else:
				return (obj, remainder)
		raise self.model.DoesNotExist('%s matching query does not exist.' % self.model._meta.object_name)


class TreeModel(models.Model):
	objects = TreeManager()
	parent = models.ForeignKey('self', related_name='children', null=True, blank=True)
	slug = models.SlugField()
	
	def get_path(self, pathsep='/', field='slug'):
		path = getattr(self, field)
		parent = self.parent
		while parent:
			path = getattr(parent, field) + pathsep + path
			parent = parent.parent
		return path
	path = property(get_path)
	
	def __unicode__(self):
		return self.path
	
	class Meta:
		abstract = True


class TreeEntity(TreeModel, Entity):
	@property
	def attributes(self):
		if self.parent:
			return QuerySetMapper(self.attribute_set, passthrough=self.parent.attributes)
		return super(TreeEntity, self).attributes
	
	@property
	def relationships(self):
		if self.parent:
			return QuerySetMapper(self.relationship_set, passthrough=self.parent.relationships)
		return super(TreeEntity, self).relationships
	
	class Meta:
		abstract = True


class Node(TreeEntity):
	instance_type = models.ForeignKey(ContentType, editable=False)
	
	def save(self, force_insert=False, force_update=False):
		if not hasattr(self, 'instance_type_ptr'):
			self.instance_type = ContentType.objects.get_for_model(self.__class__)
		super(Node, self).save(force_insert, force_update)
	
	@property
	def instance(self):
		return self.instance_type.get_object_for_this_type(id=self.id)
	
	accepts_subpath = False
	
	def render_to_response(self, request, path=None, subpath=None):
		return HttpResponseServerError()


class MultiNode(Node):
	accepts_subpath = True
	
	@property
	def urlpatterns(self):
		return []
	
	def render_to_response(self, request, path=None, subpath=None):
		if not subpath:
			subpath = ""
		subpath = "/" + subpath
		from django.core.urlresolvers import resolve
		view, args, kwargs = resolve(subpath, urlconf=self)
		return view(request, *args, **kwargs)
	
	class Meta:
		abstract = True<commit_msg>Improvements to Collections, and minor refactoring.<commit_after># encoding: utf-8
from django.utils.translation import ugettext_lazy as _
from django.contrib.auth.models import User, Group
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.contrib.sites.models import Site
from utils import fattr
from django.template import add_to_builtins as register_templatetags
from django.template import Template as DjangoTemplate
from django.template import TemplateDoesNotExist
from django.template import Context, RequestContext
from django.core.exceptions import ObjectDoesNotExist
try:
	import json
except ImportError:
	import simplejson as json
from UserDict import DictMixin
from templatetags.containers import ContainerNode
from django.template.loader_tags import ExtendsNode, ConstantIncludeNode, IncludeNode
from django.template.loader import get_template
from django.http import Http404, HttpResponse, HttpResponseServerError, HttpResponseRedirect
from django.core.servers.basehttp import FileWrapper


_value_models = {}
_value_models_ct_pks = []


def register_value_model(model):
	if issubclass(model, models.Model):
		if model not in _value_models:
			_value_models[model] = ContentType.objects.get_for_model(model)
			_value_models_ct_pks.append(_value_models[model].pk)
	else:
		raise TypeError('philo.models.register_value_model only accepts subclasses of django.db.models.Model')


def unregister_value_model(model):
	if issubclass(model, models.Model):
		if model in _value_models:
			_value_models_ct_pks.remove(_value_models[model].pk)
			del _value_models[model]
	else:
		raise TypeError('philo.models.unregister_value_model only accepts subclasses of django.db.models.Model')


class Attribute(models.Model):
	entity_content_type = models.ForeignKey(ContentType, verbose_name='Entity type')
	entity_object_id = models.PositiveIntegerField(verbose_name='Entity ID')
	entity = generic.GenericForeignKey('entity_content_type', 'entity_object_id')
	key = models.CharField(max_length=255)
	json_value = models.TextField(verbose_name='Value (JSON)', help_text='This value must be valid JSON.')
	
	def get_value(self):
		return json.loads(self.json_value)
	
	def set_value(self, value):
		self.json_value = json.dumps(value)
	
	def delete_value(self):
		self.json_value = json.dumps(None)
	
	value = property(get_value, set_value, delete_value)
	
	def __unicode__(self):
		return u'"%s": %s' % (self.key, self.value)


class Relationship(models.Model):
	entity_content_type = models.ForeignKey(ContentType, related_name='relationship_entity_set', verbose_name='Entity type')
	entity_object_id = models.PositiveIntegerField(verbose_name='Entity ID')
	entity = generic.GenericForeignKey('entity_content_type', 'entity_object_id')
	key = models.CharField(max_length=255)
	value_content_type = models.ForeignKey(ContentType, related_name='relationship_value_set', limit_choices_to={'pk__in': _value_models_ct_pks}, verbose_name='Value type')
	value_object_id = models.PositiveIntegerField(verbose_name='Value ID')
	value = generic.GenericForeignKey('value_content_type', 'value_object_id')
	
	def __unicode__(self):
		return u'"%s": %s' % (self.key, self.value)


class QuerySetMapper(object, DictMixin):
	def __init__(self, queryset, passthrough=None):
		self.queryset = queryset
		self.passthrough = passthrough
	def __getitem__(self, key):
		try:
			return self.queryset.get(key__exact=key).value
		except ObjectDoesNotExist:
			if self.passthrough:
				return self.passthrough.__getitem__(key)
			raise KeyError
	def keys(self):
		keys = set(self.queryset.values_list('key', flat=True).distinct())
		if self.passthrough:
			keys += set(self.passthrough.keys())
		return list(keys)


class Entity(models.Model):
	attribute_set = generic.GenericRelation(Attribute, content_type_field='entity_content_type', object_id_field='entity_object_id')
	relationship_set = generic.GenericRelation(Relationship, content_type_field='entity_content_type', object_id_field='entity_object_id')
	
	@property
	def attributes(self):
		return QuerySetMapper(self.attribute_set)
	
	@property
	def relationships(self):
		return QuerySetMapper(self.relationship_set)
	
	class Meta:
		abstract = True


class Collection(models.Model):
	name = models.CharField(max_length=255)
	description = models.TextField(blank=True, null=True)


class CollectionMemberManager(models.Manager):
	use_for_related_fields = True

	def with_model(self, model):
		if model in _value_models:
			return model._default_manager.filter(pk__in=self.filter(member_content_type=_value_models[model]).values_list('member_object_id', flat=True))
		else:
			raise TypeError('CollectionMemberManager.with_model only accepts models previously registered with philo.models.register_value_model')


class CollectionMember(models.Model):
	objects = CollectionMemberManager()
	collection = models.ForeignKey(Collection, related_name='members')
	index = models.PositiveIntegerField(verbose_name='Index', help_text='This will determine the ordering of the item within the collection. (Optional)', null=True, blank=True)
	member_content_type = models.ForeignKey(ContentType, limit_choices_to={'pk__in': _value_models_ct_pks}, verbose_name='Member type')
	member_object_id = models.PositiveIntegerField(verbose_name='Member ID')
	member = generic.GenericForeignKey('member_content_type', 'member_object_id')


class TreeManager(models.Manager):
	use_for_related_fields = True
	
	def roots(self):
		return self.filter(parent__isnull=True)
	
	def get_with_path(self, path, root=None, absolute_result=True, pathsep='/'):
		"""
		Returns the object with the path, or None if there is no object with that path,
		unless absolute_result is set to False, in which case it returns a tuple containing
		the deepest object found along the path, and the remainder of the path after that
		object as a string (or None in the case that there is no remaining path).
		"""
		slugs = path.split(pathsep)
		obj = root
		remaining_slugs = list(slugs)
		remainder = None
		for slug in slugs:
			remaining_slugs.remove(slug)
			if slug: # ignore blank slugs, handles for multiple consecutive pathseps
				try:
					obj = self.get(slug__exact=slug, parent__exact=obj)
				except self.model.DoesNotExist:
					if absolute_result:
						obj = None
					remaining_slugs.insert(0, slug)
					remainder = pathsep.join(remaining_slugs)
					break
		if obj:
			if absolute_result:
				return obj
			else:
				return (obj, remainder)
		raise self.model.DoesNotExist('%s matching query does not exist.' % self.model._meta.object_name)


class TreeModel(models.Model):
	objects = TreeManager()
	parent = models.ForeignKey('self', related_name='children', null=True, blank=True)
	slug = models.SlugField()
	
	def get_path(self, pathsep='/', field='slug'):
		path = getattr(self, field)
		parent = self.parent
		while parent:
			path = getattr(parent, field) + pathsep + path
			parent = parent.parent
		return path
	path = property(get_path)
	
	def __unicode__(self):
		return self.path
	
	class Meta:
		abstract = True


class TreeEntity(TreeModel, Entity):
	@property
	def attributes(self):
		if self.parent:
			return QuerySetMapper(self.attribute_set, passthrough=self.parent.attributes)
		return super(TreeEntity, self).attributes
	
	@property
	def relationships(self):
		if self.parent:
			return QuerySetMapper(self.relationship_set, passthrough=self.parent.relationships)
		return super(TreeEntity, self).relationships
	
	class Meta:
		abstract = True


class Node(TreeEntity):
	instance_type = models.ForeignKey(ContentType, editable=False)
	
	def save(self, force_insert=False, force_update=False):
		if not hasattr(self, 'instance_type_ptr'):
			self.instance_type = ContentType.objects.get_for_model(self.__class__)
		super(Node, self).save(force_insert, force_update)
	
	@property
	def instance(self):
		return self.instance_type.get_object_for_this_type(id=self.id)
	
	accepts_subpath = False
	
	def render_to_response(self, request, path=None, subpath=None):
		return HttpResponseServerError()


class MultiNode(Node):
	accepts_subpath = True
	
	urlpatterns = []
	
	def render_to_response(self, request, path=None, subpath=None):
		if not subpath:
			subpath = ""
		subpath = "/" + subpath
		from django.core.urlresolvers import resolve
		view, args, kwargs = resolve(subpath, urlconf=self)
		return view(request, *args, **kwargs)
	
	class Meta:
		abstract = True
{'fraction_non_alphanumeric': 0.09353877878336579, 'fraction_numerical': 0.0016038492381716118, 'mean_word_length': 5.919936583432421, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2292692', 'n_tokens_mistral': 5798, 'n_tokens_neox': 5319, 'n_words': 1256}