<commit_before>import re
from django.contrib.auth.models import User
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from django.core.validators import RegexValidator
from django.db import models
from django.db.models.signals import post_save, pre_save
from django.dispatch import receiver
from djangotoolbox.fields import DictField, ListField
from rest_framework.authtoken.models import Token
from oclapi.utils import reverse_resource, reverse_resource_version
from oclapi.settings.common import Common
from django.db.models import get_model

HEAD = 'HEAD'

NAMESPACE_REGEX = re.compile(r'^[a-zA-Z0-9\-\.]+$')

ACCESS_TYPE_VIEW = 'View'
ACCESS_TYPE_EDIT = 'Edit'
ACCESS_TYPE_NONE = 'None'
DEFAULT_ACCESS_TYPE = ACCESS_TYPE_VIEW
ACCESS_TYPE_CHOICES = ((ACCESS_TYPE_VIEW, 'View'),
                       (ACCESS_TYPE_EDIT, 'Edit'),
                       (ACCESS_TYPE_NONE, 'None'))


class BaseModel(models.Model):
    """
    Base model from which all resources inherit.  Contains timestamps and is_active field for logical deletion.
    """
    public_access = models.CharField(max_length=16, choices=ACCESS_TYPE_CHOICES, default=DEFAULT_ACCESS_TYPE, blank=True)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    created_by = models.TextField()
    updated_by = models.TextField()
    is_active = models.BooleanField(default=True)
    extras = DictField(null=True, blank=True)
    uri = models.TextField(null=True, blank=True)

    class Meta:
        abstract = True

    @property
    def url(self):
        return self.uri or reverse_resource(self, self.view_name)

    @property
    def view_name(self):
        return self.get_default_view_name()

    @property
    def _default_view_name(self):
        return '%(model_name)s-detail'

    def soft_delete(self):
        if self.is_active:
            self.is_active = False
            self.save()

    def undelete(self):
        if not self.is_active:
            self.is_active = True
            self.save()

    def get_default_view_name(self):
        model = self.__class__
        model_meta = model._meta
        format_kwargs = {
            'app_label': model_meta.app_label,
            'model_name': model_meta.object_name.lower()
        }
        return self._default_view_name % format_kwargs


class BaseResourceModel(BaseModel):
    """
    A base resource has a mnemonic that is unique across all objects of its type.
    A base resource may contain sub-resources.
    (An Organization is a base resource, but a Concept is not.)
    """
    mnemonic = models.CharField(max_length=255, validators=[RegexValidator(regex=NAMESPACE_REGEX)], unique=True)

    class Meta:
        abstract = True

    def __unicode__(self):
        return self.mnemonic


class SubResourceBaseModel(BaseModel):
    """
    A sub-resource is an object that exists within the scope of its parent resource.
    Its mnemonic is unique within the scope of its parent resource.
    (A Source is a sub-resource, but an Organization is not.)
    """
    mnemonic = models.CharField(max_length=255, validators=[RegexValidator(regex=NAMESPACE_REGEX)])
    parent_type = models.ForeignKey(ContentType, db_index=False)
    parent_id = models.TextField()
    parent = generic.GenericForeignKey('parent_type', 'parent_id')

    class Meta:
        abstract = True
        unique_together = ('mnemonic', 'parent_id')

    def __unicode__(self):
        return self.mnemonic

    @property
    def parent_url(self):
        return self.parent.url

    @property
    def parent_resource(self):
        return self.parent.mnemonic

    @property
    def parent_resource_type(self):
        return self.parent.resource_type

VERSION_TYPE = 'Version'


class ResourceVersionModel(BaseModel):
    """
    This model represents a version of a resource.  It has links to its base resource,
    as well as its parent and previous versions (if any).
    """
    mnemonic = models.CharField(max_length=255, validators=[RegexValidator(regex=NAMESPACE_REGEX)])
    versioned_object_id = models.TextField()
    versioned_object_type = models.ForeignKey(ContentType, db_index=False)
    versioned_object = generic.GenericForeignKey('versioned_object_type', 'versioned_object_id')
    released = models.BooleanField(default=False, blank=True)
    previous_version = models.ForeignKey('self', related_name='next', null=True, blank=True, db_index=False)
    parent_version = models.ForeignKey('self', related_name='child', null=True, blank=True, db_index=False)

    class Meta:
        abstract = True
        unique_together = ('mnemonic', 'versioned_object_id')

    def __unicode__(self):
        return self.mnemonic

    # TODO This prevents ValidationMixin from functioning correctly, we should reconsider necessity of this check
    # def clean(self):
    #     if self == self.parent_version:
    #         raise ValidationError('version cannot be its own parent')

    @property
    def url(self):
        return self.uri or reverse_resource_version(self, self.view_name)

    @property
    def previous_version_mnemonic(self):
        return self.previous_version.mnemonic if self.previous_version else None

    @property
    def previous_version_url(self):
        return self.previous_version.url if self.previous_version else None

    @property
    def parent_version_mnemonic(self):
        return self.parent_version.mnemonic if self.parent_version else None

    @property
    def parent_resource(self):
        return self.versioned_object.parent_resource

    @property
    def parent_resource_type(self):
        return self.versioned_object.parent_resource_type

    @property
    def parent_url(self):
        return self.versioned_object.parent_url

    @property
    def collections(self):
        versions = self.collection_versions
        return map(lambda v: v.versioned_object, versions)

    @property
    def collection_ids(self):
        if self.is_latest_version:
            return list(set(self._collection_ids_for_versioned_object() + self._collection_ids_for_version()))
        else:
            return self._collection_ids_for_version()

    def _collection_ids_for_versioned_object(self):
        return map(lambda c: c.id, get_model('collection', 'Collection').objects.filter(
            references={'expression': self.versioned_object.uri}))

    def _collection_ids_for_version(self):
        return map(lambda c: c.id,
                   get_model('collection', 'Collection').objects.filter(references={'expression': self.uri}))

    @property
    def collection_version_ids(self):
        return map(lambda v: v.id, self.collection_versions)

    @classmethod
    def get_latest_version_of(cls, versioned_object):
        versions = versioned_object.get_version_model().objects.filter(versioned_object_id=versioned_object.id, is_active=True).order_by('-created_at')
        return versions[0] if versions else None

    @classmethod
    def get_head_of(cls, versioned_object):
        try:
            version = versioned_object.get_version_model().objects.get(versioned_object_id=versioned_object.id, is_active=True, mnemonic=HEAD)
            return version
        except:
            return None

<commit_msg>making sure mappingversion indexing dont fail for stale data<commit_after>import re
from django.contrib.auth.models import User
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from django.core.validators import RegexValidator
from django.db import models
from django.db.models.signals import post_save, pre_save
from django.dispatch import receiver
from djangotoolbox.fields import DictField, ListField
from rest_framework.authtoken.models import Token
from oclapi.utils import reverse_resource, reverse_resource_version
from oclapi.settings.common import Common
from django.db.models import get_model

HEAD = 'HEAD'

NAMESPACE_REGEX = re.compile(r'^[a-zA-Z0-9\-\.]+$')

ACCESS_TYPE_VIEW = 'View'
ACCESS_TYPE_EDIT = 'Edit'
ACCESS_TYPE_NONE = 'None'
DEFAULT_ACCESS_TYPE = ACCESS_TYPE_VIEW
ACCESS_TYPE_CHOICES = ((ACCESS_TYPE_VIEW, 'View'),
                       (ACCESS_TYPE_EDIT, 'Edit'),
                       (ACCESS_TYPE_NONE, 'None'))


class BaseModel(models.Model):
    """
    Base model from which all resources inherit.  Contains timestamps and is_active field for logical deletion.
    """
    public_access = models.CharField(max_length=16, choices=ACCESS_TYPE_CHOICES, default=DEFAULT_ACCESS_TYPE, blank=True)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    created_by = models.TextField()
    updated_by = models.TextField()
    is_active = models.BooleanField(default=True)
    extras = DictField(null=True, blank=True)
    uri = models.TextField(null=True, blank=True)

    class Meta:
        abstract = True

    @property
    def url(self):
        return self.uri or reverse_resource(self, self.view_name)

    @property
    def view_name(self):
        return self.get_default_view_name()

    @property
    def _default_view_name(self):
        return '%(model_name)s-detail'

    def soft_delete(self):
        if self.is_active:
            self.is_active = False
            self.save()

    def undelete(self):
        if not self.is_active:
            self.is_active = True
            self.save()

    def get_default_view_name(self):
        model = self.__class__
        model_meta = model._meta
        format_kwargs = {
            'app_label': model_meta.app_label,
            'model_name': model_meta.object_name.lower()
        }
        return self._default_view_name % format_kwargs


class BaseResourceModel(BaseModel):
    """
    A base resource has a mnemonic that is unique across all objects of its type.
    A base resource may contain sub-resources.
    (An Organization is a base resource, but a Concept is not.)
    """
    mnemonic = models.CharField(max_length=255, validators=[RegexValidator(regex=NAMESPACE_REGEX)], unique=True)

    class Meta:
        abstract = True

    def __unicode__(self):
        return self.mnemonic


class SubResourceBaseModel(BaseModel):
    """
    A sub-resource is an object that exists within the scope of its parent resource.
    Its mnemonic is unique within the scope of its parent resource.
    (A Source is a sub-resource, but an Organization is not.)
    """
    mnemonic = models.CharField(max_length=255, validators=[RegexValidator(regex=NAMESPACE_REGEX)])
    parent_type = models.ForeignKey(ContentType, db_index=False)
    parent_id = models.TextField()
    parent = generic.GenericForeignKey('parent_type', 'parent_id')

    class Meta:
        abstract = True
        unique_together = ('mnemonic', 'parent_id')

    def __unicode__(self):
        return self.mnemonic

    @property
    def parent_url(self):
        return self.parent.url

    @property
    def parent_resource(self):
        return self.parent.mnemonic

    @property
    def parent_resource_type(self):
        return self.parent.resource_type

VERSION_TYPE = 'Version'


class ResourceVersionModel(BaseModel):
    """
    This model represents a version of a resource.  It has links to its base resource,
    as well as its parent and previous versions (if any).
    """
    mnemonic = models.CharField(max_length=255, validators=[RegexValidator(regex=NAMESPACE_REGEX)])
    versioned_object_id = models.TextField()
    versioned_object_type = models.ForeignKey(ContentType, db_index=False)
    versioned_object = generic.GenericForeignKey('versioned_object_type', 'versioned_object_id')
    released = models.BooleanField(default=False, blank=True)
    previous_version = models.ForeignKey('self', related_name='next', null=True, blank=True, db_index=False)
    parent_version = models.ForeignKey('self', related_name='child', null=True, blank=True, db_index=False)

    class Meta:
        abstract = True
        unique_together = ('mnemonic', 'versioned_object_id')

    def __unicode__(self):
        return self.mnemonic

    # TODO This prevents ValidationMixin from functioning correctly, we should reconsider necessity of this check
    # def clean(self):
    #     if self == self.parent_version:
    #         raise ValidationError('version cannot be its own parent')

    @property
    def url(self):
        return self.uri or reverse_resource_version(self, self.view_name)

    @property
    def previous_version_mnemonic(self):
        return self.previous_version.mnemonic if self.previous_version else None

    @property
    def previous_version_url(self):
        return self.previous_version.url if self.previous_version else None

    @property
    def parent_version_mnemonic(self):
        return self.parent_version.mnemonic if self.parent_version else None

    @property
    def parent_resource(self):
        return self.versioned_object.parent_resource

    @property
    def parent_resource_type(self):
        return self.versioned_object.parent_resource_type

    @property
    def parent_url(self):
        return self.versioned_object.parent_url

    @property
    def collections(self):
        versions = self.collection_versions
        return map(lambda v: v.versioned_object, versions)

    @property
    def collection_ids(self):
        if self.is_latest_version:
            return list(set(self._collection_ids_for_versioned_object() + self._collection_ids_for_version()))
        else:
            return self._collection_ids_for_version()

    def _collection_ids_for_versioned_object(self):
        if self.versioned_object:
            return map(lambda c: c.id, get_model('collection', 'Collection').objects.filter(
                references={'expression': self.versioned_object.uri}))
        else:
            print self.versioned_object_id
            return []

    def _collection_ids_for_version(self):
        return map(lambda c: c.id,
                   get_model('collection', 'Collection').objects.filter(references={'expression': self.uri}))

    @property
    def collection_version_ids(self):
        return map(lambda v: v.id, self.collection_versions)

    @classmethod
    def get_latest_version_of(cls, versioned_object):
        versions = versioned_object.get_version_model().objects.filter(versioned_object_id=versioned_object.id, is_active=True).order_by('-created_at')
        return versions[0] if versions else None

    @classmethod
    def get_head_of(cls, versioned_object):
        try:
            version = versioned_object.get_version_model().objects.get(versioned_object_id=versioned_object.id, is_active=True, mnemonic=HEAD)
            return version
        except:
            return None


{'fraction_non_alphanumeric': 0.08574116382206692, 'fraction_numerical': 0.0018814675446848542, 'mean_word_length': 3.4479976090854754, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6795563', 'n_tokens_mistral': 4340, 'n_tokens_neox': 4207, 'n_words': 1020}