<commit_before>import datetime
import logging
import contextlib

from django.core.exceptions import NON_FIELD_ERRORS

from google.appengine.ext import db
from google.appengine.api.datastore import Key, Delete
from google.appengine.datastore.datastore_rpc import TransactionOptions

from .unique_utils import unique_identifiers_from_entity
from .utils import key_exists
from djangae.db.backends.appengine.dbapi import IntegrityError, NotSupportedError
from django.conf import settings

DJANGAE_LOG = logging.getLogger("djangae")


def has_active_unique_constraints(model_or_instance):
    """
        Returns true if the model/instance has unique fields or unique_together fields and unique
        constraint checking is enabled on the model
    """

    django_opts = getattr(model_or_instance, "_meta", None)

    # If there are no unique fields on the model, return false
    if not django_opts.unique_together and not any(x.unique for x in django_opts.fields):
        return False

    opts = getattr(model_or_instance, "Djangae", None)
    if opts:
        if hasattr(opts, "disable_constraint_checks"):
            if opts.disable_constraint_checks:
                return False
            else:
                return True

    return not getattr(settings, "DJANGAE_DISABLE_CONSTRAINT_CHECKS", False)



class KeyProperty(db.Property):
    """A property that stores a datastore.Key reference to another object.
        Think of this as a Django GenericForeignKey which returns only the PK value, not the whole
        object, or a db.ReferenceProperty which can point to any model kind, and only returns the Key.
    """

    def validate(self, value):
        if value is None or isinstance(value, Key):
            return value
        raise ValueError("KeyProperty only accepts datastore.Key or None")


class UniqueMarker(db.Model):
    instance = KeyProperty()
    created = db.DateTimeProperty(required=True, auto_now_add=True)

    @staticmethod
    def kind():
        return "_djangae_unique_marker"


@db.transactional(propagation=TransactionOptions.INDEPENDENT, xg=True)
def acquire_identifiers(identifiers, entity_key):
    return _acquire_identifiers(identifiers, entity_key)


def _acquire_identifiers(identifiers, entity_key):
    # This must always be in a cross-group transaction, because even if there's only 1 identifider,
    # in the case where that identifier already exists, we then check if its `instance` exists
    assert entity_key
    namespace = entity_key.namespace() or None
    identifier_keys = [
        Key.from_path(UniqueMarker.kind(), identifier, namespace=namespace) for identifier in identifiers
    ]
    existing_markers = UniqueMarker.get(identifier_keys)
    markers_to_create = []
    markers = []

    for identifier_key, existing_marker in zip(identifier_keys, existing_markers):

        # Backwards compatability: we used to create the markers first in an independent transaction
        # and then create the entity and update the `instance` on the markers.  This meant that it
        # was possible that the independent marker creation transaction finished first and the outer
        # transaction failed, causing stale markers to be left behind.  We no longer do it this way
        # but we still want to ignore any old stale markers, hence if instance is None we overwrite.
        now = datetime.datetime.utcnow()
        if not existing_marker or existing_marker.instance is None:
            markers_to_create.append(UniqueMarker(
                key=identifier_key,
                instance=entity_key,
                created=now
            ))
        elif existing_marker.instance != entity_key and key_exists(existing_marker.instance):
            fields_and_values = identifier.split("|")
            table_name = fields_and_values[0]
            fields_and_values = fields_and_values[1:]
            fields = [ x.split(":")[0] for x in fields_and_values ]
            raise IntegrityError("Unique constraint violation for kind {} on fields: {}".format(table_name, ", ".join(fields)))
        elif existing_marker.instance != entity_key:
            markers_to_create.append(UniqueMarker(
                key=identifier_key,
                instance=entity_key,
                created=now
            ))
        else:
            # The marker is ours anyway
            markers.append(existing_marker)

    db.put(markers_to_create)
    return markers + markers_to_create


def get_markers_for_update(model, old_entity, new_entity):
    """
        Given an old entity state, and the new state, updates the identifiers
        appropriately. Should be called before saving the new_state
    """
    old_ids = set(unique_identifiers_from_entity(model, old_entity, ignore_pk=True))<commit_msg>Fix bug where unique constraint violation message mentioned the wrong field(s).<commit_after>import datetime
import logging
import contextlib

from django.core.exceptions import NON_FIELD_ERRORS

from google.appengine.ext import db
from google.appengine.api.datastore import Key, Delete
from google.appengine.datastore.datastore_rpc import TransactionOptions

from .unique_utils import unique_identifiers_from_entity
from .utils import key_exists
from djangae.db.backends.appengine.dbapi import IntegrityError, NotSupportedError
from django.conf import settings

DJANGAE_LOG = logging.getLogger("djangae")


def has_active_unique_constraints(model_or_instance):
    """
        Returns true if the model/instance has unique fields or unique_together fields and unique
        constraint checking is enabled on the model
    """

    django_opts = getattr(model_or_instance, "_meta", None)

    # If there are no unique fields on the model, return false
    if not django_opts.unique_together and not any(x.unique for x in django_opts.fields):
        return False

    opts = getattr(model_or_instance, "Djangae", None)
    if opts:
        if hasattr(opts, "disable_constraint_checks"):
            if opts.disable_constraint_checks:
                return False
            else:
                return True

    return not getattr(settings, "DJANGAE_DISABLE_CONSTRAINT_CHECKS", False)



class KeyProperty(db.Property):
    """A property that stores a datastore.Key reference to another object.
        Think of this as a Django GenericForeignKey which returns only the PK value, not the whole
        object, or a db.ReferenceProperty which can point to any model kind, and only returns the Key.
    """

    def validate(self, value):
        if value is None or isinstance(value, Key):
            return value
        raise ValueError("KeyProperty only accepts datastore.Key or None")


class UniqueMarker(db.Model):
    instance = KeyProperty()
    created = db.DateTimeProperty(required=True, auto_now_add=True)

    @staticmethod
    def kind():
        return "_djangae_unique_marker"


@db.transactional(propagation=TransactionOptions.INDEPENDENT, xg=True)
def acquire_identifiers(identifiers, entity_key):
    return _acquire_identifiers(identifiers, entity_key)


def _acquire_identifiers(identifiers, entity_key):
    # This must always be in a cross-group transaction, because even if there's only 1 identifider,
    # in the case where that identifier already exists, we then check if its `instance` exists
    assert entity_key
    namespace = entity_key.namespace() or None
    identifier_keys = [
        Key.from_path(UniqueMarker.kind(), identifier, namespace=namespace) for identifier in identifiers
    ]
    existing_markers = UniqueMarker.get(identifier_keys)
    markers_to_create = []
    markers = []

    for identifier_key, existing_marker in zip(identifier_keys, existing_markers):

        # Backwards compatability: we used to create the markers first in an independent transaction
        # and then create the entity and update the `instance` on the markers.  This meant that it
        # was possible that the independent marker creation transaction finished first and the outer
        # transaction failed, causing stale markers to be left behind.  We no longer do it this way
        # but we still want to ignore any old stale markers, hence if instance is None we overwrite.
        now = datetime.datetime.utcnow()
        if not existing_marker or existing_marker.instance is None:
            markers_to_create.append(UniqueMarker(
                key=identifier_key,
                instance=entity_key,
                created=now
            ))
        elif existing_marker.instance != entity_key and key_exists(existing_marker.instance):
            fields_and_values = identifier_key.name().split("|")
            table_name = fields_and_values[0]
            fields_and_values = fields_and_values[1:]
            fields = [ x.split(":")[0] for x in fields_and_values ]
            raise IntegrityError("Unique constraint violation for kind {} on fields: {}".format(table_name, ", ".join(fields)))
        elif existing_marker.instance != entity_key:
            markers_to_create.append(UniqueMarker(
                key=identifier_key,
                instance=entity_key,
                created=now
            ))
        else:
            # The marker is ours anyway
            markers.append(existing_marker)

    db.put(markers_to_create)
    return markers + markers_to_create


def get_markers_for_update(model, old_entity, new_entity):
    """
        Given an old entity state, and the new state, updates the identifiers
        appropriately. Should be called before saving the new_state
    """
    old_ids = set(unique_identifiers_from_entity(model, old_entity, ignore_pk=True))
{'fraction_non_alphanumeric': 0.06535539598293266, 'fraction_numerical': 0.000832552815069206, 'mean_word_length': 3.2093736311870344, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4118450', 'n_tokens_mistral': 2590, 'n_tokens_neox': 2493, 'n_words': 835}