<commit_before>import logging
import warnings

from itertools import islice

from django.core.exceptions import ImproperlyConfigured
from django.db.backends import (
    BaseDatabaseOperations,
    BaseDatabaseClient,
    BaseDatabaseIntrospection,
    BaseDatabaseWrapper,
    BaseDatabaseFeatures,
    BaseDatabaseValidation
)

try:
    from django.db.backends.schema import BaseDatabaseSchemaEditor
except ImportError:
    #Django < 1.6 doesn't have BaseDatabaseSchemaEditor
    class BaseDatabaseSchemaEditor(object):
        pass

from django.conf import settings
from django.utils import timezone
from django.db.backends.creation import BaseDatabaseCreation
from django.db.models.sql.subqueries import InsertQuery, DeleteQuery
from django.db import models
from google.appengine.ext.db import metadata
from google.appengine.api import datastore
from google.appengine.api.datastore_types import Key, Text
from django.db.models.sql.constants import MULTI, SINGLE, GET_ITERATOR_CHUNK_SIZE
from django.db.models.sql.where import AND, OR
from django.utils.tree import Node

from django.core.cache import cache

from google.appengine.ext import testbed

from .commands import (
    SelectCommand,
    InsertCommand,
    FlushCommand
)

class DatabaseError(Exception):
    pass

class IntegrityError(DatabaseError):
    pass

class NotSupportedError(DatabaseError):
    pass

DEFAULT_CACHE_TIMEOUT = 10

def cache_entity(model, entity):
    unique_combinations = get_uniques_from_model(model)

    unique_keys = []
    for fields in unique_combinations:
        key_parts = []
        for x in fields:
            if x == model._meta.pk.column and x not in entity:
                value = entity.key().id_or_name()
            else:
                value = entity[x]

            key_parts.append((x, value))

        unique_keys.append(generate_unique_key(model, key_parts))

    for key in unique_keys:
        #logging.error("Caching entity with key %s", key)
        cache.set(key, entity, DEFAULT_CACHE_TIMEOUT)

def uncache_entity(model, entity):
    unique_combinations = get_uniques_from_model(model)

    unique_keys = []
    for fields in unique_combinations:
        key_parts = []
        for x in fields:
            if x == model._meta.pk.column and x not in entity:
                value = entity.key().id_or_name()
            else:
                value = entity[x]
            key_parts.append((x, value))

        key = generate_unique_key(model, key_parts)
        cache.delete(key)

def get_uniques_from_model(model):
    uniques = [ [ model._meta.get_field(y).column for y in x ] for x in model._meta.unique_together ]
    uniques.extend([[x.column] for x in model._meta.fields if x.unique])
    return uniques

def generate_unique_key(model, fields_and_values):
    fields_and_values = sorted(fields_and_values, key=lambda x: x[0]) #Sort by field name

    key = '%s.%s|' % (model._meta.app_label, model._meta.db_table)
    key += '|'.join(['%s:%s' % (field, value) for field, value in fields_and_values])
    return key

def get_entity_from_cache(key):
    entity = cache.get(key)
#    if entity:
#        logging.error("Got entity from cache with key %s", key)
    return entity

class Connection(object):
    """ Dummy connection class """
    def __init__(self, wrapper, params):
        self.creation = wrapper.creation
        self.ops = wrapper.ops
        self.params = params
        self.queries = []

    def rollback(self):
        pass

    def commit(self):
        pass

    def close(self):
        pass

def django_instance_to_entity(connection, model, fields, raw, instance):
    uses_inheritance = False
    inheritance_root = model
    db_table = model._meta.db_table

    def value_from_instance(_instance, _field):
        value = getattr(_instance, _field.attname) if raw else _field.pre_save(_instance, _instance._state.adding)

        #Some types don't need converting (e.g. a date field would be converted to a string by
        # get_db_prep_save, we don't want that!
        if _field.get_internal_type() not in ("DateTimeField", "DateField", "TimeField"):
            value = _field.get_db_prep_save(
                value,
                connection = connection
            )

        if (not _field.null and not _field.primary_key) and value is None:
            raise IntegrityError("You can't set %s (a non-nullable "
                                     "field) to None!" % _field.name)

        is_primary_key = False
        if _field.primary_key and _field.model == inheritance_root:
            is_primary_key = True

        value = connection.ops.value_for_db(value, _field)

        return value, is_primary_key

    if [ x for x in model._meta.get_parent_list() if not x._meta.abstract]:
        #We can simulate multi-table inheritance by using the same approach as
        #datastore "polymodels". Here we store the classes that form the heirarchy
        #and extend the fields to include those from parent models
        classes = [ model._meta.db_table ]
        for parent in model._meta.get_parent_list():
            if not parent._meta.parents:
                #If this is the top parent, override the db_table
                db_table = parent._meta.db_table
                inheritance_root = parent

            classes.append(parent._meta.db_table)
            for field in parent._meta.fields:
                fields.append(field)

        uses_inheritance = True


    #FIXME: This will only work for two levels of inheritance
    for obj in model._meta.get_all_related_objects():
        if model in [ x for x in obj.model._meta.parents if not x._meta.abstract]:
            try:
                related_obj = getattr(instance, obj.var_name)
            except obj.model.DoesNotExist:
                #We don't have a child attached to this field
                #so ignore
                continue

            for field in related_obj._meta.fields:
                fields.append(field)

    field_values = {}
    primary_key = None
    for field in fields:
        value, is_primary_key = value_from_instance(instance, field)
        if is_primary_key:
            primary_key = value
        else:
            field_values[field.column] = value

    kwargs = {}
    if primary_key:
        if isinstance(primary_key, int):
            kwargs["id"] = primary_key
        elif isinstance(primary_key, basestring):
            if len(primary_key) >= 500:
                warnings.warn("Truncating primary key"
                    " that is over 500 characters. THIS IS AN ERROR IN YOUR PROGRAM.",
                    RuntimeWarning
                )
                primary_key = primary_key[:500]

            kwargs["name"] = primary_key
        else:
            raise ValueError("Invalid primary key value")

    entity = datastore.Entity(db_table, **kwargs)
    entity.update(field_values)

    if uses_inheritance:
        entity["class"] = classes

    #print inheritance_root.__name__ if inheritance_root else "None", model.__name__, entity
    return entity

class Cursor(object):
    """ Dummy cursor class """
    def __init__(self, connection):
        self.connection = connection
        self.start_cursor = None
        self.returned_ids = []
        self.rowcount = -1
        self.last_select_command = None
        self.last_delete_command = None

    def execute(self, sql, *params):
        if isinstance(sql, SelectCommand):
            self.last_select_command = sql
            self.last_select_command.execute()
        elif isinstance(sql, FlushCommand):
            sql.execute()
        elif isinstance(sql, InsertCommand):
            self.connection.queries.append(sql)

            self.returned_ids = datastore.Put(sql.entities)

            #Now cache them, temporarily to help avoid consistency errors
            for key, entity in zip(self.returned_ids, sql.entities):
                pk_column = sql.model._meta.pk.column

                #If there are parent models, search the parents for the
                #first primary key which isn't a relation field
                for parent in sql.model._meta.parents.keys():
                    if not parent._meta.pk.rel:
                        pk_column = parent._meta.pk.column

                entity[pk_column] = key.id_or_name()
                cache_entity(sql.model, entity)

        else:
            raise RuntimeError("Can't execute traditional SQL: '%s'", sql)

    def fix_fk_null(self, query, constraint):
        alias = constraint.alias
        table_name = query.alias_map[alias][TABLE_NAME]
        lhs_join_col, rhs_join_col = join_cols(query.alias_map[alias])
        if table_name != constraint.field.rel.to._meta.db_table or \
                rhs_join_col != constraint.field.rel.to._meta.pk.column or \
                lhs_join_col != constraint.field.column:
            return
        next_alias = query.alias_map[alias][LHS_ALIAS]
        if not next_alias:
            return
        self.unref_alias(query, alias)
        alias = next_alias
        constraint.col = constraint.field.column
        constraint.alias = alias

    def next(self):
        row = self.fetchone()
        if row is None:
            raise StopIteration
        return row

    def fetchone(self, delete_flag=False):
        try:
            if isinstance(self.last_select_command.results, int):
                #Handle aggregate (e.g. count)
                return (self.last_select_command.results, )
            else:
                entity = self.last_select_command.results.next()
        except StopIteration:
            entity = None

        if entity is None:
            return None

        if delete_flag:
            uncache_entity(self.last_select_command.model, entity)

        result = []
        for col in self.last_select_command.queried_fields:
            if col == "__key__":
                key = entity.key()
                self.returned_ids.append(key)
                result.append(key.id_or_name())
            else:
                result.append(entity.get(col))

        return result

    def fetchmany(self, size, delete_flag=False):
        if not self.last_select_command.results:
            return []

        result = []
        i = 0
        while i < size:
            entity = self.fetchone(delete_flag)
            if entity is None:
                break

            result.append(entity)
            i += 1

        return result

    def delete(self):
        #Passing the delete_flag will uncache the entities
        self.fetchmany(GET_ITERATOR_CHUNK_SIZE, delete_flag=True)
        datastore.Delete(self.returned_ids)

    @property
    def lastrowid(self):
        return self.returned_ids[-1].id_or_name()

    def __iter__(self):
        return self

class Database(object):
    """ Fake DB API 2.0 for App engine """

    Error = DatabaseError
    DataError = DatabaseError
    DatabaseError = DatabaseError
    OperationalError = DatabaseError
    IntegrityError = IntegrityError
    InternalError = DatabaseError
    ProgrammingError = DatabaseError
    NotSupportedError = NotSupportedError
    InterfaceError = DatabaseError

class DatabaseOperations(BaseDatabaseOperations):
    compiler_module = "djangae.db.backends.appengine.compiler"

    def quote_name(self, name):
        return name

    def sql_flush(self, style, tables, seqs, allow_cascade=False):
        return [ FlushCommand(table) for table in tables ]

    def value_for_db(self, value, field):
        from google.appengine.api.datastore_types import Blob, Text
        from google.appengine.api.datastore_errors import BadArgumentError, BadValueError

        if value is None:
            return None

        # Convert decimals to strings preserving order.
        field_kind = field.get_internal_type()
        if field_kind == 'DecimalField':
            value = decimal_to_string(
                value, field.max_digits, field.decimal_places)

        db_type = self.connection.creation.db_type(field)

        if db_type == 'string' or db_type == 'text':
            if isinstance(value, str):
                value = value.decode('utf-8')
            if db_type == 'text':
                value = Text(value)
        elif db_type == 'bytes':
            # Store BlobField, DictField and EmbeddedModelField values as Blobs.
            value = Blob(value)
        elif field_kind == 'DateTimeField':
            if timezone.is_aware(value):
                if settings.USE_TZ:
                    value = value.astimezone(timezone.utc).replace(tzinfo=None)
                else:
                    raise ValueError("Djangoappengine backend does not support timezone-aware datetimes when USE_TZ is False.")
        return value

    def last_insert_id(self, cursor, db_table, column):
        return cursor.lastrowid

    def fetch_returned_insert_id(self, cursor):
        return cursor.lastrowid

class DatabaseClient(BaseDatabaseClient):
    pass


class DatabaseCreation(BaseDatabaseCreation):
    data_types = {
        'AutoField':                  'key',
        'RelatedAutoField':           'key',
        'ForeignKey':                 'key',
        'OneToOneField':              'key',
        'ManyToManyField':            'key',
        'BigIntegerField':            'long',
        'BooleanField':               'bool',
        'CharField':                  'string',
        'CommaSeparatedIntegerField': 'string',
        'DateField':                  'date',
        'DateTimeField':              'datetime',
        'DecimalField':               'decimal',
        'EmailField':                 'string',
        'FileField':                  'string',
        'FilePathField':              'string',
        'FloatField':                 'float',
        'ImageField':                 'string',
        'IntegerField':               'integer',
        'IPAddressField':             'string',
        'NullBooleanField':           'bool',
        'PositiveIntegerField':       'integer',
        'PositiveSmallIntegerField':  'integer',
        'SlugField':                  'string',
        'SmallIntegerField':          'integer',<commit_msg>That last commit was rubbish, this is the proper fix - don't convert datetime values to strings<commit_after>import logging
import warnings

from itertools import islice

from django.core.exceptions import ImproperlyConfigured
from django.db.backends import (
    BaseDatabaseOperations,
    BaseDatabaseClient,
    BaseDatabaseIntrospection,
    BaseDatabaseWrapper,
    BaseDatabaseFeatures,
    BaseDatabaseValidation
)

try:
    from django.db.backends.schema import BaseDatabaseSchemaEditor
except ImportError:
    #Django < 1.6 doesn't have BaseDatabaseSchemaEditor
    class BaseDatabaseSchemaEditor(object):
        pass

from django.conf import settings
from django.utils import timezone
from django.db.backends.creation import BaseDatabaseCreation
from django.db.models.sql.subqueries import InsertQuery, DeleteQuery
from django.db import models
from google.appengine.ext.db import metadata
from google.appengine.api import datastore
from google.appengine.api.datastore_types import Key, Text
from django.db.models.sql.constants import MULTI, SINGLE, GET_ITERATOR_CHUNK_SIZE
from django.db.models.sql.where import AND, OR
from django.utils.tree import Node

from django.core.cache import cache

from google.appengine.ext import testbed

from .commands import (
    SelectCommand,
    InsertCommand,
    FlushCommand
)

class DatabaseError(Exception):
    pass

class IntegrityError(DatabaseError):
    pass

class NotSupportedError(DatabaseError):
    pass

DEFAULT_CACHE_TIMEOUT = 10

def cache_entity(model, entity):
    unique_combinations = get_uniques_from_model(model)

    unique_keys = []
    for fields in unique_combinations:
        key_parts = []
        for x in fields:
            if x == model._meta.pk.column and x not in entity:
                value = entity.key().id_or_name()
            else:
                value = entity[x]

            key_parts.append((x, value))

        unique_keys.append(generate_unique_key(model, key_parts))

    for key in unique_keys:
        #logging.error("Caching entity with key %s", key)
        cache.set(key, entity, DEFAULT_CACHE_TIMEOUT)

def uncache_entity(model, entity):
    unique_combinations = get_uniques_from_model(model)

    unique_keys = []
    for fields in unique_combinations:
        key_parts = []
        for x in fields:
            if x == model._meta.pk.column and x not in entity:
                value = entity.key().id_or_name()
            else:
                value = entity[x]
            key_parts.append((x, value))

        key = generate_unique_key(model, key_parts)
        cache.delete(key)

def get_uniques_from_model(model):
    uniques = [ [ model._meta.get_field(y).column for y in x ] for x in model._meta.unique_together ]
    uniques.extend([[x.column] for x in model._meta.fields if x.unique])
    return uniques

def generate_unique_key(model, fields_and_values):
    fields_and_values = sorted(fields_and_values, key=lambda x: x[0]) #Sort by field name

    key = '%s.%s|' % (model._meta.app_label, model._meta.db_table)
    key += '|'.join(['%s:%s' % (field, value) for field, value in fields_and_values])
    return key

def get_entity_from_cache(key):
    entity = cache.get(key)
#    if entity:
#        logging.error("Got entity from cache with key %s", key)
    return entity

class Connection(object):
    """ Dummy connection class """
    def __init__(self, wrapper, params):
        self.creation = wrapper.creation
        self.ops = wrapper.ops
        self.params = params
        self.queries = []

    def rollback(self):
        pass

    def commit(self):
        pass

    def close(self):
        pass

def django_instance_to_entity(connection, model, fields, raw, instance):
    uses_inheritance = False
    inheritance_root = model
    db_table = model._meta.db_table

    def value_from_instance(_instance, _field):
        value = getattr(_instance, _field.attname) if raw else _field.pre_save(_instance, _instance._state.adding)

        value = _field.get_db_prep_save(
            value,
            connection = connection
        )

        if (not _field.null and not _field.primary_key) and value is None:
            raise IntegrityError("You can't set %s (a non-nullable "
                                     "field) to None!" % _field.name)

        is_primary_key = False
        if _field.primary_key and _field.model == inheritance_root:
            is_primary_key = True

        value = connection.ops.value_for_db(value, _field)

        return value, is_primary_key

    if [ x for x in model._meta.get_parent_list() if not x._meta.abstract]:
        #We can simulate multi-table inheritance by using the same approach as
        #datastore "polymodels". Here we store the classes that form the heirarchy
        #and extend the fields to include those from parent models
        classes = [ model._meta.db_table ]
        for parent in model._meta.get_parent_list():
            if not parent._meta.parents:
                #If this is the top parent, override the db_table
                db_table = parent._meta.db_table
                inheritance_root = parent

            classes.append(parent._meta.db_table)
            for field in parent._meta.fields:
                fields.append(field)

        uses_inheritance = True


    #FIXME: This will only work for two levels of inheritance
    for obj in model._meta.get_all_related_objects():
        if model in [ x for x in obj.model._meta.parents if not x._meta.abstract]:
            try:
                related_obj = getattr(instance, obj.var_name)
            except obj.model.DoesNotExist:
                #We don't have a child attached to this field
                #so ignore
                continue

            for field in related_obj._meta.fields:
                fields.append(field)

    field_values = {}
    primary_key = None
    for field in fields:
        value, is_primary_key = value_from_instance(instance, field)
        if is_primary_key:
            primary_key = value
        else:
            field_values[field.column] = value

    kwargs = {}
    if primary_key:
        if isinstance(primary_key, int):
            kwargs["id"] = primary_key
        elif isinstance(primary_key, basestring):
            if len(primary_key) >= 500:
                warnings.warn("Truncating primary key"
                    " that is over 500 characters. THIS IS AN ERROR IN YOUR PROGRAM.",
                    RuntimeWarning
                )
                primary_key = primary_key[:500]

            kwargs["name"] = primary_key
        else:
            raise ValueError("Invalid primary key value")

    entity = datastore.Entity(db_table, **kwargs)
    entity.update(field_values)

    if uses_inheritance:
        entity["class"] = classes

    #print inheritance_root.__name__ if inheritance_root else "None", model.__name__, entity
    return entity

class Cursor(object):
    """ Dummy cursor class """
    def __init__(self, connection):
        self.connection = connection
        self.start_cursor = None
        self.returned_ids = []
        self.rowcount = -1
        self.last_select_command = None
        self.last_delete_command = None

    def execute(self, sql, *params):
        if isinstance(sql, SelectCommand):
            self.last_select_command = sql
            self.last_select_command.execute()
        elif isinstance(sql, FlushCommand):
            sql.execute()
        elif isinstance(sql, InsertCommand):
            self.connection.queries.append(sql)

            self.returned_ids = datastore.Put(sql.entities)

            #Now cache them, temporarily to help avoid consistency errors
            for key, entity in zip(self.returned_ids, sql.entities):
                pk_column = sql.model._meta.pk.column

                #If there are parent models, search the parents for the
                #first primary key which isn't a relation field
                for parent in sql.model._meta.parents.keys():
                    if not parent._meta.pk.rel:
                        pk_column = parent._meta.pk.column

                entity[pk_column] = key.id_or_name()
                cache_entity(sql.model, entity)

        else:
            raise RuntimeError("Can't execute traditional SQL: '%s'", sql)

    def fix_fk_null(self, query, constraint):
        alias = constraint.alias
        table_name = query.alias_map[alias][TABLE_NAME]
        lhs_join_col, rhs_join_col = join_cols(query.alias_map[alias])
        if table_name != constraint.field.rel.to._meta.db_table or \
                rhs_join_col != constraint.field.rel.to._meta.pk.column or \
                lhs_join_col != constraint.field.column:
            return
        next_alias = query.alias_map[alias][LHS_ALIAS]
        if not next_alias:
            return
        self.unref_alias(query, alias)
        alias = next_alias
        constraint.col = constraint.field.column
        constraint.alias = alias

    def next(self):
        row = self.fetchone()
        if row is None:
            raise StopIteration
        return row

    def fetchone(self, delete_flag=False):
        try:
            if isinstance(self.last_select_command.results, int):
                #Handle aggregate (e.g. count)
                return (self.last_select_command.results, )
            else:
                entity = self.last_select_command.results.next()
        except StopIteration:
            entity = None

        if entity is None:
            return None

        if delete_flag:
            uncache_entity(self.last_select_command.model, entity)

        result = []
        for col in self.last_select_command.queried_fields:
            if col == "__key__":
                key = entity.key()
                self.returned_ids.append(key)
                result.append(key.id_or_name())
            else:
                result.append(entity.get(col))

        return result

    def fetchmany(self, size, delete_flag=False):
        if not self.last_select_command.results:
            return []

        result = []
        i = 0
        while i < size:
            entity = self.fetchone(delete_flag)
            if entity is None:
                break

            result.append(entity)
            i += 1

        return result

    def delete(self):
        #Passing the delete_flag will uncache the entities
        self.fetchmany(GET_ITERATOR_CHUNK_SIZE, delete_flag=True)
        datastore.Delete(self.returned_ids)

    @property
    def lastrowid(self):
        return self.returned_ids[-1].id_or_name()

    def __iter__(self):
        return self

class Database(object):
    """ Fake DB API 2.0 for App engine """

    Error = DatabaseError
    DataError = DatabaseError
    DatabaseError = DatabaseError
    OperationalError = DatabaseError
    IntegrityError = IntegrityError
    InternalError = DatabaseError
    ProgrammingError = DatabaseError
    NotSupportedError = NotSupportedError
    InterfaceError = DatabaseError

class DatabaseOperations(BaseDatabaseOperations):
    compiler_module = "djangae.db.backends.appengine.compiler"

    def quote_name(self, name):
        return name

    def sql_flush(self, style, tables, seqs, allow_cascade=False):
        return [ FlushCommand(table) for table in tables ]

    def value_for_db(self, value, field):
        from google.appengine.api.datastore_types import Blob, Text
        from google.appengine.api.datastore_errors import BadArgumentError, BadValueError

        if value is None:
            return None

        # Convert decimals to strings preserving order.
        field_kind = field.get_internal_type()
        if field_kind == 'DecimalField':
            value = decimal_to_string(
                value, field.max_digits, field.decimal_places)

        db_type = self.connection.creation.db_type(field)

        if db_type == 'string' or db_type == 'text':
            if isinstance(value, str):
                value = value.decode('utf-8')
            if db_type == 'text':
                value = Text(value)
        elif db_type == 'bytes':
            # Store BlobField, DictField and EmbeddedModelField values as Blobs.
            value = Blob(value)
        elif field_kind == 'DateTimeField':
            if timezone.is_aware(value):
                if settings.USE_TZ:
                    value = value.astimezone(timezone.utc).replace(tzinfo=None)
                else:
                    raise ValueError("Djangoappengine backend does not support timezone-aware datetimes when USE_TZ is False.")
        return value

    def last_insert_id(self, cursor, db_table, column):
        return cursor.lastrowid

    def fetch_returned_insert_id(self, cursor):
        return cursor.lastrowid

    def value_to_db_datetime(self, value): return value
    def value_to_db_date(self, value): return value
    def value_to_db_time(self, value): return value

class DatabaseClient(BaseDatabaseClient):
    pass


class DatabaseCreation(BaseDatabaseCreation):
    data_types = {
        'AutoField':                  'key',
        'RelatedAutoField':           'key',
        'ForeignKey':                 'key',
        'OneToOneField':              'key',
        'ManyToManyField':            'key',
        'BigIntegerField':            'long',
        'BooleanField':               'bool',
        'CharField':                  'string',
        'CommaSeparatedIntegerField': 'string',
        'DateField':                  'date',
        'DateTimeField':              'datetime',
        'DecimalField':               'decimal',
        'EmailField':                 'string',
        'FileField':                  'string',
        'FilePathField':              'string',
        'FloatField':                 'float',
        'ImageField':                 'string',
        'IntegerField':               'integer',
        'IPAddressField':             'string',
        'NullBooleanField':           'bool',
        'PositiveIntegerField':       'integer',
        'PositiveSmallIntegerField':  'integer',
        'SlugField':                  'string',
        'SmallIntegerField':          'integer',
{'fraction_non_alphanumeric': 0.08353339923739585, 'fraction_numerical': 0.0014828414065809915, 'mean_word_length': 2.1570441373160945, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4995582', 'n_tokens_mistral': 7915, 'n_tokens_neox': 7635, 'n_words': 2011}