<commit_before>import datetime
import decimal
import logging
import os
import shutil

from django.db.backends.util import format_number
from django.db.utils import DatabaseError

from google.appengine.api.datastore import Delete, Query
from google.appengine.api.datastore_errors import BadArgumentError, \
    BadValueError
from google.appengine.api.datastore_types import Blob, Key, Text, \
    ValidateInteger
from google.appengine.api.namespace_manager import set_namespace
from google.appengine.ext.db.metadata import get_kinds, get_namespaces

from djangotoolbox.db.base import \
    NonrelDatabaseClient, NonrelDatabaseFeatures, \
    NonrelDatabaseIntrospection, NonrelDatabaseOperations, \
    NonrelDatabaseValidation, NonrelDatabaseWrapper

from ..boot import DATA_ROOT
from ..utils import appid, on_production_server
from .creation import DatabaseCreation
from .stubs import stub_manager


DATASTORE_PATHS = {
    'datastore_path': os.path.join(DATA_ROOT, 'datastore'),
    'blobstore_path': os.path.join(DATA_ROOT, 'blobstore'),
    #'rdbms_sqlite_path': os.path.join(DATA_ROOT, 'rdbms'),
    'prospective_search_path': os.path.join(DATA_ROOT, 'prospective-search'),
}


def key_from_path(db_table, value):
    """
    Workaround for GAE choosing not to validate integer ids when
    creating keys.

    TODO: Should be removed if it gets fixed.
    """
    if isinstance(value, (int, long)):
        ValidateInteger(value, 'id')
    return Key.from_path(db_table, value)


def get_datastore_paths(options):
    paths = {}
    for key, path in DATASTORE_PATHS.items():
        paths[key] = options.get(key, path)
    return paths


def destroy_datastore(paths):
    """Destroys the appengine datastore at the specified paths."""
    for path in paths.values():
        if not path:
            continue
        try:
            if os.path.isdir(path):
                shutil.rmtree(path)
            else:
                os.remove(path)
        except OSError, error:
            if error.errno != 2:
                logging.error("Failed to clear datastore: %s" % error)


class DatabaseFeatures(NonrelDatabaseFeatures):
    allows_primary_key_0 = True
    supports_dicts = True


class DatabaseOperations(NonrelDatabaseOperations):
    compiler_module = __name__.rsplit('.', 1)[0] + '.compiler'

    # Used when a DecimalField does not specify max_digits or when
    # encoding a float as a string, fixed to preserve comparisons.
    DEFAULT_MAX_DIGITS = 16<commit_msg>0 is not an allowed value for a primary key; dropped the supports_dict feature (no longer needed); limit fields that can be safely used as primary keys.<commit_after>import datetime
import decimal
import logging
import os
import shutil

from django.db.backends.util import format_number
from django.db.utils import DatabaseError

from google.appengine.api.datastore import Delete, Query
from google.appengine.api.datastore_errors import BadArgumentError, \
    BadValueError
from google.appengine.api.datastore_types import Blob, Key, Text, \
    ValidateInteger
from google.appengine.api.namespace_manager import set_namespace
from google.appengine.ext.db.metadata import get_kinds, get_namespaces

from djangotoolbox.db.base import \
    NonrelDatabaseClient, NonrelDatabaseFeatures, \
    NonrelDatabaseIntrospection, NonrelDatabaseOperations, \
    NonrelDatabaseValidation, NonrelDatabaseWrapper

from ..boot import DATA_ROOT
from ..utils import appid, on_production_server
from .creation import DatabaseCreation
from .stubs import stub_manager


DATASTORE_PATHS = {
    'datastore_path': os.path.join(DATA_ROOT, 'datastore'),
    'blobstore_path': os.path.join(DATA_ROOT, 'blobstore'),
    #'rdbms_sqlite_path': os.path.join(DATA_ROOT, 'rdbms'),
    'prospective_search_path': os.path.join(DATA_ROOT, 'prospective-search'),
}


def key_from_path(db_table, value):
    """
    Workaround for GAE choosing not to validate integer ids when
    creating keys.

    TODO: Should be removed if it gets fixed.
    """
    if isinstance(value, (int, long)):
        ValidateInteger(value, 'id')
    return Key.from_path(db_table, value)


def get_datastore_paths(options):
    paths = {}
    for key, path in DATASTORE_PATHS.items():
        paths[key] = options.get(key, path)
    return paths


def destroy_datastore(paths):
    """Destroys the appengine datastore at the specified paths."""
    for path in paths.values():
        if not path:
            continue
        try:
            if os.path.isdir(path):
                shutil.rmtree(path)
            else:
                os.remove(path)
        except OSError, error:
            if error.errno != 2:
                logging.error("Failed to clear datastore: %s" % error)


class DatabaseFeatures(NonrelDatabaseFeatures):

    # GAE only allow strictly positive integers (and strings) to be
    # used as key values.
    allows_primary_key_0 = False

    # Anything that results in a something different than a positive
    # integer or a string cannot be directly used as a key on GAE.
    # Note that DecimalField values are encoded as strings, so can be
    # used as keys.
    # With some encoding, we could allow most fields to be used as a
    # primary key, but for now only mark what can and what cannot be
    # safely used.
    supports_primary_key_on = \
        NonrelDatabaseFeatures.supports_primary_key_on - set((
        'FloatField', 'DateField', 'DateTimeField', 'TimeField',
        'BooleanField', 'NullBooleanField', 'TextField', 'XMLField'))


class DatabaseOperations(NonrelDatabaseOperations):
    compiler_module = __name__.rsplit('.', 1)[0] + '.compiler'

    # Used when a DecimalField does not specify max_digits or when
    # encoding a float as a string, fixed to preserve comparisons.
    DEFAULT_MAX_DIGITS = 16
{'fraction_non_alphanumeric': 0.08305992056639613, 'fraction_numerical': 0.0022448627180107063, 'mean_word_length': 3.5320813771518, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7263047', 'n_tokens_mistral': 1707, 'n_tokens_neox': 1679, 'n_words': 511}