<commit_before>#!/usr/bin/python2.7
# Copyright 2010 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""The Person Finder data model, based on PFIF (http://zesty.ca/pfif)."""

__author__ = '<EMAIL> (<NAME>) and many other Googlers'

from datetime import timedelta

from google.appengine.api import datastore_errors
from google.appengine.api import memcache
from google.appengine.ext import db

import config
import full_text_search
import indexing
import pfif
import prefix
from const import HOME_DOMAIN, NOTE_STATUS_TEXT

# default # of days for a record to expire.
DEFAULT_EXPIRATION_DAYS = 40

# ==== PFIF record IDs =====================================================

def is_original(repo, record_id):
    """Returns True if this is a record_id for a record originally created in
    the specified repository."""
    try:
        repo_id, local_id = record_id.split('/', 1)
        return repo_id == repo + '.' + HOME_DOMAIN
    except ValueError:
        raise ValueError('%r is not a valid record_id' % record_id)

def is_clone(repo, record_id):
    """Returns True if this is a record_id for a clone record (a record created
    in another repository and copied into the specified one)."""
    return not is_original(repo, record_id)

def filter_by_prefix(query, key_name_prefix):
    """Filters a query for key_names that have the given prefix.  If root_kind
    is specified, filters the query for children of any entities that are of
    that kind with the given prefix; otherwise, the results are assumed to be
    top-level entities of the kind being queried."""
    root_kind = query._model_class.__name__
    min_key = db.Key.from_path(root_kind, key_name_prefix)
    max_key = db.Key.from_path(root_kind, key_name_prefix + u'\uffff')
    return query.filter('__key__ >=', min_key).filter('__key__ <=', max_key)

def get_properties_as_dict(db_obj):
    """Returns a dictionary containing all (dynamic)* properties of db_obj."""
    properties = dict((k, v.__get__(db_obj, db_obj.__class__)) for
                      k, v in db_obj.properties().iteritems() if
                      v.__get__(db_obj, db_obj.__class__))
    dynamic_properties = dict((prop, getattr(db_obj, prop)) for
                              prop in db_obj.dynamic_properties())
    properties.update(dynamic_properties)
    return properties

def clone_to_new_type(origin, dest_class, **kwargs):
    """Clones the given entity to a new entity of the type "dest_class".
    Optionally, pass in values to kwargs to update values during cloning."""
    vals = get_properties_as_dict(origin)
    vals.update(**kwargs)
    if hasattr(origin, 'record_id'):
        vals.update(record_id=origin.record_id)
    return dest_class(key_name=origin.key().name(), **vals)

# ==== Model classes =======================================================

# Every Person or Note entity belongs to a specific repository.  To partition
# the datastore, key names consist of the repo name, a colon, and then the
# record ID.  Each repository appears to be a separate instance of the app.

# Note that the repository name doesn't necessarily have to match the original
# repository domain in the record ID!  For example, a person record created at
# foo.person-finder.appspot.com would have a key name such as:
#
#     foo:foo.person-finder.appspot.com/person.234
#
# This record would be searchable only at foo.person-finder.appspot.com --
# each repository is independent.  Copying it to bar.person-finder.appspot.com
# would produce a clone record with the key name:
#
#     bar:foo.person-finder.appspot.com/person.234
#
# That is, the clone has the same record ID but a different repository name.

class Repo(db.Model):
    """Identifier for a repository of Person and Note records.  This is a
    top-level entity, with no parent, whose existence just indicates the
    existence of a repository.  Key name: unique repository name.  In the UI,
    each repository behaves like an independent instance of the application."""

    # No properties for now; only the key_name is significant.  The repository
    # title and other settings are all in ConfigEntry entities (see config.py).
    # The per-repository 'deactivated' setting blocks UI and API access to the
    # repository, replacing all its pages with a deactivation message.
<commit_msg>Store activation status on repo objects, step 1 (#627)<commit_after>#!/usr/bin/python2.7
# Copyright 2010 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""The Person Finder data model, based on PFIF (http://zesty.ca/pfif)."""

__author__ = '<EMAIL> (<NAME>) and many other Googlers'

from datetime import timedelta

from google.appengine.api import datastore_errors
from google.appengine.api import memcache
from google.appengine.ext import db

import config
import full_text_search
import indexing
import pfif
import prefix
from const import HOME_DOMAIN, NOTE_STATUS_TEXT

# default # of days for a record to expire.
DEFAULT_EXPIRATION_DAYS = 40

# ==== PFIF record IDs =====================================================

def is_original(repo, record_id):
    """Returns True if this is a record_id for a record originally created in
    the specified repository."""
    try:
        repo_id, local_id = record_id.split('/', 1)
        return repo_id == repo + '.' + HOME_DOMAIN
    except ValueError:
        raise ValueError('%r is not a valid record_id' % record_id)

def is_clone(repo, record_id):
    """Returns True if this is a record_id for a clone record (a record created
    in another repository and copied into the specified one)."""
    return not is_original(repo, record_id)

def filter_by_prefix(query, key_name_prefix):
    """Filters a query for key_names that have the given prefix.  If root_kind
    is specified, filters the query for children of any entities that are of
    that kind with the given prefix; otherwise, the results are assumed to be
    top-level entities of the kind being queried."""
    root_kind = query._model_class.__name__
    min_key = db.Key.from_path(root_kind, key_name_prefix)
    max_key = db.Key.from_path(root_kind, key_name_prefix + u'\uffff')
    return query.filter('__key__ >=', min_key).filter('__key__ <=', max_key)

def get_properties_as_dict(db_obj):
    """Returns a dictionary containing all (dynamic)* properties of db_obj."""
    properties = dict((k, v.__get__(db_obj, db_obj.__class__)) for
                      k, v in db_obj.properties().iteritems() if
                      v.__get__(db_obj, db_obj.__class__))
    dynamic_properties = dict((prop, getattr(db_obj, prop)) for
                              prop in db_obj.dynamic_properties())
    properties.update(dynamic_properties)
    return properties

def clone_to_new_type(origin, dest_class, **kwargs):
    """Clones the given entity to a new entity of the type "dest_class".
    Optionally, pass in values to kwargs to update values during cloning."""
    vals = get_properties_as_dict(origin)
    vals.update(**kwargs)
    if hasattr(origin, 'record_id'):
        vals.update(record_id=origin.record_id)
    return dest_class(key_name=origin.key().name(), **vals)

# ==== Model classes =======================================================

# Every Person or Note entity belongs to a specific repository.  To partition
# the datastore, key names consist of the repo name, a colon, and then the
# record ID.  Each repository appears to be a separate instance of the app.

# Note that the repository name doesn't necessarily have to match the original
# repository domain in the record ID!  For example, a person record created at
# foo.person-finder.appspot.com would have a key name such as:
#
#     foo:foo.person-finder.appspot.com/person.234
#
# This record would be searchable only at foo.person-finder.appspot.com --
# each repository is independent.  Copying it to bar.person-finder.appspot.com
# would produce a clone record with the key name:
#
#     bar:foo.person-finder.appspot.com/person.234
#
# That is, the clone has the same record ID but a different repository name.

class Repo(db.Model):
    """Identifier for a repository of Person and Note records.  This is a
    top-level entity, with no parent, whose existence just indicates the
    existence of a repository.  Key name: unique repository name.  In the UI,
    each repository behaves like an independent instance of the application."""

    class ActivationStatus(object):
        """An enum for the launch/activation status of the repo."""
        # For use with repositories that have not yet been publicly launched.
        # Staging repos aren't listed on the homepage or in the repository feed,
        # but are otherwise totally usable repos for anyone who knows the URL.
        STAGING = 0
        # For repositories in active use. These repos are listed on the homepage
        # and in the repository feed.
        ACTIVE = 1
        # For repositories that have been turned down. The repository is
        # unavailable, through either the web interface or the API, and users
        # will instead see a deactivation message. These repos aren't listed on
        # the homepage or in the repository feed.
        DEACTIVATED = 2

    # TODO(nworden): actually use this field
    activation_status = db.IntegerProperty(
        required=False, default=ActivationStatus.STAGING)

    # Whether the repository is in test mode; meant for use with evergreen
    # repositories when they're not needed. Records for repos in test mode are
    # automatically deleted after 24 hours.
    test_mode = db.BooleanProperty(default=False)

    # Few properties for now; the repository title and other settings are all in
    # ConfigEntry entities (see config.py).

{'fraction_non_alphanumeric': 0.09727662749487036, 'fraction_numerical': 0.00438351053907853, 'mean_word_length': 3.717553893532776, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6382610', 'n_tokens_mistral': 2970, 'n_tokens_neox': 2861, 'n_words': 1223}