<commit_before># Copyright (C) 2016 Google Inc.
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>


"""gGRC Collection REST services implementation. Common to all gGRC collection
resources.
"""

import datetime
import hashlib
import itertools
import json
import time
from logging import getLogger
from collections import defaultdict
from exceptions import TypeError
from wsgiref.handlers import format_date_time
from urllib import urlencode

from blinker import Namespace
from flask import url_for, request, current_app, g, has_request_context
from flask.views import View
from flask.ext.sqlalchemy import Pagination
import sqlalchemy.orm.exc
from sqlalchemy import and_, or_
from sqlalchemy.exc import IntegrityError
from sqlalchemy.sql.expression import tuple_
from werkzeug.exceptions import BadRequest, Forbidden

import ggrc.builder.json
import ggrc.models
from ggrc import db, utils
from ggrc.utils import as_json, benchmark
from ggrc.fulltext import get_indexer
from ggrc.fulltext.recordbuilder import fts_record_for
from ggrc.login import get_current_user_id, get_current_user
from ggrc.models.cache import Cache
from ggrc.models.event import Event
from ggrc.models.revision import Revision
from ggrc.models.exceptions import ValidationError, translate_message
from ggrc.rbac import permissions, context_query_filter
from ggrc.services.attribute_query import AttributeQueryBuilder
from ggrc.models.background_task import BackgroundTask, create_task
from ggrc import settings


# pylint: disable=invalid-name
logger = getLogger(__name__)


CACHE_EXPIRY_COLLECTION = 60


def get_oauth_credentials():
  from flask import session
  return session.get('oauth_credentials')


def _get_cache_manager():
  from ggrc.cache import CacheManager, MemCache
  cache_manager = CacheManager()
  cache_manager.initialize(MemCache())
  return cache_manager


def get_cache_key(obj, type=None, id=None):
  """Returns a string identifier for the specified object or stub.

  `obj` can be:
    <db.Model> -- declarative model instance
    (type, id) -- tuple
    { 'type': type, 'id': id } -- dict
  """
  if isinstance(obj, tuple):
    type, id = obj
  elif isinstance(obj, dict):
    type = obj.get('type', None)
    id = obj.get('id', None)
  if isinstance(type, (str, unicode)):
    model = ggrc.models.get_model(type)
    assert model is not None, "Invalid model name: {}".format(type)
    type = ggrc.models.get_model(type)._inflector.table_plural
  if not isinstance(obj, (tuple, dict)):
    if type is None:
      type = obj._inflector.table_plural
    if id is None:
      id = obj.id
  return 'collection:{type}:{id}'.format(type=type, id=id)


def get_cache_class(obj):
  return obj.__class__.__name__


def get_related_keys_for_expiration(context, o):
  cls = get_cache_class(o)
  keys = []
  mappings = context.cache_manager.supported_mappings.get(cls, [])
  if len(mappings) > 0:
    for (cls, attr, polymorph) in mappings:
      if polymorph:
        key = get_cache_key(
            None,
            type=getattr(o, '{0}_type'.format(attr)),
            id=getattr(o, '{0}_id'.format(attr)))
        keys.append(key)
      else:
        obj = getattr(o, attr, None)
        if obj:
          if isinstance(obj, list):
            for inner_o in obj:
              key = get_cache_key(inner_o)
              keys.append(key)
          else:
            key = get_cache_key(obj)
            keys.append(key)
  return keys


def set_ids_for_new_custom_attributes(parent_obj):
  """
  When we are creating custom attribute values and definitions for
  POST requests, parent object ID is not yet defined. This is why we update
  custom attribute values at this point and set the correct attributable_id

  Args:
    parent_obj: parent object to be set as attributable

  Returns:
    None
  """
  if not hasattr(parent_obj, "PER_OBJECT_CUSTOM_ATTRIBUTABLE"):
    return

  modified_objects = get_modified_objects(db.session).new

  object_attrs = {
      "CustomAttributeValue": "attributable_id",
      "CustomAttributeDefinition": "definition_id"
  }

  for obj in modified_objects:
    if obj.type not in object_attrs:
      continue

    attr = object_attrs[obj.type]
    setattr(obj, attr, parent_obj.id)

    # Disable state updating so that a newly create object doesn't go straight
    # from Draft to Modified.
    if hasattr(obj, '_skip_os_state_update'):
      obj.skip_os_state_update()
    db.session.add(obj)
  db.session.flush()


def memcache_mark_for_deletion(context, objects_to_mark):
  """
  Mark objects for deletion from memcache

  Args:
    context: application context
    objects_to_mark: A list of objects to be deleted from memcache

  Returns:
    None
  """
  for o, _ in objects_to_mark:
    cls = get_cache_class(o)
    if cls in context.cache_manager.supported_classes:
      key = get_cache_key(o)
      context.cache_manager.marked_for_delete.append(key)
      context.cache_manager.marked_for_delete.extend(
          get_related_keys_for_expiration(context, o))


def update_memcache_before_commit(context, modified_objects, expiry_time):
  """
  Preparing the memccache entries to be updated before DB commit
  Also update the memcache to indicate the status cache operation
  'InProgress' waiting for DB commit
  Raises Exception on failures, cannot proceed with DB commit

  Args:
    context: POST/PUT/DELETE HTTP request or import Converter contextual object
    modified_objects:  objects in cache maintained prior to committing to DB
    expiry_time: Expiry time specified for memcache ADD and DELETE
  Returns:
    None

  """
  if getattr(settings, 'MEMCACHE_MECHANISM', False) is False:
    return

  context.cache_manager = _get_cache_manager()

  if modified_objects is not None:
    if len(modified_objects.new) > 0:
      memcache_mark_for_deletion(context, modified_objects.new.items())

    if len(modified_objects.dirty) > 0:
      memcache_mark_for_deletion(context, modified_objects.dirty.items())

    if len(modified_objects.deleted) > 0:
      memcache_mark_for_deletion(context, modified_objects.deleted.items())

  status_entries = {}
  for key in context.cache_manager.marked_for_delete:
    build_cache_status(status_entries, 'DeleteOp:' + key,
                       expiry_time, 'InProgress')
  if len(status_entries) > 0:
    logger.info("CACHE: status entries: %s", status_entries)
    ret = context.cache_manager.bulk_add(status_entries, expiry_time)
    if ret is not None and len(ret) == 0:
      pass
    else:
      logger.error('CACHE: Unable to add status for newly created entries %s',
                   ret)


def update_memcache_after_commit(context):
  """
  The memccache entries is updated after DB commit
  Logs error if there are errors in updating entries in cache

  Args:
    context: POST/PUT/DELETE HTTP request or import Converter contextual object
    modified_objects:  objects in cache maintained prior to committing to DB
  Returns:
    None

  """
  if getattr(settings, 'MEMCACHE_MECHANISM', False) is False:
    return

  if context.cache_manager is None:
    logger.error("CACHE: Error in initiaizing cache manager")
    return

  cache_manager = context.cache_manager

  related_objs = list()
  for val in getattr(g, "referenced_objects", {}).itervalues():
    obj_list = val.values()
    if obj_list:
      related_objs.append((obj_list[0], None))
  memcache_mark_for_deletion(context, related_objs)

  # TODO(dan): check for duplicates in marked_for_delete
  if len(cache_manager.marked_for_delete) > 0:
    delete_result = cache_manager.bulk_delete(
        cache_manager.marked_for_delete, 0)
    # TODO(dan): handling failure including network errors,
    #            currently we log errors
    if delete_result is not True:
      logger.error("CACHE: Failed to remove collection from cache")

  status_entries = []
  for key in cache_manager.marked_for_delete:
    status_entries.append('DeleteOp:' + str(key))
  if len(status_entries) > 0:
    delete_result = cache_manager.bulk_delete(status_entries, 0)
    # TODO(dan): handling failure including network errors,
    #            currently we log errors
    if delete_result is not True:
      logger.error("CACHE: Failed to remove status entries from cache")

  clear_permission_cache()
  cache_manager.clear_cache()


def build_cache_status(data, key, expiry_timeout, status):
  """
  Build the dictionary for storing operational status of cache

  Args:
    data: dictionary to update
    key: key to dictionary
    expiry_timeout: timeout for expiry cache
    status: Update status entry, e.g.InProgress
  Returns:
    None
  """
  data[key] = {'expiry': expiry_timeout, 'status': status}


def inclusion_filter(obj):
  return permissions.is_allowed_read(obj.__class__.__name__,
                                     obj.id, obj.context_id)


def get_cache(create=False):
  """
  Retrieves the cache from the Flask global object. The create arg
  indicates if a new cache should be created if none exists. If we
  are not in a request context, no cache is created (return None).
  """
  if has_request_context():
    cache = getattr(g, 'cache', None)
    if cache is None and create:
      cache = g.cache = Cache()
    return cache
  else:
    logger.warning("No request context - no cache created")
    return None


def get_modified_objects(session):
  session.flush()
  cache = get_cache()
  if cache:
    return cache.copy()
  else:
    return None


def update_index(session, cache):
  if cache:
    indexer = get_indexer()
    for obj in cache.new:
      indexer.create_record(fts_record_for(obj), commit=False)
    for obj in cache.dirty:
      indexer.update_record(fts_record_for(obj), commit=False)
    for obj in cache.deleted:
      indexer.delete_record(obj.id, obj.__class__.__name__, commit=False)
    session.commit()


def log_event(session, obj=None, current_user_id=None, flush=True):
  revisions = []
  if flush:
    session.flush()
  if current_user_id is None:
    current_user_id = get_current_user_id()
  cache = get_cache()
  if cache:
    for obj_ in cache.dirty:
      revision = Revision(obj_, current_user_id, 'modified', obj_.log_json())
      revisions.append(revision)
    for obj_ in cache.deleted:
      revision = Revision(obj_, current_user_id, 'deleted', obj_.log_json())
      revisions.append(revision)
    for obj_ in cache.new:
      revision = Revision(obj_, current_user_id, 'created', obj_.log_json())
      revisions.append(revision)
  if obj is None:
    resource_id = 0
    resource_type = None
    action = 'BULK'
    context_id = 0
  else:
    resource_id = obj.id
    resource_type = str(obj.__class__.__name__)
    action = request.method
    context_id = obj.context_id
  if revisions:
    event = Event(
        modified_by_id=current_user_id,
        action=action,
        resource_id=resource_id,
        resource_type=resource_type,
        context_id=context_id)
    event.revisions = revisions
    session.add(event)


def clear_permission_cache():
  if not getattr(settings, 'MEMCACHE_MECHANISM', False):
    return
  cache = _get_cache_manager().cache_object.memcache_client
  cached_keys_set = cache.get('permissions:list') or set()
  cached_keys_set.add('permissions:list')
  # We delete all the cached user permissions as well as
  # the permissions:list value itself
  cache.delete_multi(cached_keys_set)


class ModelView(View):
  """Basic view handler for all models"""
  # pylint: disable=protected-access
  # access to _sa_class_manager is needed for fetching the right mapper
  DEFAULT_PAGE_SIZE = 20
  MAX_PAGE_SIZE = 100
  pk = 'id'
  pk_type = 'int'

  _model = None

  # Simple accessor properties
  @property
  def request(self):
    return request

  @property
  def model(self):
    return self._model

  @property
  def modified_attr_name(self):
    return 'updated_at'

  @property
  def modified_attr(self):
    """Return the model attribute to be used for Last-Modified header and
    sorting collection elements.
    """
    return getattr(self.model, self.modified_attr_name)

  def modified_at(self, obj):
    return getattr(obj, self.modified_attr_name)

  def _get_type_select_column(self, model):
    mapper = model._sa_class_manager.mapper
    if mapper.polymorphic_on is None:
      # if len(mapper.self_and_descendants) == 1:
      type_column = sqlalchemy.literal(mapper.class_.__name__)
    else:
      # Handle polymorphic types with CASE
      type_column = sqlalchemy.case(
          value=mapper.polymorphic_on,
          whens={
              val: m.class_.__name__
              for val, m in mapper.polymorphic_map.items()
          })
    return type_column

  def _get_type_where_clause(self, model):
    mapper = model._sa_class_manager.mapper
    if mapper.polymorphic_on is None:
      return True
    else:
      mappers = list(mapper.self_and_descendants)
      polymorphic_on_values = list(
          val
          for val, m in mapper.polymorphic_map.items()
          if m in mappers)
      return mapper.polymorphic_on.in_(polymorphic_on_values)

  def _get_matching_types(self, model):
    mapper = model._sa_class_manager.mapper
    if len(list(mapper.self_and_descendants)) == 1:
      return mapper.class_.__name__
    else:
      # FIXME: Actually needs to use 'self_and_descendants'
      return [m.class_.__name__ for m in mapper.self_and_descendants]

  def get_match_columns(self, model):
    mapper = model._sa_class_manager.mapper
    columns = []
    columns.append(mapper.primary_key[0].label('id'))
    # columns.append(model.id.label('id'))
    columns.append(self._get_type_select_column(model).label('type'))
    if hasattr(mapper.c, 'context_id'):
      columns.append(mapper.c.context_id.label('context_id'))
    if hasattr(mapper.c, 'updated_at'):
      columns.append(mapper.c.updated_at.label('updated_at'))
    # columns.append(self._get_polymorphic_column(model))
    return columns

  def get_collection_matches(self, model, filter_by_contexts=True):
    columns = self.get_match_columns(self.model)
    query = db.session.query(*columns).filter(
        self._get_type_where_clause(model))
    return self.filter_query_by_request(
        query, filter_by_contexts=filter_by_contexts)

  def get_resource_match_query(self, model, id):
    columns = self.get_match_columns(model)
    query = db.session.query(*columns).filter(
        and_(
            self._get_type_where_clause(model),
            columns[0] == id))
    return query

  # Default model/DB helpers
  def get_collection(self, filter_by_contexts=True):
    if '__stubs_only' not in request.args and \
       hasattr(self.model, 'eager_query'):
      query = self.model.eager_query()
    else:
      query = db.session.query(self.model)
    return self.filter_query_by_request(
        query, filter_by_contexts=filter_by_contexts)

  def filter_query_by_request(self, query, filter_by_contexts=True):  # noqa
    joinlist = []
    if request.args:
      querybuilder = AttributeQueryBuilder(self.model)
      filter_, joinlist, _ = querybuilder.collection_filters(request.args)
      if filter_ is not None:
        for j in joinlist:
          query = query.join(j)
        query = query.filter(filter_)
    if filter_by_contexts:
      contexts = permissions.read_contexts_for(self.model.__name__)
      resources = permissions.read_resources_for(self.model.__name__)
      filter_expr = context_query_filter(self.model.context_id, contexts)
      if resources:
        filter_expr = or_(filter_expr, self.model.id.in_(resources))
      query = query.filter(filter_expr)
      for j in joinlist:
        j_class = j.property.mapper.class_
        j_contexts = permissions.read_contexts_for(j_class.__name__)
        j_resources = permissions.read_resources_for(j_class.__name__)
        if j_contexts is not None:
          j_filter_expr = context_query_filter(j_class.context_id, j_contexts)
          if resources:
            j_filter_expr = or_(j_filter_expr, self.model.id.in_(j_resources))
          query = query.filter(j_filter_expr)
        elif resources:
          query = query.filter(self.model.id.in_(resources))
    if '__search' in request.args:
      terms = request.args['__search']
      types = self._get_matching_types(self.model)
      indexer = get_indexer()
      models = indexer._get_grouped_types(types)
      search_query = indexer.get_permissions_query(models, 'read', None)
      search_query = and_(search_query, indexer._get_filter_query(terms))
      search_query = db.session.query(indexer.record_type.key).filter(
          search_query)
      if '__mywork' in request.args:
        search_query = indexer._add_owner_query(
            search_query, models, get_current_user_id())
      search_subquery = search_query.subquery()
      query = query.filter(self.model.id.in_(search_subquery))
    order_properties = []
    if '__sort' in request.args:
      sort_attrs = request.args['__sort'].split(",")
      sort_desc = request.args.get('__sort_desc', False)
      for sort_attr in sort_attrs:
        attr_desc = sort_desc
        if sort_attr.startswith('-'):
          attr_desc = not sort_desc
          sort_attr = sort_attr[1:]
        order_property = getattr(self.model, sort_attr, None)
        if order_property and hasattr(order_property, 'desc'):
          if attr_desc:
            order_property = order_property.desc()
          order_properties.append(order_property)
        else:
          # Possibly throw an exception instead,
          # if sorting by invalid attribute?
          pass
    order_properties.append(self.modified_attr.desc())
    order_properties.append(self.model.id.desc())
    query = query.order_by(*order_properties)
    if '__limit' in request.args:
      try:
        limit = int(request.args['__limit'])
        query = query.limit(limit)
      except (TypeError, ValueError):
        pass
    query = query.distinct()
    return query

  def get_object(self, id):
    # This could also use `self.pk`
    # .one() is required as long as any .eager_load() adds joins using
    #   'contains_eager()' to the core query, because 'LIMIT 1' breaks up
    #   that JOIN result (e.g. Categorizable)
    try:
      return self.get_collection(filter_by_contexts=False)\
          .filter(self.model.id == id).one()
    except sqlalchemy.orm.exc.NoResultFound:
      return None

  def not_found_message(self):
    return '{0} not found.'.format(self.model._inflector.title_singular)

  def not_found_response(self):
    return current_app.make_response((self.not_found_message(), 404, []))

  def collection_last_modified(self):
    """Calculate the last time a member of the collection was modified. This
    method relies on the fact that the collection table has an `updated_at` or
    other column with a relevant timestamp; services for models that don't have
    this field **MUST** override this method.
    """
    result = db.session.query(
        self.modified_attr).order_by(self.modified_attr.desc()).first()
    if result is not None:
      return self.modified_at(result)
    return datetime.datetime.now()

  # Routing helpers
  @classmethod
  def endpoint_name(cls):
    return cls.__name__

  @classmethod
  def url_for_preserving_querystring(cls, *args, **kwargs):
    url = cls.url_for(*args, **kwargs)
    # preserve original query string
    idx = request.url.find('?')
    querystring = '' if idx < 0 else '?' + request.url[idx + 1:]
    return url + querystring

  @classmethod
  def base_url_for(cls, _memoized_base_url={}):
    if cls not in _memoized_base_url:
      _memoized_base_url[cls] = url_for(cls.endpoint_name())
    return _memoized_base_url[cls]

  @classmethod
  def url_for(cls, *args, **kwargs):
    url = cls.base_url_for()
    if len(args) > 0:
      arg = args[0]
      id = arg if not isinstance(arg, db.Model) else arg.id
      url = url + '/' + str(id)
    if 'id' in kwargs:
      url = url + '/' + str(kwargs['id'])
      del kwargs['id']
    if len(kwargs) > 0:
      url = url + '?' + urlencode(kwargs)
    return url

  @classmethod
  def decorate_view_func(cls, view_func, decorators):
    if not isinstance(decorators, (list, tuple)):
      decorators = (decorators,)
    for decorator in reversed(decorators):
      view_func = decorator(view_func)
    return view_func


# View base class for Views handling
#   - /resources (GET, POST)
#   - /resources/<pk:pk_type> (GET, PUT, POST, DELETE)
class Resource(ModelView):
  """View base class for Views handling.  Will typically be registered with an
  application following a collection style for routes. Collection `GET` and
  `POST` will have a route like `/resources` while collection member
  resource routes will have routes likej `/resources/<pk:pk_type>`.

  To register a Resource subclass FooCollection with a Flask application:

  ..

     FooCollection.add_to(app, '/foos')

  By default will only support the `application/json` content-type.
  """

  signals = Namespace()
  model_posted = signals.signal(
      "Model POSTed",
      """
      Indicates that a model object was received via POST and will be committed
      to the database. The sender in the signal will be the model class of the
      POSTed resource. The following arguments will be sent along with the
      signal:

        :obj: The model instance created from the POSTed JSON.
        :src: The original POSTed JSON dictionary.
        :service: The instance of Resource handling the POST request.
      """,)
  collection_posted = signals.signal(
      "Collection POSTed",
      """
      Indicates that a list of models was received via POST and will be
      committed to the database. The sender in the signal will be the model
      class of the POSTed resource. The following arguments will be sent along
      with the signal:

        :objects: The model instance created from the POSTed JSON.
        :src: The original POSTed JSON dictionary.
        :service: The instance of Resource handling the POST request.
      """,)
  model_posted_after_commit = signals.signal(
      "Model POSTed - after",
      """
      Indicates that a model object was received via POST and has been
      committed to the database. The sender in the signal will be the model
      aclass of the POSTed resource. The following arguments will be sent along
      with the signal:

        :obj: The model instance created from the POSTed JSON.
        :src: The original POSTed JSON dictionary.
        :service: The instance of Resource handling the POST request.
      """,)
  model_put = signals.signal(
      "Model PUT",
      """
      Indicates that a model object update was received via PUT and will be
      updated in the database. The sender in the signal will be the model class
      of the PUT resource. The following arguments will be sent along with the
      signal:

        :obj: The model instance updated from the PUT JSON.
        :src: The original PUT JSON dictionary.
        :service: The instance of Resource handling the PUT request.
      """,)
  model_put_after_commit = signals.signal(
      "Model PUT - after",
      """
      Indicates that a model object update was received via PUT and has been
      updated in the database. The sender in the signal will be the model class
      of the PUT resource. The following arguments will be sent along with the
      signal:

        :obj: The model instance updated from the PUT JSON.
        :src: The original PUT JSON dictionary.
        :service: The instance of Resource handling the PUT request.
      """,)
  model_deleted = signals.signal(
      "Model DELETEd",
      """
      Indicates that a model object was DELETEd and will be removed from the
      databse. The sender in the signal will be the model class of the DELETEd
      resource. The followin garguments will be sent along with the signal:

        :obj: The model instance removed.
        :service: The instance of Resource handling the DELETE request.
      """,)
  model_deleted_after_commit = signals.signal(
      "Model DELETEd - after",
      """
      Indicates that a model object was DELETEd and has been removed from the
      database. The sender in the signal will be the model class of the DELETEd
      resource. The followin garguments will be sent along with the signal:

        :obj: The model instance removed.
        :service: The instance of Resource handling the DELETE request.
      """,)

  def dispatch_request(self, *args, **kwargs):  # noqa
    with benchmark("Dispatch request"):
      with benchmark("dispatch_request > Check Headers"):
        method = request.method
        if method in ('POST', 'PUT', 'DELETE')\
           and 'X-Requested-By' not in request.headers:
          raise BadRequest('X-Requested-By header is REQUIRED.')

      with benchmark("dispatch_request > Try"):
        try:
          if method == 'GET':
            if self.pk in kwargs and kwargs[self.pk] is not None:
              return self.get(*args, **kwargs)
            else:
              return self.collection_get()
          elif method == 'POST':
            if self.pk in kwargs and kwargs[self.pk] is not None:
              return self.post(*args, **kwargs)
            else:
              return self.collection_post()
          elif method == 'PUT':
            return self.put(*args, **kwargs)
          elif method == 'DELETE':
            return self.delete(*args, **kwargs)
          else:
            raise NotImplementedError()
        except (IntegrityError, ValidationError, ValueError) as err:
          message = translate_message(err)
          logger.warning(message)
          return (message, 400, [])
        except Exception as err:
          logger.exception(err)
          raise
        finally:
          # When running integration tests, cache sometimes does not clear
          # correctly
          if getattr(settings, 'TESTING', False):
            cache = get_cache()
            if cache:
              cache.clear()

  def post(*args, **kwargs):
    raise NotImplementedError()

  def get(self, id):
    """Default JSON request handlers"""
    with benchmark("Query for object"):
      obj = self.get_object(id)
    if obj is None:
      return self.not_found_response()

    accept_header = self.request.headers.get('Accept', '').strip()
    if (
        accept_header and
        accept_header != '*/*' and
        'application/json' not in accept_header
    ):
      return current_app.make_response((
          'application/json', 406, [('Content-Type', 'text/plain')]))

    with benchmark("Query read permissions"):
      if not permissions.is_allowed_read(
          self.model.__name__, obj.id, obj.context_id)\
         and not permissions.has_conditions('read', self.model.__name__):
        raise Forbidden()
      if not permissions.is_allowed_read_for(obj):
        raise Forbidden()
    with benchmark("Serialize object"):
      object_for_json = self.object_for_json(obj)

    if 'If-None-Match' in self.request.headers and \
       self.request.headers['If-None-Match'] == etag(object_for_json):
      with benchmark("Make response"):
        return current_app.make_response(
            ('', 304, [('Etag', etag(object_for_json))]))
    with benchmark("Make response"):
      return self.json_success_response(
          object_for_json, self.modified_at(obj))

  def validate_headers_for_put_or_delete(self, obj):
    """rfc 6585 defines a new status code for missing required headers"""
    required_headers = set(["If-Match", "If-Unmodified-Since"])
    missing_headers = required_headers.difference(
        set(self.request.headers.keys()))
    if missing_headers:
      return current_app.make_response((
          json.dumps({
              "message": "Missing headers: " + ", ".join(missing_headers),
          }),
          428,
          [("Content-Type", "application/json")],
      ))

    object_etag = etag(self.object_for_json(obj))
    object_timestamp = self.http_timestamp(self.modified_at(obj))
    if (request.headers["If-Match"] != object_etag or
            request.headers["If-Unmodified-Since"] != object_timestamp):
      return current_app.make_response((
          json.dumps({
              "message": "The resource could not be updated due to a conflict "
                         "with the current state on the server. Please "
                         "resolve the conflict by refreshing the resource.",
          }),
          409,
          [("Content-Type", "application/json")]
      ))
    return None

  def json_update(self, obj, src):
    ggrc.builder.json.update(obj, src)

  def put(self, id):
    with benchmark("Query for object"):
      obj = self.get_object(id)
    if obj is None:
      return self.not_found_response()
    src = self.request.json
    with benchmark("Query update permissions"):
      if not permissions.is_allowed_update(
          self.model.__name__, obj.id, obj.context_id)\
         and not permissions.has_conditions('update', self.model.__name__):
        raise Forbidden()
      if not permissions.is_allowed_update_for(obj):
        raise Forbidden()
      new_context = self.get_context_id_from_json(src)
      if new_context != obj.context_id \
         and not permissions.is_allowed_update(
              self.model.__name__, obj.id, new_context)\
         and not permissions.has_conditions('update', self.model.__name__):
        raise Forbidden()
    if self.request.mimetype != 'application/json':
      return current_app.make_response(
          ('Content-Type must be application/json', 415, []))
    header_error = self.validate_headers_for_put_or_delete(obj)
    if header_error:
      return header_error
    root_attribute = self.model._inflector.table_singular
    try:
      src = src[root_attribute]
    except KeyError:
      return current_app.make_response((
          'Required attribute "{0}" not found'.format(
              root_attribute), 400, []))
    with benchmark("Deserialize object"):
      self.json_update(obj, src)
    obj.modified_by_id = get_current_user_id()
    db.session.add(obj)
    with benchmark("Send PUTed event"):
      self.model_put.send(obj.__class__, obj=obj, src=src, service=self)
    with benchmark("Validate custom attributes"):
      if hasattr(obj, "validate_custom_attributes"):
        obj.validate_custom_attributes()
    with benchmark("Get modified objects"):
      modified_objects = get_modified_objects(db.session)
    with benchmark("Update custom attribute values"):
      set_ids_for_new_custom_attributes(obj)
    with benchmark("Log event"):
      log_event(db.session, obj)
    with benchmark("Update memcache before commit for collection PUT"):
      update_memcache_before_commit(
          self.request, modified_objects, CACHE_EXPIRY_COLLECTION)
    with benchmark("Commit"):
      db.session.commit()
    with benchmark("Query for object"):
      obj = self.get_object(id)
    with benchmark("Update index"):
      update_index(db.session, modified_objects)
    with benchmark("Update memcache after commit for collection PUT"):
      update_memcache_after_commit(self.request)
    with benchmark("Send PUT - after commit event"):
      self.model_put_after_commit.send(obj.__class__, obj=obj,
                                       src=src, service=self)
    with benchmark("Serialize collection"):
      object_for_json = self.object_for_json(obj)
    with benchmark("Make response"):
      return self.json_success_response(
          object_for_json, self.modified_at(obj))

  def delete(self, id):
    if 'X-Appengine-Taskname' not in request.headers:
      task = create_task(request.method, request.full_path)
      if getattr(settings, 'APP_ENGINE', False):<commit_msg>Postpone model_put event to after date conversion<commit_after># Copyright (C) 2016 Google Inc.
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>


"""gGRC Collection REST services implementation. Common to all gGRC collection
resources.
"""

import datetime
import hashlib
import itertools
import json
import time
from logging import getLogger
from collections import defaultdict
from exceptions import TypeError
from wsgiref.handlers import format_date_time
from urllib import urlencode

from blinker import Namespace
from flask import url_for, request, current_app, g, has_request_context
from flask.views import View
from flask.ext.sqlalchemy import Pagination
import sqlalchemy.orm.exc
from sqlalchemy import and_, or_
from sqlalchemy.exc import IntegrityError
from sqlalchemy.sql.expression import tuple_
from werkzeug.exceptions import BadRequest, Forbidden

import ggrc.builder.json
import ggrc.models
from ggrc import db, utils
from ggrc.utils import as_json, benchmark
from ggrc.fulltext import get_indexer
from ggrc.fulltext.recordbuilder import fts_record_for
from ggrc.login import get_current_user_id, get_current_user
from ggrc.models.cache import Cache
from ggrc.models.event import Event
from ggrc.models.revision import Revision
from ggrc.models.exceptions import ValidationError, translate_message
from ggrc.rbac import permissions, context_query_filter
from ggrc.services.attribute_query import AttributeQueryBuilder
from ggrc.models.background_task import BackgroundTask, create_task
from ggrc import settings


# pylint: disable=invalid-name
logger = getLogger(__name__)


CACHE_EXPIRY_COLLECTION = 60


def get_oauth_credentials():
  from flask import session
  return session.get('oauth_credentials')


def _get_cache_manager():
  from ggrc.cache import CacheManager, MemCache
  cache_manager = CacheManager()
  cache_manager.initialize(MemCache())
  return cache_manager


def get_cache_key(obj, type=None, id=None):
  """Returns a string identifier for the specified object or stub.

  `obj` can be:
    <db.Model> -- declarative model instance
    (type, id) -- tuple
    { 'type': type, 'id': id } -- dict
  """
  if isinstance(obj, tuple):
    type, id = obj
  elif isinstance(obj, dict):
    type = obj.get('type', None)
    id = obj.get('id', None)
  if isinstance(type, (str, unicode)):
    model = ggrc.models.get_model(type)
    assert model is not None, "Invalid model name: {}".format(type)
    type = ggrc.models.get_model(type)._inflector.table_plural
  if not isinstance(obj, (tuple, dict)):
    if type is None:
      type = obj._inflector.table_plural
    if id is None:
      id = obj.id
  return 'collection:{type}:{id}'.format(type=type, id=id)


def get_cache_class(obj):
  return obj.__class__.__name__


def get_related_keys_for_expiration(context, o):
  cls = get_cache_class(o)
  keys = []
  mappings = context.cache_manager.supported_mappings.get(cls, [])
  if len(mappings) > 0:
    for (cls, attr, polymorph) in mappings:
      if polymorph:
        key = get_cache_key(
            None,
            type=getattr(o, '{0}_type'.format(attr)),
            id=getattr(o, '{0}_id'.format(attr)))
        keys.append(key)
      else:
        obj = getattr(o, attr, None)
        if obj:
          if isinstance(obj, list):
            for inner_o in obj:
              key = get_cache_key(inner_o)
              keys.append(key)
          else:
            key = get_cache_key(obj)
            keys.append(key)
  return keys


def set_ids_for_new_custom_attributes(parent_obj):
  """
  When we are creating custom attribute values and definitions for
  POST requests, parent object ID is not yet defined. This is why we update
  custom attribute values at this point and set the correct attributable_id

  Args:
    parent_obj: parent object to be set as attributable

  Returns:
    None
  """
  if not hasattr(parent_obj, "PER_OBJECT_CUSTOM_ATTRIBUTABLE"):
    return

  modified_objects = get_modified_objects(db.session).new

  object_attrs = {
      "CustomAttributeValue": "attributable_id",
      "CustomAttributeDefinition": "definition_id"
  }

  for obj in modified_objects:
    if obj.type not in object_attrs:
      continue

    attr = object_attrs[obj.type]
    setattr(obj, attr, parent_obj.id)

    # Disable state updating so that a newly create object doesn't go straight
    # from Draft to Modified.
    if hasattr(obj, '_skip_os_state_update'):
      obj.skip_os_state_update()
    db.session.add(obj)
  db.session.flush()


def memcache_mark_for_deletion(context, objects_to_mark):
  """
  Mark objects for deletion from memcache

  Args:
    context: application context
    objects_to_mark: A list of objects to be deleted from memcache

  Returns:
    None
  """
  for o, _ in objects_to_mark:
    cls = get_cache_class(o)
    if cls in context.cache_manager.supported_classes:
      key = get_cache_key(o)
      context.cache_manager.marked_for_delete.append(key)
      context.cache_manager.marked_for_delete.extend(
          get_related_keys_for_expiration(context, o))


def update_memcache_before_commit(context, modified_objects, expiry_time):
  """
  Preparing the memccache entries to be updated before DB commit
  Also update the memcache to indicate the status cache operation
  'InProgress' waiting for DB commit
  Raises Exception on failures, cannot proceed with DB commit

  Args:
    context: POST/PUT/DELETE HTTP request or import Converter contextual object
    modified_objects:  objects in cache maintained prior to committing to DB
    expiry_time: Expiry time specified for memcache ADD and DELETE
  Returns:
    None

  """
  if getattr(settings, 'MEMCACHE_MECHANISM', False) is False:
    return

  context.cache_manager = _get_cache_manager()

  if modified_objects is not None:
    if len(modified_objects.new) > 0:
      memcache_mark_for_deletion(context, modified_objects.new.items())

    if len(modified_objects.dirty) > 0:
      memcache_mark_for_deletion(context, modified_objects.dirty.items())

    if len(modified_objects.deleted) > 0:
      memcache_mark_for_deletion(context, modified_objects.deleted.items())

  status_entries = {}
  for key in context.cache_manager.marked_for_delete:
    build_cache_status(status_entries, 'DeleteOp:' + key,
                       expiry_time, 'InProgress')
  if len(status_entries) > 0:
    logger.info("CACHE: status entries: %s", status_entries)
    ret = context.cache_manager.bulk_add(status_entries, expiry_time)
    if ret is not None and len(ret) == 0:
      pass
    else:
      logger.error('CACHE: Unable to add status for newly created entries %s',
                   ret)


def update_memcache_after_commit(context):
  """
  The memccache entries is updated after DB commit
  Logs error if there are errors in updating entries in cache

  Args:
    context: POST/PUT/DELETE HTTP request or import Converter contextual object
    modified_objects:  objects in cache maintained prior to committing to DB
  Returns:
    None

  """
  if getattr(settings, 'MEMCACHE_MECHANISM', False) is False:
    return

  if context.cache_manager is None:
    logger.error("CACHE: Error in initiaizing cache manager")
    return

  cache_manager = context.cache_manager

  related_objs = list()
  for val in getattr(g, "referenced_objects", {}).itervalues():
    obj_list = val.values()
    if obj_list:
      related_objs.append((obj_list[0], None))
  memcache_mark_for_deletion(context, related_objs)

  # TODO(dan): check for duplicates in marked_for_delete
  if len(cache_manager.marked_for_delete) > 0:
    delete_result = cache_manager.bulk_delete(
        cache_manager.marked_for_delete, 0)
    # TODO(dan): handling failure including network errors,
    #            currently we log errors
    if delete_result is not True:
      logger.error("CACHE: Failed to remove collection from cache")

  status_entries = []
  for key in cache_manager.marked_for_delete:
    status_entries.append('DeleteOp:' + str(key))
  if len(status_entries) > 0:
    delete_result = cache_manager.bulk_delete(status_entries, 0)
    # TODO(dan): handling failure including network errors,
    #            currently we log errors
    if delete_result is not True:
      logger.error("CACHE: Failed to remove status entries from cache")

  clear_permission_cache()
  cache_manager.clear_cache()


def build_cache_status(data, key, expiry_timeout, status):
  """
  Build the dictionary for storing operational status of cache

  Args:
    data: dictionary to update
    key: key to dictionary
    expiry_timeout: timeout for expiry cache
    status: Update status entry, e.g.InProgress
  Returns:
    None
  """
  data[key] = {'expiry': expiry_timeout, 'status': status}


def inclusion_filter(obj):
  return permissions.is_allowed_read(obj.__class__.__name__,
                                     obj.id, obj.context_id)


def get_cache(create=False):
  """
  Retrieves the cache from the Flask global object. The create arg
  indicates if a new cache should be created if none exists. If we
  are not in a request context, no cache is created (return None).
  """
  if has_request_context():
    cache = getattr(g, 'cache', None)
    if cache is None and create:
      cache = g.cache = Cache()
    return cache
  else:
    logger.warning("No request context - no cache created")
    return None


def get_modified_objects(session):
  session.flush()
  cache = get_cache()
  if cache:
    return cache.copy()
  else:
    return None


def update_index(session, cache):
  if cache:
    indexer = get_indexer()
    for obj in cache.new:
      indexer.create_record(fts_record_for(obj), commit=False)
    for obj in cache.dirty:
      indexer.update_record(fts_record_for(obj), commit=False)
    for obj in cache.deleted:
      indexer.delete_record(obj.id, obj.__class__.__name__, commit=False)
    session.commit()


def log_event(session, obj=None, current_user_id=None, flush=True):
  revisions = []
  if flush:
    session.flush()
  if current_user_id is None:
    current_user_id = get_current_user_id()
  cache = get_cache()
  if cache:
    for obj_ in cache.dirty:
      revision = Revision(obj_, current_user_id, 'modified', obj_.log_json())
      revisions.append(revision)
    for obj_ in cache.deleted:
      revision = Revision(obj_, current_user_id, 'deleted', obj_.log_json())
      revisions.append(revision)
    for obj_ in cache.new:
      revision = Revision(obj_, current_user_id, 'created', obj_.log_json())
      revisions.append(revision)
  if obj is None:
    resource_id = 0
    resource_type = None
    action = 'BULK'
    context_id = 0
  else:
    resource_id = obj.id
    resource_type = str(obj.__class__.__name__)
    action = request.method
    context_id = obj.context_id
  if revisions:
    event = Event(
        modified_by_id=current_user_id,
        action=action,
        resource_id=resource_id,
        resource_type=resource_type,
        context_id=context_id)
    event.revisions = revisions
    session.add(event)


def clear_permission_cache():
  if not getattr(settings, 'MEMCACHE_MECHANISM', False):
    return
  cache = _get_cache_manager().cache_object.memcache_client
  cached_keys_set = cache.get('permissions:list') or set()
  cached_keys_set.add('permissions:list')
  # We delete all the cached user permissions as well as
  # the permissions:list value itself
  cache.delete_multi(cached_keys_set)


class ModelView(View):
  """Basic view handler for all models"""
  # pylint: disable=protected-access
  # access to _sa_class_manager is needed for fetching the right mapper
  DEFAULT_PAGE_SIZE = 20
  MAX_PAGE_SIZE = 100
  pk = 'id'
  pk_type = 'int'

  _model = None

  # Simple accessor properties
  @property
  def request(self):
    return request

  @property
  def model(self):
    return self._model

  @property
  def modified_attr_name(self):
    return 'updated_at'

  @property
  def modified_attr(self):
    """Return the model attribute to be used for Last-Modified header and
    sorting collection elements.
    """
    return getattr(self.model, self.modified_attr_name)

  def modified_at(self, obj):
    return getattr(obj, self.modified_attr_name)

  def _get_type_select_column(self, model):
    mapper = model._sa_class_manager.mapper
    if mapper.polymorphic_on is None:
      # if len(mapper.self_and_descendants) == 1:
      type_column = sqlalchemy.literal(mapper.class_.__name__)
    else:
      # Handle polymorphic types with CASE
      type_column = sqlalchemy.case(
          value=mapper.polymorphic_on,
          whens={
              val: m.class_.__name__
              for val, m in mapper.polymorphic_map.items()
          })
    return type_column

  def _get_type_where_clause(self, model):
    mapper = model._sa_class_manager.mapper
    if mapper.polymorphic_on is None:
      return True
    else:
      mappers = list(mapper.self_and_descendants)
      polymorphic_on_values = list(
          val
          for val, m in mapper.polymorphic_map.items()
          if m in mappers)
      return mapper.polymorphic_on.in_(polymorphic_on_values)

  def _get_matching_types(self, model):
    mapper = model._sa_class_manager.mapper
    if len(list(mapper.self_and_descendants)) == 1:
      return mapper.class_.__name__
    else:
      # FIXME: Actually needs to use 'self_and_descendants'
      return [m.class_.__name__ for m in mapper.self_and_descendants]

  def get_match_columns(self, model):
    mapper = model._sa_class_manager.mapper
    columns = []
    columns.append(mapper.primary_key[0].label('id'))
    # columns.append(model.id.label('id'))
    columns.append(self._get_type_select_column(model).label('type'))
    if hasattr(mapper.c, 'context_id'):
      columns.append(mapper.c.context_id.label('context_id'))
    if hasattr(mapper.c, 'updated_at'):
      columns.append(mapper.c.updated_at.label('updated_at'))
    # columns.append(self._get_polymorphic_column(model))
    return columns

  def get_collection_matches(self, model, filter_by_contexts=True):
    columns = self.get_match_columns(self.model)
    query = db.session.query(*columns).filter(
        self._get_type_where_clause(model))
    return self.filter_query_by_request(
        query, filter_by_contexts=filter_by_contexts)

  def get_resource_match_query(self, model, id):
    columns = self.get_match_columns(model)
    query = db.session.query(*columns).filter(
        and_(
            self._get_type_where_clause(model),
            columns[0] == id))
    return query

  # Default model/DB helpers
  def get_collection(self, filter_by_contexts=True):
    if '__stubs_only' not in request.args and \
       hasattr(self.model, 'eager_query'):
      query = self.model.eager_query()
    else:
      query = db.session.query(self.model)
    return self.filter_query_by_request(
        query, filter_by_contexts=filter_by_contexts)

  def filter_query_by_request(self, query, filter_by_contexts=True):  # noqa
    joinlist = []
    if request.args:
      querybuilder = AttributeQueryBuilder(self.model)
      filter_, joinlist, _ = querybuilder.collection_filters(request.args)
      if filter_ is not None:
        for j in joinlist:
          query = query.join(j)
        query = query.filter(filter_)
    if filter_by_contexts:
      contexts = permissions.read_contexts_for(self.model.__name__)
      resources = permissions.read_resources_for(self.model.__name__)
      filter_expr = context_query_filter(self.model.context_id, contexts)
      if resources:
        filter_expr = or_(filter_expr, self.model.id.in_(resources))
      query = query.filter(filter_expr)
      for j in joinlist:
        j_class = j.property.mapper.class_
        j_contexts = permissions.read_contexts_for(j_class.__name__)
        j_resources = permissions.read_resources_for(j_class.__name__)
        if j_contexts is not None:
          j_filter_expr = context_query_filter(j_class.context_id, j_contexts)
          if resources:
            j_filter_expr = or_(j_filter_expr, self.model.id.in_(j_resources))
          query = query.filter(j_filter_expr)
        elif resources:
          query = query.filter(self.model.id.in_(resources))
    if '__search' in request.args:
      terms = request.args['__search']
      types = self._get_matching_types(self.model)
      indexer = get_indexer()
      models = indexer._get_grouped_types(types)
      search_query = indexer.get_permissions_query(models, 'read', None)
      search_query = and_(search_query, indexer._get_filter_query(terms))
      search_query = db.session.query(indexer.record_type.key).filter(
          search_query)
      if '__mywork' in request.args:
        search_query = indexer._add_owner_query(
            search_query, models, get_current_user_id())
      search_subquery = search_query.subquery()
      query = query.filter(self.model.id.in_(search_subquery))
    order_properties = []
    if '__sort' in request.args:
      sort_attrs = request.args['__sort'].split(",")
      sort_desc = request.args.get('__sort_desc', False)
      for sort_attr in sort_attrs:
        attr_desc = sort_desc
        if sort_attr.startswith('-'):
          attr_desc = not sort_desc
          sort_attr = sort_attr[1:]
        order_property = getattr(self.model, sort_attr, None)
        if order_property and hasattr(order_property, 'desc'):
          if attr_desc:
            order_property = order_property.desc()
          order_properties.append(order_property)
        else:
          # Possibly throw an exception instead,
          # if sorting by invalid attribute?
          pass
    order_properties.append(self.modified_attr.desc())
    order_properties.append(self.model.id.desc())
    query = query.order_by(*order_properties)
    if '__limit' in request.args:
      try:
        limit = int(request.args['__limit'])
        query = query.limit(limit)
      except (TypeError, ValueError):
        pass
    query = query.distinct()
    return query

  def get_object(self, id):
    # This could also use `self.pk`
    # .one() is required as long as any .eager_load() adds joins using
    #   'contains_eager()' to the core query, because 'LIMIT 1' breaks up
    #   that JOIN result (e.g. Categorizable)
    try:
      return self.get_collection(filter_by_contexts=False)\
          .filter(self.model.id == id).one()
    except sqlalchemy.orm.exc.NoResultFound:
      return None

  def not_found_message(self):
    return '{0} not found.'.format(self.model._inflector.title_singular)

  def not_found_response(self):
    return current_app.make_response((self.not_found_message(), 404, []))

  def collection_last_modified(self):
    """Calculate the last time a member of the collection was modified. This
    method relies on the fact that the collection table has an `updated_at` or
    other column with a relevant timestamp; services for models that don't have
    this field **MUST** override this method.
    """
    result = db.session.query(
        self.modified_attr).order_by(self.modified_attr.desc()).first()
    if result is not None:
      return self.modified_at(result)
    return datetime.datetime.now()

  # Routing helpers
  @classmethod
  def endpoint_name(cls):
    return cls.__name__

  @classmethod
  def url_for_preserving_querystring(cls, *args, **kwargs):
    url = cls.url_for(*args, **kwargs)
    # preserve original query string
    idx = request.url.find('?')
    querystring = '' if idx < 0 else '?' + request.url[idx + 1:]
    return url + querystring

  @classmethod
  def base_url_for(cls, _memoized_base_url={}):
    if cls not in _memoized_base_url:
      _memoized_base_url[cls] = url_for(cls.endpoint_name())
    return _memoized_base_url[cls]

  @classmethod
  def url_for(cls, *args, **kwargs):
    url = cls.base_url_for()
    if len(args) > 0:
      arg = args[0]
      id = arg if not isinstance(arg, db.Model) else arg.id
      url = url + '/' + str(id)
    if 'id' in kwargs:
      url = url + '/' + str(kwargs['id'])
      del kwargs['id']
    if len(kwargs) > 0:
      url = url + '?' + urlencode(kwargs)
    return url

  @classmethod
  def decorate_view_func(cls, view_func, decorators):
    if not isinstance(decorators, (list, tuple)):
      decorators = (decorators,)
    for decorator in reversed(decorators):
      view_func = decorator(view_func)
    return view_func


# View base class for Views handling
#   - /resources (GET, POST)
#   - /resources/<pk:pk_type> (GET, PUT, POST, DELETE)
class Resource(ModelView):
  """View base class for Views handling.  Will typically be registered with an
  application following a collection style for routes. Collection `GET` and
  `POST` will have a route like `/resources` while collection member
  resource routes will have routes likej `/resources/<pk:pk_type>`.

  To register a Resource subclass FooCollection with a Flask application:

  ..

     FooCollection.add_to(app, '/foos')

  By default will only support the `application/json` content-type.
  """

  signals = Namespace()
  model_posted = signals.signal(
      "Model POSTed",
      """
      Indicates that a model object was received via POST and will be committed
      to the database. The sender in the signal will be the model class of the
      POSTed resource. The following arguments will be sent along with the
      signal:

        :obj: The model instance created from the POSTed JSON.
        :src: The original POSTed JSON dictionary.
        :service: The instance of Resource handling the POST request.
      """,)
  collection_posted = signals.signal(
      "Collection POSTed",
      """
      Indicates that a list of models was received via POST and will be
      committed to the database. The sender in the signal will be the model
      class of the POSTed resource. The following arguments will be sent along
      with the signal:

        :objects: The model instance created from the POSTed JSON.
        :src: The original POSTed JSON dictionary.
        :service: The instance of Resource handling the POST request.
      """,)
  model_posted_after_commit = signals.signal(
      "Model POSTed - after",
      """
      Indicates that a model object was received via POST and has been
      committed to the database. The sender in the signal will be the model
      aclass of the POSTed resource. The following arguments will be sent along
      with the signal:

        :obj: The model instance created from the POSTed JSON.
        :src: The original POSTed JSON dictionary.
        :service: The instance of Resource handling the POST request.
      """,)
  model_put = signals.signal(
      "Model PUT",
      """
      Indicates that a model object update was received via PUT and will be
      updated in the database. The sender in the signal will be the model class
      of the PUT resource. The following arguments will be sent along with the
      signal:

        :obj: The model instance updated from the PUT JSON.
        :src: The original PUT JSON dictionary.
        :service: The instance of Resource handling the PUT request.
      """,)
  model_put_after_commit = signals.signal(
      "Model PUT - after",
      """
      Indicates that a model object update was received via PUT and has been
      updated in the database. The sender in the signal will be the model class
      of the PUT resource. The following arguments will be sent along with the
      signal:

        :obj: The model instance updated from the PUT JSON.
        :src: The original PUT JSON dictionary.
        :service: The instance of Resource handling the PUT request.
      """,)
  model_deleted = signals.signal(
      "Model DELETEd",
      """
      Indicates that a model object was DELETEd and will be removed from the
      databse. The sender in the signal will be the model class of the DELETEd
      resource. The followin garguments will be sent along with the signal:

        :obj: The model instance removed.
        :service: The instance of Resource handling the DELETE request.
      """,)
  model_deleted_after_commit = signals.signal(
      "Model DELETEd - after",
      """
      Indicates that a model object was DELETEd and has been removed from the
      database. The sender in the signal will be the model class of the DELETEd
      resource. The followin garguments will be sent along with the signal:

        :obj: The model instance removed.
        :service: The instance of Resource handling the DELETE request.
      """,)

  def dispatch_request(self, *args, **kwargs):  # noqa
    with benchmark("Dispatch request"):
      with benchmark("dispatch_request > Check Headers"):
        method = request.method
        if method in ('POST', 'PUT', 'DELETE')\
           and 'X-Requested-By' not in request.headers:
          raise BadRequest('X-Requested-By header is REQUIRED.')

      with benchmark("dispatch_request > Try"):
        try:
          if method == 'GET':
            if self.pk in kwargs and kwargs[self.pk] is not None:
              return self.get(*args, **kwargs)
            else:
              return self.collection_get()
          elif method == 'POST':
            if self.pk in kwargs and kwargs[self.pk] is not None:
              return self.post(*args, **kwargs)
            else:
              return self.collection_post()
          elif method == 'PUT':
            return self.put(*args, **kwargs)
          elif method == 'DELETE':
            return self.delete(*args, **kwargs)
          else:
            raise NotImplementedError()
        except (IntegrityError, ValidationError, ValueError) as err:
          message = translate_message(err)
          logger.warning(message)
          return (message, 400, [])
        except Exception as err:
          logger.exception(err)
          raise
        finally:
          # When running integration tests, cache sometimes does not clear
          # correctly
          if getattr(settings, 'TESTING', False):
            cache = get_cache()
            if cache:
              cache.clear()

  def post(*args, **kwargs):
    raise NotImplementedError()

  def get(self, id):
    """Default JSON request handlers"""
    with benchmark("Query for object"):
      obj = self.get_object(id)
    if obj is None:
      return self.not_found_response()

    accept_header = self.request.headers.get('Accept', '').strip()
    if (
        accept_header and
        accept_header != '*/*' and
        'application/json' not in accept_header
    ):
      return current_app.make_response((
          'application/json', 406, [('Content-Type', 'text/plain')]))

    with benchmark("Query read permissions"):
      if not permissions.is_allowed_read(
          self.model.__name__, obj.id, obj.context_id)\
         and not permissions.has_conditions('read', self.model.__name__):
        raise Forbidden()
      if not permissions.is_allowed_read_for(obj):
        raise Forbidden()
    with benchmark("Serialize object"):
      object_for_json = self.object_for_json(obj)

    if 'If-None-Match' in self.request.headers and \
       self.request.headers['If-None-Match'] == etag(object_for_json):
      with benchmark("Make response"):
        return current_app.make_response(
            ('', 304, [('Etag', etag(object_for_json))]))
    with benchmark("Make response"):
      return self.json_success_response(
          object_for_json, self.modified_at(obj))

  def validate_headers_for_put_or_delete(self, obj):
    """rfc 6585 defines a new status code for missing required headers"""
    required_headers = set(["If-Match", "If-Unmodified-Since"])
    missing_headers = required_headers.difference(
        set(self.request.headers.keys()))
    if missing_headers:
      return current_app.make_response((
          json.dumps({
              "message": "Missing headers: " + ", ".join(missing_headers),
          }),
          428,
          [("Content-Type", "application/json")],
      ))

    object_etag = etag(self.object_for_json(obj))
    object_timestamp = self.http_timestamp(self.modified_at(obj))
    if (request.headers["If-Match"] != object_etag or
            request.headers["If-Unmodified-Since"] != object_timestamp):
      return current_app.make_response((
          json.dumps({
              "message": "The resource could not be updated due to a conflict "
                         "with the current state on the server. Please "
                         "resolve the conflict by refreshing the resource.",
          }),
          409,
          [("Content-Type", "application/json")]
      ))
    return None

  def json_update(self, obj, src):
    ggrc.builder.json.update(obj, src)

  def put(self, id):
    with benchmark("Query for object"):
      obj = self.get_object(id)
    if obj is None:
      return self.not_found_response()
    src = self.request.json
    with benchmark("Query update permissions"):
      if not permissions.is_allowed_update(
          self.model.__name__, obj.id, obj.context_id)\
         and not permissions.has_conditions('update', self.model.__name__):
        raise Forbidden()
      if not permissions.is_allowed_update_for(obj):
        raise Forbidden()
      new_context = self.get_context_id_from_json(src)
      if new_context != obj.context_id \
         and not permissions.is_allowed_update(
              self.model.__name__, obj.id, new_context)\
         and not permissions.has_conditions('update', self.model.__name__):
        raise Forbidden()
    if self.request.mimetype != 'application/json':
      return current_app.make_response(
          ('Content-Type must be application/json', 415, []))
    header_error = self.validate_headers_for_put_or_delete(obj)
    if header_error:
      return header_error
    root_attribute = self.model._inflector.table_singular
    try:
      src = src[root_attribute]
    except KeyError:
      return current_app.make_response((
          'Required attribute "{0}" not found'.format(
              root_attribute), 400, []))
    with benchmark("Deserialize object"):
      self.json_update(obj, src)
    obj.modified_by_id = get_current_user_id()
    db.session.add(obj)
    with benchmark("Validate custom attributes"):
      if hasattr(obj, "validate_custom_attributes"):
        obj.validate_custom_attributes()
    with benchmark("Send PUT event"):
      self.model_put.send(obj.__class__, obj=obj, src=src, service=self)
    with benchmark("Get modified objects"):
      modified_objects = get_modified_objects(db.session)
    with benchmark("Update custom attribute values"):
      set_ids_for_new_custom_attributes(obj)
    with benchmark("Log event"):
      log_event(db.session, obj)
    with benchmark("Update memcache before commit for collection PUT"):
      update_memcache_before_commit(
          self.request, modified_objects, CACHE_EXPIRY_COLLECTION)
    with benchmark("Commit"):
      db.session.commit()
    with benchmark("Query for object"):
      obj = self.get_object(id)
    with benchmark("Update index"):
      update_index(db.session, modified_objects)
    with benchmark("Update memcache after commit for collection PUT"):
      update_memcache_after_commit(self.request)
    with benchmark("Send PUT - after commit event"):
      self.model_put_after_commit.send(obj.__class__, obj=obj,
                                       src=src, service=self)
    with benchmark("Serialize collection"):
      object_for_json = self.object_for_json(obj)
    with benchmark("Make response"):
      return self.json_success_response(
          object_for_json, self.modified_at(obj))

  def delete(self, id):
    if 'X-Appengine-Taskname' not in request.headers:
      task = create_task(request.method, request.full_path)
      if getattr(settings, 'APP_ENGINE', False):
{'fraction_non_alphanumeric': 0.09117563605410006, 'fraction_numerical': 0.0021753523124940885, 'mean_word_length': 3.1266506212190204, 'pattern_counts': {'":': 8, '<': 13, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1179283', 'n_tokens_mistral': 19132, 'n_tokens_neox': 18608, 'n_words': 5319}