<commit_before>"""Provide base class for collection views and utilities."""
# pylint: disable=too-many-lines; It's mostly docstrings
import functools
import itertools
import logging
import re
from collections import Sequence

from pyramid.httpexceptions import (
    HTTPNotFound,
    HTTPForbidden,
    HTTPBadRequest,
    HTTPConflict,
    HTTPUnsupportedMediaType,
    HTTPNotAcceptable,
    HTTPError,
    HTTPFailedDependency,
    HTTPInternalServerError,
    status_map,
)
import pyramid_jsonapi.jsonapi
import sqlalchemy
from sqlalchemy.ext.associationproxy import AssociationProxy
from sqlalchemy.orm import load_only
from sqlalchemy.orm.relationships import RelationshipProperty
from sqlalchemy.orm.exc import NoResultFound

ONETOMANY = sqlalchemy.orm.interfaces.ONETOMANY
MANYTOMANY = sqlalchemy.orm.interfaces.MANYTOMANY
MANYTOONE = sqlalchemy.orm.interfaces.MANYTOONE


class CollectionViewBase:
    """Base class for all view classes.

    Arguments:
        request (pyramid.request): passed by framework.
    """

    # pylint:disable=too-many-public-methods

    # Define class attributes
    # Callable attributes use lambda to keep pylint happy
    api = None
    attributes = None
    callbacks = None
    collection_name = None
    default_limit = None
    exposed_fields = None
    fields = None
    dbsession = None
    hybrid_attributes = None
    item = None
    key_column = None
    max_limit = None
    model = lambda: None
    obj_id = None
    request = None
    rel = None
    rel_class = None
    rel_view = None
    relationships = None
    relname = None
    view_classes = None
    settings = None

    def __init__(self, request):
        self.request = request
        if self.api.get_dbsession:
            self.dbsession = self.api.get_dbsession(self)
        else:
            self.dbsession = self.request.dbsession
        self.views = {}

    @staticmethod
    def id_col(item):
        """Return the column holding an item's id."""
        return getattr(item, item.__pyramid_jsonapi__['id_col_name'])

    def jsonapi_view(func):  # pylint: disable=no-self-argument
        """Decorator for view functions. Adds jsonapi boilerplate,
        and tests response validity."""

        def view_exceptions(func):
            """Decorator to intercept all exceptions raised by wrapped view methods.

            If the exception is 'valid' according to the schema, raise it.
            Else raise a generic 4xx or 5xx error and log the real one.
            """
            @functools.wraps(func)
            def new_func(self):  # pylint: disable=missing-docstring
                ep_dict = self.api.endpoint_data.endpoints
                # Get route_name from route
                _, _, endpoint = self.request.matched_route.name.split(':')
                method = self.request.method
                responses = set(
                    ep_dict['responses'].keys() |
                    ep_dict['endpoints'][endpoint]['responses'].keys() |
                    ep_dict['endpoints'][endpoint]['http_methods'][method]['responses'].keys()
                )
                try:
                    result = func(self)  # pylint: disable=not-callable
                    response_class = status_map[self.request.response.status_code]
                    if response_class not in responses:
                        logging.error(
                            "Invalid response: %s for route_name: %s path: %s",
                            response_class,
                            self.request.matched_route.name,
                            self.request.current_route_path()
                        )
                    return result
                except Exception as exc:
                    if exc.__class__ not in responses:
                        logging.exception(
                            "Invalid exception raised: %s for route_name: %s path: %s",
                            exc.__class__,
                            self.request.matched_route.name,
                            self.request.current_route_path()
                        )
                        if hasattr(exc, 'code'):
                            if 400 <= exc.code < 500:  # pylint:disable=no-member
                                raise HTTPBadRequest("Unexpected client error: {}".format(exc))
                        else:
                            raise HTTPInternalServerError("Unexpected server error.")
                    raise
            return new_func

        @functools.lru_cache()
        def get_jsonapi_accepts(request):
            """Return a set of all 'application/vnd.api' parts of the accept
            header.
            """
            accepts = re.split(
                r',\s*',
                request.headers.get('accept', '')
            )
            return {
                a for a in accepts
                if a.startswith('application/vnd.api')
            }

        def check_request_headers(request, jsonapi_accepts):
            """Check that request headers comply with spec.

            Raises:
                HTTPUnsupportedMediaType
                HTTPNotAcceptable
            """
            # Spec says to reject (with 415) any request with media type
            # params.
            if len(request.headers.get('content-type', '').split(';')) > 1:
                raise HTTPUnsupportedMediaType(
                    'Media Type parameters not allowed by JSONAPI ' +
                    'spec (http://jsonapi.org/format).'
                )
            # Spec says throw 406 Not Acceptable if Accept header has no
            # application/vnd.api+json entry without parameters.
            if jsonapi_accepts and\
                    'application/vnd.api+json' not in jsonapi_accepts:
                raise HTTPNotAcceptable(
                    'application/vnd.api+json must appear with no ' +
                    'parameters in Accepts header ' +
                    '(http://jsonapi.org/format).'
                )

        def check_request_valid_json(request):
            """Check that the body of any request is valid JSON.

            Raises:
                HTTPBadRequest
            """
            if request.content_length:
                try:
                    request.json_body
                except ValueError:
                    raise HTTPBadRequest("Body is not valid JSON.")

        @view_exceptions
        @functools.wraps(func)
        def view_wrapper(self):
            """jsonapi boilerplate function to wrap decorated functions."""
            check_request_headers(self.request, get_jsonapi_accepts(self.request))
            check_request_valid_json(self.request)

            if self.request.content_length and self.api.settings.schema_validation:
                # Validate request JSON against the JSONAPI jsonschema
                self.api.metadata.JSONSchema.validate(self.request.json_body, self.request.method)

            # Spec says throw BadRequest if any include paths reference non
            # existent attributes or relationships.
            if self.bad_include_paths:
                raise HTTPBadRequest(
                    "Bad include paths {}".format(
                        self.bad_include_paths
                    )
                )

            # Spec says set Content-Type to application/vnd.api+json.
            self.request.response.content_type = 'application/vnd.api+json'

            # Extract id and relationship from route, if provided
            self.obj_id = self.request.matchdict.get('id', None)
            self.relname = self.request.matchdict.get('relationship', None)

            if self.obj_id:
                # Try to get the object
                try:
                    self.item = self.single_return(
                        self.single_item_query(),
                        'No id {} in collection {}'.format(
                            self.obj_id,
                            self.collection_name
                        )
                    )
                except (sqlalchemy.exc.DataError, sqlalchemy.exc.StatementError):
                    # DataError is caused by e.g. id (int) = cat
                    # StatementError is caused by e.g. id (uuid) = 1
                    raise HTTPNotFound('Object {} not found in collection {}'.format(
                        self.obj_id,
                        self.collection_name
                    ))

            if self.relname:
                # Gather relationship info
                mapper = sqlalchemy.inspect(self.model).mapper
                try:
                    self.rel = self.relationships[self.relname]
                except KeyError:
                    raise HTTPNotFound('No relationship {} in collection {}'.format(
                        self.relname,
                        self.collection_name
                    ))
                if isinstance(self.rel, AssociationProxy):
                    proxy_state = self.rel.for_class(self.model)
                    self.rel_class = getattr(proxy_state.target_class, proxy_state.value_attr).mapper.class_
                else:
                    self.rel_class = self.rel.mapper.class_
                self.rel_view = self.view_instance(self.rel_class)

            # Update the dictionary with the reults of the wrapped method.
            ret = func(self)  # pylint:disable=not-callable
            if ret:
                # Include a self link unless the method is PATCH.
                if self.request.method != 'PATCH':
                    selfie = {'self': self.request.url}
                    if hasattr(ret, 'links'):
                        ret.links.update(selfie)
                    else:
                        ret.links = selfie

                # Potentially add some debug information.
                if self.api.settings.debug_meta:
                    debug = {
                        'accept_header': {
                            a: None for a in get_jsonapi_accepts(self.request)
                        },
                        'qinfo_page':
                            self.collection_query_info(self.request)['_page'],
                        'atts': {k: None for k in self.attributes.keys()},
                        'includes': {
                            k: None for k in self.requested_include_names()
                        }
                    }
                    ret.meta.update({'debug': debug})
                return ret.as_dict()
            else:
                return {}
        return view_wrapper

    @jsonapi_view
    def get(self):
        """Handle GET request for a single item.

        Get a single item from the collection, referenced by id.

        **URL (matchdict) Parameters**

            **id** (*str*): resource id

        Returns:
            jsonapi.Document: in the form:

            .. parsed-literal::

                {
                    "data": { resource object },
                    "links": {
                        "self": self url,
                        maybe other links...
                    },
                    "meta": { jsonapi specific information }
                }

        Raises:
            HTTPNotFound

        Example:

            Get person 1:

            .. parsed-literal::

                http GET http://localhost:6543/people/1
        """
        for callback in self.callbacks['after_get']:
            self.item = callback(self, self.item)
        return self.item

    @jsonapi_view
    def patch(self):
        """Handle PATCH request for a single item.

        Update an existing item from a partially defined representation.

        **URL (matchdict) Parameters**

            **id** (*str*): resource id

        **Request Body**

            **Partial resource object** (*json*)

        Returns:
            jsonapi.Document: in the form:

            .. parsed-literal::

                {
                    'meta': {
                        'updated': [
                            <attribute_name>,
                            <attribute_name>
                        ]
                    }
                }

        Raises:
            HTTPNotFound

        Todo:
            Currently does not deal with relationships.

        Example:
            PATCH person 1, changing name to alicia:

            .. parsed-literal::

                http PATCH http://localhost:6543/people/1 data:='
                {
                    "type":"people", "id": "1",
                    "attributes": {
                        "name": "alicia"
                    }
                }' Content-Type:application/vnd.api+json

            Change the author of posts/1 to people/2:

            .. parsed-literal::

                http PATCH http://localhost:6543/posts/1 data:='
                {
                    "type":"posts", "id": "1",
                    "relationships": {
                        "author": {"type": "people", "id": "2"}
                    }
                }' Content-Type:application/vnd.api+json

            Set the comments on posts/1 to be [comments/4, comments/5]:

            .. parsed-literal::

                http PATCH http://localhost:6543/posts/1 data:='
                {
                    "type":"posts", "id": "1",
                    "relationships": {
                        "comments": [
                            {"type": "comments", "id": "4"},
                            {"type": "comments", "id": "5"}
                        ]
                    }
                }' Content-Type:application/vnd.api+json
        """
        try:
            data = self.request.json_body['data']
        except KeyError:
            raise HTTPBadRequest('data attribute required in PATCHes.')
        data_id = data.get('id')
        if self.collection_name != data.get('type'):
            raise HTTPConflict(
                'JSON type ({}) does not match URL type ({}).'.format(
                    data.get('type'), self.collection_name
                )
            )
        if data_id != self.obj_id:
            raise HTTPConflict(
                'JSON id ({}) does not match URL id ({}).'.format(
                    data_id, self.obj_id
                )
            )
        for callback in self.callbacks['before_patch']:
            data = callback(self, data)
        atts = {}
        hybrid_atts = {}
        for key, value in data.get('attributes', {}).items():
            if key in self.attributes:
                atts[key] = value
            elif key in self.hybrid_attributes:
                hybrid_atts[key] = value
            else:
                raise HTTPNotFound(
                    'Collection {} has no attribute {}'.format(
                        self.collection_name, key
                    )
                )
        atts[self.key_column.name] = self.obj_id
        item = self.dbsession.merge(self.model(**atts))
        for att, value in hybrid_atts.items():
            try:
                setattr(item, att, value)
            except AttributeError:
                raise HTTPConflict(
                    'Attribute {} is read only.'.format(
                        att
                    )
                )

        rels = data.get('relationships', {})
        for relname, reldict in rels.items():
            try:
                rel = self.relationships[relname]
            except KeyError:
                raise HTTPNotFound(
                    'Collection {} has no relationship {}'.format(
                        self.collection_name, relname
                    )
                )
            rel_view = self.view_instance(rel.mapper.class_)
            try:
                data = reldict['data']
            except KeyError:
                raise HTTPBadRequest(
                    "Relationship '{}' has no 'data' member.".format(relname)
                )
            except TypeError:
                raise HTTPBadRequest(
                    "Relationship '{}' is not a dictionary with a data member.".format(relname)
                )
            if data is None:
                setattr(item, relname, None)
            elif isinstance(data, dict):
                if data.get('type') != rel_view.collection_name:
                    raise HTTPConflict(
                        'Type {} does not match relationship type {}'.format(
                            data.get('type', None), rel_view.collection_name
                        )
                    )
                if data.get('id') is None:
                    raise HTTPBadRequest(
                        'An id is required in a resource identifier.'
                    )
                rel_item = self.dbsession.query(
                    rel.mapper.class_
                ).options(
                    load_only(rel_view.key_column.name)
                ).get(data['id'])
                if not rel_item:
                    raise HTTPNotFound('{}/{} not found'.format(
                        rel_view.collection_name, data['id']
                    ))
                setattr(item, relname, rel_item)
            elif isinstance(data, list):
                rel_items = []
                for res_ident in data:
                    rel_item = self.dbsession.query(
                        rel.mapper.class_
                    ).options(
                        load_only(rel_view.key_column.name)
                    ).get(res_ident['id'])
                    if not rel_item:
                        raise HTTPNotFound('{}/{} not found'.format(
                            rel_view.collection_name, res_ident['id']
                        ))
                    rel_items.append(rel_item)
                setattr(item, relname, rel_items)

        self.dbsession.flush()
        doc = pyramid_jsonapi.jsonapi.Document()
        doc.meta = {
            'updated': {
                'attributes': [
                    att for att in atts
                    if att != self.key_column.name
                ],
                'relationships': [r for r in rels]
            }
        }
        # if an update is successful ... the server
        # responds only with top-level meta data
        doc.filter_keys = {'meta': {}}
        return doc

    @jsonapi_view
    def delete(self):
        """Handle DELETE request for single item.

        Delete the referenced item from the collection.

        **URL (matchdict) Parameters**

            **id** (*str*): resource id

        Returns:
            jsonapi.Document: Resource Identifier for deleted object.

        Raises:
            HTTPFailedDependency: if a database constraint would be broken by
            deleting the specified resource from the relationship.

        Example:
            delete person 1:

            .. parsed-literal::

                http DELETE http://localhost:6543/people/1
        """

        doc = pyramid_jsonapi.jsonapi.Document()
        item = self.single_item_query(loadonly=[self.key_column.name]).one()
        for callback in self.callbacks['before_delete']:
            callback(self, item)
        try:
            self.dbsession.delete(item)
            self.dbsession.flush()
        except sqlalchemy.exc.IntegrityError as exc:
            raise HTTPFailedDependency(str(exc))
        doc.update({
            'data': self.serialise_resource_identifier(
                self.obj_id
            )})
        return doc

    @jsonapi_view
    def collection_get(self):
        """Handle GET requests for the collection.

        Get a set of items from the collection, possibly matching search/filter
        parameters. Optionally sort the results, page them, return only certain
        fields, and include related resources.

        **Query Parameters**

            **include:** comma separated list of related resources to include
            in the include section.

            **fields[<collection>]:** comma separated list of fields
            (attributes or relationships) to include in data.

            **sort:** comma separated list of sort keys.

            **page[limit]:** number of results to return per page.

            **page[offset]:** starting index for current page.

            **filter[<attribute>:<op>]:** filter operation.

        Returns:
            jsonapi.Document: in the form:

            .. parsed-literal::

                {
                    "data": [ list of resource objects ],
                    "links": { links object },
                    "include": [ optional list of included resource objects ],
                    "meta": { implementation specific information }
                }

        Raises:
            HTTPBadRequest

        Examples:
            Get up to default page limit people resources:

            .. parsed-literal::

                http GET http://localhost:6543/people

            Get the second page of two people, reverse sorted by name and
            include the related posts as included documents:

            .. parsed-literal::

                http GET http://localhost:6543/people?page[limit]=2&page[offset]=2&sort=-name&include=posts
        """
        # Set up the query
        query = self.dbsession.query(
            self.model
        ).options(
            load_only(*self.allowed_requested_query_columns.keys())
        )
        query = self.query_add_sorting(query)
        query = self.query_add_filtering(query)
        qinfo = self.collection_query_info(self.request)
        try:
            count = query.count()
        except sqlalchemy.exc.ProgrammingError:
            raise HTTPInternalServerError(
                'An error occurred querying the database. Server logs may have details.'
            )
        query = query.offset(qinfo['page[offset]'])
        query = query.limit(qinfo['page[limit]'])

        ret = self.collection_return(query, count=count)

        # Alter return dict with any callbacks.
        for callback in self.callbacks['after_collection_get']:
            ret = callback(self, ret)
        return ret

    @jsonapi_view
    def collection_post(self):
        """Handle POST requests for the collection.

        Create a new object in collection.

        **Request Body**

            **resource object** (*json*) in the form:

            .. parsed-literal::

                {
                    "data": { resource object }
                }

        Returns:
            jsonapi.Document: in the form:

            .. parsed-literal::

                {
                    "data": { resource object },
                    "links": {
                        "self": self url,
                        maybe other links...
                    },
                    "meta": { jsonapi specific information }
                }

        Raises:
            HTTPForbidden: if an id is presented in "data" and client ids are
            not supported.

            HTTPConflict: if type is not present or is different from the
            collection name.

            HTTPNotFound: if a non existent relationship is referenced in the
            supplied resource object.

            HTTPConflict: if creating the object would break a database
            constraint (most commonly if an id is supplied by the client and
            an item with that id already exists).

            HTTPBadRequest: if the request is malformed in some other way.

        Examples:
            Create a new person with name 'monty' and let the server pick the
            id:

            .. parsed-literal::

                http POST http://localhost:6543/people data:='
                {
                    "type":"people",
                    "attributes": {
                        "name": "monty"
                    }
                }' Content-Type:application/vnd.api+json
        """
        try:
            data = self.request.json_body['data']
        except KeyError:
            raise HTTPBadRequest('data attribute required in POSTs.')

        if not isinstance(data, dict):
            raise HTTPBadRequest('data attribute must contain a single resource object.')

        # Alter data with any callbacks.
        for callback in self.callbacks['before_collection_post']:
            data = callback(self, data)

        # Check to see if we're allowing client ids
        if not self.api.settings.allow_client_ids and 'id' in data:
            raise HTTPForbidden('Client generated ids are not supported.')
        # Type should be correct or raise 409 Conflict
        datatype = data.get('type')
        if datatype != self.collection_name:
            raise HTTPConflict("Unsupported type '{}'".format(datatype))
        try:
            atts = data['attributes']
        except KeyError:
            atts = {}
        if 'id' in data:
            atts[self.model.__pyramid_jsonapi__['id_col_name']] = data['id']
        item = self.model(**atts)
        with self.dbsession.no_autoflush:
            for relname, reldict in data.get('relationships', {}).items():
                try:
                    reldata = reldict['data']
                except KeyError:
                    raise HTTPBadRequest(
                        'relationships within POST must have data member'
                    )
                try:
                    rel = sqlalchemy.inspect(self.model).mapper.relationships[relname]
                except KeyError:
                    raise HTTPNotFound(
                        'No relationship {} in collection {}'.format(
                            relname,
                            self.collection_name
                        )
                    )
                rel_type = self.api.view_classes[rel.mapper.class_].collection_name
                if rel.direction is ONETOMANY or rel.direction is MANYTOMANY:
                    # reldata should be a list/array
                    if not isinstance(reldata, Sequence) or isinstance(reldata, str):
                        raise HTTPBadRequest(
                            'Relationship data should be an array for TOMANY relationships.'
                        )
                    rel_items = []
                    for rel_identifier in reldata:
                        if rel_identifier.get('type') != rel_type:
                            raise HTTPConflict(
                                'Relationship identifier has type {} and should be {}'.format(
                                    rel_identifier.get('type'), rel_type
                                )
                            )
                        try:
                            rel_items.append(self.dbsession.query(rel.mapper.class_).get(rel_identifier['id']))
                        except KeyError:
                            raise HTTPBadRequest(
                                'Relationship identifier must have an id member'
                            )
                    setattr(item, relname, rel_items)
                else:
                    try:
                        setattr(
                            item,
                            relname,
                            self.dbsession.query(rel.mapper.class_).get(reldata['id'])
                        )
                    except KeyError:
                        raise HTTPBadRequest(
                            'No id member in relationship data.'
                        )
        try:
            self.dbsession.add(item)
            self.dbsession.flush()
        except sqlalchemy.exc.IntegrityError as exc:
            raise HTTPConflict(exc.args[0])
        self.request.response.status_code = 201
        self.request.response.headers['Location'] = self.request.route_url(
            self.api.endpoint_data.make_route_name(self.collection_name, suffix='item'),
            **{'id': self.id_col(item)}
        )
        doc = pyramid_jsonapi.jsonapi.Document()
        doc.update({'data': self.serialise_db_item(item, {})})
        return doc

    @jsonapi_view
    def related_get(self):
        """Handle GET requests for related URLs.

        Get object(s) related to a specified object.

        **URL (matchdict) Parameters**

            **id** (*str*): resource id
            **relname** (*str*): relationship name

        **Query Parameters**
            **sort:** comma separated list of sort keys.

            **filter[<attribute>:<op>]:** filter operation.

        Returns:
            jsonapi.Document: in the form:

            For a TOONE relationship (return one object):

            .. parsed-literal::

                {
                    "data": { resource object },
                    "links": {
                        "self": self url,
                        maybe other links...
                    },
                    "meta": { jsonapi specific information }
                }

            For a TOMANY relationship (return multiple objects):

            .. parsed-literal::

                {
                    "data": [ { resource object }, ... ]
                    "links": {
                        "self": self url,
                        maybe other links...
                    },
                    "meta": { jsonapi specific information }
                }

        Raises:
            HTTPNotFound: if `relname` is not found as a relationship.

            HTTPBadRequest: if a bad filter is used.

        Examples:
            Get the author of post 1:

            .. parsed-literal::

                http GET http://localhost:6543/posts/1/author
        """
        # Set up the query
        query = self.related_query(self.obj_id, self.rel)

        if self.rel.direction is ONETOMANY or self.rel.direction is MANYTOMANY:
            query = self.rel_view.query_add_sorting(query)
            query = self.rel_view.query_add_filtering(query)
            qinfo = self.rel_view.collection_query_info(self.request)
            try:
                count = query.count()
            except sqlalchemy.exc.ProgrammingError:
                raise HTTPInternalServerError(
                    'An error occurred querying the database. Server logs may have details.'
                )
            query = query.offset(qinfo['page[offset]'])
            query = query.limit(qinfo['page[limit]'])
            ret = self.rel_view.collection_return(query, count=count)
        else:
            ret = self.rel_view.single_return(query)

        # Alter return dict with any callbacks.
        for callback in self.callbacks['after_related_get']:
            ret = callback(self, ret)
        return ret

    @jsonapi_view<commit_msg>make related_get() cope with AssociationProxy.<commit_after>"""Provide base class for collection views and utilities."""
# pylint: disable=too-many-lines; It's mostly docstrings
import functools
import itertools
import logging
import re
from collections import Sequence

from pyramid.httpexceptions import (
    HTTPNotFound,
    HTTPForbidden,
    HTTPBadRequest,
    HTTPConflict,
    HTTPUnsupportedMediaType,
    HTTPNotAcceptable,
    HTTPError,
    HTTPFailedDependency,
    HTTPInternalServerError,
    status_map,
)
import pyramid_jsonapi.jsonapi
import sqlalchemy
from sqlalchemy.ext.associationproxy import AssociationProxy
from sqlalchemy.orm import load_only
from sqlalchemy.orm.relationships import RelationshipProperty
from sqlalchemy.orm.exc import NoResultFound

ONETOMANY = sqlalchemy.orm.interfaces.ONETOMANY
MANYTOMANY = sqlalchemy.orm.interfaces.MANYTOMANY
MANYTOONE = sqlalchemy.orm.interfaces.MANYTOONE


class CollectionViewBase:
    """Base class for all view classes.

    Arguments:
        request (pyramid.request): passed by framework.
    """

    # pylint:disable=too-many-public-methods

    # Define class attributes
    # Callable attributes use lambda to keep pylint happy
    api = None
    attributes = None
    callbacks = None
    collection_name = None
    default_limit = None
    exposed_fields = None
    fields = None
    dbsession = None
    hybrid_attributes = None
    item = None
    key_column = None
    max_limit = None
    model = lambda: None
    obj_id = None
    request = None
    rel = None
    rel_class = None
    rel_view = None
    relationships = None
    relname = None
    view_classes = None
    settings = None

    def __init__(self, request):
        self.request = request
        if self.api.get_dbsession:
            self.dbsession = self.api.get_dbsession(self)
        else:
            self.dbsession = self.request.dbsession
        self.views = {}

    @staticmethod
    def id_col(item):
        """Return the column holding an item's id."""
        return getattr(item, item.__pyramid_jsonapi__['id_col_name'])

    def jsonapi_view(func):  # pylint: disable=no-self-argument
        """Decorator for view functions. Adds jsonapi boilerplate,
        and tests response validity."""

        def view_exceptions(func):
            """Decorator to intercept all exceptions raised by wrapped view methods.

            If the exception is 'valid' according to the schema, raise it.
            Else raise a generic 4xx or 5xx error and log the real one.
            """
            @functools.wraps(func)
            def new_func(self):  # pylint: disable=missing-docstring
                ep_dict = self.api.endpoint_data.endpoints
                # Get route_name from route
                _, _, endpoint = self.request.matched_route.name.split(':')
                method = self.request.method
                responses = set(
                    ep_dict['responses'].keys() |
                    ep_dict['endpoints'][endpoint]['responses'].keys() |
                    ep_dict['endpoints'][endpoint]['http_methods'][method]['responses'].keys()
                )
                try:
                    result = func(self)  # pylint: disable=not-callable
                    response_class = status_map[self.request.response.status_code]
                    if response_class not in responses:
                        logging.error(
                            "Invalid response: %s for route_name: %s path: %s",
                            response_class,
                            self.request.matched_route.name,
                            self.request.current_route_path()
                        )
                    return result
                except Exception as exc:
                    if exc.__class__ not in responses:
                        logging.exception(
                            "Invalid exception raised: %s for route_name: %s path: %s",
                            exc.__class__,
                            self.request.matched_route.name,
                            self.request.current_route_path()
                        )
                        if hasattr(exc, 'code'):
                            if 400 <= exc.code < 500:  # pylint:disable=no-member
                                raise HTTPBadRequest("Unexpected client error: {}".format(exc))
                        else:
                            raise HTTPInternalServerError("Unexpected server error.")
                    raise
            return new_func

        @functools.lru_cache()
        def get_jsonapi_accepts(request):
            """Return a set of all 'application/vnd.api' parts of the accept
            header.
            """
            accepts = re.split(
                r',\s*',
                request.headers.get('accept', '')
            )
            return {
                a for a in accepts
                if a.startswith('application/vnd.api')
            }

        def check_request_headers(request, jsonapi_accepts):
            """Check that request headers comply with spec.

            Raises:
                HTTPUnsupportedMediaType
                HTTPNotAcceptable
            """
            # Spec says to reject (with 415) any request with media type
            # params.
            if len(request.headers.get('content-type', '').split(';')) > 1:
                raise HTTPUnsupportedMediaType(
                    'Media Type parameters not allowed by JSONAPI ' +
                    'spec (http://jsonapi.org/format).'
                )
            # Spec says throw 406 Not Acceptable if Accept header has no
            # application/vnd.api+json entry without parameters.
            if jsonapi_accepts and\
                    'application/vnd.api+json' not in jsonapi_accepts:
                raise HTTPNotAcceptable(
                    'application/vnd.api+json must appear with no ' +
                    'parameters in Accepts header ' +
                    '(http://jsonapi.org/format).'
                )

        def check_request_valid_json(request):
            """Check that the body of any request is valid JSON.

            Raises:
                HTTPBadRequest
            """
            if request.content_length:
                try:
                    request.json_body
                except ValueError:
                    raise HTTPBadRequest("Body is not valid JSON.")

        @view_exceptions
        @functools.wraps(func)
        def view_wrapper(self):
            """jsonapi boilerplate function to wrap decorated functions."""
            check_request_headers(self.request, get_jsonapi_accepts(self.request))
            check_request_valid_json(self.request)

            if self.request.content_length and self.api.settings.schema_validation:
                # Validate request JSON against the JSONAPI jsonschema
                self.api.metadata.JSONSchema.validate(self.request.json_body, self.request.method)

            # Spec says throw BadRequest if any include paths reference non
            # existent attributes or relationships.
            if self.bad_include_paths:
                raise HTTPBadRequest(
                    "Bad include paths {}".format(
                        self.bad_include_paths
                    )
                )

            # Spec says set Content-Type to application/vnd.api+json.
            self.request.response.content_type = 'application/vnd.api+json'

            # Extract id and relationship from route, if provided
            self.obj_id = self.request.matchdict.get('id', None)
            self.relname = self.request.matchdict.get('relationship', None)

            if self.obj_id:
                # Try to get the object
                try:
                    self.item = self.single_return(
                        self.single_item_query(),
                        'No id {} in collection {}'.format(
                            self.obj_id,
                            self.collection_name
                        )
                    )
                except (sqlalchemy.exc.DataError, sqlalchemy.exc.StatementError):
                    # DataError is caused by e.g. id (int) = cat
                    # StatementError is caused by e.g. id (uuid) = 1
                    raise HTTPNotFound('Object {} not found in collection {}'.format(
                        self.obj_id,
                        self.collection_name
                    ))

            if self.relname:
                # Gather relationship info
                mapper = sqlalchemy.inspect(self.model).mapper
                try:
                    self.rel = self.relationships[self.relname]
                except KeyError:
                    raise HTTPNotFound('No relationship {} in collection {}'.format(
                        self.relname,
                        self.collection_name
                    ))
                if isinstance(self.rel, AssociationProxy):
                    proxy_state = self.rel.for_class(self.model)
                    self.rel_class = getattr(proxy_state.target_class, proxy_state.value_attr).mapper.class_
                else:
                    self.rel_class = self.rel.mapper.class_
                self.rel_view = self.view_instance(self.rel_class)

            # Update the dictionary with the reults of the wrapped method.
            ret = func(self)  # pylint:disable=not-callable
            if ret:
                # Include a self link unless the method is PATCH.
                if self.request.method != 'PATCH':
                    selfie = {'self': self.request.url}
                    if hasattr(ret, 'links'):
                        ret.links.update(selfie)
                    else:
                        ret.links = selfie

                # Potentially add some debug information.
                if self.api.settings.debug_meta:
                    debug = {
                        'accept_header': {
                            a: None for a in get_jsonapi_accepts(self.request)
                        },
                        'qinfo_page':
                            self.collection_query_info(self.request)['_page'],
                        'atts': {k: None for k in self.attributes.keys()},
                        'includes': {
                            k: None for k in self.requested_include_names()
                        }
                    }
                    ret.meta.update({'debug': debug})
                return ret.as_dict()
            else:
                return {}
        return view_wrapper

    @jsonapi_view
    def get(self):
        """Handle GET request for a single item.

        Get a single item from the collection, referenced by id.

        **URL (matchdict) Parameters**

            **id** (*str*): resource id

        Returns:
            jsonapi.Document: in the form:

            .. parsed-literal::

                {
                    "data": { resource object },
                    "links": {
                        "self": self url,
                        maybe other links...
                    },
                    "meta": { jsonapi specific information }
                }

        Raises:
            HTTPNotFound

        Example:

            Get person 1:

            .. parsed-literal::

                http GET http://localhost:6543/people/1
        """
        for callback in self.callbacks['after_get']:
            self.item = callback(self, self.item)
        return self.item

    @jsonapi_view
    def patch(self):
        """Handle PATCH request for a single item.

        Update an existing item from a partially defined representation.

        **URL (matchdict) Parameters**

            **id** (*str*): resource id

        **Request Body**

            **Partial resource object** (*json*)

        Returns:
            jsonapi.Document: in the form:

            .. parsed-literal::

                {
                    'meta': {
                        'updated': [
                            <attribute_name>,
                            <attribute_name>
                        ]
                    }
                }

        Raises:
            HTTPNotFound

        Todo:
            Currently does not deal with relationships.

        Example:
            PATCH person 1, changing name to alicia:

            .. parsed-literal::

                http PATCH http://localhost:6543/people/1 data:='
                {
                    "type":"people", "id": "1",
                    "attributes": {
                        "name": "alicia"
                    }
                }' Content-Type:application/vnd.api+json

            Change the author of posts/1 to people/2:

            .. parsed-literal::

                http PATCH http://localhost:6543/posts/1 data:='
                {
                    "type":"posts", "id": "1",
                    "relationships": {
                        "author": {"type": "people", "id": "2"}
                    }
                }' Content-Type:application/vnd.api+json

            Set the comments on posts/1 to be [comments/4, comments/5]:

            .. parsed-literal::

                http PATCH http://localhost:6543/posts/1 data:='
                {
                    "type":"posts", "id": "1",
                    "relationships": {
                        "comments": [
                            {"type": "comments", "id": "4"},
                            {"type": "comments", "id": "5"}
                        ]
                    }
                }' Content-Type:application/vnd.api+json
        """
        try:
            data = self.request.json_body['data']
        except KeyError:
            raise HTTPBadRequest('data attribute required in PATCHes.')
        data_id = data.get('id')
        if self.collection_name != data.get('type'):
            raise HTTPConflict(
                'JSON type ({}) does not match URL type ({}).'.format(
                    data.get('type'), self.collection_name
                )
            )
        if data_id != self.obj_id:
            raise HTTPConflict(
                'JSON id ({}) does not match URL id ({}).'.format(
                    data_id, self.obj_id
                )
            )
        for callback in self.callbacks['before_patch']:
            data = callback(self, data)
        atts = {}
        hybrid_atts = {}
        for key, value in data.get('attributes', {}).items():
            if key in self.attributes:
                atts[key] = value
            elif key in self.hybrid_attributes:
                hybrid_atts[key] = value
            else:
                raise HTTPNotFound(
                    'Collection {} has no attribute {}'.format(
                        self.collection_name, key
                    )
                )
        atts[self.key_column.name] = self.obj_id
        item = self.dbsession.merge(self.model(**atts))
        for att, value in hybrid_atts.items():
            try:
                setattr(item, att, value)
            except AttributeError:
                raise HTTPConflict(
                    'Attribute {} is read only.'.format(
                        att
                    )
                )

        rels = data.get('relationships', {})
        for relname, reldict in rels.items():
            try:
                rel = self.relationships[relname]
            except KeyError:
                raise HTTPNotFound(
                    'Collection {} has no relationship {}'.format(
                        self.collection_name, relname
                    )
                )
            rel_view = self.view_instance(rel.mapper.class_)
            try:
                data = reldict['data']
            except KeyError:
                raise HTTPBadRequest(
                    "Relationship '{}' has no 'data' member.".format(relname)
                )
            except TypeError:
                raise HTTPBadRequest(
                    "Relationship '{}' is not a dictionary with a data member.".format(relname)
                )
            if data is None:
                setattr(item, relname, None)
            elif isinstance(data, dict):
                if data.get('type') != rel_view.collection_name:
                    raise HTTPConflict(
                        'Type {} does not match relationship type {}'.format(
                            data.get('type', None), rel_view.collection_name
                        )
                    )
                if data.get('id') is None:
                    raise HTTPBadRequest(
                        'An id is required in a resource identifier.'
                    )
                rel_item = self.dbsession.query(
                    rel.mapper.class_
                ).options(
                    load_only(rel_view.key_column.name)
                ).get(data['id'])
                if not rel_item:
                    raise HTTPNotFound('{}/{} not found'.format(
                        rel_view.collection_name, data['id']
                    ))
                setattr(item, relname, rel_item)
            elif isinstance(data, list):
                rel_items = []
                for res_ident in data:
                    rel_item = self.dbsession.query(
                        rel.mapper.class_
                    ).options(
                        load_only(rel_view.key_column.name)
                    ).get(res_ident['id'])
                    if not rel_item:
                        raise HTTPNotFound('{}/{} not found'.format(
                            rel_view.collection_name, res_ident['id']
                        ))
                    rel_items.append(rel_item)
                setattr(item, relname, rel_items)

        self.dbsession.flush()
        doc = pyramid_jsonapi.jsonapi.Document()
        doc.meta = {
            'updated': {
                'attributes': [
                    att for att in atts
                    if att != self.key_column.name
                ],
                'relationships': [r for r in rels]
            }
        }
        # if an update is successful ... the server
        # responds only with top-level meta data
        doc.filter_keys = {'meta': {}}
        return doc

    @jsonapi_view
    def delete(self):
        """Handle DELETE request for single item.

        Delete the referenced item from the collection.

        **URL (matchdict) Parameters**

            **id** (*str*): resource id

        Returns:
            jsonapi.Document: Resource Identifier for deleted object.

        Raises:
            HTTPFailedDependency: if a database constraint would be broken by
            deleting the specified resource from the relationship.

        Example:
            delete person 1:

            .. parsed-literal::

                http DELETE http://localhost:6543/people/1
        """

        doc = pyramid_jsonapi.jsonapi.Document()
        item = self.single_item_query(loadonly=[self.key_column.name]).one()
        for callback in self.callbacks['before_delete']:
            callback(self, item)
        try:
            self.dbsession.delete(item)
            self.dbsession.flush()
        except sqlalchemy.exc.IntegrityError as exc:
            raise HTTPFailedDependency(str(exc))
        doc.update({
            'data': self.serialise_resource_identifier(
                self.obj_id
            )})
        return doc

    @jsonapi_view
    def collection_get(self):
        """Handle GET requests for the collection.

        Get a set of items from the collection, possibly matching search/filter
        parameters. Optionally sort the results, page them, return only certain
        fields, and include related resources.

        **Query Parameters**

            **include:** comma separated list of related resources to include
            in the include section.

            **fields[<collection>]:** comma separated list of fields
            (attributes or relationships) to include in data.

            **sort:** comma separated list of sort keys.

            **page[limit]:** number of results to return per page.

            **page[offset]:** starting index for current page.

            **filter[<attribute>:<op>]:** filter operation.

        Returns:
            jsonapi.Document: in the form:

            .. parsed-literal::

                {
                    "data": [ list of resource objects ],
                    "links": { links object },
                    "include": [ optional list of included resource objects ],
                    "meta": { implementation specific information }
                }

        Raises:
            HTTPBadRequest

        Examples:
            Get up to default page limit people resources:

            .. parsed-literal::

                http GET http://localhost:6543/people

            Get the second page of two people, reverse sorted by name and
            include the related posts as included documents:

            .. parsed-literal::

                http GET http://localhost:6543/people?page[limit]=2&page[offset]=2&sort=-name&include=posts
        """
        # Set up the query
        query = self.dbsession.query(
            self.model
        ).options(
            load_only(*self.allowed_requested_query_columns.keys())
        )
        query = self.query_add_sorting(query)
        query = self.query_add_filtering(query)
        qinfo = self.collection_query_info(self.request)
        try:
            count = query.count()
        except sqlalchemy.exc.ProgrammingError:
            raise HTTPInternalServerError(
                'An error occurred querying the database. Server logs may have details.'
            )
        query = query.offset(qinfo['page[offset]'])
        query = query.limit(qinfo['page[limit]'])

        ret = self.collection_return(query, count=count)

        # Alter return dict with any callbacks.
        for callback in self.callbacks['after_collection_get']:
            ret = callback(self, ret)
        return ret

    @jsonapi_view
    def collection_post(self):
        """Handle POST requests for the collection.

        Create a new object in collection.

        **Request Body**

            **resource object** (*json*) in the form:

            .. parsed-literal::

                {
                    "data": { resource object }
                }

        Returns:
            jsonapi.Document: in the form:

            .. parsed-literal::

                {
                    "data": { resource object },
                    "links": {
                        "self": self url,
                        maybe other links...
                    },
                    "meta": { jsonapi specific information }
                }

        Raises:
            HTTPForbidden: if an id is presented in "data" and client ids are
            not supported.

            HTTPConflict: if type is not present or is different from the
            collection name.

            HTTPNotFound: if a non existent relationship is referenced in the
            supplied resource object.

            HTTPConflict: if creating the object would break a database
            constraint (most commonly if an id is supplied by the client and
            an item with that id already exists).

            HTTPBadRequest: if the request is malformed in some other way.

        Examples:
            Create a new person with name 'monty' and let the server pick the
            id:

            .. parsed-literal::

                http POST http://localhost:6543/people data:='
                {
                    "type":"people",
                    "attributes": {
                        "name": "monty"
                    }
                }' Content-Type:application/vnd.api+json
        """
        try:
            data = self.request.json_body['data']
        except KeyError:
            raise HTTPBadRequest('data attribute required in POSTs.')

        if not isinstance(data, dict):
            raise HTTPBadRequest('data attribute must contain a single resource object.')

        # Alter data with any callbacks.
        for callback in self.callbacks['before_collection_post']:
            data = callback(self, data)

        # Check to see if we're allowing client ids
        if not self.api.settings.allow_client_ids and 'id' in data:
            raise HTTPForbidden('Client generated ids are not supported.')
        # Type should be correct or raise 409 Conflict
        datatype = data.get('type')
        if datatype != self.collection_name:
            raise HTTPConflict("Unsupported type '{}'".format(datatype))
        try:
            atts = data['attributes']
        except KeyError:
            atts = {}
        if 'id' in data:
            atts[self.model.__pyramid_jsonapi__['id_col_name']] = data['id']
        item = self.model(**atts)
        with self.dbsession.no_autoflush:
            for relname, reldict in data.get('relationships', {}).items():
                try:
                    reldata = reldict['data']
                except KeyError:
                    raise HTTPBadRequest(
                        'relationships within POST must have data member'
                    )
                try:
                    rel = sqlalchemy.inspect(self.model).mapper.relationships[relname]
                except KeyError:
                    raise HTTPNotFound(
                        'No relationship {} in collection {}'.format(
                            relname,
                            self.collection_name
                        )
                    )
                rel_type = self.api.view_classes[rel.mapper.class_].collection_name
                if rel.direction is ONETOMANY or rel.direction is MANYTOMANY:
                    # reldata should be a list/array
                    if not isinstance(reldata, Sequence) or isinstance(reldata, str):
                        raise HTTPBadRequest(
                            'Relationship data should be an array for TOMANY relationships.'
                        )
                    rel_items = []
                    for rel_identifier in reldata:
                        if rel_identifier.get('type') != rel_type:
                            raise HTTPConflict(
                                'Relationship identifier has type {} and should be {}'.format(
                                    rel_identifier.get('type'), rel_type
                                )
                            )
                        try:
                            rel_items.append(self.dbsession.query(rel.mapper.class_).get(rel_identifier['id']))
                        except KeyError:
                            raise HTTPBadRequest(
                                'Relationship identifier must have an id member'
                            )
                    setattr(item, relname, rel_items)
                else:
                    try:
                        setattr(
                            item,
                            relname,
                            self.dbsession.query(rel.mapper.class_).get(reldata['id'])
                        )
                    except KeyError:
                        raise HTTPBadRequest(
                            'No id member in relationship data.'
                        )
        try:
            self.dbsession.add(item)
            self.dbsession.flush()
        except sqlalchemy.exc.IntegrityError as exc:
            raise HTTPConflict(exc.args[0])
        self.request.response.status_code = 201
        self.request.response.headers['Location'] = self.request.route_url(
            self.api.endpoint_data.make_route_name(self.collection_name, suffix='item'),
            **{'id': self.id_col(item)}
        )
        doc = pyramid_jsonapi.jsonapi.Document()
        doc.update({'data': self.serialise_db_item(item, {})})
        return doc

    @jsonapi_view
    def related_get(self):
        """Handle GET requests for related URLs.

        Get object(s) related to a specified object.

        **URL (matchdict) Parameters**

            **id** (*str*): resource id
            **relname** (*str*): relationship name

        **Query Parameters**
            **sort:** comma separated list of sort keys.

            **filter[<attribute>:<op>]:** filter operation.

        Returns:
            jsonapi.Document: in the form:

            For a TOONE relationship (return one object):

            .. parsed-literal::

                {
                    "data": { resource object },
                    "links": {
                        "self": self url,
                        maybe other links...
                    },
                    "meta": { jsonapi specific information }
                }

            For a TOMANY relationship (return multiple objects):

            .. parsed-literal::

                {
                    "data": [ { resource object }, ... ]
                    "links": {
                        "self": self url,
                        maybe other links...
                    },
                    "meta": { jsonapi specific information }
                }

        Raises:
            HTTPNotFound: if `relname` is not found as a relationship.

            HTTPBadRequest: if a bad filter is used.

        Examples:
            Get the author of post 1:

            .. parsed-literal::

                http GET http://localhost:6543/posts/1/author
        """
        # Set up the query
        query = self.related_query(self.obj_id, self.rel)

        if isinstance(self.rel, AssociationProxy):
            ps = self.rel.for_class(self.model)
            if ps.scalar:
                direction = MANYTOONE
            else:
                direction = MANYTOMANY
        else:
            direction = self.rel.direction

        if direction is ONETOMANY or direction is MANYTOMANY:
            query = self.rel_view.query_add_sorting(query)
            query = self.rel_view.query_add_filtering(query)
            qinfo = self.rel_view.collection_query_info(self.request)
            try:
                count = query.count()
            except sqlalchemy.exc.ProgrammingError:
                raise HTTPInternalServerError(
                    'An error occurred querying the database. Server logs may have details.'
                )
            query = query.offset(qinfo['page[offset]'])
            query = query.limit(qinfo['page[limit]'])
            ret = self.rel_view.collection_return(query, count=count)
        else:
            ret = self.rel_view.single_return(query)

        # Alter return dict with any callbacks.
        for callback in self.callbacks['after_related_get']:
            ret = callback(self, ret)
        return ret

    @jsonapi_view
{'fraction_non_alphanumeric': 0.08564366900274802, 'fraction_numerical': 0.00266671002780533, 'mean_word_length': 1.3266371580978322, 'pattern_counts': {'":': 84, '<': 21, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '228436', 'n_tokens_mistral': 15415, 'n_tokens_neox': 14317, 'n_words': 3942}