<commit_before>"""Rest API views."""
from copy import deepcopy
from logging import getLogger

from colander import drop
from colander import Invalid
from colander import MappingSchema
from colander import Schema
from colander import SchemaNode
from colander import SequenceSchema
from cornice.util import json_error
from cornice.util import extract_request_data
from substanced.interfaces import IUserLocator
from pyramid.events import NewResponse
from pyramid.httpexceptions import HTTPMethodNotAllowed
from pyramid.request import Request
from pyramid.view import view_config
from pyramid.view import view_defaults
from pyramid.security import remember

from adhocracy.interfaces import IResource
from adhocracy.interfaces import IItem
from adhocracy.interfaces import IItemVersion
from adhocracy.interfaces import ISimple
from adhocracy.interfaces import IPool
from adhocracy.interfaces import ILocation
from adhocracy.rest.schemas import ResourceResponseSchema
from adhocracy.rest.schemas import ItemResponseSchema
from adhocracy.rest.schemas import POSTItemRequestSchema
from adhocracy.rest.schemas import POSTLoginEmailRequestSchema
from adhocracy.rest.schemas import POSTLoginUsernameRequestSchema
from adhocracy.rest.schemas import POSTResourceRequestSchema
from adhocracy.rest.schemas import PUTResourceRequestSchema
from adhocracy.rest.schemas import GETPoolRequestSchema
from adhocracy.rest.schemas import GETItemResponseSchema
from adhocracy.rest.schemas import GETResourceResponseSchema
from adhocracy.rest.schemas import OPTIONResourceResponseSchema
from adhocracy.rest.schemas import add_get_pool_request_extra_fields
from adhocracy.schema import AbsolutePath
from adhocracy.schema import References
from adhocracy.utils import get_iresource
from adhocracy.utils import strip_optional_prefix
from adhocracy.utils import to_dotted_name
from adhocracy.utils import get_sheet
from adhocracy.utils import get_user
from adhocracy.resources.root import IRootPool
from adhocracy.sheets.user import IPasswordAuthentication
import adhocracy.sheets.pool

logger = getLogger(__name__)


def validate_post_root_versions(context, request: Request):
    """Check and transform the 'root_version' paths to resources."""
    # FIXME: make this a colander validator and move to schema.py
    # use the catalog to find IItemversions
    root_versions = request.validated.get('root_versions', [])
    valid_root_versions = []
    for root in root_versions:
        if not IItemVersion.providedBy(root):
            error = 'This resource is not a valid ' \
                    'root version: {}'.format(request.resource_url(root))
            request.errors.add('body', 'root_versions', error)
            continue
        valid_root_versions.append(root)

    request.validated['root_versions'] = valid_root_versions


def validate_request_data(context: ILocation, request: Request,
                          schema=MappingSchema(), extra_validators=[]):
    """ Validate request data.

    :param context: passed to validator functions
    :param request: passed to validator functions
    :param schema: Schema to validate. Data to validate is extracted from the
                   request.body. For schema nodes with attribute `location` ==
                   `querystring` the data is extracted from the query string.
                   The validated data (dict or list) is stored in the
                   `request.validated` attribute.
                   The `None` value is allowed to disable schema validation.
    :param extra_validators: Functions called after schema validation.
                             The passed arguments are `context` and  `request`.
                             The should append errors to `request.errors` and
                             validated data to `request.validated`.

    :raises _JSONError: HTTP 400 for bad request data.
    """
    parent = context if request.method == 'POST' else context.__parent__
    schema_with_binding = schema.bind(context=context, request=request,
                                      parent_pool=parent)
    qs, headers, body, path = extract_request_data(request)
    validate_body_or_querystring(body, qs, schema_with_binding, context,
                                 request)
    _validate_extra_validators(extra_validators, context, request)
    _raise_if_errors(request)


def validate_body_or_querystring(body, qs, schema: Schema, context: IResource,
                                 request: Request):
    """Validate the querystring if this is a GET request, the body otherwise.

    This allows using just a single schema for all kinds of requests.
    """
    if isinstance(schema, GETPoolRequestSchema):
        schema = add_get_pool_request_extra_fields(qs, schema, context,
                                                   request.registry)
    if request.method.upper() == 'GET':
        _validate_schema(qs, schema, request,
                         location='querystring')
    else:
        _validate_schema(body, schema, request, location='body')


def _validate_schema(cstruct: object, schema: MappingSchema, request: Request,
                     location='body'):
    """Validate that the :term:`cstruct` data is conform to the given schema.

    :param request: request with list like `errors` attribute to append errors
                    and the dictionary attribute `validated` to add validated
                    data.
    :param location: filter schema nodes depending on the `location` attribute.
                     The default value is `body`.
    """
    if isinstance(schema, SequenceSchema):
        _validate_list_schema(schema, cstruct, request, location)
    elif isinstance(schema, MappingSchema):
        _validate_dict_schema(schema, cstruct, request, location)
    else:
        error = 'Validation for schema {} is unsupported.'.format(str(schema))
        raise(Exception(error))


def _validate_list_schema(schema: SequenceSchema, cstruct: list,
                          request: Request, location='body'):
    if location != 'body':  # for now we only support location == body
        return
    child_cstructs = schema.cstruct_children(cstruct)
    try:
        request.validated = schema.deserialize(child_cstructs)
    except Invalid as err:
        _add_colander_invalid_error_to_request(err, request, location)


def _validate_dict_schema(schema: MappingSchema, cstruct: dict,
                          request: Request, location='body'):
    validated = {}
    nodes_with_cstruct = [n for n in schema if n.name in cstruct]
    nodes_without_cstruct = [n for n in schema if n.name not in cstruct]

    for node in nodes_without_cstruct:
        appstruct = node.deserialize()
        if appstruct is not drop:
            validated[node.name] = appstruct
    for node in nodes_with_cstruct:
        node_cstruct = cstruct[node.name]
        try:
            validated[node.name] = node.deserialize(node_cstruct)
        except Invalid as err:
            _add_colander_invalid_error_to_request(err, request, location)
    if getattr(schema.typ, 'unknown', None) == 'preserve':
        # Schema asks us to preserve other cstruct values
        for name, value in cstruct.items():
            if name not in validated:
                validated[name] = value
    request.validated.update(validated)


def _add_colander_invalid_error_to_request(error: Invalid, request: Request,
                                           location: str):
    for name, msg in error.asdict().items():
        request.errors.add(location, name, msg)


def _validate_extra_validators(validators: list, context, request: Request):
    """Run `validators` functions. Assuming schema validation run before."""
    if request.errors:
        return
    for val in validators:
        val(context, request)


def _raise_if_errors(request: Request):
    """Raise :class:`cornice.errors._JSONError` and log if request.errors."""
    if not request.errors:
        return
    logger.warning('Found %i validation errors in request: <%s>',
                   len(request.errors), request.body)
    for error in request.errors:
        logger.warning('  %s', error)
    request.validated = {}
    raise json_error(request.errors)


class RESTView:

    """Class stub with request data validation support.

    Subclasses must implement the wanted request methods
    and configure the pyramid view::

        @view_defaults(
            renderer='simplejson',
            context=IResource,
            http_cache=0,
        )
        class MySubClass(RESTView):
            validation_GET = (MyColanderSchema, [my_extra_validation_function])

            @view_config(request_method='GET')
            def get(self):
            ...
    """

    validation_OPTIONS = (None, [])
    validation_HEAD = (None, [])
    validation_GET = (None, [])
    validation_PUT = (None, [])
    validation_POST = (None, [])

    def __init__(self, context, request):
        self.context = context
        """Context Resource."""
        self.request = request
        """:class:`pyramid.request.Request`."""
        schema_class, validators = _get_schema_and_validators(self, request)
        validate_request_data(context, request,
                              schema=schema_class(),
                              extra_validators=validators)

    def options(self) -> dict:
        raise HTTPMethodNotAllowed()

    def get(self) -> dict:
        raise HTTPMethodNotAllowed()

    def put(self) -> dict:
        raise HTTPMethodNotAllowed()

    def post(self) -> dict:
        raise HTTPMethodNotAllowed()

    def delete(self) -> dict:
        raise HTTPMethodNotAllowed()


def _get_schema_and_validators(view_class, request: Request) -> tuple:
    http_method = request.method.upper()
    validation_attr = 'validation_' + http_method
    schema, validators = getattr(view_class, validation_attr, (None, []))
    return schema or MappingSchema, validators


@view_defaults(
    renderer='simplejson',
    context=IResource,
    http_cache=0,
)
class ResourceRESTView(RESTView):

    """Default view for Resources, implements get and options."""

    def __init__(self, context, request):
        super().__init__(context, request)
        self.registry = request.registry.content
        """:class:`pyramid.registry.Registry`."""

    @view_config(request_method='OPTIONS')
    def options(self) -> dict:
        """Get possible request/response data structures and http methods."""
        addables = self.registry.resource_addables(self.context, self.request)
        sheets_view = self.registry.resource_sheets(self.context, self.request,
                                                    onlyviewable=True)
        sheets_edit = self.registry.resource_sheets(self.context, self.request,
                                                    onlyeditable=True)

        cstruct_singleton = OPTIONResourceResponseSchema().serialize()
        cstruct = deepcopy(cstruct_singleton)
        for sheet in sheets_edit:
            cstruct['PUT']['request_body']['data'][sheet] = {}
            cstruct['PUT']['request_body']['content_type'] = ''
            cstruct['PUT']['response_body']['content_type'] = ''
        cstruct['PUT']['response_body']['content_type'] = ''
        for sheet in sheets_view:
            cstruct['GET']['response_body']['data'][sheet] = {}
            cstruct['GET']['response_body']['content_type'] = ''
        for type, sheets in addables.items():
            names = sheets['sheets_optional'] + sheets['sheets_mandatory']
            sheets_dict = dict([(s, {}) for s in names])
            post_data = {'content_type': type, 'data': sheets_dict}
            cstruct['POST']['request_body'].append(post_data)
            cstruct['POST']['response_body']['content_type'] = ''
        return cstruct

    @view_config(request_method='GET')
    def get(self) -> dict:
        """Get resource data."""
        schema = GETResourceResponseSchema().bind(request=self.request)
        appstruct = _get_resource_response_appstruct(self.context)
        cstruct = schema.serialize(appstruct)
        cstruct['data'] = self._get_sheets_data_cstruct()
        return cstruct<commit_msg>Fix regression: user login acceptance test is causing internal server error<commit_after>"""Rest API views."""
from copy import deepcopy
from logging import getLogger

from colander import drop
from colander import Invalid
from colander import MappingSchema
from colander import Schema
from colander import SchemaNode
from colander import SequenceSchema
from cornice.util import json_error
from cornice.util import extract_request_data
from substanced.interfaces import IUserLocator
from pyramid.events import NewResponse
from pyramid.httpexceptions import HTTPMethodNotAllowed
from pyramid.request import Request
from pyramid.view import view_config
from pyramid.view import view_defaults
from pyramid.security import remember

from adhocracy.interfaces import IResource
from adhocracy.interfaces import IItem
from adhocracy.interfaces import IItemVersion
from adhocracy.interfaces import ISimple
from adhocracy.interfaces import IPool
from adhocracy.interfaces import ILocation
from adhocracy.rest.schemas import ResourceResponseSchema
from adhocracy.rest.schemas import ItemResponseSchema
from adhocracy.rest.schemas import POSTItemRequestSchema
from adhocracy.rest.schemas import POSTLoginEmailRequestSchema
from adhocracy.rest.schemas import POSTLoginUsernameRequestSchema
from adhocracy.rest.schemas import POSTResourceRequestSchema
from adhocracy.rest.schemas import PUTResourceRequestSchema
from adhocracy.rest.schemas import GETPoolRequestSchema
from adhocracy.rest.schemas import GETItemResponseSchema
from adhocracy.rest.schemas import GETResourceResponseSchema
from adhocracy.rest.schemas import OPTIONResourceResponseSchema
from adhocracy.rest.schemas import add_get_pool_request_extra_fields
from adhocracy.schema import AbsolutePath
from adhocracy.schema import References
from adhocracy.utils import get_iresource
from adhocracy.utils import strip_optional_prefix
from adhocracy.utils import to_dotted_name
from adhocracy.utils import get_sheet
from adhocracy.utils import get_user
from adhocracy.resources.root import IRootPool
from adhocracy.sheets.user import IPasswordAuthentication
import adhocracy.sheets.pool

logger = getLogger(__name__)


def validate_post_root_versions(context, request: Request):
    """Check and transform the 'root_version' paths to resources."""
    # FIXME: make this a colander validator and move to schema.py
    # use the catalog to find IItemversions
    root_versions = request.validated.get('root_versions', [])
    valid_root_versions = []
    for root in root_versions:
        if not IItemVersion.providedBy(root):
            error = 'This resource is not a valid ' \
                    'root version: {}'.format(request.resource_url(root))
            request.errors.add('body', 'root_versions', error)
            continue
        valid_root_versions.append(root)

    request.validated['root_versions'] = valid_root_versions


def validate_request_data(context: ILocation, request: Request,
                          schema=MappingSchema(), extra_validators=[]):
    """ Validate request data.

    :param context: passed to validator functions
    :param request: passed to validator functions
    :param schema: Schema to validate. Data to validate is extracted from the
                   request.body. For schema nodes with attribute `location` ==
                   `querystring` the data is extracted from the query string.
                   The validated data (dict or list) is stored in the
                   `request.validated` attribute.
                   The `None` value is allowed to disable schema validation.
    :param extra_validators: Functions called after schema validation.
                             The passed arguments are `context` and  `request`.
                             The should append errors to `request.errors` and
                             validated data to `request.validated`.

    :raises _JSONError: HTTP 400 for bad request data.
    """
    parent = context if request.method == 'POST' else context.__parent__
    schema_with_binding = schema.bind(context=context, request=request,
                                      parent_pool=parent)
    qs, headers, body, path = extract_request_data(request)
    validate_body_or_querystring(body, qs, schema_with_binding, context,
                                 request)
    _validate_extra_validators(extra_validators, context, request)
    _raise_if_errors(request)


def validate_body_or_querystring(body, qs, schema: Schema, context: IResource,
                                 request: Request):
    """Validate the querystring if this is a GET request, the body otherwise.

    This allows using just a single schema for all kinds of requests.
    """
    if isinstance(schema, GETPoolRequestSchema):
        schema = add_get_pool_request_extra_fields(qs, schema, context,
                                                   request.registry)
    if request.method.upper() == 'GET':
        _validate_schema(qs, schema, request,
                         location='querystring')
    else:
        _validate_schema(body, schema, request, location='body')


def _validate_schema(cstruct: object, schema: MappingSchema, request: Request,
                     location='body'):
    """Validate that the :term:`cstruct` data is conform to the given schema.

    :param request: request with list like `errors` attribute to append errors
                    and the dictionary attribute `validated` to add validated
                    data.
    :param location: filter schema nodes depending on the `location` attribute.
                     The default value is `body`.
    """
    if isinstance(schema, SequenceSchema):
        _validate_list_schema(schema, cstruct, request, location)
    elif isinstance(schema, MappingSchema):
        _validate_dict_schema(schema, cstruct, request, location)
    else:
        error = 'Validation for schema {} is unsupported.'.format(str(schema))
        raise(Exception(error))


def _validate_list_schema(schema: SequenceSchema, cstruct: list,
                          request: Request, location='body'):
    if location != 'body':  # for now we only support location == body
        return
    child_cstructs = schema.cstruct_children(cstruct)
    try:
        request.validated = schema.deserialize(child_cstructs)
    except Invalid as err:
        _add_colander_invalid_error_to_request(err, request, location)


def _validate_dict_schema(schema: MappingSchema, cstruct: dict,
                          request: Request, location='body'):
    validated = {}
    nodes_with_cstruct = [n for n in schema if n.name in cstruct]
    nodes_without_cstruct = [n for n in schema if n.name not in cstruct]

    for node in nodes_without_cstruct:
        appstruct = node.deserialize()
        if appstruct is not drop:
            validated[node.name] = appstruct
    for node in nodes_with_cstruct:
        node_cstruct = cstruct[node.name]
        try:
            validated[node.name] = node.deserialize(node_cstruct)
        except Invalid as err:
            _add_colander_invalid_error_to_request(err, request, location)
    if getattr(schema.typ, 'unknown', None) == 'preserve':
        # Schema asks us to preserve other cstruct values
        for name, value in cstruct.items():
            if name not in validated:
                validated[name] = value
    request.validated.update(validated)


def _add_colander_invalid_error_to_request(error: Invalid, request: Request,
                                           location: str):
    for name, msg in error.asdict().items():
        request.errors.add(location, name, msg)


def _validate_extra_validators(validators: list, context, request: Request):
    """Run `validators` functions. Assuming schema validation run before."""
    if request.errors:
        return
    for val in validators:
        val(context, request)


def _raise_if_errors(request: Request):
    """Raise :class:`cornice.errors._JSONError` and log if request.errors."""
    if not request.errors:
        return
    logger.warning('Found %i validation errors in request: <%s>',
                   len(request.errors), request.body)
    for error in request.errors:
        logger.warning('  %s', error)
    request.validated = {}
    raise json_error(request.errors)


class RESTView:

    """Class stub with request data validation support.

    Subclasses must implement the wanted request methods
    and configure the pyramid view::

        @view_defaults(
            renderer='simplejson',
            context=IResource,
            http_cache=0,
        )
        class MySubClass(RESTView):
            validation_GET = (MyColanderSchema, [my_extra_validation_function])

            @view_config(request_method='GET')
            def get(self):
            ...
    """

    validation_OPTIONS = (None, [])
    validation_HEAD = (None, [])
    validation_GET = (None, [])
    validation_PUT = (None, [])
    validation_POST = (None, [])

    def __init__(self, context, request):
        self.context = context
        """Context Resource."""
        self.request = request
        """:class:`pyramid.request.Request`."""
        schema_class, validators = _get_schema_and_validators(self, request)
        validate_request_data(context, request,
                              schema=schema_class(),
                              extra_validators=validators)

    def options(self) -> dict:
        raise HTTPMethodNotAllowed()

    def get(self) -> dict:
        raise HTTPMethodNotAllowed()

    def put(self) -> dict:
        raise HTTPMethodNotAllowed()

    def post(self) -> dict:
        raise HTTPMethodNotAllowed()

    def delete(self) -> dict:
        raise HTTPMethodNotAllowed()


def _get_schema_and_validators(view_class, request: Request) -> tuple:
    http_method = request.method.upper()
    validation_attr = 'validation_' + http_method
    schema, validators = getattr(view_class, validation_attr, (None, []))
    return schema or MappingSchema, validators


@view_defaults(
    renderer='simplejson',
    context=IResource,
    http_cache=0,
)
class ResourceRESTView(RESTView):

    """Default view for Resources, implements get and options."""

    def __init__(self, context, request):
        super().__init__(context, request)
        self.registry = request.registry.content
        """:class:`pyramid.registry.Registry`."""

    @view_config(request_method='OPTIONS')
    def options(self) -> dict:
        """Get possible request/response data structures and http methods."""
        addables = self.registry.resource_addables(self.context, self.request)
        sheets_view = self.registry.resource_sheets(self.context, self.request,
                                                    onlyviewable=True)
        sheets_edit = self.registry.resource_sheets(self.context, self.request,
                                                    onlyeditable=True)

        cstruct_singleton = OPTIONResourceResponseSchema().serialize()
        cstruct = deepcopy(cstruct_singleton)
        for sheet in sheets_edit:
            cstruct['PUT']['request_body']['data'][sheet] = {}
            cstruct['PUT']['request_body']['content_type'] = ''
            cstruct['PUT']['response_body']['content_type'] = ''
        cstruct['PUT']['response_body']['content_type'] = ''
        for sheet in sheets_view:
            cstruct['GET']['response_body']['data'][sheet] = {}
            cstruct['GET']['response_body']['content_type'] = ''
        for type, sheets in addables.items():
            names = sheets['sheets_optional'] + sheets['sheets_mandatory']
            sheets_dict = dict([(s, {}) for s in names])
            post_data = {'content_type': type, 'data': sheets_dict}
            cstruct['POST']['request_body'].append(post_data)
            cstruct['POST']['response_body']['content_type'] = ''
        cstruct['POST']['response_body']['content_type'] = ''
        return cstruct

    @view_config(request_method='GET')
    def get(self) -> dict:
        """Get resource data."""
        schema = GETResourceResponseSchema().bind(request=self.request)
        appstruct = _get_resource_response_appstruct(self.context)
        cstruct = schema.serialize(appstruct)
        cstruct['data'] = self._get_sheets_data_cstruct()
        return cstruct
{'fraction_non_alphanumeric': 0.08656984334203656, 'fraction_numerical': 0.0004079634464751958, 'mean_word_length': 2.9562621045836024, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1351260', 'n_tokens_mistral': 6504, 'n_tokens_neox': 6274, 'n_words': 1796}