<commit_before>import json
from urllib.parse import parse_qs, urlencode
from math import ceil
from functools import wraps

import aioes
from aiohttp import web
from marshmallow import Schema, fields
from marshmallow.validate import Range

from ..connections import elastic
from ..consts import ELASTICSEARCH_INDEX


class ListParams(Schema):
    page_size = fields.Integer(validate=Range(1, 50))
    page = fields.Integer(validate=Range(1))
    expand = fields.String()
    fields = fields.String()


class ItemParams(Schema):
    expand = fields.String()
    fields = fields.String()


def with_params(schema_cls):
    def decorator(f):
        @wraps(f)
        async def wrapper(request):
            schema = schema_cls(request)
            result = schema.load(request.GET)
            if result.errors:
                return web.HTTPBadRequest(
                    text=json.dumps({'errors': result.errors}, ensure_ascii=False),
                    content_type='application/json',
                )
            else:
                return await f(request, **result.data)
        return wrapper
    return decorator


def item_handler(type_, relations={}):
    def decorator(f):
        @wraps(f)
        @with_params(ItemParams)
        async def wrapper(request, fields=None, expand=None):
            expand = expand.split(',') if expand else []

            kwargs = {}
            if fields:
                kwargs['_source'] = fields

            id_ = await f(request)
            try:
                response = await elastic.get(
                    ELASTICSEARCH_INDEX, id_, type_, **kwargs)
            except aioes.NotFoundError:
                return web.HTTPNotFound()

            item = simplify_item(response)

            for related_field in expand:
                doc_type, subfields = relations[related_field]
                if not fields or related_field in fields:
                    item = (await expand_related_items(
                        [item], related_field, doc_type, subfields))[0]

            return web.Response(
                text=json.dumps(item, ensure_ascii=False),
                content_type='application/json',
            )
        return wrapper
    return decorator


def list_handler(type_=None, relations={}):
    def decorator(f):
        @wraps(f)
        @with_params(ListParams)
        async def wrapper(request, page=1, page_size=20, fields=None, expand=None):
            fields = fields.split(',') if fields else []
            expand = expand.split(',') if expand else []
<commit_msg>Move comma separating into a marshmallow field<commit_after>import json
from urllib.parse import parse_qs, urlencode
from math import ceil
from functools import wraps

import aioes
from aiohttp import web
from marshmallow import Schema, fields
from marshmallow.validate import Range

from ..connections import elastic
from ..consts import ELASTICSEARCH_INDEX


class CommaSeparatedList(fields.List):
    def _serialize(self, value, attr, obj):
        result = super()._serialize(value, attr, obj)
        return ','.join(result) if result is not None else None

    def _deserialize(self, value, attr, data):
        if not isinstance(value, str):
            self.fail('invalid')

        value = value.split(',')
        if any(item == '' for item in value):
            self.fail('invalid')
        return super()._deserialize(value, attr, data)


class ListParams(Schema):
    page_size = fields.Integer(validate=Range(1, 50))
    page = fields.Integer(validate=Range(1))
    expand = CommaSeparatedList(fields.String())
    fields = CommaSeparatedList(fields.String())


class ItemParams(Schema):
    expand = CommaSeparatedList(fields.String())
    fields = CommaSeparatedList(fields.String())


def with_params(schema_cls):
    def decorator(f):
        @wraps(f)
        async def wrapper(request):
            schema = schema_cls(request)
            result = schema.load(request.GET)
            if result.errors:
                return web.HTTPBadRequest(
                    text=json.dumps({'errors': result.errors}, ensure_ascii=False),
                    content_type='application/json',
                )
            else:
                return await f(request, **result.data)
        return wrapper
    return decorator


def item_handler(type_, relations={}):
    def decorator(f):
        @wraps(f)
        @with_params(ItemParams)
        async def wrapper(request, fields=(), expand=()):
            id_ = await f(request)

            kwargs = {}
            if fields:
                kwargs['_source'] = ','.join(fields)

            try:
                response = await elastic.get(
                    ELASTICSEARCH_INDEX, id_, type_, **kwargs)
            except aioes.NotFoundError:
                return web.HTTPNotFound()

            item = simplify_item(response)

            for related_field in expand:
                doc_type, subfields = relations[related_field]
                if not fields or related_field in fields:
                    item = (await expand_related_items(
                        [item], related_field, doc_type, subfields))[0]

            return web.Response(
                text=json.dumps(item, ensure_ascii=False),
                content_type='application/json',
            )
        return wrapper
    return decorator


def list_handler(type_=None, relations={}):
    def decorator(f):
        @wraps(f)
        @with_params(ListParams)
        async def wrapper(request, page=1, page_size=20, fields=(), expand=()):
{'fraction_non_alphanumeric': 0.09700685178507032, 'fraction_numerical': 0.00288496213487198, 'mean_word_length': 2.189764232317424, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4714732', 'n_tokens_mistral': 1551, 'n_tokens_neox': 1473, 'n_words': 388}