<commit_before>import asyncio
import collections
import json
import logging

import aiohttp
from aiohttp import web
import voluptuous

from . import adjust
from . import exception
from . import pull
from . import repo
from . import validation

logger = logging.getLogger(__name__)

def _validated_json_endpoint(validator, coro):
    @asyncio.coroutine
    def handler(request):
        spec = yield from request.json()
        try:
            validator(spec)
        except voluptuous.MultipleInvalid as x:
            return web.Response(
                status=400,
                content_type="application/json",
                text=json.dumps(
                    obj=[e.__dict__ for e in x.errors],
                    ensure_ascii=False,
                ),
            )

        try:
            ret = yield from coro(spec, **request.app)
        except exception.DescribedError as e:
            logger.exception(coro.__name__)
            error = {k: v for k, v in e.__dict__.items() if not k.startswith("_")}
            error["error_type"] = e.__class__.__name__
            return web.Response(
                status=400,
                content_type="application/json",
                text=json.dumps(
                    obj=error,
                    ensure_ascii=False,
                ),
            )
        except Exception as e:
            logger.exception(coro.__name__)
            return web.Response(
                status=500,
                content_type="application/json",
                text=json.dumps(
                    obj={
                        "error_type": e.__class__.__name__,
                    },
                    ensure_ascii=False,
                ),
            )

        return web.Response(
            status=200,
            content_type="application/json",
            text=json.dumps(
                obj=ret,
                ensure_ascii=False,<commit_msg>Add traceback identifier to error json, for easier log referencing<commit_after>import asyncio
import collections
import hashlib
import json
import logging
import traceback

import aiohttp
from aiohttp import web
import voluptuous

from . import adjust
from . import exception
from . import pull
from . import repo
from . import validation

logger = logging.getLogger(__name__)

def create_traceback_id():
    tb = traceback.format_exc()
    h = hashlib.md5()
    h.update(tb.encode("utf-8"))
    return h.hexdigest()

def _validated_json_endpoint(validator, coro):
    @asyncio.coroutine
    def handler(request):
        spec = yield from request.json()
        try:
            validator(spec)
        except voluptuous.MultipleInvalid as x:
            return web.Response(
                status=400,
                content_type="application/json",
                text=json.dumps(
                    obj=[e.__dict__ for e in x.errors],
                    ensure_ascii=False,
                ),
            )

        try:
            ret = yield from coro(spec, **request.app)
        except exception.DescribedError as e:
            traceback_id = create_traceback_id()
            logger.exception(traceback_id)
            error = {k: v for k, v in e.__dict__.items() if not k.startswith("_")}
            error["error_type"] = e.__class__.__name__
            error["error_traceback"] = traceback_id
            return web.Response(
                status=400,
                content_type="application/json",
                text=json.dumps(
                    obj=error,
                    ensure_ascii=False,
                ),
            )
        except Exception as e:
            traceback_id = create_traceback_id()
            logger.exception(traceback_id)
            return web.Response(
                status=500,
                content_type="application/json",
                text=json.dumps(
                    obj={
                        "error_type": e.__class__.__name__,
                        "error_traceback": traceback_id,
                    },
                    ensure_ascii=False,
                ),
            )

        return web.Response(
            status=200,
            content_type="application/json",
            text=json.dumps(
                obj=ret,
                ensure_ascii=False,
{'fraction_non_alphanumeric': 0.08111266947171575, 'fraction_numerical': 0.006077606358111267, 'mean_word_length': 1.4863451481696688, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4578271', 'n_tokens_mistral': 1145, 'n_tokens_neox': 1073, 'n_words': 253}