<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import logging
import httplib
import datetime
import email.utils
import hashlib
import mimetypes
import os.path
import stat
import time

import tornado.web
import tornado.locale
from tornado.web import HTTPError
from tornado.escape import utf8, json_encode, json_decode

from torext import settings
from torext.app import TorextApp
from torext.utils import raise_exc_info


def log_response(handler):
    """
    Acturally, logging response is not a server's responsibility,
    you should use http tools like Chrome Developer Tools to analyse the response.

    Although this function and its setting(LOG_RESPONSE) is not recommended to use,
    if you are laze as I was and working in development, nothing could stop you.
    """
    content_type = handler._headers.get('Content-Type', None)
    headers_str = handler._generate_headers()
    block = 'Response Infomations:\n' + headers_str.strip()

    if content_type and ('text' in content_type or 'json' in content_type):
        limit = 0
        if 'LOG_RESPONSE_LINE_LIMIT' in settings:
            limit = settings['LOG_RESPONSE_LINE_LIMIT']

        def cut(s):
            if limit and len(s) > limit:
                return [s[:limit]] + cut(s[limit:])
            else:
                return [s]

        body = ''.join(handler._write_buffer)
        lines = []
        for i in body.split('\n'):
            lines += ['| ' + j for j in cut(i)]
        block += '\nBody:\n' + '\n'.join(lines)
    logging.info(block)


def log_request(handler):
    """
    Logging request is opposite to response, sometime its necessary,
    feel free to enable it.
    """
    block = 'Request Infomations:\n' + _format_headers_log(handler.request.headers)

    if handler.request.arguments:
        block += '+----Arguments----+\n'
        for k, v in handler.request.arguments.iteritems():
            block += '| {0:<15} | {1:<15} \n'.format(repr(k), repr(v))

    logging.info(block)


def _format_headers_log(headers):
    # length of '+-...-+' is 19
    block = '+-----Headers-----+\n'
    for k, v in headers.iteritems():
        block += '| {0:<15} | {1:<15} \n'.format(k, v)
    return block


class BaseHandler(tornado.web.RequestHandler):
    """
    Request
        header:
        body:

    Response
        status code: 200(成功), 400(参数异常), 403(不成功), 404(web,未找到), 500(服务器异常)
        header:
        body:
    """
    EXCEPTION_HANDLERS = None

    PREPARES = []

    def _exception_default_handler(self, e):
        """This method is a copy of tornado.web.RequestHandler._handle_request_exception
        """
        if isinstance(e, HTTPError):
            if e.log_message:
                format = "%d %s: " + e.log_message
                args = [e.status_code, self._request_summary()] + list(e.args)
                logging.warning(format, *args)
            if e.status_code not in httplib.responses:
                logging.error("Bad HTTP status code: %d", e.status_code)
                self.send_error(500, exc_info=sys.exc_info())
            else:
                self.send_error(e.status_code, exc_info=sys.exc_info())
        else:
            logging.error("Uncaught exception %s\n%r", self._request_summary(),
                          self.request, exc_info=True)
            self.send_error(500, exc_info=sys.exc_info())

    def _handle_request_exception(self, e):
        """This method handle HTTPError exceptions the same as how tornado does,
        leave other exceptions to be handled by user defined handler function
        maped in class attribute `EXCEPTION_HANDLERS`

        Common HTTP status codes:
            200 OK
            301 Moved Permanently
            302 Found
            400 Bad Request
            401 Unauthorized
            403 Forbidden
            404 Not Found
            405 Method Not Allowed
            500 Internal Server Error

        It is suggested only to use above HTTP status codes
        """
        handle_func = self._exception_default_handler
        if self.EXCEPTION_HANDLERS:
            for excs, func_name in self.EXCEPTION_HANDLERS.iteritems():
                if isinstance(e, excs):
                    handle_func = getattr(self, func_name)
                    break

        handle_func(e)
        if not self._finished:
            self.finish()

    @property
    def app(self):
        return TorextApp.current_app

    @property
    def db(self):
        """Rewrite this method to implement the default database connection object,
        eg. a sqlalchemy session

        Other names like ``mongodb``, ``redis``, ``memcache`` could also be used in this way.
        """
        raise NotImplementedError

    @property
    def json_decode(self):
        return json_decode

    @property
    def json_encode(self):
        return json_encode

    def flush(self, *args, **kwgs):
        """
        Before `RequestHandler.flush` was called, we got the final _write_buffer.

        This method will not be called in wsgi mode
        """
        if settings['LOG_RESPONSE'] and not self._status_code == 500:
            log_response(self)
<commit_msg>update how handler return json encoder & decoder<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import logging
import httplib
import datetime
import email.utils
import hashlib
import mimetypes
import os.path
import stat
import time

import tornado.web
import tornado.locale
from tornado.web import HTTPError
from tornado.escape import utf8

from torext import settings
from torext.app import TorextApp
from torext.utils import raise_exc_info


def log_response(handler):
    """
    Acturally, logging response is not a server's responsibility,
    you should use http tools like Chrome Developer Tools to analyse the response.

    Although this function and its setting(LOG_RESPONSE) is not recommended to use,
    if you are laze as I was and working in development, nothing could stop you.
    """
    content_type = handler._headers.get('Content-Type', None)
    headers_str = handler._generate_headers()
    block = 'Response Infomations:\n' + headers_str.strip()

    if content_type and ('text' in content_type or 'json' in content_type):
        limit = 0
        if 'LOG_RESPONSE_LINE_LIMIT' in settings:
            limit = settings['LOG_RESPONSE_LINE_LIMIT']

        def cut(s):
            if limit and len(s) > limit:
                return [s[:limit]] + cut(s[limit:])
            else:
                return [s]

        body = ''.join(handler._write_buffer)
        lines = []
        for i in body.split('\n'):
            lines += ['| ' + j for j in cut(i)]
        block += '\nBody:\n' + '\n'.join(lines)
    logging.info(block)


def log_request(handler):
    """
    Logging request is opposite to response, sometime its necessary,
    feel free to enable it.
    """
    block = 'Request Infomations:\n' + _format_headers_log(handler.request.headers)

    if handler.request.arguments:
        block += '+----Arguments----+\n'
        for k, v in handler.request.arguments.iteritems():
            block += '| {0:<15} | {1:<15} \n'.format(repr(k), repr(v))

    logging.info(block)


def _format_headers_log(headers):
    # length of '+-...-+' is 19
    block = '+-----Headers-----+\n'
    for k, v in headers.iteritems():
        block += '| {0:<15} | {1:<15} \n'.format(k, v)
    return block


class BaseHandler(tornado.web.RequestHandler):
    """
    Request
        header:
        body:

    Response
        status code: 200(成功), 400(参数异常), 403(不成功), 404(web,未找到), 500(服务器异常)
        header:
        body:
    """
    EXCEPTION_HANDLERS = None

    PREPARES = []

    def _exception_default_handler(self, e):
        """This method is a copy of tornado.web.RequestHandler._handle_request_exception
        """
        if isinstance(e, HTTPError):
            if e.log_message:
                format = "%d %s: " + e.log_message
                args = [e.status_code, self._request_summary()] + list(e.args)
                logging.warning(format, *args)
            if e.status_code not in httplib.responses:
                logging.error("Bad HTTP status code: %d", e.status_code)
                self.send_error(500, exc_info=sys.exc_info())
            else:
                self.send_error(e.status_code, exc_info=sys.exc_info())
        else:
            logging.error("Uncaught exception %s\n%r", self._request_summary(),
                          self.request, exc_info=True)
            self.send_error(500, exc_info=sys.exc_info())

    def _handle_request_exception(self, e):
        """This method handle HTTPError exceptions the same as how tornado does,
        leave other exceptions to be handled by user defined handler function
        maped in class attribute `EXCEPTION_HANDLERS`

        Common HTTP status codes:
            200 OK
            301 Moved Permanently
            302 Found
            400 Bad Request
            401 Unauthorized
            403 Forbidden
            404 Not Found
            405 Method Not Allowed
            500 Internal Server Error

        It is suggested only to use above HTTP status codes
        """
        handle_func = self._exception_default_handler
        if self.EXCEPTION_HANDLERS:
            for excs, func_name in self.EXCEPTION_HANDLERS.iteritems():
                if isinstance(e, excs):
                    handle_func = getattr(self, func_name)
                    break

        handle_func(e)
        if not self._finished:
            self.finish()

    @property
    def app(self):
        return TorextApp.current_app

    @property
    def db(self):
        """Rewrite this method to implement the default database connection object,
        eg. a sqlalchemy session

        Other names like ``mongodb``, ``redis``, ``memcache`` could also be used in this way.
        """
        raise NotImplementedError

    @property
    def json_decode(self):
        return self.app.json_decoder

    @property
    def json_encode(self):
        return self.app.json_encoder

    def flush(self, *args, **kwgs):
        """
        Before `RequestHandler.flush` was called, we got the final _write_buffer.

        This method will not be called in wsgi mode
        """
        if settings['LOG_RESPONSE'] and not self._status_code == 500:
            log_response(self)

{'fraction_non_alphanumeric': 0.09968217278243283, 'fraction_numerical': 0.013098333814889724, 'mean_word_length': 2.2208436724565757, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1516525', 'n_tokens_mistral': 3147, 'n_tokens_neox': 2959, 'n_words': 894}