<commit_before>import json

from base64 import urlsafe_b64encode, urlsafe_b64decode
from functools import wraps
from urllib import quote
import logging

from flask import Response, request

from flask.ext.sqlalchemy import get_debug_queries

from flask.ext.restful import Resource

from changes.api.serializer import serialize as serialize_func
from changes.config import db
from changes.config import statsreporter

from time import time

LINK_HEADER = '<{uri}&page={page}>; rel="{name}"'


def _as_json(context):
    try:
        return json.dumps(context)
    except TypeError:
        logging.warning(
            "unable to json-encode api response. Was the data not serialized?")
        return json.dumps(serialize_func(context))


def error(message, problems=None, http_code=400):
    """ Returns a new error response to send API clients.

    :param message: A human readable description of the error
    :param problems: List of fields that caused the error.
    :param http_code: The HTTP code to use for the response.
    """
    error_response = {'error': message}
    if problems:
        error_response['problems'] = problems
    return error_response, http_code


def param(key, validator=lambda x: x, required=True, dest=None):
    def wrapped(func):
        @wraps(func)
        def _wrapped(*args, **kwargs):
            if key in kwargs:
                value = kwargs.pop(key, '')
            elif request.method == 'POST':
                value = request.form.get(key) or ''
            else:
                value = ''

            dest_key = str(dest or key)

            value = value.strip()
            if not value:
                if required:
                    raise ParamError(key, 'value is required')
                return func(*args, **kwargs)

            try:
                value = validator(value)
            except ParamError:
                raise
            except Exception:
                raise ParamError(key, 'invalid value')

            kwargs[dest_key] = value

            return func(*args, **kwargs)

        return _wrapped
    return wrapped


class APIError(Exception):
    pass


class ParamError(APIError):

    def __init__(self, key, msg):
        self.key = key
        self.msg = msg

    def __unicode__(self):
        return '{0} is not valid: {1}'.format(self.key, self.msg)


class APIView(Resource):

    def __init__(self, *args, **kwargs):
        super(APIView, self).__init__(*args, **kwargs)

        self.start_time = 0  # used for logging performance stats

    def dispatch_request(self, *args, **kwargs):

        self.start_time = time()

        try:
            response = super(APIView, self).dispatch_request(*args, **kwargs)
        except Exception:
            db.session.rollback()
            raise
        else:
            db.session.commit()
        return response

    def paginate(self, queryset, max_per_page=100, **kwargs):
        page = int(request.args.get('page', 1))
        per_page = int(request.args.get('per_page', 25) or 0)
        if max_per_page:
            assert per_page <= max_per_page
        assert page > 0

        if per_page:
            offset = (page - 1) * per_page
            result = list(queryset[offset:offset + per_page + 1])
        else:
            page = 1
            result = list(queryset)
<commit_msg>Warn/return an error rather than asserting for paginate errors<commit_after>import json

from base64 import urlsafe_b64encode, urlsafe_b64decode
from functools import wraps
from urllib import quote
import logging

from flask import Response, request, current_app

from flask.ext.sqlalchemy import get_debug_queries

from flask.ext.restful import Resource

from changes.api.serializer import serialize as serialize_func
from changes.config import db
from changes.config import statsreporter

from time import time

LINK_HEADER = '<{uri}&page={page}>; rel="{name}"'


def _as_json(context):
    try:
        return json.dumps(context)
    except TypeError:
        logging.warning(
            "unable to json-encode api response. Was the data not serialized?")
        return json.dumps(serialize_func(context))


def error(message, problems=None, http_code=400):
    """ Returns a new error response to send API clients.

    :param message: A human readable description of the error
    :param problems: List of fields that caused the error.
    :param http_code: The HTTP code to use for the response.
    """
    error_response = {'error': message}
    if problems:
        error_response['problems'] = problems
    return error_response, http_code


def param(key, validator=lambda x: x, required=True, dest=None):
    def wrapped(func):
        @wraps(func)
        def _wrapped(*args, **kwargs):
            if key in kwargs:
                value = kwargs.pop(key, '')
            elif request.method == 'POST':
                value = request.form.get(key) or ''
            else:
                value = ''

            dest_key = str(dest or key)

            value = value.strip()
            if not value:
                if required:
                    raise ParamError(key, 'value is required')
                return func(*args, **kwargs)

            try:
                value = validator(value)
            except ParamError:
                raise
            except Exception:
                raise ParamError(key, 'invalid value')

            kwargs[dest_key] = value

            return func(*args, **kwargs)

        return _wrapped
    return wrapped


class APIError(Exception):
    pass


class ParamError(APIError):

    def __init__(self, key, msg):
        self.key = key
        self.msg = msg

    def __unicode__(self):
        return '{0} is not valid: {1}'.format(self.key, self.msg)


class APIView(Resource):

    def __init__(self, *args, **kwargs):
        super(APIView, self).__init__(*args, **kwargs)

        self.start_time = 0  # used for logging performance stats

    def dispatch_request(self, *args, **kwargs):

        self.start_time = time()

        try:
            response = super(APIView, self).dispatch_request(*args, **kwargs)
        except Exception:
            db.session.rollback()
            raise
        else:
            db.session.commit()
        return response

    def paginate(self, queryset, max_per_page=100, **kwargs):
        page = int(request.args.get('page', 1))
        per_page = int(request.args.get('per_page', 25) or 0)
        if max_per_page and per_page > max_per_page:
            msg = 'Requested more per_page than the maximum'
            current_app.logger.warning(msg)
            return error(msg, http_code=400)
        if page <= 0:
            msg = 'Page argument must be at least 1'
            current_app.logger.warning(msg)
            return error(msg, http_code=400)

        if per_page:
            offset = (page - 1) * per_page
            result = list(queryset[offset:offset + per_page + 1])
        else:
            page = 1
            result = list(queryset)

{'fraction_non_alphanumeric': 0.09335803876852908, 'fraction_numerical': 0.0075541619156214365, 'mean_word_length': 2.1982680036463083, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3262616', 'n_tokens_mistral': 2031, 'n_tokens_neox': 1931, 'n_words': 596}