<commit_before>import copy
import datetime
import logging

from google.appengine.api import lib_config
from google.appengine.ext.db import BadRequestError, BadValueError

from agar.models import ModelException

from pytz.gae import pytz

from restler.serializers import json_response as restler_json_response

from webapp2 import RequestHandler, HTTPException


class ConfigDefaults(object):
    """Configurable constants.

    To override agar.json configuration values, define values like this
    in your appengine_config.py file (in the root of your app):

        agar_json_DEBUG = True
        agar_json_DEFAULT_PAGE_SIZE = 20
        agar_json_MAX_PAGE_SIZE = 200
    """
    DEBUG = False
    DEFAULT_PAGE_SIZE = 10
    MAX_PAGE_SIZE = 100
    USE_DATA_ROOT_NODE = True

config = lib_config.register('agar_json', ConfigDefaults.__dict__)


def string_to_int(s, default=10):
    try:
        return int(s)
    except:
        return default

class JsonRequestHandler(RequestHandler):
    """A RequestHandler class to help with json web service handlers, including error handling"""
    def _setup_context(self, context):
        if not context:
            context = {}
        context['request'] = self.request
        return context
    
    def _setup_data(self, model_or_query, status_code, status_text):
        data = dict()
        data['status_code'] = status_code
        data['status_text'] = status_text
        data['timestamp'] = datetime.datetime.now(pytz.utc)
        if config.USE_DATA_ROOT_NODE:
            data['data'] = model_or_query
        else:    
            data.update(model_or_query)
        return data

    def json_response(self, model_or_query, strategy=None, status_code=200, status_text='OK', context=None):
        context = self._setup_context(context)
        data = self._setup_data(model_or_query, status_code, status_text)

        return restler_json_response(self.response, data, strategy=strategy, status_code=status_code, context=context)

    def handle_exception(self, exception, debug_mode):
        status_text = exception.message
        if isinstance(exception, HTTPException):
            code = exception.code
        elif isinstance(exception, ModelException):
            code = 400
            status_text = "BAD_REQUEST: %s" % status_text
        else:
            code = 500
            status_text = "INTERNAL_SERVER_ERROR: %s" % status_text
            logging.error("API 500 ERROR: %s" % exception)
        if code == 401:
            status_text = 'UNAUTHORIZED'
        if code == 403:
            status_text = 'FORBIDDEN'
        if code == 404:
            status_text = 'NOT_FOUND'
        if code == 405:
            status_text = 'METHOD_NOT_ALLOWED'
        if code == 400:
            logging.warning("API %s WARNING: %s" % (code, status_text))
        return self.json_response({}, status_code=code, status_text=status_text)

class MultiPageHandler(JsonRequestHandler):
    """A RequestHandler class to help with 'page_size' and 'cursor' parsing and logic"""
    @property
    def page_size(self):
        page_size = string_to_int(self.request.get('page_size', str(config.DEFAULT_PAGE_SIZE)))
        page_size = min(max(page_size, 1), config.MAX_PAGE_SIZE)
        return page_size

    def fetch_page(self, query):
        original_query = copy.deepcopy(query)
        cursor = self.request.get('cursor', None)
        if cursor is not None:
            try:
                query = query.with_cursor(cursor)
            except (BadValueError, BadRequestError):
                query = original_query
        try:
            results = query.fetch(self.page_size)
        except (BadValueError, BadRequestError):
#            self.abort(500, "BAD_REQUEST: The cursor has expired (%s)" % cursor)
            if cursor is not None:
                query = original_query
                results = query.fetch(self.page_size)
            else:
                raise
        next_page_key = None
        if len(results) == self.page_size:<commit_msg>reverted to previous version of MultiPageHandler, removed cursor workaround<commit_after>import copy
import datetime
import logging

from google.appengine.api import lib_config
from google.appengine.ext.db import BadRequestError, BadValueError

from agar.models import ModelException

from pytz.gae import pytz

from restler.serializers import json_response as restler_json_response

from webapp2 import RequestHandler, HTTPException


class ConfigDefaults(object):
    """Configurable constants.

    To override agar.json configuration values, define values like this
    in your appengine_config.py file (in the root of your app):

        agar_json_DEBUG = True
        agar_json_DEFAULT_PAGE_SIZE = 20
        agar_json_MAX_PAGE_SIZE = 200
    """
    DEBUG = False
    DEFAULT_PAGE_SIZE = 10
    MAX_PAGE_SIZE = 100
    USE_DATA_ROOT_NODE = True

config = lib_config.register('agar_json', ConfigDefaults.__dict__)


def string_to_int(s, default=10):
    try:
        return int(s)
    except:
        return default

class JsonRequestHandler(RequestHandler):
    """A RequestHandler class to help with json web service handlers, including error handling"""
    def _setup_context(self, context):
        if not context:
            context = {}
        context['request'] = self.request
        return context
    
    def _setup_data(self, model_or_query, status_code, status_text):
        data = dict()
        data['status_code'] = status_code
        data['status_text'] = status_text
        data['timestamp'] = datetime.datetime.now(pytz.utc)
        if config.USE_DATA_ROOT_NODE:
            data['data'] = model_or_query
        else:    
            data.update(model_or_query)
        return data

    def json_response(self, model_or_query, strategy=None, status_code=200, status_text='OK', context=None):
        context = self._setup_context(context)
        data = self._setup_data(model_or_query, status_code, status_text)

        return restler_json_response(self.response, data, strategy=strategy, status_code=status_code, context=context)

    def handle_exception(self, exception, debug_mode):
        status_text = exception.message
        if isinstance(exception, HTTPException):
            code = exception.code
        elif isinstance(exception, ModelException):
            code = 400
            status_text = "BAD_REQUEST: %s" % status_text
        else:
            code = 500
            status_text = "INTERNAL_SERVER_ERROR: %s" % status_text
            logging.error("API 500 ERROR: %s" % exception)
        if code == 401:
            status_text = 'UNAUTHORIZED'
        if code == 403:
            status_text = 'FORBIDDEN'
        if code == 404:
            status_text = 'NOT_FOUND'
        if code == 405:
            status_text = 'METHOD_NOT_ALLOWED'
        if code == 400:
            logging.warning("API %s WARNING: %s" % (code, status_text))
        return self.json_response({}, status_code=code, status_text=status_text)

class MultiPageHandler(JsonRequestHandler):
    """A RequestHandler class to help with 'page_size' and 'cursor' parsing and logic"""
    @property
    def page_size(self):
        page_size = string_to_int(self.request.get('page_size', str(config.DEFAULT_PAGE_SIZE)))
        page_size = min(max(page_size, 1), config.MAX_PAGE_SIZE)
        return page_size

    def fetch_page(self, query):
        cursor = self.request.get('cursor', None)
        if cursor is not None:
            try:
                query = query.with_cursor(cursor)
            except BadRequestError:
                self.abort(400, "The cursor has expired: %s" % cursor)
        results = query.fetch(self.page_size)
        next_page_key = None
        if len(results) == self.page_size:
{'fraction_non_alphanumeric': 0.08480247072448849, 'fraction_numerical': 0.011324153905546261, 'mean_word_length': 2.72757793764988, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2707817', 'n_tokens_mistral': 2302, 'n_tokens_neox': 2177, 'n_words': 568}