<commit_before>from django.conf import settings
from decorator import RequestVariableMissingError, RequestVariableConversionError
from zephyr.lib.response import json_error
from django.db import connection

import logging
import time

logger = logging.getLogger('humbug.requests')

class LogRequests(object):
    def process_request(self, request):
        request._time_started = time.time()

    def process_response(self, request, response):

        # The reverse proxy might have sent us the real external IP
        remote_ip = request.META.get('HTTP_X_REAL_IP')
        if remote_ip is None:
            remote_ip = request.META['REMOTE_ADDR']

        time_delta = -1
        # A time duration of -1 means the StartLogRequests middleware
        # didn't run for some reason
        if hasattr(request, '_time_started'):
            time_delta = time.time() - request._time_started

        # Get the amount of time spent doing database queries
        query_time = sum(float(query.get('time', 0)) for query in connection.queries)
        if settings.DEBUG:
            db_stats = "(db: %.3fs/%sq) " % (query_time, len(connection.queries),)
        else:
            db_stats = ""

        # Get the requestor's email address, if available.
        try:
            email = request._email
        except Exception:
            email = "unauth"

        logger.info('%-15s %-7s %3d %.3fs %s%s (%s)'
            % (remote_ip, request.method, response.status_code,
               time_delta, db_stats, request.get_full_path(), email))
        # Log some additional data whenever we return a 40x error
        if 400 <= response.status_code < 500:
            logger.info('status=%3d, data=%s, uid=%s' % (response.status_code, response.content, email))
        return response

class JsonErrorHandler(object):
    def process_exception(self, request, exception):
        if hasattr(exception, 'to_json_error_msg') and callable(exception.to_json_error_msg):
            return json_error(exception.to_json_error_msg())
        return None<commit_msg>Monkeypatch Django cursor to log query times even when DEBUG=False.<commit_after>from django.conf import settings
from decorator import RequestVariableMissingError, RequestVariableConversionError
from zephyr.lib.response import json_error
from django.db import connection

import logging
import time

logger = logging.getLogger('humbug.requests')

class LogRequests(object):
    def process_request(self, request):
        request._time_started = time.time()

    def process_response(self, request, response):

        # The reverse proxy might have sent us the real external IP
        remote_ip = request.META.get('HTTP_X_REAL_IP')
        if remote_ip is None:
            remote_ip = request.META['REMOTE_ADDR']

        time_delta = -1
        # A time duration of -1 means the StartLogRequests middleware
        # didn't run for some reason
        if hasattr(request, '_time_started'):
            time_delta = time.time() - request._time_started

        # Get the amount of time spent doing database queries
        query_time = sum(float(query.get('time', 0)) for query in connection.queries)

        # Get the requestor's email address, if available.
        try:
            email = request._email
        except Exception:
            email = "unauth"

        logger.info('%-15s %-7s %3d %.3fs (db: %.3fs/%sq) %s (%s)'
            % (remote_ip, request.method, response.status_code,
               time_delta, query_time, len(connection.queries),
               request.get_full_path(), email))

        # Log some additional data whenever we return a 40x error
        if 400 <= response.status_code < 500:
            logger.info('status=%3d, data=%s, uid=%s' % (response.status_code, response.content, email))
        return response

class JsonErrorHandler(object):
    def process_exception(self, request, exception):
        if hasattr(exception, 'to_json_error_msg') and callable(exception.to_json_error_msg):
            return json_error(exception.to_json_error_msg())
        return None

# Monkeypatch in time tracking to the Django non-debug cursor
# Code comes from CursorDebugWrapper
def wrapper_execute(self, action, sql, params=()):
    self.set_dirty()
    start = time.time()
    try:
        return action(sql, params)
    finally:
        stop = time.time()
        duration = stop - start
        self.db.queries.append({
                'time': "%.3f" % duration,
                })

from django.db.backends.util import CursorWrapper
def cursor_execute(self, sql, params=()):
    return wrapper_execute(self, self.cursor.execute, sql, params)
CursorWrapper.execute = cursor_execute

def cursor_executemany(self, sql, params=()):
    return wrapper_execute(self, self.cursor.executemany, sql, params)
CursorWrapper.executemany = cursor_executemany
{'fraction_non_alphanumeric': 0.09404712691194708, 'fraction_numerical': 0.007647788342290202, 'mean_word_length': 2.930950446791227, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2808910', 'n_tokens_mistral': 1389, 'n_tokens_neox': 1342, 'n_words': 391}