<commit_before>
import logging
import sys
import urllib
from threading import Lock
from typing import Any, Callable, Dict, List, Optional

import tornado.web
from django import http
from django.conf import settings
from django.core import exceptions, signals
from django.urls import resolvers
from django.core.exceptions import MiddlewareNotUsed
from django.core.handlers import base
from django.core.handlers.exception import convert_exception_to_response
from django.core.handlers.wsgi import WSGIRequest, get_script_name
from django.urls import set_script_prefix, set_urlconf
from django.http import HttpRequest, HttpResponse
from django.utils.module_loading import import_string
from tornado.wsgi import WSGIContainer

from zerver.decorator import RespondAsynchronously
from zerver.lib.response import json_response
from zerver.lib.types import ViewFuncT
from zerver.middleware import async_request_timer_restart, async_request_timer_stop
from zerver.tornado.descriptors import get_descriptor_by_handler_id

current_handler_id = 0
handlers = {}  # type: Dict[int, 'AsyncDjangoHandler']

def get_handler_by_id(handler_id: int) -> 'AsyncDjangoHandler':
    return handlers[handler_id]

def allocate_handler_id(handler: 'AsyncDjangoHandler') -> int:
    global current_handler_id
    handlers[current_handler_id] = handler
    handler.handler_id = current_handler_id
    current_handler_id += 1
    return handler.handler_id

def clear_handler_by_id(handler_id: int) -> None:
    del handlers[handler_id]

def handler_stats_string() -> str:
    return "%s handlers, latest ID %s" % (len(handlers), current_handler_id)

def finish_handler(handler_id: int, event_queue_id: str,
                   contents: List[Dict[str, Any]], apply_markdown: bool) -> None:
    err_msg = "Got error finishing handler for queue %s" % (event_queue_id,)
    try:
        # We call async_request_timer_restart here in case we are
        # being finished without any events (because another
        # get_events request has supplanted this request)
        handler = get_handler_by_id(handler_id)
        request = handler._request
        async_request_timer_restart(request)
        if len(contents) != 1:
            request._log_data['extra'] = "[%s/1]" % (event_queue_id,)
        else:
            request._log_data['extra'] = "[%s/1/%s]" % (event_queue_id, contents[0]["type"])

        handler.zulip_finish(dict(result='success', msg='',
                                  events=contents,
                                  queue_id=event_queue_id),
                             request, apply_markdown=apply_markdown)
    except IOError as e:
        if str(e) != 'Stream is closed':
            logging.exception(err_msg)
    except AssertionError as e:
        if str(e) != 'Request closed':
            logging.exception(err_msg)
    except Exception:
        logging.exception(err_msg)


# Modified version of the base Tornado handler for Django
# We mark this for nocoverage, since we only change 1 line of actual code.
class AsyncDjangoHandlerBase(tornado.web.RequestHandler, base.BaseHandler):  # nocoverage
    initLock = Lock()

    def __init__(self, *args: Any, **kwargs: Any) -> None:
        super().__init__(*args, **kwargs)

        # Set up middleware if needed. We couldn't do this earlier, because
        # settings weren't available.
        self._request_middleware = None  # type: Optional[List[Callable[[HttpRequest], HttpResponse]]]
        self.initLock.acquire()
        # Check that middleware is still uninitialised.
        if self._request_middleware is None:
            self.load_middleware()
        self.initLock.release()
        self._auto_finish = False
        # Handler IDs are allocated here, and the handler ID map must
        # be cleared when the handler finishes its response
        allocate_handler_id(self)

    def __repr__(self) -> str:
        descriptor = get_descriptor_by_handler_id(self.handler_id)
        return "AsyncDjangoHandler<%s, %s>" % (self.handler_id, descriptor)

    def load_middleware(self) -> None:
        """
        Populate middleware lists from settings.MIDDLEWARE. This is copied
        from Django. This uses settings.MIDDLEWARE setting with the old
        business logic. The middleware architecture is not compatible
        with our asynchronous handlers. The problem occurs when we return
        None from our handler. The Django middlewares throw exception
        because they can't handler None, so we can either upgrade the Django
        middlewares or just override this method to use the new setting with
        the old logic. The added advantage is that due to this our event
        system code doesn't change.
        """
        self._request_middleware = []  # type: Optional[List[Callable[[HttpRequest], HttpResponse]]]
        self._view_middleware = []  # type: List[Callable[[HttpRequest, ViewFuncT, List[str], Dict[str, Any]], Optional[HttpResponse]]]
        self._template_response_middleware = []  # type: List[Callable[[HttpRequest, HttpResponse], HttpResponse]]
        self._response_middleware = []  # type: List[Callable[[HttpRequest, HttpResponse], HttpResponse]]
        self._exception_middleware = []  # type: List[Callable[[HttpRequest, Exception], Optional[HttpResponse]]]

        handler = convert_exception_to_response(self._legacy_get_response)
        for middleware_path in settings.MIDDLEWARE:
            mw_class = import_string(middleware_path)
            try:<commit_msg>handlers: Remove duplicate type annotation.<commit_after>
import logging
import sys
import urllib
from threading import Lock
from typing import Any, Callable, Dict, List, Optional

import tornado.web
from django import http
from django.conf import settings
from django.core import exceptions, signals
from django.urls import resolvers
from django.core.exceptions import MiddlewareNotUsed
from django.core.handlers import base
from django.core.handlers.exception import convert_exception_to_response
from django.core.handlers.wsgi import WSGIRequest, get_script_name
from django.urls import set_script_prefix, set_urlconf
from django.http import HttpRequest, HttpResponse
from django.utils.module_loading import import_string
from tornado.wsgi import WSGIContainer

from zerver.decorator import RespondAsynchronously
from zerver.lib.response import json_response
from zerver.lib.types import ViewFuncT
from zerver.middleware import async_request_timer_restart, async_request_timer_stop
from zerver.tornado.descriptors import get_descriptor_by_handler_id

current_handler_id = 0
handlers = {}  # type: Dict[int, 'AsyncDjangoHandler']

def get_handler_by_id(handler_id: int) -> 'AsyncDjangoHandler':
    return handlers[handler_id]

def allocate_handler_id(handler: 'AsyncDjangoHandler') -> int:
    global current_handler_id
    handlers[current_handler_id] = handler
    handler.handler_id = current_handler_id
    current_handler_id += 1
    return handler.handler_id

def clear_handler_by_id(handler_id: int) -> None:
    del handlers[handler_id]

def handler_stats_string() -> str:
    return "%s handlers, latest ID %s" % (len(handlers), current_handler_id)

def finish_handler(handler_id: int, event_queue_id: str,
                   contents: List[Dict[str, Any]], apply_markdown: bool) -> None:
    err_msg = "Got error finishing handler for queue %s" % (event_queue_id,)
    try:
        # We call async_request_timer_restart here in case we are
        # being finished without any events (because another
        # get_events request has supplanted this request)
        handler = get_handler_by_id(handler_id)
        request = handler._request
        async_request_timer_restart(request)
        if len(contents) != 1:
            request._log_data['extra'] = "[%s/1]" % (event_queue_id,)
        else:
            request._log_data['extra'] = "[%s/1/%s]" % (event_queue_id, contents[0]["type"])

        handler.zulip_finish(dict(result='success', msg='',
                                  events=contents,
                                  queue_id=event_queue_id),
                             request, apply_markdown=apply_markdown)
    except IOError as e:
        if str(e) != 'Stream is closed':
            logging.exception(err_msg)
    except AssertionError as e:
        if str(e) != 'Request closed':
            logging.exception(err_msg)
    except Exception:
        logging.exception(err_msg)


# Modified version of the base Tornado handler for Django
# We mark this for nocoverage, since we only change 1 line of actual code.
class AsyncDjangoHandlerBase(tornado.web.RequestHandler, base.BaseHandler):  # nocoverage
    initLock = Lock()

    def __init__(self, *args: Any, **kwargs: Any) -> None:
        super().__init__(*args, **kwargs)

        # Set up middleware if needed. We couldn't do this earlier, because
        # settings weren't available.
        self._request_middleware = None  # type: Optional[List[Callable[[HttpRequest], HttpResponse]]]
        self.initLock.acquire()
        # Check that middleware is still uninitialised.
        if self._request_middleware is None:
            self.load_middleware()
        self.initLock.release()
        self._auto_finish = False
        # Handler IDs are allocated here, and the handler ID map must
        # be cleared when the handler finishes its response
        allocate_handler_id(self)

    def __repr__(self) -> str:
        descriptor = get_descriptor_by_handler_id(self.handler_id)
        return "AsyncDjangoHandler<%s, %s>" % (self.handler_id, descriptor)

    def load_middleware(self) -> None:
        """
        Populate middleware lists from settings.MIDDLEWARE. This is copied
        from Django. This uses settings.MIDDLEWARE setting with the old
        business logic. The middleware architecture is not compatible
        with our asynchronous handlers. The problem occurs when we return
        None from our handler. The Django middlewares throw exception
        because they can't handler None, so we can either upgrade the Django
        middlewares or just override this method to use the new setting with
        the old logic. The added advantage is that due to this our event
        system code doesn't change.
        """
        self._request_middleware = []
        self._view_middleware = []  # type: List[Callable[[HttpRequest, ViewFuncT, List[str], Dict[str, Any]], Optional[HttpResponse]]]
        self._template_response_middleware = []  # type: List[Callable[[HttpRequest, HttpResponse], HttpResponse]]
        self._response_middleware = []  # type: List[Callable[[HttpRequest, HttpResponse], HttpResponse]]
        self._exception_middleware = []  # type: List[Callable[[HttpRequest, Exception], Optional[HttpResponse]]]

        handler = convert_exception_to_response(self._legacy_get_response)
        for middleware_path in settings.MIDDLEWARE:
            mw_class = import_string(middleware_path)
            try:
{'fraction_non_alphanumeric': 0.08012791228871631, 'fraction_numerical': 0.0012791228871630882, 'mean_word_length': 3.426202992317024, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5187056', 'n_tokens_mistral': 3094, 'n_tokens_neox': 2986, 'n_words': 898}