<commit_before>"""
This is a middleware layer which keeps a log of all requests made
to the server. It is responsible for removing security tokens and
similar from such events, and relaying them to the event tracking
framework.
"""


import hashlib
import hmac
import json
import logging
import re
import sys

from django.conf import settings

from track import views
from track import contexts
from eventtracking import tracker


log = logging.getLogger(__name__)

CONTEXT_NAME = 'edx.request'
META_KEY_TO_CONTEXT_KEY = {
    'REMOTE_ADDR': 'ip',
    'SERVER_NAME': 'host',
    'HTTP_USER_AGENT': 'agent',
    'PATH_INFO': 'path',
    # Not a typo. See:
    # http://en.wikipedia.org/wiki/HTTP_referer#Origin_of_the_term_referer
    'HTTP_REFERER': 'referer',
    'HTTP_ACCEPT_LANGUAGE': 'accept_language',
}


class TrackMiddleware(object):
    """
    Tracks all requests made, as well as setting up context for other server
    emitted events.
    """

    def process_request(self, request):
        try:
            self.enter_request_context(request)

            if not self.should_process_request(request):
                return

            # Removes passwords from the tracking logs
            # WARNING: This list needs to be changed whenever we change
            # password handling functionality.
            #
            # As of the time of this comment, only 'password' is used
            # The rest are there for future extension.
            #
            # Passwords should never be sent as GET requests, but
            # this can happen due to older browser bugs. We censor
            # this too.
            #
            # We should manually confirm no passwords make it into log
            # files when we change this.

            censored_strings = ['password', 'newpassword', 'new_password',
                                'oldpassword', 'old_password']
            post_dict = dict(request.POST)
            get_dict = dict(request.GET)
            for string in censored_strings:
                if string in post_dict:
                    post_dict[string] = '*' * 8
                if string in get_dict:
                    get_dict[string] = '*' * 8

            event = {
                'GET': dict(get_dict),
                'POST': dict(post_dict),
            }

            # TODO: Confirm no large file uploads
            event = json.dumps(event)
            event = event[:512]

            views.server_track(request, request.META['PATH_INFO'], event)
        except:
            ## Why do we have the overly broad except?
            ##
            ## I added instrumentation so if we drop events on the
            ## floor, we at least know about it. However, we really
            ## should just return a 500 here: (1) This will translate
            ## to much more insidious user-facing bugs if we make any
            ## decisions based on incorrect data.  (2) If the system
            ## is down, we should fail and fix it.
            event = {'event-type': 'exception', 'exception': repr(sys.exc_info()[0])}
            try:
                views.server_track(request, request.META['PATH_INFO'], event)
            except:
                # At this point, things are really broken. We really
                # should fail return a 500 to the user here.  However,
                # the interim decision is to just fail in order to be
                # consistent with current policy, and expedite the PR.
                # This version of the code makes no compromises
                # relative to the code before, while a proper failure
                # here would involve shifting compromises and
                # discussion.
                pass
<commit_msg>Try and remove events truncation<commit_after>"""
This is a middleware layer which keeps a log of all requests made
to the server. It is responsible for removing security tokens and
similar from such events, and relaying them to the event tracking
framework.
"""


import hashlib
import hmac
import json
import logging
import re
import sys

from django.conf import settings

from track import views
from track import contexts
from eventtracking import tracker


log = logging.getLogger(__name__)

CONTEXT_NAME = 'edx.request'
META_KEY_TO_CONTEXT_KEY = {
    'REMOTE_ADDR': 'ip',
    'SERVER_NAME': 'host',
    'HTTP_USER_AGENT': 'agent',
    'PATH_INFO': 'path',
    # Not a typo. See:
    # http://en.wikipedia.org/wiki/HTTP_referer#Origin_of_the_term_referer
    'HTTP_REFERER': 'referer',
    'HTTP_ACCEPT_LANGUAGE': 'accept_language',
}


class TrackMiddleware(object):
    """
    Tracks all requests made, as well as setting up context for other server
    emitted events.
    """

    def process_request(self, request):
        try:
            self.enter_request_context(request)

            if not self.should_process_request(request):
                return

            # Removes passwords from the tracking logs
            # WARNING: This list needs to be changed whenever we change
            # password handling functionality.
            #
            # As of the time of this comment, only 'password' is used
            # The rest are there for future extension.
            #
            # Passwords should never be sent as GET requests, but
            # this can happen due to older browser bugs. We censor
            # this too.
            #
            # We should manually confirm no passwords make it into log
            # files when we change this.

            censored_strings = ['password', 'newpassword', 'new_password',
                                'oldpassword', 'old_password']
            post_dict = dict(request.POST)
            get_dict = dict(request.GET)
            for string in censored_strings:
                if string in post_dict:
                    post_dict[string] = '*' * 8
                if string in get_dict:
                    get_dict[string] = '*' * 8

            event = {
                'GET': dict(get_dict),
                'POST': dict(post_dict),
            }

            # TODO: Confirm no large file uploads
            event = json.dumps(event)

            views.server_track(request, request.META['PATH_INFO'], event)
        except:
            ## Why do we have the overly broad except?
            ##
            ## I added instrumentation so if we drop events on the
            ## floor, we at least know about it. However, we really
            ## should just return a 500 here: (1) This will translate
            ## to much more insidious user-facing bugs if we make any
            ## decisions based on incorrect data.  (2) If the system
            ## is down, we should fail and fix it.
            event = {'event-type': 'exception', 'exception': repr(sys.exc_info()[0])}
            try:
                views.server_track(request, request.META['PATH_INFO'], event)
            except:
                # At this point, things are really broken. We really
                # should fail return a 500 to the user here.  However,
                # the interim decision is to just fail in order to be
                # consistent with current policy, and expedite the PR.
                # This version of the code makes no compromises
                # relative to the code before, while a proper failure
                # here would involve shifting compromises and
                # discussion.
                pass

{'fraction_non_alphanumeric': 0.0728592552762468, 'fraction_numerical': 0.003360666756284447, 'mean_word_length': 1.8926905132192846, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6799053', 'n_tokens_mistral': 1936, 'n_tokens_neox': 1840, 'n_words': 740}