<commit_before>from BaseHTTPServer import BaseHTTPRequestHandler


class FilterMatchError(Exception):
    """Used to describe when a filter does not match a request."""
    pass


class WebhookRequestHandler(BaseHTTPRequestHandler):
    """Extends the BaseHTTPRequestHandler class and handles the incoming
    HTTP requests."""

    def do_POST(self):
        """Invoked on incoming POST requests"""
        from threading import Timer
        import logging

        logger = logging.getLogger()

        content_type = self.headers.getheader('content-type')
        content_length = int(self.headers.getheader('content-length'))
        request_body = self.rfile.read(content_length)
        
        # Extract request headers and make all keys to lowercase (makes them easier to compare)
        request_headers = dict(self.headers)
        request_headers = dict((k.lower(), v) for k, v in request_headers.iteritems())

        ServiceRequestParser = self.figure_out_service_from_request(request_headers, request_body)
        
        self.send_response(200)
        self.send_header('Content-type', 'text/plain')
        self.end_headers()

        # Unable to identify the source of the request
        if not ServiceRequestParser:
            logger.error('Unable to find appropriate handler for request. The source service is not supported.')
            return

        # Could be GitHubParser, GitLabParser or other
        repo_configs, ref, action = ServiceRequestParser(self._config).get_repo_params_from_request(request_headers, request_body)

        logger.info("Event details - ref: %s; action: %s" % (ref or "master", action))

        #if success:
        #    print "Successfullt handled request using %s" % ServiceHandler.__name__
        #else:
        #    print "Unable to handle request using %s" % ServiceHandler.__name__

        if len(repo_configs) == 0:
            logger.warning('Unable to find any of the repository URLs in the config: %s' % ', '.join(repo_urls))
            return

        # Wait one second before we do git pull (why?)
        Timer(1.0, self.process_repositories, (repo_configs,
                                               ref,
                                               action, request_body)).start()

    def log_message(self, format, *args):
        """Overloads the default message logging method to allow messages to
        go through our custom logger instead."""
        import logging
        logger = logging.getLogger()
        logger.info("%s - - [%s] %s\n" % (self.client_address[0],
                                          self.log_date_time_string(),
                                          format%args))

    def figure_out_service_from_request(self, request_headers, request_body):
        """Parses the incoming request and attempts to determine whether
        it originates from GitHub, GitLab or any other known service."""
        import json
        import logging
        import parsers

        logger = logging.getLogger()
        data = json.loads(request_body)

        user_agent = 'user-agent' in request_headers and request_headers['user-agent']
        content_type = 'content-type' in request_headers and request_headers['content-type']

        # Assume GitLab if the X-Gitlab-Event HTTP header is set
        if 'x-gitlab-event' in request_headers:

            logger.info("Received event from GitLab")
            return parsers.GitLabRequestParser

        # Assume GitHub if the X-GitHub-Event HTTP header is set
        elif 'x-github-event' in request_headers:

            logger.info("Received event from GitHub")
            return parsers.GitHubRequestParser

        # Assume BitBucket if the User-Agent HTTP header is set to
        # 'Bitbucket-Webhooks/2.0' (or something similar)
        elif user_agent and user_agent.lower().find('bitbucket') != -1:

            logger.info("Received event from BitBucket")
            return parsers.BitBucketRequestParser

        # Special Case for Gitlab CI
        elif content_type == "application/json" and "build_status" in data:

            logger.info('Received event from Gitlab CI')
            return parsers.GitLabCIRequestParser

        # This handles old GitLab requests and Gogs requests for example.
        elif content_type == "application/json":

            logger.info("Received event from unknown origin.")
            return parsers.GenericRequestParser

        logger.error("Unable to recognize request origin. Don't know how to handle the request.")
        return


    def process_repositories(self, repo_configs, ref, action, request_body):
        import os
        import time
        import logging
        from wrappers import GitWrapper
        from lock import Lock
        
        logger = logging.getLogger()
        data = json.loads(request_body)

        # Process each matching repository
        for repo_config in repo_configs:<commit_msg>Fixed bug due to missing import<commit_after>from BaseHTTPServer import BaseHTTPRequestHandler


class FilterMatchError(Exception):
    """Used to describe when a filter does not match a request."""
    pass


class WebhookRequestHandler(BaseHTTPRequestHandler):
    """Extends the BaseHTTPRequestHandler class and handles the incoming
    HTTP requests."""

    def do_POST(self):
        """Invoked on incoming POST requests"""
        from threading import Timer
        import logging

        logger = logging.getLogger()

        content_type = self.headers.getheader('content-type')
        content_length = int(self.headers.getheader('content-length'))
        request_body = self.rfile.read(content_length)
        
        # Extract request headers and make all keys to lowercase (makes them easier to compare)
        request_headers = dict(self.headers)
        request_headers = dict((k.lower(), v) for k, v in request_headers.iteritems())

        ServiceRequestParser = self.figure_out_service_from_request(request_headers, request_body)
        
        self.send_response(200)
        self.send_header('Content-type', 'text/plain')
        self.end_headers()

        # Unable to identify the source of the request
        if not ServiceRequestParser:
            logger.error('Unable to find appropriate handler for request. The source service is not supported.')
            return

        # Could be GitHubParser, GitLabParser or other
        repo_configs, ref, action = ServiceRequestParser(self._config).get_repo_params_from_request(request_headers, request_body)

        logger.info("Event details - ref: %s; action: %s" % (ref or "master", action))

        #if success:
        #    print "Successfullt handled request using %s" % ServiceHandler.__name__
        #else:
        #    print "Unable to handle request using %s" % ServiceHandler.__name__

        if len(repo_configs) == 0:
            logger.warning('Unable to find any of the repository URLs in the config: %s' % ', '.join(repo_urls))
            return

        # Wait one second before we do git pull (why?)
        Timer(1.0, self.process_repositories, (repo_configs,
                                               ref,
                                               action, request_body)).start()

    def log_message(self, format, *args):
        """Overloads the default message logging method to allow messages to
        go through our custom logger instead."""
        import logging
        logger = logging.getLogger()
        logger.info("%s - - [%s] %s\n" % (self.client_address[0],
                                          self.log_date_time_string(),
                                          format%args))

    def figure_out_service_from_request(self, request_headers, request_body):
        """Parses the incoming request and attempts to determine whether
        it originates from GitHub, GitLab or any other known service."""
        import json
        import logging
        import parsers

        logger = logging.getLogger()
        data = json.loads(request_body)

        user_agent = 'user-agent' in request_headers and request_headers['user-agent']
        content_type = 'content-type' in request_headers and request_headers['content-type']

        # Assume GitLab if the X-Gitlab-Event HTTP header is set
        if 'x-gitlab-event' in request_headers:

            logger.info("Received event from GitLab")
            return parsers.GitLabRequestParser

        # Assume GitHub if the X-GitHub-Event HTTP header is set
        elif 'x-github-event' in request_headers:

            logger.info("Received event from GitHub")
            return parsers.GitHubRequestParser

        # Assume BitBucket if the User-Agent HTTP header is set to
        # 'Bitbucket-Webhooks/2.0' (or something similar)
        elif user_agent and user_agent.lower().find('bitbucket') != -1:

            logger.info("Received event from BitBucket")
            return parsers.BitBucketRequestParser

        # Special Case for Gitlab CI
        elif content_type == "application/json" and "build_status" in data:

            logger.info('Received event from Gitlab CI')
            return parsers.GitLabCIRequestParser

        # This handles old GitLab requests and Gogs requests for example.
        elif content_type == "application/json":

            logger.info("Received event from unknown origin.")
            return parsers.GenericRequestParser

        logger.error("Unable to recognize request origin. Don't know how to handle the request.")
        return


    def process_repositories(self, repo_configs, ref, action, request_body):
        import os
        import time
        import logging
        from wrappers import GitWrapper
        from lock import Lock
        import json
        
        logger = logging.getLogger()
        data = json.loads(request_body)

        # Process each matching repository
        for repo_config in repo_configs:
{'fraction_non_alphanumeric': 0.07724889068172651, 'fraction_numerical': 0.002016942315449778, 'mean_word_length': 2.3731292517006803, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1178357', 'n_tokens_mistral': 2530, 'n_tokens_neox': 2405, 'n_words': 806}