<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

import sys
import traceback    # for debug
import uuid
from SocketServer import ThreadingMixIn
from BaseHTTPServer import HTTPServer
from extended_http_server import ExtendedHTTPRequestHandler
from inet_string import inet_string
from west_client import west_client
from west_parser import west_parser
from threading import currentThread, Event
import time
from debug_tools import is_debug, debug_print

'''
HTTP Proxy Handler
'''
class ProxyHandler(ExtendedHTTPRequestHandler):

    server_response_timer = 5
    waiting_for_server = None

    def do_GET(self):
        if not self.pre_process():
            return
        self.read_content()

    def do_POST(self):
        if not self.pre_process():
            return
        self.read_content()

    def pre_process(self):
        '''
        it called just immediately after the 1st line of a request is ready to
        be read.
        '''
        #
        msg = 'an HTTP client connected from %s, request %s %s' % (repr(self.client_address), self.command, self.path)
        self.log_message(msg)
        if is_debug(1, self.server.west):
            print('DEBUG:', msg)
            print('DEBUG:', self.__class__, currentThread().getName())
        #
        # check whether the incoming url can be converted into a outgoing url.
        # before it read the whole message from the cient.
        #
        if (not self.server.jc_mine.has_key(self.path) or
                not self.server.jc_mine[self.path].has_key('ou')):
            self.send_error(404)
            self.end_headers()
            msg = 'no url mapping for %s' % self.path
            self.log_message(msg)
            if is_debug(1, self.server.west):
                print('ERROR:', msg)
            return False
        return True

    def post_read(self, contents):
        '''
        it called after the whole message from the client has been read.
        '''
        #
        # convert Path into the outgoing URL, and Host into the HTTP server.
        #
        a = inet_string(self.server.jc_mine[self.path]['ou'])
        if self.headers.has_key('host'):
            if is_debug(2, self.server.west):
                print('DEBUG: convert Host into %s from %s' %
                      (self.headers['host'], a['url_host']))
            self.headers['host'] = a['url_host']
        if is_debug(2, self.server.west):
            print('DEBUG: convert Path into %s from %s' %
                  (self.path, a['url_path']))
        #
        msg_list = []
        msg_list.append(' '.join([self.command, a['url_path'],
                                  self.request_version]))
        msg_list.append('\r\n')
        msg_list.extend(['%s: %s\r\n' % (k,v) for k,v in self.headers.items()])
        msg_list.append('\r\n')
        msg_list.extend(contents)
        #
        msg = ''.join(msg_list)
        if is_debug(1, self.server.west):
            print('DEBUG: sent proxy message length=', len(msg))
            if is_debug(2, self.server.west):
                print('DEBUG: ---BEGIN OF PROXY REQUEST---')
                print(msg)
                print('DEBUG: ---END OF PROXY REQUEST---')
        #
        '''
        http sessions must be identified.
        each session must be assigned uuid.
        chunked data have to be assiened an identical uuid.
        '''
        self.session_id = str(uuid.uuid4())
        #
        if self.server.s_ws:
            response = self.server.s_ws.send(self, msg, self.session_id,
                                            proxy_protocol='http')
        else:
            msg = 'ERROR: any WS instance is not specified.'
            self.log_message(msg)
            if is_debug(1, self.server.west):
                print('DEBUG:', msg)
            return
        #
        # waiting for a response from the server
        #
        if is_debug(1, self.server.west):
            print('DEBUG: waiting for response from server in %d seconds' %
                  self.server_response_timer)
        self.waiting_for_server = Event()
        self.waiting_for_server.wait(self.server_response_timer)

    '''
    @param response http header and payload, or None
    '''
    def put_response(self, session_id, headers, content):
        if session_id != self.session_id:
            print('ERROR: unexpected session_id %s, should be %s',
                    session_id, self.session_id)
            return
        try:
            #
            # failed
            #
            if not content:
                self.send_error(404)
                self.end_headers()
                self.log_message('error in response')
                self.waiting_for_server.set()
                return
            #
            # success
            #
            self.send_response(200)
            for k,v in headers.items():
                self.send_header(k, v)
            self.end_headers()
            self.wfile.write(content)
            self.wfile.write('\r\n')    # just make it sure
            if is_debug(1, self.server.west):
                print('DEBUG: sent back message length=%d' % len(content))
                if is_debug(2, self.server.west):
                    print('DEBUG: ---BEGIN OF PROXY RESPONSE---')
                    print(content)
                    print('DEBUG: ---END OF PROXY RESPONSE---')
            self.log_message('success')
        except Exception:
            print(traceback.format_exc())
        self.waiting_for_server.set()
<commit_msg>return 503 when the peer proxy is not available.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

import sys
import traceback    # for debug
import uuid
from SocketServer import ThreadingMixIn
from BaseHTTPServer import HTTPServer
from extended_http_server import ExtendedHTTPRequestHandler
from inet_string import inet_string
from west_client import west_client
from west_parser import west_parser
from threading import currentThread, Event
import time
from debug_tools import is_debug, debug_print

'''
HTTP Proxy Handler
'''
class ProxyHandler(ExtendedHTTPRequestHandler):

    server_response_timer = 5
    waiting_for_server = None

    def do_GET(self):
        if not self.pre_process():
            return
        self.read_content()

    def do_POST(self):
        if not self.pre_process():
            return
        self.read_content()

    def pre_process(self):
        '''
        it called just immediately after the 1st line of a request is ready to
        be read.
        '''
        #
        msg = 'an HTTP client connected from %s, request %s %s' % (repr(self.client_address), self.command, self.path)
        self.log_message(msg)
        if is_debug(1, self.server.west):
            print('DEBUG:', msg)
            print('DEBUG:', self.__class__, currentThread().getName())
        #
        # check whether the incoming url can be converted into a outgoing url.
        # before it read the whole message from the cient.
        #
        if (not self.server.jc_mine.has_key(self.path) or
                not self.server.jc_mine[self.path].has_key('ou')):
            self.send_error(404)
            self.end_headers()
            msg = 'no url mapping for %s' % self.path
            self.log_message(msg)
            if is_debug(1, self.server.west):
                print('ERROR:', msg)
            return False
        return True

    def post_read(self, contents):
        '''
        it called after the whole message from the client has been read.
        '''
        #
        # convert Path into the outgoing URL, and Host into the HTTP server.
        #
        a = inet_string(self.server.jc_mine[self.path]['ou'])
        if self.headers.has_key('host'):
            if is_debug(2, self.server.west):
                print('DEBUG: convert Host into %s from %s' %
                      (self.headers['host'], a['url_host']))
            self.headers['host'] = a['url_host']
        if is_debug(2, self.server.west):
            print('DEBUG: convert Path into %s from %s' %
                  (self.path, a['url_path']))
        #
        msg_list = []
        msg_list.append(' '.join([self.command, a['url_path'],
                                  self.request_version]))
        msg_list.append('\r\n')
        msg_list.extend(['%s: %s\r\n' % (k,v) for k,v in self.headers.items()])
        msg_list.append('\r\n')
        msg_list.extend(contents)
        #
        msg = ''.join(msg_list)
        if is_debug(1, self.server.west):
            print('DEBUG: sent proxy message length=', len(msg))
            if is_debug(2, self.server.west):
                print('DEBUG: ---BEGIN OF PROXY REQUEST---')
                print(msg)
                print('DEBUG: ---END OF PROXY REQUEST---')
        #
        '''
        http sessions must be identified.
        each session must be assigned uuid.
        chunked data have to be assiened an identical uuid.
        '''
        self.session_id = str(uuid.uuid4())
        #
        if self.server.s_ws:
            response = self.server.s_ws.send(self, msg, self.session_id,
                                            proxy_protocol='http')
        else:
            msg = 'ERROR: any WS instance is not specified.'
            self.log_message(msg)
            if is_debug(1, self.server.west):
                print('DEBUG:', msg)
            self.send_error(503)
            self.end_headers()
            self.log_message('The peer proxy is not available.')
            return
        #
        # waiting for a response from the server
        #
        if is_debug(1, self.server.west):
            print('DEBUG: waiting for response from server in %d seconds' %
                  self.server_response_timer)
        self.waiting_for_server = Event()
        self.waiting_for_server.wait(self.server_response_timer)

    '''
    "self.waiting_for_server.set()" must be called when this function finishes.

    @param response http header and payload, or None
    '''
    def put_response(self, session_id, headers, content):
        if session_id != self.session_id:
            print('ERROR: unexpected session_id %s, should be %s',
                    session_id, self.session_id)
            self.waiting_for_server.set()
            return
        try:
            #
            # failed
            #
            if not content:
                self.send_error(404)
                self.end_headers()
                self.log_message('error in response')
                self.waiting_for_server.set()
                return
            #
            # success
            #
            self.send_response(200)
            for k,v in headers.items():
                self.send_header(k, v)
            self.end_headers()
            self.wfile.write(content)
            self.wfile.write('\r\n')    # just make it sure
            if is_debug(1, self.server.west):
                print('DEBUG: sent back message length=%d' % len(content))
                if is_debug(2, self.server.west):
                    print('DEBUG: ---BEGIN OF PROXY RESPONSE---')
                    print(content)
                    print('DEBUG: ---END OF PROXY RESPONSE---')
            self.log_message('success')
        except Exception:
            print(traceback.format_exc())
        self.waiting_for_server.set()

{'fraction_non_alphanumeric': 0.10468271334792123, 'fraction_numerical': 0.004551422319474836, 'mean_word_length': 1.7539166064111835, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7207787', 'n_tokens_mistral': 3236, 'n_tokens_neox': 3151, 'n_words': 839}