<commit_before>#! /usr/bin/python2.7

import argparse
import logging
import logging.handlers
import os.path as op
import Queue
import re
import socket
import threading

MIME_TYPES = {
    'aac': 'audio/aac',
    'abw': 'application/x-abiword',
    'arc': 'application/octet-stream',
    'avi': 'video/x-msvideo',
    'azw': 'application/vnd.amazon.ebook',
    'bin': 'application/octet-stream',
    'bz': 'application/x-bzip',
    'bz2': 'application/x-bzip2',
    'csh': 'application/x-csh',
    'css': 'text/css',
    'csv': 'text/csv',
    'doc': 'application/msword',
    'eot': 'application/vnd.ms-fontobject',
    'epub': 'application/epub+zip',
    'gif': 'image/gif',
    'htm': 'text/html',
    'html': 'text/html',
    'ico': 'image/x-icon',
    'ics': 'text/calendar',
    'jar': 'application/java-archive',
    'jpeg': 'image/jpeg',
    'jpg': 'image/jpeg',
    'js': 'application/javascript',
    'json': 'application/json',
    'mid': 'audio/midi',
    'midi': 'audio/midi',
    'mpeg': 'video/mpeg',
    'mpkg': 'application/vnd.apple.installer+xml',
    'odp': 'application/vnd.oasis.opendocument.presentation',
    'ods': 'application/vnd.oasis.opendocument.spreadsheet',
    'odt': 'application/vnd.oasis.opendocument.text',
    'oga': 'audio/ogg',
    'ogv': 'video/ogg',
    'ogx': 'application/ogg',
    'otf': 'font/otf',
    'png': 'image/png',
    'pdf': 'application/pdf',
    'ppt': 'application/vnd.ms-powerpoint',
    'rar': 'application/x-rar-compressed',
    'rtf': 'application/rtf',
    'sh': 'application/x-sh',
    'svg': 'image/svg+xml',
    'swf': 'application/x-shockwave-flash',
    'tar': 'application/x-tar',
    'tif': 'image/tiff',
    'tiff': 'image/tiff',
    'ts': 'application/typescript',
    'ttf': 'font/ttf',
    'vsd': 'application/vnd.visio',
    'wav': 'audio/x-wav',
    'weba': 'audio/webm',
    'webm': 'video/webm',
    'webp': 'image/webp',
    'woff': 'font/woff',
    'woff2': 'font/woff2',
    'xhtml': 'application/xhtml+xml',
    'xls': 'application/vnd.ms-excel',
    'xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
    'xml': 'application/xml',
    'xul': 'application/vnd.mozilla.xul+xml',
    'zip': 'application/zip',
    '3gp': 'video/3gpp',
    '3g2': 'video/3gpp2',
    '7z': 'application/x-7z-compressed'
}


class HTTPRequest(object):

    uri_re = re.compile('^([^?]*)\??([^#]*)#?(.*)$')
    fileext_re = re.compile('^.*\.([a-zA-Z]{2,5})$')

    def __init__(self, request):
        lines = request.splitlines()
        method, request_uri, http_version = lines[0].split()
        del lines[0]

        self.method = method
        self.request_uri = request_uri.lstrip('/.')
        self.http_version = http_version

        self.headers = {}
        self.uri = {'path': 'index.html', 'query': '', 'fragment': '', 'fileext': ''}
        self.parse_uri()

        for line in lines:
            try:
                _header, _value = line.split(': ', 1)
            except ValueError:
                pass  # empty header
            else:
                self.headers[_header] = _value

    def parse_uri(self):
        if self.request_uri == '':
            self.uri['fileext'] = HTTPRequest.fileext_re.match(self.uri['path']).group(1)
            return

        _parse = HTTPRequest.uri_re.match(self.request_uri)

        self.uri['path'] = _parse.group(1)
        self.uri['query'] = _parse.group(2)
        self.uri['fragment'] = _parse.group(3)
        self.uri['fileext'] = HTTPRequest.fileext_re.match(self.uri['path']).group(1)

    def __str__(self):
        return "Method: '{}' Request URI: '{}' Parsed URI: '{}' HTTP Version: '{}' Headers={}".format(
            self.method,
            self.request_uri,
            self.uri,
            self.http_version,
            self.headers)


class HTTPResponse(object):
    def __init__(self, http_version):
        self.http_version = http_version
        self.response_code = 200
        self.response_message = "OK"
        self.body = ""
        self.headers = {'Server': 'pymicroweb'}

    def setbody(self, data):
        self.body = data
        self.headers['Content-Length'] = len(self.body.encode('utf-8'))

    def setheader(self, header, value):
        self.headers[header] = value

    def __str__(self):
        return "{} {} {}\n{}\n\n{}".format(self.http_version,
                                           self.response_code,
                                           self.response_message,
                                           "\n".join(["{}: {}".format(x, self.headers[x]) for x in self.headers]),
                                           self.body)


class HTTPWorker(threading.Thread):

    def __init__(self, connection_queue, options, log):

        threading.Thread.__init__(self)
        self.daemon = True  # daemonise so it dies with main
        self.connq = connection_queue
        self.options = options
        self.wwwdir = options.www
        self.log = log
        self.log.debug("%s Awaiting connections", self.name)
        self.log.debug("%s WWW dir: %s", self.name, self.wwwdir)

    def run(self):

        while True:
            connection = self.connq.get()
            request = HTTPRequest(connection.recv(1024))
            log.info("%s %s %s %s",
                     self.name,
                     request.method,
                     request.request_uri,
                     request.http_version)
            log.debug(str(request))

            response = HTTPResponse(request.http_version)
            if request.method == 'GET':
                self.get(request, response)

            connection.sendall(str(response))
            connection.close()

    def get(self, request, response):
        try:<commit_msg>Moved a few things around to get debugging of HTTPResponse objects and make Request and Response objects consistent. Implemented HEAD<commit_after>#! /usr/bin/python2.7

import argparse
import logging
import logging.handlers
import os.path as op
import Queue
import re
import socket
import threading

MIME_TYPES = {
    'aac': 'audio/aac',
    'abw': 'application/x-abiword',
    'arc': 'application/octet-stream',
    'avi': 'video/x-msvideo',
    'azw': 'application/vnd.amazon.ebook',
    'bin': 'application/octet-stream',
    'bz': 'application/x-bzip',
    'bz2': 'application/x-bzip2',
    'csh': 'application/x-csh',
    'css': 'text/css',
    'csv': 'text/csv',
    'doc': 'application/msword',
    'eot': 'application/vnd.ms-fontobject',
    'epub': 'application/epub+zip',
    'gif': 'image/gif',
    'htm': 'text/html',
    'html': 'text/html',
    'ico': 'image/x-icon',
    'ics': 'text/calendar',
    'jar': 'application/java-archive',
    'jpeg': 'image/jpeg',
    'jpg': 'image/jpeg',
    'js': 'application/javascript',
    'json': 'application/json',
    'mid': 'audio/midi',
    'midi': 'audio/midi',
    'mpeg': 'video/mpeg',
    'mpkg': 'application/vnd.apple.installer+xml',
    'odp': 'application/vnd.oasis.opendocument.presentation',
    'ods': 'application/vnd.oasis.opendocument.spreadsheet',
    'odt': 'application/vnd.oasis.opendocument.text',
    'oga': 'audio/ogg',
    'ogv': 'video/ogg',
    'ogx': 'application/ogg',
    'otf': 'font/otf',
    'png': 'image/png',
    'pdf': 'application/pdf',
    'ppt': 'application/vnd.ms-powerpoint',
    'rar': 'application/x-rar-compressed',
    'rtf': 'application/rtf',
    'sh': 'application/x-sh',
    'svg': 'image/svg+xml',
    'swf': 'application/x-shockwave-flash',
    'tar': 'application/x-tar',
    'tif': 'image/tiff',
    'tiff': 'image/tiff',
    'ts': 'application/typescript',
    'ttf': 'font/ttf',
    'vsd': 'application/vnd.visio',
    'wav': 'audio/x-wav',
    'weba': 'audio/webm',
    'webm': 'video/webm',
    'webp': 'image/webp',
    'woff': 'font/woff',
    'woff2': 'font/woff2',
    'xhtml': 'application/xhtml+xml',
    'xls': 'application/vnd.ms-excel',
    'xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
    'xml': 'application/xml',
    'xul': 'application/vnd.mozilla.xul+xml',
    'zip': 'application/zip',
    '3gp': 'video/3gpp',
    '3g2': 'video/3gpp2',
    '7z': 'application/x-7z-compressed'
}


class HTTPRequest(object):

    uri_re = re.compile('^([^?]*)\??([^#]*)#?(.*)$')
    fileext_re = re.compile('^.*\.([a-zA-Z]{2,5})$')

    def __init__(self, request):
        self._request = request
        lines = request.splitlines()
        method, request_uri, http_version = lines[0].split()
        del lines[0]

        self.method = method
        self.request_uri = request_uri.lstrip('/.')
        self.http_version = http_version

        self.headers = {}
        self.uri = {'path': 'index.html', 'query': '', 'fragment': '', 'fileext': ''}
        self.parse_uri()

        for line in lines:
            try:
                _header, _value = line.split(': ', 1)
            except ValueError:
                pass  # empty header
            else:
                self.headers[_header] = _value

    def parse_uri(self):
        if self.request_uri == '':
            self.uri['fileext'] = HTTPRequest.fileext_re.match(self.uri['path']).group(1)
            return

        _parse = HTTPRequest.uri_re.match(self.request_uri)

        self.uri['path'] = _parse.group(1)
        self.uri['query'] = _parse.group(2)
        self.uri['fragment'] = _parse.group(3)
        self.uri['fileext'] = HTTPRequest.fileext_re.match(self.uri['path']).group(1)

    def __str__(self):
        return self._request

    def debug(self):
        return "HTTPRequest Method: '{}' Request URI: '{}' Parsed URI: '{}' HTTP Version: '{}' Headers={}".format(
            self.method,
            self.request_uri,
            self.uri,
            self.http_version,
            self.headers)


class HTTPResponse(object):
    def __init__(self, http_version, http_method):
        self.http_version = http_version
        self.method = http_method
        self.response_code = 200
        self.response_message = "OK"
        self.body = ""
        self.headers = {'Server': 'pymicroweb'}

    def setbody(self, data):
        self.body = data
        self.headers['Content-Length'] = len(self.body.encode('utf-8'))

    def setheader(self, header, value):
        self.headers[header] = value

    def debug(self):
        return "HTTPResponse Method: '{}' Code: '{}' Message: '{}' HTTP Version: '{}' Headers={}".format(
            self.method,
            self.response_code,
            self.response_message,
            self.http_version,
            self.headers)

    def __str__(self):
        if self.method == 'HEAD':
            return "{} {} {}\n{}".format(self.http_version,
                                         self.response_code,
                                         self.response_message,
                                         "\n".join(["{}: {}".format(x, self.headers[x]) for x in self.headers]))

        return "{} {} {}\n{}\n\n{}".format(self.http_version,
                                           self.response_code,
                                           self.response_message,
                                           "\n".join(["{}: {}".format(x, self.headers[x]) for x in self.headers]),
                                           self.body)


class HTTPWorker(threading.Thread):

    def __init__(self, connection_queue, options, log):

        threading.Thread.__init__(self)
        self.daemon = True  # daemonise so it dies with main
        self.connq = connection_queue
        self.options = options
        self.wwwdir = options.www
        self.log = log
        self.log.debug("%s Awaiting connections", self.name)
        self.log.debug("%s WWW dir: %s", self.name, self.wwwdir)

    def run(self):

        while True:
            connection = self.connq.get()
            request = HTTPRequest(connection.recv(1024))
            log.info("%s %s %s %s",
                     self.name,
                     request.method,
                     request.request_uri,
                     request.http_version)
            log.debug(request.debug())

            response = HTTPResponse(request.http_version, request.method)
            if request.method == 'GET' or request.method == 'HEAD':
                self.get(request, response)

            log.debug(response.debug())
            connection.sendall(str(response))
            connection.close()

    def get(self, request, response):
        try:
{'fraction_non_alphanumeric': 0.16839482356723737, 'fraction_numerical': 0.0051442810063499714, 'mean_word_length': 2.20174987133299, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1006429', 'n_tokens_mistral': 4102, 'n_tokens_neox': 3948, 'n_words': 736}