<commit_before># vim:nowrap:textwidth=0
#
# Sheared -- non-blocking network programming library for Python
# Copyright (C) 2003  <NAME> <<EMAIL>>
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#

import sys, traceback, types, errno

from sheared import error
from sheared.protocol import http
from sheared.python import io

import oh_nine, one_oh

class HTTPServer:
    def __init__(self):
        self.hosts = {}
        self.default_host = None

        self.oh_nine = oh_nine.Server()
        self.one_oh = one_oh.Server()

    def addVirtualHost(self, name, vhost):
        self.hosts[name] = vhost

    def setDefaultHost(self, name):
        self.default_host = name

    def startup(self, transport):
        try:
            transport = io.BufferedReader(transport)

            rl = transport.readline().rstrip()
            if not rl:
                transport.close()
                return

            try:
                requestline = http.HTTPRequestLine(rl)
            except ValueError:
                raise error.web.BadRequestError, 'could not parse request-line: %r' % rl

            if requestline.version[0] == 0: # HTTP/0.9
                request, reply = self.oh_nine.parse(transport, requestline)
                self.handle(request, reply)

            elif requestline.version[0] == 1: # HTTP/1.x
                request, reply = self.one_oh.parse(transport, requestline)
                self.handle(request, reply)

            else:
                # FIXME -- is this the Right Thing?
                raise error.web.NotImplementedError, 'HTTP Version not supported'

        except error.web.WebServerError, e:
            if e.args:
                traceback.print_exc(1)

            if len(e.args) == 1 and isinstance(e.args[0], types.StringType):
                err = e.args[0]
            else:
                err = 'Unknown why?!'

            transport.write('HTTP/1.0 %d %s\r\n' % (e.statusCode, err))
            transport.write('Content-Type: text/plain\r\n\r\n')
            transport.write('Crashing in flames!\r\n')

        except OSError, e:
            if e.errno in (errno.ECONNRESET, errno.EPIPE):
                pass
            else:
                raise

        except:
            self.logInternalError(sys.exc_info())

        transport.close()

    def handle(self, request, reply):
        try:
            if request.requestline.uri[0] or request.requestline.uri[1]:
                raise error.web.ForbiddenError

            if request.headers.has_key('Host'):
                vhost = self.hosts.get(request.headers['Host'], None)
            else:
                vhost = None
            if vhost is None and self.default_host:
                vhost = self.hosts[self.default_host]

            if vhost:
                try:
                    vhost.handle(request, reply)
                except error.web.WebServerError:
                    raise
                except:
                    self.logInternalError(sys.exc_info())
                    raise error.web.InternalServerError
            else:
                raise error.web.NotFoundError

        except error.web.WebServerError, e:
            if not reply.decapitated:
                reply.setStatusCode(e.statusCode)
                reply.headers.setHeader('Content-Type', 'text/plain')
                self.handleWebServerError(e, request, reply)

        self.handleCompletedRequest(request, reply)
     
    def handleCompletedRequest(self, request, reply):
        print reply.status,
        print '<%s>' % request.requestline.wire_uri,
        if request.headers.has_key('Referer'):
           print '<%s>' % request.headers.get('Referer'),
        else:
            print '<>',
        print reply.transport.other,
        print

    def handleWebServerError(self, e, request, reply):
        if isinstance(e, error.web.Moved):
            reply.send("This page has moved. You can now find it here:\r\n"
                       "  %s\r\n" % reply.headers.get('Location'))

        elif isinstance(e, error.web.UnauthorizedError):
            reply.send("I need to see some credentials.\r\n")

        elif isinstance(e, error.web.ForbiddenError):
            reply.send("Forbidden.\r\n")

        elif isinstance(e, error.web.NotFoundError):
            reply.send("Not found.\r\n")

        else:
            reply.send("I am terribly sorry, but an error (%d) occured "
                       "while processing your request.\r\n" % e.statusCode)
            traceback.print_exc()
        
    def logInternalError(self, (tpe, val, tb)):
        traceback.print_exception(tpe, val, tb)
<commit_msg>Add logging in the combined format.<commit_after># vim:nowrap:textwidth=0
#
# Sheared -- non-blocking network programming library for Python
# Copyright (C) 2003  <NAME> <<EMAIL>>
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#

import sys, traceback, types, errno, time

from sheared import error
from sheared.protocol import http
from sheared.python import io

import oh_nine, one_oh

class HTTPServer:
    def __init__(self):
        self.hosts = {}
        self.default_host = None

        self.oh_nine = oh_nine.Server()
        self.one_oh = one_oh.Server()

        self.accesslog = None
        self.errorlog = None

    def addVirtualHost(self, name, vhost):
        self.hosts[name] = vhost

    def setDefaultHost(self, name):
        self.default_host = name

    def setAccessLog(self, l):
        self.accesslog = l
    def setErrorLog(self, l):
        self.errorlog = l

    def startup(self, transport):
        try:
            transport = io.BufferedReader(transport)

            rl = transport.readline().rstrip()
            if not rl:
                transport.close()
                return

            try:
                requestline = http.HTTPRequestLine(rl)
            except ValueError:
                raise error.web.BadRequestError, 'could not parse request-line: %r' % rl

            if requestline.version[0] == 0: # HTTP/0.9
                request, reply = self.oh_nine.parse(transport, requestline)
                self.handle(request, reply)

            elif requestline.version[0] == 1: # HTTP/1.x
                request, reply = self.one_oh.parse(transport, requestline)
                self.handle(request, reply)

            else:
                # FIXME -- is this the Right Thing?
                raise error.web.NotImplementedError, 'HTTP Version not supported'

        except error.web.WebServerError, e:
            if e.args:
                traceback.print_exc(1)

            if len(e.args) == 1 and isinstance(e.args[0], types.StringType):
                err = e.args[0]
            else:
                err = 'Unknown why?!'

            transport.write('HTTP/1.0 %d %s\r\n' % (e.statusCode, err))
            transport.write('Content-Type: text/plain\r\n\r\n')
            transport.write('Crashing in flames!\r\n')

        except OSError, e:
            if e.errno in (errno.ECONNRESET, errno.EPIPE):
                pass
            else:
                raise

        except:
            self.logInternalError(sys.exc_info())

        transport.close()

    def handle(self, request, reply):
        try:
            if request.requestline.uri[0] or request.requestline.uri[1]:
                raise error.web.ForbiddenError

            if request.headers.has_key('Host'):
                vhost = self.hosts.get(request.headers['Host'], None)
            else:
                vhost = None
            if vhost is None and self.default_host:
                vhost = self.hosts[self.default_host]

            if vhost:
                try:
                    vhost.handle(request, reply)
                except error.web.WebServerError:
                    raise
                except:
                    self.logInternalError(sys.exc_info())
                    raise error.web.InternalServerError
            else:
                raise error.web.NotFoundError

        except error.web.WebServerError, e:
            if not reply.decapitated:
                reply.setStatusCode(e.statusCode)
                reply.headers.setHeader('Content-Type', 'text/plain')
                self.handleWebServerError(e, request, reply)

        self.logCompletedRequest(request, reply)
     
    def handleWebServerError(self, e, request, reply):
        if isinstance(e, error.web.Moved):
            reply.send("This page has moved. You can now find it here:\r\n"
                       "  %s\r\n" % reply.headers.get('Location'))

        elif isinstance(e, error.web.UnauthorizedError):
            reply.send("I need to see some credentials.\r\n")

        elif isinstance(e, error.web.ForbiddenError):
            reply.send("Forbidden.\r\n")

        elif isinstance(e, error.web.NotFoundError):
            reply.send("Not found.\r\n")

        else:
            reply.send("I am terribly sorry, but an error (%d) occured "
                       "while processing your request.\r\n" % e.statusCode)
            self.logInternalError()
        
    def logCompletedRequest(self, request, reply):
        if self.accesslog:
            ip, port = reply.transport.other
            ident = '-'
            user, _ = request.authentication(require=0)
            user = user or '-'
            date = time.strftime('%d/%m/%Y:%H:%M:%S %z')
            req = request.requestline.raw
            code = reply.status
            length = 0

            if request.headers.has_key('Referer'):
                referer = request.headers.get('Referer')
            else:
                referer = ''

            if request.headers.has_key('User-Agent'):
                agent = request.headers.get('User-Agent')
            else:
                agent = ''

            fmt = '%s %s %s [%s] "%s" %d %d "%s" "%s"\n'
            self.accesslog.write(fmt % (ip, ident, user, date,
                                        req, code, length,
                                        referer, agent))

    def logInternalError(self, ex):
        if self.errorlog:
            self.errorlog.exception(ex)

{'fraction_non_alphanumeric': 0.09887896304081276, 'fraction_numerical': 0.006305832895428271, 'mean_word_length': 1.8132544961813255, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3923609', 'n_tokens_mistral': 3241, 'n_tokens_neox': 3066, 'n_words': 869}