<commit_before>#
# Copyright (c) 2004-2007 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

from mod_python import apache
from mod_python.util import FieldStorage
import os
import sys
import time
import xmlrpclib
import zlib

from conary.lib import log, util
from conary.repository import changeset, errors, netclient
from conary.repository.netrepos import proxy
from conary.repository.filecontainer import FileContainer
from conary.web.webauth import getAuth

BUFFER=1024 * 256

def post(port, isSecure, repos, req):
    authToken = getAuth(req)
    if authToken is None:
        return apache.HTTP_BAD_REQUEST

    if authToken[0] != "anonymous" and not isSecure and repos.forceSecure:
        return apache.HTTP_FORBIDDEN

    if isSecure:
        protocol = "https"
    else:
        protocol = "http"

    extraInfo = None
    repos.log.reset()
    if req.headers_in['Content-Type'] == "text/xml":
        # handle XML-RPC requests
        encoding = req.headers_in.get('Content-Encoding', None)
        sio = util.BoundedStringIO()
        try:
            util.copyStream(req, sio)
        except IOError, e:
            # if we got a read timeout, marshal an exception back
            # to the client
            print >> sys.stderr, 'error reading from client: %s' %e
            method = 'unknown - client timeout'
            result = (False, True, ('ClientTimeout',
                                    'The server was not able to read the '
                                    'XML-RPC request sent by this client. '
                                    'This is sometimes caused by MTU problems '
                                    'on your network connection.  Using a '
                                    'smaller MTU may work around this '
                                    'problem.'))
            startTime = time.time()
        else:
            # otherwise, we've read the data, let's process it
            if encoding == 'deflate':
                sio.seek(0)
                try:
                    sio = util.decompressStream(sio)
                except zlib.error, error:
                    req.log_error("zlib inflate error in POST: %s" % error)
                    return apache.HTTP_BAD_REQUEST

            startTime = time.time()
            sio.seek(0)
            try:
                (params, method) = util.xmlrpcLoad(sio)
            except xmlrpclib.ResponseError:
                req.log_error('error parsing XMLRPC request')
                return apache.HTTP_BAD_REQUEST
            repos.log(3, "decoding=%s" % method, authToken[0],
                      "%.3f" % (time.time()-startTime))
            # req.connection.local_addr[0] is the IP address the server
            # listens on, not the IP address of the accepted socket. Most of
            # the times it will be 0.0.0.0 which is not very useful. We're
            # using local_ip instead, and we grab just the port from
            # local_addr.
            localAddr = "%s:%s" % (req.connection.local_ip,
                                   req.connection.local_addr[1])

            remoteIp = req.connection.remote_ip
            # Get the IP address of the original request in the case
            # of a proxy, otherwise use the connection's remote_ip
            if 'X-Forwarded-For' in req.headers_in:
                # pick the right-most client, since that is
                # the one closest to us.  For example, if
                # we have "X-Forwarded-For: 1.2.3.4, 4.5.6.7"
                # we want to use 4.5.6.7
                clients = req.headers_in['X-Forwarded-For']
                remoteIp = clients.split(',')[-1].strip()<commit_msg>Catch unicode problems while parsing the XMLRPC request<commit_after>#
# Copyright (c) 2004-2007 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

from mod_python import apache
from mod_python.util import FieldStorage
import os
import sys
import time
import xmlrpclib
import zlib

from conary.lib import log, util
from conary.repository import changeset, errors, netclient
from conary.repository.netrepos import proxy
from conary.repository.filecontainer import FileContainer
from conary.web.webauth import getAuth

BUFFER=1024 * 256

def post(port, isSecure, repos, req):
    authToken = getAuth(req)
    if authToken is None:
        return apache.HTTP_BAD_REQUEST

    if authToken[0] != "anonymous" and not isSecure and repos.forceSecure:
        return apache.HTTP_FORBIDDEN

    if isSecure:
        protocol = "https"
    else:
        protocol = "http"

    extraInfo = None
    repos.log.reset()
    if req.headers_in['Content-Type'] == "text/xml":
        # handle XML-RPC requests
        encoding = req.headers_in.get('Content-Encoding', None)
        sio = util.BoundedStringIO()
        try:
            util.copyStream(req, sio)
        except IOError, e:
            # if we got a read timeout, marshal an exception back
            # to the client
            print >> sys.stderr, 'error reading from client: %s' %e
            method = 'unknown - client timeout'
            result = (False, True, ('ClientTimeout',
                                    'The server was not able to read the '
                                    'XML-RPC request sent by this client. '
                                    'This is sometimes caused by MTU problems '
                                    'on your network connection.  Using a '
                                    'smaller MTU may work around this '
                                    'problem.'))
            startTime = time.time()
        else:
            # otherwise, we've read the data, let's process it
            if encoding == 'deflate':
                sio.seek(0)
                try:
                    sio = util.decompressStream(sio)
                except zlib.error, error:
                    req.log_error("zlib inflate error in POST: %s" % error)
                    return apache.HTTP_BAD_REQUEST

            startTime = time.time()
            sio.seek(0)
            try:
                (params, method) = util.xmlrpcLoad(sio)
            except xmlrpclib.ResponseError:
                req.log_error('error parsing XMLRPC request')
                return apache.HTTP_BAD_REQUEST
            except UnicodeDecodeError:
                req.log_error('unicode decode error parsing XMLRPC request')
                return apache.HTTP_BAD_REQUEST
            repos.log(3, "decoding=%s" % method, authToken[0],
                      "%.3f" % (time.time()-startTime))
            # req.connection.local_addr[0] is the IP address the server
            # listens on, not the IP address of the accepted socket. Most of
            # the times it will be 0.0.0.0 which is not very useful. We're
            # using local_ip instead, and we grab just the port from
            # local_addr.
            localAddr = "%s:%s" % (req.connection.local_ip,
                                   req.connection.local_addr[1])

            remoteIp = req.connection.remote_ip
            # Get the IP address of the original request in the case
            # of a proxy, otherwise use the connection's remote_ip
            if 'X-Forwarded-For' in req.headers_in:
                # pick the right-most client, since that is
                # the one closest to us.  For example, if
                # we have "X-Forwarded-For: 1.2.3.4, 4.5.6.7"
                # we want to use 4.5.6.7
                clients = req.headers_in['X-Forwarded-For']
                remoteIp = clients.split(',')[-1].strip()
{'fraction_non_alphanumeric': 0.07963980820956613, 'fraction_numerical': 0.01029119401239621, 'mean_word_length': 1.8187211601845747, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 8}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4798349', 'n_tokens_mistral': 2369, 'n_tokens_neox': 2236, 'n_words': 804}