<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
huTools/http/__init__.py

Provides a simple interface for doing HTTP requests on stock Python and on Google Appengine. Provides
unicode aware application/x-www-form-urlencoded encoding and an do multipart/form-data (file upload)
without aditional code. On Google Appengine it incerass the timeout from 5 seconds to 10.

Usage is without suprises::

    >>> status, header, body = fetch('http://www.postbin.org/o0ds54',
                                     {'küh': 'Iñtërnâtiônàlizætiøn', 'just a test': 212},
                                     headers={'X-Foo': 'Bar'})

File Upload just works::

    >>> status, header, body = fetch('http://www.postbin.org/o0ds54',
                                     {'hosts': open('/etc/hosts', 'r')}, 'POST')

`fetch2xx()` throws a `WrongStatusCode` if the server returns a status code outside the 200-299 range.
`fetch_json2xx()` in addition decodes a JSON reply and returns that.
"""

# Created by <NAME> on 2010-10-24.
# Copyright (c) 2010, 2011 HUDORA. All rights reserved.

from huTools import hujson
from huTools.http import exceptions
from huTools.http import tools
import cgi
import poster_encode
import urllib
import urlparse


# This is somewhat clumsy to make static code checkers happy
request = None
try:
    import engine_appengine
    request = engine_appengine.request
except ImportError:
    import engine_httplib2
    request = engine_httplib2.request


def prepare_headers(url, content='', method='GET', credentials=None, headers=None, multipart=False, ua='',
                    timeout=25):
    """Prepares a request, returns (url, method, content, headers, timeout)"""

    myheaders = {'Accept-Encoding': 'gzip',
                 'User-Agent': '%s/huTools.http (gzip)' % ua}
    if headers:
        myheaders.update(headers)
    if method == 'POST':
        if hasattr(content, 'items'):
            # we assume content is a dict which needs to be encoded
            # decide to use multipart/form-data encoding or application/x-www-form-urlencoded
            for val in content.values():
                if hasattr(val, 'read'):  # file() or StringIO()
                    multipart = True
            if multipart:
                datagen, mp_headers = poster_encode.multipart_encode(content)
                myheaders.update(mp_headers)
                content = "".join(datagen)
            else:
                myheaders.update({'Content-Type': 'application/x-www-form-urlencoded'})
                content = tools.urlencode(content)
    else:
        # url parmater encoding
        if hasattr(content, 'items'):
            scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
            qdict = cgi.parse_qs(query)
            # ugly Unicode issues, see http://bugs.python.org/issue1712522
            qdict.update(content)
            query = tools.urlencode(qdict)
            url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
            content = ''
    # convert all header values to strings (what about unicode?)
    for key, val in myheaders.items():
        myheaders[key] = str(val)
    # add authentication
    if credentials and not 'Authorization' in myheaders.keys():
        authheader = "Basic %s" % credentials.encode('base64').strip()
        myheaders["Authorization"] = authheader
    return url, method, content, myheaders, timeout


def fetch(url, content='', method='GET', credentials=None, headers=None, multipart=False, ua='', timeout=25):
    """Does a HTTP request with method `method` to `url`.

    Returns (status, headers, content) whereas `status` is an integer status code, `headers` is a dict<commit_msg>HTTP-Client: Lange Credentials werden nicht richtig zu Base64 gewandelt.<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
huTools/http/__init__.py

Provides a simple interface for doing HTTP requests on stock Python and on Google Appengine. Provides
unicode aware application/x-www-form-urlencoded encoding and an do multipart/form-data (file upload)
without aditional code. On Google Appengine it incerass the timeout from 5 seconds to 10.

Usage is without suprises::

    >>> status, header, body = fetch('http://www.postbin.org/o0ds54',
                                     {'küh': 'Iñtërnâtiônàlizætiøn', 'just a test': 212},
                                     headers={'X-Foo': 'Bar'})

File Upload just works::

    >>> status, header, body = fetch('http://www.postbin.org/o0ds54',
                                     {'hosts': open('/etc/hosts', 'r')}, 'POST')

`fetch2xx()` throws a `WrongStatusCode` if the server returns a status code outside the 200-299 range.
`fetch_json2xx()` in addition decodes a JSON reply and returns that.
"""

# Created by <NAME> on 2010-10-24.
# Copyright (c) 2010, 2011 HUDORA. All rights reserved.

from huTools import hujson
from huTools.http import exceptions
from huTools.http import tools
import cgi
import poster_encode
import urllib
import urlparse


# This is somewhat clumsy to make static code checkers happy
request = None
try:
    import engine_appengine
    request = engine_appengine.request
except ImportError:
    import engine_httplib2
    request = engine_httplib2.request


def prepare_headers(url, content='', method='GET', credentials=None, headers=None, multipart=False, ua='',
                    timeout=25):
    """Prepares a request, returns (url, method, content, headers, timeout)"""

    myheaders = {'Accept-Encoding': 'gzip',
                 'User-Agent': '%s/huTools.http (gzip)' % ua}
    if headers:
        myheaders.update(headers)
    if method == 'POST':
        if hasattr(content, 'items'):
            # we assume content is a dict which needs to be encoded
            # decide to use multipart/form-data encoding or application/x-www-form-urlencoded
            for val in content.values():
                if hasattr(val, 'read'):  # file() or StringIO()
                    multipart = True
            if multipart:
                datagen, mp_headers = poster_encode.multipart_encode(content)
                myheaders.update(mp_headers)
                content = "".join(datagen)
            else:
                myheaders.update({'Content-Type': 'application/x-www-form-urlencoded'})
                content = tools.urlencode(content)
    else:
        # url parmater encoding
        if hasattr(content, 'items'):
            scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
            qdict = cgi.parse_qs(query)
            # ugly Unicode issues, see http://bugs.python.org/issue1712522
            qdict.update(content)
            query = tools.urlencode(qdict)
            url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
            content = ''
    # convert all header values to strings (what about unicode?)
    for key, val in myheaders.items():
        myheaders[key] = str(val)
    # add authentication
    if credentials and not 'Authorization' in myheaders.keys():
        # ''.encode('b64') zerbricht lange Strings (also bei langen Credentials)
        # in mehrere Zeilen, was einer Verwendung als Wert im HTTP-Header extrem
        # abtraeglich ist. Aufgetaucht ist das Problem bei den Login-Daten fuer
        # den Import der Shopify-Auftraege aus dem hySkate-Shop. Zur Loesung werden
        # deshalb alle moeglichen Einzelzeilen wieder zu einer Zeile zusammengezogen.
        authheader = 'Basic %s' % ''.join(credentials.encode('base64').strip().split())
        myheaders["Authorization"] = authheader
    return url, method, content, myheaders, timeout


def fetch(url, content='', method='GET', credentials=None, headers=None, multipart=False, ua='', timeout=25):
    """Does a HTTP request with method `method` to `url`.

    Returns (status, headers, content) whereas `status` is an integer status code, `headers` is a dict
{'fraction_non_alphanumeric': 0.10697145743874716, 'fraction_numerical': 0.013639808032331396, 'mean_word_length': 2.742438563327032, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 10, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2193062', 'n_tokens_mistral': 2331, 'n_tokens_neox': 2200, 'n_words': 723}