<commit_before># -*- coding: utf-8 -*-

import pdb

"""
Note, requests are much better than this
"""

import logging as mod_logging
import urllib2 as mod_urllib2
import urllib as mod_urllib
import collections as mod_collections

mod_logging.basicConfig(level=mod_logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')

VALID_METHODS = ['GET', 'DELETE', 'PUT', 'POST']

# For those methods params will be urlencoded and put in request body:
PARAMS_IN_BODY_METHODS = ['PUT', 'POST']

CustomHttpResponse = mod_collections.namedtuple(
        'CustomHttpResponse',
        ('status_code', 'headers', 'content'))

class CustomRequest(mod_urllib2.Request):

    def __init__(self, method, url, *args1, **args2):
        mod_urllib2.Request.__init__(self, url, *args1, **args2)
        self.method = method

    def get_method(self):
        return self.method

def add_params(url, urlencoded_params):
    if '?' in url:
        return url + '&' + urlencoded_params
    else:
        return url + '?' + urlencoded_params

def parse_headers(headers):
    result = {}
    if not headers:
        return result
    for header in headers:
        pos = header.find(':')
        if pos > 0:
            result[header[:pos]] = header[pos + 1:].strip()

    return result

def urlencode_params(params):
    if hasattr(params, 'items') or isinstance(params, tuple):
        return mod_urllib.urlencode(params)

    return params

def execute_request(method, url, data=None, headers=None):
    if not method in VALID_METHODS:
        raise Exception('Invalid method %s' % method)

    mod_logging.debug('%s request to %s', method, url)

    body = None
    opener = mod_urllib2.build_opener(mod_urllib2.HTTPHandler)<commit_msg>params when logging http requests<commit_after># -*- coding: utf-8 -*-

import pdb

"""
Note, requests are much better than this
"""

import logging as mod_logging
import urllib2 as mod_urllib2
import urllib as mod_urllib
import collections as mod_collections

mod_logging.basicConfig(level=mod_logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')

VALID_METHODS = ['GET', 'DELETE', 'PUT', 'POST']

# For those methods params will be urlencoded and put in request body:
PARAMS_IN_BODY_METHODS = ['PUT', 'POST']

CustomHttpResponse = mod_collections.namedtuple(
        'CustomHttpResponse',
        ('status_code', 'headers', 'content'))

class CustomRequest(mod_urllib2.Request):

    def __init__(self, method, url, *args1, **args2):
        mod_urllib2.Request.__init__(self, url, *args1, **args2)
        self.method = method

    def get_method(self):
        return self.method

def add_params(url, urlencoded_params):
    if '?' in url:
        return url + '&' + urlencoded_params
    else:
        return url + '?' + urlencoded_params

def parse_headers(headers):
    result = {}
    if not headers:
        return result
    for header in headers:
        pos = header.find(':')
        if pos > 0:
            result[header[:pos]] = header[pos + 1:].strip()

    return result

def urlencode_params(params):
    if hasattr(params, 'items') or isinstance(params, tuple):
        return mod_urllib.urlencode(params)

    return params

def execute_request(method, url, data=None, headers=None):
    if not method in VALID_METHODS:
        raise Exception('Invalid method %s' % method)

    mod_logging.debug('%s to %s with %s', method, url, data)

    body = None
    opener = mod_urllib2.build_opener(mod_urllib2.HTTPHandler)
{'fraction_non_alphanumeric': 0.11912943871706758, 'fraction_numerical': 0.009163802978235968, 'mean_word_length': 3.6021080368906455, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2431461', 'n_tokens_mistral': 1133, 'n_tokens_neox': 1109, 'n_words': 303}