<commit_before>__all__ = ['log', 'appendArgs', 'toBase64', 'fromBase64']

import binascii
import sys

from urllib import urlencode

def log(message, unused_level=0):
    sys.stderr.write(message)
    sys.stderr.write('\n')

def appendArgs(url, args):
    if len(args) == 0:
        return url

    if '?' in url:
        sep = '&'
    else:
        sep = '?'

    return '%s%s%s' % (url, sep, urlencode(args))

def toBase64(s):
    """Represent string s as base64, omitting newlines"""
    return binascii.b2a_base64(s)[:-1]

def fromBase64(s):
    try:
        return binascii.a2b_base64(s)
    except binascii.Error:
        return ''<commit_msg>[project @ Move getOpenIDParamaters, quoteMinimal, and normalizeURL to oidutil]<commit_after>__all__ = ['log', 'appendArgs', 'toBase64', 'fromBase64']

import binascii
import sys
import urlparse

from urllib import urlencode

def log(message, unused_level=0):
    sys.stderr.write(message)
    sys.stderr.write('\n')

def appendArgs(url, args):
    if len(args) == 0:
        return url

    if '?' in url:
        sep = '&'
    else:
        sep = '?'

    return '%s%s%s' % (url, sep, urlencode(args))

def toBase64(s):
    """Represent string s as base64, omitting newlines"""
    return binascii.b2a_base64(s)[:-1]

def fromBase64(s):
    try:
        return binascii.a2b_base64(s)
    except binascii.Error:
        return ''


def getOpenIDParameters(query):
    params = {}
    for k, v in query.iteritems():
        if k.startswith('openid.'):
            params[k] = v
    return params


def quoteMinimal(s):
    # Do not escape anything that is already 7-bit safe, so we do the
    # minimal transform on the identity URL
    res = []
    for c in s:
        if c >= u'\x80':
            for b in c.encode('utf8'):
                res.append('%%%02X' % ord(b))
        else:
            res.append(c)
    return str(''.join(res))


def normalizeUrl(url):
    assert isinstance(url, (str, unicode)), type(url)

    url = url.strip()
    parsed = urlparse.urlparse(url)

    if parsed[0] == '' or parsed[1] == '':
        url = 'http://' + url
        parsed = urlparse.urlparse(url)

    if isinstance(url, unicode):
        authority = parsed[1].encode('idna')
    else:
        authority = str(parsed[1])

    tail = map(quoteMinimal, parsed[2:])
    if tail[0] == '':
        tail[0] = '/'
    encoded = (str(parsed[0]), authority) + tuple(tail)
    url = urlparse.urlunparse(encoded)
    assert type(url) is str

    return url

{'fraction_non_alphanumeric': 0.14457831325301204, 'fraction_numerical': 0.020883534136546186, 'mean_word_length': 2.6364963503649634, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7103885', 'n_tokens_mistral': 894, 'n_tokens_neox': 852, 'n_words': 220}