<commit_before>import re

from .headers import HeaderElement
from .parsers import MultipartParser
from .parsers import QueryStringParser


def process_multipart(request, params):
    headers = request.headers

    ctype = headers.elements("Content-Type")
    if ctype:
        ctype = ctype[0]
    else:
        ctype = HeaderElement.from_str("application/x-www-form-urlencoded")

    ib = ""
    if "boundary" in ctype.params:
        # http://tools.ietf.org/html/rfc2046#section-5.1.1
        # "The grammar for parameters on the Content-type field is such that it
        # is often necessary to enclose the boundary parameter values in quotes
        # on the Content-type line"
        ib = ctype.params["boundary"].strip("\"")

    if not re.match("^[ -~]{0,200}[!-~]$", ib):
        raise ValueError("Invalid boundary in multipart form: %r" % (ib,))

    parser = MultipartParser(request.body, ib)
    for part in parser:
        if part.filename or not part.is_buffered():
            params[part.name] = part
        else:
            params[part.name] = part.value


def process_urlencoded(request, params, encoding="utf-8"):
    params.update(QueryStringParser(request.qs).result)
    body = request.body.getvalue().decode(encoding)
    params.update(QueryStringParser(body).result)


def process(request, params):
    ctype = request.headers.get("Content-Type")
    if not ctype:
        return

    mtype, mencoding = ctype.split("/", 1) if "/" in ctype else (ctype, None)

    if mtype == "multipart":
        process_multipart(request, params)
    elif mtype == "application" and mencoding == "x-www-form-urlencoded":
        process_urlencoded(request, params)<commit_msg>Fixed processors where Content-Type specifies a charset<commit_after>import re
from cgi import parse_header

from .headers import HeaderElement
from .parsers import MultipartParser
from .parsers import QueryStringParser


def process_multipart(request, params):
    headers = request.headers

    ctype = headers.elements("Content-Type")
    if ctype:
        ctype = ctype[0]
    else:
        ctype = HeaderElement.from_str("application/x-www-form-urlencoded")

    ib = ""
    if "boundary" in ctype.params:
        # http://tools.ietf.org/html/rfc2046#section-5.1.1
        # "The grammar for parameters on the Content-type field is such that it
        # is often necessary to enclose the boundary parameter values in quotes
        # on the Content-type line"
        ib = ctype.params["boundary"].strip("\"")

    if not re.match("^[ -~]{0,200}[!-~]$", ib):
        raise ValueError("Invalid boundary in multipart form: %r" % (ib,))

    parser = MultipartParser(request.body, ib)
    for part in parser:
        if part.filename or not part.is_buffered():
            params[part.name] = part
        else:
            params[part.name] = part.value


def process_urlencoded(request, params, encoding="utf-8"):
    params.update(QueryStringParser(request.qs).result)
    body = request.body.getvalue().decode(encoding)
    params.update(QueryStringParser(body).result)


def process(request, params):
    ctype = request.headers.get("Content-Type")
    if not ctype:
        return

    mtype, mencoding = ctype.split("/", 1) if "/" in ctype else (ctype, None)
    mencoding, extra = parse_header(mencoding)

    charset = extra.get("charset", "utf-8")

    if mtype == "multipart":
        process_multipart(request, params)
    elif mtype == "application" and mencoding == "x-www-form-urlencoded":
        process_urlencoded(request, params, encoding=charset)
{'fraction_non_alphanumeric': 0.12060725330334551, 'fraction_numerical': 0.008152937868990722, 'mean_word_length': 3.1180555555555554, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4907444', 'n_tokens_mistral': 1089, 'n_tokens_neox': 1036, 'n_words': 282}