<commit_before>import string
import regex
from collections import deque
from flanker.mime.message.headers import encodedword, parametrized
from flanker.mime.message.headers.wrappers import ContentType, WithParams
from flanker.mime.message.errors import DecodingError
from flanker.mime.message.utils import to_unicode
from flanker.utils import is_pure_ascii

MAX_LINE_LENGTH = 10000


def normalize(header):
    return string.capwords(header.lower(), '-')


def parse_stream(stream):
    """Reads the incoming stream and returns list of tuples"""
    out = deque()
    for header in unfold(split(stream)):
        out.append(parse_header(header))
    return out


def parse_header(header):
    """ Accepts a raw header with name, colons and newlines
    and returns it's parsed value
    """
    name, val = split2(header)
    if not is_pure_ascii(name):
        raise DecodingError("Non-ascii header name")
    return name, parse_header_value(name, encodedword.unfold(val))


def parse_header_value(name, val):
    if not is_pure_ascii(val):
        if parametrized.is_parametrized(name, val):
            raise DecodingError("Unsupported value in content- header")
        return to_unicode(val)
    else:
        if parametrized.is_parametrized(name, val):
            val, params = parametrized.decode(val)
            if name == 'Content-Type':
                main, sub = parametrized.fix_content_type(val)
                return ContentType(main, sub, params)
            else:
                return WithParams(val, params)
        else:
            return val


def is_empty(line):
    return line in ('\r\n', '\r', '\n')


RE_HEADER = regex.compile(r'^(From |[\041-\071\073-\176]+:|[\t ])')


def split(fp):
    """Read lines with headers until the start of body"""
    lines = deque()<commit_msg>Handle non-ascii characters in parameter values.<commit_after>import string
import regex
from collections import deque
from flanker.mime.message.headers import encodedword, parametrized
from flanker.mime.message.headers.wrappers import ContentType, WithParams
from flanker.mime.message.errors import DecodingError
from flanker.mime.message.utils import to_unicode
from flanker.utils import is_pure_ascii

MAX_LINE_LENGTH = 10000


def normalize(header):
    return string.capwords(header.lower(), '-')


def parse_stream(stream):
    """Reads the incoming stream and returns list of tuples"""
    out = deque()
    for header in unfold(split(stream)):
        out.append(parse_header(header))
    return out


def parse_header(header):
    """ Accepts a raw header with name, colons and newlines
    and returns it's parsed value
    """
    name, val = split2(header)
    if not is_pure_ascii(name):
        raise DecodingError("Non-ascii header name")
    return name, parse_header_value(name, encodedword.unfold(val))


def parse_header_value(name, val):
    if parametrized.is_parametrized(name, val):
        val, params = parametrized.decode(val)
        if name == 'Content-Type':
            main, sub = parametrized.fix_content_type(val)
            return ContentType(main, sub, params)
        else:
            return WithParams(val, params)
    else:
        return val if is_pure_ascii(val) else to_unicode(val)


def is_empty(line):
    return line in ('\r\n', '\r', '\n')


RE_HEADER = regex.compile(r'^(From |[\041-\071\073-\176]+:|[\t ])')


def split(fp):
    """Read lines with headers until the start of body"""
    lines = deque()
{'fraction_non_alphanumeric': 0.10584152689415847, 'fraction_numerical': 0.010410641989589358, 'mean_word_length': 3.3619167717528375, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6946069', 'n_tokens_mistral': 1113, 'n_tokens_neox': 1006, 'n_words': 295}