<commit_before># -*- coding: utf-8 -
#
# This file is part of gunicorn released under the MIT license. 
# See the NOTICE for more information.

import re
import urlparse

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from gunicorn.http.body import ChunkedReader, LengthReader, EOFReader, Body
from gunicorn.http.errors import InvalidHeader, InvalidHeaderName, NoMoreData, \
InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion

class Message(object):
    def __init__(self, unreader):
        self.unreader = unreader
        self.version = None
        self.connection_hdr = None
        self.headers = []
        self.trailers = []
        self.body = None

        self.hdrre = re.compile("[\x00-\x1F\x7F()<>@,;:\[\]={} \t\\\\\"]")

        unused = self.parse(self.unreader)
        self.unreader.unread(unused)
        self.set_body_reader()
    
    def parse(self):
        raise NotImplementedError()

    def parse_headers(self, data):
        headers = []

        # Split lines on \r\n keeping the \r\n on each line
        lines = [line + "\r\n" for line in data.split("\r\n")]

        # Parse headers into key/value pairs paying attention
        # to continuation lines.
        while len(lines):
            # Parse initial header name : value pair.
            curr = lines.pop(0)
            if curr.find(":") < 0:
                raise InvalidHeader(curr.strip())
            name, value = curr.split(":", 1)
            name = name.rstrip(" \t").upper()
            if self.hdrre.search(name):
                raise InvalidHeaderName(name)
            name, value = name.strip(), [value.lstrip()]
            
            # Consume value continuation lines
            while len(lines) and lines[0].startswith((" ", "\t")):
                value.append(lines.pop(0))
            value = ''.join(value).rstrip()
           
            if name == "CONNECTION":
                self.connection_hdr = value

            headers.append((name, value))
        return headers

    def set_body_reader(self):
        chunked = False
        clength = None
        for (name, value) in self.headers:
            if name == "CONTENT-LENGTH":
                try:
                    clength = int(value)
                except ValueError:
                    clength = None
            elif name == "TRANSFER-ENCODING":
                chunked = value.lower() == "chunked"
            elif name == "SEC-WEBSOCKET-KEY1":
                clength = 8

            if clength is not None or chunked:
                break

        if chunked:
            self.body = Body(ChunkedReader(self, self.unreader))
        elif clength is not None:
            self.body = Body(LengthReader(self.unreader, clength))
        else:<commit_msg>again no need to iter headers. Get the values we need the first time<commit_after># -*- coding: utf-8 -
#
# This file is part of gunicorn released under the MIT license. 
# See the NOTICE for more information.

import re
import urlparse

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from gunicorn.http.body import ChunkedReader, LengthReader, EOFReader, Body
from gunicorn.http.errors import InvalidHeader, InvalidHeaderName, NoMoreData, \
InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion

class Message(object):
    def __init__(self, unreader):
        self.unreader = unreader
        self.version = None
        self.connection_hdr = None
        self.chunked = False
        self.clength = None
        self.headers = []
        self.trailers = []
        self.body = None

        self.hdrre = re.compile("[\x00-\x1F\x7F()<>@,;:\[\]={} \t\\\\\"]")

        unused = self.parse(self.unreader)
        self.unreader.unread(unused)
        self.set_body_reader()
    
    def parse(self):
        raise NotImplementedError()

    def parse_headers(self, data):
        headers = []

        # Split lines on \r\n keeping the \r\n on each line
        lines = [line + "\r\n" for line in data.split("\r\n")]

        # Parse headers into key/value pairs paying attention
        # to continuation lines.
        while len(lines):
            # Parse initial header name : value pair.
            curr = lines.pop(0)
            if curr.find(":") < 0:
                raise InvalidHeader(curr.strip())
            name, value = curr.split(":", 1)
            name = name.rstrip(" \t").upper()
            if self.hdrre.search(name):
                raise InvalidHeaderName(name)
            name, value = name.strip(), [value.lstrip()]
            
            # Consume value continuation lines
            while len(lines) and lines[0].startswith((" ", "\t")):
                value.append(lines.pop(0))
            value = ''.join(value).rstrip()
           
            if name == "CONNECTION":
                self.connection_hdr = value
            elif name == "CONTENT-LENGTH":
                try:
                    self.clength = int(value)
                except ValueError:
                    pass
            elif name == "TRANSFER-ENCODING":
                 self.chunked = value.lower() == "chunked"
            elif  name == "SEC-WEBSOCKET-KEY1":
                self.clength = 8

            headers.append((name, value))
        return headers

    def set_body_reader(self):
        if self.chunked:
            self.body = Body(ChunkedReader(self, self.unreader))
        elif self.clength is not None:
            self.body = Body(LengthReader(self.unreader, self.clength))
        else:
{'fraction_non_alphanumeric': 0.1098546042003231, 'fraction_numerical': 0.004308023694130318, 'mean_word_length': 1.838512480896587, 'pattern_counts': {'":': 12, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2849341', 'n_tokens_mistral': 1587, 'n_tokens_neox': 1538, 'n_words': 424}