<commit_before>from collections import namedtuple
import cgi
import enum

from streaming_form_data.finder import Finder
from streaming_form_data.targets import NullTarget
from streaming_form_data.part import Part


HYPHEN = 45

CR = 13

LF = 10


class ParseFailedException(Exception):
    pass


class ParserState(enum.Enum):
    START = -1

    STARTING_BOUNDARY = 0
    READING_BOUNDARY = 1
    ENDING_BOUNDARY = 2

    READING_HEADER = 3
    ENDING_HEADER = 4
    ENDED_HEADER = 5

    ENDING_ALL_HEADERS = 6

    READING_BODY = 7

    END = 8


def parse_content_boundary(headers):
    content_type = headers.get('Content-Type')
    if not content_type:
        raise ParseFailedException()

    value, params = cgi.parse_header(content_type)

    if not value or value.lower() != 'multipart/form-data':
        raise ParseFailedException('Content-Type not multipart/form-data')

    boundary = params.get('boundary')
    if not boundary:
        raise ParseFailedException('Boundary not found')

    return boundary.encode('utf-8')


Position = namedtuple('Position', ['buffer_start', 'buffer_end'])


class StreamingFormDataParser(object):
    """Parse multipart/form-data in chunks, one byte at a time.
    """

    def __init__(self, expected_parts, headers):
        self.expected_parts = expected_parts
        self.headers = headers

        self._raw_boundary = parse_content_boundary(headers)

        self._boundary = b'--' + self._raw_boundary
        self._delimiter = self._boundary + b'\r\n'
        self._ender = self._boundary + b'--\r\n'

        self.state = ParserState.START
        self._active_part = None

        self._default_part = Part('_default', NullTarget())

        # stores the index of the byte we're currently looking at
        self._index = -1

        # stores the indices where the current buffer starts (inclusive) and
        # ends (exclusive)
        self._buffer_start = -1
        self._buffer_end = -1

        self._max_buffer_size = 1024

        self._leftover_buffer = None

        self._delimiter_finder = Finder(self._delimiter)
        self._ender_finder = Finder(self._ender)

    def data_received(self, data):
        if not self.expected_parts or not data:
            return

        if self._leftover_buffer:
            chunk = self._leftover_buffer + data

            index = len(self._leftover_buffer)
            buffer_start = 0
            buffer_end = index

            self._leftover_buffer = None
        else:
            chunk = data

            index = 0
            buffer_start = 0
            buffer_end = 0

        self._parse(chunk, index, buffer_start, buffer_end)

    def _parse(self, chunk, index, buffer_start, buffer_end):
        position = Position(buffer_start, buffer_end)

        def expand_buffer():
            return position._replace(buffer_end=position.buffer_end+1)

        def buffer_length():
            return position.buffer_end - position.buffer_start

        def reset_buffer():
            return position._replace(buffer_start=index+1,
                                     buffer_end=index+1)

        def truncate_buffer(suffix):
            if buffer_length() <= len(suffix):
                return position

            index = position.buffer_end - len(suffix)

            self._active_part.data_received(
                chunk[position.buffer_start: index - 2])

            return reset_buffer()

        def try_flush_buffer():
            if buffer_length() <= self._max_buffer_size:
                return position<commit_msg>avoid variable naming confusion<commit_after>from collections import namedtuple
import cgi
import enum

from streaming_form_data.finder import Finder
from streaming_form_data.targets import NullTarget
from streaming_form_data.part import Part


HYPHEN = 45

CR = 13

LF = 10


class ParseFailedException(Exception):
    pass


class ParserState(enum.Enum):
    START = -1

    STARTING_BOUNDARY = 0
    READING_BOUNDARY = 1
    ENDING_BOUNDARY = 2

    READING_HEADER = 3
    ENDING_HEADER = 4
    ENDED_HEADER = 5

    ENDING_ALL_HEADERS = 6

    READING_BODY = 7

    END = 8


def parse_content_boundary(headers):
    content_type = headers.get('Content-Type')
    if not content_type:
        raise ParseFailedException()

    value, params = cgi.parse_header(content_type)

    if not value or value.lower() != 'multipart/form-data':
        raise ParseFailedException('Content-Type not multipart/form-data')

    boundary = params.get('boundary')
    if not boundary:
        raise ParseFailedException('Boundary not found')

    return boundary.encode('utf-8')


Position = namedtuple('Position', ['buffer_start', 'buffer_end'])


class StreamingFormDataParser(object):
    """Parse multipart/form-data in chunks, one byte at a time.
    """

    def __init__(self, expected_parts, headers):
        self.expected_parts = expected_parts
        self.headers = headers

        self._raw_boundary = parse_content_boundary(headers)

        self._boundary = b'--' + self._raw_boundary
        self._delimiter = self._boundary + b'\r\n'
        self._ender = self._boundary + b'--\r\n'

        self.state = ParserState.START
        self._active_part = None

        self._default_part = Part('_default', NullTarget())

        # stores the index of the byte we're currently looking at
        self._index = -1

        # stores the indices where the current buffer starts (inclusive) and
        # ends (exclusive)
        self._buffer_start = -1
        self._buffer_end = -1

        self._max_buffer_size = 1024

        self._leftover_buffer = None

        self._delimiter_finder = Finder(self._delimiter)
        self._ender_finder = Finder(self._ender)

    def data_received(self, data):
        if not self.expected_parts or not data:
            return

        if self._leftover_buffer:
            chunk = self._leftover_buffer + data

            index = len(self._leftover_buffer)
            buffer_start = 0
            buffer_end = index

            self._leftover_buffer = None
        else:
            chunk = data

            index = 0
            buffer_start = 0
            buffer_end = 0

        self._parse(chunk, index, buffer_start, buffer_end)

    def _parse(self, chunk, index, buffer_start, buffer_end):
        position = Position(buffer_start, buffer_end)

        def expand_buffer():
            return position._replace(buffer_end=position.buffer_end+1)

        def buffer_length():
            return position.buffer_end - position.buffer_start

        def reset_buffer():
            return position._replace(buffer_start=index+1,
                                     buffer_end=index+1)

        def truncate_buffer(suffix):
            if buffer_length() <= len(suffix):
                return position

            idx = position.buffer_end - len(suffix)

            self._active_part.data_received(
                chunk[position.buffer_start: idx - 2])

            return reset_buffer()

        def try_flush_buffer():
            if buffer_length() <= self._max_buffer_size:
                return position
{'fraction_non_alphanumeric': 0.081787521079258, 'fraction_numerical': 0.008993816750983699, 'mean_word_length': 2.3194962686567164, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1337386', 'n_tokens_mistral': 2136, 'n_tokens_neox': 1939, 'n_words': 488}