<commit_before>#
# growler/http.py
#

__all__ = ['HTTPRequest', 'HTTPResponse', 'HTTPParser', 'HTTPError']

import asyncio
import sys
import time
import json
from urllib.parse import (quote, urlparse, parse_qs)
from pprint import PrettyPrinter

from datetime import (datetime, timezone, timedelta)

import mimetypes
from termcolor import colored

mimetypes.init()

import growler

KB = 1024
MB = KB ** 2

MAX_REQUEST_LENGTH = 4096 # 4KB
MAX_POST_LENGTH = 2 * 1024**3 # 2MB

# MAX_REQUEST_LENGTH = 96

# End of line and end of header
# EOL = "\r\n"
EOL = "\n"
HEADER_DELIM = EOL * 2

HTTPCodes = {
  200 : "OK",
  301 : "Moved Permanently",
  302 : "Found"
}

class HTTPParser(object):

  def __init__(self, req, instream = None, color = 'white'):
    """
    Create a Growler HTTP Parser.

    req: a Growler HTTPRequest object
    instream: the asyncio.streams.StreamReader (defaults to req._stream)"
    """
    self._stream = req._stream if not instream else instream
    self._req = req
    self._buffer = ''
    self.EOL_TOKEN = None
    self.read_buffer = ''
    self.bytes_read = 0
    self.max_read_size = MAX_REQUEST_LENGTH
    self.max_bytes_read = MAX_REQUEST_LENGTH
    self.data_future = asyncio.Future()
    self.c = color

  @asyncio.coroutine
  def _read_data(self):
    """Reads in a block of data (at most self.max_read_size bytes) and returns the decoded string"""
    data = yield from self._stream.read(self.max_read_size)
    self.bytes_read += len(data)
    return data.decode()

  @asyncio.coroutine
  def read_next_line(self, line_future = None):
    """
    Returns a single line read from the stream. If the EOL char has not been
    determined, it will wait for '\n' and set EOL to either LF or CRLF. This
    returns a single line, and stores the rest of the read in data in self._buffer
    """
    ## Keep reading data until newline is found
    while self.EOL_TOKEN == None:
      next_str = yield from self._read_data()
      line_end_pos = next_str.find("\n")
      self._buffer += next_str
      if line_end_pos != -1:
        self.EOL_TOKEN = "\r\n" if next_str[line_end_pos-1] == '\r' else "\n"

    line_ends_at = self._buffer.find(self.EOL_TOKEN)

    while line_ends_at == -1:
      self._buffer += yield from self._read_data()
      line_ends_at = self._buffer.find(self.EOL_TOKEN)

    ## split!
    line, self._buffer = self._buffer.split(self.EOL_TOKEN, 1)

    # yield from asyncio.sleep(0.75) # artificial delay
    return line

  @asyncio.coroutine
  def read_next_header(self, header = None):
    """
    A coroutine to generate headers. It uses read_next_line to get headers
    line by line - allowing quick response to invalid requests.
    """
    line = yield from self.read_next_line()

    if line == '': return None

    try:
      key, value = map(str.strip, line.split(":", 1))
    except ValueError as e:
      print (colored("ERROR parsing headers. Input '{}'".format(line), 'red'))
      raise HTTPBadRequest(e)

    return {'key':key, 'value':value}

  @asyncio.coroutine
  def read_body(self):
    """
    Finishes reading the request. This method expects content-length to be defined in self.headers, and will
    read that many bytes from the stream
    """
    # There is no body - return None
    if self.headers['content-length'] == 0:
      return None

    # Get length of whatever is already read into the buffer
    bytes_read = len(self._buffer)
    if self.headers['content-length'] < bytes_read:
      raise "Body too large!"

    self.max_read_size = self.headers['content-length'] - bytes_read

    while self.max_read_size != 0:
      next_str = yield from self._read_data()
      bytes_read += len(next_str)
      self.max_read_size = self.headers['content-length'] - bytes_read
      self._buffer += next_str<commit_msg>Implemented multi-line headers<commit_after>#
# growler/http.py
#

__all__ = ['HTTPRequest', 'HTTPResponse', 'HTTPParser', 'HTTPError']

import asyncio
import sys
import time
import json
from urllib.parse import (quote, urlparse, parse_qs)
from pprint import PrettyPrinter

from datetime import (datetime, timezone, timedelta)

import mimetypes
from termcolor import colored

mimetypes.init()

import growler

KB = 1024
MB = KB ** 2

MAX_REQUEST_LENGTH = 4096 # 4KB
MAX_POST_LENGTH = 2 * 1024**3 # 2MB

# MAX_REQUEST_LENGTH = 96

# End of line and end of header
# EOL = "\r\n"
EOL = "\n"
HEADER_DELIM = EOL * 2

HTTPCodes = {
  200 : "OK",
  301 : "Moved Permanently",
  302 : "Found"
}

class HTTPParser(object):

  def __init__(self, req, instream = None, color = 'white'):
    """
    Create a Growler HTTP Parser.

    req: a Growler HTTPRequest object
    instream: the asyncio.streams.StreamReader (defaults to req._stream)"
    """
    self._stream = req._stream if not instream else instream
    self._req = req
    self._buffer = ''
    self.EOL_TOKEN = None
    self.read_buffer = ''
    self.bytes_read = 0
    self.max_read_size = MAX_REQUEST_LENGTH
    self.max_bytes_read = MAX_REQUEST_LENGTH
    self.data_future = asyncio.Future()
    self.c = color
    self._header_buffer = ''

  @asyncio.coroutine
  def _read_data(self):
    """Reads in a block of data (at most self.max_read_size bytes) and returns the decoded string"""
    data = yield from self._stream.read(self.max_read_size)
    self.bytes_read += len(data)
    return data.decode()

  @asyncio.coroutine
  def read_next_line(self, line_future = None):
    """
    Returns a single line read from the stream. If the EOL char has not been
    determined, it will wait for '\n' and set EOL to either LF or CRLF. This
    returns a single line, and stores the rest of the read in data in self._buffer
    """
    ## Keep reading data until newline is found
    while self.EOL_TOKEN == None:
      next_str = yield from self._read_data()
      line_end_pos = next_str.find("\n")
      self._buffer += next_str
      if line_end_pos != -1:
        self.EOL_TOKEN = "\r\n" if next_str[line_end_pos-1] == '\r' else "\n"

    line_ends_at = self._buffer.find(self.EOL_TOKEN)

    while line_ends_at == -1:
      self._buffer += yield from self._read_data()
      line_ends_at = self._buffer.find(self.EOL_TOKEN)

    ## split!
    line, self._buffer = self._buffer.split(self.EOL_TOKEN, 1)

    # yield from asyncio.sleep(0.75) # artificial delay
    return line

  @asyncio.coroutine
  def read_next_header(self, header = None):
    """
    A coroutine to generate headers. It uses read_next_line to get headers
    line by line - allowing quick response to invalid requests.
    """
    # A _header_buffer set to None is the key that we've hit the end of headers
    if self._header_buffer == None:
      return None
    # if there was a header saved, use that (and clear the buffer)
    elif self._header_buffer != '':
      line, self._header_buffer = self._header_buffer, ''
    else:
      line = yield from self.read_next_line()

    if line == '': return None

    if line[0] == ' ':
      print ("WARNING: Header leads with whitespace")

    next_line = yield from self.read_next_line()
    
    # Loop through 
    if next_line == '':
      self._header_buffer = None
    else:
      while next_line[0] == ' ':
        line += next_line
        next_line = yield from self.read_next_line()
        if next_line == '':
          self._header_buffer = None
          break

    if self._header_buffer != None:
      self._header_buffer = next_line

    try:
      key, value = map(str.strip, line.split(":", 1))
    except ValueError as e:
      print (colored("ERROR parsing headers. Input '{}'".format(line), 'red'))
      raise HTTPBadRequest(e)

    return {'key':key, 'value':value}

  @asyncio.coroutine
  def read_body(self):
    """
    Finishes reading the request. This method expects content-length to be defined in self.headers, and will
    read that many bytes from the stream
    """
    # There is no body - return None
    if self.headers['content-length'] == 0:
      return None

    # Get length of whatever is already read into the buffer
    bytes_read = len(self._buffer)
    if self.headers['content-length'] < bytes_read:
      raise "Body too large!"

    self.max_read_size = self.headers['content-length'] - bytes_read

    while self.max_read_size != 0:
      next_str = yield from self._read_data()
      bytes_read += len(next_str)
      self.max_read_size = self.headers['content-length'] - bytes_read
      self._buffer += next_str
{'fraction_non_alphanumeric': 0.10496386684042175, 'fraction_numerical': 0.009714488804644, 'mean_word_length': 3.0528084493518963, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6091007', 'n_tokens_mistral': 2787, 'n_tokens_neox': 2708, 'n_words': 833}