<commit_before># -*- coding: utf8 -*-
__author__ = '<NAME>'

import sys
sys.path.append('../..')

from core.Plugins.ProtocolDissector import *
from cStringIO import StringIO
from contextlib import closing

def getClassReference():
    return HTTP11

# Parses HTTP Requests / Responses according to http://tools.ietf.org/html/rfc7230
class HTTP11(ProtocolDissector):
    defaultPorts = [80, 8080, 8000, 443]

    @classmethod
    def getProtocolName(cls):
        return "HTTP 1.1"

    @classmethod
    def getRequestPayload(cls, data):
        return cls.getResponsePayload(data)     # No special case found yet that has to be handled differently

    @classmethod
    def getResponsePayload(cls, data):
        headers = cls.parseHeaders(data)
        if 'Content-Length' in headers:
            length = int(headers['Content-Length'])
            return data.read(length)

        return None

    @classmethod
    def parseHeaders(cls, data):
        headers = dict()
        line = data.readline()
        while line not in ['\r\n','']:
            keyval = line.split(':')
            headers[keyval[0].strip()] = keyval[1].strip()
            line = data.readline()
        return headers

    @classmethod
    def parseData(cls, data):
        with closing(StringIO(data)) as data:
            # check start line for HTTP 1.1 tag
            line = data.readline()
            if 'HTTP/1.1' not in line:
                return None

            payloads = []

            #loop to allow HTTP pipelining
            while line != '':<commit_msg>added HTTP Content Encoding capabilities<commit_after># -*- coding: utf8 -*-
__author__ = '<NAME>'

import sys
sys.path.append('../..')

from core.Plugins.ProtocolDissector import *
from cStringIO import StringIO
from contextlib import closing
from gzip import GzipFile

def getClassReference():
    return HTTP11


# Parses HTTP Requests / Responses according to http://tools.ietf.org/html/rfc7230
class HTTP11(ProtocolDissector):
    defaultPorts = [80, 8080, 8000, 443]

    decoders = {
        'gzip':     lambda x: GzipFile(fileobj=StringIO(x)).read(),
        'x-gzip':   lambda x: GzipFile(fileobj=StringIO(x)).read(),
        'deflate':  lambda x: x.decode('zlib'),
    }

    @classmethod
    def getProtocolName(cls):
        return "HTTP 1.1"

    @classmethod
    def getRequestPayload(cls, data):
        return cls.getResponsePayload(data)     # No special case found yet that has to be handled differently

    @classmethod
    def decode(cls, payload, encoding):
        if not payload:
            return None

        if encoding not in cls.decoders.keys():
            return payload

        try:
            return cls.decoders[encoding](payload)
        except Exception as e:
            return payload

    @classmethod
    def getResponsePayload(cls, data):
        payload = None
        encoding = None
        headers = cls.parseHeaders(data)
        if 'Content-Length' in headers:
            length = int(headers['Content-Length'])
            payload = data.read(length)

        if 'Content-Encoding' in headers:
            encoding = headers['Content-Encoding']
            encoding = encoding.split(':')[-1].strip().lower()

        if 'Transfer-Encoding' in headers:
            encoding = headers['Transfer-Encoding']
            encoding = encoding.split(':')[-1].strip().lower()

        return cls.decode(payload, encoding)

    @classmethod
    def parseHeaders(cls, data):
        headers = dict()
        line = data.readline()
        while line not in ['\r\n','']:
            keyval = line.split(':')
            headers[keyval[0].strip()] = keyval[1].strip()
            line = data.readline()
        return headers

    @classmethod
    def parseData(cls, data):
        with closing(StringIO(data)) as data:
            # check start line for HTTP 1.1 tag
            line = data.readline()
            if 'HTTP/1.1' not in line:
                return None

            payloads = []

            #loop to allow HTTP pipelining
            while line != '':
{'fraction_non_alphanumeric': 0.11127523387493846, 'fraction_numerical': 0.015263417035942885, 'mean_word_length': 2.201733648542159, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '233725', 'n_tokens_mistral': 1187, 'n_tokens_neox': 1129, 'n_words': 326}