<commit_before># -*- coding: utf-8 -*-

'''
gds.burp.models
~~~~~~~~~~~~~~~

This module contains the primary objects that make working with
Burp's IHttpRequestResponse object's more... Pythonic.
'''
import java.lang
from Cookie import SimpleCookie
from cgi import parse_qs
from urlparse import urlparse

from .structures import CaseInsensitiveDict


CRLF = '\r\n'
SP = chr(0x20)


class HttpRequest(object):
    '''The :class:`HttpRequest <HttpRequest>` object. Pass Burp's
    IHttpRequestResponse object to the constructor.

    Optional init arguments:
    :param callbacks: IBurpExtenderCallbacks
    '''
    def __init__(self, messageInfo, callbacks=None):
        self._messageInfo = messageInfo
        self._request = None
        self._callbacks = callbacks

        self.host = messageInfo.getHost()
        self.port = messageInfo.getPort()
        self.protocol = messageInfo.getProtocol()

        self.method = None
        self.url = None
        self.version = None
        self.headers = CaseInsensitiveDict()
        self.cookies = SimpleCookie()
        self.body = None
        self.response = None

        self.url = urlparse(messageInfo.getUrl().toString())

        if messageInfo.getRequest():
            self._request = messageInfo.getRequest().tostring()

        self.method, self._uri, self.version, self.headers, self.body = \
            _parse_message(self._request)

        self.parameters = _parse_parameters(self)
        self.cookies.load(self.headers.get('cookie', ''))

        self.response = HttpResponse(getattr(messageInfo, 'response', None),
                                     request=self)


    def __len__(self):
        return int(self.headers.get('content-length', len(self.body or '')))


    def __repr__(self):
        return '<HttpRequest [%s]>' % (self.url.path,)


    @property
    def content_type(self):
        '''
        Content-Type of the HTTP request.
        '''
        return self.headers.get('content-type', '')


    @property
    def is_secure(self):
        '''
        True if the HTTP request was sent over HTTPS.
        '''
        return True if self.protocol == 'https' else False


    @property
    def raw(self):
        '''
        Returns the raw, unparsed HTTP request.
        '''
        return self._request


    @property
    def raw_headers(self):
        '''
        Returns just the raw, unparsed HTTP request headers.
        '''
        request_headers, _ = self._request.split(CRLF + CRLF, 1)
        return request_headers


    def add_comment(self, comment, append=True):
        '''
        Helper method to add comment to underlying IHttpRequestResponse
        object.

        :param comment: the comment
        :param append: if True, append comment to existing comment if any.
        '''
        try:
            if append is True:
                _comment = self._messageInfo.getComment()
                if _comment:
                    comment = '%s, %s' % (_comment, comment)
                self._messageInfo.setComment(comment)
            else:
                self._messageInfo.setComment(comment)
        except java.lang.Exception:
            print '[*] Could not set comment %s' % (comment,)

        return


class HttpResponse(object):
    def __init__(self, messageInfo, request=None):
        self._response = None
        self.request = request

        self.version = None
        self.status_code = None
        self.reason = None
        self.encoding = None
        self.headers = CaseInsensitiveDict()
        self.cookies = SimpleCookie()
        self.body = None

        if messageInfo is not None:
            self._response = messageInfo.tostring()
            self.version, self.status_code, self.reason, self.headers, self.body = \
                _parse_message(self._response)

        self.cookies.load(self.headers.get('set-cookie', ''))

    def __len__(self):
        return int(self.headers.get('content-length', len(self.body or '')))


    def __repr__(self):
        return '<HttpResponse [%s]>' % (self.status_code, )


    @property
    def content_type(self):
        '''
        Content-Type of the HTTP response.
        '''
        return self.headers.get('content-type', '')


    @property
    def raw(self):
        '''
        Returns the raw, unparsed HTTP response.
        '''
        return self._response


    @property
    def raw_headers(self):
        '''
        Returns just the raw, unparsed HTTP response headers.
        '''
        response_headers , _ = self._response.split(CRLF + CRLF, 1)
        return response_headers


    def add_comment(self, comment, append=True):
        '''
        Helper method to add comment to underlying IHttpRequestResponse
        object.

        :param comment: the comment
        :param append: if True, append comment to existing comment if any.
        '''
        try:
            self.request.add_comment(comment, append=append)
        except java.lang.Exception, reason:
            print '[*] Could not set comment %s: %s' % (comment, reason,)

        return


def _parse_message(message):
    is_response = False
    pos = idx = 0

    idx = message.find(CRLF, pos)

    if idx != -1:
        start_line = message[pos:idx]

        if start_line.startswith('HTTP/'):
            is_response = True

        _idx = start_line.find(SP)

        if _idx != -1:
            if is_response:
                version = start_line[0:_idx]
            else:
                method = start_line[0:_idx]

            _pos = _idx + 1

        if is_response:
            _idx = start_line.find(SP, _pos)

            status = start_line[_pos:_idx]
            if not status.isdigit():
                raise ValueError('status code %r is not a number' % (status,))

            status = int(status)

            _pos = _idx + 1
            reason = start_line[_pos:]

        else:
            # work out the http version by looking in reverse
            _ridx = start_line.rfind(SP)
            version = start_line[_ridx + 1:]
            if not version.startswith('HTTP/'):
                raise ValueError('Invalid HTTP version: %r' % (version,))

            # request-uri will be everything in-between.
            # some clients might not encode space into a plus or %20
            uri = start_line[_pos:_ridx]
            if not uri or uri.isspace():
                raise ValueError('Invalid URI: %r' % (uri,))

        pos = idx + 2
    else:
        raise ValueError('Could not parse start-line from message')

    headers = CaseInsensitiveDict()

    while (idx != -1):
        idx = message.find(CRLF, pos)

        if idx == pos:
            # we've reached the end of the request headers
            # advance 4 bytes (2 * CRLF)
            pos = idx + 2
            break

        if idx != -1:
            header = message[pos:idx]
            _idx = header.find(':')

            if _idx != -1:
                name = header[:_idx].strip()
                value = header[_idx+1:].strip()

                has_value = headers.get(name)

                if has_value and has_value != value:
                    value = ', '.join([has_value, value])

                headers[name] = value
            else:
                raise ValueError('Error parsing header: %r' % (header,))

            pos = idx + 2
        else:
            # looks like we reached the end of the message before EOL
            break

    if idx < len(message):
        body = message[pos:]
    else:
        raise ValueError('Parsed past message body??')

    if not is_response:
        return method, uri, version, headers, body
    else:
        return version, status, reason, headers, body


def _parse_parameters(request):
    parameters = {}

    parameters['query'] = parse_qs(request.url.query, keep_blank_values=True)

    _type = request.headers.get('content-type')

    if _type == 'application/x-www-form-urlencoded':
        parameters['body'] = parse_qs(request.body, keep_blank_values=True)

    elif _type in ('application/json', ):
        try:
            parameters['body'] = json.loads(request.body)
        except (NameError, TypeError):
            pass

    elif _type == 'application/x-amf':
        pass

    elif _type == 'text/x-gwt-rpc':
        pass

    elif _type == 'application/xml':
        pass

    return parameters
<commit_msg>Add __contains__ special method to test if item is in request body or response body<commit_after># -*- coding: utf-8 -*-

'''
gds.burp.models
~~~~~~~~~~~~~~~

This module contains the primary objects that make working with
Burp's IHttpRequestResponse object's more... Pythonic.
'''
import java.lang
from Cookie import SimpleCookie
from cgi import parse_qs
from urlparse import urlparse

from .structures import CaseInsensitiveDict


CRLF = '\r\n'
SP = chr(0x20)


class HttpRequest(object):
    '''The :class:`HttpRequest <HttpRequest>` object. Pass Burp's
    IHttpRequestResponse object to the constructor.

    Optional init arguments:
    :param callbacks: IBurpExtenderCallbacks
    '''
    def __init__(self, messageInfo, callbacks=None):
        self._messageInfo = messageInfo
        self._request = None
        self._callbacks = callbacks

        self.host = messageInfo.getHost()
        self.port = messageInfo.getPort()
        self.protocol = messageInfo.getProtocol()

        self.method = None
        self.url = None
        self.version = None
        self.headers = CaseInsensitiveDict()
        self.cookies = SimpleCookie()
        self.body = None
        self.response = None

        self.url = urlparse(messageInfo.getUrl().toString())

        if messageInfo.getRequest():
            self._request = messageInfo.getRequest().tostring()

        self.method, self._uri, self.version, self.headers, self.body = \
            _parse_message(self._request)

        self.parameters = _parse_parameters(self)
        self.cookies.load(self.headers.get('cookie', ''))

        self.response = HttpResponse(getattr(messageInfo, 'response', None),
                                     request=self)


    def __contains__(self, item):
        return item in self.body if self.body else False


    def __len__(self):
        return int(self.headers.get('content-length', len(self.body or '')))


    def __repr__(self):
        return '<HttpRequest [%s]>' % (self.url.path,)


    @property
    def content_type(self):
        '''
        Content-Type of the HTTP request.
        '''
        return self.headers.get('content-type', '')


    @property
    def is_secure(self):
        '''
        True if the HTTP request was sent over HTTPS.
        '''
        return True if self.protocol == 'https' else False


    @property
    def raw(self):
        '''
        Returns the raw, unparsed HTTP request.
        '''
        return self._request


    @property
    def raw_headers(self):
        '''
        Returns just the raw, unparsed HTTP request headers.
        '''
        request_headers, _ = self._request.split(CRLF + CRLF, 1)
        return request_headers


    def add_comment(self, comment, append=True):
        '''
        Helper method to add comment to underlying IHttpRequestResponse
        object.

        :param comment: the comment
        :param append: if True, append comment to existing comment if any.
        '''
        try:
            if append is True:
                _comment = self._messageInfo.getComment()
                if _comment:
                    comment = '%s, %s' % (_comment, comment)
                self._messageInfo.setComment(comment)
            else:
                self._messageInfo.setComment(comment)
        except java.lang.Exception:
            print '[*] Could not set comment %s' % (comment,)

        return


class HttpResponse(object):
    def __init__(self, messageInfo, request=None):
        self._response = None
        self.request = request

        self.version = None
        self.status_code = None
        self.reason = None
        self.encoding = None
        self.headers = CaseInsensitiveDict()
        self.cookies = SimpleCookie()
        self.body = None

        if messageInfo is not None:
            self._response = messageInfo.tostring()
            self.version, self.status_code, self.reason, self.headers, self.body = \
                _parse_message(self._response)

        self.cookies.load(self.headers.get('set-cookie', ''))


    def __contains__(self, item):
        return item in self.body if self.body else False


    def __len__(self):
        return int(self.headers.get('content-length', len(self.body or '')))


    def __repr__(self):
        return '<HttpResponse [%s]>' % (self.status_code, )


    @property
    def content_type(self):
        '''
        Content-Type of the HTTP response.
        '''
        return self.headers.get('content-type', '')


    @property
    def raw(self):
        '''
        Returns the raw, unparsed HTTP response.
        '''
        return self._response


    @property
    def raw_headers(self):
        '''
        Returns just the raw, unparsed HTTP response headers.
        '''
        response_headers , _ = self._response.split(CRLF + CRLF, 1)
        return response_headers


    def add_comment(self, comment, append=True):
        '''
        Helper method to add comment to underlying IHttpRequestResponse
        object.

        :param comment: the comment
        :param append: if True, append comment to existing comment if any.
        '''
        try:
            self.request.add_comment(comment, append=append)
        except java.lang.Exception, reason:
            print '[*] Could not set comment %s: %s' % (comment, reason,)

        return


def _parse_message(message):
    is_response = False
    pos = idx = 0

    idx = message.find(CRLF, pos)

    if idx != -1:
        start_line = message[pos:idx]

        if start_line.startswith('HTTP/'):
            is_response = True

        _idx = start_line.find(SP)

        if _idx != -1:
            if is_response:
                version = start_line[0:_idx]
            else:
                method = start_line[0:_idx]

            _pos = _idx + 1

        if is_response:
            _idx = start_line.find(SP, _pos)

            status = start_line[_pos:_idx]
            if not status.isdigit():
                raise ValueError('status code %r is not a number' % (status,))

            status = int(status)

            _pos = _idx + 1
            reason = start_line[_pos:]

        else:
            # work out the http version by looking in reverse
            _ridx = start_line.rfind(SP)
            version = start_line[_ridx + 1:]
            if not version.startswith('HTTP/'):
                raise ValueError('Invalid HTTP version: %r' % (version,))

            # request-uri will be everything in-between.
            # some clients might not encode space into a plus or %20
            uri = start_line[_pos:_ridx]
            if not uri or uri.isspace():
                raise ValueError('Invalid URI: %r' % (uri,))

        pos = idx + 2
    else:
        raise ValueError('Could not parse start-line from message')

    headers = CaseInsensitiveDict()

    while (idx != -1):
        idx = message.find(CRLF, pos)

        if idx == pos:
            # we've reached the end of the request headers
            # advance 4 bytes (2 * CRLF)
            pos = idx + 2
            break

        if idx != -1:
            header = message[pos:idx]
            _idx = header.find(':')

            if _idx != -1:
                name = header[:_idx].strip()
                value = header[_idx+1:].strip()

                has_value = headers.get(name)

                if has_value and has_value != value:
                    value = ', '.join([has_value, value])

                headers[name] = value
            else:
                raise ValueError('Error parsing header: %r' % (header,))

            pos = idx + 2
        else:
            # looks like we reached the end of the message before EOL
            break

    if idx < len(message):
        body = message[pos:]
    else:
        raise ValueError('Parsed past message body??')

    if not is_response:
        return method, uri, version, headers, body
    else:
        return version, status, reason, headers, body


def _parse_parameters(request):
    parameters = {}

    parameters['query'] = parse_qs(request.url.query, keep_blank_values=True)

    _type = request.headers.get('content-type')

    if _type == 'application/x-www-form-urlencoded':
        parameters['body'] = parse_qs(request.body, keep_blank_values=True)

    elif _type in ('application/json', ):
        try:
            parameters['body'] = json.loads(request.body)
        except (NameError, TypeError):
            pass

    elif _type == 'application/x-amf':
        pass

    elif _type == 'text/x-gwt-rpc':
        pass

    elif _type == 'application/xml':
        pass

    return parameters

{'fraction_non_alphanumeric': 0.10498440534337669, 'fraction_numerical': 0.0029423880421349967, 'mean_word_length': 1.9269720978298313, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1632741', 'n_tokens_mistral': 4818, 'n_tokens_neox': 4566, 'n_words': 1300}