<commit_before>import os.path

import mock
import io

import netlib.utils
from netlib.http import Headers
from mitmproxy import filt, controller, flow
from mitmproxy.contrib import tnetstring
from mitmproxy.exceptions import FlowReadException, ScriptException
from mitmproxy.models import Error
from mitmproxy.models import Flow
from mitmproxy.models import HTTPFlow
from mitmproxy.models import HTTPRequest
from mitmproxy.models import HTTPResponse
from mitmproxy.proxy.config import HostMatcher
from mitmproxy.proxy import ProxyConfig
from mitmproxy.proxy.server import DummyServer
from mitmproxy.models.connections import ClientConnection
from . import tutils


def test_app_registry():
    ar = flow.AppRegistry()
    ar.add("foo", "domain", 80)

    r = HTTPRequest.wrap(netlib.tutils.treq())
    r.host = "domain"
    r.port = 80
    assert ar.get(r)

    r.port = 81
    assert not ar.get(r)

    r = HTTPRequest.wrap(netlib.tutils.treq())
    r.host = "domain2"
    r.port = 80
    assert not ar.get(r)
    r.headers["host"] = "domain"
    assert ar.get(r)


class TestStickyCookieState:

    def _response(self, cookie, host):
        s = flow.StickyCookieState(filt.parse(".*"))
        f = tutils.tflow(req=netlib.tutils.treq(host=host, port=80), resp=True)
        f.response.headers["Set-Cookie"] = cookie
        s.handle_response(f)
        return s, f

    def test_domain_match(self):
        s = flow.StickyCookieState(filt.parse(".*"))
        assert s.domain_match("www.google.com", ".google.com")
        assert s.domain_match("google.com", ".google.com")

    def test_response(self):
        c = "SSID=mooo; domain=.google.com, FOO=bar; Domain=.google.com; Path=/; " \
            "Expires=Wed, 13-Jan-2021 22:23:01 GMT; Secure; "

        s, f = self._response(c, "host")
        assert not s.jar.keys()

        s, f = self._response(c, "www.google.com")
        assert s.jar.keys()

        s, f = self._response("SSID=mooo", "www.google.com")
        assert list(s.jar.keys())[0] == ('www.google.com', 80, '/')

        # Test setting of multiple cookies
        c1 = "somecookie=test; Path=/"
        c2 = "othercookie=helloworld; Path=/"
        s, f = self._response(c1, "www.google.com")
        f.response.headers["Set-Cookie"] = c2
        s.handle_response(f)
        googlekey = list(s.jar.keys())[0]
        assert len(s.jar[googlekey].keys()) == 2

        # Test setting of weird cookie keys
        s = flow.StickyCookieState(filt.parse(".*"))
        f = tutils.tflow(req=netlib.tutils.treq(host="www.google.com", port=80), resp=True)
        cs = [
            "foo/bar=hello",
            "foo:bar=world",
            "foo@bar=fizz",
            "foo,bar=buzz",
        ]
        for c in cs:
            f.response.headers["Set-Cookie"] = c
            s.handle_response(f)
        googlekey = list(s.jar.keys())[0]
        assert len(s.jar[googlekey]) == len(cs)

        # Test overwriting of a cookie value
        c1 = "somecookie=helloworld; Path=/"
        c2 = "somecookie=newvalue; Path=/"
        s, f = self._response(c1, "www.google.com")
        f.response.headers["Set-Cookie"] = c2
        s.handle_response(f)
        googlekey = list(s.jar.keys())[0]
        assert len(s.jar[googlekey]) == 1
        assert list(s.jar[googlekey]["somecookie"].values())[0] == "newvalue"

    def test_request(self):
        s, f = self._response("SSID=mooo", b"www.google.com")
        assert "cookie" not in f.request.headers
        s.handle_request(f)
        assert "cookie" in f.request.headers


class TestStickyAuthState:<commit_msg>Add tests for deletion of stickycookies<commit_after>import os.path

import mock
import io

import netlib.utils
from netlib.http import Headers
from mitmproxy import filt, controller, flow
from mitmproxy.contrib import tnetstring
from mitmproxy.exceptions import FlowReadException, ScriptException
from mitmproxy.models import Error
from mitmproxy.models import Flow
from mitmproxy.models import HTTPFlow
from mitmproxy.models import HTTPRequest
from mitmproxy.models import HTTPResponse
from mitmproxy.proxy.config import HostMatcher
from mitmproxy.proxy import ProxyConfig
from mitmproxy.proxy.server import DummyServer
from mitmproxy.models.connections import ClientConnection
from . import tutils


def test_app_registry():
    ar = flow.AppRegistry()
    ar.add("foo", "domain", 80)

    r = HTTPRequest.wrap(netlib.tutils.treq())
    r.host = "domain"
    r.port = 80
    assert ar.get(r)

    r.port = 81
    assert not ar.get(r)

    r = HTTPRequest.wrap(netlib.tutils.treq())
    r.host = "domain2"
    r.port = 80
    assert not ar.get(r)
    r.headers["host"] = "domain"
    assert ar.get(r)


class TestStickyCookieState:

    def _response(self, cookie, host):
        s = flow.StickyCookieState(filt.parse(".*"))
        f = tutils.tflow(req=netlib.tutils.treq(host=host, port=80), resp=True)
        f.response.headers["Set-Cookie"] = cookie
        s.handle_response(f)
        return s, f

    def test_domain_match(self):
        s = flow.StickyCookieState(filt.parse(".*"))
        assert s.domain_match("www.google.com", ".google.com")
        assert s.domain_match("google.com", ".google.com")

    def test_response(self):
        c = (
            "SSID=mooo; domain=.google.com, FOO=bar; Domain=.google.com; Path=/; "
            "Expires=Wed, 13-Jan-2021 22:23:01 GMT; Secure; "
        )

        s, f = self._response(c, "host")
        assert not s.jar.keys()

        s, f = self._response(c, "www.google.com")
        assert list(s.jar.keys())[0] == ('.google.com', 80, '/')

        s, f = self._response("SSID=mooo", "www.google.com")
        assert list(s.jar.keys())[0] == ('www.google.com', 80, '/')

        # Test setting of multiple cookies
        c1 = "somecookie=test; Path=/"
        c2 = "othercookie=helloworld; Path=/"
        s, f = self._response(c1, "www.google.com")
        f.response.headers["Set-Cookie"] = c2
        s.handle_response(f)
        googlekey = list(s.jar.keys())[0]
        assert len(s.jar[googlekey].keys()) == 2

        # Test setting of weird cookie keys
        s = flow.StickyCookieState(filt.parse(".*"))
        f = tutils.tflow(req=netlib.tutils.treq(host="www.google.com", port=80), resp=True)
        cs = [
            "foo/bar=hello",
            "foo:bar=world",
            "foo@bar=fizz",
            "foo,bar=buzz",
        ]
        for c in cs:
            f.response.headers["Set-Cookie"] = c
            s.handle_response(f)
        googlekey = list(s.jar.keys())[0]
        assert len(s.jar[googlekey]) == len(cs)

        # Test overwriting of a cookie value
        c1 = "somecookie=helloworld; Path=/"
        c2 = "somecookie=newvalue; Path=/"
        s, f = self._response(c1, "www.google.com")
        f.response.headers["Set-Cookie"] = c2
        s.handle_response(f)
        googlekey = list(s.jar.keys())[0]
        assert len(s.jar[googlekey]) == 1
        assert list(s.jar[googlekey]["somecookie"].values())[0] == "newvalue"

    def test_response_delete(self):
        c = "duffer=zafar; Path=/", "www.google.com"

        # Test that a cookie is be deleted
        # by setting the expire time in the past
        s, f = self._response(*c)
        f.response.headers["Set-Cookie"] = "duffer=; Expires=Thu, 01-Jan-1970 00:00:00 GMT"
        s.handle_response(f)
        assert not s.jar.keys()

        # or by setting Max-Age to 0
        s, f = self._response(*c)
        f.response.headers["Set-Cookie"] = "duffer=; Max-Age=0"
        s.handle_response(f)
        assert not s.jar.keys()

        # or both
        s, f = self._response(*c)
        f.response.headers["Set-Cookie"] = "duffer=; Expires=Thu, 01-Jan-1970 00:00:00 GMT; Max-Age=0"
        s.handle_response(f)
        assert not s.jar.keys()

    def test_request(self):
        s, f = self._response("SSID=mooo", b"www.google.com")
        assert "cookie" not in f.request.headers
        s.handle_request(f)
        assert "cookie" in f.request.headers


class TestStickyAuthState:
{'fraction_non_alphanumeric': 0.1401369010578718, 'fraction_numerical': 0.01418792781580585, 'mean_word_length': 2.6214511041009465, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 17, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5740661', 'n_tokens_mistral': 2787, 'n_tokens_neox': 2648, 'n_words': 588}