<commit_before>import tutil
import unittest
import sys

import webvulnscan.log
import webvulnscan.attacks.exotic_characters
from webvulnscan.page import Page

try:
    from urllib.parse import unquote
except ImportError:
    from urllib2 import unquote


class ExoticCharacterTest(unittest.TestCase):
    def setUp(self):
        webvulnscan.log.do_print = True

    def test_static_site(self):
        default_page = Page("/", "<html></html>", {}, 200)

        class StaticSite(tutil.ClientSite):
            def download_page(self, url, parameters=None,
                              remember_visited=None):
                return default_page

        webvulnscan.attacks.exotic_characters(default_page, StaticSite())

        output = sys.stdout.getvalue().strip()
        self.assertEqual(output, "")

    def test_url_vulnerable_site(self):
        default_page = Page("/?test=random", "<html></html>", {}, 200)

        class UrlVulnerableSite(tutil.ClientSite):
            def download_page(self, url, parameters=None,
                              remember_visited=None):
                try:
                    url = url.decode('ascii', 'replace')
                except AttributeError:
                    url = url
                return Page("/", "<html>" + url.replace('>', '')
                            + "</html>", {}, 200)

        webvulnscan.attacks.exotic_characters(default_page,
                                              UrlVulnerableSite())

        output = sys.stdout.getvalue().strip()
        self.assertNotEqual(output, "")

    def test_post_vulnerable_site(self):
        default_page = Page("/?test",
                            '<html><form action="/test">'
                            '<input type="text" name="random" />'
                            '</form></html>', {}, 200)<commit_msg>exotic_characters.py: Added additional unit test<commit_after>import tutil
import unittest
import sys

import webvulnscan.log
import webvulnscan.attacks.exotic_characters
from webvulnscan.page import Page

try:
    from urllib.parse import unquote
except ImportError:
    from urllib2 import unquote


class ExoticCharacterTest(unittest.TestCase):
    def setUp(self):
        webvulnscan.log.do_print = True

    def test_static_site(self):
        default_page = Page("/", "<html></html>", {}, 200)

        class StaticSite(tutil.ClientSite):
            def download_page(self, url, parameters=None,
                              remember_visited=None):
                return default_page

        webvulnscan.attacks.exotic_characters(default_page, StaticSite())

        output = sys.stdout.getvalue().strip()
        self.assertEqual(output, "")

    def test_valid_parsing(self):
        default_page = Page("/?test=random", "<html></html>", {}, 200)

        class UrlVulnerableSite(tutil.ClientSite):
            def download_page(self, url, parameters=None,
                              remember_visited=None):
                try:
                    url = url.decode('ascii', 'xmlcharrefreplace')
                except AttributeError:
                    url = url
                return Page("/", "<html>" + url
                            + "</html>", {}, 200)

        webvulnscan.attacks.exotic_characters(default_page,
                                              UrlVulnerableSite())

        output = sys.stdout.getvalue().strip()
        self.assertNotEqual(output, "")

    def test_url_vulnerable_site(self):
        default_page = Page("/?test=random", "<html></html>", {}, 200)

        class UrlVulnerableSite(tutil.ClientSite):
            def download_page(self, url, parameters=None,
                              remember_visited=None):
                try:
                    url = url.decode('ascii', 'replace')
                except AttributeError:
                    url = url
                return Page("/", "<html>" + url.replace('>', '')
                            + "</html>", {}, 200)

        webvulnscan.attacks.exotic_characters(default_page,
                                              UrlVulnerableSite())

        output = sys.stdout.getvalue().strip()
        self.assertNotEqual(output, "")

    def test_post_vulnerable_site(self):
        default_page = Page("/?test",
                            '<html><form action="/test">'
                            '<input type="text" name="random" />'
                            '</form></html>', {}, 200)
{'fraction_non_alphanumeric': 0.11784587652095539, 'fraction_numerical': 0.007210455159981974, 'mean_word_length': 1.843689942344651, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1984863', 'n_tokens_mistral': 1252, 'n_tokens_neox': 1181, 'n_words': 223}