<commit_before>from cement.utils import test
from common.testutils import decallmethods, xml_validate
from lxml import etree
from mock import patch, MagicMock
from plugins.drupal import Drupal
from requests.exceptions import ConnectionError
from tests import BaseTest
import hashlib
import requests
import responses

@decallmethods(responses.activate)
class FingerprintTests(BaseTest):
    """
        Tests related to version fingerprinting for all plugins.
    """

    versions_xsd = "common/versions.xsd"
    xml_file = "tests/versions.xml"

    class MockHash():
        files = None
        def mock_func(self, *args, **kwargs):
            url = kwargs['file_url']
            return self.files[url]

    def setUp(self):
        super(FingerprintTests, self).setUp()
        self.add_argv(["drupal"])
        self.add_argv(["--method", "forbidden"])
        self.add_argv(self.param_version)
        self.scanner = Drupal()

    def mock_xml(self, xml_file, version_to_mock):
        """
            generates all mock data, and patches Drupal.get_hash
        """
        with open(xml_file) as f:
            doc = etree.fromstring(f.read())
            files_xml = doc.xpath("//cms/files/file")

            files = {}
            for file in files_xml:
                url = file.get("url")
                versions = file.xpath("version")
                for file_version in versions:
                    version_number = file_version.get("nb")
                    md5 = file_version.get("md5")

                    if version_number == version_to_mock:
                        files[url] = md5

        mock_hash = self.MockHash()
        mock_hash.files = files
        mock = MagicMock(side_effect=mock_hash.mock_func)

        return mock

    @patch('common.VersionsFile.files_get', return_value=['misc/drupal.js'])
    def test_calls_version(self, m):
        responses.add(responses.HEAD, self.base_url + self.scanner.changelog)
        responses.add(responses.GET, self.base_url + 'misc/drupal.js')
        # with no mocked calls, any HTTP req will cause a ConnectionError.
        self.app.run()

    @test.raises(ConnectionError)
    def test_calls_version_no_mock(self):
        # with no mocked calls, any HTTP req will cause a ConnectionError.
        self.app.run()

    @patch('common.warn')
    @patch('common.VersionsFile.files_get', return_value=['misc/drupal.js'])
    def test_fingerprint_warns_if_changelog(self, m, warn):
        responses.add(responses.HEAD, self.base_url + self.scanner.changelog)
        responses.add(responses.GET, self.base_url + 'misc/drupal.js')

        self.app.run()

        assert warn.called, "should have warned about changelog being present."

    def test_xml_validates_all(self):
        assert False
        # make it so that it searches for versions.xml and makes sure all verify
        # against the XSD
        drupal = Drupal()

        xml_validate(drupal.versions_file, self.versions_xsd)

    def test_determines_version(self):
        # mock the URLs needed
        real_version = "7.26"
        self.scanner.enumerate_file_hash = self.mock_xml(self.xml_file, real_version)

        version, is_empty = self.scanner.enumerate_version(self.base_url, self.xml_file)

        assert version[0] == real_version
        assert is_empty == False

    def test_enumerate_hash(self):
        file_url = "/misc/drupal.js"
        body = "zjyzjy2076"
        responses.add(responses.GET, self.base_url + file_url, body=body)

        actual_md5 = hashlib.md5(body).hexdigest()

        md5 = self.scanner.enumerate_file_hash(self.base_url, file_url)

        assert md5 == actual_md5

    @patch('common.VersionsFile.files_get', return_value=['misc/drupal.js'])
    def test_fingerprint_correct_verb(self, patch):
        responses.add(responses.HEAD, self.base_url + self.scanner.changelog)
        # this needs to be a get, otherwise, how are going to get the request body?
        responses.add(responses.GET, self.base_url + 'misc/drupal.js')

        # will exception if attempts to HEAD<commit_msg>add validation to all versions.xml files.<commit_after>from cement.utils import test
from common.testutils import decallmethods, xml_validate
from glob import glob
from lxml import etree
from mock import patch, MagicMock
from plugins.drupal import Drupal
from requests.exceptions import ConnectionError
from tests import BaseTest
import hashlib
import requests
import responses

@decallmethods(responses.activate)
class FingerprintTests(BaseTest):
    """
        Tests related to version fingerprinting for all plugins.
    """

    versions_xsd = "common/versions.xsd"
    xml_file = "tests/versions.xml"

    class MockHash():
        files = None
        def mock_func(self, *args, **kwargs):
            url = kwargs['file_url']
            return self.files[url]

    def setUp(self):
        super(FingerprintTests, self).setUp()
        self.add_argv(["drupal"])
        self.add_argv(["--method", "forbidden"])
        self.add_argv(self.param_version)
        self.scanner = Drupal()

    def mock_xml(self, xml_file, version_to_mock):
        """
            generates all mock data, and patches Drupal.get_hash
        """
        with open(xml_file) as f:
            doc = etree.fromstring(f.read())
            files_xml = doc.xpath("//cms/files/file")

            files = {}
            for file in files_xml:
                url = file.get("url")
                versions = file.xpath("version")
                for file_version in versions:
                    version_number = file_version.get("nb")
                    md5 = file_version.get("md5")

                    if version_number == version_to_mock:
                        files[url] = md5

        mock_hash = self.MockHash()
        mock_hash.files = files
        mock = MagicMock(side_effect=mock_hash.mock_func)

        return mock

    @patch('common.VersionsFile.files_get', return_value=['misc/drupal.js'])
    def test_calls_version(self, m):
        responses.add(responses.HEAD, self.base_url + self.scanner.changelog)
        responses.add(responses.GET, self.base_url + 'misc/drupal.js')
        # with no mocked calls, any HTTP req will cause a ConnectionError.
        self.app.run()

    @test.raises(ConnectionError)
    def test_calls_version_no_mock(self):
        # with no mocked calls, any HTTP req will cause a ConnectionError.
        self.app.run()

    @patch('common.warn')
    @patch('common.VersionsFile.files_get', return_value=['misc/drupal.js'])
    def test_fingerprint_warns_if_changelog(self, m, warn):
        responses.add(responses.HEAD, self.base_url + self.scanner.changelog)
        responses.add(responses.GET, self.base_url + 'misc/drupal.js')

        self.app.run()

        assert warn.called, "should have warned about changelog being present."

    def test_xml_validates_all(self):
        for xml_path in glob("plugins/*/versions.xml"):
            xml_validate(xml_path, self.versions_xsd)

    def test_determines_version(self):
        # mock the URLs needed
        real_version = "7.26"
        self.scanner.enumerate_file_hash = self.mock_xml(self.xml_file, real_version)

        version, is_empty = self.scanner.enumerate_version(self.base_url, self.xml_file)

        assert version[0] == real_version
        assert is_empty == False

    def test_enumerate_hash(self):
        file_url = "/misc/drupal.js"
        body = "zjyzjy2076"
        responses.add(responses.GET, self.base_url + file_url, body=body)

        actual_md5 = hashlib.md5(body).hexdigest()

        md5 = self.scanner.enumerate_file_hash(self.base_url, file_url)

        assert md5 == actual_md5

    @patch('common.VersionsFile.files_get', return_value=['misc/drupal.js'])
    def test_fingerprint_correct_verb(self, patch):
        responses.add(responses.HEAD, self.base_url + self.scanner.changelog)
        # this needs to be a get, otherwise, how are going to get the request body?
        responses.add(responses.GET, self.base_url + 'misc/drupal.js')

        # will exception if attempts to HEAD
{'fraction_non_alphanumeric': 0.10159436410826844, 'fraction_numerical': 0.003955011741441108, 'mean_word_length': 2.6565747853592407, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 33}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4862518', 'n_tokens_mistral': 2500, 'n_tokens_neox': 2431, 'n_words': 548}