<commit_before>import unittest

import requests
import responses
from mock import patch

from MyCapytain.retrievers.cts5 import *


class TestEndpointsCts5(unittest.TestCase):

    """ Test Cts5 Endpoint request making """

    def setUp(self):
        self.cts = HttpCtsRetriever("http://domainname.com/rest/cts")

    def test_request_Cts_getCapabilities_arguments(self):
        """ Tests that methods getCapabilities maps correctly to request"""
        with patch('requests.get') as patched_get:
            self.cts.getCapabilities(inventory="inventory")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetCapabilities"
                }
            )

    def test_request_Cts_getValidReff_arguments(self):
        """ Tests that methods getValidReff maps correctly to request"""
        with patch('requests.get') as patched_get:
            self.cts.getValidReff(urn="urn", inventory="inventory", level=1)
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetValidReff",
                    "level": "1",
                    "urn": "urn"
                }
            )
        with patch('requests.get') as patched_get:
            self.cts.getValidReff(urn="urn", inventory="inventory")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetValidReff",
                    "urn": "urn"
                }
            )

    def test_request_Cts_getPassage_arguments(self):
        """ Tests that methods getPassage maps correctly to request"""
        with patch('requests.get') as patched_get:
            self.cts.getPassage(urn="urn", inventory="inventory", context=1)
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetPassage",
                    "context": "1",
                    "urn": "urn"
                }
            )
        with patch('requests.get') as patched_get:
            self.cts.getPassage(urn="urn", inventory="inventory")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetPassage",
                    "urn": "urn"
                }
            )

    def test_call_with_default(self):
        inv = HttpCtsRetriever("http://domainname.com/rest/cts", inventory="annotsrc")
        with patch('requests.get') as patched_get:
            inv.getPassage(urn="urn")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "annotsrc",
                    "request": "GetPassage",
                    "urn": "urn"
                }
            )

    def test_request_Cts_getPassagePlus_arguments(self):
        """ Tests that methods getPassagePlus maps correctly to request"""
        with patch('requests.get') as patched_get:
            self.cts.getPassagePlus(
                urn="urn", inventory="inventory", context=1)
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetPassagePlus",
                    "context": "1",
                    "urn": "urn"
                }
            )
        with patch('requests.get') as patched_get:
            self.cts.getPassagePlus(urn="urn", inventory="inventory")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetPassagePlus",
                    "urn": "urn"
                }
            )

    def test_request_Cts_getFirstUrn_arguments(self):
        """ Tests that methods getFirstUrn maps correctly to request"""
        with patch('requests.get') as patched_get:
            self.cts.getFirstUrn(urn="urn", inventory="inventory")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetFirstUrn",
                    "urn": "urn"
                }
            )

    def test_request_Cts_getPrevNextUrn_arguments(self):
        """ Tests that methods getPrevNextUrn maps correctly to request"""
        with patch('requests.get') as patched_get:
            self.cts.getPrevNextUrn(urn="urn", inventory="inventory")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetPrevNextUrn",
                    "urn": "urn"
                }
            )

    def test_request_Cts_getLabel_arguments(self):
        """ Tests that methods getLabel maps correctly to request"""
        with patch('requests.get') as patched_get:
            self.cts.getLabel(urn="urn", inventory="inventory")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetLabel",
                    "urn": "urn"
                }
            )

    def test_get_siblings(self):
        """ Ensure Citable CtsTextMetadata Service getMetadata is correctly routed """
        with patch('requests.get') as patched_get:
            self.cts.getSiblings("urn:cts:latinLit:phi1294.phi002.perseus-lat2", "1.1")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "request": "GetPrevNextUrn",
                    "urn": "urn:cts:latinLit:phi1294.phi002.perseus-lat2:1.1"
                }
            )

    def test_get_children(self):
        """ Ensure Citable CtsTextMetadata Service getMetadata is correctly routed """
        with patch('requests.get') as patched_get:
            self.cts.getReffs("urn:cts:latinLit:phi1294.phi002.perseus-lat2")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "request": "GetValidReff",
                    "urn": "urn:cts:latinLit:phi1294.phi002.perseus-lat2",
                    "level": "1"
                }
            )
        with patch('requests.get') as patched_get:
            self.cts.getReffs("urn:cts:latinLit:phi1294.phi002.perseus-lat2", subreference="1.1")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "request": "GetValidReff",
                    "urn": "urn:cts:latinLit:phi1294.phi002.perseus-lat2:1.1",
                    "level": "3"
                }
            )

        with patch('requests.get') as patched_get:
            self.cts.getReffs("urn:cts:latinLit:phi1294.phi002.perseus-lat2", subreference="1", level=2)
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "request": "GetValidReff",
                    "urn": "urn:cts:latinLit:phi1294.phi002.perseus-lat2:1",
                    "level": "3"
                }
            )

    def test_get_metadata(self):
        """ Ensure Citable CtsTextMetadata Service getMetadata is correctly routed """
        with patch('requests.get') as patched_get:
            self.cts.getMetadata()
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "request": "GetCapabilities"
                }
            )
        with patch('requests.get') as patched_get:
            self.cts.getMetadata(objectId="urn")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "request": "GetCapabilities",
                    "urn": "urn"
                }
            )

    def test_get_text(self):
        """ Ensure Citable CtsTextMetadata Service getText is correctly routed """
        with patch('requests.get') as patched_get:
            self.cts.getTextualNode(textId="urn", metadata=True)
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "request": "GetPassagePlus",
                    "urn": "urn"
                }
            )
        with patch('requests.get') as patched_get:
            self.cts.getTextualNode(textId="urn", subreference="1.1")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "request": "GetPassage",
                    "urn": "urn:1.1"
                }
            )

    @responses.activate
    def test_error_handling(self):
        responses.add(
            responses.GET,
            "http://domainname.com/rest/cts?urn=urn%3A1.1&request=GetPassage",
            body="Internal Server Error", status=500,
        )
        with self.assertRaises(requests.HTTPError):
            self.cts.getTextualNode(textId="urn", subreference="1.1")<commit_msg>Added test case for response encoding<commit_after>import unittest

import requests
import responses
from mock import patch, MagicMock

from MyCapytain.retrievers.cts5 import *


class TestEndpointsCts5(unittest.TestCase):

    """ Test Cts5 Endpoint request making """

    def setUp(self):
        self.cts = HttpCtsRetriever("http://domainname.com/rest/cts")

    def test_request_Cts_getCapabilities_arguments(self):
        """ Tests that methods getCapabilities maps correctly to request"""
        with patch('requests.get') as patched_get:
            self.cts.getCapabilities(inventory="inventory")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetCapabilities"
                }
            )

    def test_request_Cts_getValidReff_arguments(self):
        """ Tests that methods getValidReff maps correctly to request"""
        with patch('requests.get') as patched_get:
            self.cts.getValidReff(urn="urn", inventory="inventory", level=1)
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetValidReff",
                    "level": "1",
                    "urn": "urn"
                }
            )
        with patch('requests.get') as patched_get:
            self.cts.getValidReff(urn="urn", inventory="inventory")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetValidReff",
                    "urn": "urn"
                }
            )

    def test_request_Cts_getPassage_arguments(self):
        """ Tests that methods getPassage maps correctly to request"""
        with patch('requests.get') as patched_get:
            self.cts.getPassage(urn="urn", inventory="inventory", context=1)
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetPassage",
                    "context": "1",
                    "urn": "urn"
                }
            )
        with patch('requests.get') as patched_get:
            self.cts.getPassage(urn="urn", inventory="inventory")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetPassage",
                    "urn": "urn"
                }
            )

    def test_call_with_default(self):
        inv = HttpCtsRetriever("http://domainname.com/rest/cts", inventory="annotsrc")
        with patch('requests.get') as patched_get:
            inv.getPassage(urn="urn")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "annotsrc",
                    "request": "GetPassage",
                    "urn": "urn"
                }
            )

    def test_request_Cts_getPassagePlus_arguments(self):
        """ Tests that methods getPassagePlus maps correctly to request"""
        with patch('requests.get') as patched_get:
            self.cts.getPassagePlus(
                urn="urn", inventory="inventory", context=1)
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetPassagePlus",
                    "context": "1",
                    "urn": "urn"
                }
            )
        with patch('requests.get') as patched_get:
            self.cts.getPassagePlus(urn="urn", inventory="inventory")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetPassagePlus",
                    "urn": "urn"
                }
            )

    def test_request_Cts_getFirstUrn_arguments(self):
        """ Tests that methods getFirstUrn maps correctly to request"""
        with patch('requests.get') as patched_get:
            self.cts.getFirstUrn(urn="urn", inventory="inventory")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetFirstUrn",
                    "urn": "urn"
                }
            )

    def test_request_Cts_getPrevNextUrn_arguments(self):
        """ Tests that methods getPrevNextUrn maps correctly to request"""
        with patch('requests.get') as patched_get:
            self.cts.getPrevNextUrn(urn="urn", inventory="inventory")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetPrevNextUrn",
                    "urn": "urn"
                }
            )

    def test_request_Cts_getLabel_arguments(self):
        """ Tests that methods getLabel maps correctly to request"""
        with patch('requests.get') as patched_get:
            self.cts.getLabel(urn="urn", inventory="inventory")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "inv": "inventory",
                    "request": "GetLabel",
                    "urn": "urn"
                }
            )

    def test_get_siblings(self):
        """ Ensure Citable CtsTextMetadata Service getMetadata is correctly routed """
        with patch('requests.get') as patched_get:
            self.cts.getSiblings("urn:cts:latinLit:phi1294.phi002.perseus-lat2", "1.1")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "request": "GetPrevNextUrn",
                    "urn": "urn:cts:latinLit:phi1294.phi002.perseus-lat2:1.1"
                }
            )

    def test_get_children(self):
        """ Ensure Citable CtsTextMetadata Service getMetadata is correctly routed """
        with patch('requests.get') as patched_get:
            self.cts.getReffs("urn:cts:latinLit:phi1294.phi002.perseus-lat2")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "request": "GetValidReff",
                    "urn": "urn:cts:latinLit:phi1294.phi002.perseus-lat2",
                    "level": "1"
                }
            )
        with patch('requests.get') as patched_get:
            self.cts.getReffs("urn:cts:latinLit:phi1294.phi002.perseus-lat2", subreference="1.1")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "request": "GetValidReff",
                    "urn": "urn:cts:latinLit:phi1294.phi002.perseus-lat2:1.1",
                    "level": "3"
                }
            )

        with patch('requests.get') as patched_get:
            self.cts.getReffs("urn:cts:latinLit:phi1294.phi002.perseus-lat2", subreference="1", level=2)
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "request": "GetValidReff",
                    "urn": "urn:cts:latinLit:phi1294.phi002.perseus-lat2:1",
                    "level": "3"
                }
            )

    def test_get_metadata(self):
        """ Ensure Citable CtsTextMetadata Service getMetadata is correctly routed """
        with patch('requests.get') as patched_get:
            self.cts.getMetadata()
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "request": "GetCapabilities"
                }
            )
        with patch('requests.get') as patched_get:
            self.cts.getMetadata(objectId="urn")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "request": "GetCapabilities",
                    "urn": "urn"
                }
            )

    def test_get_text(self):
        """ Ensure Citable CtsTextMetadata Service getText is correctly routed """
        with patch('requests.get') as patched_get:
            self.cts.getTextualNode(textId="urn", metadata=True)
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "request": "GetPassagePlus",
                    "urn": "urn"
                }
            )
        with patch('requests.get') as patched_get:
            self.cts.getTextualNode(textId="urn", subreference="1.1")
            patched_get.assert_called_once_with(
                "http://domainname.com/rest/cts", params={
                    "request": "GetPassage",
                    "urn": "urn:1.1"
                }
            )

    @responses.activate
    def test_error_handling(self):
        responses.add(
            responses.GET,
            "http://domainname.com/rest/cts?urn=urn%3A1.1&request=GetPassage",
            body="Internal Server Error", status=500,
        )
        with self.assertRaises(requests.HTTPError):
            self.cts.getTextualNode(textId="urn", subreference="1.1")

    @responses.activate
    def test_encoding(self):
        """ Ensure retriever falls back to UTF-8 when no encoding is present on responses """
        response = MagicMock(encoding=None)
        with patch('requests.get', return_value=response) as patched_get:
            self.cts.getTextualNode(textId="urn", subreference="1.1")
            self.assertEqual(response.encoding, "utf-8")
        response = MagicMock(encoding="latin1")
        with patch('requests.get', return_value=response) as patched_get:
            self.cts.getTextualNode(textId="urn", subreference="1.1")
            self.assertEqual(response.encoding, "latin1")
{'fraction_non_alphanumeric': 0.12008978675645342, 'fraction_numerical': 0.010509131721252934, 'mean_word_length': 1.7405284496015658, 'pattern_counts': {'":': 106, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2242973', 'n_tokens_mistral': 5802, 'n_tokens_neox': 5490, 'n_words': 903}