<commit_before>"""Unit tests for django-sekh"""
from django.test import TestCase
from django.http import HttpRequest
from django.http import HttpResponse
from django.template import Context
from django.template import Template
from django.template import TemplateSyntaxError

from sekh.excerpt import excerpt
from sekh.highlighting import highlight
from sekh.utils import remove_duplicates
from sekh.middleware import KeywordsHighlightingMiddleware


HTML_CONTENT = '<html><body><p>Hello world !</p></body></html>'


class TestRemoveDuplicates(TestCase):
    """Tests of remove_duplicates function"""

    def test_remove_duplicates(self):
        self.assertEquals(
            remove_duplicates(['titi', 'toto', 'tata', 'titi']),
            ['titi', 'toto', 'tata'])

    def test_remove_duplicates_with_spaces(self):
        self.assertEquals(
            remove_duplicates(['titi', 'toto', 'tata', ' titi']),
            ['titi', 'toto', 'tata'])

    def test_remove_duplicates_with_void_value(self):
        self.assertEquals(
            remove_duplicates(['titi', ' ', 'toto', '', ' titi']),
            ['titi', 'toto'])


class TestHighlight(TestCase):
    """Tests of highlight function"""

    def test_highlight(self):
        self.assertEquals(
            highlight(HTML_CONTENT, ['Hello', 'world']),
            '<html><body><p><span class="highlight term-1">Hello</span> '
            '<span class="highlight term-2">world</span> !</p></body></html>')

    def test_highlight_case(self):
        self.assertEquals(
            highlight(HTML_CONTENT, ['HELLO', 'World']),
            '<html><body><p><span class="highlight term-1">Hello</span> '
            '<span class="highlight term-2">world</span> !</p></body></html>')

    def test_highlight_none(self):
        self.assertEquals(
            highlight(HTML_CONTENT, []), HTML_CONTENT)

    def test_highlight_multiple_in_one_markup(self):
        self.assertEquals(
            highlight(HTML_CONTENT.replace('!', 'hello'), ['hello']),
            '<html><body><p><span class="highlight term-1">Hello</span> '
            'world <span class="highlight term-1">hello</span></p>'
            '</body></html>')

    def test_dont_highlight_highlightings(self):
        self.assertEquals(
            highlight(HTML_CONTENT.replace('world', 'highlight'),
                      ['hello', 'highlight']),
            '<html><body><p><span class="highlight term-1">Hello</span> '
            '<span class="highlight term-2">highlight</span> !</p>'
            '</body></html>')

    def test_dont_highlight_protected_markups(self):
        self.assertEquals(
            highlight(HTML_CONTENT.replace('world', '<pre>world</pre>'),
                      ['world']),
            '<html><body><p>Hello <pre>world</pre> !</p></body></html>')


class TestExcerpt(TestCase):
    """Test of excerpt function"""
    content = """
    Lorem ipsum dolor sit amet, consectetur adipiscing elit.
    In in nunc eros! Suspendisse a feugiat eros, et pharetra nisl ?
    Cras pulvinar varius enim et aliquet. Sed sit amet ultricies libero.
    Etiam facilisis, lectus ut tristique rutrum, leo libero
    elementum eros, sed lobortis urna lacus sit amet velit.
    Quisque ut leo eu dolor aliquet eleifend mattis et urna.
    Praesent vitae viverra purus.
    """

    def test_excerpt(self):
        self.assertEquals(
            excerpt(self.content, ['lacus']),
            '')
        self.assertEquals(
            excerpt(self.content, ['aliquet']),
            '')
        self.assertEquals(
            excerpt(self.content, ['aliquet', 'lacus']),
            '')

    def test_excerpt_case(self):
        self.assertEquals(
            excerpt(self.content, ['LACUS']),
            '')

    def test_excerpt_not_present(self):
        self.assertEquals(
            excerpt(self.content, ['toto']),
            self.content)

    def test_excerpt_none(self):
        self.assertEquals(
            excerpt(self.content, []),
            self.content)


class TestKeywordsHighlightingMiddleware(TestCase):
    """Tests of the Sekh middleware"""

    def _get_request(self, GET={}, referer=None):
        request = HttpRequest()
        request.GET = GET
        if referer:
            request.META['HTTP_REFERER'] = referer
        return request

    def test_referer(self):
        response = KeywordsHighlightingMiddleware().process_response(
            self._get_request(referer='http://toto.com/?q=Hello'),
            HttpResponse(HTML_CONTENT))
        self.assertEquals(response.content, HTML_CONTENT)

        response = KeywordsHighlightingMiddleware().process_response(
            self._get_request(referer='http://www.google.com/?q=world'),
            HttpResponse(HTML_CONTENT))
        self.assertEquals(response.content,
                          '<html><body><p>Hello <span class="highlight '
                          'term-1">world</span> !</p></body></html>')

    def test_get(self):
        response = KeywordsHighlightingMiddleware().process_response(
            self._get_request(),
            HttpResponse(HTML_CONTENT))
        self.assertEquals(response.content, HTML_CONTENT)

        response = KeywordsHighlightingMiddleware().process_response(
            self._get_request({'highlight': 'ziltoid'}),
            HttpResponse(HTML_CONTENT))
        self.assertEquals(response.content, HTML_CONTENT)

        response = KeywordsHighlightingMiddleware().process_response(
            self._get_request({'highlight': 'Hello world'}),
            HttpResponse(HTML_CONTENT))
        self.assertEquals(
            response.content,
            '<html><body><p><span class="highlight term-1">Hello</span> '
            '<span class="highlight term-2">world</span> !</p></body></html>')

        response = KeywordsHighlightingMiddleware().process_response(
            self._get_request({'highlight': 'Hello',
                               'hl': 'World',
                               'q': 'Hello'}),
            HttpResponse(HTML_CONTENT))
        self.assertEquals(
            response.content,
            '<html><body><p><span class="highlight term-1">Hello</span> '
            '<span class="highlight term-2">world</span> !</p></body></html>')

    def test_non_html(self):
        response = KeywordsHighlightingMiddleware().process_response(
            self._get_request({'highlight': 'Hello world'}),
            HttpResponse(HTML_CONTENT, content_type='text/xml'))
        self.assertEquals(response.content, HTML_CONTENT)


class TestHighlightFilter(TestCase):
    """Tests of Highlight filter"""
    response = '<p><span class="highlight term-1">Coding</span> is ' \
               '<span class="highlight term-2">fun</span> :).</p>'

    def test_filter(self):
        t = Template("""
        {% load sekh_tags %}
        <p>{{ content|highlight:"coding,fun" }}</p>
        """)
        html = t.render(Context({'content': 'Coding is fun :).'}))
        self.assertEquals(html.strip(), self.response)

        t = Template("""
        {% load sekh_tags %}
        <p>{{ content|highlight:"coding, fun" }}</p>
        """)
        html = t.render(Context({'content': 'Coding is fun :).'}))
        self.assertEquals(html.strip(), self.response)

        t = Template("""
        {% load sekh_tags %}
        <p>{{ content|highlight:"coding fun" }}</p>
        """)
        html = t.render(Context({'content': 'Coding is fun :).'}))
        self.assertEquals(html.strip(), self.response)

        t = Template("""
        {% load sekh_tags %}
        <p>{{ content|highlight:"coding coding fun" }}</p>
        """)
        html = t.render(Context({'content': 'Coding is fun :).'}))
        self.assertEquals(html.strip(), self.response)

    def test_filter_with_variable(self):
        t = Template("""
        {% load sekh_tags %}
        <p>{{ content|highlight:query }}</p>
        """)
        html = t.render(Context({'content': 'Coding is fun :).',
                                 'query': 'coding, fun'}))
        self.assertEquals(html.strip(), self.response)


class TestHighlightTag(TestCase):
    """Test for Highlight tag"""
    response = '<p><span class="highlight term-1">Coding</span> is ' \
               '<span class="highlight term-2">fun</span> :).</p>'

    def test_tag(self):
        t = Template("""
        {% load sekh_tags %}
        {% highlight "coding fun" %}
        <p>{{ content }}</p>
        {% endhighlight %}
        """)
        html = t.render(Context({'content': 'Coding is fun :).'}))
        self.assertEquals(html.strip(), self.response)

        t = Template("""
        {% load sekh_tags %}
        {% highlight "coding,fun" %}
        <p>Coding is fun :).</p>
        {% endhighlight %}
        """)
        html = t.render(Context())
        self.assertEquals(html.strip(), self.response)

        t = Template("""
        {% load sekh_tags %}
        {% highlight "coding, fun" %}
        <p>Coding is fun :).</p>
        {% endhighlight %}
        """)
        html = t.render(Context())
        self.assertEquals(html.strip(), self.response)

        t = Template("""
        {% load sekh_tags %}
        {% highlight "coding fun" %}
        <p>Coding is fun :).</p>
        {% endhighlight %}
        """)
        html = t.render(Context())
        self.assertEquals(html.strip(), self.response)

        t = Template("""
        {% load sekh_tags %}
        {% highlight "coding coding fun" %}
        <p>Coding is fun :).</p>
        {% endhighlight %}
        """)
        html = t.render(Context())
        self.assertEquals(html.strip(), self.response)

    def test_tag_variable_content(self):
        t = Template("""
        {% load sekh_tags %}
        {% highlight query %}
        <p>{{ content }}</p>
        {% endhighlight %}
        """)
        html = t.render(Context({'content': 'Coding is fun :).',
                                 'query': 'coding, fun'}))
        self.assertEquals(html.strip(), self.response)

    def test_tag_error(self):
        with self.assertRaises(TemplateSyntaxError):
            Template("""
            {% load sekh_tags %}
            {% highlight %}
            <p>Coding is fun :).</p>
            {% endhighlight %}
            """)
<commit_msg>Complete the tests for the excerpt function<commit_after>"""Unit tests for django-sekh"""
from django.test import TestCase
from django.http import HttpRequest
from django.http import HttpResponse
from django.template import Context
from django.template import Template
from django.template import TemplateSyntaxError

from sekh.excerpt import excerpt
from sekh.highlighting import highlight
from sekh.utils import remove_duplicates
from sekh.middleware import KeywordsHighlightingMiddleware


HTML_CONTENT = '<html><body><p>Hello world !</p></body></html>'


class TestRemoveDuplicates(TestCase):
    """Tests of remove_duplicates function"""

    def test_remove_duplicates(self):
        self.assertEquals(
            remove_duplicates(['titi', 'toto', 'tata', 'titi']),
            ['titi', 'toto', 'tata'])

    def test_remove_duplicates_with_spaces(self):
        self.assertEquals(
            remove_duplicates(['titi', 'toto', 'tata', ' titi']),
            ['titi', 'toto', 'tata'])

    def test_remove_duplicates_with_void_value(self):
        self.assertEquals(
            remove_duplicates(['titi', ' ', 'toto', '', ' titi']),
            ['titi', 'toto'])


class TestHighlight(TestCase):
    """Tests of highlight function"""

    def test_highlight(self):
        self.assertEquals(
            highlight(HTML_CONTENT, ['Hello', 'world']),
            '<html><body><p><span class="highlight term-1">Hello</span> '
            '<span class="highlight term-2">world</span> !</p></body></html>')

    def test_highlight_case(self):
        self.assertEquals(
            highlight(HTML_CONTENT, ['HELLO', 'World']),
            '<html><body><p><span class="highlight term-1">Hello</span> '
            '<span class="highlight term-2">world</span> !</p></body></html>')

    def test_highlight_none(self):
        self.assertEquals(
            highlight(HTML_CONTENT, []), HTML_CONTENT)

    def test_highlight_multiple_in_one_markup(self):
        self.assertEquals(
            highlight(HTML_CONTENT.replace('!', 'hello'), ['hello']),
            '<html><body><p><span class="highlight term-1">Hello</span> '
            'world <span class="highlight term-1">hello</span></p>'
            '</body></html>')

    def test_dont_highlight_highlightings(self):
        self.assertEquals(
            highlight(HTML_CONTENT.replace('world', 'highlight'),
                      ['hello', 'highlight']),
            '<html><body><p><span class="highlight term-1">Hello</span> '
            '<span class="highlight term-2">highlight</span> !</p>'
            '</body></html>')

    def test_dont_highlight_protected_markups(self):
        self.assertEquals(
            highlight(HTML_CONTENT.replace('world', '<pre>world</pre>'),
                      ['world']),
            '<html><body><p>Hello <pre>world</pre> !</p></body></html>')


class TestExcerpt(TestCase):
    """Test of excerpt function"""
    content = (
        'Lorem ipsum dolor sit amet, consectetur adipiscing elit. '
        'In in nunc eros! Suspendisse a feugiat eros, et pharetra nisl ? '
        'Cras pulvinar varius enim et aliquet. Sed sit amet ultricies libero. '
        'Etiam facilisis, lectus ut tristique rutrum, leo libero '
        'elementum eros, sed lobortis urna lacus sit amet velit. '
        'Quisque ut leo eu dolor aliquet eleifend mattis et urna. '
        'Praesent vitae viverra purus.')

    def test_excerpt(self):
        self.assertEquals(
            excerpt(self.content, ['lacus']),
            'et aliquet. Sed sit amet ultricies libero. Etiam facilisis, '
            'lectus ut tristique rutrum, leo libero elementum eros, sed '
            'lobortis urna lacus sit amet velit. Quisque ut leo eu dolor '
            'aliquet eleifend mattis et urna. Praesent vitae viverra purus.')
        self.assertEquals(
            excerpt(self.content, ['lacus'], 10),
            'elementum eros, sed lobortis urna lacus sit amet velit. Quisque ut')
        self.assertEquals(
            excerpt(self.content, ['aliquet']),
            'consectetur adipiscing elit. In in nunc eros! Suspendisse a '
            'feugiat eros, et pharetra nisl ? Cras pulvinar varius enim '
            'et aliquet. Sed sit amet ultricies libero. Etiam facilisis, '
            'lectus ut tristique rutrum, leo libero elementum eros, sed '
            'lobortis urna lacus sit')
        self.assertEquals(
            excerpt(self.content, ['aliquet'], 10),
            'Cras pulvinar varius enim et aliquet. Sed sit amet ultricies libero.')

    def test_excerpt_multi_terms(self):
        result = (
            'lectus ut tristique rutrum, leo libero elementum eros, sed '
            'lobortis urna lacus sit amet velit. Quisque ut leo eu dolor '
            'aliquet eleifend mattis et urna. Praesent vitae viverra purus.'
        )
        self.assertEquals(
            excerpt(self.content, ['aliquet', 'lacus']), result)
        self.assertEquals(
            excerpt(self.content, ['lacus', 'aliquet']), result)
        self.assertEquals(
            excerpt(self.content, ['aliquet', 'lacus'], 20),
            'urna lacus sit amet velit. Quisque ut leo '
            'eu dolor aliquet eleifend')

    def test_excerpt_case(self):
        self.assertEquals(
            excerpt(self.content, ['LACUS']),
            'et aliquet. Sed sit amet ultricies libero. Etiam facilisis, '
            'lectus ut tristique rutrum, leo libero elementum eros, sed '
            'lobortis urna lacus sit amet velit. Quisque ut leo eu dolor '
            'aliquet eleifend mattis et urna. Praesent vitae viverra purus.')

    def test_excerpt_not_present(self):
        self.assertEquals(
            excerpt(self.content, ['toto']),
            'Lorem ipsum dolor sit amet, consectetur adipiscing elit. '
            'In in nunc eros! Suspendisse a feugiat eros, et pharetra '
            'nisl ? Cras pulvinar varius enim et aliquet. Sed sit amet '
            'ultricies libero. Etiam facilisis, lectus ut tristique '
            'rutrum, leo libero elementum')
        self.assertEquals(
            excerpt(self.content, ['toto'], 10),
            'Lorem ipsum dolor sit amet, consectetur adipiscing elit. '
            'In in')

    def test_excerpt_none(self):
        self.assertEquals(
            excerpt(self.content, []),
            'Lorem ipsum dolor sit amet, consectetur adipiscing elit. '
            'In in nunc eros! Suspendisse a feugiat eros, et pharetra '
            'nisl ? Cras pulvinar varius enim et aliquet. Sed sit amet '
            'ultricies libero. Etiam facilisis, lectus ut tristique '
            'rutrum, leo libero elementum')
        self.assertEquals(
            excerpt(self.content, [], 10),
            'Lorem ipsum dolor sit amet, consectetur adipiscing elit. '
            'In in')


class TestKeywordsHighlightingMiddleware(TestCase):
    """Tests of the Sekh middleware"""

    def _get_request(self, GET={}, referer=None):
        request = HttpRequest()
        request.GET = GET
        if referer:
            request.META['HTTP_REFERER'] = referer
        return request

    def test_referer(self):
        response = KeywordsHighlightingMiddleware().process_response(
            self._get_request(referer='http://toto.com/?q=Hello'),
            HttpResponse(HTML_CONTENT))
        self.assertEquals(response.content, HTML_CONTENT)

        response = KeywordsHighlightingMiddleware().process_response(
            self._get_request(referer='http://www.google.com/?q=world'),
            HttpResponse(HTML_CONTENT))
        self.assertEquals(response.content,
                          '<html><body><p>Hello <span class="highlight '
                          'term-1">world</span> !</p></body></html>')

    def test_get(self):
        response = KeywordsHighlightingMiddleware().process_response(
            self._get_request(),
            HttpResponse(HTML_CONTENT))
        self.assertEquals(response.content, HTML_CONTENT)

        response = KeywordsHighlightingMiddleware().process_response(
            self._get_request({'highlight': 'ziltoid'}),
            HttpResponse(HTML_CONTENT))
        self.assertEquals(response.content, HTML_CONTENT)

        response = KeywordsHighlightingMiddleware().process_response(
            self._get_request({'highlight': 'Hello world'}),
            HttpResponse(HTML_CONTENT))
        self.assertEquals(
            response.content,
            '<html><body><p><span class="highlight term-1">Hello</span> '
            '<span class="highlight term-2">world</span> !</p></body></html>')

        response = KeywordsHighlightingMiddleware().process_response(
            self._get_request({'highlight': 'Hello',
                               'hl': 'World',
                               'q': 'Hello'}),
            HttpResponse(HTML_CONTENT))
        self.assertEquals(
            response.content,
            '<html><body><p><span class="highlight term-1">Hello</span> '
            '<span class="highlight term-2">world</span> !</p></body></html>')

    def test_non_html(self):
        response = KeywordsHighlightingMiddleware().process_response(
            self._get_request({'highlight': 'Hello world'}),
            HttpResponse(HTML_CONTENT, content_type='text/xml'))
        self.assertEquals(response.content, HTML_CONTENT)


class TestHighlightFilter(TestCase):
    """Tests of Highlight filter"""
    response = '<p><span class="highlight term-1">Coding</span> is ' \
               '<span class="highlight term-2">fun</span> :).</p>'

    def test_filter(self):
        t = Template("""
        {% load sekh_tags %}
        <p>{{ content|highlight:"coding,fun" }}</p>
        """)
        html = t.render(Context({'content': 'Coding is fun :).'}))
        self.assertEquals(html.strip(), self.response)

        t = Template("""
        {% load sekh_tags %}
        <p>{{ content|highlight:"coding, fun" }}</p>
        """)
        html = t.render(Context({'content': 'Coding is fun :).'}))
        self.assertEquals(html.strip(), self.response)

        t = Template("""
        {% load sekh_tags %}
        <p>{{ content|highlight:"coding fun" }}</p>
        """)
        html = t.render(Context({'content': 'Coding is fun :).'}))
        self.assertEquals(html.strip(), self.response)

        t = Template("""
        {% load sekh_tags %}
        <p>{{ content|highlight:"coding coding fun" }}</p>
        """)
        html = t.render(Context({'content': 'Coding is fun :).'}))
        self.assertEquals(html.strip(), self.response)

    def test_filter_with_variable(self):
        t = Template("""
        {% load sekh_tags %}
        <p>{{ content|highlight:query }}</p>
        """)
        html = t.render(Context({'content': 'Coding is fun :).',
                                 'query': 'coding, fun'}))
        self.assertEquals(html.strip(), self.response)


class TestHighlightTag(TestCase):
    """Test for Highlight tag"""
    response = '<p><span class="highlight term-1">Coding</span> is ' \
               '<span class="highlight term-2">fun</span> :).</p>'

    def test_tag(self):
        t = Template("""
        {% load sekh_tags %}
        {% highlight "coding fun" %}
        <p>{{ content }}</p>
        {% endhighlight %}
        """)
        html = t.render(Context({'content': 'Coding is fun :).'}))
        self.assertEquals(html.strip(), self.response)

        t = Template("""
        {% load sekh_tags %}
        {% highlight "coding,fun" %}
        <p>Coding is fun :).</p>
        {% endhighlight %}
        """)
        html = t.render(Context())
        self.assertEquals(html.strip(), self.response)

        t = Template("""
        {% load sekh_tags %}
        {% highlight "coding, fun" %}
        <p>Coding is fun :).</p>
        {% endhighlight %}
        """)
        html = t.render(Context())
        self.assertEquals(html.strip(), self.response)

        t = Template("""
        {% load sekh_tags %}
        {% highlight "coding fun" %}
        <p>Coding is fun :).</p>
        {% endhighlight %}
        """)
        html = t.render(Context())
        self.assertEquals(html.strip(), self.response)

        t = Template("""
        {% load sekh_tags %}
        {% highlight "coding coding fun" %}
        <p>Coding is fun :).</p>
        {% endhighlight %}
        """)
        html = t.render(Context())
        self.assertEquals(html.strip(), self.response)

    def test_tag_variable_content(self):
        t = Template("""
        {% load sekh_tags %}
        {% highlight query %}
        <p>{{ content }}</p>
        {% endhighlight %}
        """)
        html = t.render(Context({'content': 'Coding is fun :).',
                                 'query': 'coding, fun'}))
        self.assertEquals(html.strip(), self.response)

    def test_tag_error(self):
        with self.assertRaises(TemplateSyntaxError):
            Template("""
            {% load sekh_tags %}
            {% highlight %}
            <p>Coding is fun :).</p>
            {% endhighlight %}
            """)

{'fraction_non_alphanumeric': 0.14549197906837094, 'fraction_numerical': 0.0018872780303680191, 'mean_word_length': 2.3570914326853853, 'pattern_counts': {'":': 0, '<': 243, '<?xml version=': 0, '>': 243, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6560135', 'n_tokens_mistral': 7130, 'n_tokens_neox': 6768, 'n_words': 1472}