<commit_before>import os
import unittest
import urlparse

from scrapy.xlib.BeautifulSoup import BeautifulSoup
from scrapy.http import Response, TextResponse, HtmlResponse
from scrapy.utils.response import body_or_str, get_base_url, get_meta_refresh, \
    response_httprepr, get_cached_beautifulsoup, open_in_browser

class ResponseUtilsTest(unittest.TestCase):
    dummy_response = TextResponse(url='http://example.org/', body='dummy_response')

    def test_body_or_str_input(self):
        self.assertTrue(isinstance(body_or_str(self.dummy_response), basestring))
        self.assertTrue(isinstance(body_or_str('text'), basestring))
        self.assertRaises(Exception, body_or_str, 2)

    def test_body_or_str_extraction(self):
        self.assertEqual(body_or_str(self.dummy_response), 'dummy_response')
        self.assertEqual(body_or_str('text'), 'text')

    def test_body_or_str_encoding(self):
        self.assertTrue(isinstance(body_or_str(self.dummy_response, unicode=False), str))
        self.assertTrue(isinstance(body_or_str(self.dummy_response, unicode=True), unicode))

        self.assertTrue(isinstance(body_or_str('text', unicode=False), str))
        self.assertTrue(isinstance(body_or_str('text', unicode=True), unicode))

        self.assertTrue(isinstance(body_or_str(u'text', unicode=False), str))
        self.assertTrue(isinstance(body_or_str(u'text', unicode=True), unicode))

    def test_get_base_url(self):
        response = HtmlResponse(url='https://example.org', body="""\
            <html>\
            <head><title>Dummy</title><base href='http://example.org/something' /></head>\
            <body>blahablsdfsal&amp;</body>\
            </html>""")
        self.assertEqual(get_base_url(response), 'http://example.org/something')

        # relative url with absolute path
        response = HtmlResponse(url='https://example.org', body="""\
            <html>\
            <head><title>Dummy</title><base href='/absolutepath' /></head>\
            <body>blahablsdfsal&amp;</body>\
            </html>""")
        self.assertEqual(get_base_url(response), 'https://example.org/absolutepath')

        # no scheme url
        response = HtmlResponse(url='https://example.org', body="""\
            <html>\
            <head><title>Dummy</title><base href='//noscheme.com/path' /></head>\
            <body>blahablsdfsal&amp;</body>\
            </html>""")
        self.assertEqual(get_base_url(response), 'https://noscheme.com/path')

    def test_get_meta_refresh(self):
        body = """
            <html>
            <head><title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
            <body>blahablsdfsal&amp;</body>
            </html>"""
        response = TextResponse(url='http://example.org', body=body)
        self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage'))

        # refresh without url should return (None, None)
        body = """<meta http-equiv="refresh" content="5" />"""
        response = TextResponse(url='http://example.org', body=body)
        self.assertEqual(get_meta_refresh(response), (None, None))

        body = """<meta http-equiv="refresh" content="5;
            url=http://example.org/newpage" /></head>"""
        response = TextResponse(url='http://example.org', body=body)
        self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage'))

        # meta refresh in multiple lines
        body = """<html><head>
               <META
               HTTP-EQUIV="Refresh"
               CONTENT="1; URL=http://example.org/newpage">"""
        response = TextResponse(url='http://example.org', body=body)
        self.assertEqual(get_meta_refresh(response), (1, 'http://example.org/newpage'))

        # entities in the redirect url
        body = """<meta http-equiv="refresh" content="3; url=&#39;http://www.example.com/other&#39;">"""
        response = TextResponse(url='http://example.com', body=body)
        self.assertEqual(get_meta_refresh(response), (3, 'http://www.example.com/other'))

        # relative redirects
        body = """<meta http-equiv="refresh" content="3; url=other.html">"""
        response = TextResponse(url='http://example.com/page/this.html', body=body)
        self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/page/other.html'))

        # non-standard encodings (utf-16)
        body = """<meta http-equiv="refresh" content="3; url=http://example.com/redirect">"""
        body = body.decode('ascii').encode('utf-16')
        response = TextResponse(url='http://example.com', body=body, encoding='utf-16')
        self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/redirect'))

        # non-ascii chars in the url (default encoding - utf8)
        body = """<meta http-equiv="refresh" content="3; url=http://example.com/to\xc2\xa3">"""
        response = TextResponse(url='http://example.com', body=body)
        self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3'))

        # non-ascii chars in the url (custom encoding - latin1)
        body = """<meta http-equiv="refresh" content="3; url=http://example.com/to\xa3">"""<commit_msg>made encoding explicit in test_get_meta_refresh, to avoid depending on unreliable UnicodeDammit criteria.<commit_after>import os
import unittest
import urlparse

from scrapy.xlib.BeautifulSoup import BeautifulSoup
from scrapy.http import Response, TextResponse, HtmlResponse
from scrapy.utils.response import body_or_str, get_base_url, get_meta_refresh, \
    response_httprepr, get_cached_beautifulsoup, open_in_browser

class ResponseUtilsTest(unittest.TestCase):
    dummy_response = TextResponse(url='http://example.org/', body='dummy_response')

    def test_body_or_str_input(self):
        self.assertTrue(isinstance(body_or_str(self.dummy_response), basestring))
        self.assertTrue(isinstance(body_or_str('text'), basestring))
        self.assertRaises(Exception, body_or_str, 2)

    def test_body_or_str_extraction(self):
        self.assertEqual(body_or_str(self.dummy_response), 'dummy_response')
        self.assertEqual(body_or_str('text'), 'text')

    def test_body_or_str_encoding(self):
        self.assertTrue(isinstance(body_or_str(self.dummy_response, unicode=False), str))
        self.assertTrue(isinstance(body_or_str(self.dummy_response, unicode=True), unicode))

        self.assertTrue(isinstance(body_or_str('text', unicode=False), str))
        self.assertTrue(isinstance(body_or_str('text', unicode=True), unicode))

        self.assertTrue(isinstance(body_or_str(u'text', unicode=False), str))
        self.assertTrue(isinstance(body_or_str(u'text', unicode=True), unicode))

    def test_get_base_url(self):
        response = HtmlResponse(url='https://example.org', body="""\
            <html>\
            <head><title>Dummy</title><base href='http://example.org/something' /></head>\
            <body>blahablsdfsal&amp;</body>\
            </html>""")
        self.assertEqual(get_base_url(response), 'http://example.org/something')

        # relative url with absolute path
        response = HtmlResponse(url='https://example.org', body="""\
            <html>\
            <head><title>Dummy</title><base href='/absolutepath' /></head>\
            <body>blahablsdfsal&amp;</body>\
            </html>""")
        self.assertEqual(get_base_url(response), 'https://example.org/absolutepath')

        # no scheme url
        response = HtmlResponse(url='https://example.org', body="""\
            <html>\
            <head><title>Dummy</title><base href='//noscheme.com/path' /></head>\
            <body>blahablsdfsal&amp;</body>\
            </html>""")
        self.assertEqual(get_base_url(response), 'https://noscheme.com/path')

    def test_get_meta_refresh(self):
        body = """
            <html>
            <head><title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
            <body>blahablsdfsal&amp;</body>
            </html>"""
        response = TextResponse(url='http://example.org', body=body)
        self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage'))

        # refresh without url should return (None, None)
        body = """<meta http-equiv="refresh" content="5" />"""
        response = TextResponse(url='http://example.org', body=body)
        self.assertEqual(get_meta_refresh(response), (None, None))

        body = """<meta http-equiv="refresh" content="5;
            url=http://example.org/newpage" /></head>"""
        response = TextResponse(url='http://example.org', body=body)
        self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage'))

        # meta refresh in multiple lines
        body = """<html><head>
               <META
               HTTP-EQUIV="Refresh"
               CONTENT="1; URL=http://example.org/newpage">"""
        response = TextResponse(url='http://example.org', body=body)
        self.assertEqual(get_meta_refresh(response), (1, 'http://example.org/newpage'))

        # entities in the redirect url
        body = """<meta http-equiv="refresh" content="3; url=&#39;http://www.example.com/other&#39;">"""
        response = TextResponse(url='http://example.com', body=body)
        self.assertEqual(get_meta_refresh(response), (3, 'http://www.example.com/other'))

        # relative redirects
        body = """<meta http-equiv="refresh" content="3; url=other.html">"""
        response = TextResponse(url='http://example.com/page/this.html', body=body)
        self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/page/other.html'))

        # non-standard encodings (utf-16)
        body = """<meta http-equiv="refresh" content="3; url=http://example.com/redirect">"""
        body = body.decode('ascii').encode('utf-16')
        response = TextResponse(url='http://example.com', body=body, encoding='utf-16')
        self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/redirect'))

        # non-ascii chars in the url (utf8)
        body = """<meta http-equiv="refresh" content="3; url=http://example.com/to\xc2\xa3">"""
        response = TextResponse(url='http://example.com', body=body, encoding='utf-8')
        self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3'))

        # non-ascii chars in the url (latin1)
        body = """<meta http-equiv="refresh" content="3; url=http://example.com/to\xa3">"""
{'fraction_non_alphanumeric': 0.16531835918911203, 'fraction_numerical': 0.006567050537736747, 'mean_word_length': 3.3403552251135893, 'pattern_counts': {'":': 0, '<': 97, '<?xml version=': 0, '>': 97, 'https://': 10, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6420666', 'n_tokens_mistral': 3262, 'n_tokens_neox': 3178, 'n_words': 503}