<commit_before># -*- coding: utf-8 -*-
import re
import weakref
import six
import unittest
from parsel import Selector


class SelectorTestCase(unittest.TestCase):

    sscls = Selector

    def test_simple_selection(self):
        """Simple selector tests"""
        body = u"<p><input name='a'value='1'/><input name='b'value='2'/></p>"
        sel = self.sscls(text=body)

        xl = sel.xpath('//input')
        self.assertEqual(2, len(xl))
        for x in xl:
            assert isinstance(x, self.sscls)

        self.assertEqual(sel.xpath('//input').extract(),
                         [x.extract() for x in sel.xpath('//input')])

        self.assertEqual([x.extract() for x in sel.xpath("//input[@name='a']/@name")],
                         [u'a'])
        self.assertEqual([x.extract() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
                         [u'12.0'])

        self.assertEqual(sel.xpath("concat('xpath', 'rules')").extract(),
                         [u'xpathrules'])
        self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
                         [u'12'])

    def test_simple_selection_with_variables(self):
        """Using XPath variables"""
        body = u"<p><input name='a' value='1'/><input name='b' value='2'/></p>"
        sel = self.sscls(text=body)

        self.assertEqual([x.extract() for x in sel.xpath("//input[@value=$number]/@name", number=1)],
                         [u'a'])
        self.assertEqual([x.extract() for x in sel.xpath("//input[@name=$letter]/@value", letter='b')],
                         [u'2'])

        self.assertEqual(sel.xpath("count(//input[@value=$number or @name=$letter])",
                                   number=2, letter='a').extract(),
                         [u'2.0'])

        # you can also pass booleans
        self.assertEqual(sel.xpath("boolean(count(//input)=$cnt)=$test",
                                   cnt=2, test=True).extract(),
                         [u'1'])
        self.assertEqual(sel.xpath("boolean(count(//input)=$cnt)=$test",
                                   cnt=4, test=True).extract(),
                         [u'0'])
        self.assertEqual(sel.xpath("boolean(count(//input)=$cnt)=$test",
                                   cnt=4, test=False).extract(),
                         [u'1'])

        # for named nodes, you need to use "name()=node_name"
        self.assertEqual(sel.xpath("boolean(count(//*[name()=$tag])=$cnt)=$test",
                                   tag="input", cnt=2, test=True).extract(),
                         [u'1'])

    def test_simple_selection_with_variables_escape_friendly(self):
        """Using XPath variables with quotes that would need escaping with string formatting"""
        body = u"""<p>I'm mixing single and <input name='a' value='I say "Yeah!"'/>
        "double quotes" and I don't care :)</p>"""
        sel = self.sscls(text=body)

        t = 'I say "Yeah!"'
        # naive string formatting with give something like:
        # ValueError: XPath error: Invalid predicate in //input[@value="I say "Yeah!""]/@name
        self.assertRaises(ValueError, sel.xpath, '//input[@value="{}"]/@name'.format(t))

        # with XPath variables, escaping is done for you
        self.assertEqual([x.extract() for x in sel.xpath("//input[@value=$text]/@name", text=t)],
                         [u'a'])
        lt = """I'm mixing single and "double quotes" and I don't care :)"""
        # the following gives you something like
        # ValueError: XPath error: Invalid predicate in //p[normalize-space()='I'm mixing single and "double quotes" and I don't care :)']//@name
        self.assertRaises(ValueError, sel.xpath, "//p[normalize-space()='{}']//@name".format(lt))

        self.assertEqual([x.extract() for x in sel.xpath("//p[normalize-space()=$lng]//@name",
            lng=lt)],
                         [u'a'])

    def test_representation_slice(self):
        body = u"<p><input name='{}' value='\xa9'/></p>".format(50 * 'b')
        sel = self.sscls(text=body)

        representation = "<Selector xpath='//input/@name' data='{}'>".format(40 * 'b')
        if six.PY2:
            representation = "<Selector xpath='//input/@name' data=u'{}'>".format(40 * 'b')

        self.assertEqual(
            [repr(it) for it in sel.xpath('//input/@name')],
            [representation]
        )

    def test_representation_unicode_query(self):
        body = u"<p><input name='{}' value='\xa9'/></p>".format(50 * 'b')

        representation = '<Selector xpath=\'//input[@value="©"]/@value\' data=\'©\'>'
        if six.PY2:
            representation = "<Selector xpath=u'//input[@value=\"\\xa9\"]/@value' data=u'\\xa9'>"

        sel = self.sscls(text=body)
        self.assertEqual(
            [repr(it) for it in sel.xpath(u'//input[@value="\xa9"]/@value')],
            [representation]
        )

    def test_check_text_argument_type(self):
        self.assertRaisesRegexp(TypeError, 'text argument should be of type',
                                self.sscls, b'<html/>')

    def test_extract_first(self):
        """Test if extract_first() returns first element"""
        body = u'<ul><li id="1">1</li><li id="2">2</li></ul>'
        sel = self.sscls(text=body)

        self.assertEqual(sel.xpath('//ul/li/text()').extract_first(),
                         sel.xpath('//ul/li/text()').extract()[0])

        self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').extract_first(),
                         sel.xpath('//ul/li[@id="1"]/text()').extract()[0])

        self.assertEqual(sel.xpath('//ul/li[2]/text()').extract_first(),
                         sel.xpath('//ul/li/text()').extract()[1])

        self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').extract_first(), None)

    def test_extract_first_default(self):
        """Test if extract_first() returns default value when no results found"""
        body = u'<ul><li id="1">1</li><li id="2">2</li></ul>'
        sel = self.sscls(text=body)

        self.assertEqual(sel.xpath('//div/text()').extract_first(default='missing'), 'missing')

    def test_selector_get_alias(self):
        """Test if get() returns extracted value on a Selector"""
        body = u'<ul><li id="1">1</li><li id="2">2</li></ul>'
        sel = self.sscls(text=body)

        self.assertEqual(sel.xpath('//ul/li/text()')[0].get(),
                         sel.xpath('//ul/li/text()')[0].extract()[0])

    def test_selectorlist_get_alias(self):
        """Test if get() returns first element for a selection call"""
        body = u'<ul><li id="1">1</li><li id="2">2</li></ul>'
        sel = self.sscls(text=body)

        self.assertEqual(sel.xpath('//ul/li/text()').get(),
                         sel.xpath('//ul/li/text()').extract()[0])

    def test_re_first(self):
        """Test if re_first() returns first matched element"""
        body = u'<ul><li id="1">1</li><li id="2">2</li></ul>'
        sel = self.sscls(text=body)

        self.assertEqual(sel.xpath('//ul/li/text()').re_first('\d'),
                         sel.xpath('//ul/li/text()').re('\d')[0])

        self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').re_first('\d'),
                         sel.xpath('//ul/li[@id="1"]/text()').re('\d')[0])

        self.assertEqual(sel.xpath('//ul/li[2]/text()').re_first('\d'),
                         sel.xpath('//ul/li/text()').re('\d')[1])

        self.assertEqual(sel.xpath('/ul/li/text()').re_first('\w+'), None)
        self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').re_first('\d'), None)

    def test_extract_first_default(self):
        """Test if re_first() returns default value when no results found"""
        body = u'<ul><li id="1">1</li><li id="2">2</li></ul>'
        sel = self.sscls(text=body)

        self.assertEqual(sel.xpath('//div/text()').re_first('\w+', default='missing'), 'missing')
        self.assertEqual(sel.xpath('/ul/li/text()').re_first('\w+', default='missing'), 'missing')

    def test_select_unicode_query(self):
        body = u"<p><input name='\xa9' value='1'/></p>"
        sel = self.sscls(text=body)<commit_msg>Update tests<commit_after># -*- coding: utf-8 -*-
import re
import weakref
import six
import unittest
from parsel import Selector


class SelectorTestCase(unittest.TestCase):

    sscls = Selector

    def test_simple_selection(self):
        """Simple selector tests"""
        body = u"<p><input name='a'value='1'/><input name='b'value='2'/></p>"
        sel = self.sscls(text=body)

        xl = sel.xpath('//input')
        self.assertEqual(2, len(xl))
        for x in xl:
            assert isinstance(x, self.sscls)

        self.assertEqual(sel.xpath('//input').extract(),
                         [x.extract() for x in sel.xpath('//input')])

        self.assertEqual([x.extract() for x in sel.xpath("//input[@name='a']/@name")],
                         [u'a'])
        self.assertEqual([x.extract() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
                         [u'12.0'])

        self.assertEqual(sel.xpath("concat('xpath', 'rules')").extract(),
                         [u'xpathrules'])
        self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
                         [u'12'])

    def test_simple_selection_with_variables(self):
        """Using XPath variables"""
        body = u"<p><input name='a' value='1'/><input name='b' value='2'/></p>"
        sel = self.sscls(text=body)

        self.assertEqual([x.extract() for x in sel.xpath("//input[@value=$number]/@name", number=1)],
                         [u'a'])
        self.assertEqual([x.extract() for x in sel.xpath("//input[@name=$letter]/@value", letter='b')],
                         [u'2'])

        self.assertEqual(sel.xpath("count(//input[@value=$number or @name=$letter])",
                                   number=2, letter='a').extract(),
                         [u'2.0'])

        # you can also pass booleans
        self.assertEqual(sel.xpath("boolean(count(//input)=$cnt)=$test",
                                   cnt=2, test=True).extract(),
                         [u'1'])
        self.assertEqual(sel.xpath("boolean(count(//input)=$cnt)=$test",
                                   cnt=4, test=True).extract(),
                         [u'0'])
        self.assertEqual(sel.xpath("boolean(count(//input)=$cnt)=$test",
                                   cnt=4, test=False).extract(),
                         [u'1'])

        # for named nodes, you need to use "name()=node_name"
        self.assertEqual(sel.xpath("boolean(count(//*[name()=$tag])=$cnt)=$test",
                                   tag="input", cnt=2, test=True).extract(),
                         [u'1'])

    def test_simple_selection_with_variables_escape_friendly(self):
        """Using XPath variables with quotes that would need escaping with string formatting"""
        body = u"""<p>I'm mixing single and <input name='a' value='I say "Yeah!"'/>
        "double quotes" and I don't care :)</p>"""
        sel = self.sscls(text=body)

        t = 'I say "Yeah!"'
        # naive string formatting with give something like:
        # ValueError: XPath error: Invalid predicate in //input[@value="I say "Yeah!""]/@name
        self.assertRaises(ValueError, sel.xpath, '//input[@value="{}"]/@name'.format(t))

        # with XPath variables, escaping is done for you
        self.assertEqual([x.extract() for x in sel.xpath("//input[@value=$text]/@name", text=t)],
                         [u'a'])
        lt = """I'm mixing single and "double quotes" and I don't care :)"""
        # the following gives you something like
        # ValueError: XPath error: Invalid predicate in //p[normalize-space()='I'm mixing single and "double quotes" and I don't care :)']//@name
        self.assertRaises(ValueError, sel.xpath, "//p[normalize-space()='{}']//@name".format(lt))

        self.assertEqual([x.extract() for x in sel.xpath("//p[normalize-space()=$lng]//@name",
            lng=lt)],
                         [u'a'])

    def test_representation_slice(self):
        body = u"<p><input name='{}' value='\xa9'/></p>".format(50 * 'b')
        sel = self.sscls(text=body)

        representation = "<Selector xpath='//input/@name' data='{}'>".format(40 * 'b')
        if six.PY2:
            representation = "<Selector xpath='//input/@name' data=u'{}'>".format(40 * 'b')

        self.assertEqual(
            [repr(it) for it in sel.xpath('//input/@name')],
            [representation]
        )

    def test_representation_unicode_query(self):
        body = u"<p><input name='{}' value='\xa9'/></p>".format(50 * 'b')

        representation = '<Selector xpath=\'//input[@value="©"]/@value\' data=\'©\'>'
        if six.PY2:
            representation = "<Selector xpath=u'//input[@value=\"\\xa9\"]/@value' data=u'\\xa9'>"

        sel = self.sscls(text=body)
        self.assertEqual(
            [repr(it) for it in sel.xpath(u'//input[@value="\xa9"]/@value')],
            [representation]
        )

    def test_check_text_argument_type(self):
        self.assertRaisesRegexp(TypeError, 'text argument should be of type',
                                self.sscls, b'<html/>')

    def test_extract_first(self):
        """Test if extract_first() returns first element"""
        body = u'<ul><li id="1">1</li><li id="2">2</li></ul>'
        sel = self.sscls(text=body)

        self.assertEqual(sel.xpath('//ul/li/text()').extract_first(),
                         sel.xpath('//ul/li/text()').extract()[0])

        self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').extract_first(),
                         sel.xpath('//ul/li[@id="1"]/text()').extract()[0])

        self.assertEqual(sel.xpath('//ul/li[2]/text()').extract_first(),
                         sel.xpath('//ul/li/text()').extract()[1])

        self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').extract_first(), None)

    def test_extract_first_default(self):
        """Test if extract_first() returns default value when no results found"""
        body = u'<ul><li id="1">1</li><li id="2">2</li></ul>'
        sel = self.sscls(text=body)

        self.assertEqual(sel.xpath('//div/text()').extract_first(default='missing'), 'missing')

    def test_selector_get_alias(self):
        """Test if get() returns extracted value on a Selector"""
        body = u'<ul><li id="1">1</li><li id="2">2</li><li id="3">3</li></ul>'
        sel = self.sscls(text=body)

        self.assertEqual(sel.xpath('//ul/li[position()>1]')[0].get(), u'<li id="2">2</li>')
        self.assertEqual(sel.xpath('//ul/li[position()>1]/text()')[0].get(), u'2')

    def test_selectorlist_get_alias(self):
        """Test if get() returns first element for a selection call"""
        body = u'<ul><li id="1">1</li><li id="2">2</li><li id="3">3</li></ul>'
        sel = self.sscls(text=body)

        self.assertEqual(sel.xpath('//ul/li').get(), u'<li id="1">1</li>')
        self.assertEqual(sel.xpath('//ul/li/text()').get(), u'1')

    def test_re_first(self):
        """Test if re_first() returns first matched element"""
        body = u'<ul><li id="1">1</li><li id="2">2</li></ul>'
        sel = self.sscls(text=body)

        self.assertEqual(sel.xpath('//ul/li/text()').re_first('\d'),
                         sel.xpath('//ul/li/text()').re('\d')[0])

        self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').re_first('\d'),
                         sel.xpath('//ul/li[@id="1"]/text()').re('\d')[0])

        self.assertEqual(sel.xpath('//ul/li[2]/text()').re_first('\d'),
                         sel.xpath('//ul/li/text()').re('\d')[1])

        self.assertEqual(sel.xpath('/ul/li/text()').re_first('\w+'), None)
        self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').re_first('\d'), None)

    def test_extract_first_default(self):
        """Test if re_first() returns default value when no results found"""
        body = u'<ul><li id="1">1</li><li id="2">2</li></ul>'
        sel = self.sscls(text=body)

        self.assertEqual(sel.xpath('//div/text()').re_first('\w+', default='missing'), 'missing')
        self.assertEqual(sel.xpath('/ul/li/text()').re_first('\w+', default='missing'), 'missing')

    def test_select_unicode_query(self):
        body = u"<p><input name='\xa9' value='1'/></p>"
        sel = self.sscls(text=body)
{'fraction_non_alphanumeric': 0.20800869722775867, 'fraction_numerical': 0.010388355378389806, 'mean_word_length': 2.3342730567861456, 'pattern_counts': {'":': 0, '<': 133, '<?xml version=': 0, '>': 135, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2095090', 'n_tokens_mistral': 5401, 'n_tokens_neox': 5191, 'n_words': 910}