<commit_before>#!/usr/bin/env python
# vim:fileencoding=utf-8
# License: Apache 2.0 Copyright: 2017, <NAME> <kovid at kovidgoyal.net>

from __future__ import absolute_import, division, print_function, unicode_literals

from functools import partial

from lxml import etree

from __init__ import TestCase
from html5_parser import parse

nsparse = partial(parse, namespace_elements=True)
XHTML = "http://www.w3.org/1999/xhtml"
SVG = "http://www.w3.org/2000/svg"
XLINK = "http://www.w3.org/1999/xlink"
XML = "http://www.w3.org/XML/1998/namespace"


def tostring(root):
    return etree.tostring(root, encoding='unicode')


class BasicTests(TestCase):

    def test_single_namespace(self):
        root = nsparse('<p>xxx')
        self.ae(
            tostring(root), '<html xmlns="{}"><head/><body><p>xxx</p></body></html>'.format(XHTML))
        for tag in root.iter('*'):
            self.ae(tag.nsmap, {None: XHTML})
            self.assertIsNone(tag.prefix)
            self.ae(tag.tag.rpartition('}')[0][1:], XHTML, 'no namespace for {}'.format(tag.tag))
        self.ae(len(tuple(root.iterdescendants('{%s}p' % XHTML))), 1)

    def test_multiple_namespace(self):
        root = nsparse('<p>xxx<svg a=1><image b=2 xlink:href="xxx"></svg><p>yyy')
        self.ae(len(tuple(root.iterdescendants('{%s}p' % XHTML))), 2)
        self.ae(len(tuple(root.iterdescendants('{%s}svg' % SVG))), 1)
        self.ae(
            tostring(root),
            '<html xmlns="http://www.w3.org/1999/xhtml" xmlns:xlink="http://www.w3.org/1999/xlink">'
            '<head/><body><p>xxx<svg xmlns="http://www.w3.org/2000/svg" a="1">'
            '<image b="2" xlink:href="xxx"/></svg></p><p>yyy</p></body></html>')
        root = nsparse('<p>xxx<svg a=1><image b=2 xlink:href="href"></svg><p id=1>yyy')
        self.ae(root.xpath('//@id'), ['1'])
        self.ae(root.xpath('//@a'), ['1'])
        self.ae(root.xpath('//@b'), ['2'])
        self.ae(root.xpath('//@xlink:href', namespaces={'xlink': XLINK}), ['href'])

    def test_xml_ns(self):
        root = nsparse('<html xml:lang="fr" lang="es"><svg xml:lang="1">xxx', maybe_xhtml=True)
        self.ae(
            tostring(root), '<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="fr"'
            ' lang="es"><head/><body><svg xmlns="http://www.w3.org/2000/svg" xml:lang="1">'
            'xxx</svg></body></html>')
        self.ae(root.xpath('//@lang'), ['es'])
        self.assertIn('{%s}lang' % XML, root.attrib)
        self.ae(root.xpath('//@xml:lang'), ['fr', '1'])<commit_msg>More xml: tests<commit_after>#!/usr/bin/env python
# vim:fileencoding=utf-8
# License: Apache 2.0 Copyright: 2017, <NAME> <kovid at kovidgoyal.net>

from __future__ import absolute_import, division, print_function, unicode_literals

from functools import partial

from lxml import etree

from __init__ import TestCase
from html5_parser import parse

nsparse = partial(parse, namespace_elements=True)
XHTML = "http://www.w3.org/1999/xhtml"
SVG = "http://www.w3.org/2000/svg"
XLINK = "http://www.w3.org/1999/xlink"
XML = "http://www.w3.org/XML/1998/namespace"


def tostring(root):
    return etree.tostring(root, encoding='unicode')


class BasicTests(TestCase):

    def test_single_namespace(self):
        root = nsparse('<p>xxx')
        self.ae(
            tostring(root), '<html xmlns="{}"><head/><body><p>xxx</p></body></html>'.format(XHTML))
        for tag in root.iter('*'):
            self.ae(tag.nsmap, {None: XHTML})
            self.assertIsNone(tag.prefix)
            self.ae(tag.tag.rpartition('}')[0][1:], XHTML, 'no namespace for {}'.format(tag.tag))
        self.ae(len(tuple(root.iterdescendants('{%s}p' % XHTML))), 1)

    def test_multiple_namespace(self):
        root = nsparse('<p>xxx<svg a=1><image b=2 xlink:href="xxx"></svg><p>yyy')
        self.ae(len(tuple(root.iterdescendants('{%s}p' % XHTML))), 2)
        self.ae(len(tuple(root.iterdescendants('{%s}svg' % SVG))), 1)
        self.ae(
            tostring(root),
            '<html xmlns="http://www.w3.org/1999/xhtml" xmlns:xlink="http://www.w3.org/1999/xlink">'
            '<head/><body><p>xxx<svg xmlns="http://www.w3.org/2000/svg" a="1">'
            '<image b="2" xlink:href="xxx"/></svg></p><p>yyy</p></body></html>')
        root = nsparse('<p>xxx<svg a=1><image b=2 xlink:href="href"></svg><p id=1>yyy')
        self.ae(root.xpath('//@id'), ['1'])
        self.ae(root.xpath('//@a'), ['1'])
        self.ae(root.xpath('//@b'), ['2'])
        self.ae(root.xpath('//@xlink:href', namespaces={'xlink': XLINK}), ['href'])

    def test_xml_ns(self):
        root = nsparse('<html xml:lang="fr" lang="es"><svg xml:lang="1">xxx', maybe_xhtml=True)
        self.ae(
            tostring(root), '<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="fr"'
            ' lang="es"><head/><body><svg xmlns="http://www.w3.org/2000/svg" xml:lang="1">'
            'xxx</svg></body></html>')
        self.ae(root.xpath('//@lang'), ['es'])
        self.assertIn('{%s}lang' % XML, root.attrib)
        self.ae(root.xpath('//@xml:lang'), ['fr', '1'])
        root = nsparse('<html xml:lang="fr" lang="es"><svg xml:lang="1">xxx')
        self.ae(root.xpath('//@xml:lang'), ['1'])
        self.assertIn('xml:lang', root.attrib)
{'fraction_non_alphanumeric': 0.2231578947368421, 'fraction_numerical': 0.02755980861244019, 'mean_word_length': 3.294165981922761, 'pattern_counts': {'":': 0, '<': 87, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 18, 'xml': 31}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '720214', 'n_tokens_mistral': 2044, 'n_tokens_neox': 1935, 'n_words': 296}