<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Python 2 compatibility
from __future__ import unicode_literals
import xml.etree.ElementTree as Etree
import htmlement
import examples
import tempfile
import pytest
import io
import os


def quick_parsehtml(html, encoding=""):
    obj = htmlement.HTMLement(encoding=encoding)
    obj.feed(html)
    root = obj.close()
    assert Etree.iselement(root)
    return root


def quick_parse_filter(html, tag, attrs=None, encoding=""):
    obj = htmlement.HTMLement(tag, attrs, encoding=encoding)
    obj.feed(html)
    return obj.close()


def test_initialization():
    # Check that the parser even starts
    obj = htmlement.HTMLement()
    assert isinstance(obj, htmlement.HTMLement)


# ############################# HTML Test ############################## #


def test_basic_tree():
    # Check that I can parse a simple tree
    html = "<html><body></body></html>"
    root = quick_parsehtml(html)
    assert root.tag == "html"
    assert root[0].tag == "body"


def test_basic_partial():
    # Check that I can parse a simple tree segment at a time
    html = "<html><body></body></html>"
    obj = htmlement.HTMLement()
    obj.feed(html[:9])
    obj.feed(html[9:])
    root = obj.close()
    assert Etree.iselement(root)
    assert root.tag == "html"
    assert root[0].tag == "body"


def test_nohtml_tree():
    # Check the the missing html starting tag is created
    html = "<body></body>"
    root = quick_parsehtml(html)
    assert root.tag == "html"
    assert root[0].tag == "body"
    assert Etree.tostring(root, method="html") == b'<html><body></body></html>'


def test_text():
    html = "<html><body>text</body></html>"
    root = quick_parsehtml(html)
    assert root.tag == "html"
    assert root[0].tag == "body"
    assert root[0].attrib == {}
    assert root[0].text == "text"


def test_attrib():
    html = "<html><body test='yes'>text</body></html>"
    root = quick_parsehtml(html)
    assert root[0].attrib == {"test": "yes"}


def test_tail():
    html = "<html><body test='yes'><p>text</p>tail</body></html>"
    root = quick_parsehtml(html)
    assert root[0][0].tail == "tail"


def test_self_closing_normal():
    html = "<html><check test='self closing'/></html>"
    root = quick_parsehtml(html)
    assert root[0].attrib.get("test") == "self closing"


def test_self_closing_void():
    html = "<html><img src='http://myimages.com/myimage.jpg'/></html>"
    root = quick_parsehtml(html)
    assert root[0].tag == "img"
    assert root[0].attrib.get("src") == "http://myimages.com/myimage.jpg"


def test_open_void():
    html = "<html><img src='http://myimages.com/myimage.jpg'></html>"
    root = quick_parsehtml(html)
    assert root[0].tag == "img"
    assert root[0].attrib.get("src") == "http://myimages.com/myimage.jpg"


def test_comment():
    html = "<html><body><!--This is a comment.--><p>This is a paragraph.</p></body></html>"
    root = quick_parsehtml(html)
    assert root[0][0].text == "This is a comment."
    assert root[0][1].tag == "p"
    assert root[0][1].text == "This is a paragraph."


def test_missing_end_tag():
    # Test for a missing 'a' end tag
    html = "<html><body><a href='http://google.ie/'>link</body></html>"
    root = quick_parsehtml(html)
    assert root.find(".//a").get("href") == "http://google.ie/"
    assert Etree.tostring(root, method="html") == b'<html><body><a href="http://google.ie/">link</a></body></html>'


def test_extra_tag():
    # Check that a extra tag that should not exist was removed
    html = "<html><body></div></body></html>"
    root = quick_parsehtml(html)
    assert len(root[0]) == 0
    assert Etree.tostring(root, method="html") == b'<html><body></body></html>'


# ############################# HTML Entity ############################## #


def test_entity_name_euro():
    html = "<html><body>cost is &euro;49.99</body></html>"
    root = quick_parsehtml(html)
    assert root[0].text == "cost is €49.99"


def test_entity_number_euro():
    html = "<html><body>cost is &#8364;49.99</body></html>"
    root = quick_parsehtml(html)
    assert root[0].text == "cost is €49.99"


def test_entity_hex_euro():
    html = "<html><body>cost is &#x20AC;49.99</body></html>"
    root = quick_parsehtml(html)
    assert root[0].text == "cost is €49.99"


def test_entity_name_euro_fail():
    html = "<html><body>cost is &euros;49.99</body></html>"
    root = quick_parsehtml(html)
    assert "euros" in root[0].text


def test_entity_hex_euro_fail():
    html = "<html><body>cost is &#xD899;49.99</body></html>"
    root = quick_parsehtml(html)
    assert "€" not in root[0].text


# ############################# Filter Test ############################## #


def test_tag_match():
    html = "<html><body><div test='attribute'><p>text</p></div></body></html>"
    root = quick_parse_filter(html, "div")
    assert root.tag == "div"
    assert root[0].tag == "p"


def test_tag_no_match():
    html = "<html><body></body></html>"
    with pytest.raises(RuntimeError) as excinfo:
        quick_parse_filter(html, "div")
    excinfo.match("Unable to find requested section with tag of")


def test_attrib_match():
    html = "<html><body><div test='attribute'><p>text</p></div><div test='yes'>text</div></body></html>"<commit_msg>tweak test test_entity_hex_euro_fail<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Python 2 compatibility
from __future__ import unicode_literals
import xml.etree.ElementTree as Etree
import htmlement
import examples
import tempfile
import pytest
import io
import os


def quick_parsehtml(html, encoding=""):
    obj = htmlement.HTMLement(encoding=encoding)
    obj.feed(html)
    root = obj.close()
    assert Etree.iselement(root)
    return root


def quick_parse_filter(html, tag, attrs=None, encoding=""):
    obj = htmlement.HTMLement(tag, attrs, encoding=encoding)
    obj.feed(html)
    return obj.close()


def test_initialization():
    # Check that the parser even starts
    obj = htmlement.HTMLement()
    assert isinstance(obj, htmlement.HTMLement)


# ############################# HTML Test ############################## #


def test_basic_tree():
    # Check that I can parse a simple tree
    html = "<html><body></body></html>"
    root = quick_parsehtml(html)
    assert root.tag == "html"
    assert root[0].tag == "body"


def test_basic_partial():
    # Check that I can parse a simple tree segment at a time
    html = "<html><body></body></html>"
    obj = htmlement.HTMLement()
    obj.feed(html[:9])
    obj.feed(html[9:])
    root = obj.close()
    assert Etree.iselement(root)
    assert root.tag == "html"
    assert root[0].tag == "body"


def test_nohtml_tree():
    # Check the the missing html starting tag is created
    html = "<body></body>"
    root = quick_parsehtml(html)
    assert root.tag == "html"
    assert root[0].tag == "body"
    assert Etree.tostring(root, method="html") == b'<html><body></body></html>'


def test_text():
    html = "<html><body>text</body></html>"
    root = quick_parsehtml(html)
    assert root.tag == "html"
    assert root[0].tag == "body"
    assert root[0].attrib == {}
    assert root[0].text == "text"


def test_attrib():
    html = "<html><body test='yes'>text</body></html>"
    root = quick_parsehtml(html)
    assert root[0].attrib == {"test": "yes"}


def test_tail():
    html = "<html><body test='yes'><p>text</p>tail</body></html>"
    root = quick_parsehtml(html)
    assert root[0][0].tail == "tail"


def test_self_closing_normal():
    html = "<html><check test='self closing'/></html>"
    root = quick_parsehtml(html)
    assert root[0].attrib.get("test") == "self closing"


def test_self_closing_void():
    html = "<html><img src='http://myimages.com/myimage.jpg'/></html>"
    root = quick_parsehtml(html)
    assert root[0].tag == "img"
    assert root[0].attrib.get("src") == "http://myimages.com/myimage.jpg"


def test_open_void():
    html = "<html><img src='http://myimages.com/myimage.jpg'></html>"
    root = quick_parsehtml(html)
    assert root[0].tag == "img"
    assert root[0].attrib.get("src") == "http://myimages.com/myimage.jpg"


def test_comment():
    html = "<html><body><!--This is a comment.--><p>This is a paragraph.</p></body></html>"
    root = quick_parsehtml(html)
    assert root[0][0].text == "This is a comment."
    assert root[0][1].tag == "p"
    assert root[0][1].text == "This is a paragraph."


def test_missing_end_tag():
    # Test for a missing 'a' end tag
    html = "<html><body><a href='http://google.ie/'>link</body></html>"
    root = quick_parsehtml(html)
    assert root.find(".//a").get("href") == "http://google.ie/"
    assert Etree.tostring(root, method="html") == b'<html><body><a href="http://google.ie/">link</a></body></html>'


def test_extra_tag():
    # Check that a extra tag that should not exist was removed
    html = "<html><body></div></body></html>"
    root = quick_parsehtml(html)
    assert len(root[0]) == 0
    assert Etree.tostring(root, method="html") == b'<html><body></body></html>'


# ############################# HTML Entity ############################## #


def test_entity_name_euro():
    html = "<html><body>cost is &euro;49.99</body></html>"
    root = quick_parsehtml(html)
    assert root[0].text == "cost is €49.99"


def test_entity_number_euro():
    html = "<html><body>cost is &#8364;49.99</body></html>"
    root = quick_parsehtml(html)
    assert root[0].text == "cost is €49.99"


def test_entity_hex_euro():
    html = "<html><body>cost is &#x20AC;49.99</body></html>"
    root = quick_parsehtml(html)
    assert root[0].text == "cost is €49.99"


def test_entity_name_euro_fail():
    html = "<html><body>cost is &euros;49.99</body></html>"
    root = quick_parsehtml(html)
    assert "euros" in root[0].text


def test_entity_hex_euro_fail():
    html = "<html><body>cost is &#xDB9900;49.99</body></html>"
    root = quick_parsehtml(html)
    assert "€" not in root[0].text


# ############################# Filter Test ############################## #


def test_tag_match():
    html = "<html><body><div test='attribute'><p>text</p></div></body></html>"
    root = quick_parse_filter(html, "div")
    assert root.tag == "div"
    assert root[0].tag == "p"


def test_tag_no_match():
    html = "<html><body></body></html>"
    with pytest.raises(RuntimeError) as excinfo:
        quick_parse_filter(html, "div")
    excinfo.match("Unable to find requested section with tag of")


def test_attrib_match():
    html = "<html><body><div test='attribute'><p>text</p></div><div test='yes'>text</div></body></html>"
{'fraction_non_alphanumeric': 0.20815672977300556, 'fraction_numerical': 0.013845719129697655, 'mean_word_length': 3.8863322595490106, 'pattern_counts': {'":': 2, '<': 215, '<?xml version=': 0, '>': 215, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5659893', 'n_tokens_mistral': 3653, 'n_tokens_neox': 3551, 'n_words': 793}