<commit_before>#  test_fude.py
#
#  Created by <NAME>
#  Copyright (c) 2012 <NAME>. All rights reserved.
#
import unittest
from fude import html

class HTMLTextCase(unittest.TestCase):
    def test_escape_attrs(self):
        data = ['tag#<>.<>']
        expt = '<tag class="&lt;&gt;" id="&lt;&gt;" />'
        self.assertEqual(html(data), expt)

    def test_tag(self):
        data = []
        expt = ''
        self.assertEqual(html(data), expt)

        data = ['tag']
        expt = '<tag />'
        self.assertEqual(html(data), expt)

        data = ['100']
        expt = '<100 />'
        self.assertEqual(html(data), expt)

        data = ['tag#id']
        expt = '<tag id="id" />'
        self.assertEqual(html(data), expt)

        data = ['tag.class']
        expt = '<tag class="class" />'
        self.assertEqual(html(data), expt)

        data = ['tag.class.a']
        expt = '<tag class="class a" />'
        self.assertEqual(html(data), expt)

        data = ['tag#id.class']
        expt = '<tag class="class" id="id" />'
        self.assertEqual(html(data), expt)

        data = ['tag#id.class.a']
        expt = '<tag class="class a" id="id" />'
        self.assertEqual(html(data), expt)

        data = ['tag', {'data': 'aaa'}]
        expt = '<tag data="aaa" />'
        self.assertEqual(html(data), expt)

        data = ['tag', {'data': 'aaa', 'id': 'id', 'class': 'class'}]
        expt = '<tag class="class" data="aaa" id="id" />'
        self.assertEqual(html(data), expt)

        data = ['tag#id.class', {'data': 'aaa'}]
        expt = '<tag class="class" data="aaa" id="id" />'
        self.assertEqual(html(data), expt)

        data = ['tag#id.class', {'data': 'aaa', 'id': 'ID', 'class': 'CLASS'}]
        expt = '<tag class="CLASS" data="aaa" id="ID" />'
        self.assertEqual(html(data), expt)

    def test_abbr_tag(self):
        data = ['#id']
        expt = '<div id="id" />'
        self.assertEqual(html(data), expt)

        data = ['.class']
        expt = '<div class="class" />'
        self.assertEqual(html(data), expt)

        data = ['.class.a']
        expt = '<div class="class a" />'
        self.assertEqual(html(data), expt)

        data = ['#id.class']
        expt = '<div class="class" id="id" />'
        self.assertEqual(html(data), expt)

        data = ['#id.class.a']
        expt = '<div class="class a" id="id" />'
        self.assertEqual(html(data), expt)

        data = ['', {'data': 'aaa'}]
        expt = '<div data="aaa" />'
        self.assertEqual(html(data), expt)

        data = ['', {'data': 'aaa', 'id': 'id', 'class': 'class'}]
        expt = '<div class="class" data="aaa" id="id" />'
        self.assertEqual(html(data), expt)

        data = ['#id.class', {'data': 'aaa'}]
        expt = '<div class="class" data="aaa" id="id" />'
        self.assertEqual(html(data), expt)

        data = ['#id.class', {'data': 'aaa', 'id': 'ID', 'class': 'CLASS'}]
        expt = '<div class="CLASS" data="aaa" id="ID" />'
        self.assertEqual(html(data), expt)

    def test_complex_tags(self):
        data = [['a'], ['b'], [''], []]
        expt = '<a /><b /><div />'
        self.assertEqual(html(data), expt)

        data = [['a', '100'], ['b', 200], [''], []]
        expt = '<a>100</a><b>200</b><div />'
        self.assertEqual(html(data), expt)

        data = ['html', []]
        expt = '<html></html>'
        self.assertEqual(html(data), expt)

        data = ['html', ['head'], ['body', ['p', 'hello world']]]
        expt = '<html><head /><body><p>hello world</p></body></html>'
        self.assertEqual(html(data), expt)

        data = ['html', ['head'], ['body', {'data': 'foobar'}, ['p#contents', 'hello world'], ['br.line'], 'foo', []]]
        expt = '<html><head /><body data="foobar"><p id="contents">hello world</p><br class="line" />foo</body></html>'
        self.assertEqual(html(data), expt)

if __name__ == "__main__":
    unittest.main()
<commit_msg>added test for text.<commit_after>#  test_fude.py
#
#  Created by <NAME>
#  Copyright (c) 2012 <NAME>. All rights reserved.
#
import unittest
from fude import html

class HTMLTextCase(unittest.TestCase):
    def test_escape_attrs(self):
        data = ['tag#<>.<>']
        expt = '<tag class="&lt;&gt;" id="&lt;&gt;" />'
        self.assertEqual(html(data), expt)

    def test_text(self):
        data = ''
        expt = ''
        self.assertEqual(html(data), expt)

        data = 'text'
        expt = 'text'
        self.assertEqual(html(data), expt)

    def test_tag(self):
        data = []
        expt = ''
        self.assertEqual(html(data), expt)

        data = ['tag']
        expt = '<tag />'
        self.assertEqual(html(data), expt)

        data = ['100']
        expt = '<100 />'
        self.assertEqual(html(data), expt)

        data = ['tag#id']
        expt = '<tag id="id" />'
        self.assertEqual(html(data), expt)

        data = ['tag.class']
        expt = '<tag class="class" />'
        self.assertEqual(html(data), expt)

        data = ['tag.class.a']
        expt = '<tag class="class a" />'
        self.assertEqual(html(data), expt)

        data = ['tag#id.class']
        expt = '<tag class="class" id="id" />'
        self.assertEqual(html(data), expt)

        data = ['tag#id.class.a']
        expt = '<tag class="class a" id="id" />'
        self.assertEqual(html(data), expt)

        data = ['tag', {'data': 'aaa'}]
        expt = '<tag data="aaa" />'
        self.assertEqual(html(data), expt)

        data = ['tag', {'data': 'aaa', 'id': 'id', 'class': 'class'}]
        expt = '<tag class="class" data="aaa" id="id" />'
        self.assertEqual(html(data), expt)

        data = ['tag#id.class', {'data': 'aaa'}]
        expt = '<tag class="class" data="aaa" id="id" />'
        self.assertEqual(html(data), expt)

        data = ['tag#id.class', {'data': 'aaa', 'id': 'ID', 'class': 'CLASS'}]
        expt = '<tag class="CLASS" data="aaa" id="ID" />'
        self.assertEqual(html(data), expt)

    def test_abbr_tag(self):
        data = ['#id']
        expt = '<div id="id" />'
        self.assertEqual(html(data), expt)

        data = ['.class']
        expt = '<div class="class" />'
        self.assertEqual(html(data), expt)

        data = ['.class.a']
        expt = '<div class="class a" />'
        self.assertEqual(html(data), expt)

        data = ['#id.class']
        expt = '<div class="class" id="id" />'
        self.assertEqual(html(data), expt)

        data = ['#id.class.a']
        expt = '<div class="class a" id="id" />'
        self.assertEqual(html(data), expt)

        data = ['', {'data': 'aaa'}]
        expt = '<div data="aaa" />'
        self.assertEqual(html(data), expt)

        data = ['', {'data': 'aaa', 'id': 'id', 'class': 'class'}]
        expt = '<div class="class" data="aaa" id="id" />'
        self.assertEqual(html(data), expt)

        data = ['#id.class', {'data': 'aaa'}]
        expt = '<div class="class" data="aaa" id="id" />'
        self.assertEqual(html(data), expt)

        data = ['#id.class', {'data': 'aaa', 'id': 'ID', 'class': 'CLASS'}]
        expt = '<div class="CLASS" data="aaa" id="ID" />'
        self.assertEqual(html(data), expt)

    def test_complex_tags(self):
        data = [['a'], ['b'], [''], []]
        expt = '<a /><b /><div />'
        self.assertEqual(html(data), expt)

        data = [['a', '100'], ['b', 200], [''], []]
        expt = '<a>100</a><b>200</b><div />'
        self.assertEqual(html(data), expt)

        data = ['html', []]
        expt = '<html></html>'
        self.assertEqual(html(data), expt)

        data = ['html', ['head'], ['body', ['p', 'hello world']]]
        expt = '<html><head /><body><p>hello world</p></body></html>'
        self.assertEqual(html(data), expt)

        data = ['html', ['head'], ['body', {'data': 'foobar'}, ['p#contents', 'hello world'], ['br.line'], 'foo', []]]
        expt = '<html><head /><body data="foobar"><p id="contents">hello world</p><br class="line" />foo</body></html>'
        self.assertEqual(html(data), expt)

if __name__ == "__main__":
    unittest.main()

{'fraction_non_alphanumeric': 0.22706027060270603, 'fraction_numerical': 0.005412054120541206, 'mean_word_length': 2.366873706004141, 'pattern_counts': {'":': 2, '<': 103, '<?xml version=': 0, '>': 103, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2837285', 'n_tokens_mistral': 2879, 'n_tokens_neox': 2748, 'n_words': 595}