<commit_before><?php

class Test_PureHTMLDefinition extends UnitTestCase
{
    
    var $def;
    
    function Test_PureHTMLDefinition() {
        $this->UnitTestCase();
        $this->def = new PureHTMLDefinition();
        $this->def->loadData();
    }
    
    function test_removeForeignElements() {
        
        $inputs = array();
        $expect = array();
        
        $inputs[0] = array();
        $expect[0] = $inputs[0];
        
        $inputs[1] = array(
            new MF_Text('This is ')
           ,new MF_StartTag('b', array())
           ,new MF_Text('bold')
           ,new MF_EndTag('b')
           ,new MF_Text(' text')
            );
        $expect[1] = $inputs[1];
        
        $inputs[2] = array(
            new MF_StartTag('asdf')
           ,new MF_EndTag('asdf')
           ,new MF_StartTag('d', array('href' => 'bang!'))
           ,new MF_EndTag('d')
           ,new MF_StartTag('pooloka')
           ,new MF_StartTag('poolasdf')
           ,new MF_StartTag('ds', array('moogle' => '&'))
           ,new MF_EndTag('asdf')
           ,new MF_EndTag('asdf')
            );
        $expect[2] = array(
            new MF_Text('<asdf>')
           ,new MF_Text('</asdf>')
           ,new MF_Text('<d href="bang!">')
           ,new MF_Text('</d>')
           ,new MF_Text('<pooloka>')
           ,new MF_Text('<poolasdf>')
           ,new MF_Text('<ds moogle="&amp;">')
           ,new MF_Text('</asdf>')
           ,new MF_Text('</asdf>')
            );
        
        foreach ($inputs as $i => $input) {
            $result = $this->def->removeForeignElements($input);
            $this->assertEqual($expect[$i], $result);
            paintIf($result, $result != $expect[$i]);
        }
        
    }
    
    function test_makeWellFormed() {
        
        $inputs = array();
        $expect = array();
        
        $inputs[0] = array();
        $expect[0] = $inputs[0];
        
        $inputs[1] = array(
            new MF_Text('This is ')
           ,new MF_StartTag('b')
           ,new MF_Text('bold')
           ,new MF_EndTag('b')
           ,new MF_Text(' text')
            );
        $expect[1] = $inputs[1];
        
        $inputs[2] = array(
            new MF_StartTag('b')
           ,new MF_Text('Unclosed tag, gasp!')
            );
        $expect[2] = array(
            new MF_StartTag('b')
           ,new MF_Text('Unclosed tag, gasp!')
           ,new MF_EndTag('b')
            );
        
        $inputs[3] = array(
            new MF_StartTag('b')
           ,new MF_StartTag('i')
           ,new MF_Text('The b is closed, but the i is not')
           ,new MF_EndTag('b')
            );
        $expect[3] = array(
            new MF_StartTag('b')
           ,new MF_StartTag('i')
           ,new MF_Text('The b is closed, but the i is not')
           ,new MF_EndTag('i')
           ,new MF_EndTag('b')
            );
        
        $inputs[4] = array(
            new MF_Text('Hey, recycle unused end tags!')
           ,new MF_EndTag('b')
            );
        $expect[4] = array(
            new MF_Text('Hey, recycle unused end tags!')
           ,new MF_Text('</b>')
            );
        
        $inputs[5] = array(new MF_StartTag('br'));
        $expect[5] = array(new MF_EmptyTag('br'));
        
        $inputs[6] = array(new MF_EmptyTag('div'));
        $expect[6] = array(
            new MF_StartTag('div')
           ,new MF_EndTag('div')
            );
        
        foreach ($inputs as $i => $input) {
            $result = $this->def->makeWellFormed($input);
            $this->assertEqual($expect[$i], $result);
            paintIf($result, $result != $expect[$i]);
        }
        
    }
    
}

?><commit_msg>Refine tests to include attribute checks and some other cases.<commit_after><?php

class Test_PureHTMLDefinition extends UnitTestCase
{
    
    var $def;
    
    function Test_PureHTMLDefinition() {
        $this->UnitTestCase();
        $this->def = new PureHTMLDefinition();
        $this->def->loadData();
    }
    
    function test_removeForeignElements() {
        
        $inputs = array();
        $expect = array();
        
        $inputs[0] = array();
        $expect[0] = $inputs[0];
        
        $inputs[1] = array(
            new MF_Text('This is ')
           ,new MF_StartTag('b', array())
           ,new MF_Text('bold')
           ,new MF_EndTag('b')
           ,new MF_Text(' text')
            );
        $expect[1] = $inputs[1];
        
        $inputs[2] = array(
            new MF_StartTag('asdf')
           ,new MF_EndTag('asdf')
           ,new MF_StartTag('d', array('href' => 'bang!'))
           ,new MF_EndTag('d')
           ,new MF_StartTag('pooloka')
           ,new MF_StartTag('poolasdf')
           ,new MF_StartTag('ds', array('moogle' => '&'))
           ,new MF_EndTag('asdf')
           ,new MF_EndTag('asdf')
            );
        $expect[2] = array(
            new MF_Text('<asdf>')
           ,new MF_Text('</asdf>')
           ,new MF_Text('<d href="bang!">')
           ,new MF_Text('</d>')
           ,new MF_Text('<pooloka>')
           ,new MF_Text('<poolasdf>')
           ,new MF_Text('<ds moogle="&amp;">')
           ,new MF_Text('</asdf>')
           ,new MF_Text('</asdf>')
            );
        
        foreach ($inputs as $i => $input) {
            $result = $this->def->removeForeignElements($input);
            $this->assertEqual($expect[$i], $result);
            paintIf($result, $result != $expect[$i]);
        }
        
    }
    
    function test_makeWellFormed() {
        
        $inputs = array();
        $expect = array();
        
        $inputs[0] = array();
        $expect[0] = $inputs[0];
        
        $inputs[1] = array(
            new MF_Text('This is ')
           ,new MF_StartTag('b')
           ,new MF_Text('bold')
           ,new MF_EndTag('b')
           ,new MF_Text(' text')
           ,new MF_EmptyTag('br')
            );
        $expect[1] = $inputs[1];
        
        $inputs[2] = array(
            new MF_StartTag('b')
           ,new MF_Text('Unclosed tag, gasp!')
            );
        $expect[2] = array(
            new MF_StartTag('b')
           ,new MF_Text('Unclosed tag, gasp!')
           ,new MF_EndTag('b')
            );
        
        $inputs[3] = array(
            new MF_StartTag('b')
           ,new MF_StartTag('i')
           ,new MF_Text('The b is closed, but the i is not')
           ,new MF_EndTag('b')
            );
        $expect[3] = array(
            new MF_StartTag('b')
           ,new MF_StartTag('i')
           ,new MF_Text('The b is closed, but the i is not')
           ,new MF_EndTag('i')
           ,new MF_EndTag('b')
            );
        
        $inputs[4] = array(
            new MF_Text('Hey, recycle unused end tags!')
           ,new MF_EndTag('b')
            );
        $expect[4] = array(
            new MF_Text('Hey, recycle unused end tags!')
           ,new MF_Text('</b>')
            );
        
        $inputs[5] = array(new MF_StartTag('br', array('style' => 'clear:both;')));
        $expect[5] = array(new MF_EmptyTag('br', array('style' => 'clear:both;')));
        
        $inputs[6] = array(new MF_EmptyTag('div', array('style' => 'clear:both;')));
        $expect[6] = array(
            new MF_StartTag('div', array('style' => 'clear:both;'))
           ,new MF_EndTag('div')
            );
        
        // demonstrates start tag precedence
        
        foreach ($inputs as $i => $input) {
            $result = $this->def->makeWellFormed($input);
            $this->assertEqual($expect[$i], $result);
            paintIf($result, $result != $expect[$i]);
        }
        
    }
    
}

?>
{'fraction_non_alphanumeric': 0.1579343193782903, 'fraction_numerical': 0.006016545500125345, 'mean_word_length': 1.4142208774583964, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '639219', 'n_tokens_mistral': 2589, 'n_tokens_neox': 2423, 'n_words': 477}