<commit_before>#!/usr/bin/env python
#:coding=utf-8:

from base import HTMLSanitizationTest

from lifestream.util import * 

class TagStrippingTest(HTMLSanitizationTest):
    test_html = (
        (
            u'<b>This is a test</b>', 
            u'<b>This is a test</b>',
        ),
        (
            u'<script type="text/javascript">alert("DANGER!!");</script> <NAME>',
            u'alert(&quot;DANGER!!&quot;); <NAME>',
        ),
        (
            u'<a href="http://www.ianlewis.org/" rel="me" onclick="alert(\'woah!!\')">This is a test</a>',
            u'<a href="http://www.ianlewis.org/" rel="me">This is a test</a>',
        ),
        (
            u'<CRazY>Crazy Text</crazy>',
            u'Crazy Text',
        )
    )

class EntitiesTest(HTMLSanitizationTest):
    test_html = (
        (
            u'<b>Ian\'s Homepage</b>', 
            u'<b>Ian&apos;s Homepage</b>',
        ),
        (
            u'"I CaN HaZ SoMe <TeXt"', u'&quot;I CaN HaZ SoMe &lt;TeXt&gt;&quot;',
        ),
    )
<commit_msg>Added tests for css sanitization of style tags<commit_after>#!/usr/bin/env python
#:coding=utf-8:

from base import HTMLSanitizationTest

from lifestream.util import * 

class TagStrippingTest(HTMLSanitizationTest):
    test_html = (
        (
            u'<b>This is a test</b>', 
            u'<b>This is a test</b>',
        ),
        (
            u'<script type="text/javascript">alert("DANGER!!");</script> <NAME>',
            u'alert(&quot;DANGER!!&quot;); <NAME>',
        ),
        (
            u'<a href="http://www.ianlewis.org/" rel="me" onclick="alert(\'woah!!\')">This is a test</a>',
            u'<a href="http://www.ianlewis.org/" rel="me">This is a test</a>',
        ),
        (
            u'<CRazY>Crazy Text</crazy>',
            u'Crazy Text',
        )
    )

class EntitiesTest(HTMLSanitizationTest):
    test_html = (
        (
            u'<b>Ian\'s Homepage</b>', 
            u'<b>Ian&apos;s Homepage</b>',
        ),
        (
            u'"I CaN HaZ SoMe <TeXt"',
            u'&quot;I CaN HaZ SoMe &lt;TeXt&gt;&quot;',
        ),
    )

class CSSSanitizationTest(HTMLSanitizationTest):
    valid_tags = {
        'div': ('style',),
        'span': ('style',),
    }
    valid_styles = (
        "color",
        "font-weight",
    )
    test_html = (
        (
            u'<span style="color:#FFF;position:absolute;">My Homepage</span>', 
            u'<span style="color:#FFF;">My Homepage</span>', 
        ),
        (
            u'<span style="color:#FFF;position:absolute;font-weight:bold;">My Homepage</span>', 
            u'<span style="color:#FFF;font-weight:bold;">My Homepage</span>', 
        ),
        (
            u'<span style="color:#FFF;position:absolute   ">My Homepage</span>', 
            u'<span style="color:#FFF;">My Homepage</span>', 
        ),
        (
            u'<span style="  color:#FFF;  position:absolute;\tfont-weight:bold  ">My Homepage</span>', 
            u'<span style="color:#FFF;font-weight:bold;">My Homepage</span>', 
        ),
        (
            u'<span style="  color:#FFF;  position:absolute;\tfont-weight:bold;  aaaaa">My Homepage</span>', 
            u'<span style="color:#FFF;font-weight:bold;">My Homepage</span>', 
        ),
    )

{'fraction_non_alphanumeric': 0.18610345883073157, 'fraction_numerical': 0.0006121824303642485, 'mean_word_length': 1.984474885844749, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1530746', 'n_tokens_mistral': 1158, 'n_tokens_neox': 1127, 'n_words': 175}