<commit_before>"""
Text types
"""
from scrapely.extractors import safehtml, htmlregion

def escape_html(text):
    """escape text for use in an html page"""
    # import here so jinja2 is only required when displaying html
    from jinja2.utils import escape
    return escape(text)

class _BaseTextProcessor(object):
    """basic text processor, defines identity functions, some of which 
    are overridden in subclasses
    """
    def extract(self, text):
        """Matches and extracts any string, as it is"""
        return text
    
    def adapt(self, text, htmlpage):
        return text
    
    def render(self, field_name, field_value, item):
        return escape_html(field_value)

class RawFieldTypeProcessor(_BaseTextProcessor):
    """Extracts the raw data, without processing. Data is escaped for presentation
    
    >>> r = RawFieldTypeProcessor()
    >>> html = htmlregion(u'<p>test</p>')
    >>> r.extract(html)
    u'<p>test</p>'
    >>> r.adapt(html, None)
    u'<p>test</p>'
    >>> str(r.render(None, '<p>test</p>', {}))
    '&lt;p&gt;test&lt;/p&gt;'
    """
    name = 'raw html'
    description = 'raw html as it appears in the page'

class TextFieldTypeProcessor(_BaseTextProcessor):
    """Extracts strings, removing all HTML markup

    Extraction and rendering does nothing, but adapt converts html into text
    >>> p = TextFieldTypeProcessor()
    >>> html = htmlregion(u'<p>test</p><!-- comment --><script> // script</script>!')
    >>> extracted = p.extract(html)
    >>> extracted
    u'test !'
    >>> p.adapt(extracted, None)
    u'test !'
    """
    name = 'text'
    description = 'extracts text from web pages, cleaning all markup'
    
    def extract(self, htmlregion):
        return htmlregion.text_content

class SafeHtmlFieldTypeProcessor(_BaseTextProcessor):
    """Extracts strings, with only a safe subset of HTML remaining

    Extraction and rendering does nothing, but adapt transforms the HTML
    >>> p = SafeHtmlFieldTypeProcessor()
    >>> html = htmlregion(u'<p>test</p> <blink>foo')<commit_msg>fix text extraction to avoid to match whitespace chars<commit_after>"""
Text types
"""
from scrapely.extractors import text as extract_text, safehtml, htmlregion

def escape_html(text):
    """escape text for use in an html page"""
    # import here so jinja2 is only required when displaying html
    from jinja2.utils import escape
    return escape(text)

class _BaseTextProcessor(object):
    """basic text processor, defines identity functions, some of which 
    are overridden in subclasses
    """
    def extract(self, text):
        """Matches and extracts any string, as it is"""
        return text
    
    def adapt(self, text, htmlpage):
        return text
    
    def render(self, field_name, field_value, item):
        return escape_html(field_value)

class RawFieldTypeProcessor(_BaseTextProcessor):
    """Extracts the raw data, without processing. Data is escaped for presentation
    
    >>> r = RawFieldTypeProcessor()
    >>> html = htmlregion(u'<p>test</p>')
    >>> r.extract(html)
    u'<p>test</p>'
    >>> r.adapt(html, None)
    u'<p>test</p>'
    >>> str(r.render(None, '<p>test</p>', {}))
    '&lt;p&gt;test&lt;/p&gt;'
    """
    name = 'raw html'
    description = 'raw html as it appears in the page'

class TextFieldTypeProcessor(_BaseTextProcessor):
    """Extracts strings, removing all HTML markup

    Extraction and rendering does nothing, but adapt converts html into text
    >>> p = TextFieldTypeProcessor()
    >>> html = htmlregion(u'<p>test</p><!-- comment --><script> // script</script>!')
    >>> extracted = p.extract(html)
    >>> extracted
    u'test !'
    >>> p.adapt(extracted, None)
    u'test !'
    >>> html = htmlregion(u'<p>&nbsp;\\n<p>')
    >>> p.extract(html)
    u''
    """
    name = 'text'
    description = 'extracts text from web pages, cleaning all markup'
    
    def extract(self, htmlregion):
        return extract_text(htmlregion.text_content)

    
class SafeHtmlFieldTypeProcessor(_BaseTextProcessor):
    """Extracts strings, with only a safe subset of HTML remaining

    Extraction and rendering does nothing, but adapt transforms the HTML
    >>> p = SafeHtmlFieldTypeProcessor()
    >>> html = htmlregion(u'<p>test</p> <blink>foo')
{'fraction_non_alphanumeric': 0.128451099672438, 'fraction_numerical': 0.0009358914365933552, 'mean_word_length': 3.402677651905252, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 115, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3188248', 'n_tokens_mistral': 1285, 'n_tokens_neox': 1257, 'n_words': 390}