<commit_before>from __future__ import absolute_import, division, print_function

from HTMLParser import HTMLParser
from itertools import izip
import logging

from streamcorpus import InvalidXpathError, Offset, OffsetType, XpathRange


logger = logging.getLogger(__name__)


# In HTML5, void elements are technically distinct from self-closing elements.
# Void elements have a "start" but no end tag, while self-closing elements
# have cause both the "start" and "end" events to fire.
#
# In our HTML parser, we need to be careful with void element handling,
# otherwise our state gets corrupt (because we cannot assume every start
# tag has an end tag).
VOID_ELEMENTS = set(['area', 'base', 'br', 'col', 'embed', 'hr', 'img',
                     'input', 'keygen', 'link', 'meta', 'param', 'source',
                     'track', 'wbr'])


# To be written transform.
class xpath_offsets(object):
    config_name = 'xpath_offsets'
    default_config = {}

    def __init__(self, config):
        self.config = config

    def __call__(self, si, context=None):
        if not si.body or not si.body.clean_html or not si.body.clean_visible:
            logger.warning('stream item %s: has no clean_{html,visible}, '
                           'so we will not generate Xpath offsets',
                           si.stream_id)
            return si
        add_xpaths_to_stream_item(si)
        try:
            stream_item_roundtrip_xpaths(si, quick=False)
        except XpathMismatchError:
            logger.warning('stream item %s: Failed xpath roundtrip test, '
                           'dropping', si.stream_id, exc_info=True)
            return None
        return si
<commit_msg>Put XPath offset validation behind a configuration knob.<commit_after>from __future__ import absolute_import, division, print_function

from HTMLParser import HTMLParser
from itertools import izip
import logging

from streamcorpus import InvalidXpathError, Offset, OffsetType, XpathRange


logger = logging.getLogger(__name__)


# In HTML5, void elements are technically distinct from self-closing elements.
# Void elements have a "start" but no end tag, while self-closing elements
# have cause both the "start" and "end" events to fire.
#
# In our HTML parser, we need to be careful with void element handling,
# otherwise our state gets corrupt (because we cannot assume every start
# tag has an end tag).
VOID_ELEMENTS = set(['area', 'base', 'br', 'col', 'embed', 'hr', 'img',
                     'input', 'keygen', 'link', 'meta', 'param', 'source',
                     'track', 'wbr'])


# To be written transform.
class xpath_offsets(object):
    '''Add XPath offsets to all tokens.

    This is needed for specific display flows that try to present the
    document in a browser and need to highlight tokens.

    This stage has one configuration option, `validate`.  Computing
    the XPath expressions involves parsing HTML and this may encounter
    issues.  If `validate` is set to :const:`True`, after the XPath
    expressions are generated, this stage runs an additional check
    that the XPath expression range selects the original token text.
    A failure generates a log message and drops the stream item from
    the pipeline.  Validation is disabled by default.

    '''
    config_name = 'xpath_offsets'
    default_config = {
        'validate': False,
    }

    def __init__(self, config):
        self.validate = config.get('validate',
                                   self.default_config['validate'])

    def __call__(self, si, context=None):
        if not si.body or not si.body.clean_html or not si.body.clean_visible:
            logger.warning('stream item %s: has no clean_{html,visible}, '
                           'so we will not generate Xpath offsets',
                           si.stream_id)
            return si
        add_xpaths_to_stream_item(si)
        if self.validate:
            try:
                stream_item_roundtrip_xpaths(si, quick=False)
            except XpathMismatchError:
                logger.warning('stream item %s: Failed xpath roundtrip test, '
                               'dropping', si.stream_id, exc_info=True)
                return None
        return si

{'fraction_non_alphanumeric': 0.08140085186938002, 'fraction_numerical': 0.000473260766682442, 'mean_word_length': 2.650259067357513, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1763458', 'n_tokens_mistral': 1149, 'n_tokens_neox': 1105, 'n_words': 437}