<commit_before>"""
XPath selectors based on lxml
"""

from lxml import etree
import six

from parsel.utils import extract_regex
from parsel.utils import flatten, iflatten
from .csstranslator import ScrapyHTMLTranslator, ScrapyGenericTranslator


class SafeXMLParser(etree.XMLParser):
    def __init__(self, *args, **kwargs):
        kwargs.setdefault('resolve_entities', False)
        super(SafeXMLParser, self).__init__(*args, **kwargs)

_ctgroup = {
    'html': {'_parser': etree.HTMLParser,
             '_csstranslator': ScrapyHTMLTranslator(),
             '_tostring_method': 'html'},
    'xml': {'_parser': SafeXMLParser,
            '_csstranslator': ScrapyGenericTranslator(),
            '_tostring_method': 'xml'},
}


def _st(st):
    if st is None:
        return 'html'
    elif st in _ctgroup:
        return st
    else:
        raise ValueError('Invalid type: %s' % st)


class Selector(object):

    __slots__ = ['text', 'namespaces', 'type', '_expr', '_root',
                 '__weakref__', '_parser', '_csstranslator', '_tostring_method']

    _default_type = None
    _default_namespaces = {
        "re": "http://exslt.org/regular-expressions",

        # supported in libxslt:
        # set:difference
        # set:has-same-node
        # set:intersection
        # set:leading
        # set:trailing
        "set": "http://exslt.org/sets"
    }
    _lxml_smart_strings = False

    def __init__(self, text=None, type=None, namespaces=None, root=None, _expr=None):
        self.type = st = _st(type or self._default_type)
        self._parser = _ctgroup[st]['_parser']
        self._csstranslator = _ctgroup[st]['_csstranslator']
        self._tostring_method = _ctgroup[st]['_tostring_method']

        if text is not None:
            root = self._get_root(text)
        elif root is None:
            raise ValueError("Selector needs either text or root argument")

        self.namespaces = dict(self._default_namespaces)
        if namespaces is not None:
            self.namespaces.update(namespaces)
        self._root = root
        self._expr = _expr

    def _get_root(self, text):
        body = text.strip().encode('utf8') or u'<html/>'
        parser = self._parser(recover=True, encoding='utf8')
        return etree.fromstring(body, parser=parser)

    def xpath(self, query):<commit_msg>fix fallback literal to be bytes<commit_after>"""
XPath selectors based on lxml
"""

from lxml import etree
import six

from parsel.utils import extract_regex
from parsel.utils import flatten, iflatten
from .csstranslator import ScrapyHTMLTranslator, ScrapyGenericTranslator


class SafeXMLParser(etree.XMLParser):
    def __init__(self, *args, **kwargs):
        kwargs.setdefault('resolve_entities', False)
        super(SafeXMLParser, self).__init__(*args, **kwargs)

_ctgroup = {
    'html': {'_parser': etree.HTMLParser,
             '_csstranslator': ScrapyHTMLTranslator(),
             '_tostring_method': 'html'},
    'xml': {'_parser': SafeXMLParser,
            '_csstranslator': ScrapyGenericTranslator(),
            '_tostring_method': 'xml'},
}


def _st(st):
    if st is None:
        return 'html'
    elif st in _ctgroup:
        return st
    else:
        raise ValueError('Invalid type: %s' % st)


class Selector(object):

    __slots__ = ['text', 'namespaces', 'type', '_expr', '_root',
                 '__weakref__', '_parser', '_csstranslator', '_tostring_method']

    _default_type = None
    _default_namespaces = {
        "re": "http://exslt.org/regular-expressions",

        # supported in libxslt:
        # set:difference
        # set:has-same-node
        # set:intersection
        # set:leading
        # set:trailing
        "set": "http://exslt.org/sets"
    }
    _lxml_smart_strings = False

    def __init__(self, text=None, type=None, namespaces=None, root=None, _expr=None):
        self.type = st = _st(type or self._default_type)
        self._parser = _ctgroup[st]['_parser']
        self._csstranslator = _ctgroup[st]['_csstranslator']
        self._tostring_method = _ctgroup[st]['_tostring_method']

        if text is not None:
            root = self._get_root(text)
        elif root is None:
            raise ValueError("Selector needs either text or root argument")

        self.namespaces = dict(self._default_namespaces)
        if namespaces is not None:
            self.namespaces.update(namespaces)
        self._root = root
        self._expr = _expr

    def _get_root(self, text):
        body = text.strip().encode('utf8') or b'<html/>'
        parser = self._parser(recover=True, encoding='utf8')
        return etree.fromstring(body, parser=parser)

    def xpath(self, query):
{'fraction_non_alphanumeric': 0.1192562513357555, 'fraction_numerical': 0.0008548835221201111, 'mean_word_length': 2.8048780487804876, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2049374', 'n_tokens_mistral': 1491, 'n_tokens_neox': 1462, 'n_words': 336}