<commit_before>import re

from scrapy.selector import Selector, SelectorList

_UNSUPPORTED_XPATH_ENDING = re.compile(r'.*/((@)?([^/()]+)(\(\))?)$')
_UNSUPPORTED_CSS_ENDING = re.compile(r'.*(::text|::attr\(([\w-]+)\))$')


class WebdriverXPathSelector(Selector):
    """Scrapy selector that works using XPath selectors in a remote browser.

    Based on some code from <NAME>:
        https://github.com/marconi/ghost-selector

    """
    def __init__(self, response=None, webdriver=None, element=None,
                 *args, **kwargs):
        kwargs['response'] = response
        super(WebdriverXPathSelector, self).__init__(*args, **kwargs)
        self.response = response
        self.webdriver = webdriver or response.webdriver
        self.element = element

    def _make_result(self, result):
        if type(result) is not list:
            result = [result]
        return [self.__class__(webdriver=self.webdriver, element=e)
                for e in result]

    def css(self, css):
        """Return elements using the webdriver `find_elements_by_css` method.
        """
        elem = self.element if self.element else self.webdriver
        ending = _UNSUPPORTED_CSS_ENDING.match(css)
        if ending:
            psuedo, attr = ending.groups()
            css = css[:-len(psuedo)]
        result = self._make_result(elem.find_elements_by_css_selector(css))
        if ending:
            if psuedo == '::text':
                result = (_TextNode(self.webdriver, r.element) for r in result)
            elif attr:
                result = (_NodeAttribute(r.element, attr) for r in result)
        return SelectorList(result)

    def xpath(self, xpath):
        """Return elements using the webdriver `find_elements_by_xpath` method.

        Some XPath features are not supported by the webdriver implementation.
        Namely, selecting text content or attributes:
          - /some/element/text()<commit_msg>improve comment<commit_after>import re

from scrapy.selector import Selector, SelectorList

_UNSUPPORTED_XPATH_ENDING = re.compile(r'.*/((@)?([^/()]+)(\(\))?)$')
_UNSUPPORTED_CSS_ENDING = re.compile(r'.*(::text|::attr\(([\w-]+)\))$')


class WebdriverXPathSelector(Selector):
    """Scrapy selector that works using XPath selectors in a remote browser.

    Based on some code from <NAME>:
        https://github.com/marconi/ghost-selector

    """
    def __init__(self, response=None, webdriver=None, element=None,
                 *args, **kwargs):
        kwargs['response'] = response
        super(WebdriverXPathSelector, self).__init__(*args, **kwargs)
        self.response = response
        self.webdriver = webdriver or response.webdriver
        self.element = element

    def _make_result(self, result):
        if type(result) is not list:
            result = [result]
        return [self.__class__(webdriver=self.webdriver, element=e)
                for e in result]

    def css(self, css):
        """Return elements using the webdriver `find_elements_by_css` method.
        This add support for useful css psuedo-selectors:
          - a.clicky::attr(href)
          - h2.heading::text
        """
        elem = self.element if self.element else self.webdriver
        ending = _UNSUPPORTED_CSS_ENDING.match(css)
        if ending:
            psuedo, attr = ending.groups()
            css = css[:-len(psuedo)]
        result = self._make_result(elem.find_elements_by_css_selector(css))
        if ending:
            if psuedo == '::text':
                result = (_TextNode(self.webdriver, r.element) for r in result)
            elif attr:
                result = (_NodeAttribute(r.element, attr) for r in result)
        return SelectorList(result)

    def xpath(self, xpath):
        """Return elements using the webdriver `find_elements_by_xpath` method.

        Some XPath features are not supported by the webdriver implementation.
        Namely, selecting text content or attributes:
          - /some/element/text()
{'fraction_non_alphanumeric': 0.11584754262788365, 'fraction_numerical': 0.00025075225677031093, 'mean_word_length': 2.5936936936936936, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4094541', 'n_tokens_mistral': 1168, 'n_tokens_neox': 1139, 'n_words': 299}