<commit_before>"""
A simple XPath-like language for tree traversal.

This works by creating a filter chain of generator functions.  Each
function selects a part of the expression, e.g. a child node, a
specific descendant or a node that holds an attribute.
"""

from __future__ import absolute_import

import re
import operator

path_tokenizer = re.compile(
    "("
    "'[^']*'|\"[^\"]*\"|"
    "//?|"
    "\(\)|"
    "==?|"
    "[/.*\[\]\(\)@])|"
    "([^/\[\]\(\)@=\s]+)|"
    "\s+"
    ).findall

def iterchildren(node, attr_name):
    # returns an iterable of all child nodes of that name
    child = getattr(node, attr_name)
    if child is not None:
        if type(child) is list:
            return child
        else:
            return [child]
    else:
        return ()

def _get_first_or_none(it):
    try:
        try:
            _next = it.next
        except AttributeError:
            return next(it)
        else:
            return _next()
    except StopIteration:
        return None
<commit_msg>Fix some Python warnings about unknown/unnecessary string escape sequences.<commit_after>"""
A simple XPath-like language for tree traversal.

This works by creating a filter chain of generator functions.  Each
function selects a part of the expression, e.g. a child node, a
specific descendant or a node that holds an attribute.
"""

from __future__ import absolute_import

import re
import operator

path_tokenizer = re.compile(
    r"("
    r"'[^']*'|\"[^\"]*\"|"
    r"//?|"
    r"\(\)|"
    r"==?|"
    r"[/.*\[\]()@])|"
    r"([^/\[\]()@=\s]+)|"
    r"\s+"
    ).findall

def iterchildren(node, attr_name):
    # returns an iterable of all child nodes of that name
    child = getattr(node, attr_name)
    if child is not None:
        if type(child) is list:
            return child
        else:
            return [child]
    else:
        return ()

def _get_first_or_none(it):
    try:
        try:
            _next = it.next
        except AttributeError:
            return next(it)
        else:
            return _next()
    except StopIteration:
        return None

{'fraction_non_alphanumeric': 0.13307984790874525, 'fraction_numerical': 0.0, 'mean_word_length': 2.194233687405159, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5495260', 'n_tokens_mistral': 666, 'n_tokens_neox': 655, 'n_words': 218}