<commit_before>from codecs import getdecoder
import cgi
from datetime import datetime
from fnmatch import fnmatchcase
from heapq import merge
from itertools import chain, compress, groupby, izip_longest
import json
from operator import itemgetter
import os
from os import stat
from os.path import dirname
import shutil
import subprocess
import sys
from sys import exc_info
from traceback import format_exc
from warnings import warn

from concurrent.futures import as_completed, ProcessPoolExecutor
from jinja2 import Markup

from dxr.config import Config
from dxr.plugins import load_htmlifiers, load_indexers
import dxr.languages
import dxr.mime
from dxr.utils import load_template_env, connect_database, open_log


def linked_pathname(path, tree_name):
    """Return a list of (server-relative URL, subtree name) tuples that can be
    used to display linked path components in the headers of file or folder
    pages.

    :arg path: The path that will be split

    """
    # Hold the root of the tree:
    components = [('/%s/source' % tree_name, tree_name)]

    # Populate each subtree:
    dirs = path.split(os.sep)  # TODO: Trips on \/ in path.

    # A special case when we're dealing with the root tree. Without
    # this, it repeats:
    if not path:
        return components

    for idx in range(1, len(dirs)+1):
        subtree_path = os.path.join('/', tree_name, 'source', *dirs[:idx])
        subtree_name = os.path.split(subtree_path)[1] or tree_name<commit_msg>Supply fallback for itertools.compress for Python 2.6<commit_after>from codecs import getdecoder
import cgi
from datetime import datetime
from fnmatch import fnmatchcase
from heapq import merge
from itertools import chain, groupby, izip_longest
import json
from operator import itemgetter
import os
from os import stat
from os.path import dirname
import shutil
import subprocess
import sys
from sys import exc_info
from traceback import format_exc
from warnings import warn

from concurrent.futures import as_completed, ProcessPoolExecutor
from jinja2 import Markup

from dxr.config import Config
from dxr.plugins import load_htmlifiers, load_indexers
import dxr.languages
import dxr.mime
from dxr.utils import load_template_env, connect_database, open_log

try:
    from itertools import compress
except ImportError:
    from itertools import izip
    def compress(data, selectors):
        return (d for d, s in izip(data, selectors) if s)


def linked_pathname(path, tree_name):
    """Return a list of (server-relative URL, subtree name) tuples that can be
    used to display linked path components in the headers of file or folder
    pages.

    :arg path: The path that will be split

    """
    # Hold the root of the tree:
    components = [('/%s/source' % tree_name, tree_name)]

    # Populate each subtree:
    dirs = path.split(os.sep)  # TODO: Trips on \/ in path.

    # A special case when we're dealing with the root tree. Without
    # this, it repeats:
    if not path:
        return components

    for idx in range(1, len(dirs)+1):
        subtree_path = os.path.join('/', tree_name, 'source', *dirs[:idx])
        subtree_name = os.path.split(subtree_path)[1] or tree_name
{'fraction_non_alphanumeric': 0.0657439446366782, 'fraction_numerical': 0.0031456432840515887, 'mean_word_length': 3.9302325581395348, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5578267', 'n_tokens_mistral': 950, 'n_tokens_neox': 927, 'n_words': 382}