<commit_before>import os
from distutils import log

from setuptools.extern.six.moves import map


class Installer:

    def install_namespaces(self):
        nsp = self._get_all_ns_packages()
        if not nsp:
            return
        filename, ext = os.path.splitext(self.target)
        filename += '-nspkg.pth'
        self.outputs.append(filename)
        log.info("Installing %s", filename)
        lines = map(self._gen_nspkg_line, nsp)

        if self.dry_run:
            # always generate the lines, even in dry run
            list(lines)
            return

        with open(filename, 'wt') as f:
            f.writelines(lines)

    _nspkg_tmpl = (
        "import sys, types, os",
        "pep420 = sys.version_info > (3, 3)",
        "p = os.path.join(sys._getframe(1).f_locals['sitedir'], *%(pth)r)",
        "ie = os.path.exists(os.path.join(p,'__init__.py'))",
        "m = not ie and not pep420 and "
            "sys.modules.setdefault(%(pkg)r, types.ModuleType(%(pkg)r))",
        "mp = (m or []) and m.__dict__.setdefault('__path__',[])",
        "(p not in mp) and mp.append(p)",
    )
    "lines for the namespace installer"

    _nspkg_tmpl_multi = (
        'm and setattr(sys.modules[%(parent)r], %(child)r, m)',
    )
    "additional line(s) when a parent package is indicated"

    @classmethod
    def _gen_nspkg_line(cls, pkg):
        # ensure pkg is not a unicode string under Python 2.7
        pkg = str(pkg)
        pth = tuple(pkg.split('.'))
        tmpl_lines = cls._nspkg_tmpl
        parent, sep, child = pkg.rpartition('.')
        if parent:
            tmpl_lines += cls._nspkg_tmpl_multi
        return ';'.join(tmpl_lines) % locals() + '\n'

    def _get_all_ns_packages(self):
        """Return sorted list of all package namespaces"""
        nsp = set()
        pkgs = self.distribution.namespace_packages or []
        for pkg in pkgs:
            pkg = pkg.split('.')
            while pkg:
                nsp.add('.'.join(pkg))
                pkg.pop()
        return sorted(nsp)<commit_msg>Extract _pkg_names function and add test.<commit_after>import os
from distutils import log
import itertools

from setuptools.extern.six.moves import map


flatten = itertools.chain.from_iterable


class Installer:

    def install_namespaces(self):
        nsp = self._get_all_ns_packages()
        if not nsp:
            return
        filename, ext = os.path.splitext(self.target)
        filename += '-nspkg.pth'
        self.outputs.append(filename)
        log.info("Installing %s", filename)
        lines = map(self._gen_nspkg_line, nsp)

        if self.dry_run:
            # always generate the lines, even in dry run
            list(lines)
            return

        with open(filename, 'wt') as f:
            f.writelines(lines)

    _nspkg_tmpl = (
        "import sys, types, os",
        "pep420 = sys.version_info > (3, 3)",
        "p = os.path.join(sys._getframe(1).f_locals['sitedir'], *%(pth)r)",
        "ie = os.path.exists(os.path.join(p,'__init__.py'))",
        "m = not ie and not pep420 and "
            "sys.modules.setdefault(%(pkg)r, types.ModuleType(%(pkg)r))",
        "mp = (m or []) and m.__dict__.setdefault('__path__',[])",
        "(p not in mp) and mp.append(p)",
    )
    "lines for the namespace installer"

    _nspkg_tmpl_multi = (
        'm and setattr(sys.modules[%(parent)r], %(child)r, m)',
    )
    "additional line(s) when a parent package is indicated"

    @classmethod
    def _gen_nspkg_line(cls, pkg):
        # ensure pkg is not a unicode string under Python 2.7
        pkg = str(pkg)
        pth = tuple(pkg.split('.'))
        tmpl_lines = cls._nspkg_tmpl
        parent, sep, child = pkg.rpartition('.')
        if parent:
            tmpl_lines += cls._nspkg_tmpl_multi
        return ';'.join(tmpl_lines) % locals() + '\n'

    def _get_all_ns_packages(self):
        """Return sorted list of all package namespaces"""
        pkgs = self.distribution.namespace_packages or []
        return sorted(flatten(map(self._pkg_names, pkgs)))

    @staticmethod
    def _pkg_names(pkg):
        """
        Given a namespace package, yield the components of that
        package.

        >>> names = Installer._pkg_names('a.b.c')
        >>> set(names) == set(['a', 'a.b', 'a.b.c'])
        True
        """
        parts = pkg.split('.')
        while parts:
            yield '.'.join(parts)
            parts.pop()
{'fraction_non_alphanumeric': 0.13276134567622488, 'fraction_numerical': 0.004967261232783924, 'mean_word_length': 2.159771754636234, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '889072', 'n_tokens_mistral': 1499, 'n_tokens_neox': 1439, 'n_words': 346}