<commit_before># -*- coding: utf-8; -*-

import os.path
import itertools


class SpaceList(list):
    def __add__(self, other):
        return SpaceList(super(SpaceList, self).__add__(other))

    def __str__(self):
        return ' '.join(map(str, self))

    def paths(self):
        return SpaceList(getattr(x, 'path', x) for x in self)


class FileMap(dict):
    __iter__ = dict.iteritems

    def sources(self):
        return SpaceList(self.iterkeys())

    def targets(self):
        return SpaceList(self.itervalues())

    def iterpaths(self):
        for source, target in self:
            yield (source.path, target.path)

    def target_paths(self):
        return SpaceList(x.path for x in self.targets())


def list_subdirs(dirname, recursive=False, exclude=[]):
    entries = [e for e in os.listdir(dirname) if e not in exclude]
    paths = [os.path.join(dirname, e) for e in entries]
    dirs = filter(os.path.isdir, paths)
    if recursive:
        sub = itertools.chain.from_iterable(
            list_subdirs(d, recursive=True, exclude=exclude) for d in dirs)
        dirs.extend(sub)
    return dirs


def format_available_options(items, head_width, head_color='cyan', 
                             default=None, default_mark="[DEFAULT]", 
                             default_mark_color='red'):
    from ino.filters import colorize
    default_mark = colorize(default_mark + ' ', default_mark_color)
    lines = ['%s: %s%s' % (colorize('%%%ds' % head_width % key, head_color), 
                           default_mark if key == default else '', 
                           val) 
             for key, val in items]
    return '\n'.join(lines)
<commit_msg>fix: do not include dot directories (like .git) within libraries to -I flags<commit_after># -*- coding: utf-8; -*-

import os.path
import itertools


class SpaceList(list):
    def __add__(self, other):
        return SpaceList(super(SpaceList, self).__add__(other))

    def __str__(self):
        return ' '.join(map(str, self))

    def paths(self):
        return SpaceList(getattr(x, 'path', x) for x in self)


class FileMap(dict):
    __iter__ = dict.iteritems

    def sources(self):
        return SpaceList(self.iterkeys())

    def targets(self):
        return SpaceList(self.itervalues())

    def iterpaths(self):
        for source, target in self:
            yield (source.path, target.path)

    def target_paths(self):
        return SpaceList(x.path for x in self.targets())


def list_subdirs(dirname, recursive=False, exclude=[]):
    entries = [e for e in os.listdir(dirname) if e not in exclude and not e.startswith('.')]
    paths = [os.path.join(dirname, e) for e in entries]
    dirs = filter(os.path.isdir, paths)
    if recursive:
        sub = itertools.chain.from_iterable(
            list_subdirs(d, recursive=True, exclude=exclude) for d in dirs)
        dirs.extend(sub)
    return dirs


def format_available_options(items, head_width, head_color='cyan', 
                             default=None, default_mark="[DEFAULT]", 
                             default_mark_color='red'):
    from ino.filters import colorize
    default_mark = colorize(default_mark + ' ', default_mark_color)
    lines = ['%s: %s%s' % (colorize('%%%ds' % head_width % key, head_color), 
                           default_mark if key == default else '', 
                           val) 
             for key, val in items]
    return '\n'.join(lines)

{'fraction_non_alphanumeric': 0.1182108626198083, 'fraction_numerical': 0.0005808887598024978, 'mean_word_length': 2.471774193548387, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5746123', 'n_tokens_mistral': 1076, 'n_tokens_neox': 1038, 'n_words': 281}