<commit_before>#! /usr/bin/python
# -*- coding: utf8 -*-

from __future__ import absolute_import, division, print_function

import os
import sys
import logging
from argparse import Namespace

import tldp

from tldp.inventory import status_classes, status_types
from tldp.utils import arg_isloglevel
from tldp.sources import arg_issourcedoc

logformat = '%(levelname)-9s %(name)s %(filename)s#%(lineno)s %(funcName)s %(message)s'
logging.basicConfig(stream=sys.stderr, format=logformat, level=logging.ERROR)
logger = logging.getLogger(__name__)


def summary(config, inv=None):
    if inv is None:
        inv = tldp.inventory.Inventory(config.pubdir, config.sourcedir)
    width = Namespace()
    width.status = max([len(x) for x in status_types])
    width.count = len(str(len(inv.source.keys())))
    for status in status_types:
        if status == 'all':
            continue
        count = len(getattr(inv, status, 0))
        s = '{0:{w.status}}  {1:{w.count}}  '.format(status, count, w=width)
        print(s, end="")
        if config.verbose:
            print(', '.join(getattr(inv, status).keys()))
        else:
            abbrev = getattr(inv, status).keys()
            s = ''
            if abbrev:
                s = s + abbrev.pop(0)
                while abbrev and len(s) < 50:
                    s = s + ', ' + abbrev.pop(0)
                if abbrev:
                    s = s + ', and %d more ...' % (len(abbrev))
            print(s)
    return 0


def detail(config, docs, inv, **kwargs):
    if inv is None:
        inv = tldp.inventory.Inventory(config.pubdir, config.sourcedir)
    if not docs:
        docs = inv.work.values()
    width = Namespace()
    width.status = max([len(x) for x in status_types])
    width.stem = max([len(x) for x in inv.source.keys()])
    # -- if user just said "list" with no args, then give the user something
    #    sane, "all"; it would make sense for this to be "work", too, but
    #    "all" seems to be less surprising
    #
    for doc in docs:
        stdout = kwargs.get('file', sys.stdout)
        doc.detail(width, config.verbose, file=stdout)
    return 0


def build(config, docs, inv):
    if inv is None:
        inv = tldp.inventory.Inventory(config.pubdir, config.sourcedir)
    if not docs:
        docs = inv.work.values()
    for source in docs:
        if source.stem in config.skip:
            logger.info("%s skipping build per request", source.stem)
            continue
        if not source.output:
            dirname = os.path.join(config.pubdir, source.stem)
            source.output = tldp.outputs.OutputDirectory(dirname)
        if not source.doctype:
            logger.warning("%s skipping document of unknown doctype",
                           source.stem)
            continue
        output = source.output
        runner = source.doctype(source=source, output=output, config=config)
        runner.generate()
    return 0


def script(config, docs, inv):
    return 0


def getDocumentNames(args):<commit_msg>allow file to be specified for summary<commit_after>#! /usr/bin/python
# -*- coding: utf8 -*-

from __future__ import absolute_import, division, print_function

import os
import sys
import logging
from argparse import Namespace

import tldp

from tldp.inventory import status_classes, status_types
from tldp.utils import arg_isloglevel
from tldp.sources import arg_issourcedoc

logformat = '%(levelname)-9s %(name)s %(filename)s#%(lineno)s %(funcName)s %(message)s'
logging.basicConfig(stream=sys.stderr, format=logformat, level=logging.ERROR)
logger = logging.getLogger(__name__)


def summary(config, inv=None, **kwargs):
    if inv is None:
        inv = tldp.inventory.Inventory(config.pubdir, config.sourcedir)
    file = kwargs.get('file', sys.stdout)
    width = Namespace()
    width.status = max([len(x) for x in status_types])
    width.count = len(str(len(inv.source.keys())))
    for status in status_types:
        if status == 'all':
            continue
        count = len(getattr(inv, status, 0))
        s = '{0:{w.status}}  {1:{w.count}}  '.format(status, count, w=width)
        print(s, end="", file=file)
        if config.verbose:
            print(', '.join(getattr(inv, status).keys()), file=file)
        else:
            abbrev = getattr(inv, status).keys()
            s = ''
            if abbrev:
                s = s + abbrev.pop(0)
                while abbrev and len(s) < 50:
                    s = s + ', ' + abbrev.pop(0)
                if abbrev:
                    s = s + ', and %d more ...' % (len(abbrev))
            print(s, file=file)
    return 0


def detail(config, docs, inv, **kwargs):
    if inv is None:
        inv = tldp.inventory.Inventory(config.pubdir, config.sourcedir)
    if not docs:
        docs = inv.work.values()
    width = Namespace()
    width.status = max([len(x) for x in status_types])
    width.stem = max([len(x) for x in inv.source.keys()])
    # -- if user just said "list" with no args, then give the user something
    #    sane, "all"; it would make sense for this to be "work", too, but
    #    "all" seems to be less surprising
    #
    for doc in docs:
        stdout = kwargs.get('file', sys.stdout)
        doc.detail(width, config.verbose, file=stdout)
    return 0


def build(config, docs, inv, **kwargs):
    if inv is None:
        inv = tldp.inventory.Inventory(config.pubdir, config.sourcedir)
    if not docs:
        docs = inv.work.values()
    for source in docs:
        if not source.output:
            dirname = os.path.join(config.pubdir, source.stem)
            source.output = tldp.outputs.OutputDirectory(dirname)
        if source.stem in config.skip:
            logger.info("%s skipping build per request", source.stem)
            continue
        if not source.doctype:
            logger.warning("%s skipping document of unknown doctype",
                           source.stem)
            continue
        output = source.output
        runner = source.doctype(source=source, output=output, config=config)
        runner.generate()
    return 0


# def script(config, docs, inv, **kwargs):
#     return 0


def getDocumentNames(args):
{'fraction_non_alphanumeric': 0.12078469520103761, 'fraction_numerical': 0.004215304798962386, 'mean_word_length': 2.3691971600218458, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '56400', 'n_tokens_mistral': 1980, 'n_tokens_neox': 1872, 'n_words': 538}