<commit_before>#!/usr/bin/python
'''
See help(Pipeline) for details on configuring a Pipeline.

This software is released under an MIT/X11 open source license.

Copyright 2012 Diffeo, Inc.
'''
import os
import sys
from _getch import getch
from _task_queues import ZookeeperTaskQueue

if __name__ == '__main__':
    import yaml
    import argparse
    parser = argparse.ArgumentParser(
        description='',
        usage='python -m kba.pipeline.load config.yaml')
    parser.add_argument(
        'config', metavar='config.yaml', 
        help='configuration parameters for a pipeline run')
    parser.add_argument(
        '--delete-all', action='store_true', default=False,
        dest='delete_all',
        help='Delete all data in the namespace.')
    parser.add_argument(
        '--load', 
        help='Load a file of one task string per line, defaults to stdin.')
    parser.add_argument(
        '--counts', action='store_true', default=False,
        help='Display counts for the three different states.')
    parser.add_argument(
        '--set-completed', action='store_true', default=False,
        dest='set_completed',
        help='Must be used in conjunction with --load, and causes all ' + \
            'loaded tasks to be set to "completed".')
    parser.add_argument(
        '--list-completed', action='store_true', default=False,
        dest='list_completed',
        help='List all completed tasks.')
    parser.add_argument(
        '--cleanup', action='store_true', default=False,
        help='Cleans up "available" and "pending" to match "state" of tasks.')
    parser.add_argument(
        '--terminate', action='store_true', default=False,
        help='End all current task workers.')
    parser.add_argument(
        '--finish', action='store_true', default=False,
        help='Finish the queue and then stop all current task workers.')
    parser.add_argument(
        '--reset-pending', action='store_true', default=False,
        help='Move all tasks from "pending" back to "available".')
    args = parser.parse_args()

    assert os.path.exists(args.config), '%s does not exist' % args.config
    config = yaml.load(open(args.config))
    config = config['kba.pipeline']['zookeeper']
    tq = ZookeeperTaskQueue(config)

    if args.delete_all:
        sys.stdout.write('Are you sure you want to delete everything in %r?  (y/N): ' \
                            % config['namespace'])
        ch = getch()
        if ch.lower() == 'y':
            sys.stdout.write('\nDeleting ...')
            tq.delete_all()
            print('')
        else:
            print(' ... Aborting.')

    elif args.load:
        if args.load == '-':
            args.load = sys.stdin
        else:
            args.load = open(args.load)
        sys.stdout.write('Loading...')
        num = tq.push(*args.load, completed=args.set_completed)
        print(' %d pushed new tasks.' % num)

    if args.terminate:
        tq.set_mode( tq.TERMINATE )

    if args.finish:
        tq.set_mode( tq.FINISH )

    if args.reset_pending:
        tq.reset_pending()

    if args.cleanup:
        tq.cleanup()

    if args.counts:
        counts = tq.counts
        print('\n'.join(['\t%s:\t%s' % (k, v) for k, v in counts.items()]))
        print('Total: %d' % sum([counts[k] for k in ['available', 'pending', 'completed']]))
        print('Num Tasks: %d' % counts['tasks'])

    if args.list_completed:
        for completed in tq.completed:
            print completed['i_str']
<commit_msg>making the "result" from commit(result) visible in the completed tasks diagnostic<commit_after>#!/usr/bin/python
'''
See help(Pipeline) for details on configuring a Pipeline.

This software is released under an MIT/X11 open source license.

Copyright 2012 Diffeo, Inc.
'''
import os
import sys
from _getch import getch
from _task_queues import ZookeeperTaskQueue

if __name__ == '__main__':
    import yaml
    import argparse
    parser = argparse.ArgumentParser(
        description='',
        usage='python -m kba.pipeline.load config.yaml')
    parser.add_argument(
        'config', metavar='config.yaml', 
        help='configuration parameters for a pipeline run')
    parser.add_argument(
        '--delete-all', action='store_true', default=False,
        dest='delete_all',
        help='Delete all data in the namespace.')
    parser.add_argument(
        '--load', 
        help='Load a file of one task string per line, defaults to stdin.')
    parser.add_argument(
        '--counts', action='store_true', default=False,
        help='Display counts for the three different states.')
    parser.add_argument(
        '--set-completed', action='store_true', default=False,
        dest='set_completed',
        help='Must be used in conjunction with --load, and causes all ' + \
            'loaded tasks to be set to "completed".')
    parser.add_argument(
        '--list-completed', action='store_true', default=False,
        dest='list_completed',
        help='List all completed tasks.')
    parser.add_argument(
        '--cleanup', action='store_true', default=False,
        help='Cleans up "available" and "pending" to match "state" of tasks.')
    parser.add_argument(
        '--terminate', action='store_true', default=False,
        help='End all current task workers.')
    parser.add_argument(
        '--finish', action='store_true', default=False,
        help='Finish the queue and then stop all current task workers.')
    parser.add_argument(
        '--reset-pending', action='store_true', default=False,
        help='Move all tasks from "pending" back to "available".')
    args = parser.parse_args()

    assert os.path.exists(args.config), '%s does not exist' % args.config
    config = yaml.load(open(args.config))
    config = config['kba.pipeline']['zookeeper']
    tq = ZookeeperTaskQueue(config)

    if args.delete_all:
        sys.stdout.write('Are you sure you want to delete everything in %r?  (y/N): ' \
                            % config['namespace'])
        ch = getch()
        if ch.lower() == 'y':
            sys.stdout.write('\nDeleting ...')
            tq.delete_all()
            print('')
        else:
            print(' ... Aborting.')

    elif args.load:
        if args.load == '-':
            args.load = sys.stdin
        else:
            args.load = open(args.load)
        sys.stdout.write('Loading...')
        num = tq.push(*args.load, completed=args.set_completed)
        print(' %d pushed new tasks.' % num)

    if args.terminate:
        tq.set_mode( tq.TERMINATE )

    if args.finish:
        tq.set_mode( tq.FINISH )

    if args.reset_pending:
        tq.reset_pending()

    if args.cleanup:
        tq.cleanup()

    if args.counts:
        counts = tq.counts
        print('\n'.join(['\t%s:\t%s' % (k, v) for k, v in counts.items()]))
        print('Total: %d' % sum([counts[k] for k in ['available', 'pending', 'completed']]))
        print('Num Tasks: %d' % counts['tasks'])

    if args.list_completed:
        for completed in tq.completed:
            print 'in:  %s\nout: %s' % (completed['i_str'], completed['result'])

{'fraction_non_alphanumeric': 0.13605728727885424, 'fraction_numerical': 0.0016849199663016006, 'mean_word_length': 2.47802734375, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3677956', 'n_tokens_mistral': 2172, 'n_tokens_neox': 2118, 'n_words': 574}