<commit_before>import functools
import sys
from concurrent.futures import ProcessPoolExecutor, as_completed
from os import makedirs, getcwd

from cs251tk.common import chdir
from cs251tk.toolkit import process_student
from cs251tk.toolkit import process_args
from cs251tk.toolkit import progress_bar
from cs251tk.toolkit import save_recordings, gist_recordings
from cs251tk.toolkit import tabulate
from cs251tk.specs import load_all_specs


def make_progress_bar(students, no_progress=False):
    if no_progress:
        return lambda _: None

    size = len(students)
    remaining = set(students)
    invocation_count = 0

    def increment(username):
        nonlocal remaining
        nonlocal invocation_count
        remaining.remove(username)
        invocation_count += 1
        msg = ', '.join(remaining)
        progress_bar(size, invocation_count, message=msg)

    msg = ', '.join(remaining)
    progress_bar(size, invocation_count, message=msg)
    return increment


def main():
    args = process_args()
    basedir = getcwd()
    if update_available():
        print(dedent('v{} is available (you have v{}) - try "pip3 install --no-cache --user cs251tk"'))

    if args['date']:
        print('Checking out {}'.format(args['date']))

    specs = load_all_specs(basedir)
    if not specs:
        print('no specs loaded!')
        sys.exit(1)

    print_progress = make_progress_bar(args['students'])

    results = []
    records = []
    makedirs('./students', exist_ok=True)
    with chdir('./students'):
        single = functools.partial(process_student, args=args, specs=specs, basedir=basedir, debug=args['debug'])

        if args['workers'] > 1 and not args['debug']:
            with ProcessPoolExecutor(max_workers=args['workers']) as pool:
                futures = [pool.submit(single, student) for student in args['students']]
                for future in as_completed(futures):
                    result, recording = future.result()
                    print_progress(result['username'])<commit_msg>fix up message in cli/cs251tk<commit_after>import functools
import sys
from concurrent.futures import ProcessPoolExecutor, as_completed
from os import makedirs, getcwd

from cs251tk.common import chdir
from cs251tk.toolkit import update_available
from cs251tk.toolkit import process_student
from cs251tk.toolkit import process_args
from cs251tk.toolkit import progress_bar
from cs251tk.toolkit import save_recordings, gist_recordings
from cs251tk.toolkit import tabulate
from cs251tk.specs import load_all_specs


def make_progress_bar(students, no_progress=False):
    if no_progress:
        return lambda _: None

    size = len(students)
    remaining = set(students)
    invocation_count = 0

    def increment(username):
        nonlocal remaining
        nonlocal invocation_count
        remaining.remove(username)
        invocation_count += 1
        msg = ', '.join(remaining)
        progress_bar(size, invocation_count, message=msg)

    msg = ', '.join(remaining)
    progress_bar(size, invocation_count, message=msg)
    return increment


def main():
    args = process_args()
    basedir = getcwd()
    if update_available():
        print('v{} is available: you have v{}. Try "pip3 install --no-cache --user cs251tk" to update.')

    if args['date']:
        print('Checking out {}'.format(args['date']))

    specs = load_all_specs(basedir)
    if not specs:
        print('no specs loaded!')
        sys.exit(1)

    print_progress = make_progress_bar(args['students'])

    results = []
    records = []
    makedirs('./students', exist_ok=True)
    with chdir('./students'):
        single = functools.partial(process_student, args=args, specs=specs, basedir=basedir, debug=args['debug'])

        if args['workers'] > 1 and not args['debug']:
            with ProcessPoolExecutor(max_workers=args['workers']) as pool:
                futures = [pool.submit(single, student) for student in args['students']]
                for future in as_completed(futures):
                    result, recording = future.result()
                    print_progress(result['username'])
{'fraction_non_alphanumeric': 0.101308773630635, 'fraction_numerical': 0.015511391177896268, 'mean_word_length': 3.2855659397715473, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1741285', 'n_tokens_mistral': 1278, 'n_tokens_neox': 1193, 'n_words': 311}