<commit_before>import argparse, common, sys, tests
from features import check_features, get_features


def arguments(argv=sys.argv[1:]):
    parser = argparse.ArgumentParser()

    parser.add_argument(
        'tests', nargs='*', help='The list of tests to run')

    parser.add_argument(
        '--features', '-f', default=[], action='append',
        help='A list of features separated by colons')

    args = parser.parse_args(argv)

    if args.tests:
        all_tests = [(t, getattr(tests, t, None)) for t in args.tests]
        bad_tests = [t for (t, a) in all_tests if a is None]
        if bad_tests:
            raise ValueError('Bad test names: ' + ', '.join(bad_tests))
        all_tests = tuple(a for (t, a) in all_tests)
    else:
        all_tests = tests.__all__

    if args.features:
        features = set(':'.join(args.features).split(':'))
        check_features(features)

    else:
        features = get_features()

    return all_tests, features<commit_msg>Improve help messages from release_test<commit_after>import argparse, common, sys, tests
from features import check_features, get_features, FEATURES


def arguments(argv=sys.argv[1:]):
    parser = argparse.ArgumentParser()

    names = [t.__name__.split('.')[1] for t in tests.__all__]
    names = ', '.join(names)

    parser.add_argument(
        'tests', nargs='*',
        help='The list of tests to run.  Tests are:' + names)

    features = ', '.join(FEATURES)
    parser.add_argument(
        '--features', '-f', default=[], action='append',
        help='A list of features separated by colons.  Features are: ' +
        features)

    args = parser.parse_args(argv)

    if args.tests:
        all_tests = [(t, getattr(tests, t, None)) for t in args.tests]
        bad_tests = [t for (t, a) in all_tests if a is None]
        if bad_tests:
            raise ValueError('Bad test names: ' + ', '.join(bad_tests))
        all_tests = tuple(a for (t, a) in all_tests)
    else:
        all_tests = tests.__all__

    if args.features:
        features = set(':'.join(args.features).split(':'))
        check_features(features)

    else:
        features = get_features()

    return all_tests, features
{'fraction_non_alphanumeric': 0.13035551504102097, 'fraction_numerical': 0.0013673655423883319, 'mean_word_length': 2.752136752136752, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7038167', 'n_tokens_mistral': 689, 'n_tokens_neox': 669, 'n_words': 194}