<commit_before>import fastavro as avro
import json
from sys import stdout

encoding = stdout.encoding or "UTF-8"

def main(argv=None):
    import sys
    from argparse import ArgumentParser

    argv = argv or sys.argv

    parser = ArgumentParser(
        description='iter over avro file, emit records as JSON')
    parser.add_argument('file', help='file(s) to parse', nargs='+')
    parser.add_argument('--schema', help='dump schema instead of records',
                        action='store_true', default=False)
    parser.add_argument('--version', action='version',
            version=avro.__version__)
    args = parser.parse_args(argv[1:])

    for filename in args.file:
        if filename == '-':
            fo = sys.stdin
        else:
            try:
                fo = open(filename, 'rb')
            except IOError as e:
                raise SystemExit(
                    'error: cannot open {0} - {1}'.format(args.filename, e))

        reader = avro.reader(fo)
        if args.schema:
            json.dump(reader.schema, stdout, indent=4, encoding=encoding)
            sys.stdout.write('\n')
            continue

        try:
            for record in reader:
                json.dump(record, stdout, encoding=encoding)
                sys.stdout.write('\n')
        except (IOError, KeyboardInterrupt):
            pass

if __name__ == '__main__':
    main()<commit_msg>name in version<commit_after>import fastavro as avro
import json
from sys import stdout

encoding = stdout.encoding or "UTF-8"

def main(argv=None):
    import sys
    from argparse import ArgumentParser

    argv = argv or sys.argv

    parser = ArgumentParser(
        description='iter over avro file, emit records as JSON')
    parser.add_argument('file', help='file(s) to parse', nargs='+')
    parser.add_argument('--schema', help='dump schema instead of records',
                        action='store_true', default=False)
    parser.add_argument('--version', action='version',
            version='fastavro {0}'.format(avro.__version__))
    args = parser.parse_args(argv[1:])

    for filename in args.file:
        if filename == '-':
            fo = sys.stdin
        else:
            try:
                fo = open(filename, 'rb')
            except IOError as e:
                raise SystemExit(
                    'error: cannot open {0} - {1}'.format(args.filename, e))

        reader = avro.reader(fo)
        if args.schema:
            json.dump(reader.schema, stdout, indent=4, encoding=encoding)
            sys.stdout.write('\n')
            continue

        try:
            for record in reader:
                json.dump(record, stdout, encoding=encoding)
                sys.stdout.write('\n')
        except (IOError, KeyboardInterrupt):
            pass

if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.11354793066855323, 'fraction_numerical': 0.0038910505836575876, 'mean_word_length': 2.0245989304812833, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1834868', 'n_tokens_mistral': 801, 'n_tokens_neox': 782, 'n_words': 216}