<commit_before>#!/usr/bin/python3

"""Simple python script to 'pretty' json files.

It attempts to make use of the C based simplejson, but will fall back to using
the core json module if simplejson is not available.

"""

import argparse
import os

try:
    import simplejson as json
except ImportError:
    import json


def _parser():
    """Parse arguments and return an argparse namespace."""
    parser = argparse.ArgumentParser()
    parser.add_argument('file', help='A JSON file to clean')
    parser.add_argument('-i', '--indent',
                        type=int,
                        default=4,
                        help='The number of spaces to indent the file by')
    parser.add_argument('-p', '--preserve',
                        action='store_true',
                        help='Save the original file as a <file>.bak')
    return parser.parse_args()


def main():
    """Main function."""
    args = _parser()

    # Read in the contents of the old file as json data
    with open(args.file, 'r') as f:
        content = json.load(f)

    # If requested save the old file with a .bak extension
    if args.preserve:
        os.rename(args.file, args.file + '.bak')

    # Finally dump the updated file
    with open(args.file, 'w') as f:
        json.dump(content, f, indent=args.indent)


if __name__ == '__main__':
    main()<commit_msg>jsontidy.py: Allow reading from stdin and writing to stdout<commit_after>#!/usr/bin/python3

"""Simple python script to 'pretty' json data.

It attempts to make use of the C based simplejson, but will fall back to using
the core json module if simplejson is not available.

This script can read both data from stdin, or from a stored text file.

"""

import argparse
import os
import sys

try:
    import simplejson as json
except ImportError:
    import json


def _parser():
    """Parse arguments and return an argparse namespace."""
    # The use of type, default, and nargs allows the stdin/stdout to work,
    # since one would either provide a file or stdin/out data

    # FIXME: This will not allow stdin/file to be used. Is there an elegent way
    # to solve that?
    parser = argparse.ArgumentParser()
    parser.add_argument('input',
                        type=argparse.FileType('r'),
                        default=sys.stdin,
                        nargs='?',
                        help='JSON data to pretty. Default: stdin')
    parser.add_argument('output',
                        type=argparse.FileType('w'),
                        default=sys.stdout,
                        nargs='?',
                        help='Location to place the updated data. '
                             'Default: stdout')
    parser.add_argument('-i', '--indent',
                        type=int,
                        default=4,
                        help='The number of spaces to indent the file by')
    parser.add_argument('-p', '--preserve',
                        action='store_true',
                        help='Save the original file as a <file>.bak, if '
                             'overwritting. Has no effect if the destination '
                             'is not the same as the input')
    return parser.parse_args()


def main():
    """Main function."""
    args = _parser()

    # Read in the contents of the old data as json data
    content = json.load(args.input)

    # If requested save the old file with a .bak extension
    if (args.preserve and
            os.path.abspath(args.input) == os.path.abspath(args.output)):
        os.rename(args.input, args.input + '.bak')

    # Finally dump the updated data
    json.dump(content, args.output, indent=args.indent)


if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.09921811809112968, 'fraction_numerical': 0.0010784578053383662, 'mean_word_length': 2.018714401952807, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3651272', 'n_tokens_mistral': 1026, 'n_tokens_neox': 987, 'n_words': 374}