<commit_before>import json
import argparse

import righter
from righter.log import logger
from righter.competitor import ginger

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--input-file', help='File with one json per line containing the key text', required=True)
    parser.add_argument('-o', '--file-output', help='Save analysis to output file', required=True)
    parser.add_argument('-a', '--algorithm', help='Name of the algorithm to use')
    args = parser.parse_args()

    if args.algorithm is None:
        check = righter.check
    elif args.algorithm == 'ginger':
        check = ginger.check

    with open(args.input_file, 'r') as input_fp:
        with open(args.file_output, 'w') as output_fp:
            for line in input_fp:
                writing = json.loads(line.strip())
                try:
                    writing['changes'] = check(writing['text'])
                except ginger.SentenceTooBigError:
                    logger.error("Unable to process <%s> on ginger (Sentence too big)", writing['id'])
                else:
                    print(json.dumps(writing), file=output_fp)<commit_msg>Import righter only when needed<commit_after>import json
import argparse

from righter.log import logger

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--input-file', help='File with one json per line containing the key text', required=True)
    parser.add_argument('-o', '--file-output', help='Save analysis to output file', required=True)
    parser.add_argument('-a', '--algorithm', help='Name of the algorithm to use')
    args = parser.parse_args()

    if args.algorithm is None or args.algorithm == 'righter':
        import righter
        check = righter.check
    elif args.algorithm == 'ginger':
        from righter.competitor import ginger
        check = ginger.check

    with open(args.input_file, 'r') as input_fp:
        with open(args.file_output, 'w') as output_fp:
            for line in input_fp:
                writing = json.loads(line.strip())
                try:
                    writing['changes'] = check(writing['text'])
                except ginger.SentenceTooBigError:
                    logger.error("Unable to process <%s> on ginger (Sentence too big)", writing['id'])
                else:
                    print(json.dumps(writing), file=output_fp)
{'fraction_non_alphanumeric': 0.11742892459826947, 'fraction_numerical': 0.0, 'mean_word_length': 2.570588235294118, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4475356', 'n_tokens_mistral': 688, 'n_tokens_neox': 664, 'n_words': 196}