<commit_before>#!/usr/bin/env python

import argparse
import logging
import sys

import rt

class Parser(argparse.ArgumentParser):

    def error(self, message):
        self.print_help()
        sys.stderr.write('\n{}\n'.format(message))
        sys.exit(2)

def main():

    parser = Parser(description='Real-time adaptive translation with cdec.  (See README.md)')
    parser.add_argument('-c', '--config', required=True, help='Config directory')
    parser.add_argument('-s', '--state', help='Load state file (saved incremental data)')
    parser.add_argument('-n', '--normalize', help='Normalize text (tokenize, translate, detokenize)', action='store_true')
    parser.add_argument('-T', '--temp', help='Temp directory (default /tmp)', default='/tmp')
    parser.add_argument('-a', '--cache', help='Grammar cache size (default 5)', default='5')
    parser.add_argument('-v', '--verbose', help='Info to stderr', action='store_true')
    args = parser.parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.INFO)

    with rt.RealtimeDecoder(args.config, tmpdir=args.temp, cache_size=int(args.cache), norm=args.normalize) as rtd:

        try:
            # Load state if given
            if args.state:
                rtd.load_state(args.state)
            # Read lines and commands
            while True:
                line = sys.stdin.readline()
                if not line:
                    break
                line = line.strip()
                if '|||' in line:
                    rtd.command_line(line)
                else:
                    hyp = rtd.decode(line)
                    sys.stdout.write('{}\n'.format(hyp))
                    sys.stdout.flush()
    
        # Clean exit on ctrl+c
        except KeyboardInterrupt:
            logging.info('Caught KeyboardInterrupt, exiting')

if __name__ == '__main__':
    main()<commit_msg>Open stream to state file<commit_after>#!/usr/bin/env python

import argparse
import logging
import sys

import rt

class Parser(argparse.ArgumentParser):

    def error(self, message):
        self.print_help()
        sys.stderr.write('\n{}\n'.format(message))
        sys.exit(2)

def main():

    parser = Parser(description='Real-time adaptive translation with cdec.  (See README.md)')
    parser.add_argument('-c', '--config', required=True, help='Config directory')
    parser.add_argument('-s', '--state', help='Load state file (saved incremental data)')
    parser.add_argument('-n', '--normalize', help='Normalize text (tokenize, translate, detokenize)', action='store_true')
    parser.add_argument('-T', '--temp', help='Temp directory (default /tmp)', default='/tmp')
    parser.add_argument('-a', '--cache', help='Grammar cache size (default 5)', default='5')
    parser.add_argument('-v', '--verbose', help='Info to stderr', action='store_true')
    args = parser.parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.INFO)

    with rt.RealtimeDecoder(args.config, tmpdir=args.temp, cache_size=int(args.cache), norm=args.normalize) as rtd:

        try:
            # Load state if given
            if args.state:
                with open(args.state) as input:
                    rtd.load_state(input)
            # Read lines and commands
            while True:
                line = sys.stdin.readline()
                if not line:
                    break
                line = line.strip()
                if '|||' in line:
                    rtd.command_line(line)
                else:
                    hyp = rtd.decode(line)
                    sys.stdout.write('{}\n'.format(hyp))
                    sys.stdout.flush()
    
        # Clean exit on ctrl+c
        except KeyboardInterrupt:
            logging.info('Caught KeyboardInterrupt, exiting')

if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.13734688558769872, 'fraction_numerical': 0.001563721657544957, 'mean_word_length': 2.255301102629347, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '73709', 'n_tokens_mistral': 1148, 'n_tokens_neox': 1100, 'n_words': 259}