<commit_before>import pickle
import logging
import sys
import traceback

from pymachine.src.lexicon import Lexicon
from pymachine.src.definition_parser import read as read_defs

def dump(argv):
    definitions = read_defs(file(argv[0]), 2, add_indices=True)
    pickle.dump(definitions, open(argv[1], "w"))

def test_static(argv):
    fn = "/home/zseder/Proj/machine/pymachine/src/definitions.dump"
    if len(argv) > 0:
        fn = argv[0]
    definitions = pickle.load(open(fn))
    l = Lexicon()
    try:
        l.add_static(definitions.itervalues())
    except Exception as e:
        print "Exception", e
        print traceback.print_exc(file=sys.stdout)
    l.finalize_static()
    print "\n\n===========\n\n"

    for pn, machines in l.static.iteritems():
        print pn
        for m in machines:
            print m.to_debug_str()
        print

if __name__ == "__main__":
    logging.basicConfig(level=logging.DEBUG, format=
            "%(asctime)s : %(module)s (%(lineno)s) - %(levelname)s - %(message)s")

    command = sys.argv[1]
    if command == "dump":
        dump(sys.argv[2:])
    elif command == "add_static":
        test_static(sys.argv[2:])
    else:
        raise Exception("Unknown command")

<commit_msg>Exception printed to stderr as well. 	modified:   test_static.py<commit_after>import pickle
import logging
import sys
import traceback

from pymachine.src.lexicon import Lexicon
from pymachine.src.definition_parser import read as read_defs

def dump(argv):
    definitions = read_defs(file(argv[0]), 2, add_indices=True)
    pickle.dump(definitions, open(argv[1], "w"))

def test_static(argv):
    fn = "/home/zseder/Proj/machine/pymachine/src/definitions.dump"
    if len(argv) > 0:
        fn = argv[0]
    definitions = pickle.load(open(fn))
    l = Lexicon()
    try:
        l.add_static(definitions.itervalues())
    except Exception as e:
        print "Exception", e
        print traceback.print_exc(file=sys.stdout)
        print traceback.print_exc(file=sys.stderr)
    l.finalize_static()
    print "\n\n===========\n\n"

    for pn, machines in l.static.iteritems():
        print pn
        for m in machines:
            print m.to_debug_str()
        print

if __name__ == "__main__":
    logging.basicConfig(level=logging.DEBUG, format=
            "%(asctime)s : %(module)s (%(lineno)s) - %(levelname)s - %(message)s")

    command = sys.argv[1]
    if command == "dump":
        dump(sys.argv[2:])
    elif command == "add_static":
        test_static(sys.argv[2:])
    else:
        raise Exception("Unknown command")


{'fraction_non_alphanumeric': 0.13664596273291926, 'fraction_numerical': 0.006211180124223602, 'mean_word_length': 2.9585253456221197, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6300297', 'n_tokens_mistral': 876, 'n_tokens_neox': 872, 'n_words': 193}