<commit_before>import pickle
from matplotlib import pyplot as plt
import numpy as np
from indra.tools import plot_formatting as pf
from collections import Counter
from indra.statements import *
from indra.preassembler import Preassembler
from indra.preassembler.hierarchy_manager import hierarchies
from copy import deepcopy
pf.set_fig_params()

print "Loading REACH results"
with open('reach_stmts.pkl') as f:
    results = pickle.load(f)

counts_per_paper = [(pmid, len(stmts)) for pmid, stmts in results.items()]
zero_pmids = [pmid for pmid, stmts in results.items() if len(stmts) == 0]
counts = np.array([tup[1] for tup in counts_per_paper])

# Preassemble to remove duplicates

# Combining duplicates
pa = Preassembler(hierarchies)
for paper_stmts in results.values():
    pa.add_statements(paper_stmts)
pa.combine_duplicates()<commit_msg>Pass stmts file to reach_results_stats as arg<commit_after>import pickle
from matplotlib import pyplot as plt
import numpy as np
from indra.tools import plot_formatting as pf
from collections import Counter
from indra.statements import *
from indra.preassembler import Preassembler
from indra.preassembler.hierarchy_manager import hierarchies
from copy import deepcopy
import sys

pf.set_fig_params()

if len(sys.argv) < 2:
    print "Usage: %s reach_stmts_file" % sys.argv[0]
    sys.exit(1)

stmts_file = sys.argv[1]

print "Loading REACH results"
with open(stmts_file) as f:
    results = pickle.load(f)

counts_per_paper = [(pmid, len(stmts)) for pmid, stmts in results.items()]
zero_pmids = [pmid for pmid, stmts in results.items() if len(stmts) == 0]
counts = np.array([tup[1] for tup in counts_per_paper])

# Preassemble to remove duplicates

# Combining duplicates
pa = Preassembler(hierarchies)
for paper_stmts in results.values():
    pa.add_statements(paper_stmts)
pa.combine_duplicates()
{'fraction_non_alphanumeric': 0.08518719479110147, 'fraction_numerical': 0.004340748779164406, 'mean_word_length': 5.657039711191335, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6602169', 'n_tokens_mistral': 654, 'n_tokens_neox': 642, 'n_words': 190}