<commit_before>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import pickle
from os.path import join as pjoin
from indra.tools import assemble_corpus as ac
from indra.tools.gene_network import GeneNetwork

import process_data, process_r3, process_sparser, process_trips
from read_phosphosite import read_phosphosite
from assemble_sif import assemble_sif
from assemble_cx import assemble_cx
from assemble_pysb import assemble_pysb

def build_prior(genes, out_file):
    gn = GeneNetwork(genes, 'korkut')
    stmts = gn.get_statements(filter=False)
    ac.dump_statements(stmts, out_file)
    return stmts

def read_sources():
    trips_stmts = process_trips.read_stmts(process_trips.base_folder)
    sparser_stmts = process_sparser.read_stmts(process_sparser.base_folder)
    r3_stmts = process_r3.read_stmts(process_r3.active_forms_file)
    stmts = trips_stmts + sparser_stmts + r3_stmts
    return stmts

def get_prior_genes(fname):
    """Get the list of prior genes."""
    with open(fname, 'rt') as fh:
        genes = fh.read().strip().split('\n')
        return genes

if __name__ == '__main__':
    outf = 'output/'
    data = process_data.read_data(process_data.data_file)
    data_genes = process_data.get_all_gene_names(data)
    reassemble = False
    if not reassemble:
        stmts = ac.load_statements(pjoin(outf, 'preassembled.pkl'))
    else:
        #prior_stmts = build_prior(data_genes, pjoin(outf, 'prior.pkl'))
        prior_stmts = ac.load_statements(pjoin(outf, 'prior.pkl'))
        prior_stmts = ac.map_grounding(prior_stmts,
                                       save=pjoin(outf, 'gmapped_prior.pkl'))
        reading_stmts = ac.load_statements(pjoin(outf, 'phase3_stmts.pkl'))
        reading_stmts = ac.map_grounding(reading_stmts,
                                    save=pjoin(outf, 'gmapped_reading.pkl'))
        stmts = prior_stmts + reading_stmts

        stmts = ac.filter_grounded_only(stmts)
        stmts = ac.filter_genes_only(stmts, specific_only=False)
        stmts = ac.filter_human_only(stmts)
        stmts = ac.expand_families(stmts)
        stmts = ac.filter_gene_list(stmts, data_genes, 'one')
        stmts = ac.map_sequence(stmts, save=pjoin(outf, 'smapped.pkl'))
        stmts = ac.run_preassembly(stmts, return_toplevel=False,
                                   save=pjoin(outf, 'preassembled.pkl'))
<commit_msg>Integrate extra sources into assembly pipeline<commit_after>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import pickle
from os.path import join as pjoin
from indra.tools import assemble_corpus as ac
from indra.tools.gene_network import GeneNetwork

import process_data, process_r3, process_sparser, process_trips
from read_phosphosite import read_phosphosite
from assemble_sif import assemble_sif
from assemble_cx import assemble_cx
from assemble_pysb import assemble_pysb

def build_prior(genes, out_file):
    gn = GeneNetwork(genes, 'korkut')
    stmts = gn.get_statements(filter=False)
    ac.dump_statements(stmts, out_file)
    return stmts

def read_extra_sources():
    trips_stmts = process_trips.read_stmts(process_trips.base_folder)
    sparser_stmts = process_sparser.read_stmts(process_sparser.base_folder)
    r3_stmts = process_r3.read_stmts(process_r3.active_forms_file)
    stmts = trips_stmts + sparser_stmts + r3_stmts
    return stmts

def get_prior_genes(fname):
    """Get the list of prior genes."""
    with open(fname, 'rt') as fh:
        genes = fh.read().strip().split('\n')
        return genes

if __name__ == '__main__':
    outf = 'output/'
    data = process_data.read_data(process_data.data_file)
    data_genes = process_data.get_all_gene_names(data)
    reassemble = True
    if not reassemble:
        stmts = ac.load_statements(pjoin(outf, 'preassembled.pkl'))
    else:
        #prior_stmts = build_prior(data_genes, pjoin(outf, 'prior.pkl'))
        prior_stmts = ac.load_statements(pjoin(outf, 'prior.pkl'))
        prior_stmts = ac.map_grounding(prior_stmts,
                                       save=pjoin(outf, 'gmapped_prior.pkl'))
        reach_stmts = ac.load_statements(pjoin(outf, 'phase3_stmts.pkl'))
        extra_stmts = ac.load_statements(pjoin(outf, 'extra_stmts.pkl'))
        reading_stmts = reach_stmts + extra_stmts
        reading_stmts = ac.map_grounding(reading_stmts,
                                    save=pjoin(outf, 'gmapped_reading.pkl'))
        stmts = prior_stmts + reading_stmts + extra_stmts

        stmts = ac.filter_grounded_only(stmts)
        stmts = ac.filter_genes_only(stmts, specific_only=False)
        stmts = ac.filter_human_only(stmts)
        stmts = ac.expand_families(stmts)
        stmts = ac.filter_gene_list(stmts, data_genes, 'one')
        stmts = ac.map_sequence(stmts, save=pjoin(outf, 'smapped.pkl'))
        stmts = ac.run_preassembly(stmts, return_toplevel=False,
                                   save=pjoin(outf, 'preassembled.pkl'))

{'fraction_non_alphanumeric': 0.09101772555267874, 'fraction_numerical': 0.0023899621589324837, 'mean_word_length': 3.5365853658536586, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1350951', 'n_tokens_mistral': 1875, 'n_tokens_neox': 1775, 'n_words': 320}