<jupyter_start><jupyter_text>Bacillus subtilis Modulome<jupyter_code>from pymodulon.core import IcaData
from pymodulon.io import *
from pymodulon.imodulondb import *
from pymodulon.util import *
import pandas as pd
from os import path
ica_data = load_json_model(path.join('..','data','processed_data','bsu.json.gz'))
pd.set_option('display.max_colwidth', None)

table_issues, tf_issues, missing_gene_links, missing_dois = \
    imodulondb_compatibility(ica_data)

print('--Table Issues--')
display(table_issues)
print('--TF Issues--')
display(tf_issues)
print('--Missing Gene Links--')
display(missing_gene_links.values)
print('--Missing DOIs--')
display(missing_dois.values)<jupyter_output>--Table Issues--<jupyter_text>Add the iModulonDB Table<jupyter_code>ica_data.imodulondb_table  = {
     'organism': 'Bacillus subtilis',
     'dataset': 'Modulome',
     'strain': '168',
     'publication_name': 'Sastry, et al., Forthcoming',
     'publication_link': '',
     'gene_link_db': '<i>Subti</i>Wiki',
     'organism_folder': 'b_subtilis',
     'dataset_folder': 'modulome'
}<jupyter_output><empty_output><jupyter_text>Add the Explained Variance<jupyter_code>for k in ica_data.imodulon_table.index:
    ica_data.imodulon_table.loc[k, 'exp_var'] = explained_variance(
        ica_data, imodulons=k)<jupyter_output><empty_output><jupyter_text>Rename some iModulon columns<jupyter_code>ica_data.imodulon_table = ica_data.imodulon_table.rename({
    'category':'broad_category',
    'function':'category'
    }, axis = 1)<jupyter_output><empty_output><jupyter_text>Links<jupyter_code>prefix = 'http://subtiwiki.uni-goettingen.de/v3/gene/search/exact/'

# add gene links
for g in ica_data.gene_table.index:
    ica_data.gene_links[g] = prefix + g

# add regulator links
for r in tf_issues.index[(tf_issues.has_gene.astype(bool))]:
    ica_data.tf_links[r] = prefix + ica_data.name2num(r)
    
# some regulators don't have genes but do have pages
# here, I link to the direct page since I had to check their existence anyway
other_regs = {
    'SwrA':'http://subtiwiki.uni-goettingen.de/v3/gene/view/5D479874B43F521DB52EDC2C27CDE4967F22DE47',
    'SigK':'http://subtiwiki.uni-goettingen.de/v3/gene/view/24F7FD5C7C3A68BB2760ABB8CBD8FBD65E5FF7D4',
    'YlxR':'http://subtiwiki.uni-goettingen.de/v3/gene/view/F4097349A563503468A2A14F062AEAC532C7917A',
    'LnrK':'http://subtiwiki.uni-goettingen.de/v3/gene/view/387EF370CE24F7A3C20789A57329A02EBED46F53',
    'KipR':'http://subtiwiki.uni-goettingen.de/v3/gene/view/7DA9A79876C546B78B716A64706A3A3716018C2E'
}
for k, v in other_regs.items():
    ica_data.tf_links[k] = v<jupyter_output><empty_output><jupyter_text>Sample Table<jupyter_code># add replicate numbers
for name, group in ica_data.sample_table.groupby(['project', 'condition']):
    ica_data.sample_table.loc[group.index, 'replicate'] = range(1, group.shape[0]+1)
    
# make sample names that include replicate number
ica_data.sample_table = ica_data.sample_table.rename({'full_name':'full_condition_name'}, axis = 1)
ica_data.sample_table['name'] = \
    ica_data.sample_table.full_condition_name + ':' + \
    ica_data.sample_table.replicate.astype(int).astype(str)

# important: adjust naming to this new column
ica_data.sample_table = ica_data.sample_table.rename({
    'Sample': 'SRS_Sample',
    'name': 'sample'
    }, axis = 1)
#ica_data.sample_table.to_csv('bsub_sample_table.csv')

# TODO 
# add a DOI column and then replace the existing sample_table<jupyter_output><empty_output><jupyter_text>Double-check Compatibility<jupyter_code>table_issues, tf_issues, missing_gene_links, missing_dois = \
    imodulondb_compatibility(ica_data)

print('--Table Issues--')
display(table_issues)
print('--TF Issues--')
display(tf_issues)
print('--Missing Gene Links--')
display(missing_gene_links.values)
print('--Missing DOIs--')
display(missing_dois.values)<jupyter_output>--Table Issues--<jupyter_text>Save & Export<jupyter_code>save_to_json(ica_data, path.join('..','data','processed_data','bsu_imdb.json.gz'))
categories = [
    'Carbon Metabolism',
    'AA/Nucleotide Metabolism',
    'Misc. Metabolism',
    'Homeostasis',
    'Lifestyles',
    'Cellular Processes',
    'Stress Response',
    'Prophages',
    'Other',
    'Single Gene',
    'Uncharacterized'
]
pd.set_option('mode.chained_assignment',None)
imodulondb_export(ica_data, '../iModulonDB', cat_order = categories)<jupyter_output>Writing main site files...
Done writing main site files. Writing plot files...
Two progress bars will appear below. The second will take significantly longer than the first.
Writing iModulon page files (1/2)
{'fraction_non_alphanumeric': 0.13535791757049892, 'fraction_numerical': 0.0316702819956616, 'mean_word_length': 5.634532374100719, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '409006', 'license': 'MIT', 'n_tokens_mistral': 1861, 'n_tokens_neox': 1720, 'n_words': 276, 'path': '5_characterize_iModulons/5_iModulonDB_setup.ipynb', 'repo_name': 'kevin-rychel/modulome-workflow', 'size': '4610'}