<jupyter_start><jupyter_text>A notebook to demonstrate the use of the analysis functions for lda dictionaries<jupyter_code>original_dict_file = '/Users/simon/Dropbox/BioResearch/Meta_clustering/KRD/mzml sylvia/molnet130918/carnegie_lda.dict'<jupyter_output><empty_output><jupyter_text>Load the dictionary<jupyter_code>import pickle
with open(original_dict_file,'r') as f:
    lda_dict = pickle.loads(f.read())
print lda_dict.keys()<jupyter_output>['phi', 'features', 'doc_metadata', 'alpha', 'K', 'word_index', 'topic_index', 'beta', 'topic_metadata', 'theta', 'corpus', 'overlap_scores', 'doc_index', 'gamma']<jupyter_text>User can set some parameters here<jupyter_code>overlap_thresh = 0.3
probability_thresh = 0.1<jupyter_output><empty_output><jupyter_text>Compute motif degrees<jupyter_code>%load_ext autoreload
%autoreload 2
from lda_analysis_functions import compute_motif_degrees
motif_degree_dict,motif_degree_list = compute_motif_degrees(lda_dict,probability_thresh,overlap_thresh)<jupyter_output><empty_output><jupyter_text>print the top 10 motifs by degree<jupyter_code>for m,d in motif_degree_list[:10]:
    print m,d<jupyter_output>motif_283 1132
motif_200 411
motif_297 118
motif_71 97
motif_144 89
motif_205 89
motif_241 83
motif_87 79
motif_233 76
motif_236 73<jupyter_text>Print the degree of any motif<jupyter_code>motif = 'motif_236'
print motif_degree_dict[motif]<jupyter_output>73<jupyter_text>Plot a motif<jupyter_code>from lda_analysis_functions import plot_motif
import pylab as plt
%matplotlib inline
# plot_motif(lda_dict,'motif_22')
plot_motif(lda_dict,'motif_106',figsize=(20,10))
plot_motif(lda_dict,'motif_125',figsize=(20,10))<jupyter_output><empty_output><jupyter_text>List the available document metadata fields<jupyter_code>from lda_analysis_functions import list_metadata_fields
mdf = list_metadata_fields(lda_dict)
print mdf
from lda_analysis_functions import print_mols
print_mols(lda_dict,['1'])<jupyter_output>Spectra name        	precursormass       	parentrt            	
1                   	679.510192871       	41.2024<jupyter_text>Print all molecules in a particular motif<jupyter_code>motif = 'motif_40'
from lda_analysis_functions import get_motif_mols
mols = get_motif_mols(lda_dict,motif,probability_thresh,overlap_thresh)
print_mols(lda_dict,
           mols,
           fields = ['precursormass','parentrt','scanno'])<jupyter_output>Spectra name        	precursormass       	parentrt            	scanno              	
5499                	890.594482422       	1082.4358           	1802                	
5741                	809.169128418       	689.9597            	1163                	
5301                	921.750915527       	1760.4542           	2934                	
6021                	950.711425781       	1675.0987           	2811<jupyter_text>Plot a document<jupyter_code>mol = mols[3]
from lda_analysis_functions import plot_mol
plot_mol(lda_dict,mol,color_motifs = True,figsize=(20,10))
# The optional xlim parameter allows us to zoom in
plot_mol(lda_dict,mol,color_motifs = True,xlim = [255,260],figsize=(20,10))<jupyter_output><empty_output><jupyter_text>Now for some motif matchingFirstly, load motifdb<jupyter_code>motifdb_path = '/Users/simon/git/motifdb/'
import sys,os
sys.path.append(os.path.join(motifdb_path,'code','utilities'))
from motifdb_loader import load_db,MotifFilter
dbs_to_load = ['massbank_binned_005','gnps_binned_005']
db_spectra,db_metadata = load_db(dbs_to_load,motifdb_path+'motifs')
mf = MotifFilter(db_spectra,db_metadata)
db_spectra,db_metadata = mf.filter()
from lda_analysis_functions import match_motifs
matches = match_motifs(lda_dict,db_spectra,threshold = 0.00)

match_idx = 11
plot_motif(lda_dict,matches[match_idx][0],xlim=[0,500],figsize=(10,5))
from lda_analysis_functions import plot_motif_from_dict
plot_motif_from_dict(db_spectra[matches[match_idx][1]],xlim=[0,500],figsize=(10,5))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09394945111054379, 'fraction_numerical': 0.06714322185345928, 'mean_word_length': 5.497512437810945, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '16476', 'license': 'MIT', 'n_tokens_mistral': 1629, 'n_tokens_neox': 1444, 'n_words': 228, 'path': 'lda/offline_analysis/ExampleAnalysis.ipynb', 'repo_name': 'RP0001/ms2ldaviz', 'size': '3917'}