<jupyter_start><jupyter_text>Comparing Jesse's previous output to new program output<jupyter_code>import random as rand
import pandas as pd
from sklearn.metrics.pairwise import cosine_similarity
import matplotlib.pyplot as plt
import numpy as np
pd.set_option('display.max_columns', None)
def plot_spec(SPECTRA, COLOR):
    plt.vlines(SPECTRA.columns, np.repeat(0, len(SPECTRA.columns)), SPECTRA, colors=COLOR)
def plot_line(v, m):
    plt.hlines(y=0.7,xmin=0,xmax=m,colors='red')
jesse_df = pd.read_csv('Data/MSPLIToutput_cantRead.txt', sep='\t')
caleb_df = pd.read_csv('Data/fullOutput_lib31_match3_ppm10_sqrt.csv')

var1 = '#shared'
var2 = 'shared'
#var1 = 'cosine'
#var2 = 'cosine'


jesse_df['ID'] = list(zip(jesse_df['Scan#'].tolist(),jesse_df['Peptide'].tolist()))
jesse_df = jesse_df.loc[:, jesse_df.columns.intersection(['ID',var1])]
jesse_dict = jesse_df.set_index('ID')[var1].to_dict()

caleb_df['ID'] = list(zip(caleb_df['scan'].tolist(),caleb_df['peptide'].tolist()))
caleb_df = caleb_df[caleb_df['shared'] >= 15].reset_index(drop=True)

caleb_df = caleb_df.loc[:, caleb_df.columns.intersection(['ID',var2])]
caleb_df_high = caleb_df.drop(caleb_df[caleb_df[var2] < 0.7].index) 
caleb_df_low = caleb_df.drop(caleb_df[caleb_df[var2] >= 0.7].index) 

caleb_dict_high = caleb_df_high.set_index('ID')[var2].to_dict()
caleb_dict_low = caleb_df_low.set_index('ID')[var2].to_dict()
len(jesse_dict.keys())
len(caleb_dict_high.keys())
len(caleb_dict_low.keys())<jupyter_output><empty_output><jupyter_text>Intersect between Jesse's data and Caleb's data (0.7 and higher)<jupyter_code>intersection = sorted(list(set(jesse_dict.keys()) & set(caleb_dict_high.keys())))
len(intersection)
jesse_intersect = pd.DataFrame([[jesse_dict[k] for k in intersection]])
caleb_intersect = pd.DataFrame([[caleb_dict_high[k] for k in intersection]])
diff_intersect = pd.DataFrame([[abs(caleb_dict_high[k]-jesse_dict[k]) for k in intersection]])
compare = pd.DataFrame([[caleb_dict_high[k] for k in intersection],[jesse_dict[k] for k in intersection],intersection])
compare
plot_spec(caleb_intersect, 'blue')
plot_spec(jesse_intersect, 'green')
plot_spec(-diff_intersect, 'red')
plot_line(0.7,len(intersection))<jupyter_output><empty_output><jupyter_text>first 150 (to improve visuals)<jupyter_code>jesse_intersect = jesse_intersect.iloc[:,:150]
caleb_intersect = caleb_intersect.iloc[:,:150]
diff_intersect = diff_intersect.iloc[:,:150]
plot_spec(caleb_intersect, 'blue')
plot_spec(jesse_intersect, 'green')
plot_spec(-diff_intersect, 'red')
plot_line(0.7,150)<jupyter_output><empty_output><jupyter_text>Intersect between Jesse's data and Caleb's data (less than 0.7)<jupyter_code>intersection = sorted(list(set(jesse_dict.keys()) & set(caleb_dict_low.keys())))
len(intersection)
jesse_intersect = pd.DataFrame([[jesse_dict[k] for k in intersection]])
caleb_intersect = pd.DataFrame([[caleb_dict_low[k] for k in intersection]])
diff_intersect = pd.DataFrame([[abs(caleb_dict_low[k]-jesse_dict[k]) for k in intersection]])
compare = pd.DataFrame([[caleb_dict_low[k] for k in intersection],[jesse_dict[k] for k in intersection],intersection])
compare
plot_spec(jesse_intersect, 'green')
plot_spec(caleb_intersect, 'blue')
plot_spec(-diff_intersect, 'red')
plot_line(0.7,len(intersection))<jupyter_output><empty_output><jupyter_text>Difference - Jesse's Data (no match in Caleb's 0.7 and higher data)<jupyter_code>diff = sorted(list(set(jesse_dict.keys()) - set(caleb_dict_high.keys())))
print(len(diff))
jesse_difference = pd.DataFrame([[jesse_dict[k] for k in diff],diff])
jesse_difference
plot_spec(jesse_difference, 'green')
plot_line(0.7,len(diff))<jupyter_output><empty_output><jupyter_text>first 150 (to improve visuals)<jupyter_code>jesse_diff_150 = jesse_difference.iloc[:,:150]
plot_spec(jesse_diff_150, 'green')
plot_line(0.7,150)<jupyter_output><empty_output><jupyter_text>Difference - Caleb's 0.7 or higher Data (no match in Jesse's)<jupyter_code>diff = sorted(list(set(caleb_dict_high.keys()) - set(jesse_dict.keys())))
print(len(diff))
caleb_difference = pd.DataFrame([[caleb_dict_high[k] for k in diff],diff])
caleb_difference
plot_spec(caleb_difference, 'blue')
plot_line(0.7,len(diff))<jupyter_output><empty_output><jupyter_text>first 150 (to improve visuals)<jupyter_code>caleb_diff_150 = caleb_difference.iloc[:,:150]
plot_spec(caleb_diff_150, 'blue')
plot_line(0.7,150)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1542480690595184, 'fraction_numerical': 0.02112676056338028, 'mean_word_length': 9.95273631840796, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '634352', 'license': 'MIT', 'n_tokens_mistral': 1850, 'n_tokens_neox': 1730, 'n_words': 255, 'path': '.ipynb_checkpoints/Comparing_Jesse_and_Caleb_outputs-checkpoint.ipynb', 'repo_name': 'CCranney/CsoDIAq', 'size': '4402'}