<jupyter_start><jupyter_text>Pairwise disorder comparison between effectors and reference proteomes - Disopred<jupyter_code>import json
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import glob
import seaborn as sns
from scipy.stats import mannwhitneyu

import sys
sys.path.append('../src/')

import aepathdisorder as aepd

%load_ext autoreload
%autoreload 2

%matplotlib inline
# Load maps
bug_map = aepd.load_map('../data/maps/reference_taxa.json')
CR_map = aepd.load_map('../data/maps/CR_taxa.json')
EHEC_map = aepd.load_map('../data/maps/EHEC_taxa.json')
EPEC_map = aepd.load_map('../data/maps/EPEC_taxa.json')

# Load Disopred results
bug_disopred = glob.glob('../data/disopred_agg-clas/proteomes/*/*.table')
EHEC_disopred = glob.glob('../data/disopred_agg-clas/EHEC_effectors/*.table')
EPEC_disopred = glob.glob('../data/disopred_agg-clas/EPEC_effectors/*.table')
CR_disopred = glob.glob('../data/disopred_agg-clas/CR_effectors/*.table')
#human_cat_short = glob.glob('../data/iupredagg-clas/human_proteome/*short*.table')


#human_df = concatenate_results(human_disopred)
bug_df = aepd.concatenate_results(bug_disopred)
EHEC_df = aepd.concatenate_results(EHEC_disopred)
EPEC_df = aepd.concatenate_results(EPEC_disopred)
CR_df = aepd.concatenate_results(CR_disopred)

effector_types = ['EHEC', 'EPEC', 'CR']
effector_dfs = [EHEC_df, EPEC_df, CR_df]
effector_maps = [EHEC_map, EPEC_map, CR_map]

for df, mapdict in zip(effector_dfs, effector_maps):
    #df.drop(['dataset'], axis=1,  inplace=True)
    df['dataset'] = df['protein_ac'].map(mapdict)
    df['collection_type'] = 'Effector'

for df, effector_type in zip(effector_dfs, effector_types):
    df['effector_type'] = effector_type
# Make bug taxa strings (stored as int)
bug_df['dataset'] = bug_df['dataset'].astype(str)
# Define references as such
bug_df['collection_type'] = 'Reference'

merged_effector_df = pd.concat(effector_dfs)

bug_efftype_map = {}
for k, v in bug_map.items():
    bug_efftype_map[k] = v['type']

bug_efftype_map
effector_taxa  = set(merged_effector_df['dataset'])
reference_taxa = set(bug_df['dataset'])

paired_taxa = effector_taxa & reference_taxa

paired_effectors = merged_effector_df[merged_effector_df['dataset'].isin(paired_taxa)]
paired_bugs = bug_df[bug_df['dataset'].isin(paired_taxa)]
paired_bugs['effector_type'] = paired_bugs['dataset'].map(bug_efftype_map) 
final_df = pd.concat([paired_effectors, paired_bugs], ignore_index=True)
final_df.reset_index(inplace=True)
# Drop effectors from Reference collections
final_df = final_df.sort_values(by='collection_type').drop_duplicates(subset='protein_ac')
len(final_df)
sns.catplot(
    x='effector_type',
    y='disorder_fraction',
    hue='collection_type',
    data=final_df,
    kind='violin',
    cut=0)

plt.savefig('../figures/pairwise_disopred.png',
            dpi=300)
mwu_stat_df = aepd.calc_mannwithney(final_df)
mwu_stat_df.to_csv('../data/disopred_agg-clas/mannwithney-disopred.tsv', sep='\t', index_label='Effector collection')

mwu_stat_df

ks_stat_df = aepd.calc_kolmogorovsmirnov(final_df)
ks_stat_df.to_csv('../data/disopred_agg-clas/kolmogorovsmirnov-disopred.tsv', sep='\t', index_label='Effector collection')
ks_stat_df
sns.catplot(
    x='effector_type',
    y='disorder_fraction',
    data=final_df[final_df['collection_type']=='Reference'],
    kind='violin')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1397341211225997, 'fraction_numerical': 0.0017725258493353029, 'mean_word_length': 6.425438596491228, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '369647', 'license': 'MIT', 'n_tokens_mistral': 1376, 'n_tokens_neox': 1321, 'n_words': 191, 'path': 'notebooks/disorder_pairwise_comparison_disopred.ipynb', 'repo_name': 'miguelarbesu/AE_pathogens', 'size': '3385'}