<jupyter_start><jupyter_text>Plot box-violin plot<jupyter_code>names = ['<NAME>.', 'MatrixREDUCE', 'RNAcontext', 'DeepBind', 'DLPRB', 'RCK', 'cDeepbind', 'ThermoNet', 'ResidualBind']
data = [ray, matrixreduce, rnacontext, deepbind, rck, dlprb, cdeepbind, thermonet, residualbind]

fig = plt.figure(figsize=(12,5))
vplot = plt.violinplot(data, 
                       showextrema=False);

data = [ray, matrixreduce, rnacontext, deepbind, rck, dlprb, cdeepbind, thermonet, residualbind]


import matplotlib.cm as cm
cmap = cm.ScalarMappable(cmap='tab10')
test_mean = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]


    
for patch, color in zip(vplot['bodies'], cmap.to_rgba(test_mean)):
    patch.set_facecolor(color)
    patch.set_edgecolor('black')
    

medianprops = dict(color="red",linewidth=2)
    
bplot = plt.boxplot(data, 
                    notch=True, patch_artist=True, 
                    widths=0.2,
                    medianprops=medianprops);

for patch, color in zip(bplot['boxes'], cmap.to_rgba(test_mean)):
    patch.set_facecolor(color)
    patch.set_edgecolor('black')
    #patch.set(color=colors[i])
plt.xticks(range(1,len(names)+1), names, rotation=40, fontsize=14, ha='right');
ax = plt.gca();
plt.setp(ax.get_yticklabels(),fontsize=14)
plt.ylabel('Pearson correlation', fontsize=14);


plot_path = '../results/rnacompete_2013/'
outfile = os.path.join(plot_path, 'Performance_comparison.pdf')
fig.savefig(outfile, format='pdf', dpi=200, bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>plot comparison between ResidualBind and ThermoNet<jupyter_code>fig = plt.figure(figsize=(3,3))
ax = plt.subplot(111)
plt.hist(residualbind-thermonet, bins=20);

plt.setp(ax.get_yticklabels(),fontsize=14)
plt.ylabel('Counts', fontsize=14);
plt.setp(ax.get_xticklabels(),fontsize=14)
plt.xlabel('$\Delta$ Pearson r', fontsize=14);


plot_path = '../results/rnacompete_2013/'
outfile = os.path.join(plot_path, 'Performance_comparison_hist.pdf')
fig.savefig(outfile, format='pdf', dpi=200, bbox_inches='tight')

from scipy import stats
stats.wilcoxon(residualbind, thermonet)<jupyter_output><empty_output><jupyter_text>Compare performance based on binding score normalization and different input features<jupyter_code>data_path = '../results/rnacompete_2013/clip_norm_seq_performance.tsv'
df = pd.read_csv(data_path, delimiter='\t')
clip_norm_seq = df['Pearson score'].to_numpy()

data_path = '../results/rnacompete_2013/clip_norm_pu_performance.tsv'
df = pd.read_csv(data_path, delimiter='\t')
clip_norm_pu = df['Pearson score'].to_numpy()

data_path = '../results/rnacompete_2013/clip_norm_struct_performance.tsv'
df = pd.read_csv(data_path, delimiter='\t')
clip_norm_struct = df['Pearson score'].to_numpy()

data_path = '../results/rnacompete_2013/log_norm_seq_performance.tsv'
df = pd.read_csv(data_path, delimiter='\t')
log_norm_seq = df['Pearson score'].to_numpy()

data_path = '../results/rnacompete_2013/log_norm_pu_performance.tsv'
df = pd.read_csv(data_path, delimiter='\t')
log_norm_pu = df['Pearson score'].to_numpy()

data_path = '../results/rnacompete_2013/log_norm_struct_performance.tsv'
df = pd.read_csv(data_path, delimiter='\t')
log_norm_struct = df['Pearson score'].to_numpy()
names = ['Clip-norm', 'Log-norm']
data = [clip_norm_seq, log_norm_seq]

fig = plt.figure(figsize=(3,3))
vplot = plt.violinplot(data, 
                       showextrema=False);


import matplotlib.cm as cm
cmap = cm.ScalarMappable(cmap='viridis')
test_mean = [0.1,  0.5,  0.9]

   
for patch, color in zip(vplot['bodies'], cmap.to_rgba(test_mean)):
    patch.set_facecolor(color)
    patch.set_edgecolor('black')
    

medianprops = dict(color="red",linewidth=2)
    
bplot = plt.boxplot(data, 
                    notch=True, patch_artist=True, 
                    widths=0.2,
                    medianprops=medianprops);

for patch, color in zip(bplot['boxes'], cmap.to_rgba(test_mean)):
    patch.set_facecolor(color)
    patch.set_edgecolor('black')
    #patch.set(color=colors[i])
plt.xticks(range(1,len(names)+1), names, rotation=40, fontsize=14, ha='right');
ax = plt.gca();
plt.setp(ax.get_yticklabels(),fontsize=14)
plt.ylabel('Pearson correlation', fontsize=14);


plot_path = '../results/rnacompete_2013/'
outfile = os.path.join(plot_path, 'Performance_comparison_clip_vs_log.pdf')
fig.savefig(outfile, format='pdf', dpi=200, bbox_inches='tight')
names = ['Sequence', 'Sequence + PU', 'Sequence + PHIME']
data = [clip_norm_seq, clip_norm_pu, clip_norm_struct]

fig = plt.figure(figsize=(5,5))
vplot = plt.violinplot(data, 
                       showextrema=False);


import matplotlib.cm as cm
cmap = cm.ScalarMappable(cmap='viridis')
test_mean = [0.1,  0.5,  0.9]

   
for patch, color in zip(vplot['bodies'], cmap.to_rgba(test_mean)):
    patch.set_facecolor(color)
    patch.set_edgecolor('black')
    

medianprops = dict(color="red",linewidth=2)
    
bplot = plt.boxplot(data, 
                    notch=True, patch_artist=True, 
                    widths=0.2,
                    medianprops=medianprops);

for patch, color in zip(bplot['boxes'], cmap.to_rgba(test_mean)):
    patch.set_facecolor(color)
    patch.set_edgecolor('black')
    #patch.set(color=colors[i])
plt.xticks(range(1,len(names)+1), names, rotation=40, fontsize=14, ha='right');
ax = plt.gca();
plt.setp(ax.get_yticklabels(),fontsize=14)
plt.ylabel('Pearson correlation', fontsize=14);


plot_path = '../results/rnacompete_2013/'
outfile = os.path.join(plot_path, 'Performance_comparison_clip_structure.pdf')
fig.savefig(outfile, format='pdf', dpi=200, bbox_inches='tight')
names = ['Sequence', 'Sequence + PU', 'Sequence + PHIME']
data = [log_norm_seq, log_norm_pu, log_norm_struct]

fig = plt.figure(figsize=(5,3))
vplot = plt.violinplot(data, 
                       showextrema=False);


import matplotlib.cm as cm
cmap = cm.ScalarMappable(cmap='viridis')
test_mean = [0.1,  0.5,  0.9]

   
for patch, color in zip(vplot['bodies'], cmap.to_rgba(test_mean)):
    patch.set_facecolor(color)
    patch.set_edgecolor('black')
    

medianprops = dict(color="red",linewidth=2)
    
bplot = plt.boxplot(data, 
                    notch=True, patch_artist=True, 
                    widths=0.2,
                    medianprops=medianprops);

for patch, color in zip(bplot['boxes'], cmap.to_rgba(test_mean)):
    patch.set_facecolor(color)
    patch.set_edgecolor('black')
    #patch.set(color=colors[i])
plt.xticks(range(1,len(names)+1), names, rotation=40, fontsize=14, ha='right');
ax = plt.gca();
plt.setp(ax.get_yticklabels(),fontsize=14)
plt.ylabel('Pearson correlation', fontsize=14);


plot_path = '../results/rnacompete_2013/'
outfile = os.path.join(plot_path, 'Performance_comparison_log_structure.pdf')
fig.savefig(outfile, format='pdf', dpi=200, bbox_inches='tight')
data = [clip_norm_seq, clip_norm_pu, clip_norm_struct, log_norm_seq, log_norm_pu, log_norm_struct]
name = ['clip_norm_seq', 'clip_norm_pu', 'clip_norm_struct', 'log_norm_seq', 'log_norm_pu', 'log_norm_struct']

for n,x in zip(name, data):
    print(n, np.mean(x), np.std(x))<jupyter_output>clip_norm_seq 0.688619262295082 0.16975689898179339
clip_norm_pu 0.690047131147541 0.17045535408919055
clip_norm_struct 0.6886290983606558 0.17097206323661115
log_norm_seq 0.6862725609756097 0.17848520377502483
log_norm_pu 0.6837483606557376 0.18308362249306023
log_norm_struct 0.6827598360655739 0.1832164039652027<jupyter_text>compare PHIME vs seq only<jupyter_code>fig = plt.figure(figsize=(3,3))
ax = plt.subplot(111)
plt.hist(clip_norm_seq-clip_norm_struct, bins=15)

plt.setp(ax.get_yticklabels(),fontsize=14)
plt.ylabel('Counts', fontsize=14);
plt.setp(ax.get_xticklabels(),fontsize=14)
plt.xlabel('$\Delta$ Pearson r', fontsize=14);


plot_path = '../results/rnacompete_2013/'
outfile = os.path.join(plot_path, 'Performance_comparison_hist_seq_vs_struct.pdf')
fig.savefig(outfile, format='pdf', dpi=200, bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>2009 RNAcompete analysis<jupyter_code>data_path = '../results/rnacompete_2009/log_norm_seq_performance.tsv'
df = pd.read_csv(data_path, delimiter='\t')
log_norm_seq = df['Pearson score'].to_numpy()

data_path = '../results/rnacompete_2009/log_norm_pu_performance.tsv'
df = pd.read_csv(data_path, delimiter='\t')
log_norm_pu = df['Pearson score'].to_numpy()

names = ['Sequence', 'Sequence + PU']
data = [log_norm_seq, log_norm_pu]

fig = plt.figure(figsize=(5,5))
vplot = plt.violinplot(data, 
                       showextrema=False);


import matplotlib.cm as cm
cmap = cm.ScalarMappable(cmap='viridis')
test_mean = [0.1,  0.5,  0.9]

   
for patch, color in zip(vplot['bodies'], cmap.to_rgba(test_mean)):
    patch.set_facecolor(color)
    patch.set_edgecolor('black')
    

medianprops = dict(color="red",linewidth=2)
    
bplot = plt.boxplot(data, 
                    notch=True, patch_artist=True, 
                    widths=0.2,
                    medianprops=medianprops);

for patch, color in zip(bplot['boxes'], cmap.to_rgba(test_mean)):
    patch.set_facecolor(color)
    patch.set_edgecolor('black')
    #patch.set(color=colors[i])
plt.xticks(range(1,len(names)+1), names, rotation=40, fontsize=14, ha='right');
ax = plt.gca();
plt.setp(ax.get_yticklabels(),fontsize=14)
plt.ylabel('Pearson correlation', fontsize=14);


plot_path = '../results/rnacompete_2013/'
outfile = os.path.join(plot_path, 'Performance_comparison_log_structure_2009.pdf')
fig.savefig(outfile, format='pdf', dpi=200, bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Compare log vs clip as a scatter plot<jupyter_code>data_path = '../data/RNAcompete_2013/rnacompete2013.h5'
results_path = helper.make_directory('../results', 'rnacompete_2013')

experiment = 'RNCMPT00169'
rbp_index = helper.find_experiment_index(data_path, experiment)

normalization = 'clip_norm'   # 'log_norm' or 'clip_norm'
ss_type = 'seq'                  # 'seq', 'pu', or 'struct'
save_path = helper.make_directory(results_path, normalization+'_'+ss_type)

# load rbp dataset
train, valid, test = helper.load_rnacompete_data(data_path, 
                                                 ss_type=ss_type, 
                                                 normalization=normalization, 
                                                 rbp_index=rbp_index)

# load residualbind model
input_shape = list(train['inputs'].shape)[1:]
weights_path = os.path.join(save_path, experiment + '_weights.hdf5')    
model = ResidualBind(input_shape, weights_path)

# load pretrained weights
model.load_weights()

# get predictions for test sequences
predictions_clip = model.predict(test['inputs'])
y = test['targets']


fig = plt.figure(figsize=(3,3))
plt.scatter(predictions_clip, y, alpha=0.5, rasterized=True)
plt.plot([-2,9],[-2,9],'--k')
plt.xlabel('Predicted binding scores', fontsize=14)
plt.ylabel('Experimental binding scores', fontsize=14)
plt.xticks([-2, 0, 2, 4, 6, 8], fontsize=14)
plt.yticks([-2, 0, 2, 4, 6, 8], fontsize=14)
outfile = os.path.join(results_path, experiment+'_scatter_clip.pdf')
fig.savefig(outfile, format='pdf', dpi=600, bbox_inches='tight')

normalization = 'log_norm'   # 'log_norm' or 'clip_norm'
ss_type = 'seq'                  # 'seq', 'pu', or 'struct'
save_path = helper.make_directory(results_path, normalization+'_'+ss_type)

# load rbp dataset
train, valid, test = helper.load_rnacompete_data(data_path, 
                                                 ss_type=ss_type, 
                                                 normalization=normalization, 
                                                 rbp_index=rbp_index)

# load residualbind model
input_shape = list(train['inputs'].shape)[1:]
weights_path = os.path.join(save_path, experiment + '_weights.hdf5')    
model = ResidualBind(input_shape, weights_path)

# load pretrained weights
model.load_weights()

# get predictions for test sequences
predictions_log = model.predict(test['inputs'])

y2 = test['targets']


fig = plt.figure(figsize=(3,3))
plt.scatter(predictions_log, y2, alpha=0.5, rasterized=True)
plt.plot([-2,9],[-2,9],'--k')
plt.xlabel('Predicted binding scores', fontsize=14)
plt.ylabel('Experimental binding scores', fontsize=14)
plt.xticks([-2, 0, 2, 4, 6, 8,], fontsize=14)
plt.yticks([-2, 0, 2, 4, 6, 8], fontsize=14)
outfile = os.path.join(results_path, experiment+'_scatter_log.pdf')
fig.savefig(outfile, format='pdf', dpi=600, bbox_inches='tight')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14968178522516717, 'fraction_numerical': 0.042858293724321275, 'mean_word_length': 4.527159394479074, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '11960', 'license': 'MIT', 'n_tokens_mistral': 4996, 'n_tokens_neox': 4567, 'n_words': 729, 'path': 'Figure1_performance_analysis.ipynb', 'repo_name': 'p-koo/residualbind', 'size': '12416'}