<jupyter_start><jupyter_text>predict clinvar w/ mtr* eval on missesne* must have mtr score<jupyter_code>import pandas, pydot, pydotplus, graphviz
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import linear_model, metrics, tree, svm
from sklearn.neural_network import MLPClassifier
from sklearn.externals.six import StringIO
from IPython.display import HTML
%matplotlib inline
def calc_final_sig(row):
    sig_set = set(str(row['clinSig'].split('|')))
    has_benign = '2' in sig_set or '3' in sig_set
    has_path = '4' in sig_set or '5' in sig_set
    if has_path and not has_benign:
        return 1
    if not has_path and has_benign:
        return 0
    return -1

clin_file = '../data/interim/clinvar/clinvar.dat'
clinvar_df_pre = pandas.read_csv(clin_file, sep='\t').fillna(0)
clinvar_df_pre.loc[:, "y"] = clinvar_df_pre.apply(calc_final_sig, axis=1)
p = clinvar_df_pre[ (clinvar_df_pre.y==1) & (clinvar_df_pre.mtr>0) & (clinvar_df_pre.eff=='missense_variant')]
b = clinvar_df_pre[ (clinvar_df_pre.y==0) & (clinvar_df_pre.mtr>0) & (clinvar_df_pre.eff=='missense_variant')]
print('# clinvar variants with missesense pathogenic and mtr score', len(p))
print('# clinvar variants with missense benign and mtr score', len(b))

p = len(p[['chrom','pos']].drop_duplicates())
b = len(b[['chrom','pos']].drop_duplicates())
print('# clinvar positions with missesnse pathogenic and mtr score', p)
print('# clinvar positions with missense benign and mtr score', b)

clinvar_df = clinvar_df_pre[(clinvar_df_pre.mtr>0)
                            & (clinvar_df_pre.eff=='missense_variant')
                            & (clinvar_df_pre.y!=-1)]
scores = clinvar_df['mtr'].values
truth = clinvar_df['y'].values
fpr_mpc, tpr_mpc, _ = metrics.roc_curve(truth, scores, pos_label=1)
plt.plot(fpr_mpc, tpr_mpc, label='mtr', color='black')
plt.savefig('../docs/plots/clinvar_mis_mtr_roc.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.133298755186722, 'fraction_numerical': 0.008298755186721992, 'mean_word_length': 4.953703703703703, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '77258', 'license': 'MIT', 'n_tokens_mistral': 766, 'n_tokens_neox': 718, 'n_words': 161, 'path': 'notebooks/predict-clinvar-mtr.ipynb', 'repo_name': 'samesense/mahdi_epi', 'size': '1928'}