<jupyter_start><jupyter_text>Data Pre-processing and reformatIntegrating My Data0. After checking the given data, filter the correlations. 1. Exchanging gene's name and ID.<jupyter_code>myData = pd.merge(myData,geneName,how = 'left',left_on=['geneid'],right_on=['loadedanno.gene_id'])
myData
myData1 =  pd.merge(myData1,geneName,how = 'left',left_on=['geneid'],right_on=['loadedanno.gene_id'])
myData1
relation_M1 = myData1[['loadedanno.gene_name','peakname','q_value']]
relation_M = myData[['loadedanno.gene_name','peakname','q_value']]
relation_G = givenData[['Linked_Gene','Peak_Name','Raw_FDR','Raw_Correlation']]

comparing = pd.merge(relation_G,relation_M,how='inner',left_on=['Linked_Gene'],right_on=['loadedanno.gene_name'])
comparing
comparing1 = comparing[comparing['Peak_Name']==comparing['peakname']]
del comparing1['loadedanno.gene_name']
del comparing1['peakname']
comparing1 = pd.merge(comparing1,peakAnno,how='left',left_on='Peak_Name',right_on='name')
del comparing1['Peak_Name']
del comparing1['end']
comparing1 = pd.merge(comparing1,geneAnno,how='left',left_on='Linked_Gene',right_on='loadedanno.gene_name')
del comparing1['loadedanno.gene_name']

del comparing1['loadedanno.type']
del comparing1['width']
del comparing1['exonlen']
del comparing1['chr']
del comparing1['q_value']
distence = comparing1['start_x']-comparing1['start_y']
comparing1['dist'] = distence
comparing1.to_csv('final_result1.csv',index=0)
see = comparing1['Linked_Gene'].value_counts()
see

one =  see.index 
geneidfre = pd.DataFrame({'geneid':one,'times':see})
geneidfre.to_csv('gene_times.txt',sep='\t',index=0)
comparing2 = pd.merge(relation_G,relation_M1,how='inner',left_on=['Linked_Gene'],right_on=['loadedanno.gene_name'])
comparing2 = comparing2[comparing2['Peak_Name']==comparing2['peakname']]
comparing2
comparing3 = pd.merge(relation_M,relation_M1,how='inner',on=['loadedanno.gene_name'])
comparing3 = comparing3[comparing3['peakname_x']==comparing3['peakname_y']]
comparing3
relation_M
import  numpy as np
meanGiven = np.mean(givenData['Raw_FDR'])
print(str(meanGiven))
meanRemain = np.mean(comparing1['Raw_FDR'])
print(str(meanRemain))
import matplotlib.pyplot as plt
distence.value_counts()
from scipy import stats
import numpy as np
x = np.linspace(-3,3,100000)
y = stats.norm.pdf(x,0,1)
z = stats.t.pdf(x,9999)
plt.plot(x,y,c="red")
plt.plot(x,z,c="blue")
plt.legend(['norm','t'])
plt.title('PDF of norm&t distribution')


plt.savefig('normT.jpg')
plt.show()
teatPearson = pd.read_csv('test_pearson_onBRCA_11.csv',header=0)
relation_G = givenData[['Linked_Gene','Peak_Name','Raw_FDR','Raw_Correlation']]
myData2 =  pd.merge(teatPearson,geneName,how = 'left',left_on=['geneid'],right_on=['loadedanno.gene_id'])
comparing22 = pd.merge(relation_G,myData2,how='inner',left_on=['Linked_Gene'],right_on=['loadedanno.gene_name'])

comparing22 = comparing22[comparing22['Peak_Name']==comparing22['peakname']]
comparing22

teatPearson.sort_values(by='p',ascending=True,inplace=True)
teatPearson.reset_index(inplace=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17251557887832075, 'fraction_numerical': 0.02492620531321745, 'mean_word_length': 11.815126050420169, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '155361', 'license': 'MIT', 'n_tokens_mistral': 1212, 'n_tokens_neox': 1140, 'n_words': 138, 'path': 'correlation research/Assesment.ipynb', 'repo_name': 'SYNALEAN/Mycode', 'size': '3049'}