<jupyter_start><jupyter_text>Loading h2 results<jupyter_code>h2_neale_result=pd.read_csv(h2_path,sep='\t')
h2_neale_result=h2_neale_result.set_index('phenotype')
h2_neale_sorted=h2_neale_result.sort_values('h2_observed',ascending=False)
h2_neale_sorted.columns = h2_neale_sorted.columns.map(lambda x: 'neale.'+str(x))
h2_neale2_result=pd.read_csv(h2_v2_path,sep='\t')
h2_neale2_result=h2_neale2_result.set_index('phenotype')
h2_neale2_sorted=h2_neale2_result.sort_values('h2_observed',ascending=False)
h2_neale2_sorted.columns = h2_neale2_sorted.columns.map(lambda x: 'neale2.'+str(x))<jupyter_output><empty_output><jupyter_text>from my ldsc result of univariate<jupyter_code>h2_myuni_sorted=read_ldsc('uni',False)
h2_myuni_sorted.head()<jupyter_output><empty_output><jupyter_text>from my ldsc result of various BP scales<jupyter_code>suffix_bp_list_power="bp300 bp128 bp64 bp32 bp16 bp8".split()
suffix_bp_list_full="bp300 bp128 bp64 bp32 bp16 bp8".split()

h2_bp_sorted_list=[]
for suffix in suffix_bp_list_full:
    h2_bp_sorted_list.append(read_ldsc(suffix,verbose=True))
    print(suffix,"was loaded")<jupyter_output>bp300 was loaded
bp128 was loaded
bp64 was loaded
bp32 was loaded
bp16 was loaded
bp8 was loaded<jupyter_text>merging data.<jupyter_code>h2_total=h2_bp_sorted_list[0].copy()
for h2_bp_sorted in h2_bp_sorted_list[1:]:
    h2_total=h2_total.join(h2_bp_sorted)

h2_total=h2_total.join(h2_myuni_sorted)
h2_total=h2_total.join(h2_neale_sorted)
h2_total=h2_total.join(h2_neale2_sorted)
h2_total['uni.Total h2 z']=h2_total['uni.Total Observed scale h2']/h2_total['uni.Total Observed scale h2_se']
h2_total['uni.Total h2 p']=st.norm.sf(h2_total['uni.Total Observed scale h2']/h2_total['uni.Total Observed scale h2_se'])
h2_total.shape
%matplotlib inline

#basic settings for plt
"""
plt.rcParams['font.size']=22
plt.rcParams["figure.figsize"] = (5,5)
plt.rcParams['lines.linewidth'] = 2
plt.rcParams['lines.color'] = 'r'
"""
plt.rcParams['axes.grid'] = True

plt.title("h2 value from different source")
plt.scatter(h2_total['neale.h2_observed'], h2_total['neale2.h2_observed'])
plt.xlabel("h2 observed calculated by neale group")
plt.ylabel("h2 observed calculated by me(univariate)")
plt.show()

plt.title("h2 value from different source")
plt.scatter(h2_total['neale.h2_liability'], h2_total['neale2.h2_liability'])
plt.xlabel("h2 observed calculated by neale group")
plt.ylabel("h2 observed calculated by me(univariate)")
plt.show()

plt.title("h2 value from different source")
plt.scatter(h2_total['neale.h2_liability'], h2_total['uni.Total Liability scale h2'])
plt.xlabel("h2 observed calculated by neale group")
plt.ylabel("h2 observed calculated by me(univariate)")
plt.show()

plt.title("h2 value from different source")
plt.scatter(h2_total['neale2.h2_liability'], h2_total['uni.Total Liability scale h2'])
plt.xlabel("h2 observed calculated by neale group")
plt.ylabel("h2 observed calculated by me(univariate)")
plt.show()


def plot_compare_h2(suffix='cm300',mode='all'):
    if mode=='all':
        idx=h2_total.index
    elif mode=='continuous':
        idx=h2_total[h2_total['uni.Total Liability scale h2']==h2_total['uni.Total Observed scale h2']].index
    elif mode=='binary':
        idx=h2_total[h2_total['uni.Total Liability scale h2']!=h2_total['uni.Total Observed scale h2']].index
        
    x_data=h2_total.loc[idx]['uni.Total Liability scale h2']
    y_data=h2_total.loc[idx]['{}.Total Liability scale h2'.format(suffix)]
    #x_data=h2_total[idx]['uni.Total Observed scale h2']
    #y_data=h2_total[idx]['{}.Total Observed scale h2'.format(suffix)]
    #c=h2_total[idx]['cm300.Total Liability scale h2']/h2_total[idx]['cm300.Total Observed scale h2']
    #x_data=h2_total[idx]['uni.Total Observed scale h2']*c
    #y_data=h2_total[idx]['{}.Total Observed scale h2'.format(suffix)]*c
    
    hue_data=np.log10(st.norm.sf(h2_total.loc[idx]['{}.Total Observed scale h2'.format(suffix)]/h2_total.loc[idx]['{}.Total Observed scale h2_se'.format(suffix)]))
    
    plt.title("h2 values of {} traits from different source".format(mode))
    sns.scatterplot(x_data,y_data,hue=hue_data)
    plt.xlabel("h2 calculated by me(univariate)")
    plt.ylabel("h2 calculated by me({})".format(suffix))
    plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.15), fontsize=10,fancybox=True, shadow=True)
    plt.show()

    

    
for suffix in suffix_bp_list_full:
    plot_compare_h2(suffix,mode='binary')

    
for suffix in suffix_bp_list_full:
    plot_compare_h2(suffix,mode='continuous')
#plot_compare_h2("cm15")
#plot_compare_h2("cm10")
#plot_compare_h2("cm2")

a=pd.read_pickle('23115_irnt.pickle')

[(chr,a[a.CHR==chr]['BP'].iloc[-1]) for chr in range(1,22+1)]
h2_total
suffix_bp_list_power="bp300 bp128 bp64 bp32 bp16 bp8".split()
regression_result_bp_list_dict=dict([make_regression_result_list(h2_total,phenotype,suffix_bp_list_power) for phenotype in phenotypes_par_filtered.index])
h2_bp8_df=pd.DataFrame([[phenotype]+(regression_result_list[-1]['h2']/regression_result_list[-1]['h2'].sum()).tolist() for phenotype,regression_result_list in regression_result_bp_list_dict.items()])
h2_bp8_df=h2_bp8_df.set_index(0)
sum([regression_result_bp_list_dict['23115_irnt'][i].shape[0] for i in range(len(regression_result_bp_list_dict['23115_irnt']))])*497<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14406461307287755, 'fraction_numerical': 0.0473328324567994, 'mean_word_length': 7.616504854368932, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '94675', 'license': 'MIT', 'n_tokens_mistral': 2255, 'n_tokens_neox': 2107, 'n_words': 316, 'path': 'test.ipynb', 'repo_name': 'ch6845/regional_heritability_analysis', 'size': '5324'}