<jupyter_start><jupyter_text>Make GS file<jupyter_code># GENE_SCORE_PATH='/n/holystore01/LABS/price_lab/Users/mjzhang/scTRS_data/trs_gene_scores/gwas_max_abs_z'

# # Top 500 GWAS genes 
# df_gs = pd.DataFrame(columns=['TRAIT', 'GENESET'])
# for file in os.listdir(GENE_SCORE_PATH):
#     trait=file.replace('.csv','')
#     temp_df = pd.read_csv(join(GENE_SCORE_PATH, file), sep=',')
#     temp_df = temp_df.loc[~temp_df['MAX_ABS_Z'].isna()]
#     temp_df = temp_df.sort_values(by='MAX_ABS_Z', ascending=False)
#     df_gs.loc[trait] = [trait,','.join(temp_df['GENE'][0:500])]
# df_gs.to_csv('/n/holystore01/LABS/price_lab/Users/mjzhang/scTRS_data/gs_file/gwas_max_abs_z.top500.gs',
#              sep='\t', index=False)

# BATCH_SIZE=5
# for i_batch in range(np.ceil(df_gs.shape[0]/BATCH_SIZE).astype(int)):
#     df_gs.iloc[i_batch*BATCH_SIZE:(i_batch+1)*BATCH_SIZE].to_csv(
#         '/n/holystore01/LABS/price_lab/Users/mjzhang/scTRS_data/gs_file/gwas_max_abs_z.top500.batch%d.gs'%i_batch,
#                 sep='\t', index=False)
# GENE_SCORE_PATH='/n/holystore01/LABS/price_lab/Users/mjzhang/scTRS_data/trs_gene_scores/gwas_max_abs_z'

# # Top 100 GWAS genes 
# df_gs = pd.DataFrame(columns=['TRAIT', 'GENESET'])
# for file in os.listdir(GENE_SCORE_PATH):
#     trait=file.replace('.csv','')
#     temp_df = pd.read_csv(join(GENE_SCORE_PATH, file), sep=',')
#     temp_df = temp_df.loc[~temp_df['MAX_ABS_Z'].isna()]
#     temp_df = temp_df.sort_values(by='MAX_ABS_Z', ascending=False)
#     df_gs.loc[trait] = [trait,','.join(temp_df['GENE'][0:100])]
# df_gs.to_csv('/n/holystore01/LABS/price_lab/Users/mjzhang/scTRS_data/gs_file/gwas_max_abs_z.top100.gs',
#              sep='\t', index=False)

# BATCH_SIZE=5
# for i_batch in range(np.ceil(df_gs.shape[0]/BATCH_SIZE).astype(int)):
#     df_gs.iloc[i_batch*BATCH_SIZE:(i_batch+1)*BATCH_SIZE].to_csv(
#         '/n/holystore01/LABS/price_lab/Users/mjzhang/scTRS_data/gs_file'
#         '/gwas_max_abs_z.top100.gs.batch/gwas_max_abs_z.top100.batch%d.gs'%i_batch,
#                 sep='\t', index=False)
# df_gs = pd.read_csv('/n/holystore01/LABS/price_lab/Users/mjzhang/scTRS_data/gs_file/magma_10kb_1000.gs',
#                     sep='\t')

# BATCH_SIZE=3
# for i_batch in range(np.ceil(df_gs.shape[0]/BATCH_SIZE).astype(int)):
#     df_gs.iloc[i_batch*BATCH_SIZE:(i_batch+1)*BATCH_SIZE].to_csv(
#         '/n/holystore01/LABS/price_lab/Users/mjzhang/scTRS_data/gs_file'
#         '/magma_10kb_1000.gs.batch/magma_10kb_1000.batch%d.gs'%i_batch,
#                 sep='\t', index=False)
# # Make a small gs file containing the 74 analyzed traits 
# df_gs = pd.read_csv('/n/holystore01/LABS/price_lab/Users/mjzhang/scDRS_data/gs_file/magma_10kb_1000.gs', sep='\t')
# df_trait = pd.read_csv('/n/holystore01/LABS/price_lab/Users/mjzhang/scDRS_data/supp_table/trait_info.tsv', sep='\t')

# df_gs_small = df_gs.loc[[x in df_trait['Trait_Identifier'].values for x in df_gs['TRAIT']]].copy()
# df_gs_small.to_csv('/n/holystore01/LABS/price_lab/Users/mjzhang/scDRS_data/gs_file/magma_10kb_1000.74_traits.gs',
#                    sep='\t', index=False)

# BATCH_SIZE=3
# for i_batch in range(np.ceil(df_gs_small.shape[0]/BATCH_SIZE).astype(int)):
#     df_gs_small.iloc[i_batch*BATCH_SIZE:(i_batch+1)*BATCH_SIZE].to_csv(
#         '/n/holystore01/LABS/price_lab/Users/mjzhang/scDRS_data/gs_file'
#         '/magma_10kb_1000.74_traits.gs.batch/magma_10kb_1000.batch%d.gs'%i_batch,
#                 sep='\t', index=False)
# # Make a small gs file containing IBD,SCZ,TG
# df_gs = pd.read_csv('/n/holystore01/LABS/price_lab/Users/mjzhang/scDRS_data/gs_file/magma_10kb_1000.gs', sep='\t')
# df_trait = pd.read_csv('/n/holystore01/LABS/price_lab/Users/mjzhang/scDRS_data/supp_table/trait_info.tsv', sep='\t')

# df_gs_small = df_gs.loc[[x in df_trait['Trait_Identifier'].values for x in df_gs['TRAIT']]].copy()
# df_gs_small.to_csv('/n/holystore01/LABS/price_lab/Users/mjzhang/scDRS_data/gs_file/magma_10kb_1000.74_traits.gs',
#                    sep='\t', index=False)

# BATCH_SIZE=3
# for i_batch in range(np.ceil(df_gs_small.shape[0]/BATCH_SIZE).astype(int)):
#     df_gs_small.iloc[i_batch*BATCH_SIZE:(i_batch+1)*BATCH_SIZE].to_csv(
#         '/n/holystore01/LABS/price_lab/Users/mjzhang/scDRS_data/gs_file'
#         '/magma_10kb_1000.74_traits.gs.batch/magma_10kb_1000.batch%d.gs'%i_batch,
#                 sep='\t', index=False)<jupyter_output><empty_output><jupyter_text>Check unfinished files<jupyter_code># # Check unfinished files 
# DATA_PATH='/n/holystore01/LABS/price_lab/Users/mjzhang/scTRS_data'
# GS_FILE=DATA_PATH+'/gs_file/gwas_max_abs_z.top500.gs'
# SCORE_FILE=DATA_PATH+'/score_file/score.tms_droplet_with_cov.gwas_max_abs_z.top500/@t.full_score.gz'

# trait_list = []
# df_gs = pd.read_csv(GS_FILE, sep='\t', index_col=0)
# for trait in df_gs.index:
#     if os.path.exists(SCORE_FILE.replace('@t',trait)) is False:
#         print(trait)
#         trait_list.append(trait)
        
# df_gs.loc[trait_list].to_csv(DATA_PATH+'/gs_file/gwas_max_abs_z.top500.unfinished.gs', sep='\t')
# # Check unfinished files 
# DATA_PATH='/n/holystore01/LABS/price_lab/Users/mjzhang/scTRS_data'
# GS_FILE=DATA_PATH+'/gs_file/tcell_signature.gs'
# SCORE_FILE=DATA_PATH+'/score_file/score.tms_droplet_with_cov.tcell_sig/@t.full_score.gz'

# trait_list = []
# df_gs = pd.read_csv(GS_FILE, sep='\t', index_col=0)
# for trait in df_gs.index:
#     if os.path.exists(SCORE_FILE.replace('@t',trait)) is False:
#         print(trait)
#         trait_list.append(trait)
        
# df_gs.loc[trait_list].to_csv(DATA_PATH+'/gs_file/tcell_signature.unfinished.gs', sep='\t')
# Check unfinished files 
DATA_PATH='/n/holystore01/LABS/price_lab/Users/mjzhang/scDRS_data'
GS_FILE=DATA_PATH+'/gs_file/ploidy.gs'
SCORE_FILE=DATA_PATH+'/score_file/score.tms_droplet_with_cov.hep_metabolic/@t.full_score.gz'

trait_list = []
df_gs = pd.read_csv(GS_FILE, sep='\t', index_col=0)
for trait in df_gs.index:
    if os.path.exists(SCORE_FILE.replace('@t',trait)) is False:
        print(trait)
        trait_list.append(trait)
        
df_gs.loc[trait_list].to_csv(DATA_PATH+'/gs_file/ploidy.unfinished.gs', sep='\t')
df_gs
# Check unfinished files 
DATA_PATH='/n/holystore01/LABS/price_lab/Users/mjzhang/scDRS_data'
GS_FILE=DATA_PATH+'/gs_file/magma_10kb_1000.74_traits.gs'
SCORE_FILE=DATA_PATH+'/score_file/score.tms_facs_with_cov.magma_10kb_1000.score_od/@t.score.gz'
SUFFIX = 'score_od'

trait_list = []
df_gs = pd.read_csv(GS_FILE, sep='\t', index_col=0)
for trait in df_gs.index:
    if os.path.exists(SCORE_FILE.replace('@t',trait)) is False:
        print(trait)
        trait_list.append(trait)
df_gs.loc[trait_list].to_csv(DATA_PATH+'/gs_file/magma_10kb_1000.74_traits.unfinished.%s.gs'%SUFFIX, sep='\t')

df_gs = df_gs.loc[trait_list].copy()
BATCH_SIZE=3
for i_batch in range(np.ceil(df_gs.shape[0]/BATCH_SIZE).astype(int)):
    df_gs.iloc[i_batch*BATCH_SIZE:(i_batch+1)*BATCH_SIZE].to_csv(DATA_PATH+
        '/gs_file/magma_10kb_1000.74_traits.gs.unfinished.batch/magma_10kb_1000.batch%d.unfinished%s.gs'
                                                                 %(i_batch,SUFFIX), 
        sep='\t', index=True)
df_gs = df_gs.loc[trait_list].copy()
BATCH_SIZE=3
for i_batch in range(np.ceil(df_gs.shape[0]/BATCH_SIZE).astype(int)):
    df_gs.iloc[i_batch*BATCH_SIZE:(i_batch+1)*BATCH_SIZE].to_csv(
        '/n/holystore01/LABS/price_lab/Users/mjzhang/scDRS_data/gs_file'
        '/magma_10kb_1000.gs.batch/magma_10kb_1000.batch%d.unfinished.gs'%i_batch,
                sep='\t', index=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16477272727272727, 'fraction_numerical': 0.030258985200845665, 'mean_word_length': 4.449244060475162, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '106970', 'license': 'MIT', 'n_tokens_mistral': 3463, 'n_tokens_neox': 3312, 'n_words': 281, 'path': 'experiments/job.curate_data/curate_gs_file.ipynb', 'repo_name': 'martinjzhang/scDRS', 'size': '7568'}