<jupyter_start><jupyter_text>Perform initial data transformations for 5G TPM data1. Log ratio relative to umax1. Log ratio relative to noCu1. Box - cox transformFor all of these, we'll use promoter windows of 300bp (no truncation) and we'll optionally skip operon-likely genes (2204 total)<jupyter_code>import altair as alt
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

import random
random.seed(7) #10



import utils as u
import torch_utils as tu
#upstream_region_file = 'all_seq_info/all_loci_upstream_regions_w100_min20.fa'
upstream_region_file = 'all_proms/all_loci_upstream_regions_w300_min20.fa'
op_file = 'all_proms/loci_in_top_100perc.txt'
data_mat_file = 'data/extract_TPM_counts.tsv'
sample2cond_file = 'data/sample2condition.txt'
sample_file = None
condition_file = 'data/conditions_to_include.txt'
coded_meta_file = 'data/5G_exp_metadata_coded.tsv'

COND_COLS = ['carbon_source','oxygen_level','nitrate_level',
             'copper_level','lanthanum_level','growth_rate',
             'growth_mode']


loc2seq, op_leads, tpm_df, sample2condition, samples, conditions, meta_df = u.load_data(
    upstream_region_file,
    op_file,
    data_mat_file, 
    sample2cond_file, 
    sample_file, 
    condition_file,
    coded_meta_file
)
print('EQU24_RS19310' in op_leads)
print('EQU24_RS19315' in op_leads)
tpm_df.head()
print(np.mean(tpm_df['5GB1_pA9_red_tpm']))
print(np.mean(tpm_df['5GB1C-5G-La-BR1_tpm']))
# save locus info separately - easier to use in other files
tpm_df[['locus_tag','gene_symbol','product','type']].to_csv("data/locus2info.tsv",sep='\t',index=False)
locus_info_df = pd.read_csv("data/locus2info.tsv",sep='\t')
locus_info_df
locus2info = u.make_info_dict(locus_info_df)
locus2info['EQU24_RS19315']
df_means = u.get_gene_means_by_condition(tpm_df,samples,sample2condition)
df_means<jupyter_output><empty_output><jupyter_text>Check df_means distributions<jupyter_code>psuedo_count = 0.01
df_meansT = df_means.set_index("exp_condition").T + psuedo_count

df_meansT.describe()
# check a quick boxplot of mean tpm data
df_meansT_melt = pd.melt(df_meansT.reset_index(), id_vars=['locus_tag'],var_name="exp_condition",value_name='tpm')
df_meansT_melt
sns.boxplot(data=df_meansT_melt,x="exp_condition",y="tpm")
plt.ylim(-1,1000)
plt.xticks(rotation=90)
plt.show()<jupyter_output><empty_output><jupyter_text>Make log ratio versions of df<jupyter_code>lr_umax = u.make_lr_df(df_meansT, 'uMax')
lr_nocu = u.make_lr_df(df_meansT, 'NoCu')
# spot check log calculation
df_meansT[["highCu","medCu","lowCu","NoCu","uMax"]]
print(np.log2(47.861477/52.045884))
lr_umax[["highCu","medCu","lowCu","NoCu","uMax"]]
# check distribution of log ratio/uMax
def check_dist_of_lr_df(lr_df,name):
    lr_df_melt = pd.melt(lr_df.reset_index(), id_vars=['locus_tag'],var_name="exp_condition",value_name='lr_tpm')

    sns.boxplot(data=lr_df_melt,x="exp_condition",y="lr_tpm")
    #plt.ylim(-1,1000)
    plt.xticks(rotation=90)
    plt.title(f"Distribution of Log Ratio TPM /{name}")
    plt.show()
    
check_dist_of_lr_df(lr_umax,'uMax')
check_dist_of_lr_df(lr_nocu,'NoCu')
cu_conds = ['NoCu','lowCu','medCu','highCu']
check_dist_of_lr_df(lr_umax[cu_conds],'uMax')
check_dist_of_lr_df(lr_nocu[cu_conds],'NoCu')<jupyter_output><empty_output><jupyter_text>Save data frames<jupyter_code>XY_umax = u.make_XY_df(lr_umax, conditions, loc2seq, op_leads)
XY_nocu = u.make_XY_df(lr_nocu, conditions, loc2seq, op_leads)
XY_umax
XY_umax_noOpFilt = u.make_XY_df(lr_umax, conditions, loc2seq, op_leads, op_filt=False)
XY_nocu_noOpFilt = u.make_XY_df(lr_nocu, conditions, loc2seq, op_leads, op_filt=False)
XY_umax_noOpFilt
# save
XY_umax.to_csv("data/XY_lr_uMax_opFilt.tsv",sep='\t',index=False)
XY_nocu.to_csv("data/XY_lr_noCu_opFilt.tsv",sep='\t',index=False)

XY_umax_noOpFilt.to_csv("data/XY_lr_uMax.tsv",sep='\t',index=False)
XY_nocu_noOpFilt.to_csv("data/XY_lr_noCu.tsv",sep='\t',index=False)<jupyter_output><empty_output><jupyter_text>Raw TPM<jupyter_code>XY_tpm = u.make_XY_df(df_meansT, conditions, loc2seq, op_leads, op_filt=True)
XY_tpm_noOpFilt = u.make_XY_df(df_meansT, conditions, loc2seq, op_leads, op_filt=False)

XY_tpm.to_csv("data/XY_TPM_opFilt.tsv",sep='\t',index=False)
XY_tpm_noOpFilt.to_csv("data/XY_TPM.tsv",sep='\t',index=False)<jupyter_output><empty_output><jupyter_text>Log TPM<jupyter_code>log_df_meansT = np.log2(df_meansT)
XY_log = u.make_XY_df(log_df_meansT, conditions, loc2seq, op_leads, op_filt=True)
XY_log_noOpFilt = u.make_XY_df(log_df_meansT, conditions, loc2seq, op_leads, op_filt=False)

XY_log.to_csv("data/XY_logTPM_opFilt.tsv",sep='\t',index=False)
XY_log_noOpFilt.to_csv("data/XY_logTPM.tsv",sep='\t',index=False)<jupyter_output><empty_output><jupyter_text>Box-Cox<jupyter_code>from scipy import stats
# do box cox calculation for highcu
hc = df_meansT['highCu'].values
hcbc = stats.boxcox(hc)
# log ratio highCu/noCu
hclr = lr_nocu['highCu'].values
# log highCu
lhc = np.log2(hc)
hcbc[0]
plt.scatter(hc, hcbc[0],alpha = 0.2,label='TPM vs Box Cox')
plt.scatter(hc, hcl,alpha = 0.2,label='TPM vs log ratio TPM')
plt.scatter(hc, lhc,alpha = 0.2,label='TPM vs log TPM')
plt.xlabel("high copper TPM values")
plt.ylabel("BoxCox transform of high copper values")
plt.legend()
#plt.xscale('log')
plt.show()
plt.scatter(hc, hcbc[0],alpha = 0.2,label='TPM vs Box Cox')
plt.scatter(hc, hcl,alpha = 0.2,label='TPM vs logRatio TPM')
plt.scatter(hc, lhc,alpha = 0.2,label='TPM vs log TPM')
plt.xlabel("high copper TPM values")
plt.ylabel("BoxCox transform of high copper values")
plt.legend()
plt.xscale('log')
plt.show()
plt.hist(hcbc[0],bins=20,label='BoxCox',alpha=0.5)
plt.hist(lhc,bins=20,label="log(TPM)",alpha=0.5)
plt.hist(hcl,bins=20,label="logRatio(TPM)",alpha=0.5)
plt.title("distribution of transformed highCu scores")
plt.xlabel("new score")
plt.ylabel('count')
plt.legend()
plt.show()

plt.hist(hcbc[0],bins=20,label='BoxCox',alpha=0.5)
plt.hist(lhc,bins=20,label="log(TPM)",alpha=0.5)
plt.hist(hcl,bins=20,label="logRatio(TPM)",alpha=0.5)
plt.title("distribution of transformed highCu scores")
plt.xlabel("new score")
plt.ylabel('count')
plt.ylim(0,100)
plt.legend()
plt.show()


plt.hist(hc,bins=200)
plt.title("distribution of highCu scores")
plt.xlabel("tpm score")
plt.ylabel('count')
plt.show()

plt.hist(hc,bins=200)
plt.title("distribution of highCu scores")
plt.xlabel("tpm score")
plt.ylabel('count')
plt.xscale('log')
plt.show()
plt.scatter(hcl, hcbc[0],alpha = 0.2,label='Log ratio TPM vs Box Cox')
plt.xlabel("LOG high copper TPM values")
plt.ylabel("BoxCox transform of high copper values")
plt.legend()
#plt.xscale('log')
plt.show()
stats.probplot(hc,plot=plt)
plt.yscale('log')
list(XY_log['highCu'].values) == list(log_df_meansT['highCu'].values)
# make box cox df
bcdf = pd.DataFrame()
bcdf['locus_tag'] = df_meansT.reset_index()['locus_tag']

for cond in conditions:
    bc_vals,other = stats.boxcox(df_meansT[cond].values)
    print(cond, other)
    bcdf[cond] = bc_vals
list(bcdf['highCu'].values) == list(hcbc[0])
XY_bc = u.make_XY_df(bcdf, conditions, loc2seq, op_leads, op_filt=True)
XY_bc_noOpFilt = u.make_XY_df(bcdf, conditions, loc2seq, op_leads, op_filt=False)
XY_bc.to_csv("data/XY_bc_opFilt.tsv",sep='\t',index=False)
XY_bc_noOpFilt.to_csv("data/XY_bc.tsv",sep='\t',index=False)
list(XY_bc_noOpFilt['highCu'].values) == list(hcbc[0])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15292841648590022, 'fraction_numerical': 0.0227765726681128, 'mean_word_length': 7.909420289855072, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '416061', 'license': 'MIT', 'n_tokens_mistral': 3380, 'n_tokens_neox': 3162, 'n_words': 455, 'path': 'transform_5g_data.ipynb', 'repo_name': 'erinhwilson/deep-5g-sandbox', 'size': '7376'}