<jupyter_start><jupyter_text>SCENIC  Load the expression matrix<jupyter_code>SC_EXP_FNAME = "Data/Cells/CD4/cd4.matrix.csv"
ex_matrix = pd.read_csv(SC_EXP_FNAME, sep=' ', header=0, index_col=0).T
ex_matrix.index = ex_matrix.index.str.replace('.', '-')
ex_matrix<jupyter_output><empty_output><jupyter_text>__METADATA CLEANING__<jupyter_code>BASE_FOLDER = "Data/Cells/CD4/"
df_cluster = pd.read_csv(BASE_FOLDER+"cd4.clusters.csv")
df_tumor = pd.read_csv(BASE_FOLDER+"cd4.tumor.csv")
df_cluster['x'] = pd.Series(df_cluster['x'], dtype="category")
df_metadata = df_cluster.merge(df_tumor, on="Unnamed: 0")
df_metadata.columns = ["Cell ID", "Cluster", "Tumor"]
df_metadata = df_metadata.set_index("Cell ID")
df_metadata.to_csv(BASE_FOLDER+"SCENIC/cd4.metadata.csv", index=False)
df_metadata
# Regulons
BASE_FOLDER = "Data/Cells/CD4/SCENIC/"
derive_regulons(BASE_FOLDER)

with open(BASE_FOLDER+'regulons.dat', 'rb') as f:
    regulons = pickle.load(f)
BASE_FOLDER = "Data/Cells/CD4/SCENIC/"
REGULONS_DAT_FNAME = os.path.join(BASE_FOLDER, 'regulons.dat')
with open(REGULONS_DAT_FNAME, 'rb') as f:
    regulons = pickle.load(f)
regulon_df = []
for i,regulon in enumerate(regulons):
    gene2weight = pd.Series(dict(regulon.gene2weight)).sort_values(ascending=False).round(2)
    regulon_df.append([regulon.name, len(regulon.genes), regulon.score, list(zip(gene2weight.index, gene2weight))])
regulon_df = pd.DataFrame(regulon_df, columns=["Regulon", "Size", "Score", "Genes"]).sort_values(by="Score", ascending=False).set_index('Regulon')
regulon_df.to_excel(BASE_FOLDER+"regulon_df.xlsx")
regulon_df
# Calculate AUC Matrix
auc_mtx = aucell(ex_matrix, regulons, seed=42)
auc_mtx.to_csv(BASE_FOLDER+"auc.csv")
auc_mtx
# %%capture
# from tqdm import tqdm_notebook as tqdm
# tqdm().pandas()

# for start in tqdm(range(0, len(auc_mtx.columns))):
#     print(auc_mtx.iloc[:,start:start+1].columns)
#     binary_mtx, auc_thresholds = binarize(auc_mtx.iloc[:,start:start+1], seed=42, num_workers=1)
# auc_mtx['Klf2'].to_csv(BASE_FOLDER+"error_column.txt")
# auc_mtx = auc_mtx.drop('Klf2', axis=1)
# Calculate Binary Matrix
binary_mtx, auc_thresholds = binarize(auc_mtx, seed=42, num_workers=1)
binary_mtx.to_csv(BASE_FOLDER+"binary.csv")
binary_mtx<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15826702033598586, 'fraction_numerical': 0.013262599469496022, 'mean_word_length': 7.80544747081712, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '501675', 'license': 'MIT', 'n_tokens_mistral': 985, 'n_tokens_neox': 959, 'n_words': 124, 'path': 'Code/CD4SCENICAnalysis.ipynb', 'repo_name': 'andrewni4313/IL33Th17', 'size': '2262'}