<jupyter_start><jupyter_text>This notebook follows 05_make_rentina_subsets_for_teaching.This takes data from retina_1_digital_expression and subsets them into three groups: big_clusters, amacrine, and differential.Inputs:1. retina_clusteridentities.txt2. mouse_gene_metadata was output from 04-make-mouse-gene-metadata3. GSM1626793_P14Retina_1.digital_expressionn.txt was downloaded from [here](https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE63472).This notebook outputs the following files:1. big_clusters_expression.csv2. big_clusters_cell_metadata.csv3. big_clusters_gene_metadata.csv4. amacrine_expression.csv5. amacrine_cell_metadata.csv6. amacrine_gene_metadata.csv7. differential_expression.csv8. differential_cell_metadata.csv9. differential_gene_metadata.csv<jupyter_code>import os
import pandas as pd
import numpy as np

from functions.data import ClusterToCellType, preprocess_digital_expression
cluster_to_celltype = ClusterToCellType()
to_save=True<jupyter_output><empty_output><jupyter_text>Get Data<jupyter_code># cell_cluster information
cluster_df = pd.read_csv("data/downloads/retina_clusteridentities.txt",
                         sep='\t', header=None, names=['cell', 'cluster_no'], index_col=0)
cluster_df['cluster_name'] = cluster_df['cluster_no'].apply(lambda x: f'cluster_{str(x).zfill(2)}')
print(cluster_df.shape)
cluster_df.head()
# Get mouse_gene_metadata
mouse_gene_metadata = pd.read_csv('data/mouse_gene_metadata.csv', index_col=0)

# For the purposes of teaching, we'll rename "retina" to "cluster" so it's consistent with the cell metadata
mouse_gene_metadata.columns = [x.replace('retina', 'cluster') for x in mouse_gene_metadata]
mouse_gene_metadata
# Digital expression
digital_expression = pd.read_csv("data/downloads/GSM1626793_P14Retina_1.digital_expression.txt",
                     sep='\t')
digital_expression = preprocess_digital_expression(digital_expression)
digital_expression
# No need for this
# digital_expression.to_csv('data/retina_1_digital_expression.csv', index=True)<jupyter_output><empty_output><jupyter_text>Big Clusters<jupyter_code>cluster_sizes = digital_expression.groupby(cluster_df['cluster_name'], axis=0).size()
big_clusters = cluster_sizes[cluster_sizes > 100].index.to_list()  # define big_clusters

# find cells in big_clusters
cells_in_big_clusters = cluster_df[(cluster_df['cluster_name'].isin(big_clusters))].index.to_list()  # get cells from cluster_df

# find mouse_genes in big_clusters
mouse_genes_in_big_clusters = mouse_gene_metadata[(mouse_gene_metadata[big_clusters].any(axis=1))].index.to_list()
# Subset rows by cells_in_big_clusters and columns by mouse_genes_in_big_clusters
big_clusters_expression_subset = digital_expression.loc[
    digital_expression.index.isin(cells_in_big_clusters),
    mouse_genes_in_big_clusters
]

# Take 50 random cells from each cluster
np.random.seed(2017)
big_clusters_expression_random_subset = big_clusters_expression_subset.groupby(
    cluster_df['cluster_name'], as_index=False, group_keys=False).apply(
        lambda x: x.loc[np.random.choice(x.index, size=50, replace=False)])

big_clusters_expression_random_subset
# subset metadata
big_clusters_cell_metadata = cluster_df.loc[big_clusters_expression_random_subset.index]
big_clusters_gene_metadata = mouse_gene_metadata.loc[mouse_genes_in_big_clusters, big_clusters]
if to_save:
    os.makedirs("data/big_clusters", exist_ok=True)
    big_clusters_expression_random_subset.reset_index().to_csv('data/big_clusters/big_clusters_expression.csv', index=False)
    big_clusters_cell_metadata.reset_index().to_csv('data/big_clusters/big_clusters_cell_metadata.csv', index=False)
    big_clusters_gene_metadata.reset_index().to_csv('data/big_clusters/big_clusters_gene_metadata.csv', index=False)<jupyter_output><empty_output><jupyter_text>Amacrine Cells<jupyter_code>amacrine_clusters = ['cluster_{}'.format(str(i).zfill(2)) for i in range(3, 24)]

# find cells in amacrine_clusters
cells_in_amacrine_clusters = cluster_df[(cluster_df['cluster_name'].isin(amacrine_clusters))].index.to_list()  # get cells from cluster_df

# find mouse_genes in amacrine_clusters
mouse_genes_in_amacrine_clusters = mouse_gene_metadata[(mouse_gene_metadata[amacrine_clusters].any(axis=1))].index.to_list()
# Subset rows by cells_in_amacrine_clusters and columns by mouse_genes_in_amacrine_clusters
amacrine_clusters_expression_subset = digital_expression.loc[
    digital_expression.index.isin(cells_in_amacrine_clusters),
    mouse_genes_in_amacrine_clusters
]

amacrine_clusters_expression_subset
# subset metadata
amacrine_cell_metadata = cluster_df.loc[amacrine_clusters_expression_subset.index]
amacrine_gene_metadata = mouse_gene_metadata.loc[mouse_genes_in_amacrine_clusters, amacrine_clusters]
if to_save:
    os.makedirs("data/amacrine", exist_ok=True)
    amacrine_clusters_expression_subset.to_csv('data/amacrine/amacrine_expression.csv', index=True)
    amacrine_cell_metadata.to_csv('data/amacrine/amacrine_cell_metadata.csv')
    amacrine_gene_metadata.to_csv('data/amacrine/amacrine_gene_metadata.csv')
# check
# amacrine_clusters_expression_subset['NAV1']<jupyter_output><empty_output><jupyter_text>Retina (aka. Differential)<jupyter_code>retina_clusters = list(filter(lambda x: 'cluster' in x, mouse_gene_metadata.columns))

# find cells in retina_clusters
cells_in_retina_clusters = cluster_df[(cluster_df['cluster_name'].isin(retina_clusters))].index.to_list()  # get cells from cluster_df

# find mouse_genes in retina_clusters
mouse_genes_in_retina_clusters = mouse_gene_metadata[(mouse_gene_metadata[retina_clusters].any(axis=1))].index.to_list()
# Subset rows by cells_in_retina_clusters and columns by mouse_genes_in_retina_clusters
retina_clusters_expression_subset = digital_expression.loc[
    digital_expression.index.isin(cells_in_retina_clusters),
    mouse_genes_in_retina_clusters]

retina_clusters_expression_subset
# subset metadata
retina_cell_metadata = cluster_df.loc[retina_clusters_expression_subset.index]
retina_gene_metadata = mouse_gene_metadata.loc[mouse_genes_in_retina_clusters, retina_clusters]
if to_save:
    os.makedirs("data/retina", exist_ok=True)
    retina_clusters_expression_subset.to_csv('data/retina/differential_clusters_expression.csv')
    retina_cell_metadata.to_csv('data/retina/differential_clusters_gene_metadata.csv')
    retina_gene_metadata.to_csv('data/retina/differential_clusters_cell_metadata.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09150023223409197, 'fraction_numerical': 0.010218300046446818, 'mean_word_length': 8.817629179331307, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 22, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '446491', 'license': 'MIT', 'n_tokens_mistral': 2335, 'n_tokens_neox': 2153, 'n_words': 296, 'path': '03-subset-digital_expression.ipynb', 'repo_name': 'harrisonized/cshl-singlecell-2017-v2', 'size': '6459'}