<jupyter_start><jupyter_text>Imports<jupyter_code>import glob
import os

import pandas as pd
import scanpy as sc
import seaborn as sns
from nb_utils import describe

from celltype_utils import get_shared_adata
from path_constants import sig_outdir_base, FIGURE_FOLDER<jupyter_output><empty_output><jupyter_text>Figure folder<jupyter_code>figure_folder = os.path.join(FIGURE_FOLDER, 'unaligned_kmers')
! mkdir -p $figure_folder<jupyter_output><empty_output><jupyter_text>Read one2one h5ad<jupyter_code>adata = get_shared_adata()
adata<jupyter_output><empty_output><jupyter_text>Read parquet files  File paths<jupyter_code>sketch_id = "alphabet-dayhoff__ksize-51__scaled-10"


parquets = glob.glob(
    os.path.join(
        sig_outdir_base,
        "*",
        "2--single-cell-kmers",
        sketch_id,
        "hash2kmer__unique_kmers_per_celltype.parquet",
    )
)
parquets<jupyter_output><empty_output><jupyter_text>Bat<jupyter_code>bat_parquet = [x for x in parquets if 'bat' in x][0]
bat_parquet
bat = pd.read_parquet(bat_parquet)
describe(bat)<jupyter_output>(69988437, 13)
--- First 5 entries ---<jupyter_text>Human<jupyter_code>human_parquet = [x for x in parquets if 'human' in x][0]
human_parquet
human = pd.read_parquet(human_parquet)
describe(human)
bat.groupby('broad_group').cell_id.nunique()
human.groupby('broad_group').cell_id.nunique()
bat.groupby('broad_group').apply(lambda x: x.hashval.nunique()/x.cell_id.nunique())
bat_unaligned = bat.query('alignment_status == "unaligned"')
bat_unaligned.head()
bat_aligned = bat.query('alignment_status == "aligned"')
describe(bat_aligned)
bat_aligned.dtypes
bat_hashval_to_ncells = bat_aligned.groupby(['hashval', 'broad_group']).cell_id.nunique()
bat_hashval_to_ncells.name = 'n_cells_bat'
bat_hashval_to_ncells.head()
bat_unaligned.dtypes
bat_unaligned_hashes = set(bat_unaligned.hashval)
len(bat_unaligned_hashes)
human.hashval = human.hashval.astype(str)
human.hashval.dtypes
%%time

human_hashvals_bat_unaligned = human.query(
    "hashval in @bat_unaligned_hashes"
)
describe(human_hashvals_bat_unaligned)
human_hashvals_bat_unaligned_gene_names = human_hashvals_bat_unaligned.groupby('broad_group').gene_name.value_counts()
human_hashvals_bat_unaligned_gene_names.name = 'n_hashes'
human_hashvals_bat_unaligned_gene_names = human_hashvals_bat_unaligned_gene_names.reset_index()
describe(human_hashvals_bat_unaligned_gene_names)<jupyter_output>(33379, 3)
--- First 5 entries ---<jupyter_text>Read bat unannotated genes<jupyter_code>bat_unannotated = pd.read_csv('/home/olga/data_lg/data_sm_copy/immune-evolution/databases/unannotated-genes-in-bat-compared-to-human-GRCh38/unannotated_gene_in_bat_compared_to_GRCH38p13.csv')
bat_unannotated = bat_unannotated.dropna(how='all', axis=1)
bat_unannotated = {k: v for k, v in bat_unannotated.iteritems()}
# print(bat_unannotated.shape)
# bat_unannotated.head()
bat_unannotated.keys()
dfs = []

for gene_category, gene_names in bat_unannotated.items():
#     print(f'--- {gene_category} ---')
    df = human_hashvals_bat_unaligned_gene_names.query('gene_name in @gene_names')
    df['gene_category'] = gene_category
    dfs.append(df)
bat_unaligned_in_human = pd.concat(dfs)
describe(bat_unaligned_in_human)

bat_unaligned_in_human.to_csv(
    os.path.join(figure_folder, "bat_unaligned_in_human.csv"), index=False
)
bat_unaligned_in_human.query('gene_category == "ISG" and broad_group == "Alveolar Epithelial Type 2"')
bat_unaligned_in_human_n_genes = bat_unaligned_in_human.groupby(['gene_category', 'broad_group']).size()
bat_unaligned_in_human_n_genes.name = 'n_genes'
bat_unaligned_in_human_n_genes = bat_unaligned_in_human_n_genes.reset_index()
bat_unaligned_in_human_n_genes = bat_unaligned_in_human_n_genes.sort_values(['gene_category', 'n_genes'], ascending=False)
bat_unaligned_in_human_n_genes<jupyter_output><empty_output><jupyter_text>Celltype palette<jupyter_code>celltype_palette = dict(
    zip(
        sorted(bat_unaligned_in_human_n_genes.broad_group.unique()),
        sns.color_palette("Dark2", n_colors=10),
    ),
)<jupyter_output><empty_output><jupyter_text>one plot per category<jupyter_code>for gene_group, df in bat_unaligned_in_human_n_genes.groupby("gene_category"):
    df = df.sort_values("n_genes", ascending=False)
    g = sns.catplot(
        y="broad_group",
        x="n_genes",
        col="gene_category",
        sharex=False,
        data=df,
        order=df.broad_group,
        palette=celltype_palette,
        kind="bar",
        height=3,
        aspect=1.5,
        #     sharey=False,
    )
    g.set_titles("{col_name}")
    pdf = os.path.join(figure_folder, f'barplot__unannotated_genes_found_in_unaligned_kmers__{gene_group}.pdf')
    g.savefig(pdf)
    png = pdf.replace('.pdf', '.png')
    g.savefig(png)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11507191994996874, 'fraction_numerical': 0.008755472170106316, 'mean_word_length': 6.579778830963665, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '238984', 'license': 'MIT', 'n_tokens_mistral': 1895, 'n_tokens_neox': 1819, 'n_words': 228, 'path': 'notebooks/supplementary_figure_6B-E.ipynb', 'repo_name': 'czbiohub/scrnaseq-for-the-99-percent', 'size': '4797'}