<jupyter_start><jupyter_text>Imports<jupyter_code>%load_ext autoreload
%autoreload 2
import glob
import os

import matplotlib.pyplot as plt
import pandas as pd
import scanpy as sc
import seaborn as sns
from path_constants import H5AD
from nb_utils import describe

from celltype_utils import get_shared_adata
from path_constants import sig_outdir_base<jupyter_output><empty_output><jupyter_text>Read one2one h5ad<jupyter_code>adata_shared = get_shared_adata()<jupyter_output>/home/olga/miniconda3/envs/immune-evolution/lib/python3.7/site-packages/anndata/_core/anndata.py:1094: FutureWarning: is_categorical is deprecated and will be removed in a future version.  Use is_categorical_dtype instead
  if not is_categorical(df_full[k]):<jupyter_text>Read parquet files  File paths<jupyter_code>sketch_id = 'alphabet-dayhoff__ksize-51__scaled-10'
sig_outdir_base = "/home/olga/data_lg/data_sm_copy/immune-evolution/kmer-signatures"


parquets = glob.glob(f'/home/olga/data_lg/data_sm_copy/immune-evolution/kmer-signatures/*/2--single-cell-kmers/{sketch_id}/hash2kmer__unique_kmers_per_celltype.parquet')
parquets
# parquets = glob.glob(f'/home/olga/data_lg/data_sm_copy/immune-evolution/kmer-signatures/*/2--single-cell-kmers/*/hash2kmer__unique_kmers_per_celltype.parquet')
# parquets
%%time
dfs = []

for parquet in parquets:
    ! ls -lha $parquet
    %time df = pd.read_parquet(parquet)
    df = df.rename(columns={'broad_group': 'groundtruth_celltype'})
    dfs.append(df)
    
hash2kmer = pd.concat(dfs)
describe(hash2kmer)<jupyter_output>-rw-r--r-- 1 olga czb 3.3G Jun 23 11:12 /home/olga/data_lg/data_sm_copy/immune-evolution/kmer-signatures/1--train-mouse--to-publish/2--single-cell-kmers/alphabet-dayhoff__ksize-51__scaled-10/hash2kmer__unique_kmers_per_celltype.parquet
CPU times: user 1min 23s, sys: 48.5 s, total: 2min 12s
Wall time: 1min 34s
-rw-r--r-- 1 olga czb 2.4G Jun 24 00:45 /home/olga/data_lg/data_sm_copy/immune-evolution/kmer-signatures/2--test-human/2--single-cell-kmers/alphabet-dayhoff__ksize-51__scaled-10/hash2kmer__unique_kmers_per_celltype.parquet
CPU times: user 48.5 s, sys: 32.3 s, total: 1min 20s
Wall time: 57.5 s
-rw-r--r-- 1 olga czb 5.0G Jun 24 05:18 /home/olga/data_lg/data_sm_copy/immune-evolution/kmer-signatures/3--test-bat/2--single-cell-kmers/alphabet-dayhoff__ksize-51__scaled-10/hash2kmer__unique_kmers_per_celltype.parquet
CPU times: user 2min 24s, sys: 1min 4s, total: 3min 28s
Wall time: 2min 33s
-rw-r--r-- 1 olga czb 149M Jun 24 06:59 /home/olga/data_lg/data_sm_copy/immune-evolution/kmer-sign[...]<jupyter_text>Read predicted cell types<jupyter_code>predicted_cells_min_5_percent = pd.read_parquet(
    os.path.join(sig_outdir_base, "aggregated-predicted-cells.parquet")
)
predicted_cells_min_5_percent = predicted_cells_min_5_percent.query('sketch_id == @sketch_id')
describe(predicted_cells_min_5_percent)<jupyter_output>(28578, 43)
--- First 5 entries ---<jupyter_text>Join hash2kmer with predicted cell types<jupyter_code>celltype_predictions = predicted_cells_min_5_percent[
    ["predicted_celltype", "groundtruth_compartment", "predicted_compartment"]
]
celltype_predictions.index = celltype_predictions.index.str.upper()
describe(celltype_predictions)
hash2kmer.cell_id = hash2kmer.cell_id.str.upper()
hash2kmer_with_predictions = hash2kmer.join(celltype_predictions, on='cell_id')
describe(hash2kmer_with_predictions)
hash2kmer_with_predictions['hashval'] = hash2kmer_with_predictions['hashval'].astype(str)<jupyter_output><empty_output><jupyter_text>Write joined to parquet<jupyter_code>hash2kmer_with_predictions.to_parquet(
    os.path.join(sig_outdir_base, f"aggregated-hash2kmer-with-predicted-cells__{sketch_id}.parquet")
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11272531611514662, 'fraction_numerical': 0.03685768092547754, 'mean_word_length': 6.526315789473684, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '340068', 'license': 'MIT', 'n_tokens_mistral': 1570, 'n_tokens_neox': 1448, 'n_words': 214, 'path': 'notebooks/figure_3G--00_aggregate_hash2kmer.ipynb', 'repo_name': 'czbiohub/scrnaseq-for-the-99-percent', 'size': '3717'}