<jupyter_start><jupyter_text>Read data<jupyter_code>adata = []
for fname in sorted(pathlib.Path("../download/Domcke-2020").glob("*.pkl.gz")):
    print(f"Processing {fname}...")
    with gzip.open(fname, "rb") as f:
        d = pickle.load(f)
        adata.append(anndata.AnnData(
            X=d["X"], obs=d["obs"],
            var=pd.DataFrame(index=d["var_names"])
        ))
adata = anndata.AnnData.concatenate(*adata, batch_key="file", index_unique=None)
adata.obs.index.name, adata.var.index.name = "cells", "peaks"
adata<jupyter_output><empty_output><jupyter_text>Process meta<jupyter_code>adata.obs["domain"] = "scATAC-seq"
adata.obs["protocol"] = "sci-ATAC-seq3"
adata.obs["dataset"] = "Domcke-2020"
adata.obs.head()
adata.var["chrom"] = np.vectorize(lambda x: x.split("-")[0])(adata.var_names)
adata.var["chromStart"] = np.vectorize(lambda x: int(x.split("-")[1]))(adata.var_names)
adata.var["chromEnd"] = np.vectorize(lambda x: int(x.split("-")[2]))(adata.var_names)
adata.var["genome"] = "hg19"
adata.var.head()<jupyter_output><empty_output><jupyter_text>Clean data<jupyter_code>blacklist_overlap = scglue.genomics.window_graph(
    scglue.genomics.Bed(adata.var.assign(name=adata.var_names)),
    "../genome/Blacklist/lists/hg19-blacklist.v2.bed.gz",
    window_size=0
)
retained_peaks = np.asarray(biadjacency_matrix(
    blacklist_overlap, adata.var_names
).sum(axis=1)).ravel() == 0
adata = adata[:, retained_peaks]
adata.var = adata.var.astype({"chromStart": int, "chromEnd": int})
adata
sc.pp.filter_genes(adata, min_counts=1)
adata<jupyter_output><empty_output><jupyter_text>Save data<jupyter_code>adata.write_h5ad("../dataset/Domcke-2020.h5ad", compression="gzip")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19194865810968495, 'fraction_numerical': 0.01575262543757293, 'mean_word_length': 5.8326693227091635, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '522206', 'license': 'MIT', 'n_tokens_mistral': 701, 'n_tokens_neox': 674, 'n_words': 75, 'path': 'data/collect/Domcke-2020.ipynb', 'repo_name': 'gao-lab/GLUE', 'size': '1714'}