<jupyter_start><jupyter_text>Test datasetThis dataset is used to run the notebooks on the CI.<jupyter_code>%load_ext autoreload
%autoreload 2
from pathlib import Path
import csv
import shutil

import pandas as pd
from opencadd.databases.klifs import setup_remote, setup_local

from src.paths import PATH_DATA_KLIFS_DOWNLOAD, PATH_DATA
HERE = Path(_dh[-1])  # noqa: F821<jupyter_output><empty_output><jupyter_text>Prepare KLIFS test datasetUse KLIFS reference structures and a few DFG-out structures.<jupyter_code>klifs_reference_structures = pd.read_csv(
    PATH_DATA / "external/structures/klifs_reference_structures.csv", delimiter=";"
)
session = setup_local(PATH_DATA_KLIFS_DOWNLOAD)
# Get KLIFS reference structures
klifs_reference_structures.loc[
    klifs_reference_structures["structure.alternate_model"] == " ", 
    "structure.alternate_model"
] = "-"
klifs_reference_structures["structure.pdb_id"] = (
    klifs_reference_structures["structure.pdb_id"].str.lower()
)
# Look them up in local KLIFS download (contains filepaths)
structures_selected = []
for _, x in klifs_reference_structures.iterrows():
    structure = session.structures.by_structure_pdb_id(
        x["structure.pdb_id"],
        None if x["structure.alternate_model"] == " " else x["structure.alternate_model"],
        x["structure.chain"],
    )
    structures_selected.append(structure)
structures_selected = pd.concat(structures_selected)
# Add some DFG-out structures, incl. ABL1
structures = session.structures.all_structures()
structures_selected = pd.concat(
    [
        structures_selected,
        structures[structures["structure.dfg"] == "out"].sample(5),
        #structures[structures["kinase.klifs_name"] == "ABL1"].sample(1)
    ]
)
# Add more kinases
structures = session.structures.by_kinase_name(
    ["ErbB2", "p110a", "MET", "SLK", "KDR", "BRAF", "p38a", "LOK", "GAK"]  #, "ABL1"
)
structures = structures[structures["structure.dfg"] == "in"]
structures = structures.groupby("kinase.klifs_name").first().reset_index()
structures_selected = pd.concat(
    [structures_selected, structures]
)<jupyter_output><empty_output><jupyter_text>Write test `KLIFS_export.csv`<jupyter_code>klifs_export = pd.read_csv(PATH_DATA_KLIFS_DOWNLOAD / "KLIFS_export.csv")
print(klifs_export.shape)
klifs_export_selected = pd.merge(
    klifs_export, 
    structures_selected, 
    how="inner", 
    left_on=["PDB", "CHAIN", "ALTERNATE_MODEL"], 
    right_on=["structure.pdb_id", "structure.chain", "structure.alternate_model"]
)[klifs_export.columns]
klifs_export_selected.shape
klifs_export_selected.to_csv(
    HERE / "data/external/klifs_test/KLIFS_export.csv", 
    quoting=csv.QUOTE_ALL, 
    index=None
)<jupyter_output><empty_output><jupyter_text>Write test `overview.csv`<jupyter_code>structures_selected.loc[
    structures_selected["structure.alternate_model"] == "-", 
    "structure.alternate_model"
] = " "
overview = pd.read_csv(PATH_DATA_KLIFS_DOWNLOAD / "overview.csv")
print(overview.shape)
overview_selected = pd.merge(
    overview, 
    structures_selected, 
    how="inner", 
    left_on=["pdb", "chain", "alt"], 
    right_on=["structure.pdb_id", "structure.chain", "structure.alternate_model"]
)[overview.columns]
overview_selected.shape
overview_selected.to_csv(
    HERE / "data/external/klifs_test/overview.csv", 
    index=None
)<jupyter_output><empty_output><jupyter_text>Copy test PDB files<jupyter_code>for folder in structures_selected["structure.filepath"]:
    shutil.copytree(
        src=PATH_DATA_KLIFS_DOWNLOAD / folder, 
        dst=HERE / "data/external/klifs_test" / folder
    )<jupyter_output><empty_output><jupyter_text>Check KLIFS test folder<jupyter_code>setup_local(HERE / "data/external/klifs_test")<jupyter_output><empty_output><jupyter_text>Write test structure KLIFS IDs files<jupyter_code>structures_selected["structure.klifs_id"].to_csv(
    HERE / f"data/processed/structure_klifs_ids_all.txt", index=None, header=None
)
for dfg in ["in", "out"]:
    structures_selected[
        structures_selected["structure.dfg"] == dfg
    ]["structure.klifs_id"].to_csv(
        HERE / f"data/processed/structure_klifs_ids_dfg_{dfg}.txt", index=None, header=None
    )<jupyter_output><empty_output><jupyter_text>Generate `kissim` fingerprints and distances  All<jupyter_code>!bash src/kissim_encode_compare.sh all klifs_test normalized full
!bash src/kissim_weights.sh all<jupyter_output># packages in environment at /home/dominique/.local/miniconda/envs/kissim-app-dev:
#
# Name                    Version                   Build  Channel
kissim                    0.4.1+9.g378e5b5           dev_0    <develop>
Job settings
------------
Structure subset: all
INFO:kissim.api.weights:Read feature distances from /home/dominique/Documents/GitHub/kissim_app/test/results/all/feature_distances.csv.bz2...
INFO:kissim.comparison.fingerprint_distance_generator:GENERATE FINGERPRINT DISTANCES
INFO:kissim.comparison.fingerprint_distance_generator:Fingerprint distance generation started at: 2021-11-22 13:13:26.357916
INFO:kissim.comparison.fingerprint_distance_generator:Feature weights: [0.125 0.125 0.125 0.125 0.125 0.125 0.125 0.125 0.    0.    0.    0.
 0.    0.    0.   ]
Calculate pairwise fingerprint distance: 100%|█| 703/703 [00:00<00:00, 29447.97i
Calculate pairwise fingerprint coverage: 100%|█| 703/703 [00:00<00:00, 31958.59i
INFO:kissim.comparison.fingerprint_distance_generator[...]<jupyter_text>DFG-in<jupyter_code>!bash src/kissim_encode_compare.sh dfg_in klifs_test normalized full
!bash src/kissim_weights.sh dfg_in<jupyter_output># packages in environment at /home/dominique/.local/miniconda/envs/kissim-app-dev:
#
# Name                    Version                   Build  Channel
kissim                    0.4.1+9.g378e5b5           dev_0    <develop>
Job settings
------------
Structure subset: dfg_in
INFO:kissim.api.weights:Read feature distances from /home/dominique/Documents/GitHub/kissim_app/test/results/dfg_in/feature_distances.csv.bz2...
INFO:kissim.comparison.fingerprint_distance_generator:GENERATE FINGERPRINT DISTANCES
INFO:kissim.comparison.fingerprint_distance_generator:Fingerprint distance generation started at: 2021-11-22 13:14:21.348687
INFO:kissim.comparison.fingerprint_distance_generator:Feature weights: [0.125 0.125 0.125 0.125 0.125 0.125 0.125 0.125 0.    0.    0.    0.
 0.    0.    0.   ]
Calculate pairwise fingerprint distance: 100%|█| 496/496 [00:00<00:00, 29119.77i
Calculate pairwise fingerprint coverage: 100%|█| 496/496 [00:00<00:00, 27277.16i
INFO:kissim.comparison.fingerprint_distance_gen[...]<jupyter_text>DFG-out<jupyter_code>!bash src/kissim_encode_compare.sh dfg_out klifs_test normalized full
!bash src/kissim_weights.sh dfg_out<jupyter_output># packages in environment at /home/dominique/.local/miniconda/envs/kissim-app-dev:
#
# Name                    Version                   Build  Channel
kissim                    0.4.1+9.g378e5b5           dev_0    <develop>
Job settings
------------
Structure subset: dfg_out
INFO:kissim.api.weights:Read feature distances from /home/dominique/Documents/GitHub/kissim_app/test/results/dfg_out/feature_distances.csv.bz2...
INFO:kissim.comparison.fingerprint_distance_generator:GENERATE FINGERPRINT DISTANCES
INFO:kissim.comparison.fingerprint_distance_generator:Fingerprint distance generation started at: 2021-11-22 13:14:49.299712
INFO:kissim.comparison.fingerprint_distance_generator:Feature weights: [0.125 0.125 0.125 0.125 0.125 0.125 0.125 0.125 0.    0.    0.    0.
 0.    0.    0.   ]
Calculate pairwise fingerprint distance: 100%|█| 10/10 [00:00<00:00, 23523.86it/
Calculate pairwise fingerprint coverage: 100%|█| 10/10 [00:00<00:00, 25842.91it/
INFO:kissim.comparison.fingerprint_distance_g[...]
{'fraction_non_alphanumeric': 0.12020162853819309, 'fraction_numerical': 0.04730515703761148, 'mean_word_length': 4.644055433989789, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '351168', 'license': 'MIT', 'n_tokens_mistral': 3003, 'n_tokens_neox': 2647, 'n_words': 453, 'path': 'test/test_dataset.ipynb', 'repo_name': 'volkamerlab/kissim_app', 'size': '7737'}