<jupyter_start><jupyter_text>SummaryCheck that all results have been calculcated. Executing```bashexport NOTEBOOK_PATH="$(realpath 20_el2_stability.ipynb)"export DATASET_NAME="elaspic-core-mutation-local"export ORIGINAL_ARRAY_TASK_COUNT=26sbatch --export=DATASET_NAME,NOTEBOOK_PATH,ORIGINAL_ARRAY_TASK_COUNT --array=10,15 --ntasks-per-node=48 ../scripts/run_notebook_cpu.shexport NOTEBOOK_PATH="$(realpath 20_el2_stability.ipynb)"export DATASET_NAME="uniprot-domain-mutation"export ORIGINAL_ARRAY_TASK_COUNT=2141export SLURM_TASK_ID_OFFSET=1000sbatch --export=DATASET_NAME,NOTEBOOK_PATH,ORIGINAL_ARRAY_TASK_COUNT,SLURM_TASK_ID_OFFSET --array=817,826,827,830,832,849,883 --ntasks-per-node=48 ../scripts/run_notebook_cpu.shexport NOTEBOOK_PATH="$(realpath 20_el2_affinity.ipynb)"export DATASET_NAME="elaspic-interface-mutation-local"export ORIGINAL_ARRAY_TASK_COUNT=9sbatch --export=DATASET_NAME,NOTEBOOK_PATH,ORIGINAL_ARRAY_TASK_COUNT --array= --ntasks-per-node=40 --mem=0 ../scripts/run_notebook_cpu.shexport NOTEBOOK_PATH="$(realpath 20_el2_affinity.ipynb)"export DATASET_NAME="uniprot-domain-pair-mutation"export ORIGINAL_ARRAY_TASK_COUNT=1358export SLURM_TASK_ID_OFFSET=900sbatch --export=DATASET_NAME,NOTEBOOK_PATH,ORIGINAL_ARRAY_TASK_COUNT,SLURM_TASK_ID_OFFSET --array=6,139 ../scripts/run_notebook_cpu.sh On Cedar--ntasks-per-node=48  On Niagara,--ntasks-per-node=40 --mem=0```---  Imports<jupyter_code>import glob
import os
import socket
import tempfile
from pathlib import Path

from tqdm.notebook import tqdm<jupyter_output><empty_output><jupyter_text>Parameters<jupyter_code>NOTEBOOK_DIR = Path("20_el2_validation").resolve()
NOTEBOOK_DIR.mkdir(exist_ok=True)

NOTEBOOK_DIR
if "DATAPKG_OUTPUT_DIR" in os.environ:
    OUTPUT_DIR = Path(os.getenv("DATAPKG_OUTPUT_DIR")).joinpath("elaspic2").resolve()
else:
    OUTPUT_DIR = NOTEBOOK_DIR.parent
OUTPUT_DIR.mkdir(exist_ok=True)

OUTPUT_DIR
if (slurm_tmpdir := os.getenv("SLURM_TMPDIR")) is not None:
    os.environ["TMPDIR"] = slurm_tmpdir
    
print(tempfile.gettempdir())
if "scinet" in socket.gethostname():
    CPU_COUNT = 40
else:
    CPU_COUNT = max(1, len(os.sched_getaffinity(0)))

CPU_COUNT = max(1, CPU_COUNT // 2)

CPU_COUNT
# DATASET_NAME = "elaspic-core-mutation-local"  # done
# DATASET_NAME = "uniprot-domain-mutation"  # done
# DATASET_NAME = "elaspic-interface-mutation-local"  # done
DATASET_NAME = "uniprot-domain-pair-mutation"  # done
if any(q in DATASET_NAME for q in ["interface", "pair"]):
    DATASET_DIR = OUTPUT_DIR.joinpath("20_el2_affinity", DATASET_NAME)
else:
    DATASET_DIR = OUTPUT_DIR.joinpath("20_el2_stability", DATASET_NAME)
DATASET_ARRAYS = {
    "elaspic-core-mutation-local": 26,
    "uniprot-domain-mutation": 2141,
    "elaspic-interface-mutation-local": 9,
    "uniprot-domain-pair-mutation": 1358,
}<jupyter_output><empty_output><jupyter_text>Workspace  Find missing tasks<jupyter_code>missing_tasks = []

task_count = DATASET_ARRAYS[DATASET_NAME]
for task_id in range(1, task_count + 1):
    output_file = DATASET_DIR.joinpath(f"{DATASET_NAME}-{task_id:04d}-{task_count:04d}.SUCCESS")
    if not output_file.is_file():
        missing_tasks.append(task_id)
        
print(DATASET_NAME, len(missing_tasks), ",".join((str(i) for i in missing_tasks)))
DATASET_NAME
assert not missing_tasks<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12266425557564799, 'fraction_numerical': 0.03134418324291742, 'mean_word_length': 7.114914425427873, 'pattern_counts': {'":': 4, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '92116', 'license': 'MIT', 'n_tokens_mistral': 1360, 'n_tokens_neox': 1302, 'n_words': 167, 'path': 'notebooks/20_el2_validation.ipynb', 'repo_name': 'elaspic/elaspic2', 'size': '3318'}