<jupyter_start><jupyter_text>Summary  Imports<jupyter_code>import os
import runpy
import shutil
import subprocess
from pathlib import Path

import numpy as np
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq
import yaml
pd.set_option("max_columns", 100)<jupyter_output><empty_output><jupyter_text>Parameters<jupyter_code>NOTEBOOK_PATH = Path('train_network')
NOTEBOOK_PATH
OUTPUT_PATH = Path(os.getenv('OUTPUT_DIR', NOTEBOOK_PATH.name)).resolve()
OUTPUT_PATH.mkdir(parents=True, exist_ok=True)
OUTPUT_PATH
DEBUG = "CI" not in os.environ
if DEBUG:
    assert "SLURM_JOB_ID" not in os.environ
DEBUG<jupyter_output><empty_output><jupyter_text>Train network<jupyter_code>DATAPKG_OUTPUT_PATH = Path(os.environ['DATAPKG_OUTPUT_DIR'])
DATAPKG_OUTPUT_PATH
def parse_slurm_runtime(sbatch_timelimit: str) -> float:
    runtime = 0
    multipliers = [3600, 60, 1]
    while sbatch_timelimit:
        sbatch_timelimit, _, value = sbatch_timelimit.rpartition(':')
        multipier = multipliers.pop()
        if value:
            runtime += int(value) * multipier
    return runtime

            
parse_slurm_runtime("1:20:11")
if DEBUG:
    runtime = 60  # 1m
else:
    runtime = parse_slurm_runtime(os.environ['SBATCH_TIMELIMIT']) - 1200  # Total time - 20m
runtime
from pagnn.training.dcn import Args, main
args = Args(
    root_path=OUTPUT_PATH,
    training_data_path=(
        DATAPKG_OUTPUT_PATH
        .joinpath("adjacency-net-v2", "master", "training_dataset", "adjacency_matrix.parquet")
    ),
#     training_data_cache=(
#         DATAPKG_OUTPUT_PATH
#         .joinpath("adjacency-net-v2", "master", "training_dataset", f"array_id_{Args().array_id}")
#     ),
    gpu=-1,
    verbosity=1,
    network_name=f"DCN_{OUTPUT_PATH.name}",
    num_negative_examples=63,
)

assert isinstance(args.runtime, float)
args.runtime = runtime

args
args_file = OUTPUT_PATH.joinpath("args.yaml")

with args_file.open("wt") as fout:
    yaml.dump(args.to_dict(), fout, default_flow_style=False)
model_file = OUTPUT_PATH.joinpath("model.py")

with Path("../src/model.py").open('rt') as fin, model_file.open("wt") as fout:
    sub1, sub2 = False, False
    for line in fin:
        if line == "class Custom(nn.Module):\n":
            sub1 = True
            fout.write(f"class {args.network_name}(nn.Module):\n")
        elif line == "pagnn.models.dcn.Custom = Custom\n":
            sub2 = True
            fout.write(f"pagnn.models.dcn.{args.network_name} = {args.network_name}\n")
        else:
            fout.write(line)
    assert sub1 and sub2
    
runpy.run_path(model_file.as_posix(), globals())
None
main(args)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12194203989461799, 'fraction_numerical': 0.013925479864508844, 'mean_word_length': 3.9682242990654206, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '351496', 'license': 'MIT', 'n_tokens_mistral': 977, 'n_tokens_neox': 947, 'n_words': 177, 'path': 'notebooks2/03-train_network.ipynb', 'repo_name': 'ostrokach/protein-adjacency-net', 'size': '2657'}