<jupyter_start><jupyter_text>Run learners in job scripts  Define the learnersWe need the following variables:* `learners` a list of learners* `fnames` a list of file names, one for each learner<jupyter_code>%%writefile learners_file.py

import adaptive
from functools import partial


def h(x, offset=0):
    import numpy as np
    import random

    for _ in range(10):  # Burn some CPU time just because
        np.linalg.eig(np.random.rand(1000, 1000))

    a = 0.01
    return x + a ** 2 / (a ** 2 + (x - offset) ** 2)


offset = [i / 20 - 0.5 for i in range(20)]

combos = adaptive.utils.named_product(offset=offset)

learners = []
fnames = []

for i, combo in enumerate(combos):
    f = partial(h, offset=combo["offset"])
    learner = adaptive.Learner1D(f, bounds=(-1, 1))
    fnames.append(f"data/{combo}")
    learners.append(learner)

learner = adaptive.BalancingLearner(learners)
# Execute the previous code block and plot the learners
from learners_file import *
adaptive.notebook_extension()
learner.load(fnames)
learner.plot()<jupyter_output><empty_output><jupyter_text>Option 1, the simple wayAfter defining the `learners` and `fnames` in an file (above) we can start to run these learners.We split up all learners into seperate jobs, all you need to do is to specify how many cores per job you want.<jupyter_code>import adaptive_scheduler

def goal(learner):
    return learner.npoints > 200

run_manager = adaptive_scheduler.server_support.RunManager(
    learners_file="learners_file.py",
    goal=goal,
    cores_per_job=12,
    log_interval=30,
    save_interval=30,
)
run_manager.start()
# See the current queue with
import pandas as pd
pd.DataFrame(adaptive_scheduler.slurm.queue()).transpose()
# Read the logfiles and put it in a `pandas.DataFrame`.
# This only returns something when there are log-files to parse!
# So after `run_manager.log_interval` has passed.
run_manager.parse_log_files()
# See the database
pd.DataFrame(run_manager.get_database())
# Run this to STOP managing the database and jobs
run_manager.cancel(), run_manager.cleanup()<jupyter_output><empty_output><jupyter_text>Option 2, the manual way The `adaptive_scheduler.server_support.RunManager` above essentially does everything we do below.  The Python script that is run on the nodes<jupyter_code># Make sure to use the headnode's address in the next cell
from adaptive_scheduler import server_support
server_support.get_allowed_url()
%%writefile run_learner.py

import adaptive
from adaptive_scheduler import client_support
from mpi4py.futures import MPIPoolExecutor

from learners_file import learners, fnames

if __name__ == "__main__":  # ← use this, see warning @ https://bit.ly/2HAk0GG
    url = "tcp://10.75.0.5:57101"
    learner, fname = client_support.get_learner(url, learners, fnames)
    learner.load(fname)
    runner = adaptive.Runner(
        learner, executor=MPIPoolExecutor(), shutdown_executor=True, goal=None
    )
    runner.start_periodic_saving(dict(fname=fname), interval=600)
    client_support.log_info(runner, interval=600)  # log info in the job output script
    runner.ioloop.run_until_complete(runner.task)  # wait until runner goal reached
    client_support.tell_done(url, fname)<jupyter_output><empty_output><jupyter_text>Create a new database<jupyter_code>from adaptive_scheduler import server_support
from learners_file import learners, fnames

db_fname = 'running.json'
server_support.create_empty_db(db_fname, fnames)<jupyter_output><empty_output><jupyter_text>Check the running learners in the databaseAll the ones that are `None` are still `PENDING`, reached their goal, or are not scheduled.<jupyter_code>server_support.get_database(db_fname)<jupyter_output><empty_output><jupyter_text>Start the job scripts<jupyter_code>import asyncio
from adaptive_scheduler import server_support, slurm
from learners_file import learners, fnames

# create unique names for the jobs
job_names = [f"test-job-{i}" for i in range(len(learners))]

# start the "job manager" and the "database manager"
database_task = server_support.start_database_manager("tcp://10.75.0.5:57101", db_fname)

job_task = server_support.start_job_manager(
    job_names,
    db_fname=db_fname,
    cores=2,
    interval=60,
    run_script="run_learner.py",  # optional
    job_script_function=slurm.make_job_script,  # optional
)
job_task.print_stack()
database_task.print_stack()
# Run this to STOP managing the database and jobs
from adaptive_scheduler import cancel_jobs
job_task.cancel(), database_task.cancel(), cancel_jobs(job_names)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09190371991247265, 'fraction_numerical': 0.015536105032822756, 'mean_word_length': 5.4109396914446, 'pattern_counts': {'":': 1, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '235842', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1562, 'n_tokens_neox': 1466, 'n_words': 412, 'path': 'example.ipynb', 'repo_name': 'farhad-karimi/adaptive-scheduler', 'size': '4570'}