<jupyter_start><jupyter_text>Dask Adaptive Scaling<jupyter_code>from dask_jobqueue import PBSCluster

import dask

dask.config.set(
    {
        'distributed.dashboard.link': 'https://jupyterhub.hpc.ucar.edu/stable/user/{USER}/proxy/{port}/status'
    }
)
cluster = PBSCluster(
    cores=2,
    processes=1,
    queue="casper",
    walltime="00:30:00",
    resource_spec="select=1:ncpus=1:mem=10GB",
)
cluster
!qstat -u abanihi -l<jupyter_output>240546.casper-* abanihi  jhublog* STDIN      217212   1   1    4gb 720:0 R 214:2
251037.casper-* abanihi  jhublog* STDIN      238092   1   1    4gb 720:0 R 137:5
279371.casper-* abanihi  htc      dask-work* 194610   1   1   10gb 00:30 R 00:00
279372.casper-* abanihi  htc      dask-work* 101838   1   1   10gb 00:30 R 00:00
279373.casper-* abanihi  htc      dask-work* 134639   1   1   10gb 00:30 R 00:00
279374.casper-* abanihi  htc      dask-work* 145193   1   1   10gb 00:30 R 00:00
279375.casper-* abanihi  htc      dask-work* 145273   1   1   10gb 00:30 R 00:00<jupyter_text>Batch Job Script<jupyter_code>print(cluster.job_script())<jupyter_output>#!/usr/bin/env bash

#PBS -N dask-worker
#PBS -q casper
#PBS -A NIOW0001
#PBS -l select=1:ncpus=1:mem=10GB
#PBS -l walltime=00:30:00
#PBS -e /glade/scratch/abanihi/
#PBS -o /glade/scratch/abanihi/

/glade/work/abanihi/opt/miniconda/envs/playground/bin/python -m distributed.cli.dask_worker tcp://10.12.206.47:40433 --nthreads 2 --memory-limit 101.51GiB --name dummy-name --nanny --death-timeout 60 --local-directory /glade/scratch/abanihi --interface ib0 --protocol tcp://<jupyter_text>Cluster Scaling APIs<jupyter_code>cluster.scale(5)
cluster.scale(jobs=1)
cluster.adapt(minimum_jobs=1, maximum_jobs=20)<jupyter_output><empty_output><jupyter_text>Connect cluster to the client<jupyter_code>from distributed import Client

client = Client(cluster)<jupyter_output><empty_output><jupyter_text>Run some computation<jupyter_code>import dask.array as da

x = da.random.random((200, 10_000, 5_000), chunks=(20, 1_000, 1_000))
x
y = x.std(axis=0)
y = y.persist()
y<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12978927203065135, 'fraction_numerical': 0.11637931034482758, 'mean_word_length': 3.9976076555023923, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '567986', 'license': 'MIT', 'n_tokens_mistral': 1041, 'n_tokens_neox': 855, 'n_words': 174, 'path': 'dask/dask-jobqueue-adaptive-scaling.ipynb', 'repo_name': 'andersy005/sandbox', 'size': '2088'}