<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Imports2&nbsp;&nbsp;Data3&nbsp;&nbsp;Diagnostic distributed3.1&nbsp;&nbsp;diagnostic distributed progressbar  Imports<jupyter_code>import numpy as np
import pandas as pd

import dask
import dask.dataframe as dd
import dask.array as da
import dask_ml
import pyarrow

print([(x.__name__,x.__version__) for x in 
       [np,pd, dask, dask_ml,pyarrow]])<jupyter_output>[('numpy', '1.18.5'), ('pandas', '1.0.5'), ('dask', '2.20.0'), ('dask_ml', '1.5.0'), ('pyarrow', '0.17.1')]<jupyter_text>Data<jupyter_code>a = da.random.normal(size=(10_000, 10_000), chunks=(1000, 1000)) # data
res = a.dot(a.T).mean(axis=0) # operation<jupyter_output>/Users/poudel/opt/miniconda3/envs/dsk/lib/python3.7/site-packages/dask/array/routines.py:272: PerformanceWarning: Increasing number of chunks by factor of 10
  axes=(left_axes, right_axes),<jupyter_text>Diagnostic distributed- https://docs.dask.org/en/latest/diagnostics-distributed.html- https://docs.dask.org/en/latest/diagnostics-local.htmlIt is typically served at http://localhost:8787/status , but may be served elsewhere if this port is taken. The address of the dashboard will be displayed if you are in a Jupyter Notebook, or can be queried from `client.scheduler_info()['services']`.  diagnostic distributed progressbar<jupyter_code># from dask.diagnostics import ProgressBar # for single machine

# a = da.random.normal(size=(2000, 2000), chunks=(1000, 1000))
# res = a.dot(a.T).mean(axis=0)
# with ProgressBar():
#     out = res.compute()
# Distributed scheduler ProgressBar

from dask.distributed import Client, progress

client = Client(processes=False)  # use dask.distributed by default
client
client.scheduler_info()['services']
res = res.persist()  # start computation in the background
progress(res)      # watch progress
res.compute()      # convert to final result when done if desired<jupyter_output>distributed.scheduler - ERROR - Couldn't gather keys {"('mean_agg-aggregate-8a3aa5d5a7c11071fd31b41c9dfb8b0c', 0)": ['inproc://192.168.0.188/1007/3'], "('mean_agg-aggregate-8a3aa5d5a7c11071fd31b41c9dfb8b0c', 6)": ['inproc://192.168.0.188/1007/3'], "('mean_agg-aggregate-8a3aa5d5a7c11071fd31b41c9dfb8b0c', 9)": ['inproc://192.168.0.188/1007/3'], "('mean_agg-aggregate-8a3aa5d5a7c11071fd31b41c9dfb8b0c', 2)": ['inproc://192.168.0.188/1007/3'], "('mean_agg-aggregate-8a3aa5d5a7c11071fd31b41c9dfb8b0c', 3)": ['inproc://192.168.0.188/1007/3'], "('mean_agg-aggregate-8a3aa5d5a7c11071fd31b41c9dfb8b0c', 7)": ['inproc://192.168.0.188/1007/3'], "('mean_agg-aggregate-8a3aa5d5a7c11071fd31b41c9dfb8b0c', 8)": ['inproc://192.168.0.188/1007/3'], "('mean_agg-aggregate-8a3aa5d5a7c11071fd31b41c9dfb8b0c', 4)": ['inproc://192.168.0.188/1007/3'], "('mean_agg-aggregate-8a3aa5d5a7c11071fd31b41c9dfb8b0c', 5)": ['inproc://192.168.0.188/1007/3']} state: ['memory', 'memory', 'memory', 'memory', 'memory', 'memory', 'memo[...]
{'fraction_non_alphanumeric': 0.1699212058924289, 'fraction_numerical': 0.1260705721137376, 'mean_word_length': 6.087378640776699, 'pattern_counts': {'":': 9, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 9, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '186276', 'license': 'Apache-2.0', 'n_tokens_mistral': 1352, 'n_tokens_neox': 1154, 'n_words': 197, 'path': 'a03_Dask/notebooks/dask_diagnostic_distributed.ipynb', 'repo_name': 'mindis/Big_Data_Analysis', 'size': '2919'}