<jupyter_start><jupyter_text>Serial spawningI'll need to explain the data layout a bit...  This is using 2D chunked arrays.  Each nodelet has a chunk of consecutive memory.  Every nodelet has a pointer to each nodelet's local memory.  On the Emu platform, *all* pointers are global.```c// define INDEX(PTR, BLOCK, I) (PTR[I/BLOCK][I%BLOCK])define INDEX(PTR, BLOCK, I) (PTR[I >> PRIORITY(BLOCK)][I&(BLOCK-1)])noinline voidserial_spawn_add_worker(long begin, long end, global_stream_data *data){    long block_sz = data->n / NODELETS();    for (long i = begin; i < end; ++i) {        INDEX(data->c, block_sz, i) = INDEX(data->a, block_sz, i) + INDEX(data->b, block_sz, i);    }}// serial_spawn - spawn one thread to handle each grain-sized chunk of the rangevoidglobal_stream_add_serial_spawn(global_stream_data * data){    long grain = data->n / data->num_threads;    for (long i = 0; i n; i += grain) {        long begin = i;        long end = begin + grain n ? begin + grain : data->n;        cilk_spawn serial_spawn_add_worker(begin, end, data);    }    cilk_sync;}```But first, some boring building...<jupyter_code>%%bash
mkdir build
cd build
cmake ../emu-microbench -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=../emu-microbench/cmake/emu-toolchain.cmake
make global_stream
%%bash
cp build/global_stream.mwx .
mkdir serial_data
mkdir serial_remote_data
mkdir recursive_remote_data<jupyter_output><empty_output><jupyter_text>Now the actual running, using small parameters for emulation<jupyter_code>!cd serial_data && emusim.x --capture_timing_queues -- ../global_stream.mwx serial_spawn 12 512 1
!cd serial_data && make_cdc_plots.py global_stream.cdc && make_tqd_plots.py global_stream.tqd
from IPython.display import Image, display
display(Image(filename="serial_data/global_stream_migration_map.png"))
display(Image(filename="serial_data/global_stream_local_memops.png"))
display(Image(filename="serial_data/global_stream_remotes_map.png"))
display(Image(filename="serial_data/global_stream_NLET_REGSETS_USED.png"))<jupyter_output><empty_output><jupyter_text>Remote spawningNote, this decodes the data earlier...```cnoinline voidserial_remote_spawn_level2(long begin, long end, long * a, long * b, long * c){    for (long i = begin; i < end; ++i) {        c[i] = a[i] + b[i];    }}noinline voidserial_remote_spawn_level1(long * a, long * b, long * c, long n, long grain){    for (long i = 0; i < n; i += grain) {        long begin = i;        long end = begin + grain <= n ? begin + grain : n;        cilk_spawn serial_remote_spawn_level2(begin, end, a, b, c);    }    cilk_sync;}// serial_remote_spawn - remote spawn a thread on each nodelet, then do a serial spawn locallyvoidglobal_stream_add_serial_remote_spawn(global_stream_data * data){    // Each thread will be responsible for the elements on one nodelet    long local_n = data->n / NODELETS();    // Calculate the grain so we get the right number of threads globally    long grain = data->n / data->num_threads;    // Spawn a thread on each nodelet    for (long i = 0; i < NODELETS(); ++i) {        cilk_spawn_at(data->a[i]) serial_remote_spawn_level1(data->a[i], data->b[i], data->c[i], local_n, grain);    }    cilk_sync;}```<jupyter_code>!cd serial_remote_data && emusim.x --capture_timing_queues -- ../global_stream.mwx serial_remote_spawn 12 512 1
!cd serial_remote_data && make_cdc_plots.py global_stream.cdc && make_tqd_plots.py global_stream.tqd
display(Image(filename="serial_remote_data/global_stream_migration_map.png"))
display(Image(filename="serial_remote_data/global_stream_local_memops.png"))
display(Image(filename="serial_remote_data/global_stream_remotes_map.png"))
display(Image(filename="serial_remote_data/global_stream_NLET_REGSETS_USED.png"))<jupyter_output><empty_output><jupyter_text>*Recursive* remote spawning```cnoinline voidrecursive_remote_spawn_level2_worker(long begin, long end, long * a, long * b, long * c){    for (long i = begin; i < end; ++i) {        c[i] = a[i] + b[i];    }}define RECURSIVE_CILK_SPAWN(BEGIN, END, GRAIN, FUNC, ...)          \do {                                                                \    long low = BEGIN;                                               \    long high = END;                                                \                                                                    \    for (;;) {                                                      \        /* How many elements in my range? */                        \        long count = high - low;                                    \                                                                    \        /* Break out when my range is smaller than the grain size */\        if (count <= GRAIN) break;                                  \                                                                    \        /* Divide the range in half */                              \        /* Invariant: count >= 2 */                                 \        long mid = low + count / 2;                                 \                                                                    \        /* Spawn a thread to deal with the lower half */            \        cilk_spawn FUNC(low, mid, GRAIN, __VA_ARGS__);              \                                                                    \        low = mid;                                                  \    }                                                               \                                                                    \    /* Recursive base case: call worker function */                 \    FUNC  _worker(low, high, __VA_ARGS__);                        \} while (0)noinline voidrecursive_remote_spawn_level2(long begin, long end, long grain, long * a, long * b, long * c){    RECURSIVE_CILK_SPAWN(begin, end, grain, recursive_remote_spawn_level2, a, b, c);}noinline voidrecursive_remote_spawn_level1(long low, long high, global_stream_data * data){    for (;;) {        long count = high - low;        if (count == 1) break;        long mid = low + count / 2;        cilk_spawn_at(data->a[low]) recursive_remote_spawn_level1(low, mid, data);        low = mid;    }    /* Recursive base case: call worker function */    long local_n = data->n / NODELETS();    long grain = data->n / data->num_threads;    recursive_remote_spawn_level2(0, local_n, grain, data->a[low], data->b[low], data->c[low]);}// recursive_remote_spawn - Recursively spawns threads to divice up the loop range, using remote spawns where possible.voidglobal_stream_add_recursive_remote_spawn(global_stream_data * data){    recursive_remote_spawn_level1(0, NODELETS(), data);}```<jupyter_code>!cd recursive_remote_data && emusim.x --capture_timing_queues -- ../global_stream.mwx recursive_remote_spawn 12 512 1
!cd recursive_remote_data && make_cdc_plots.py global_stream.cdc && make_tqd_plots.py global_stream.tqd
display(Image(filename="recursive_remote_data/global_stream_migration_map.png"))
display(Image(filename="recursive_remote_data/global_stream_local_memops.png"))
display(Image(filename="recursive_remote_data/global_stream_remotes_map.png"))
display(Image(filename="recursive_remote_data/global_stream_NLET_REGSETS_USED.png"))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10675713098864581, 'fraction_numerical': 0.005677097756854057, 'mean_word_length': 2.395862717442407, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 49, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '121148', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 2196, 'n_tokens_neox': 2105, 'n_words': 523, 'path': 'code/05-stream-example/NB-05-stream-emu-chick.ipynb', 'repo_name': 'gt-crnch-rg/pearc-tutorial-2021', 'size': '7222'}