<jupyter_start><jupyter_text>Read-While-Writing Luster file systemThe test were performed with the *drpneh* Luster file system. The specs are:* 12 OSS each with 2 OSTs  (drp-neh-ffb101 - drp-neh-ffb112)* each ost: 8 SSDs in raidz1 with total size of 2.8TB.* SSDs: 480GB INTEL SSDSC2BB48* Lustre version on OSS: lustre-2.12.7 with zfs-0.8.6* clients are drp-neh-cmp0NN* Lustre client: lustre-client-2.12.0 except drp-neh-cmp022: lustre-client-2.12.7 TestsThe tool _~wilko/psdm/dev/psdm-testing/filesystem/rww/rww.go_ is used for the reader and writer:1) write a 100GB file (default blocksize is 1MiB)    * rww --mode write --wsize 100000 --fn /drp/neh/wktst/f12) wait a few minutes and start the readere on a different client machine    * rww --mode read --wsize 100000 --fn /drp/neh/wktst/f1     Setting Lustre cache size    % ctl set_param llite.*.max_cached_mb=16384    In order to persist is run on the above command with the _-P_ option on the MDS. It requires a remount on the client.<jupyter_code>from bokeh.io import output_notebook
from bokeh.plotting import figure, show, save
from bokeh.themes import built_in_themes
from bokeh.io import curdoc

import jmespath
from datetime import datetime, timedelta
import numpy as np
import pandas as pd
import pyarrow.feather as feather

output_notebook()
curdoc().theme = 'light_minimal'
rate_data = "rww_20210929.feather"<jupyter_output><empty_output><jupyter_text>Save the Prometheus IB rates Read from prometheus and store them in a file. Prometheus keeps the data only for 3-4 weeks.<jupyter_code>def read_test():
    import promquery as pq
    from itertools import cycle, chain
    
    tst_start = datetime(2021, 9, 29, 21, 21, 0)
    tst_dt = timedelta(seconds=1200)
    trange = pq.time_range(tst_start, tst_dt)
    
    rate_ib_out = pq.get_data_prom(
        'irate(node_infiniband_port_data_transmitted_bytes_total{job="nehdrp",instance=~"drp-neh-cmp014:9100|drp-neh-cmp013:9100"}[30s])', 
        *trange)
    rate_ib_in = pq.get_data_prom(
        'irate(node_infiniband_port_data_received_bytes_total{job="nehdrp",instance="drp-neh-cmp013:9100"}[30s])', 
        *trange)
    
    out = zip(rate_ib_out['data']['result'], cycle(["out"]))
    ind = zip(rate_ib_in['data']['result'], cycle(["in"]))

    to_mb = pow(2,20)
    df = pd.DataFrame({
            f"{r['metric']['instance'][8:14]}_{n}":
                pd.Series((np.float64(v[1])/to_mb for v in r['values']), index=(pd.Timestamp(v[0], unit='s') for v in r['values']))
            for r,n in chain(ind, out) })
    return df

do_create_file = False
if do_create_file:
    import os
    import pyarrow as pa
    import pyarrow.feather as feather

    df = read_test()
    if os.path.exists(rate_data):
        print("data file already exists", rate_data)
    else:
        table = pa.Table.from_pandas(df, preserve_index=True)
        feather.write_feather(table, rate_data)
        print("Created data file", rate_data)<jupyter_output><empty_output><jupyter_text>Read the data from a feather file<jupyter_code># read the data from file
df = feather.read_feather(rate_data)

from bokeh.models import ColumnDataSource
cds = ColumnDataSource(df)
from bokeh.models.formatters import DatetimeTickFormatter
p = figure(title="Read and write for RWW tests", x_axis_label="time", y_axis_label="rate [MiB/s]", x_axis_type='datetime', width=1000, height=600)
p.line("index", "cmp014_out", source=cds, line_color="red", legend_label="writer")
p.line("index", "cmp013_in", source=cds, line_color="blue", legend_label="reader")
p.xaxis.major_label_orientation = 3.14/4
p.xaxis.formatter = DatetimeTickFormatter(minutes="%H:%M")
p.xgrid.grid_line_color = "black"
p.ygrid.grid_line_color = "black"
p.xgrid.grid_line_alpha = .3
p.ygrid.grid_line_alpha = .3

#p.xaxis.major_label_text_font_size = "20pt"
show(p)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11905380816220432, 'fraction_numerical': 0.03769170782427866, 'mean_word_length': 4.435028248587571, 'pattern_counts': {'":': 1, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '201750', 'license': 'Apache-2.0', 'n_tokens_mistral': 1500, 'n_tokens_neox': 1379, 'n_words': 346, 'path': 'fs/read_while_write/rww_drpneh_2021.ipynb', 'repo_name': 'slac-lcls/psdm-tests', 'size': '3847'}