<jupyter_start><jupyter_text>Memory usage for FlexDriver and CPU driverThis notebook estimates the memory usage for an mlx5 driver for RDMA and compares it to the memory usage for FlexDriver.<jupyter_code>%config InlineBackend.figure_format = 'retina'

from copy import copy

import numpy as np
import pandas as pd
import plotnine as p9

from common import *
def log2(value):
    return np.ceil(np.log2(value)).astype('int')
def round_pow2(value):
    return np.left_shift(1, log2(value))
def cuckoo_hash(tag_size, value_size):
    '''Compute the size of the cuckoo hash table in bytes.'''
    num_banks = 4
    num_entries = 2 * round_pow2(value_size) / num_banks
    tag_width = log2(tag_size)
    value_width = log2(value_size)
    return num_banks * num_entries * (tag_width + value_width) / 8
def bdp(df):
    '''Compute the bandwidth-delay product.'''
    linerate = df['linerate_gbps']
    df[f'packet_rate_mpps'] = linerate / (df['min_packet_size_bytes'] + 20) * 125
    for dir in ('tx', 'rx'):
        latency = df[f'latency_{dir}_usec']
        df[f'{dir}_bdp_bytes'] = np.ceil(linerate * latency * 125)
        df[f'{dir}_bdp_descs'] = np.ceil(df['packet_rate_mpps'] * latency)
def outstanding_buffer_fragmentation(df):
    '''Compute necessary buffer size for outstanding receive/transmit packets,
    taking fragmentation into account.'''

    # Use double the computed size for fragmentation (assumes minimal packet size >= block size)
    assert (df['fld_tx_data_block_bytes'] <= df['min_packet_size_bytes']).all()
    rx_stride = 64 # byte
    assert (rx_stride <= df['min_packet_size_bytes']).all()
    # Standard SRQ uses same computation as unshared tx buffer
    for dir in ('tx', 'rx'):
        df[f'{dir}_descs'] = df[f'{dir}_bdp_descs']
    
        # FLD
        outstanding_fragmentation_bytes = df[f'{dir}_bdp_bytes'].rename(f'{dir}_outstanding_fragmentation_bytes') * 2
        # not FLD
        outstanding_fragmentation_bytes = outstanding_fragmentation_bytes.where(df['fld'],
                                            df[f'{dir}_descs'] * df['max_packet_size_bytes'])

        df[f'{dir}_outstanding_bytes'] = np.maximum(df[f'{dir}_bdp_bytes'], outstanding_fragmentation_bytes)
def srq(df):
    '''Compute the size of the receive queue with FLD and with a standard driver.'''
    df['rwqe_bytes'] = np.where(df['fld'], 32, 16)
    df['cqe_bytes'] = np.where(df['fld'], 120 / 8, 64)
    rx_descs_pow2 = round_pow2(df['rx_descs'])
    df['srq_wq_bytes'] = (rx_descs_pow2 * df['rwqe_bytes']).where(~df['fld'], 0)
    df['rcq_bytes'] = rx_descs_pow2 * df['cqe_bytes']
    df['srq_doorbell_record_bytes'] = 4
wqebb_bytes = 64
fld_wqe_bytes = 8

def txq(df):
    '''Compute the size of the transmit queues.'''

    tx_descs_pow2 = round_pow2(df['tx_descs'])
    df['twqe_bytes'] = np.where(df['fld'], fld_wqe_bytes, wqebb_bytes)
    txq_wq_bytes_fld = tx_descs_pow2 * df['twqe_bytes']
    txq_wq_bytes_no_fld = tx_descs_pow2 * df['num_tx_queues'] * df['twqe_bytes']

    df['txq_wq_trans_table_tag_size'] = tx_descs_pow2 * df['num_tx_queues']
    txq_wq_translation_table_bytes_fld = cuckoo_hash(tag_size=df['txq_wq_trans_table_tag_size'],
                                                        value_size=df['tx_descs'])
    df['txq_data_trans_table_tag_size'] = (df['tx_outstanding_bytes'] * df['num_tx_queues'] / df['fld_tx_data_block_bytes'])
    txq_data_translation_table_bytes_fld = cuckoo_hash(tag_size=df['txq_data_trans_table_tag_size'],
                                                            value_size=df['tx_outstanding_bytes'] / df['fld_tx_data_block_bytes'])

    df['txq_wq_bytes'] = txq_wq_bytes_fld.where(df['fld'], txq_wq_bytes_no_fld)
    df['txq_wq_translation_table_bytes'] = txq_wq_translation_table_bytes_fld.where(df['fld'], 0)
    df['txq_data_translation_table_bytes'] = txq_data_translation_table_bytes_fld.where(df['fld'], 0)

    df['tcq_bytes'] = tx_descs_pow2 * df['cqe_bytes']
    df['txq_doorbell_record_bytes'] = 4 * df['num_tx_queues']

metadata_fields = ['srq_wq_bytes', 'rcq_bytes', 'txq_wq_bytes', 'tcq_bytes',
                    'srq_doorbell_record_bytes', 'txq_doorbell_record_bytes',
                    'txq_wq_translation_table_bytes',
                    'txq_data_translation_table_bytes']
data_fields = [f'{dir}_outstanding_bytes' for dir in ('tx', 'rx')]

def mem_usage(df):
    '''Compute usage of the different driver components with FlexDriver and for
    a CPU driver.'''
    keys = list(df.columns)
    bdp(df)
    outstanding_buffer_fragmentation(df)
    srq(df)
    txq(df)
    df = df.set_index(keys)
    df['total_pi_bytes'] = df['txq_doorbell_record_bytes'] + df['srq_doorbell_record_bytes']
    df['total_cq_bytes'] = df['rcq_bytes'] + df['tcq_bytes']
    df['total_metadata_bytes'] = df.groupby(keys)[metadata_fields].sum().sum(axis=1)
    df['total_data_bytes'] = df.groupby(keys)[data_fields].sum().sum(axis=1)
    df['total_bytes'] = df['total_metadata_bytes'] + df['total_data_bytes']
    return df.reset_index()
# Scenarios: FLD / CPU x Linerate x No. TX queues
df = (pd.DataFrame({'fld': (False, True)})
      .merge(pd.DataFrame({'linerate_gbps': [50 << i for i in range(4)]}), how='cross')
      .merge(pd.DataFrame({'num_tx_queues': [1 << i for i in range(0, 13)]}), how='cross')
).assign(latency_tx_usec=25, latency_rx_usec=5, min_packet_size_bytes=256, max_packet_size_bytes=1 << 14, fld_tx_data_block_bytes=256)
df = mem_usage(df)

# add KiB/MiB units results
for col in df.columns:
      if col.endswith('bytes'):
            df[col.split('_bytes')[0] + '_kib'] = df[col] / (1 << 10)
            df[col.split('_bytes')[0] + '_mib'] = df[col] / (1 << 20)

interesting_rows = (df.linerate_gbps.isin((100,))) & (df.num_tx_queues == 512)
keys = ['linerate_gbps', 'fld', '<KEY>']
table_2a_cols = ['min_packet_size_bytes', 'max_packet_size_kib', 'latency_rx_usec', 'latency_tx_usec', 'packet_rate_mpps', 'tx_bdp_descs', 'rx_bdp_descs', 'tx_bdp_kib', 'rx_bdp_kib']
df[interesting_rows][keys + table_2a_cols]
table_2b_cols = ['twqe_bytes', 'rwqe_bytes', 'cqe_bytes']
df[interesting_rows][keys + table_2b_cols]
table_3_cols = [
    'txq_wq_mib', 'txq_wq_kib', 
    'txq_wq_translation_table_kib', 
    'tx_outstanding_mib', 'tx_outstanding_kib', 
    'txq_data_translation_table_kib',
    'rx_outstanding_mib', 'rx_outstanding_kib', 
    'total_cq_kib',
    'srq_wq_mib', 'srq_wq_kib', 
    'total_pi_bytes', 
    'total_mib',
    'total_kib']
df[interesting_rows][keys + table_3_cols]
# Color palette
software_color, fld_color, fpga_color = pal(3) 
def linerate_labeller(linerate):
    return f'{linerate} Gbps'

def log2_labeller(breaks):
    return [f'$2^{{{i}}}$' for i in np.log2(breaks).astype('int')]

def minor_breaks(limits):
    major = [10 ** i for i in range(0, 5)]
    res = [m * 1.5 for m in major
           if m * 1.5 < limits[1]]
    return res

limit_xcku15p_df = pd.DataFrame({'type': ['BRAM', 'URAM', 'LUTRAM'],
  'Mibit': [34.6, 36, 9.8]})
limit_xcku15p_mib = limit_xcku15p_df['Mibit'].sum() / 8

grid_color = p9.options.current_theme.themeables['panel_grid_major'].theme_element.properties['color']
ticks_color = "#000000"

df['total_mib'] = df['total_bytes'] / (2 ** 20)
df['fld_name'] = np.where(df['fld'], 'FLD', 'software')
plt = (p9.ggplot(df, p9.aes(x='num_tx_queues', y='total_mib', linetype='fld_name', color='fld_name')) +
  p9.geom_line() +
  p9.geom_hline(mapping=p9.aes(linetype=['XCKU15P limit'], color=['XCKU15P limit'], yintercept=limit_xcku15p_mib)) + 
  p9.scale_x_continuous(trans='log2', name='# transmit queues', labels=log2_labeller) +
  p9.scale_y_continuous(trans='log10', name='shared memory [MiB]',
      minor_breaks=minor_breaks) +
  p9.facet_grid('~ linerate_gbps', labeller=linerate_labeller) + 
  p9.theme(
    figure_size=(fig_width, 1),
    subplots_adjust={'top': 0.77},
    axis_ticks_minor_y=p9.element_line(ticks_color),
    panel_grid_minor_y=p9.element_line(color=grid_color, size=0.5),
  ) +
  top_legend + \
  p9.scale_color_manual(values=[fld_color, software_color, fpga_color])
)
plt
plt.save('mem_usage.pdf')
reduction = copy(df)
reduction['tx_meta_total_bytes'] = reduction['txq_wq_bytes'] + reduction['txq_wq_translation_table_bytes']
reduction['tx_data_total_bytes'] = reduction['tx_outstanding_bytes'] + reduction['txq_data_translation_table_bytes']
reduction_cols = ['tx_meta_total_bytes', 'tx_data_total_bytes', 'total_cq_bytes', 'srq_wq_bytes', 'rx_outstanding_bytes', 'total_bytes']
reduction = reduction.pivot_table(values=reduction_cols, columns=['fld'], index=['linerate_gbps', 'num_tx_queues'])

def slice_fld(df, fld):
    new_df = df.loc[:, [(col, fld) for col in reduction_cols]]
    new_df.columns = new_df.columns.droplevel(1)
    return new_df

no_fld = slice_fld(reduction, False)
fld = slice_fld(reduction, True)
ratio = no_fld / fld
ratio.columns = pd.MultiIndex.from_tuples((c, 'ratio') for c in ratio.columns)
reduction = pd.concat([reduction, ratio], axis=1)#.sort_values(by=1, axis=1)
reduction.columns.names = ['type', 'col']
reduction = reduction.sort_values(by=['type', 'col'], axis=1)
reduction[[(col, 'ratio') for col in reduction_cols]].loc[(100, 512)]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14917729105372127, 'fraction_numerical': 0.018415604227961208, 'mean_word_length': 4.808860759493671, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '12559', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 3649, 'n_tokens_neox': 3515, 'n_words': 600, 'path': 'mem_usage.ipynb', 'repo_name': 'acsl-technion/flexdriver-model', 'size': '9185'}