<jupyter_start><jupyter_text>Quick Analysis of Chunk Size Tests<jupyter_code>import pandas as pd

def find_fastest_schedule(data):
    for mat_size in data['matrix size'].unique():
        max_rate_i = data[(data['matrix size'] == mat_size)]\
                   ['Gigaentries per second'].idxmax()
        print(data.loc[max_rate_i])
        print("ratio of chunk size to mat size:", 
              data.loc[max_rate_i, 'chunk size']/data.loc[max_rate_i, 'matrix size'],
              '\n')
print('outer row loop')
df1 = pd.read_csv('./chunk-size-tests-01-outer-row-loop.csv')
find_fastest_schedule(df1)
print('\n\ninner row loop')
df2 = pd.read_csv('./chunk-size-tests-02-inner-row-loop.csv')
find_fastest_schedule(df2)
print('\n\ncollapsed for loops')
df3 = pd.read_csv('./chunk-size-tests-03-collapsed-for-loops.csv')
find_fastest_schedule(df3)

import matplotlib.pyplot as plt
%matplotlib inline

df = pd.read_csv('./chunk-size-tests-03-collapsed-for-loops.csv')
fig = plt.figure(figsize=(10,5))
schedules = ['static', 'dynamic', 'guided']
mat_size = 2048
for schedule in schedules:
    plt.plot(df[(df['schedule type'] == schedule) & (df['matrix size'] == mat_size)]['chunk size'], 
             df[(df['schedule type'] == schedule) & (df['matrix size'] == mat_size)]['Gigaentries per second'],
             marker='o')
plt.legend(schedules)
plt.title('performance per chunk size of various schedule types for matrix size ' + str(mat_size))
plt.xlabel('chunk size')
plt.ylabel('gigaentries per second')
plt.yscale('log')
plt.xscale('log')
plt.show()<jupyter_output><empty_output><jupyter_text>ResultsCollpase(1) with static scheduling and a small chunk size seems to be the best. Let's explore more chunk sizes around 16<jupyter_code>print('chunk sizes near 16')
df = pd.read_csv('./chunk-size-tests-04-chunk-size-near-16.csv')
find_fastest_schedule(df)<jupyter_output>chunk sizes near 16
schedule type               static
chunk size                      22
matrix size                    256
time (ms)                  1.13805
Gigaentries per second    0.005759
Name: 12462, dtype: object
ratio of chunk size to mat size: 0.0859375 

schedule type               static
chunk size                      21
matrix size                    512
time (ms)                  3.82318
Gigaentries per second    0.006857
Name: 12522, dtype: object
ratio of chunk size to mat size: 0.041015625 

schedule type               static
chunk size                      18
matrix size                   1024
time (ms)                  20.6195
Gigaentries per second    0.005085
Name: 868, dtype: object
ratio of chunk size to mat size: 0.017578125 

schedule type              static
chunk size                     29
matrix size                  2048
time (ms)                 230.436
Gigaentries per second    0.00182
Name: 2404, dtype: object
ratio of chunk size to mat size: 0.014160[...]<jupyter_text>Graph of Above Results<jupyter_code>import numpy as np

fig, axes = plt.subplots(2, 2, figsize=(15,10))

i = 0
j = 0
for mat_size in df['matrix size'].unique():
    df2 = df[df['matrix size'] == mat_size]
    
    x = df2[df2['schedule type'] == 'static']['chunk size']
    y = df2[df2['schedule type'] == 'static']['Gigaentries per second']
    axes[i][j].scatter(x, y, alpha=0.2)
    axes[i][j].set_title("Gigaentires per second for matrix of size" + str(mat_size))
    axes[i][j].set_xticks(np.arange(64, step=4))
    axes[i][j].set_xlabel("chunk size")
    axes[i][j].set_ylabel("gigaentries per second")
    
    i += 1
    if i >= 2:
        i = 0
        j += 1
        
plt.show()<jupyter_output><empty_output><jupyter_text>Sequential vs Parallel vs Sequential-Vectorized vs Parallel-Vectorized<jupyter_code>seq = pd.read_csv('./speedtest-sequential.csv')
par = pd.read_csv('./speedtest-parallel.csv')
seq_vec = pd.read_csv('./speedtest-sequential-vectorized.csv')
par_vec = pd.read_csv('./speedtest-parallel-vectorized.csv')

fig = plt.figure(figsize=(10,5))
plt.plot(seq['matrix size'], seq['Gigaentries per second'], marker='o')
plt.plot(par['matrix size'], par['Gigaentries per second'], marker='o')
plt.plot(seq_vec['matrix size'], seq_vec['Gigaentries per second'], marker='o')
plt.plot(par_vec['matrix size'], par_vec['Gigaentries per second'], marker='o')
plt.legend(['sequential', 'parallel', 'sequential vectorized', 'parallel vectorized'])
plt.xlabel('matrix size')
plt.ylabel('gigaentries per second')
plt.yscale('log')
plt.xscale('log')
plt.show()<jupyter_output><empty_output><jupyter_text>Final Performance<jupyter_code>seq = pd.read_csv('./speedtest-sequential.csv')
combined = pd.read_csv('./speedtest-combined-cpu.csv')

fig = plt.figure(figsize=(10,5))
plt.plot(seq['matrix size'], seq['Gigaentries per second'], marker='o')
plt.plot(combined['matrix size'], combined['Gigaentries per second'], marker='o')
plt.legend(['sequential', 'combined'])
plt.xlabel('matrix size')
plt.ylabel('gigaentries per second')
plt.yscale('log')
plt.xscale('log')
plt.show()<jupyter_output><empty_output><jupyter_text>Speedup<jupyter_code>speedup = combined['Gigaentries per second']/seq['Gigaentries per second']

fig = plt.figure(figsize=(10,5))
plt.plot(seq['matrix size'], speedup, marker='o')
plt.xlabel('matrix size')
plt.ylabel('speedup')
plt.xscale('log')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14835061262959473, 'fraction_numerical': 0.035438265786993405, 'mean_word_length': 3.8148820326678767, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '158219', 'license': 'MIT', 'n_tokens_mistral': 1918, 'n_tokens_neox': 1754, 'n_words': 413, 'path': 'experiments/Experiment Analysis.ipynb', 'repo_name': 'paigeweber13/fast-lu-decomposition', 'size': '5305'}