<jupyter_start><jupyter_text>[12/30/21] Data Diet and LR at Rewind 200<jupyter_code>import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from pathlib import Path
import seaborn as sns
plt.style.use('default')
sns.set_theme(
    style='ticks', 
    font_scale=1.1, 
    rc={'axes.grid': True, 'figure.constrained_layout.use': True, 'grid.linewidth': 0.8},
)
exp_0000_paths = [
    '/home/mansheej/open_lth_data/train_5a34cb6d43c2973a3b7d79c142998062/replicate_1',
    '/home/mansheej/open_lth_data/train_636197b3accade336da6b753dd045679/replicate_1',
    '/home/mansheej/open_lth_data/train_2553e6eeb4c5d6117cf66bf188784438/replicate_1',
    '/home/mansheej/open_lth_data/train_cbb8baeb0e6508d612f54aee60a67ca1/replicate_1',
]
exp_0002_paths = [
    '/home/mansheej/open_lth_data/lottery_552853e4489055ee2e791f4114308ed8/replicate_1',
    '/home/mansheej/open_lth_data/lottery_483f1e5e0e4ba3d7e9613e2373a8a7a7/replicate_1',
    '/home/mansheej/open_lth_data/lottery_0af95436e66f8d252d8224b5d7f5fe22/replicate_1',
    '/home/mansheej/open_lth_data/lottery_4756c01f8724a4aef59a37c1be23e16b/replicate_1',
]
exp_0003_paths = [
    '/home/mansheej/open_lth_data/lottery_89f530202cd01be92b5c6a5223869a21/replicate_1',
    '/home/mansheej/open_lth_data/lottery_f98203f5d288abf1fd4c05f62a9259ac/replicate_1',
    '/home/mansheej/open_lth_data/lottery_bb608172f6cad6f3c6edb8c2b17872e8/replicate_1',
    '/home/mansheej/open_lth_data/lottery_c5e78ab5da3cf5960708a2511eb2d9ac/replicate_1',
]
exp_0004_paths = [
    '/home/mansheej/open_lth_data/lottery_ebcfc5850c4a261e8512b1f3119d3084/replicate_1',
    '/home/mansheej/open_lth_data/lottery_52adbeeffb94cc7c439cb05dc8105efa/replicate_1',
    '/home/mansheej/open_lth_data/lottery_36659649aac62013577a2270a7a5224a/replicate_1',
    '/home/mansheej/open_lth_data/lottery_4e928f04467dd6d114fd27966875317f/replicate_1',
]
exp_0005_paths = [
    '/home/mansheej/open_lth_data/lottery_e53e8295f978524fc33019fbddd5deb1/replicate_1',
    '/home/mansheej/open_lth_data/lottery_8d3e1146c72dbb4c9f6babd3e6a87cd3/replicate_1',
    '/home/mansheej/open_lth_data/lottery_3c540fc2879ded4648e688bcd0665469/replicate_1',
    '/home/mansheej/open_lth_data/lottery_2890a5dbcb90d65ec56e44a906030114/replicate_1',
]
exp_0007_paths = [
    '/home/mansheej/open_lth_data/lottery_0d19e9b22e7bf14fb959ce61cf289681/replicate_1',
    '/home/mansheej/open_lth_data/lottery_bcbd3556e03f94c178dca05aae7831f1/replicate_1',
    '/home/mansheej/open_lth_data/lottery_fd36b1855b2f4587fb9c363a67ddb36e/replicate_1',
    '/home/mansheej/open_lth_data/lottery_7d3151728ed8838affb54ab23e6b317e/replicate_1',
]
exp_0010_paths = [
    '/home/mansheej/open_lth_data/lottery_70872dc82c3dd4c3a16a83c43d8aba5a/replicate_1',
    '/home/mansheej/open_lth_data/lottery_072a61de6128444d0d461d6369b9efc4/replicate_1',
    '/home/mansheej/open_lth_data/lottery_ec7ad021bfd68e1a8435f87b40d8484f/replicate_1',
    '/home/mansheej/open_lth_data/lottery_6c529618055d4e7fe94e1607322be062/replicate_1',
]
exp_0011_paths = [
    '/home/mansheej/open_lth_data/lottery_1aedd32afa467e1ce0bf7b15649c1677/replicate_1',
    '/home/mansheej/open_lth_data/lottery_3b652a65d59a0208c22b4503de6b66a9/replicate_1',
    '/home/mansheej/open_lth_data/lottery_1b33c7f5fcdd880ac2cb88477ea5b49b/replicate_1',
    '/home/mansheej/open_lth_data/lottery_d2e623663d5b21de1183986ae5f7fff5/replicate_1',
]
exp_0012_paths = [
    '/home/mansheej/open_lth_data/lottery_0afbcc380fc0692cdf6473d9cb3ee9d5/replicate_1',
    '/home/mansheej/open_lth_data/lottery_ffe2b29c04e821986a21bce056def080/replicate_1',
    '/home/mansheej/open_lth_data/lottery_9ad494882674a1fcc9328828ca5ce1a2/replicate_1',
    '/home/mansheej/open_lth_data/lottery_e92876f126f585e5862518336778c371/replicate_1',
]
exp_0013_paths = [
    '/home/mansheej/open_lth_data/lottery_1d9a2a869f79f5aeed26732b157c643c/replicate_1',
    '/home/mansheej/open_lth_data/lottery_ef77c24828e1b43b3e5f9b71e30df51d/replicate_1',
    '/home/mansheej/open_lth_data/lottery_3cbf3bbaf48c0ce8f8101b5ed62dc7ef/replicate_1',
    '/home/mansheej/open_lth_data/lottery_51541e587fa9aa6573aed133594836da/replicate_1',
]
exp_0014_paths = [
    '/home/mansheej/open_lth_data/lottery_596ab20298165c3f9c36cd0d18ee47be/replicate_1',
    '/home/mansheej/open_lth_data/lottery_7e9228e12de8ec0b4f4fff77b53ef0eb/replicate_1',
    '/home/mansheej/open_lth_data/lottery_53f7c36ae2748072b60aecc2776b084d/replicate_1',
    '/home/mansheej/open_lth_data/lottery_61486fa8aa327636a2856caa7c2b0975/replicate_1',
]
exp_0015_paths = [
    '/home/mansheej/open_lth_data/lottery_5db8fe68e13e06852a37bc2c66ee601a/replicate_1',
    '/home/mansheej/open_lth_data/lottery_33ea4bd5c73c15afb27491a77a594f1d/replicate_1',
    '/home/mansheej/open_lth_data/lottery_6d7ce23016649431ba7dc70a888f6abc/replicate_1',
    '/home/mansheej/open_lth_data/lottery_c6d65d469a40f758c798beb046967124/replicate_1',
]
exp_0016_paths = [
    '/home/mansheej/open_lth_data/lottery_5b4860cd3b30950ef909b301d78a2910/replicate_1',
    '/home/mansheej/open_lth_data/lottery_bc44470dc5786c59485a087ec6b955d1/replicate_1',
    '/home/mansheej/open_lth_data/lottery_a1591b6b2d64dfd9917c8923a66587fd/replicate_1',
    '/home/mansheej/open_lth_data/lottery_855e34773f13e67fcd5057f240832253/replicate_1',
]
exp_0000_paths = [Path(p) for p in exp_0000_paths]
exp_0002_paths = [Path(p) for p in exp_0002_paths]
exp_0003_paths = [Path(p) for p in exp_0003_paths]
exp_0004_paths = [Path(p) for p in exp_0004_paths]
exp_0005_paths = [Path(p) for p in exp_0005_paths]
exp_0007_paths = [Path(p) for p in exp_0007_paths]
exp_0010_paths = [Path(p) for p in exp_0010_paths]
exp_0011_paths = [Path(p) for p in exp_0011_paths]
exp_0012_paths = [Path(p) for p in exp_0012_paths]
exp_0013_paths = [Path(p) for p in exp_0013_paths]
exp_0014_paths = [Path(p) for p in exp_0014_paths]
exp_0015_paths = [Path(p) for p in exp_0015_paths]
exp_0016_paths = [Path(p) for p in exp_0016_paths]
plt.figure(figsize=(8, 4.8))
ls = []

acc_run_step = []
for p in exp_0000_paths:
    df = pd.read_csv(p / 'main/logger', header=None)
    mask = df[0] == 'test_accuracy'
    acc_step = df[2][mask].to_numpy()
    acc_run_step.append(acc_step)
acc_run_step = np.stack(acc_run_step)
ys = acc_run_step[:, -1]
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C7'
l = plt.plot([0, 15], [y_mean] * 2, c=c)
ls.append(l[0])
plt.fill_between(
    [0, 15], [y_mean - y_std] * 2, [y_mean + y_std] * 2, color=c, alpha=0.2
)

num_levels = 15
acc_run_level = []
for p in exp_0010_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C0'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

num_levels = 15
acc_run_level = []
for p in exp_0011_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C1'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

num_levels = 15
acc_run_level = []
for p in exp_0012_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C2'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

plt.legend(
    ls, 
    [
        'Vanilla', 
        'Pretrain Step 200 Offset 0 (CB)',
        'Pretrain Step 200 Offset 5000 (CB)', 
        'Pretrain Step 200 Offset 10000 (CB)', 
    ], 
    fontsize='small',
)
plt.xlim(0, 15)
plt.ylim(0.815, 0.925)
plt.xticks(np.arange(0, 16, 2), [f'{f*100:.1f}' for f in 0.8**np.arange(0, 16, 2)])
plt.xlabel('% Weights Remaining')
plt.ylabel('Test Accuracy')
plt.savefig('/home/mansheej/open_lth/figs/0002.svg')
plt.show()
plt.figure(figsize=(8, 4.8))
ls = []

acc_run_step = []
for p in exp_0000_paths:
    df = pd.read_csv(p / 'main/logger', header=None)
    mask = df[0] == 'test_accuracy'
    acc_step = df[2][mask].to_numpy()
    acc_run_step.append(acc_step)
acc_run_step = np.stack(acc_run_step)
ys = acc_run_step[:, -1]
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C7'
l = plt.plot([0, 15], [y_mean] * 2, c=c)
ls.append(l[0])
plt.fill_between(
    [0, 15], [y_mean - y_std] * 2, [y_mean + y_std] * 2, color=c, alpha=0.2
)

num_levels = 15
acc_run_level = []
for p in exp_0004_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C2'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

num_levels = 15
acc_run_level = []
for p in exp_0003_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C1'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

num_levels = 15
acc_run_level = []
for p in exp_0002_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C4'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

num_levels = 15
acc_run_level = []
for p in exp_0010_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C0'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

plt.legend(
    ls, 
    [
        'Vanilla', 
        'Pretrain Step 500',
        'Pretrain Step 250', 
        'Pretrain Step 200 Random Examples',
        'Pretrain Step 200 Smallest Error Norm Scores at 1000 it', 
    ], 
    fontsize='small',
)
plt.xlim(0, 15)
plt.ylim(0.815, 0.925)
plt.xticks(np.arange(0, 16, 2), [f'{f*100:.1f}' for f in 0.8**np.arange(0, 16, 2)])
plt.xlabel('% Weights Remaining')
plt.ylabel('Test Accuracy')
plt.savefig('/home/mansheej/open_lth/figs/0003.svg')
plt.show()
plt.figure(figsize=(8, 4.8))
ls = []

acc_run_step = []
for p in exp_0000_paths:
    df = pd.read_csv(p / 'main/logger', header=None)
    mask = df[0] == 'test_accuracy'
    acc_step = df[2][mask].to_numpy()
    acc_run_step.append(acc_step)
acc_run_step = np.stack(acc_run_step)
ys = acc_run_step[:, -1]
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C7'
l = plt.plot([0, 15], [y_mean] * 2, c=c)
ls.append(l[0])
plt.fill_between(
    [0, 15], [y_mean - y_std] * 2, [y_mean + y_std] * 2, color=c, alpha=0.2
)

num_levels = 15
acc_run_level = []
for p in exp_0004_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C2'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

num_levels = 15
acc_run_level = []
for p in exp_0003_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C0'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

num_levels = 15
acc_run_level = []
for p in exp_0013_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C1'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

plt.legend(
    ls, 
    [
        'Vanilla', 
        'Pretrain Step 500',
        'Pretrain Step 250 Random Examples', 
        'Pretrain Step 250 Smalles Error Norm Scores at 1000 it',
    ], 
    fontsize='small',
)
plt.xlim(0, 15)
plt.ylim(0.815, 0.925)
plt.xticks(np.arange(0, 16, 2), [f'{f*100:.1f}' for f in 0.8**np.arange(0, 16, 2)])
plt.xlabel('% Weights Remaining')
plt.ylabel('Test Accuracy')
plt.savefig('/home/mansheej/open_lth/figs/0004.svg')
plt.show()
plt.figure(figsize=(8, 4.8))
ls = []

acc_run_step = []
for p in exp_0000_paths:
    df = pd.read_csv(p / 'main/logger', header=None)
    mask = df[0] == 'test_accuracy'
    acc_step = df[2][mask].to_numpy()
    acc_run_step.append(acc_step)
acc_run_step = np.stack(acc_run_step)
ys = acc_run_step[:, -1]
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C7'
l = plt.plot([0, 15], [y_mean] * 2, c=c)
ls.append(l[0])
plt.fill_between(
    [0, 15], [y_mean - y_std] * 2, [y_mean + y_std] * 2, color=c, alpha=0.2
)

num_levels = 15
acc_run_level = []
for p in exp_0014_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C0'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

num_levels = 15
acc_run_level = []
for p in exp_0015_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C1'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

num_levels = 15
acc_run_level = []
for p in exp_0016_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C2'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

plt.legend(
    ls, 
    [
        'Vanilla',
        'Pretrain Step 200 LR 0.2 Smallest Scores at 1000 it',
        'Pretrain Step 200 LR 0.4 Smallest Scores at 1000 it',
        'Pretrain Step 200 LR 0.8 Smallest Scores at 1000 it',
    ], 
    fontsize='small',
)
plt.xlim(0, 15)
plt.ylim(0.815, 0.925)
plt.xticks(np.arange(0, 16, 2), [f'{f*100:.1f}' for f in 0.8**np.arange(0, 16, 2)])
plt.xlabel('% Weights Remaining')
plt.ylabel('Test Accuracy')
plt.savefig('/home/mansheej/open_lth/figs/0005.svg')
plt.show()
plt.figure(figsize=(8, 4.8))
ls = []

acc_run_step = []
for p in exp_0000_paths:
    df = pd.read_csv(p / 'main/logger', header=None)
    mask = df[0] == 'test_accuracy'
    acc_step = df[2][mask].to_numpy()
    acc_run_step.append(acc_step)
acc_run_step = np.stack(acc_run_step)
ys = acc_run_step[:, -1]
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C7'
l = plt.plot([0, 15], [y_mean] * 2, c=c)
ls.append(l[0])
plt.fill_between(
    [0, 15], [y_mean - y_std] * 2, [y_mean + y_std] * 2, color=c, alpha=0.2
)

num_levels = 15
acc_run_level = []
for p in exp_0005_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C4'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

num_levels = 15
acc_run_level = []
for p in exp_0004_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C2'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

num_levels = 15
acc_run_level = []
for p in exp_0002_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C3'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

num_levels = 15
acc_run_level = []
for p in exp_0007_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C0'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

num_levels = 15
acc_run_level = []
for p in exp_0014_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C1'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

plt.legend(
    ls, 
    [
        'Vanilla',
        'Pretrain Step 1000 LR 0.1 Random Examples',
        'Pretrain Step 500 LR 0.1 Random Examples',
        'Pretrain Step 200 LR 0.1 Random Examples',
        'Pretrain Step 200 LR 0.2 Random Examples',
        'Pretrain Step 200 LR 0.2 Smallest Error Norm Scores at 1000 it',
    ], 
    fontsize='small',
)

plt.xlim(0, 15)
plt.ylim(0.815, 0.925)
plt.xticks(np.arange(0, 16, 2), [f'{f*100:.1f}' for f in 0.8**np.arange(0, 16, 2)])
plt.xlabel('% Weights Remaining')
plt.ylabel('Test Accuracy')
plt.savefig('/home/mansheej/open_lth/figs/0006.svg')
plt.show()
plt.figure(figsize=(8, 4.8))
ls = []

acc_run_step = []
for p in exp_0000_paths:
    df = pd.read_csv(p / 'main/logger', header=None)
    mask = df[0] == 'test_accuracy'
    acc_step = df[2][mask].to_numpy()
    acc_run_step.append(acc_step)
acc_run_step = np.stack(acc_run_step)
ys = acc_run_step[:, -1]
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C7'
l = plt.plot([0, 15], [y_mean] * 2, c=c)
ls.append(l[0])
plt.fill_between(
    [0, 15], [y_mean - y_std] * 2, [y_mean + y_std] * 2, color=c, alpha=0.2
)

num_levels = 15
acc_run_level = []
for p in exp_0005_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C4'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

num_levels = 15
acc_run_level = []
for p in exp_0004_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C2'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

num_levels = 15
acc_run_level = []
for p in exp_0002_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C3'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

# num_levels = 15
# acc_run_level = []
# for p in exp_0007_paths:
#     acc_level = []
#     for l in range(num_levels + 1):
#         df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
#         acc_level.append(df[2].iloc[-2])
#     acc_run_level.append(acc_level)
# acc_run_level = np.array(acc_run_level)
# x = np.arange(16)
# ys = acc_run_level
# y_mean, y_std = ys.mean(0), ys.std(0)
# c = 'C0'
# l = plt.plot(x, y_mean, c=c)
# ls.append(l[0])
# plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

num_levels = 15
acc_run_level = []
for p in exp_0014_paths:
    acc_level = []
    for l in range(num_levels + 1):
        df = pd.read_csv(p / f'level_{l}/main/logger', header=None)
        acc_level.append(df[2].iloc[-2])
    acc_run_level.append(acc_level)
acc_run_level = np.array(acc_run_level)
x = np.arange(16)
ys = acc_run_level
y_mean, y_std = ys.mean(0), ys.std(0)
c = 'C1'
l = plt.plot(x, y_mean, c=c)
ls.append(l[0])
plt.fill_between(x, y_mean + y_std, y_mean - y_std, color=c, alpha=0.2)

plt.legend(
    ls, 
    [
        'Vanilla',
        'Pretrain Step 1000 LR 0.1 Random Examples',
        'Pretrain Step 500 LR 0.1 Random Examples',
        'Pretrain Step 200 LR 0.1 Random Examples',
        # 'Pretrain Step 200 LR 0.2 Random Examples',
        'Pretrain Step 200 LR 0.2 Smallest Error Norm Scores at 1000 it',
    ], 
    loc=3, 
    fontsize='small',
)

plt.xlim(0, 15)
plt.ylim(0.815, 0.925)
plt.xticks(np.arange(0, 16, 2), [f'{f*100:.1f}' for f in 0.8**np.arange(0, 16, 2)])
plt.xlabel('% Weights Remaining')
plt.ylabel('Test Accuracy')
plt.savefig('/home/mansheej/open_lth/figs/0006f.svg')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15310159077518729, 'fraction_numerical': 0.08947058328423534, 'mean_word_length': 4.151311510947322, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '525398', 'license': 'MIT', 'n_tokens_mistral': 12123, 'n_tokens_neox': 10865, 'n_words': 1769, 'path': 'nbs/21_12_30__Data_Diet_and_LR_at_Rewind_200.ipynb', 'repo_name': 'mansheej/open_lth', 'size': '23762'}