<jupyter_start><jupyter_text>Hessian<jupyter_code>untrained = result_df.query("step == @result_df['step'].min()")
trained = result_df.query("step == @result_df['step'].max()")
x = 'N_del/h'
y = 'train_loss'

plt.scatter(untrained[x], untrained[y])
plt.scatter(trained[x], trained[y])

for step in  sorted(result_df['step'].unique()):
    df = result_df.query("step == @step")
    # Row with minimum value of N_del/N where train loss is non-zero and N_del/N >= 1 (underparameterized)
    row = df.query('(train_loss > 5e-2) and (N_del/h >= 1)').sort_values('N_del/h').iloc[0]
    plt.scatter(row[x], row[y], c='k')
    
plt.yscale('log')
plt.xscale('log')
from matplotlib import cm
sm = cm.ScalarMappable(norm=norm, cmap=cmap)

steps = sorted(result_df['step'].unique(), reverse=False)
norm.autoscale(steps)

for i, step in enumerate(np.array(steps)): #[20::-1]): #[20::1]):
    df = result_df.query("step == @step")
    row = df.query('(train_loss > 5e-2) and (N_del/h >= 1)').sort_values('N_del/h').iloc[0]

    vals = np.sqrt(row.eigs0)
    hist, edges = np.histogram(np.log(vals), 'sturges', density=True)
    dx = np.mean(np.diff(edges))
    edges = np.concatenate((edges[[0]]-dx, edges[1:]/2 + edges[:-1]/2 , edges[[-1]]+dx))
    edges = np.exp(edges)
    hist = np.concatenate(([0], hist, [0]))

    plt.plot(edges, hist, c=sm.to_rgba(np.clip(step, norm.vmin, norm.vmax)), alpha=1.)
plt.colorbar(sm, label='Training Steps')

plt.xscale('symlog',linthreshx=1e-1)
plt.xlim(0, None)
plt.ylim(0, None)
plt.xlabel(r'$\sqrt{\lambda}$')
plt.ylabel(r'$P\left(\sqrt{\lambda}\right)$')
plt.title("Hessian Spectrum as a Function of Training Steps")
from matplotlib import cm
sm = cm.ScalarMappable(norm=norm, cmap=cmap, )

df = untrained.query('(N_del/h >= 1)')
losses = np.logspace(np.log10(max(5e-2, min(df.train_loss))), np.log10(df.train_loss.max()))
norm.autoscale(losses)

last_loss = None
for i, loss in enumerate(losses[-5::-1]): 
    row = df.query('(train_loss >= @loss) ').sort_values('N_del/h').iloc[0]
    vals = np.sqrt(row.eigs0)
    hist, edges = np.histogram(np.log(vals), 'sturges', density=True)
    dx = np.mean(np.diff(edges))
    edges = np.concatenate((edges[[0]]-dx, edges[1:]/2 + edges[:-1]/2 , edges[[-1]]+dx))
    edges = np.exp(edges)
    hist = np.concatenate(([0], hist, [0]))
    
    plt.plot(edges, hist, c=sm.to_rgba(np.clip(row.train_loss, norm.vmin, norm.vmax)), alpha=1.)
plt.colorbar(sm, label='Train Loss')

plt.xscale('symlog',linthreshx=1e-1)
plt.xlim(0, None)
plt.ylim(0, None)
plt.xlabel(r'$\sqrt{\lambda}$')
plt.ylabel(r'$P\left(\sqrt{\lambda}\right)$')
plt.title("Hessian Spectrum as a Function of Train Loss\nRandom Features")
from matplotlib import cm
sm = cm.ScalarMappable(norm=norm, cmap=cmap)

df = trained.query('(N_del/h >= 1)')
losses = np.logspace(np.log10(max(5e-2, min(df.train_loss))), np.log10(df.train_loss.max()))
norm.autoscale(losses)

last_loss = None
for i, loss in enumerate(losses[-5::-1]): 
    row = df.query('(train_loss >= @loss) ').sort_values('N_del/h').iloc[0]
    vals = np.sqrt(row.eigs0)
    hist, edges = np.histogram(np.log(vals), 'sturges', density=True)
    dx = np.mean(np.diff(edges))
    edges = np.concatenate((edges[[0]]-dx, edges[1:]/2 + edges[:-1]/2 , edges[[-1]]+dx))
    edges = np.exp(edges)
    hist = np.concatenate(([0], hist, [0]))

    plt.plot(edges, hist, c=sm.to_rgba(np.clip(row.train_loss, norm.vmin, norm.vmax)), alpha=1.)
plt.colorbar(sm, label='Train Loss')

plt.xscale('symlog',linthreshx=1e-1)
plt.xlim(0, None)
plt.ylim(0, None)
plt.xlabel(r'$\sqrt{\lambda}$')
plt.ylabel(r'$P\left(\sqrt{\lambda}\right)$')
plt.title("Hessian Spectrum as a Function of Train Loss\nTrained Features")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.21459343026386646, 'fraction_numerical': 0.020732364028002152, 'mean_word_length': 4.850393700787402, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '356930', 'license': 'MIT', 'n_tokens_mistral': 1591, 'n_tokens_neox': 1532, 'n_words': 260, 'path': 'code/progressive_training/L=2/plotting.ipynb', 'repo_name': 'hudson-ai/random-features-jamming', 'size': '3714'}