<jupyter_start><jupyter_text>Aug. Likelihood Sweep  cSGLD Aug. Scale v/s Prior Scale<jupyter_code>_, metrics = get_summary_metrics('deeplearn/data-aug-likelihood/e1sdwmzy',
                                 config_keys=['aug_scale', 'prior_scale', 'temperature'])
metrics
metrics.temperature.unique()
T = 1e-3
plt_data = metrics[metrics.temperature == T][['run_id', 'aug_scale', 'prior_scale', 'test/bma_acc']]
plt_data = plt_data.pivot(index='aug_scale', columns='prior_scale', values='test/bma_acc')

fig, ax = plt.subplots(figsize=(7, 7))
g = sns.heatmap(data=plt_data, ax=ax, fmt='.4f', linewidths=1.,
                annot=True, annot_kws=dict(fontsize=14),
                cbar=True, cbar_kws=dict(shrink=.5),
                yticklabels=plt_data.index,
                cmap=sns.color_palette('summer'))
g.set(xlabel=r'Prior Scale ($\sigma)$', ylabel=r'Aug. Scale ($\epsilon$)', title=rf'cSGLD at $T = {T}$ with Random Aug.')
g.set_xticklabels(g.get_xticklabels(), fontdict=dict(size=16))
g.set_yticklabels(g.get_yticklabels(), fontdict=dict(size=12))
fig.tight_layout()

# fig.savefig('csgld_aug_lik_scales_cold.png', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>SGD Aug. Scale v/s Prior Scale<jupyter_code>_, metrics = get_summary_metrics('deeplearn/data-aug-likelihood/mre7c18c',
                                 config_keys=['aug_scale', 'prior_scale'])
metrics
plt_data = metrics[['run_id', 'aug_scale', 'prior_scale', 'test/best_acc']]
plt_data = plt_data.pivot(index='aug_scale', columns='prior_scale', values='test/best_acc')

fig, ax = plt.subplots(figsize=(7, 7))
g = sns.heatmap(data=plt_data, ax=ax, fmt='.3f', linewidths=1.,
                annot=True, annot_kws=dict(fontsize=14),
                cbar=True, cbar_kws=dict(shrink=.5),
                yticklabels=plt_data.index,
                cmap=sns.color_palette('summer'))
g.set(xlabel=r'Prior Scale ($\sigma)$', ylabel=r'Aug. Scale ($\epsilon$)', title='SGD with Fixed Aug.')
g.set_xticklabels(g.get_xticklabels(), fontdict=dict(size=16))
g.set_yticklabels(g.get_yticklabels(), fontdict=dict(size=12))
fig.tight_layout()

# fig.savefig('sgd_aug_lik_scales.png', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>SGD Prior Scale with or without Random Data AugmentationThis is the usual way of doing things, just with an explicit prior.<jupyter_code>_, metrics = get_summary_metrics('deeplearn/data-aug-likelihood/zay9l79c',
                                 config_keys=['aug', 'prior_scale'])
metrics
plt_data = metrics[['run_id', 'aug', 'prior_scale', 'test/best_acc']]
plt_data = plt_data.pivot(index='aug', columns='prior_scale', values='test/best_acc')

fig, ax = plt.subplots(figsize=(7, 3))
g = sns.heatmap(data=plt_data, ax=ax, fmt='.3f', linewidths=1.,
                annot=True, annot_kws=dict(fontsize=14),
                cbar=True, cbar_kws=dict(shrink=.5),
                yticklabels=plt_data.index,
                cmap=sns.color_palette('summer'))
g.set(xlabel=r'Prior Scale ($\sigma)$', ylabel=r'Aug. (?)', title='SGD with Random Data Aug.')
g.set_xticklabels(g.get_xticklabels(), fontdict=dict(size=16))
g.set_yticklabels(g.get_yticklabels(), fontdict=dict(size=12))
fig.tight_layout()

# fig.savefig('sgd_lik_scales.png', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>SGD Prior Scale with Fixed Set of Data Augmentations<jupyter_code>_, metrics = get_summary_metrics('deeplearn/data-aug-likelihood/sjcc8rjk',
                                 config_keys=['prior_scale'])
metrics
plt_data = metrics[['run_id', 'train/epoch', 'prior_scale', 'test/best_acc']]
plt_data = plt_data.pivot(index='train/epoch', columns='prior_scale', values='test/best_acc')

fig, ax = plt.subplots(figsize=(7, 2))
g = sns.heatmap(data=plt_data, ax=ax, fmt='.3f', linewidths=1.,
                annot=True, annot_kws=dict(fontsize=14),
                cbar=True, cbar_kws=dict(shrink=.5),
                yticklabels=plt_data.index,
                cmap=sns.color_palette('summer'))
g.set(xlabel=r'Prior Scale ($\sigma)$', ylabel=r'', title='SGD with Fixed Data Aug.')
g.set_xticklabels(g.get_xticklabels(), fontdict=dict(size=16))
g.set_yticklabels([], fontdict=dict(size=12))
fig.tight_layout()

fig.savefig('sgd_lik_scales_fixed_aug.png', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>cSGLD with Fixed Set of Data Augmentations<jupyter_code>_, metrics = get_summary_metrics('deeplearn/data-aug-likelihood/o7gy31ps',
                                 config_keys=['aug', 'prior_scale', 'temperature'])
metrics
T = .5
plt_data = metrics[metrics.temperature == T][['run_id', 'aug', 'prior_scale', 'test/bma_acc']]
plt_data = plt_data.pivot(index='aug', columns='prior_scale', values='test/bma_acc')

fig, ax = plt.subplots(figsize=(7, 3))
g = sns.heatmap(data=plt_data, ax=ax, fmt='.3f', linewidths=1.,
                annot=True, annot_kws=dict(fontsize=14),
                cbar=True, cbar_kws=dict(shrink=.5),
                yticklabels=plt_data.index,
                cmap=sns.color_palette('summer'))
g.set(xlabel=r'Prior Scale ($\sigma)$', ylabel=r'Aug. Scale ($\epsilon$)', title=rf'cSGLD at $T = {T}$ with Fixed Aug.')
g.set_xticklabels(g.get_xticklabels(), fontdict=dict(size=16))
g.set_yticklabels(g.get_yticklabels(), fontdict=dict(size=12))
fig.tight_layout()

fig.savefig('csgld_fixed_aug_cold.png', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>cSGLD + Fixed Aug + FRN Layer<jupyter_code>_, metrics = get_summary_metrics('deeplearn/data-aug-likelihood/9vf05eep',
                                 config_keys=['aug', 'prior_scale', 'temperature'])
metrics
T = 1
plt_data = metrics[['run_id', 'aug', 'prior_scale', 'test/bma_acc']]
plt_data = plt_data.pivot(index='aug', columns='prior_scale', values='test/bma_acc')

fig, ax = plt.subplots(figsize=(7, 3))
g = sns.heatmap(data=plt_data, ax=ax, fmt='.3f', linewidths=1.,
                annot=True, annot_kws=dict(fontsize=14),
                cbar=True, cbar_kws=dict(shrink=.5),
                yticklabels=plt_data.index,
                cmap=sns.color_palette('summer'))
g.set(xlabel=r'Prior Scale ($\sigma)$', ylabel=r'Aug. Scale ($\epsilon$)', title=rf'cSGLD at $T = {T}$ with Fixed Aug + FRN Layer')
g.set_xticklabels(g.get_xticklabels(), fontdict=dict(size=16))
g.set_yticklabels(g.get_yticklabels(), fontdict=dict(size=12))
fig.tight_layout()

# fig.savefig('csgld_fixed_aug_frn.png', bbox_inches='tight')
_, metrics = get_summary_metrics('deeplearn/data-aug-likelihood/5lady5ar',
                                 config_keys=['aug_scale', 'prior_scale'])
metrics
T = 1
plt_data = metrics[['run_id', 'aug_scale', 'prior_scale', 'test/bma_acc']]
plt_data = plt_data.pivot(index='aug_scale', columns='prior_scale', values='test/bma_acc')

fig, ax = plt.subplots(figsize=(7, 7))
g = sns.heatmap(data=plt_data, ax=ax, fmt='.3f', linewidths=1.,
                annot=True, annot_kws=dict(fontsize=14),
                cbar=True, cbar_kws=dict(shrink=.5),
                yticklabels=plt_data.index,
                cmap=sns.color_palette('summer'))
g.set(xlabel=r'Prior Scale ($\sigma)$', ylabel=r'Aug. Scale ($\epsilon$)', title=rf'cSGLD at $T = {T}$ with Avg. Aug + FRN Layer')
g.set_xticklabels(g.get_xticklabels(), fontdict=dict(size=16))
g.set_yticklabels(g.get_yticklabels(), fontdict=dict(size=12))
fig.tight_layout()

# fig.savefig('csgld_avg_aug_frn.png', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Noisy Dirichlet Likelihoods<jupyter_code>## Noisy Dirichlet
_, metrics = get_summary_metrics('deeplearn/data-aug-likelihood/wou01xes',
                                 config_keys=['augment', 'temperature'])

## Noisy Dirichlet + KL Consistency
_, metrics_consistency = get_summary_metrics('deeplearn/data-aug-likelihood/2irgmpw7',
                                 config_keys=['temperature'])
metrics_consistency['augment'] = -1
metrics = pd.concat([metrics, metrics_consistency]).reset_index()
metrics
g = sns.relplot(data=metrics, x='temperature', y='csgld/test/bma_acc', hue='augment', kind='line',
                markers=True, dashes=False,
                height=5, aspect=4/3, linewidth=3, palette=sns.color_palette('tab10', 3))
g.set(xscale='log', xlabel=r'$T$', ylabel='BMA Accuracy')
g.fig.savefig('save.png', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Noisy Labels<jupyter_code>## Softmax
_, softmax_metrics = get_summary_metrics('deeplearn/data-aug-likelihood/4ypwsj0k',
                                         config_keys=['label_noise', 'temperature'])
softmax_metrics['kind'] = 'softmax'
softmax_metrics['noise'] = float('NaN')

## Noisy Dirichlet
_, dirichlet_metrics = get_summary_metrics('deeplearn/data-aug-likelihood/zboj4e6h',
                                           config_keys=['label_noise', 'noise', 'temperature'])
dirichlet_metrics['kind'] = 'dirichlet'

metrics = pd.concat([softmax_metrics, dirichlet_metrics[dirichlet_metrics.temperature == 1]]).reset_index()
# metrics = pd.read_csv('label_noise.csv')
best_metrics = metrics.iloc[metrics.groupby(by=['kind', 'label_noise'])['csgld/test/bma_acc'].idxmax().values].reset_index()
g = sns.relplot(data=best_metrics, x='label_noise', y='csgld/test/bma_acc', hue='kind',
                kind='line', marker='o', markersize=10,
                height=5, aspect=4/3, linewidth=3, palette=sns.color_palette('tab10', 2))

sns.lineplot(ax=g.ax, data=metrics[metrics.kind == 'dirichlet'], x='label_noise', y='csgld/test/bma_acc',
             style='noise', legend=False,
             linewidth=1, alpha=.5, color=sns.color_palette('tab10', 2)[0])

sns.lineplot(ax=g.ax, data=metrics[metrics.kind == 'softmax'], x='label_noise', y='csgld/test/bma_acc',
             style='temperature', legend=False,
             linewidth=1, alpha=.5, color=sns.color_palette('tab10', 2)[1])

g.set(xlabel='Label Noise', ylabel='Test BMA Accuracy')

g.fig.savefig('label_noise.pdf', bbox_inches='tight')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1658639373871162, 'fraction_numerical': 0.013646397752358017, 'mean_word_length': 4.497517926089355, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '348878', 'license': 'Apache-2.0', 'n_tokens_mistral': 3879, 'n_tokens_neox': 3604, 'n_words': 511, 'path': 'notebooks/viz_perf.ipynb', 'repo_name': 'activatedgeek/uncertainty-da-bayesian-classification', 'size': '9966'}