<jupyter_start><jupyter_text>Hyperparameter Scaling<jupyter_code>%matplotlib inline
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
np.random.seed(0)<jupyter_output><empty_output><jupyter_text>Learning Rate $ \alpha $ Pick values between 0 and 1Problem is that 1e-4 to 1e-3 regions, 1e-3 to 1e-2, 1e-2 to 1e-1 regions are not equally represented.<jupyter_code>random_values = np.random.rand(100)
r = -4 * random_values
alpha = 10 ** r
plt.scatter(r, alpha);
ranges_alpha = [1e-4, 1e-3, 1e-2, 1e-1, 5e-1, 1]
# There's a very small number of values sampled from 1e-4 to 1e-3 and 1e-3 to 1e-2 ranges (in fact 1 in total)
pd.cut(random_values, ranges_alpha).value_counts()
# Whereas this sampling allows for more uniform sampling
pd.cut(alpha, ranges_alpha).value_counts()<jupyter_output><empty_output><jupyter_text>Beta $\beta$ Ranging between 0.9 and 0.999<jupyter_code>random_values_beta = 2 * random_values - 3
beta = 1 - 10 ** random_values_beta
pd.Series(random_values_beta).hist();
plt.scatter(r, beta);
ranges_beta = [0.9, 0.99, 0.999]
pd.cut(beta, ranges_beta).value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11915269196822595, 'fraction_numerical': 0.05383936451897617, 'mean_word_length': 5.631578947368421, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '248208', 'license': 'MIT', 'n_tokens_mistral': 475, 'n_tokens_neox': 429, 'n_words': 113, 'path': 'deep-learning/Hyperparameter-Scaling.ipynb', 'repo_name': 'ashishpatel26/learning', 'size': '1133'}