<jupyter_start><jupyter_text>Lockout: Quick User's Guide A synthetic dataset was generated and partitioned into training, validation, and testing subsets and it is used througout this guide.  Create DataLoadersChange this section according to your particular problem to build training, validation, and testing DataLoaders.<jupyter_code>from lockout.pytorch_utils import load_data_reg, normalize_xy, make_DataLoaders
from lockout.utils import read_list

# Read Data
_, x_train, x_valid, x_test, _, y_train, y_valid, y_test = load_data_reg('dataset/')

# Normalize Data
xtrain, xvalid, xtest, ytrain, yvalid, ytest = normalize_xy(x_train, x_valid, x_test, 
                                                            y_train, y_valid, y_test)

# Create DataLoaders
dl_train, dl_valid, dl_test = make_DataLoaders(xtrain, xvalid, xtest, ytrain, yvalid, ytest)<jupyter_output><empty_output><jupyter_text>Neural Network ArchitectureChange this section to use a different architecture.<jupyter_code>from lockout.pytorch_utils import FCNN

# Number of Input features
n_features  = 100       

# Nodes per layer:
# - len(layer_sizes) = number of layers in the network
# - layer_sizes[i]   = number of nodes in the ith layer
layer_sizes = [10, 1]   

# Instantiate Neural Network (with ReLUs)
model_init = FCNN(n_features, layer_sizes)
model_init<jupyter_output><empty_output><jupyter_text>Unconstrained TrainingChange this section to use different hyperparameters. This problem is a regression task.<jupyter_code>from lockout import Lockout

# Learning rate
lr = 1e-2

# Loss function id:
# - loss_type = 1: Mean Squared Error
# - loss_type = 2: Mean Cross Entropy Loss
loss_type = 1

# Optimizer id:
# - optim_id = 1: Stochastic Gradient Descend
# - optim_id = 2: Adam
optim_id = 1

# Instantiate Lockout
lockout_forward = Lockout(model_init, 
                          lr=lr, 
                          loss_type=loss_type, 
                          optim_id=optim_id)

# Train Neural Network Without Regularization
lockout_forward.train(dl_train, dl_valid, 
                      train_how="unconstrained",
                      epochs=10000,              # Max number of epochs
                      early_stopping=20,         # Epoch used for convergence condition
                      tol_loss=1e-6,             # Tolerance in training loss used for convergence condition
                      reset_weights=True)        # Whether or not to reset weights before training
from lockout.pytorch_utils import save_model

# Retrieve and Save Unconstrained Model
model_forward_unconstrained = lockout_forward.model_last
save_model(model_forward_unconstrained, 'model_forward_unconstrained.pth')

# Retrieve and Save Model At Validation Minimum
model_forward_best = lockout_forward.model_best_valid
save_model(model_forward_best, 'model_forward_best.pth')
# Path Data
df0 = lockout_forward.path_data
df0.head()
# Plot loss vs iteration
import matplotlib.pyplot as plt
import numpy as np

fig, axes = plt.subplots(figsize=(9,6))

axes.plot(df0["iteration"], df0["train_loss"], label="Training", linewidth=4)
axes.plot(df0["iteration"], df0["valid_loss"], label="Validation", linewidth=4)
axes.legend(fontsize=16)
axes.set_xlabel("iteration", fontsize=16)
axes.set_ylabel("Loss Function", fontsize=16)
axes.tick_params(axis='both', which='major', labelsize=14)
axes.set_title("Unconstrained", fontsize=16)
axes.grid(True, zorder=2)

plt.tight_layout()
plt.savefig("loss_vs_iter_forward.pdf", bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Lockout Training: Option 1Within this option, the network is first trained until the regularization path is found (path 1). Then, the constraint value $t_0$ is iteratively decreased (path 2) with a step size\begin{equation}    \Delta t_0 =\frac{{t_0}_{\mathrm{final}} - {t_0}_{\mathrm{initial}}}{\mathrm{epochs}}.\end{equation}A small $\Delta t_0$ is necessary to stay on the regularization path.<jupyter_code>from lockout import Lockout

# List of tuples (or dictionary) of the form [(layer_name, regul_id)] where:
# - layer_name: layer name in the input model (string)
# - regul_id = 1: L1 regularization
# - regul_id = 2: Log regularization
regul_type = [('linear_layers.0.weight', 1)]

# List of tuples (or dictionary) of the form [(layer_name, path_flg)] where:
# - path_flg = True: the constraint t0 will be iteratively changed in this layer
# - path_flg = False: the constraint t0 will be kept constant in this layer
regul_path = [('linear_layers.0.weight', True)]

# Instantiate Lockout
lockout_option1 = Lockout(lockout_forward.model_best_valid,  # initial network
                          lr=1e-2, 
                          loss_type=1,
                          regul_type=regul_type,
                          regul_path=regul_path)

# Train Neural Network With Lockout
lockout_option1.train(dl_train, dl_valid, 
                      train_how="decrease_t0", 
                      epochs=5000,              # Max number of epochs used in Path 1
                      epochs2=20000,            # Max number of epochs used in Path 2
                      early_stopping=20, 
                      tol_loss=1e-5)
# Save Model At Validation Minimum
model_lockout_option1 = lockout_option1.model_best_valid
save_model(model_lockout_option1, 'model_lockout_option1.pth')
# Path Data
df1 = lockout_option1.path_data
df1.head()
# Plot loss vs iteration
import matplotlib.pyplot as plt
import numpy as np

fig, axes = plt.subplots(figsize=(8,6))

axes.plot(df1["iteration"], df1["train_loss"], label="Training", linewidth=4)
axes.plot(df1["iteration"], df1["valid_loss"], label="Validation", linewidth=4)
axes.legend(fontsize=16)
axes.set_xlabel("iteration", fontsize=16)
axes.set_ylabel("Loss Function", fontsize=16)
axes.tick_params(axis='both', which='major', labelsize=14)
axes.set_title("Unconstrained", fontsize=16)
axes.grid(True, zorder=2)

plt.tight_layout()
plt.savefig("loss_vs_iter_lockout1.pdf", bbox_inches='tight')
plt.show()
# Plot loss vs iteration
import matplotlib.pyplot as plt
import numpy as np

fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(16,7))

axes[0].plot(df0["iteration"], df0["train_loss"], label="Training", linewidth=4)
axes[0].plot(df0["iteration"], df0["valid_loss"], label="Validation", linewidth=4)
axes[0].legend(fontsize=16)
axes[0].set_xlabel("iteration", fontsize=16)
axes[0].set_ylabel("Loss Function", fontsize=16)
axes[0].tick_params(axis='both', which='major', labelsize=14)
axes[0].set_title("Unconstrained", fontsize=16)
axes[0].grid(True, zorder=2)
axes[0].set_ylim(0,1.3)

axes[1].plot(df1["iteration"], df1["train_loss"], label="Training", linewidth=4)
axes[1].plot(df1["iteration"], df1["valid_loss"], label="Validation", linewidth=4)
axes[1].legend(fontsize=16)
axes[1].set_xlabel("iteration", fontsize=16)
axes[1].set_ylabel("Loss Function", fontsize=16)
axes[1].tick_params(axis='both', which='major', labelsize=14)
axes[1].set_title("Lockout", fontsize=16)
axes[1].grid(True, zorder=2)
axes[1].set_ylim(0,1.3)

plt.tight_layout()
plt.show()
# Compute Test Accuracy
import torch
from lockout.pytorch_utils import dataset_r2

device = torch.device('cpu')
r2_test_forward, _  = dataset_r2(dl_test, model_forward_best, device)
r2_test_lockout1, _ = dataset_r2(dl_test, model_lockout_option1, device)
print("Test R2 (unconstrained) = {:.3f}".format(r2_test_forward))
print("Test R2 (lockout)       = {:.3f}".format(r2_test_lockout1))
# Graph Feature Importance
import matplotlib.pyplot as plt
import numpy as np
from lockout.pytorch_utils import get_features_importance

importance = get_features_importance(model_lockout_option1, 'linear_layers.0.weight')

fig, axes = plt.subplots(figsize=(9,6))
x_pos = np.arange(len(importance))
axes.bar(x_pos, importance, zorder=2)
axes.set_xticks(x_pos)
axes.set_xticklabels(importance.index, rotation='vertical')
axes.set_xlim(-1,len(x_pos))
axes.tick_params(axis='both', which='major', labelsize=14)
axes.set_ylabel('Importance', fontsize=16)
axes.set_xlabel('feature', fontsize=16)
axes.set_title('Lockout', fontsize=16)
axes.grid(True, zorder=1)

plt.tight_layout()
plt.savefig('feature_importance_lockout1.pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Lockout Training: Option 2Within this option, a discrete set $t_0$ values are sampled. The set of $t_0$s can be either:a) Entered externally as a 1D torch tensor:<jupyter_code>from lockout import Lockout

regul_type = [('linear_layers.0.weight', 1)]
regul_path = [('linear_layers.0.weight', True)]

# List of tuples (or dictionary) of the form [(layer_name, t0_sampled)] where:
# - layer_name: layer name in the input model (string)
# - t0_sampled: 1D tensor with the constraint values t0 to be sampled in the layer
t0_sampled = torch.from_numpy(np.geomspace(53.504620, 1e-3, num=100, endpoint=True))
t0_grid    = {'linear_layers.0.weight': t0_sampled}

# Instantiate Lockout
lockout_option2a = Lockout(lockout_forward.model_best_valid,  # initial network
                          lr=1e-2, 
                          loss_type=1,
                          regul_type=regul_type,
                          regul_path=regul_path, 
                          t0_grid=t0_grid)

# Train Neural Network With Lockout
lockout_option2a.train(dl_train, dl_valid, 
                      train_how="sampling_t0", 
                      epochs=5000,             # Max number of epochs used for first t0
                      epochs2=200,             # Max number of epochs used for the rest of the t0 values
                      early_stopping=20, 
                      tol_loss=1e-4)
# Save Model At Validation Minimum
model_lockout_option2a = lockout_option2a.model_best_valid
save_model(model_lockout_option2a, 'model_lockout_option2a.pth')
# Path Data
df2a = lockout_option2a.path_data
df2a.head()
# Graph Feature Importance
import matplotlib.pyplot as plt
import numpy as np
from lockout.pytorch_utils import get_features_importance

importance = get_features_importance(model_lockout_option2a, 'linear_layers.0.weight')

fig, axes = plt.subplots(figsize=(9,6))
x_pos = np.arange(len(importance))
axes.bar(x_pos, importance, zorder=2)
axes.set_xticks(x_pos)
axes.set_xticklabels(importance.index, rotation='vertical')
axes.set_xlim(-1,len(x_pos))
axes.tick_params(axis='both', which='major', labelsize=14)
axes.set_ylabel('Importance', fontsize=16)
axes.set_xlabel('feature', fontsize=16)
axes.set_title('Lockout', fontsize=16)
axes.grid(True, zorder=1)

plt.tight_layout()
plt.savefig('feature_importance_lockout2a.pdf', bbox_inches='tight')
plt.show()
# Compute Test Accuracy
import torch
from lockout.pytorch_utils import dataset_r2

device = torch.device('cpu')
r2_test_forward, _   = dataset_r2(dl_test, model_forward_best, device)
r2_test_lockout2a, _ = dataset_r2(dl_test, model_lockout_option2a, device)
print("Test R2 (unconstrained) = {:.3f}".format(r2_test_forward))
print("Test R2 (lockout)       = {:.3f}".format(r2_test_lockout2a))<jupyter_output>Test R2 (unconstrained) = 0.655
Test R2 (lockout)       = 0.953<jupyter_text>b) Generated internally, in which case the $t_0$ points are linearly sampled<jupyter_code>from lockout import Lockout

regul_type = [('linear_layers.0.weight', 1)]
regul_path = [('linear_layers.0.weight', True)]

# List of tuples (or dictionary) of the form [(layer_name, t0_number)] where:
# - layer_name: layer name in the input model (string)
# - t0_number: number of constraint values t0 to be linearly sampled (integer)
t0_points    = {'linear_layers.0.weight': 200}

# Instantiate Lockout
lockout_option2b = Lockout(lockout_forward.model_best_valid,  # initial network
                          lr=1e-2, 
                          loss_type=1,
                          regul_type=regul_type,
                          regul_path=regul_path, 
                          t0_points=t0_points)

# Train Neural Network With Lockout
lockout_option2b.train(dl_train, dl_valid, 
                      train_how="sampling_t0", 
                      epochs=5000,             # Max number of epochs used for first t0
                      epochs2=200,             # Max number of epochs used for the rest of the t0 values
                      early_stopping=20, 
                      tol_loss=1e-4)
# Path Data
df2b = lockout_option2b.path_data
df2b.head()
# Save Model At Validation Minimum
model_lockout_option2b = lockout_option2b.model_best_valid
save_model(model_lockout_option2b, 'model_lockout_option2b.pth')
# Compute Test Accuracy
import torch
from lockout.pytorch_utils import dataset_r2

device = torch.device('cpu')
r2_test_forward, _   = dataset_r2(dl_test, model_forward_best, device)
r2_test_lockout2b, _ = dataset_r2(dl_test, model_lockout_option2b, device)
print("Test R2 (unconstrained) = {:.3f}".format(r2_test_forward))
print("Test R2 (lockout)       = {:.3f}".format(r2_test_lockout2b))
# Graph Feature Importance
import matplotlib.pyplot as plt
import numpy as np
from lockout.pytorch_utils import get_features_importance

importance = get_features_importance(model_lockout_option2b, 'linear_layers.0.weight')

fig, axes = plt.subplots(figsize=(9,6))
x_pos = np.arange(len(importance))
axes.bar(x_pos, importance, zorder=2)
axes.set_xticks(x_pos)
axes.set_xticklabels(importance.index, rotation='vertical')
axes.set_xlim(-1,len(x_pos))
axes.tick_params(axis='both', which='major', labelsize=14)
axes.set_ylabel('Importance', fontsize=16)
axes.set_xlabel('feature', fontsize=16)
axes.set_title('Lockout', fontsize=16)
axes.grid(True, zorder=1)

plt.tight_layout()
plt.savefig('feature_importance_lockout2b.pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Lockout Training: Option 3Within this option, the neural network is trained with a single (constant) value of $t_0$ until the regularization path is found.<jupyter_code>from lockout import Lockout

regul_type = [('linear_layers.0.weight', 1)]

# List of tuples (or dictionary) of the form [(layer_name, t0_value)] where:
# - layer_name: layer name in the input model (string)
# - t0_value: constraint value t0 used in the layer (tensor)
t0 = {'linear_layers.0.weight': torch.tensor(0.4)}

# Instantiate Lockout
lockout_option3 = Lockout(model_lockout_option2b,  # initial network
                          lr=1e-3, 
                          loss_type=1,
                          regul_type=regul_type,
                          t0=t0)

# Train Neural Network With Lockout
lockout_option3.train(dl_train, dl_valid, 
                      train_how="constant_t0", 
                      epochs=10000,              # Max number of epochs
                      early_stopping=20, 
                      tol_loss=1e-5)
# Path Data
df3 = lockout_option3.path_data
df3.tail()
# Save Model At Regularization Path
from lockout.pytorch_utils import save_model

model_lockout_option3 = lockout_option3.model_last
save_model(model_lockout_option3, 'model_lockout_option3.pth')
# Compute Test Accuracy
import torch
from lockout.pytorch_utils import dataset_r2

device = torch.device('cpu')
r2_test_forward, _  = dataset_r2(dl_test, model_forward_best, device)
r2_test_lockout3, _ = dataset_r2(dl_test, model_lockout_option3, device)
print("Test R2 (unconstrained) = {:.3f}".format(r2_test_forward))
print("Test R2 (lockout)       = {:.3f}".format(r2_test_lockout3))
# Graph Feature Importance
import matplotlib.pyplot as plt
import numpy as np
from lockout.pytorch_utils import get_features_importance

importance = get_features_importance(model_lockout_option3, 'linear_layers.0.weight')

fig, axes = plt.subplots(figsize=(9,6))
x_pos = np.arange(len(importance))
axes.bar(x_pos, importance, zorder=2)
axes.set_xticks(x_pos)
axes.set_xticklabels(importance.index, rotation='vertical')
axes.set_xlim(-1,len(x_pos))
axes.tick_params(axis='both', which='major', labelsize=14)
axes.set_ylabel('Importance', fontsize=16)
axes.set_xlabel('feature', fontsize=16)
axes.set_title('Lockout', fontsize=16)
axes.grid(True, zorder=1)

plt.tight_layout()
plt.savefig('feature_importance_lockout3.pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Lockout Training: Option 4Within this option, the neural network is trained until the regularization path is found. The contraint value used, $t_0$ is computed from the input model.<jupyter_code>from lockout import Lockout

regul_type = [('linear_layers.0.weight', 1)]

# Instantiate Lockout
lockout_option4 = Lockout(model_forward_best,  # initial network
                          lr=1e-2, 
                          loss_type=1,
                          regul_type=regul_type)

# Train Neural Network With Lockout
lockout_option4.train(dl_train, dl_valid, 
                      train_how="until_path", 
                      epochs=10000,            # Max number of epochs
                      early_stopping=20, 
                      tol_loss=1e-5)
# Path Data
df4 = lockout_option4.path_data
df4.tail()
# Plot loss vs iteration
import matplotlib.pyplot as plt
import numpy as np

fig, axes = plt.subplots(figsize=(8,6))

axes.plot(df4["iteration"], df4["train_loss"], label="Training", linewidth=4)
axes.plot(df4["iteration"], df4["valid_loss"], label="Validation", linewidth=4)
axes.legend(fontsize=16)
axes.set_xlabel("iteration", fontsize=16)
axes.set_ylabel("Loss Function", fontsize=16)
axes.tick_params(axis='both', which='major', labelsize=14)
axes.set_title("Unconstrained", fontsize=16)
axes.grid(True, zorder=2)

plt.tight_layout()
# plt.savefig("loss_vs_iter_lockout4.pdf", bbox_inches='tight')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10836458688487328, 'fraction_numerical': 0.027105352881009206, 'mean_word_length': 4.191445427728613, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '418280', 'license': 'MIT', 'n_tokens_mistral': 6249, 'n_tokens_neox': 5857, 'n_words': 1308, 'path': 'Doc/lockout_user_guide.ipynb', 'repo_name': 'warbelo/Lockout', 'size': '17598'}