<jupyter_start><jupyter_text>Data Collection <NAME>  <NAME><jupyter_code>from configuration_generator import Configurations
c = Configurations()
c.generateConfigurations()<jupyter_output><empty_output><jupyter_text>PyTorch Trainer<jupyter_code>%run trainer_pytorch.py -c "0,1" -d "data/" -s "training_data/"
%run trainer_pytorch.py -d data/ -e 1 -s training_data/
%run trainer_pytorch.py -d "data/" -s "training_data/" -w 0 -e 3 -b 32 -lr 0.1<jupyter_output><empty_output><jupyter_text>Data Parallel<jupyter_code>import torch.nn as nn
model_p = nn.DataParallel(model)<jupyter_output><empty_output><jupyter_text>Distributed Data Parallel<jupyter_code>!python distributed_trainer.py --num-gpus 4 -e 10 -w 16 -b 2048 -lr 0.1<jupyter_output><empty_output><jupyter_text>Tensorflow<jupyter_code>import tensorflow as tf
from tensorflow.keras import regularizers
from tensorflow.keras.models import Sequential
from tensorflow.keras.datasets import cifar10
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.callbacks import Callback
from tensorflow.keras.layers import Dense, Input, BatchNormalization, Flatten, Dropout
from tensorflow.keras.losses import SparseCategoricalCrossentropy, CategoricalCrossentropy
from tensorflow.keras.optimizers import Adagrad, RMSprop, Adadelta, Adam, Nadam
import matplotlib.pyplot as plt
from time import time
from IPython.display import clear_output
from itertools import accumulate<jupyter_output><empty_output><jupyter_text>Loading dataset<jupyter_code>(x_train, y_train), (x_test, y_test) = cifar10.load_data()
print('Training feature set shape: ' + str(x_train.shape))
print('Training label set shape: ' + str(y_train.shape))
print('Test feature set shape: ' + str(x_test.shape))
print('Test label set shape: ' + str(y_test.shape))
x_train = x_train[:5000]
y_train = y_train[:5000]
x_test = x_test[:2000]
y_test = y_test[:2000]
# Set numeric type to float32
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')

# Normalize feature values
x_train /= 255
x_test /= 255

# Transform lables to one-hot encoding
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)

# Reshape the dataset into 4D array
x_train = x_train.reshape(x_train.shape[0], 32, 32, 3)
x_test = x_test.reshape(x_test.shape[0], 32, 32, 3)
class timer(Callback):
    def __init__(self):
        self.epoch_timings = []
        self.total_time_at_epochs = []
        self.training_time = 0
        
    def on_epoch_begin(self, epoch, logs=None):
        self.epoch_start = time()
        
    def on_epoch_end(self, epoch, logs=None):
        self.epoch_timings.append(time() - self.epoch_start)
        
    def on_train_begin(self, logs=None):
        self.training_start = time()
        
    def on_train_end(self, logs=None):
        self.training_time = time() - self.training_start
        self.total_time_at_epochs = list(accumulate(self.epoch_timings))
def run_trainings(layers = None):
    results = {}
    
    for method in [0.01,0.001,0.0001]:
        print(f'Training with Learning Rate: {method}')
        results[method], t = {}, timer()
        model = Sequential(layers)
        model.compile(optimizer=RMSprop(learning_rate=method),
                      loss=CategoricalCrossentropy(),
                      metrics=['accuracy'])
        history = model.fit(
            x = x_train,
            y = y_train,
            batch_size = 128,
            epochs = 25,
            validation_data = (x_test, y_test),
            verbose = 0,
            callbacks=[t]
        )

        results[method]['test_performance'] = model.evaluate(x=x_test, y=y_test)
        results[method]['accuracy'] = history.history['accuracy']
        results[method]['loss'] = history.history['loss']
        results[method]['total_time_at_epochs'] = t.total_time_at_epochs
        results[method]['training_time'] = t.training_time
        results[method]['epoch_timings'] = t.epoch_timings
        
        print('\n')
    
    return results
def visualize_training(results, opts='-', title='Adaptive Learning Rate Methods'):
    fig, ax = plt.subplots(nrows=1, ncols=4, figsize=(24, 6))
    fig.suptitle(title, fontsize=16)

    for method in results:
        ax[0].plot([None] + results[method]['accuracy'], opts, label=method)
    ax[0].legend(loc = 'best')
    ax[0].set_title('Training Accuracy per Epoch')
    ax[0].set_xlabel('Epoch')
    ax[0].set_ylabel('Accuracy')
    #ax[0].set_ylim(0.2, 0.8)

    for method in results:
        ax[1].plot([None] + results[method]['loss'], opts, label=method)
    ax[1].legend(loc = 'best')
    ax[1].set_title('Training Loss per Epoch')
    ax[1].set_xlabel('Epoch')
    ax[1].set_ylabel('Loss')
    #ax[1].set_ylim(1, 4.5)
    
    for method in results:
        ax[2].plot([None] + results[method]['total_time_at_epochs'], opts, label=method)
    ax[2].legend(loc = 'best')
    ax[2].set_title('Total Training Time at Epoch')
    ax[2].set_xlabel('Epoch')
    ax[2].set_ylabel('Training Time')
    # ax[2].set_ylim(-100, 950)

    for method in results:
        ax[3].plot([None] + results[method]['epoch_timings'], opts, label=method)
    ax[3].legend(loc = 'best')
    ax[3].set_title('Epoch Timing')
    ax[3].set_xlabel('Epoch')
    ax[3].set_ylabel('Training Time')
    # ax[2].set_ylim(-100, 950)

    fig.tight_layout()
    plt.show()
layers = [ 
    Flatten(input_shape=(32, 32, 3)),
    Dense(1000, activation='relu', kernel_regularizer=regularizers.l2(0.001)),
    Dense(1000, activation='relu', kernel_regularizer=regularizers.l2(0.001)),
    Dense(10, activation='softmax')
]
results = run_trainings(layers)
visualize_training(results)
results = run_trainings(layers)
visualize_training(results)
results = run_trainings(layers)
visualize_training(results)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13255091473938557, 'fraction_numerical': 0.02744218156713842, 'mean_word_length': 4.030381944444445, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '159377', 'license': 'Apache-2.0', 'n_tokens_mistral': 2123, 'n_tokens_neox': 2018, 'n_words': 390, 'path': 'Data Collection/Data Collection.ipynb', 'repo_name': 'sukumarh/distributed-training', 'size': '5813'}