<jupyter_start><jupyter_text>Deep Learning Toolkit for Splunk - Notebook for PyTorch  Logistic Regression in PyTorchThis notebook contains an example for a simple logistic regression in PyTorch.By default every time you save this notebook the cells are exported into a python module which is then used for executing your custom model invoked by Splunk MLTK Container App.   Stage 0 - import librariesAt stage 0 we define all imports necessary to run our subsequent code depending on various libraries.<jupyter_code># this definition exposes all python module imports that should be available in all subsequent commands
import json
import datetime
import numpy as np
import scipy as sp
import pandas as pd
import torch
# global constants
MODEL_DIRECTORY = "/srv/app/model/data/"
# THIS CELL IS NOT EXPORTED - free notebook cell for testing purposes
print("numpy version: " + np.__version__)
print("scipy version: " + sp.__version__)
print("pandas version: " + pd.__version__)
print("PyTorch: " + torch.__version__)
if torch.cuda.is_available():
    print(f"There are {torch.cuda.device_count()} CUDA devices available")
    for i in range(0,torch.cuda.device_count()):
        print(f"Device {i:0}: {torch.cuda.get_device_name(i)} ")
else:
    print("No GPU found")<jupyter_output><empty_output><jupyter_text>Stage 1 - get a data sample from SplunkIn Splunk run a search to pipe a prepared sample dataset into this environment. | inputlookup iris.csv | fit MLTKContainer algo=pytorch_logistic_regression epochs=1000 mode=stage species from petal_length petal_width sepal_length sepal_width into app:PyTorch_iris_model After you run this search your data set sample is available as a csv inside the container to develop your model. The name is taken from the into keyword ("PyTorch_iris_model" in the example above) or set to "default" if no into keyword is present. This step is intended to work with a subset of your data to create your custom model.<jupyter_code># this cell is not executed from MLTK and should only be used for staging data into the notebook environment
def stage(name):
    with open("data/"+name+".csv", 'r') as f:
        df = pd.read_csv(f)
    with open("data/"+name+".json", 'r') as f:
        param = json.load(f)
    return df, param
# THIS CELL IS NOT EXPORTED - free notebook cell for testing purposes
df, param = stage("PyTorch_iris_model")
#print(param)
print(df.describe)<jupyter_output><empty_output><jupyter_text>Stage 2 - create and initialize a model<jupyter_code>def init(df,param):
    X = df[param['feature_variables']]
    Y = df[param['target_variables']]
    input_size = int(X.shape[1])
    num_classes = len(np.unique(Y.to_numpy()))
    learning_rate = 0.001
    mapping = { key: value for value,key in enumerate(np.unique(Y.to_numpy().reshape(-1))) }
    print("FIT build logistic regression model with input shape " + str(X.shape))
    print("FIT build model with target classes " + str(num_classes))
    model = {
        "input_size": input_size,
        "num_classes": num_classes,
        "learning_rate": learning_rate,
        "mapping": mapping,
        "num_epochs": 10000,
        "batch_size": 100,
    }
    if 'options' in param:
        if 'params' in param['options']:
            if 'epochs' in param['options']['params']:
                model['num_epochs'] = int(param['options']['params']['epochs'])
            if 'batch_size' in param['options']['params']:
                model['batch_size'] = int(param['options']['params']['batch_size'])
    # Simple logistic regression model
    model['model'] = torch.nn.Linear(input_size, num_classes)
    # Define loss and optimizer
    model['criterion'] = torch.nn.CrossEntropyLoss()  
    model['optimizer'] = torch.optim.SGD(model['model'].parameters(), lr=learning_rate)      
    return model
model = init(df,param)
print(model)<jupyter_output><empty_output><jupyter_text>Stage 3 - fit the model<jupyter_code>def fit(model,df,param):
    returns = {}
    X = df[param['feature_variables']].astype('float32').to_numpy()
    Y = df[param['target_variables']].to_numpy().reshape(-1)
    mapping = { key: value for value,key in enumerate(np.unique(Y)) }
    Y = df[param['target_variables']].replace( {param['target_variables'][0]:mapping } ).to_numpy().reshape(-1)
    #Y = df[param['target_variables']].to_numpy().reshape(-1)
    #Y = pd.get_dummies(Y).astype('float32').to_numpy()
    #Ymap = df[param['target_variables']].replace( {param['target_variables'][0]:mapping } ).to_numpy()
    if 'options' in param:
        if 'params' in param['options']:
            if 'epochs' in param['options']['params']:
                model['num_epochs'] = int(param['options']['params']['epochs'])
            if 'batch_size' in param['options']['params']:
                model['batch_size'] = int(param['options']['params']['batch_size'])
    print(model['num_epochs'])
    for epoch in range(model['num_epochs']):
        inputs = torch.from_numpy(X)
        targets = torch.from_numpy(Y)
        outputs = model['model'](inputs)
        loss = model['criterion'](outputs, targets)
        model['optimizer'].zero_grad()
        loss.backward()
        model['optimizer'].step()
        if (epoch+1) % (model['num_epochs']/10) == 0:
            print ('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1, model['num_epochs'], loss.item()))                
    # memorize parameters
    returns['model_epochs'] = model['num_epochs']
    returns['model_batch_size'] = model['batch_size']
    returns['model_loss_acc'] = loss.item()
    return returns
returns = fit(model,df,param)
print(returns['model_loss_acc'])<jupyter_output><empty_output><jupyter_text>Stage 4 - apply the model<jupyter_code>def apply(model,df,param):
    X = df[param['feature_variables']].astype('float32').to_numpy()
    classes = {v: k for k, v in model['mapping'].items()}
    with torch.no_grad():
        input = torch.from_numpy(X)
        output = model['model'](input)
        y_hat = output.data
        _, predicted = torch.max(output.data, 1)
        prediction = [classes[key] for key in predicted.numpy()]
    return prediction
y_hat = apply(model,df,param)
y_hat<jupyter_output><empty_output><jupyter_text>Stage 5 - save the model<jupyter_code># save model to name in expected convention "<algo_name>_<model_name>.h5"
def save(model,name):
    torch.save(model, MODEL_DIRECTORY + name + ".pt")
    return model<jupyter_output><empty_output><jupyter_text>Stage 6 - load the model<jupyter_code># load model from name in expected convention "<algo_name>_<model_name>.h5"
def load(name):
    model = torch.load(MODEL_DIRECTORY + name + ".pt")
    return model<jupyter_output><empty_output><jupyter_text>Stage 7 - provide a summary of the model<jupyter_code># return model summary
def summary(model=None):
    returns = {"version": {"pytorch": torch.__version__} }
    if model is not None:
        if 'model' in model:
            returns["summary"] = str(model)
    return returns<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12989985693848355, 'fraction_numerical': 0.007010014306151645, 'mean_word_length': 4.167036215816704, 'pattern_counts': {'":': 8, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '106372', 'license': 'Apache-2.0', 'n_tokens_mistral': 2206, 'n_tokens_neox': 2174, 'n_words': 590, 'path': 'notebooks/pytorch_logistic_regression.ipynb', 'repo_name': 'splunk/splunk-mltk-container-docker', 'size': '6990'}