<jupyter_start><jupyter_text>Customizing Deep500In this tutorial, we will create manual versions of three different aspects of Deep500 training: [Operators](Operators), [Datasets](Datasets), and [Optimizers](Optimizers).After setting the environment, we show how to [combine all components](Putting-it-All-Together).<jupyter_code>import deep500 as d5
from deep500.frameworks import pytorch as d5fw<jupyter_output>WARNING: mpi4py not available, distributed optimization disabled
WARNING: mpi4py not available, distributed optimization disabled<jupyter_text>Operators PythonCreating a Python custom operator is easy - just fill in the `forward` and `backward` functions:<jupyter_code>import numpy as np  # for data types

class IPowOp(d5.CustomPythonOp):
    def __init__(self, power):
        # Supply input and output tensor types in constructor
        super(IPowOp, self).__init__([d5.tensordesc.runtime_shape(np.float32)], [d5.tensordesc.runtime_shape(np.float32)])
        
        self.power = power
        assert int(power) == power # integral

    def forward(self, inputs):
        return inputs ** self.power

    def backward(self, grads, fwd_inputs, fwd_outputs):
        return (grads[0] * self.power * 
           (fwd_inputs[0] ** (self.power - 1)))<jupyter_output><empty_output><jupyter_text>Now, wrap the operator so that the executor can use it as-is:<jupyter_code>fw_pow = d5fw.custom_op(IPowOp(3.0))
import torch
t = torch.tensor([1,2,3,4,5], dtype=torch.float32)
fw_pow(t)<jupyter_output><empty_output><jupyter_text>Same for gradients:<jupyter_code>from torch.autograd import Variable
v = Variable(t, requires_grad=True)
output, = fw_pow(v)
result = torch.sum(output)
result.backward()
v.grad<jupyter_output><empty_output><jupyter_text>C++Operators can also be defined in C++ almost as easily as in Python, but some extra steps need to be taken to compile the code.First, we define the C++ source code as an inline string (files are also supported):<jupyter_code>cpp_src = """
#include <deep500/deep500.h>   // Main include file
#include <cmath>

// Operator class definition
template<typename T>
class ipowop : public deep500::CustomOperator {
protected:
    int m_len;
public:
    ipowop(size_t len) : m_len((int)len) {}
    virtual ~ipowop() {}
    virtual bool supports_cuda() {return false;}

    // Forward part
    void forward(const T *input, T *output) {
        #pragma omp parallel for
        for (int i = 0; i < m_len; ++i)
            output[i] = std::pow(input[i], DPOWER);
    }

    // Backward part
    void backward(const T *nextop_grad,
                  const T *fwd_input_tensor,
                  const T *fwd_output_tensor,
                  T *input_tensor_grad) {
        #pragma omp parallel for
        for (int i = 0; i < m_len; ++i) {
            input_tensor_grad[i] = nextop_grad[i] * DPOWER * 
                std::pow(fwd_input_tensor[i], DPOWER - 1);
        }
    }
};

// Function called with runtime sizes to create the operator
D500_EXPORTED void *create_new_op(deep500::tensor_t *input_descriptors, int num_inputs,
                                  deep500::tensor_t *output_descriptors, int num_outputs) {
    size_t totalsz = 1;
    for (int i = 0; i < input_descriptors[0].dims; ++i)
        totalsz *= input_descriptors[0].sizes[i];
    return new ipowop<float>(totalsz);
}

// Register the operator
D500_REGISTER_OP(ipowop<float>);
"""
# Define the operator descriptor
opdesc = d5.compile_custom_cppop_inline('ipowop', cpp_src,
                                        # Input tensor shapes
                                        [d5.tensordesc(np.float32, [5])],
                                        # Output tensor shapes
                                        [d5.tensordesc(np.float32, [5])],
                                        live_output=True,
                                        # Running on CPU
                                        is_cuda=False,
                                        additional_definitions={'DPOWER': '3'})
# Compile and get a handle to the operator
fwcpp_op = d5fw.custom_op(opdesc)<jupyter_output>-- Selecting Windows SDK version 10.0.17763.0 to target Windows 10.0.18362.
-- Configuring done
-- Generating done
-- Build files have been written to: C:/Users/XL/Desktop/university/eurompi19/ipowop_pytorch_build

Microsoft (R) Build Engine version 15.9.21+g9802d43bc3 for .NET Framework
Copyright (C) Microsoft Corporation. All rights reserved.
  pytorch.cpp
C:\Users\XL\miniconda3\envs\denv\lib\site-packages\torch\include\c10/util/Exception.h(28): warning C4275: non dll-interface class 'std::exception' used as base for dll-interface class 'c10::Error' [C:\Users\XL\Desktop\university\eurompi19\ipowop_pytorch_build\ipowop.vcxproj]
  C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\include\vcruntime_exception.h(44): note: see declaration of 'std::exception'
  C:\Users\XL\miniconda3\envs\denv\lib\site-packages\torch\include\c10/util/Exception.h(28): note: see declaration of 'c10::Error'
C:\Users\XL\miniconda3\envs\denv\lib\site-packages\torch\include\[...]<jupyter_text>Similarly to Python, the function is now a native PyTorch operator and can be used normally:<jupyter_code>fwcpp_op(torch.tensor([5., 4., 3., 2., 1.], dtype=torch.float32))<jupyter_output><empty_output><jupyter_text>CUDAWorks exactly the same as the C++ version, but with a different flag set and `forward/backward_cuda` functions:<jupyter_code>cuda_src = """
#include <deep500/deep500.h>   // Main include file
#include <cmath>

// Kernels
template<typename T>
__global__ void fwd_kernel(const T* inp, T *out, int n) {
    int tid = blockIdx.x * 32 + threadIdx.x;
    if (tid >= n) return;
    out[tid] = std::pow(inp[tid], DPOWER) + T(1); // Adding term to CUDA kernel to ensure it ran
}

template<typename T>
__global__ void bwd_kernel(const T* gradin, const T* fwdin, T *gradout, int n) {
    int tid = blockIdx.x * 32 + threadIdx.x;
    if (tid >= n) return;
    gradout[tid] = gradin[tid] * DPOWER * std::pow(fwdin[tid], DPOWER - 1);
}

// Operator class definition
template<typename T>
class ipowop2 : public deep500::CustomOperator {
protected:
    int m_len;
public:
    ipowop2(size_t len) : m_len((int)len) {}
    virtual ~ipowop2() {}
    virtual bool supports_cuda() {return true;}   // NOTE: This is now true!

    // Forward part (CPU)
    void forward(const T *input, T *output) {
        #pragma omp parallel for
        for (int i = 0; i < m_len; ++i)
            output[i] = std::pow(input[i], DPOWER);
    }

    // Backward part (CPU)
    void backward(const T *nextop_grad,
                  const T *fwd_input_tensor,
                  const T *fwd_output_tensor,
                  T *input_tensor_grad) {
        #pragma omp parallel for
        for (int i = 0; i < m_len; ++i) {
            input_tensor_grad[i] = nextop_grad[i] * DPOWER * 
                std::pow(fwd_input_tensor[i], DPOWER - 1);
        }
    }

    // Forward part (GPU, invokes a kernel)
    void forward_cuda(const T *input, T *output) {
        fwd_kernel<T> <<<(m_len + 32 - 1) / 32, 32>>>(input, output, m_len);
    }

    // Backward part (GPU, invokes a kernel)
    void backward_cuda(const T *nextop_grad,
                  const T *fwd_input_tensor,
                  const T *fwd_output_tensor,
                  T *input_tensor_grad) {
        bwd_kernel<T> <<<(m_len + 32 - 1) / 32, 32>>>(nextop_grad, fwd_input_tensor, 
                                                      input_tensor_grad, m_len);
    }
};

// Function called with runtime sizes to create the operator
D500_EXPORTED void *create_new_op(deep500::tensor_t *input_descriptors, int num_inputs,
                                  deep500::tensor_t *output_descriptors, int num_outputs) {
    size_t totalsz = 1;
    for (int i = 0; i < input_descriptors[0].dims; ++i)
        totalsz *= input_descriptors[0].sizes[i];
    return new ipowop2<float>(totalsz);
}

// Register the operator
D500_REGISTER_OP(ipowop2<float>);
"""
# Define the operator descriptor
opdesc = d5.compile_custom_cppop_inline('ipowop2', cuda_src,
                                        # Input tensor shapes
                                        [d5.tensordesc(np.float32, [5])],
                                        # Output tensor shapes
                                        [d5.tensordesc(np.float32, [5])],
                                        live_output=True,
                                        # Running on GPU
                                        is_cuda=True,
                                        additional_definitions={'DPOWER': '3'})
# Compile and get a handle to the operator
fwcuda_op = d5fw.custom_op(opdesc)
fwcuda_op(torch.tensor([5., 4., 3., 2., 1.], dtype=torch.float32).cuda())<jupyter_output><empty_output><jupyter_text>If the result is correct, the output should be $x^3 + 1$  DatasetsThe dataset interface only consists of a `get(index)` method. Implementing a new dataset format, or a synthetic dataset, is possible by providing the functionality at that level. If a `Sampler` class loads using multiple threads or processes, the dataset will be called multiple times. It only needs to take care of decoding/generating data, rather than loading it.<jupyter_code>import random

class MyDataset(d5.Dataset):
    def __init__(self, input_node, label_node):
        super().__init__()
        self.input_node = input_node
        self.label_node = label_node
        
    # There are three possible inputs to __getitem__: Integer element, a slice (e.g., 0:N:2), or a list of elements
    def __getitem__(self, index):
        if isinstance(index, int): # One element
            num = np.array([random.random()], np.float32)
        else: # Minibatch
            if isinstance(index, slice): # Slice
                length = len(range(*index.indices(len(self))))
            else: # List of elements
                length = len(index)
            num = np.random.rand(length, 1).astype(np.float32)
            
        # In any case, return a mapping (random number -> 3 * (random number ^ 3)
        return {self.input_node: num, self.label_node: 3 * (num ** 3)}

    def __len__(self):
        return 1000 # Bogus number to determine epoch length<jupyter_output><empty_output><jupyter_text>Example usage:<jupyter_code>ds = MyDataset('dummy_input', 'dummy_target')
print('Element 0:', ds[0])
print('Element 0 (again):', ds[0])
print('Elements 1 to 50, skipping 10:', ds[1:50:10])<jupyter_output>Element 0: {'dummy_input': array([0.80738384], dtype=float32), 'dummy_target': array([1.5789247], dtype=float32)}
Element 0 (again): {'dummy_input': array([0.44469085], dtype=float32), 'dummy_target': array([0.26381278], dtype=float32)}
Elements 1 to 50, skipping 10: {'dummy_input': array([[0.6502744 ],
       [0.9222002 ],
       [0.54256546],
       [0.50194806],
       [0.7920214 ]], dtype=float32), 'dummy_target': array([[0.82491887],
       [2.3528643 ],
       [0.47915685],
       [0.37940025],
       [1.4905001 ]], dtype=float32)}<jupyter_text>OptimizersCreating a custom optimizer only requires providing an implementation of a `deep500.Optimizer` subclass. In this case, we can extend the reference SGD optimizer to contain "Stochastic Learning Rate", in which we simply randomize it at every step:<jupyter_code>import numpy as np
from deep500.frameworks import reference as d5ref

class StochasticUpdateRule(d5ref.GradientDescent):
    def update_rule(self, grad, old_param, param_name):
        return old_param - (np.random.random_sample() * self.lr) * grad<jupyter_output><empty_output><jupyter_text>Putting it All TogetherWe can now run our custom model in a native PyTorch environment, reading from the synthetic dataset, and optimizing using the Stochastic Update Rule:<jupyter_code># Define our custom native PyTorch model
from torch import nn
class MyNet(nn.Module):
    def __init__(self):
        super(MyNet, self).__init__()
        self.pow_op = fwcuda_op
        self.linear = nn.Linear(1, 1)

    def forward(self, inp):
        x, = self.pow_op(inp)
        x = x[0:inp.shape[0]].reshape(inp.shape)  # reshape to right size
        return self.linear(x - 1)  # "x - 1" negates CUDA kernel

model = MyNet()

# Set a native graph executor with our module and MSE loss
executor = d5fw.PyTorchNativeGraphExecutor(module=model, loss=nn.MSELoss(), device=d5.GPUDevice())
# Create the data necessary for recipe-less training
train_set = MyDataset(executor.innode, executor.labelnode)
val_set = MyDataset(executor.innode, executor.labelnode)
optimizer = StochasticUpdateRule(executor, lr=0.1, loss=executor.lossnode)
# Reset parameters
with torch.no_grad():
    model.linear.weight.data[:] = 1
    model.linear.bias.data[:] = 1
          
# Train (batch size must match compiled kernel)
d5.test_training(executor, train_set, val_set, optimizer, epochs=5, batch_size=5, metrics=[])<jupyter_output><empty_output><jupyter_text>We can now inspect the parameters (should be close to $3,0$):<jupyter_code>model.linear.weight, model.linear.bias<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11249332213996795, 'fraction_numerical': 0.031977409753491566, 'mean_word_length': 2.9721127614428613, 'pattern_counts': {'":': 0, '<': 72, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '472413', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 4407, 'n_tokens_neox': 4143, 'n_words': 1117, 'path': '3_custom.ipynb', 'repo_name': 'deep500/eurompi19', 'size': '13103'}