<commit_before>#![feature(zero_one)]

extern crate deeplearn;
extern crate matrix;

use std::fs::File;
use std::io::{
    self,
    BufReader,
    Read,
};
use std::num::{Zero, One};
use std::path::Path;

use deeplearn::Graph;
use deeplearn::op::{Add, MatMul, Mse, Relu};
use deeplearn::init;
use matrix::Matrix;

fn main() {
    println!("Reading training labels...");
    let train_labels = read_mnist_labels("data/mnist/train-labels-idx1-ubyte");
    let train_labels: Vec<Matrix<f32>> = train_labels.unwrap().into_iter().map(|x| one_hot(x, 10)).collect();
    println!("Label count: {}", train_labels.len());

    println!("Reading training images...");
    let (rows, columns, train_images) = read_mnist_images("data/mnist/train-images-idx3-ubyte").unwrap();

    let ctx = matrix::Context::new();

    // Setup the graph. It's going to have 2 inputs, 3 hidden nodes in it's 1 hidden layer, and 1
    // output node
    let mut graph = Graph::new();

    //////////////////////////
    // Layer 1

    // Input.  1 batches of rows*columns inputs each.
    let input = graph.add_variable(&ctx, (1, rows*columns), vec![0.0; rows*columns]);
    // Weights for layer 1: [2 inputs x 3 nodes]
    let l1_w = graph.add_variable(&ctx, (rows*columns, 1000), init::Normal(0.001, 0.005));
    // Use matrix multiplication to do a fully connected layer
    let l1_mat_mul = graph.add_node(&ctx,
                                    MatMul::new(&ctx, (1, rows*columns), (rows*columns, 1000)),
                                    vec![input, l1_w],
                                    &[(1, 1000)]); // out shape: [5x2]*[2x3] = [5 batches x 3 outputs]
    // Grab VarIndex for l1_mat_mul's output
    let l1_mat_mul_out = l1_mat_mul.get(&graph).outputs[0]; 
    // 3 biases; one for each node in layer 1
    let l1_b = graph.add_variable(&ctx, (1, 1000), init::Normal(0.001, 0.005));
    // Here we add the biases to the matrix multiplication output
    let l1_biased = graph.add_node(&ctx,
                               Add::new(0),
                               vec![l1_mat_mul_out, l1_b],
                               &[(1, 1000)]);
    // Grab VarIndex for l1_biased's output
    let l1_biased_out = l1_biased.get(&graph).outputs[0];
    // Run the biased input*weight sums through an ReLU activation
    let l1_relu = graph.add_node(&ctx,
                                 Relu::new(),
                                 vec![l1_biased_out],
                                 &[(1, 1000)]);
    let l1_relu_out = l1_relu.get(&graph).outputs[0];

    //////////////////////////
    // Layer 2

    // Weights for layer 2: [3 inputs x 1 nodes]
    let l2_w = graph.add_variable(&ctx, (1000, 10), init::Normal(0.001, 0.005));
    // Fully connected layer 2. Use layer 1's output as layer 2's input
    let l2_mat_mul = graph.add_node(&ctx,
                                    MatMul::new(&ctx, (1, 1000), (1000, 10)),
                                    vec![l1_relu_out, l2_w],
                                    &[(1, 10)]); // out shape: [5x3]*[3x1] = [5 batches x 1 output]
    // Grab VarIndex for l2_mat_mul's output
    let l2_mat_mul_out = l2_mat_mul.get(&graph).outputs[0];
    // 1 bias for 1 output node
    let l2_b = graph.add_variable(&ctx, (1, 10), init::Normal(0.001, 0.005));
    // Here we add the bias to the matrix multiplication output
    let l2_biased = graph.add_node(&ctx,
                               Add::new(0),
                               vec![l2_mat_mul_out, l2_b],
                               &[(1, 10)]);
    // Grab VarIndex for l2_biased's output
    let l2_biased_out = l2_biased.get(&graph).outputs[0];
    // Run the biased input*weight sums through an ReLU activation
    let l2_relu = graph.add_node(&ctx,
                                 Relu::new(),
                                 vec![l2_biased_out],
                                 &[(1, 10)]);
    // Grab VarIndex for l2_relu's output
    let l2_relu_out = l2_relu.get(&graph).outputs[0];

    //////////////////////////
    // Loss

    // Add a variable for the training outputs: [5 batches x 1 output]
    let train_out = graph.add_variable(&ctx, (1, 10), vec![0.0; 10]);
    // Use mean squared error loss function
    let loss = graph.add_node(&ctx,
                              Mse::new(),
                              vec![l2_relu_out, train_out],
                              &[(1, 10)]);
    // Grab the VarIndex for loss's output
    let loss_out = loss.get(&graph).outputs[0];
    // Create a gradient to apply to the loss function
    let loss_d = graph.add_gradient(&ctx, loss, 0);

    ////////////////////////////////////////////////////////////////////////////////////////////////

    // We apply a gradient of -0.1 to the loss function
    let loss_d_cpu = Matrix::from_vec(1, 10, vec![-0.001; 10]);
    loss_d.get(&graph).set(&ctx, &loss_d_cpu);

    /////////////////////////
    // Run/Train the network
    for epoch in 0..100000 {
        // Upload training data
        let train_sample = epoch%train_images.len();
        input.get(&graph).set(&ctx, &train_images[train_sample]);
        train_out.get(&graph).set(&ctx, &train_labels[train_sample]);

        graph.run(&ctx);
        let out = l2_relu_out.get(&graph).get(&ctx);
        let out_d = l2_relu.get(&graph)
                           .out_grad[0]
                           .gradient()
                           .get(&graph)
                           .get(&ctx);
        let l = loss_out.get(&graph).get(&ctx);
        let l1_w_d = graph.get_input_gradient(l1_w).unwrap().get(&graph);
        let l2_w_d = graph.get_input_gradient(l2_w).unwrap().get(&graph);
        let l1_b_d = graph.get_input_gradient(l1_b).unwrap().get(&graph);
        let l2_b_d = graph.get_input_gradient(l2_b).unwrap().get(&graph);
        l1_w.get(&graph).add(&ctx, -1, &*l1_w_d, &*l1_w.get(&graph));
        l2_w.get(&graph).add(&ctx, -1, &*l2_w_d, &*l2_w.get(&graph));
        l1_b.get(&graph).add(&ctx, -1, &*l1_b_d, &*l1_b.get(&graph));
        l2_b.get(&graph).add(&ctx, -1, &*l2_b_d, &*l2_b.get(&graph));
        if epoch % 1000 == 1 {
            println!("===================");
            println!("Epoch: {}", epoch);
            println!("out = {:?}", out);
            println!("out_d = {:?}", out_d);
            println!("loss = {:?}", l);
        }
    }
}
<commit_msg>Added accuracy score to mnist example output<commit_after>#![feature(zero_one)]

extern crate deeplearn;
extern crate matrix;

use std::fs::File;
use std::io::{
    self,
    BufReader,
    Read,
};
use std::num::{Zero, One};
use std::path::Path;

use deeplearn::Graph;
use deeplearn::op::{Add, MatMul, Mse, Relu};
use deeplearn::init;
use matrix::Matrix;

fn main() {
    println!("Reading training labels...");
    let train_labels = read_mnist_labels("data/mnist/train-labels-idx1-ubyte").unwrap();
    let train_labels_logits: Vec<Matrix<f32>> = train_labels.iter().cloned().map(|x| one_hot(x, 10)).collect();
    println!("Label count: {}", train_labels.len());

    println!("Reading training images...");
    let (rows, columns, train_images) = read_mnist_images("data/mnist/train-images-idx3-ubyte").unwrap();

    let ctx = matrix::Context::new();

    // Setup the graph. It's going to have 2 inputs, 3 hidden nodes in it's 1 hidden layer, and 1
    // output node
    let mut graph = Graph::new();

    //////////////////////////
    // Layer 1

    // Input.  1 batches of rows*columns inputs each.
    let input = graph.add_variable(&ctx, (1, rows*columns), vec![0.0; rows*columns]);
    // Weights for layer 1: [2 inputs x 3 nodes]
    let l1_w = graph.add_variable(&ctx, (rows*columns, 1000), init::Normal(0.001, 0.005));
    // Use matrix multiplication to do a fully connected layer
    let l1_mat_mul = graph.add_node(&ctx,
                                    MatMul::new(&ctx, (1, rows*columns), (rows*columns, 1000)),
                                    vec![input, l1_w],
                                    &[(1, 1000)]); // out shape: [5x2]*[2x3] = [5 batches x 3 outputs]
    // Grab VarIndex for l1_mat_mul's output
    let l1_mat_mul_out = l1_mat_mul.get(&graph).outputs[0]; 
    // 3 biases; one for each node in layer 1
    let l1_b = graph.add_variable(&ctx, (1, 1000), init::Normal(0.001, 0.005));
    // Here we add the biases to the matrix multiplication output
    let l1_biased = graph.add_node(&ctx,
                               Add::new(0),
                               vec![l1_mat_mul_out, l1_b],
                               &[(1, 1000)]);
    // Grab VarIndex for l1_biased's output
    let l1_biased_out = l1_biased.get(&graph).outputs[0];
    // Run the biased input*weight sums through an ReLU activation
    let l1_relu = graph.add_node(&ctx,
                                 Relu::new(),
                                 vec![l1_biased_out],
                                 &[(1, 1000)]);
    let l1_relu_out = l1_relu.get(&graph).outputs[0];

    //////////////////////////
    // Layer 2

    // Weights for layer 2: [3 inputs x 1 nodes]
    let l2_w = graph.add_variable(&ctx, (1000, 10), init::Normal(0.001, 0.005));
    // Fully connected layer 2. Use layer 1's output as layer 2's input
    let l2_mat_mul = graph.add_node(&ctx,
                                    MatMul::new(&ctx, (1, 1000), (1000, 10)),
                                    vec![l1_relu_out, l2_w],
                                    &[(1, 10)]); // out shape: [5x3]*[3x1] = [5 batches x 1 output]
    // Grab VarIndex for l2_mat_mul's output
    let l2_mat_mul_out = l2_mat_mul.get(&graph).outputs[0];
    // 1 bias for 1 output node
    let l2_b = graph.add_variable(&ctx, (1, 10), init::Normal(0.001, 0.005));
    // Here we add the bias to the matrix multiplication output
    let l2_biased = graph.add_node(&ctx,
                               Add::new(0),
                               vec![l2_mat_mul_out, l2_b],
                               &[(1, 10)]);
    // Grab VarIndex for l2_biased's output
    let l2_biased_out = l2_biased.get(&graph).outputs[0];
    // Run the biased input*weight sums through an ReLU activation
    let l2_relu = graph.add_node(&ctx,
                                 Relu::new(),
                                 vec![l2_biased_out],
                                 &[(1, 10)]);
    // Grab VarIndex for l2_relu's output
    let l2_relu_out = l2_relu.get(&graph).outputs[0];

    //////////////////////////
    // Loss

    // Add a variable for the training outputs: [5 batches x 1 output]
    let train_out = graph.add_variable(&ctx, (1, 10), vec![0.0; 10]);
    // Use mean squared error loss function
    let loss = graph.add_node(&ctx,
                              Mse::new(),
                              vec![l2_relu_out, train_out],
                              &[(1, 10)]);
    // Grab the VarIndex for loss's output
    let loss_out = loss.get(&graph).outputs[0];
    // Create a gradient to apply to the loss function
    let loss_d = graph.add_gradient(&ctx, loss, 0);

    ////////////////////////////////////////////////////////////////////////////////////////////////

    // We apply a gradient of -0.1 to the loss function
    let loss_d_cpu = Matrix::from_vec(1, 10, vec![-0.001; 10]);
    loss_d.get(&graph).set(&ctx, &loss_d_cpu);

    /////////////////////////
    // Run/Train the network
    let mut correct = 0;
    for epoch in 0..100000 {
        // Upload training data
        let train_sample = epoch%train_images.len();
        input.get(&graph).set(&ctx, &train_images[train_sample]);
        train_out.get(&graph).set(&ctx, &train_labels_logits[train_sample]);

        graph.run(&ctx);
        let out = l2_relu_out.get(&graph).get(&ctx);
        let out_d = l2_relu.get(&graph)
                           .out_grad[0]
                           .gradient()
                           .get(&graph)
                           .get(&ctx);
        let l = loss_out.get(&graph).get(&ctx);
        let l1_w_d = graph.get_input_gradient(l1_w).unwrap().get(&graph);
        let l2_w_d = graph.get_input_gradient(l2_w).unwrap().get(&graph);
        let l1_b_d = graph.get_input_gradient(l1_b).unwrap().get(&graph);
        let l2_b_d = graph.get_input_gradient(l2_b).unwrap().get(&graph);
        l1_w.get(&graph).add(&ctx, -1, &*l1_w_d, &*l1_w.get(&graph));
        l2_w.get(&graph).add(&ctx, -1, &*l2_w_d, &*l2_w.get(&graph));
        l1_b.get(&graph).add(&ctx, -1, &*l1_b_d, &*l1_b.get(&graph));
        l2_b.get(&graph).add(&ctx, -1, &*l2_b_d, &*l2_b.get(&graph));

        let (mut max_index, mut max_value) = (0, out[(0, 0)]);
        for (i, val) in out.buffer().iter().enumerate() {
            if *val > max_value {
                max_index = i;
                max_value = *val;
            }
        }

        if max_index == train_labels[train_sample] as usize {
            correct += 1;
        }

        if epoch % 1000 == 999 {
            println!("===================");
            println!("Epoch: {}", epoch);
            println!("out = {:?}", out);
            println!("out_d = {:?}", out_d);
            println!("loss = {:?}", l);
            println!("Accuracy: {}%", (correct as f32)/(1000 as f32) * 100.0);
            correct = 0;
        }
    }
}

{'fraction_non_alphanumeric': 0.17764003326026154, 'fraction_numerical': 0.03779575175750246, 'mean_word_length': 1.8245089666951324, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2176978', 'n_tokens_mistral': 4583, 'n_tokens_neox': 4211, 'n_words': 1008}