<commit_before>import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf
import org.apache.spark.rdd.cl._
import Array._
import scala.math._
import org.apache.spark.rdd._
import java.net._

import org.apache.spark.mllib.linalg.DenseVector
import org.apache.spark.mllib.linalg.Vectors

object SparkNN {
    def main(args : Array[String]) {
        if (args.length < 1) {
            println("usage: SparkNN cmd")
            return;
        }

        val cmd = args(0)

        if (cmd == "convert") {
            convert(args.slice(1, args.length))
        } else if (cmd == "run") {
            run_nn(args.slice(2, args.length), args(1).toBoolean)
        } else if (cmd == "check") {
            // TODO
            val correct = run_nn(args.slice(1, args.length), false)
            val actual = run_nn(args.slice(1, args.length), true)
            System.err.println("PASSED")
        }
    }

    def get_spark_context(appName : String) : SparkContext = {
        val conf = new SparkConf()
        conf.setAppName(appName)

        val localhost = InetAddress.getLocalHost
        conf.setMaster("spark://" + localhost.getHostName + ":7077") // 7077 is the default port

        return new SparkContext(conf)
    }

    def sigmoid(z : Double) : Double = {
      1.0 / (1.0 + Math.exp(-z))
    }

    def inv_sigmoid(a : Double) : Double = {
      -1.0 * Math.log((1.0 / a) - 1.0)
    }

    def sigmoid_prime(z : Double) : Double = {
      sigmoid(z) * (1.0 - sigmoid(z))
    }

    def get_nabla_w(delta : RDD[DenseVector], activation : RDD[DenseVector]) :
        RDD[DenseVector] = {
      val delta_with_activation = delta.zip(activation)
      delta_with_activation.map(d_with_a => {
        val d : DenseVector = d_with_a._1
        val a : DenseVector = d_with_a._2
        val layerSize = d.size
        val prevLayerSize = a.size
        val new_w : Array[Double] = new Array[Double](prevLayerSize * layerSize)

        var i = 0
        while (i < prevLayerSize * layerSize) {
          new_w(i) = 0.0
          i += 1
        }

        i = 0
        while (i < prevLayerSize) {
            var j = 0
            while (j < layerSize) {
                new_w(i * layerSize + j)  += (a(i) * d(j))
                j += 1
            }
            i += 1
        }
        Vectors.dense(new_w).asInstanceOf[DenseVector]
      })
    }

    def reduce_sum(rdd : RDD[DenseVector]) : DenseVector = {
      rdd.reduce(
        (a : DenseVector, b : DenseVector) => {
          val size = a.size
          val combined : Array[Double] = new Array[Double](size)
          var i = 0
          while (i < size) {
            combined(i) = a(i) + b(i)
            i += 1
          }
          Vectors.dense(combined).asInstanceOf[DenseVector]
        })
    }

    // Return the weights and biases of each layer?
    def run_nn(args : Array[String], useSwat : Boolean) :
          Tuple2[Array[DenseVector], Array[DenseVector]] = {
        if (args.length != 4) {
            System.err.println("usage: SparkNN run info-file " +
                    "training-data-path correct-data-path niters")
            return (new Array[DenseVector](0), new Array[DenseVector](0))
        }
        val infoFilename = args(0)
        val trainingDataPath = args(1)
        val correctDataPath = args(2)
        val iters = args(3).toInt
        val sc = get_spark_context("Spark NN");

        var rand : java.util.Random = new java.util.Random(1)

        val infoLines = scala.io.Source.fromFile(infoFilename).getLines()
        val layerDimensionalities : Array[Int] = new Array[Int](infoLines.length)<commit_msg>add some documentation of inputs<commit_after>import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf
import org.apache.spark.rdd.cl._
import Array._
import scala.math._
import org.apache.spark.rdd._
import java.net._

import org.apache.spark.mllib.linalg.DenseVector
import org.apache.spark.mllib.linalg.Vectors

object SparkNN {
    def main(args : Array[String]) {
        if (args.length < 1) {
            println("usage: SparkNN cmd")
            return;
        }

        val cmd = args(0)

        if (cmd == "convert") {
            convert(args.slice(1, args.length))
        } else if (cmd == "run") {
            run_nn(args.slice(2, args.length), args(1).toBoolean)
        } else if (cmd == "check") {
            // TODO
            val correct = run_nn(args.slice(1, args.length), false)
            val actual = run_nn(args.slice(1, args.length), true)
            System.err.println("PASSED")
        }
    }

    def get_spark_context(appName : String) : SparkContext = {
        val conf = new SparkConf()
        conf.setAppName(appName)

        val localhost = InetAddress.getLocalHost
        conf.setMaster("spark://" + localhost.getHostName + ":7077") // 7077 is the default port

        return new SparkContext(conf)
    }

    def sigmoid(z : Double) : Double = {
      1.0 / (1.0 + Math.exp(-z))
    }

    def inv_sigmoid(a : Double) : Double = {
      -1.0 * Math.log((1.0 / a) - 1.0)
    }

    def sigmoid_prime(z : Double) : Double = {
      sigmoid(z) * (1.0 - sigmoid(z))
    }

    def get_nabla_w(delta : RDD[DenseVector], activation : RDD[DenseVector]) :
        RDD[DenseVector] = {
      val delta_with_activation = delta.zip(activation)
      delta_with_activation.map(d_with_a => {
        val d : DenseVector = d_with_a._1
        val a : DenseVector = d_with_a._2
        val layerSize = d.size
        val prevLayerSize = a.size
        val new_w : Array[Double] = new Array[Double](prevLayerSize * layerSize)

        var i = 0
        while (i < prevLayerSize * layerSize) {
          new_w(i) = 0.0
          i += 1
        }

        i = 0
        while (i < prevLayerSize) {
            var j = 0
            while (j < layerSize) {
                new_w(i * layerSize + j)  += (a(i) * d(j))
                j += 1
            }
            i += 1
        }
        Vectors.dense(new_w).asInstanceOf[DenseVector]
      })
    }

    def reduce_sum(rdd : RDD[DenseVector]) : DenseVector = {
      rdd.reduce(
        (a : DenseVector, b : DenseVector) => {
          val size = a.size
          val combined : Array[Double] = new Array[Double](size)
          var i = 0
          while (i < size) {
            combined(i) = a(i) + b(i)
            i += 1
          }
          Vectors.dense(combined).asInstanceOf[DenseVector]
        })
    }

    // Return the weights and biases of each layer?
    def run_nn(args : Array[String], useSwat : Boolean) :
          Tuple2[Array[DenseVector], Array[DenseVector]] = {
        if (args.length != 4) {
            System.err.println("usage: SparkNN run info-file " +
                    "training-data-path correct-data-path niters")
            return (new Array[DenseVector](0), new Array[DenseVector](0))
        }
        /*
         * infoFilename should have one line for each layer in the neural net,
         * containing a single integer that is the number of neurons in that
         * layer. This includes the input layer and output layer.
         */
        val infoFilename = args(0)
        /*
         * Path to the input training data to use. This should be in object file
         * format and consists of DenseVector inputs, one for each input data
         * point. The dimensionality of these input vectors must equal the value
         * on the first line of infoFilename.
         */
        val trainingDataPath = args(1)
        /*
         * The expected output for each of the input points in trainingDataPath.
         * Also in object file format and containing DenseVectors, the size of
         * each of these vectors should equal the value on the last line of
         * infoFilename.
         */
        val correctDataPath = args(2)
        // Number of iters to train the neural net over
        val iters = args(3).toInt
        val sc = get_spark_context("Spark NN");

        var rand : java.util.Random = new java.util.Random(1)

        val infoLines = scala.io.Source.fromFile(infoFilename).getLines()
        val layerDimensionalities : Array[Int] = new Array[Int](infoLines.length)
{'fraction_non_alphanumeric': 0.11512798738323426, 'fraction_numerical': 0.01164624529904161, 'mean_word_length': 1.9338078291814946, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3429235', 'n_tokens_mistral': 2697, 'n_tokens_neox': 2570, 'n_words': 639}