<commit_before>package pipelines

import breeze.stats.distributions.Rand
import breeze.linalg._
import breeze.numerics._
import evaluation.MulticlassClassifierEvaluator
import loaders.{CifarLoader, MnistLoader, SmallMnistLoader}
import nodes.images._
import nodes.learning.{BlockLeastSquaresEstimator, BlockWeightedLeastSquaresEstimator}
import nodes.stats.{StandardScaler, Sampler}
import nodes.util.{Identity, Cacher, ClassLabelIndicatorsFromIntLabels, TopKClassifier, MaxClassifier}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD
import pipelines.Logging
import scopt.OptionParser
import workflow.Pipeline
import utils.{Image, MatrixUtils, Stats, ImageMetadata, LabeledImage}


import scala.reflect.BeanProperty
import org.yaml.snakeyaml.Yaml
import org.yaml.snakeyaml.constructor.Constructor

object CKM extends Serializable with Logging {
  val appName = "CKM"

  def run(sc: SparkContext, conf: CKMConf) {
    val data: Dataset = loadData(sc, conf.dataset)

    val (xDim, yDim, numChannels) = getInfo(data)

    var currX = xDim
    var currY = yDim

    var convKernel: Pipeline[Image, Image] = new Identity()
    var numInputFeatures = numChannels

    for (i <- 0 until conf.layers) {
      var numOutputFeatures = conf.filters(i)
      val patchSize = math.pow(conf.patch_sizes(i), 2).toInt
      val W = DenseMatrix.rand(numOutputFeatures, numInputFeatures*patchSize, Rand.gaussian) :* conf.bandwidth(i)
      val b = DenseVector.rand(numOutputFeatures, Rand.uniform) :* (2*math.Pi)
      val ccap = new CCaP(W, b, currX, currY, numInputFeatures, 2, 2)
      convKernel = convKernel andThen ccap

      currX = ccap.outX
      currY = ccap.outY

      numInputFeatures = numOutputFeatures
    }


    val featurizer = ImageExtractor andThen convKernel andThen ImageVectorizer andThen new Cacher[DenseVector[Double]]

    val XTrain = featurizer(data.train)
    val count = XTrain.count()
    val XTest = featurizer(data.test)
    XTest.count()
    val labelVectorizer = ClassLabelIndicatorsFromIntLabels(conf.numClasses)

    val yTrain = labelVectorizer(LabelExtractor(data.train))
    val yTest = labelVectorizer(LabelExtractor(data.test)).map(convert(_, Int).toArray)
    val numFeatures = XTrain.take(1)(0).size
    println(s"numFeatures: ${numFeatures}, count: ${count}")
    println(data.train.take(1)(0).label)
    val clf = new BlockWeightedLeastSquaresEstimator(numFeatures, 1, conf.reg, 0.5).fit(XTrain, yTrain) andThen MaxClassifier

    val yTrainPred = clf.apply(XTrain)
    val yTestPred =  clf.apply(XTest)

    val trainEval = MulticlassClassifierEvaluator(yTrainPred, LabelExtractor(data.train), 10)
    val testEval = MulticlassClassifierEvaluator(yTestPred, LabelExtractor(data.test), 10)
    println(s"total training accuracy ${1 - trainEval.totalError}")<commit_msg>controlling randomness<commit_after>package pipelines

import breeze.stats.distributions._
import breeze.linalg._
import breeze.numerics._
import evaluation.MulticlassClassifierEvaluator
import loaders.{CifarLoader, MnistLoader, SmallMnistLoader}
import nodes.images._
import nodes.learning.{BlockLeastSquaresEstimator, BlockWeightedLeastSquaresEstimator}
import nodes.stats.{StandardScaler, Sampler}
import nodes.util.{Identity, Cacher, ClassLabelIndicatorsFromIntLabels, TopKClassifier, MaxClassifier}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD
import pipelines.Logging
import scopt.OptionParser
import workflow.Pipeline
import utils.{Image, MatrixUtils, Stats, ImageMetadata, LabeledImage}
import org.apache.log4j.Logger
import org.apache.log4j.Level
import org.apache.commons.math3.random.MersenneTwister


import scala.reflect.BeanProperty
import org.yaml.snakeyaml.Yaml
import org.yaml.snakeyaml.constructor.Constructor

import java.io.{File, BufferedWriter, FileWriter}

object CKM extends Serializable with Logging {
  val appName = "CKM"

  def run(sc: SparkContext, conf: CKMConf) {
    val data: Dataset = loadData(sc, conf.dataset)

    val (xDim, yDim, numChannels) = getInfo(data)
    var currX = xDim
    var currY = yDim

    var convKernel: Pipeline[Image, Image] = new Identity()
    var numInputFeatures = numChannels

      implicit val randBasis: RandBasis = new RandBasis(new ThreadLocalRandomGenerator(new MersenneTwister(conf.seed)))
      val gaussian = new Gaussian(0, 1)
      val uniform = new Uniform(0, 1)

    for (i <- 0 until conf.layers) {

      var numOutputFeatures = conf.filters(i)
      val patchSize = math.pow(conf.patch_sizes(i), 2).toInt
      val W = DenseMatrix.rand(numOutputFeatures, numInputFeatures*patchSize, gaussian) :* conf.bandwidth(i)
      val b = DenseVector.rand(numOutputFeatures, uniform) :* (2*math.Pi)
      val ccap = new CCaP(W, b, currX, currY, numInputFeatures, 2, 2)
      convKernel = convKernel andThen ccap

      currX = ccap.outX
      currY = ccap.outY

      numInputFeatures = numOutputFeatures
    }


    val featurizer = ImageExtractor andThen convKernel andThen ImageVectorizer andThen new Cacher[DenseVector[Double]]

    val XTrain = featurizer(data.train)
    val count = XTrain.count()
    val XTest = featurizer(data.test)
    XTest.count()
    val labelVectorizer = ClassLabelIndicatorsFromIntLabels(conf.numClasses)

    val yTrain = labelVectorizer(LabelExtractor(data.train))
    val yTest = labelVectorizer(LabelExtractor(data.test)).map(convert(_, Int).toArray)
    val numFeatures = XTrain.take(1)(0).size
    println(s"numFeatures: ${numFeatures}, count: ${count}")
    println(data.train.take(1)(0).label)
    val clf = new BlockWeightedLeastSquaresEstimator(numFeatures, 1, conf.reg, 0.5).fit(XTrain, yTrain) andThen MaxClassifier

    val yTrainPred = clf.apply(XTrain)
    val yTestPred =  clf.apply(XTest)

    val trainEval = MulticlassClassifierEvaluator(yTrainPred, LabelExtractor(data.train), 10)
    val testEval = MulticlassClassifierEvaluator(yTestPred, LabelExtractor(data.test), 10)
    println(s"total training accuracy ${1 - trainEval.totalError}")
{'fraction_non_alphanumeric': 0.09651741293532339, 'fraction_numerical': 0.0067993366500829185, 'mean_word_length': 4.782358581016299, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6955479', 'n_tokens_mistral': 2012, 'n_tokens_neox': 1911, 'n_words': 432}