<commit_before>package BIDMach.models

// Random Forest model (new version).
// Computes a matrix representing the Forest.


import BIDMat.{SBMat,CMat,CSMat,DMat,Dict,IDict,FMat,GMat,GIMat,GSMat,HMat,IMat,LMat,Mat,SMat,SDMat}
import BIDMach.Learner
import BIDMach.datasources.{DataSource,MatDS}
import BIDMach.updaters.Batch
import BIDMat.MatFunctions._
import BIDMat.SciFunctions._
import edu.berkeley.bid.CUMAT
import edu.berkeley.bid.CUMACH
import scala.util.hashing.MurmurHash3
import java.util.Arrays

class RandomForest(override val opts:RandomForest.Opts = new RandomForest.Options) extends Model(opts) {
  
  val ITree = 0; val INode = 1; val JFeat = 2; val IFeat = 3; val IVFeat = 4; val ICat = 5
  
  var nnodes = 0;
  var ntrees = 0;
  var nsamps = 0;
  var nfeats = 0;
  var nbits = 0;
  var ncats = 0;
  var iblock = 0;
  var batchSize = 0;
  var tmpinds:LMat = null;
  var tmpcounts:IMat = null;
  var gtmpinds:GIMat = null;
  var gtmpcounts:GIMat = null;
  var midinds:LMat = null;
  var midcounts:IMat = null;
  var gmidinds:GIMat = null;
  var gmidcounts:GIMat = null;
  var totalinds:LMat = null;
  var totalcounts:IMat = null;
  var nodecounts:IMat = null;
  var itrees:IMat = null;                   // Index of left child (right child is at this value + 1)
  var ftrees:IMat = null;                   // The feature index for this node
  var vtrees:IMat = null;                   // The value to compare with for this node
  var ctrees:FMat = null;                   // Majority class for this node
  var gtrees:GIMat = null;
  var gout:GIMat = null;
  var gtnodes:GIMat = null;
  var outv:IMat = null;                     // Threshold values returned by minImpurity
  var outf:IMat = null;                     // Features returned by minImpurity
  var outn:IMat = null;                     // Node numbers returned by minImpurity
  var outg:FMat = null;                     // Node impurity gain returned by minImpurity
  var outc:FMat = null;                     // Category label (or avg) returned by minImpurity
  var outleft:FMat = null;                  // child categories returned by minImpurity     
  var outright:FMat = null;
  var jc:IMat = null;
  var xnodes:IMat = null;
  var ynodes:FMat = null;
  var gains:FMat = null;
  var igains:FMat = null; 
  val fieldlengths = izeros(1,6);
  var gfieldlengths:GIMat = null;
  var fieldmasks:IMat = null;
  var fieldshifts:IMat = null;
  var cspine:IMat = null;
  var gspine:GIMat = null;
  var t0 = 0f;
  var t1 = 0f;
  var t2 = 0f;
  var t3 = 0f; 
  var t4 = 0f;
  var t5 = 0f;
  var t6 = 0f;
  var midstep = 0;
  val runtimes = zeros(8,1);
  
  def init() = {
    mats = datasource.next;
    nfeats = mats(0).nrows;
    val nc = mats(0).ncols;
    batchSize = nc;
    val nnz = mats(1).nnz;
    datasource.reset;   
    nnodes = opts.nnodes; 
    ntrees = opts.ntrees;
    nsamps = opts.nsamps;
    nbits = opts.nbits;
    ncats = if (opts.ncats > 0) opts.ncats else (maxi(mats(1)).dv.toInt + 1);
    fieldlengths(ITree) = RandomForest.countbits(ntrees);
    fieldlengths(INode) = RandomForest.countbits(nnodes);
    fieldlengths(JFeat) = RandomForest.countbits(nsamps);
    fieldlengths(IFeat) = RandomForest.countbits(nfeats);
    fieldlengths(IVFeat) = nbits;
    fieldlengths(ICat) = RandomForest.countbits(ncats);
    fieldmasks = getFieldMasks(fieldlengths);
    fieldshifts = getFieldShifts(fieldlengths);
    //    if (useGPU) 
    gfieldlengths = GIMat(fieldlengths);
    if (refresh) {
    	if (sum(fieldlengths).v > 64) {
    		throw new RuntimeException("RandomForest: Too many bits in treepack! "+ sum(fieldlengths).v);
    	}
    	opts.asInstanceOf[Learner.Options].npasses = opts.depth;                   // Make sure we make the correct number of passes
    	itrees = izeros(nnodes, ntrees);
    	ftrees = izeros(nnodes, ntrees);
    	vtrees = izeros(nnodes, ntrees);
    	ctrees = zeros(nnodes, ntrees);
    	gains = zeros(ntrees,1);
    	igains = zeros(ntrees,1);
    	nodecounts = iones(opts.ntrees, 1);
    	midstep = opts.midstep;
    	ctrees.set(-1);
    	ctrees(0,?) = 0;
    	ftrees.set(-1)
    	if (opts.useGPU && Mat.hasCUDA > 0) gtrees = GIMat(ftrees);
    	modelmats = Array(itrees, ftrees, vtrees, ctrees);
    	// Small buffers hold results of batch treepack and sort
    	val bsize = (opts.catsPerSample * batchSize * ntrees * nsamps).toInt;
    	tmpinds = lzeros(1, bsize);
    	tmpcounts = izeros(1, bsize);
    	gtmpinds = gizeros(2, bsize);
    	gtmpcounts = gizeros(1, bsize);
    	midinds = new LMat(1, 0, new Array[Long](midstep*bsize));
    	midcounts = new IMat(1, 0, new Array[Int](midstep*bsize));
    	gmidinds = new GIMat(1, 0, GIMat(1, midstep*bsize*2).data, midstep*bsize*2)
    	gmidcounts = new GIMat(1, 0, GIMat(1, midstep*bsize).data, midstep*bsize)
    	// Allocate about half our memory to the main buffers
    	val bufsize = (math.min(java.lang.Runtime.getRuntime().maxMemory()/20, 2000000000)).toInt;
    	totalinds = new LMat(1, 0, new Array[Long](bufsize));
    	totalcounts = new IMat(1, 0, new Array[Int](bufsize));
    	outv = IMat(nsamps, nnodes);
    	outf = IMat(nsamps, nnodes);
    	outn = IMat(nsamps, nnodes);
    	outg = FMat(nsamps, nnodes);
    	outc = FMat(nsamps, nnodes);
    	outleft = FMat(nsamps, nnodes);
    	outright = FMat(nsamps, nnodes);
    	jc = IMat(1, ntrees * nnodes * nsamps);
    	gout = GIMat(2, batchSize * nsamps * ntrees);
    	gtnodes = GIMat(ntrees, batchSize);
    	cspine = IMat(512, 1);
    	gspine = GIMat(512,1);
    }       
  } 
  
  def doblock(gmats:Array[Mat], ipass:Int, i:Long) = {
    val data = gmats(0);
    val cats = gmats(1);
    val nnodes:Mat = if (gmats.length > 2) gmats(2) else null
    val t0 = toc
    val (inds, counts):(LMat, IMat) = (data, cats) match {
      case (fdata:FMat, icats:IMat) => {
//        println("i=%d, hash=%f" format (i, sum(sum(fdata,2)).v))
        xnodes = if (nnodes.asInstanceOf[AnyRef] != null) {
          val nn = nnodes.asInstanceOf[IMat];
          treeStep(fdata, nn, null, itrees, ftrees, vtrees, ctrees, false);
          Mat.nflops += fdata.ncols;
          nn;
        } else {
          Mat.nflops += fdata.ncols * ipass;
          val nn = izeros(ntrees, fdata.ncols)
          treeWalk(fdata, nn, null, itrees, ftrees, vtrees, ctrees, ipass, false);
          nn;
        }
        //         print("xnodes="+xnodes.toString)
        t1 = toc;
        runtimes(0) += t1 - t0;
        val nxvals = gtreePack(fdata, xnodes, icats, gout, ipass + opts.seed*341431);
//        println("lt "+lt(0->100).toString)
        Mat.nflops += icats.length * nsamps * ntrees * 6;
        t2 = toc;
        runtimes(1) += t2 - t1;
        val lt = gpsort(nxvals, gout, tmpinds);  
        Mat.nflops += lt.length * math.log(lt.length).toLong;
        t3 = toc;
        runtimes(2) += t3 - t2;
        makeV(lt, tmpcounts);
      }
      case _ => {
        throw new RuntimeException("RandomForest doblock types dont match %s %s" format (data.mytype, cats.mytype))
      }
    }
    val (g1, g2) = gmakeV(gout, gtmpinds, gtmpcounts);
    val ginds = LMat(g1);
    val gcnts = IMat(g2);
    println("sizes %d %d %d" format(gout.length, inds.length, ginds.length))
    println("indsdelta = %ld %ld" format (maxi(ginds - inds).v, mini(ginds - inds).v))
    println("countsdelta = %d %d" format (maxi(gcnts - counts).v, mini(ginds - inds).v))
    t4 = toc;
    runtimes(3) += t4 - t3;
    val (inds1, counts1) = addV(inds, counts, midinds, midcounts);
    midinds = inds1;
    midcounts = counts1;
    iblock += 1;<commit_msg>speed up makeV<commit_after>package BIDMach.models

// Random Forest model (new version).
// Computes a matrix representing the Forest.


import BIDMat.{SBMat,CMat,CSMat,DMat,Dict,IDict,FMat,GMat,GIMat,GSMat,HMat,IMat,LMat,Mat,SMat,SDMat}
import BIDMach.Learner
import BIDMach.datasources.{DataSource,MatDS}
import BIDMach.updaters.Batch
import BIDMat.MatFunctions._
import BIDMat.SciFunctions._
import edu.berkeley.bid.CUMAT
import edu.berkeley.bid.CUMACH
import scala.util.hashing.MurmurHash3
import java.util.Arrays

class RandomForest(override val opts:RandomForest.Opts = new RandomForest.Options) extends Model(opts) {
  
  val ITree = 0; val INode = 1; val JFeat = 2; val IFeat = 3; val IVFeat = 4; val ICat = 5
  
  var nnodes = 0;
  var ntrees = 0;
  var nsamps = 0;
  var nfeats = 0;
  var nbits = 0;
  var ncats = 0;
  var iblock = 0;
  var batchSize = 0;
  var tmpinds:LMat = null;
  var tmpcounts:IMat = null;
  var gtmpinds:GIMat = null;
  var gtmpcounts:GIMat = null;
  var midinds:LMat = null;
  var midcounts:IMat = null;
  var gmidinds:GIMat = null;
  var gmidcounts:GIMat = null;
  var totalinds:LMat = null;
  var totalcounts:IMat = null;
  var nodecounts:IMat = null;
  var itrees:IMat = null;                   // Index of left child (right child is at this value + 1)
  var ftrees:IMat = null;                   // The feature index for this node
  var vtrees:IMat = null;                   // The value to compare with for this node
  var ctrees:FMat = null;                   // Majority class for this node
  var gtrees:GIMat = null;
  var gout:GIMat = null;
  var gtnodes:GIMat = null;
  var outv:IMat = null;                     // Threshold values returned by minImpurity
  var outf:IMat = null;                     // Features returned by minImpurity
  var outn:IMat = null;                     // Node numbers returned by minImpurity
  var outg:FMat = null;                     // Node impurity gain returned by minImpurity
  var outc:FMat = null;                     // Category label (or avg) returned by minImpurity
  var outleft:FMat = null;                  // child categories returned by minImpurity     
  var outright:FMat = null;
  var jc:IMat = null;
  var xnodes:IMat = null;
  var ynodes:FMat = null;
  var gains:FMat = null;
  var igains:FMat = null; 
  val fieldlengths = izeros(1,6);
  var gfieldlengths:GIMat = null;
  var fieldmasks:IMat = null;
  var fieldshifts:IMat = null;
  var cspine:IMat = null;
  var gspine:GIMat = null;
  var t0 = 0f;
  var t1 = 0f;
  var t2 = 0f;
  var t3 = 0f; 
  var t4 = 0f;
  var t5 = 0f;
  var t6 = 0f;
  var midstep = 0;
  val runtimes = zeros(8,1);
  
  def init() = {
    mats = datasource.next;
    nfeats = mats(0).nrows;
    val nc = mats(0).ncols;
    batchSize = nc;
    val nnz = mats(1).nnz;
    datasource.reset;   
    nnodes = opts.nnodes; 
    ntrees = opts.ntrees;
    nsamps = opts.nsamps;
    nbits = opts.nbits;
    ncats = if (opts.ncats > 0) opts.ncats else (maxi(mats(1)).dv.toInt + 1);
    fieldlengths(ITree) = RandomForest.countbits(ntrees);
    fieldlengths(INode) = RandomForest.countbits(nnodes);
    fieldlengths(JFeat) = RandomForest.countbits(nsamps);
    fieldlengths(IFeat) = RandomForest.countbits(nfeats);
    fieldlengths(IVFeat) = nbits;
    fieldlengths(ICat) = RandomForest.countbits(ncats);
    fieldmasks = getFieldMasks(fieldlengths);
    fieldshifts = getFieldShifts(fieldlengths);
    //    if (useGPU) 
    gfieldlengths = GIMat(fieldlengths);
    if (refresh) {
    	if (sum(fieldlengths).v > 64) {
    		throw new RuntimeException("RandomForest: Too many bits in treepack! "+ sum(fieldlengths).v);
    	}
    	opts.asInstanceOf[Learner.Options].npasses = opts.depth;                   // Make sure we make the correct number of passes
    	itrees = izeros(nnodes, ntrees);
    	ftrees = izeros(nnodes, ntrees);
    	vtrees = izeros(nnodes, ntrees);
    	ctrees = zeros(nnodes, ntrees);
    	gains = zeros(ntrees,1);
    	igains = zeros(ntrees,1);
    	nodecounts = iones(opts.ntrees, 1);
    	midstep = opts.midstep;
    	ctrees.set(-1);
    	ctrees(0,?) = 0;
    	ftrees.set(-1)
    	if (opts.useGPU && Mat.hasCUDA > 0) gtrees = GIMat(ftrees);
    	modelmats = Array(itrees, ftrees, vtrees, ctrees);
    	// Small buffers hold results of batch treepack and sort
    	val bsize = (opts.catsPerSample * batchSize * ntrees * nsamps).toInt;
    	tmpinds = lzeros(1, bsize);
    	tmpcounts = izeros(1, bsize);
    	gtmpinds = gizeros(2, bsize);
    	gtmpcounts = gizeros(1, bsize);
    	midinds = new LMat(1, 0, new Array[Long](midstep*bsize));
    	midcounts = new IMat(1, 0, new Array[Int](midstep*bsize));
    	gmidinds = new GIMat(1, 0, GIMat(1, midstep*bsize*2).data, midstep*bsize*2)
    	gmidcounts = new GIMat(1, 0, GIMat(1, midstep*bsize).data, midstep*bsize)
    	// Allocate about half our memory to the main buffers
    	val bufsize = (math.min(java.lang.Runtime.getRuntime().maxMemory()/20, 2000000000)).toInt;
    	totalinds = new LMat(1, 0, new Array[Long](bufsize));
    	totalcounts = new IMat(1, 0, new Array[Int](bufsize));
    	outv = IMat(nsamps, nnodes);
    	outf = IMat(nsamps, nnodes);
    	outn = IMat(nsamps, nnodes);
    	outg = FMat(nsamps, nnodes);
    	outc = FMat(nsamps, nnodes);
    	outleft = FMat(nsamps, nnodes);
    	outright = FMat(nsamps, nnodes);
    	jc = IMat(1, ntrees * nnodes * nsamps);
    	gout = GIMat(2, batchSize * nsamps * ntrees);
    	gtnodes = GIMat(ntrees, batchSize);
    	cspine = IMat(512, 1);
    	gspine = GIMat(512,1);
    }       
  } 
  
  def doblock(gmats:Array[Mat], ipass:Int, i:Long) = {
    val data = gmats(0);
    val cats = gmats(1);
    val nnodes:Mat = if (gmats.length > 2) gmats(2) else null
    val t0 = toc
    val (inds, counts):(LMat, IMat) = (data, cats) match {
      case (fdata:FMat, icats:IMat) => {
//        println("i=%d, hash=%f" format (i, sum(sum(fdata,2)).v))
        xnodes = if (nnodes.asInstanceOf[AnyRef] != null) {
          val nn = nnodes.asInstanceOf[IMat];
          treeStep(fdata, nn, null, itrees, ftrees, vtrees, ctrees, false);
          Mat.nflops += fdata.ncols;
          nn;
        } else {
          Mat.nflops += fdata.ncols * ipass;
          val nn = izeros(ntrees, fdata.ncols)
          treeWalk(fdata, nn, null, itrees, ftrees, vtrees, ctrees, ipass, false);
          nn;
        }
        //         print("xnodes="+xnodes.toString)
        t1 = toc;
        runtimes(0) += t1 - t0;
        val nxvals = gtreePack(fdata, xnodes, icats, gout, ipass + opts.seed*341431);
//        println("lt "+lt(0->100).toString)
        Mat.nflops += icats.length * nsamps * ntrees * 6;
        t2 = toc;
        runtimes(1) += t2 - t1;
        val lt = gpsort(nxvals, gout, tmpinds);  
        gout = new GIMat(2, nxvals, gout.data, gout.realsize);
        println("lengths %d %d" format (lt.length, gout.length))
        Mat.nflops += lt.length * math.log(lt.length).toLong;
        t3 = toc;
        runtimes(2) += t3 - t2;
        makeV(lt, tmpcounts);
      }
      case _ => {
        throw new RuntimeException("RandomForest doblock types dont match %s %s" format (data.mytype, cats.mytype))
      }
    }
    val (g1, g2) = gmakeV(gout, gtmpinds, gtmpcounts);
    val ginds = LMat(g1);
    val gcnts = IMat(g2);
    println("sizes %d %d %d" format(gout.length, inds.length, ginds.length))
    println("indsdelta = %ld %ld" format (maxi(ginds - inds).v, mini(ginds - inds).v))
    println("countsdelta = %d %d" format (maxi(gcnts - counts).v, mini(ginds - inds).v))
    t4 = toc;
    runtimes(3) += t4 - t3;
    val (inds1, counts1) = addV(inds, counts, midinds, midcounts);
    midinds = inds1;
    midcounts = counts1;
    iblock += 1;
{'fraction_non_alphanumeric': 0.12760365332807674, 'fraction_numerical': 0.0178066890071621, 'mean_word_length': 2.6888027144934563, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6264475', 'n_tokens_mistral': 6063, 'n_tokens_neox': 5818, 'n_words': 1389}