<commit_before>package BIDMach.networks

import BIDMat.{Mat,SBMat,CMat,DMat,FMat,IMat,LMat,HMat,GMat,GDMat,GIMat,GLMat,GSMat,GSDMat,JSON,SMat,SDMat}
import BIDMat.MatFunctions._
import BIDMat.SciFunctions._
import BIDMach.datasources._
import BIDMach.updaters._
import BIDMach.mixins._
import BIDMach.models._
import BIDMach._
import scala.util.hashing.MurmurHash3;
import java.util.HashMap;

/**
 * Basic Net class. Learns a supervised map from input blocks to output (target) data blocks. 
 *
 * The network topology is specified by opts.layers which is a sequence of "LayerOptions" objects. There is a LayerOptions
 * Class for each Layer class, which holds the params for defining that layer. There is also an inputs parameter which points
 * to the set of Layer.Options instances that mirror the final network structure. 
 *
 */

class Net(override val opts:Net.Opts = new Net.Options) extends Model(opts) {
  var layers:Array[Layer] = null;
  var output_layers:Array[Layer] = null;
  var targmap:Mat = null;
  var mask:Mat = null;
  var bufmat:Mat = null;
  var modelMap:HashMap[String,Int] = null;
  var batchSize = -1;
  var imodel = 0;
  var initialize = false;

  override def init() = {
	  mats = datasource.next;
	  var nfeats = mats(0).nrows;
	  batchSize = mats(0).ncols
	  targmap = if (opts.targmap.asInstanceOf[AnyRef] != null) convertMat(opts.targmap) else null;
	  mask = if (opts.dmask.asInstanceOf[AnyRef] != null) convertMat(opts.dmask) else null;
	  createLayers;
    if (output_layers == null) output_layers = Array(layers(layers.length-1));
	  if (modelMap == null) {
	  	modelMap = new HashMap[String,Int];
	  	imodel = 0;
	  	layers.map(_.getModelMats(this));
	  }
	  if (refresh) {
	  	setmodelmats(new Array[Mat](imodel + modelMap.size));
	  }
	  if (updatemats == null) updatemats = new Array[Mat](modelmats.length);
	  for (i <- 0 until modelmats.length) {
	  	if (modelmats(i).asInstanceOf[AnyRef] != null) modelmats(i) = convertMat(modelmats(i));
	  	if (updatemats(i).asInstanceOf[AnyRef] != null) {
        updatemats(i) = convertMat(updatemats(i));
        updatemats(i).clear;
	  	}
	  };
	  if (useGPU) copyMats(mats, gmats);
	  val pb = putBack;
	  putBack = -1;
    initialize = true;
    evalbatch(gmats, 0, 0);
    initialize = false;<commit_msg>fix getmodelmap problem<commit_after>package BIDMach.networks

import BIDMat.{Mat,SBMat,CMat,DMat,FMat,IMat,LMat,HMat,GMat,GDMat,GIMat,GLMat,GSMat,GSDMat,JSON,SMat,SDMat}
import BIDMat.MatFunctions._
import BIDMat.SciFunctions._
import BIDMach.datasources._
import BIDMach.updaters._
import BIDMach.mixins._
import BIDMach.models._
import BIDMach._
import scala.util.hashing.MurmurHash3;
import java.util.HashMap;

/**
 * Basic Net class. Learns a supervised map from input blocks to output (target) data blocks. 
 *
 * The network topology is specified by opts.layers which is a sequence of "LayerOptions" objects. There is a LayerOptions
 * Class for each Layer class, which holds the params for defining that layer. There is also an inputs parameter which points
 * to the set of Layer.Options instances that mirror the final network structure. 
 *
 */

class Net(override val opts:Net.Opts = new Net.Options) extends Model(opts) {
  var layers:Array[Layer] = null;
  var output_layers:Array[Layer] = null;
  var targmap:Mat = null;
  var mask:Mat = null;
  var bufmat:Mat = null;
  var modelMap:HashMap[String,Int] = null;
  var batchSize = -1;
  var imodel = 0;
  var initialize = false;

  override def init() = {
	  mats = datasource.next;
	  var nfeats = mats(0).nrows;
	  batchSize = mats(0).ncols
	  targmap = if (opts.targmap.asInstanceOf[AnyRef] != null) convertMat(opts.targmap) else null;
	  mask = if (opts.dmask.asInstanceOf[AnyRef] != null) convertMat(opts.dmask) else null;
	  createLayers;
    if (output_layers == null) output_layers = Array(layers(layers.length-1));
	  if (modelMap == null) {
	  	modelMap = new HashMap[String,Int];
	  	imodel = 0;
	  }
	  layers.map(_.getModelMats(this));
	  if (refresh) {
	  	setmodelmats(new Array[Mat](imodel + modelMap.size));
	  }
	  if (updatemats == null) updatemats = new Array[Mat](modelmats.length);
	  for (i <- 0 until modelmats.length) {
	  	if (modelmats(i).asInstanceOf[AnyRef] != null) modelmats(i) = convertMat(modelmats(i));
	  	if (updatemats(i).asInstanceOf[AnyRef] != null) {
        updatemats(i) = convertMat(updatemats(i));
        updatemats(i).clear;
	  	}
	  };
	  if (useGPU) copyMats(mats, gmats);
	  val pb = putBack;
	  putBack = -1;
    initialize = true;
    evalbatch(gmats, 0, 0);
    initialize = false;
{'fraction_non_alphanumeric': 0.11952017448200654, 'fraction_numerical': 0.004798255179934569, 'mean_word_length': 4.073008849557522, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6974934', 'n_tokens_mistral': 1702, 'n_tokens_neox': 1622, 'n_words': 411}