<commit_before>package BIDMach
import BIDMat.{Mat,BMat,CMat,CSMat,DMat,FMat,IMat,HMat,GMat,GIMat,GSMat,SMat,SDMat}
import BIDMat.MatFunctions._
import BIDMat.SciFunctions._
import scala.actors._
import java.io._

abstract class DataSource(opts:DataSource.Options = new DataSource.Options) { 
  val options = opts
  
  def next:Array[Mat]
  
  def hasNext:Boolean
}

class MatDataSource(mats:Array[Mat], opts:DataSource.Options = new DataSource.Options) extends DataSource(opts) { 
  val sizeMargin = opts.sizeMargin
  var here = 0
  val blockSize = opts.blockSize
  var omats:Array[Mat] = null
  
  def init() = {
    here = 0
    omats = new Array[Mat](mats.length)
    for (i <- 0 until mats.length) {
      omats(i) = mats(i) match {
        case mm:SMat => SMat(mats(i).nrows, blockSize, (mats(i).nnz * sizeMargin * blockSize / mats(i).ncols).toInt)
        case mm:SDMat => SDMat(mats(i).nrows, blockSize, (mats(i).nnz * sizeMargin * blockSize / mats(i).ncols).toInt)
        case _ => mats(i).zeros(mats(i).nrows, blockSize)
      }      
    }    
  }
  
  def next:Array[Mat] = {
    val there = math.min(here+blockSize, mats(0).ncols)
  	for (i <- 0 until mats.length) {
  	  omats(i) = mats(i).colslice(here, there, omats(i))
  	}
    here = math.min(here+blockSize, mats(0).ncols)
  	omats
  }
  
  def hasNext:Boolean = {
    here < mats(0).ncols
  }

}

class FilesDataSource(fnames:CSMat, dirname:(Int)=>String, nstart:Int, nend:Int, transpose:IMat=null,
		opts:FilesDataSource.Options = new FilesDataSource.Options) extends DataSource(opts) { 
  val sizeMargin = opts.sizeMargin
  val blockSize = opts.blockSize
  var fileno = nstart
  var colno = 0
  var omats:Array[Mat] = null
  var matqueue:Array[Array[Mat]] = null
  var ready:IMat = null
  
  def init = {
    fileno = nstart                                           // Number of the current output file
    colno = 0                                                 // Column number in the current output file
    omats = new Array[Mat](fnames.size)
    matqueue = new Array[Array[Mat]](fnames.size)             // Queue of matrices for each output matrix
    ready = -iones(opts.lookahead, 1)                         // Numbers of files currently loaded in queue
    for (i <- 0 until fnames.size) {
      var mm = HMat.loadMat(dirname(nstart) + fnames(i))
      if (transpose.asInstanceOf[AnyRef] != null && transpose(i) == 1) mm = mm.t
      omats(i) = mm match {
      case mm:SMat => SMat(mm.nrows, blockSize, (mm.nnz * sizeMargin * blockSize / mm.ncols).toInt)
      case mm:SDMat => SDMat(mm.nrows, blockSize, (mm.nnz * sizeMargin * blockSize / mm.ncols).toInt)
      case _ => mm.zeros(mm.nrows, blockSize)
      } 
      matqueue(i) = new Array[Mat](opts.lookahead)
    }
    for (i <- 0 until opts.lookahead) {
      Actor.actor {
        prefetch(nstart + i)
      }
    }
  }
  
  def next:Array[Mat] = {
    val filex = fileno % opts.lookahead
    var donextfile = false
    var todo = 0
    while (ready(filex) < fileno) Thread.`yield`
    for (i <- 0 until fnames.size) {
    	val matq = matqueue(i)(filex)
    	val ccols = math.min(colno + blockSize, matq.ncols)
    	omats(i) = matq(?, colno -> ccols)
    	todo = blockSize - ccols + colno
    }
    if (todo > 0) {    	
    	var done = false
    	while (!done && fileno+1 < nend) {
    		val filey = (fileno+1) % opts.lookahead<commit_msg>fixing DataSource<commit_after>package BIDMach
import BIDMat.{Mat,BMat,CMat,CSMat,DMat,FMat,IMat,HMat,GMat,GIMat,GSMat,SMat,SDMat}
import BIDMat.MatFunctions._
import BIDMat.SciFunctions._
import scala.actors._
import java.io._

abstract class DataSource(opts:DataSource.Options = new DataSource.Options) { 
  val options = opts
  
  def next:Array[Mat]
  
  def hasNext:Boolean
}

class MatDataSource(mats:Array[Mat], opts:DataSource.Options = new DataSource.Options) extends DataSource(opts) { 
  val sizeMargin = opts.sizeMargin
  var here = 0
  val blockSize = opts.blockSize
  var omats:Array[Mat] = null
  
  def init() = {
    here = 0
    omats = new Array[Mat](mats.length)
    for (i <- 0 until mats.length) {
      omats(i) = mats(i) match {
        case mm:SMat => SMat(mats(i).nrows, blockSize, (mats(i).nnz * sizeMargin * blockSize / mats(i).ncols).toInt)
        case mm:SDMat => SDMat(mats(i).nrows, blockSize, (mats(i).nnz * sizeMargin * blockSize / mats(i).ncols).toInt)
        case _ => mats(i).zeros(mats(i).nrows, blockSize)
      }      
    }    
  }
  
  def next:Array[Mat] = {
    val there = math.min(here+blockSize, mats(0).ncols)
  	for (i <- 0 until mats.length) {
  	  omats(i) = mats(i).colslice(here, there, omats(i))
  	}
    here = math.min(here+blockSize, mats(0).ncols)
  	omats
  }
  
  def hasNext:Boolean = {
    here < mats(0).ncols
  }

}

class FilesDataSource(fnames:CSMat, dirname:(Int)=>String, nstart:Int, nend:Int, transpose:IMat=null,
		opts:FilesDataSource.Options = new FilesDataSource.Options) extends DataSource(opts) { 
  val sizeMargin = opts.sizeMargin
  val blockSize = opts.blockSize
  var fileno = nstart
  var colno = 0
  var omats:Array[Mat] = null
  var matqueue:Array[Array[Mat]] = null
  var ready:IMat = null
  
  def init = {
    fileno = nstart                                           // Number of the current output file
    colno = 0                                                 // Column number in the current output file
    omats = new Array[Mat](fnames.size)
    matqueue = new Array[Array[Mat]](fnames.size)             // Queue of matrices for each output matrix
    ready = -iones(opts.lookahead, 1)                         // Numbers of files currently loaded in queue
    for (i <- 0 until fnames.size) {
      var mm = HMat.loadMat(dirname(nstart) + fnames(i))
      if (transpose.asInstanceOf[AnyRef] != null && transpose(i) == 1) mm = mm.t
      omats(i) = mm match {
      case mm:SMat => SMat(mm.nrows, blockSize, (mm.nnz * sizeMargin * blockSize / mm.ncols).toInt)
      case mm:SDMat => SDMat(mm.nrows, blockSize, (mm.nnz * sizeMargin * blockSize / mm.ncols).toInt)
      case _ => mm.zeros(mm.nrows, blockSize)
      } 
      matqueue(i) = new Array[Mat](opts.lookahead)
    }
    for (i <- 0 until opts.lookahead) {
      Actor.actor {
        prefetch(nstart + i)
      }
    }
  }
  
  def next:Array[Mat] = {
    val filex = fileno % opts.lookahead
    var donextfile = false
    var todo = 0
    while (ready(filex) < fileno) Thread.`yield`
    for (i <- 0 until fnames.size) {
    	val matq = matqueue(i)(filex)
    	if (matq != null) {
    		val ccols = math.min(colno + blockSize, matq.ncols)
    		omats(i) = matq(?, colno -> ccols)
    		todo = blockSize - ccols + colno
    	}
    }
    if (todo > 0) {    	
    	var done = false
    	while (!done && fileno+1 < nend) {
    		val filey = (fileno+1) % opts.lookahead
{'fraction_non_alphanumeric': 0.1282918934824187, 'fraction_numerical': 0.005296454318081506, 'mean_word_length': 2.568503937007874, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7235474', 'n_tokens_mistral': 2498, 'n_tokens_neox': 2447, 'n_words': 551}