<commit_before>package edu.stanford.graphics.shapenet.data

import com.jme3.math.Vector3f
import edu.stanford.graphics.shapenet.Constants
import edu.stanford.graphics.shapenet.common._
import edu.stanford.graphics.shapenet.jme3.loaders.{AssetGroups, ModelLoadOptions}
import edu.stanford.graphics.shapenet.util.{BatchSampler, CSVFile, Loggable, IOUtils}

import scala.util.Random
import scala.util.matching.Regex

/**
 *  Database of 3D models
  *
  *  @author <NAME>
 */
trait ModelsDb {
  val name: String

  def getName(): String = name

  def getSources(): Set[String]

  def getModelIds(): Seq[String]

  def getModelInfo(id: String): Option[ModelInfo]

  def getModelInfos(): Seq[ModelInfo]

  def getModelInfos(source: String, category: String): Seq[ModelInfo] = {
    val modelInfos = if (source != null) {
      getModelInfos.filter( x => {
        val fullId = FullId(x.fullId)
        fullId.source == source
      })
    } else {
      getModelInfos
    }
    val filtered = if (category != null) {
      modelInfos.filter( modelInfo => {
        CategoryUtils.hasCategory(modelInfo, category)
      })
    } else {
      modelInfos
    }
    filtered
  }

  def getModelIds(source: String, category: String): Seq[String] = {
    getModelInfos(source, category).map( x => x.fullId )
  }

  def getRandomModelIds(source: String = null, category: String = null, nModels:Int = 1): Seq[String] = {
    val filteredIds = getModelIds(source, category)
    if (filteredIds.nonEmpty) {
      val sampler = new BatchSampler
      sampler.sampleWithoutReplacement(filteredIds, nModels)
    } else {
      Seq()
    }
  }

  def getRandomModelId(source: String = null, category: String = null): String = {
    val filteredIds = getModelIds(source, category)
    if (filteredIds.nonEmpty) {
      filteredIds(Random.nextInt(filteredIds.size))
    } else {
      null
    }
  }

  def getModelLoadOptions(fullId:FullId, format:String): ModelLoadOptions = {
    val modelInfo = getModelInfo(fullId.fullid)
    var x = AssetGroups.getModelLoadOptions(fullId.source, format)
    if (modelInfo.isDefined) {
      x = x.copy( unit = Some(modelInfo.get.unit),
                  up = if (modelInfo.get.up != null) Some(modelInfo.get.up) else x.up,
                  front = if (modelInfo.get.front != null) Some(modelInfo.get.front) else x.front )
    }
    x
  }

  def getDefaultModelInfo(fullId:FullId, format:String = null): DefaultModelInfo = {
    AssetGroups.getDefaultModelInfo(fullId.source)
  }
}

class CombinedModelsDb(override val name: String,
                       val defaultModelInfo: DefaultModelInfo = DefaultModelInfo()) extends ModelsDb with Loggable {
  lazy val modelsDbsByName = new scala.collection.mutable.HashMap[String, scala.collection.mutable.ArrayBuffer[ModelsDb]]()
  lazy val modelsDbsBySource = new scala.collection.mutable.HashMap[String, scala.collection.mutable.ArrayBuffer[ModelsDb]]()
  lazy val modelsDbs = new scala.collection.mutable.ArrayBuffer[ModelsDb]
  override def getModelLoadOptions(fullId:FullId, format:String) = {
    import scala.util.control.Breaks._
    var opts: ModelLoadOptions = null
    breakable {
      for (db <- modelsDbs) {
        val dbOpts = db.getModelLoadOptions(fullId, format)
        if (dbOpts != null) {
          if (opts == null || (dbOpts.format != opts.format && dbOpts.format == format)
            || (dbOpts.format == opts.format && dbOpts.path.isDefined && opts.path.isEmpty)) {
            // this one is better
            opts = dbOpts
          }
        }
      }
    }
    if (opts == null) {
      opts = super.getModelLoadOptions(fullId, format)
    }
    opts
  }<commit_msg>Minor cleanup.<commit_after>package edu.stanford.graphics.shapenet.data

import com.jme3.math.Vector3f
import edu.stanford.graphics.shapenet.Constants
import edu.stanford.graphics.shapenet.common._
import edu.stanford.graphics.shapenet.jme3.loaders.{AssetGroups, ModelLoadOptions}
import edu.stanford.graphics.shapenet.util.{BatchSampler, CSVFile, Loggable, IOUtils}

import scala.util.Random
import scala.util.matching.Regex

/**
 *  Database of 3D models
  *
  *  @author <NAME>
 */
trait ModelsDb {
  val name: String

  def getName(): String = name

  def getSources(): Set[String]

  def getModelIds(): Seq[String]

  def getModelInfo(id: String): Option[ModelInfo]

  def getModelInfos(): Seq[ModelInfo]

  def getModelInfos(source: String, category: String): Seq[ModelInfo] = {
    val modelInfos = if (source != null) {
      getModelInfos.filter( x => {
        val fullId = FullId(x.fullId)
        fullId.source == source
      })
    } else {
      getModelInfos
    }
    val filtered = if (category != null) {
      modelInfos.filter( modelInfo => {
        CategoryUtils.hasCategory(modelInfo, category)
      })
    } else {
      modelInfos
    }
    filtered
  }

  def getModelIds(source: String, category: String): Seq[String] = {
    getModelInfos(source, category).map( x => x.fullId )
  }

  def getRandomModelIds(source: String = null, category: String = null, nModels:Int = 1): Seq[String] = {
    val filteredIds = getModelIds(source, category)
    if (filteredIds.nonEmpty) {
      val sampler = new BatchSampler
      sampler.sampleWithoutReplacement(filteredIds, nModels)
    } else {
      Seq()
    }
  }

  def getRandomModelId(source: String = null, category: String = null): String = {
    val filteredIds = getModelIds(source, category)
    if (filteredIds.nonEmpty) {
      filteredIds(Random.nextInt(filteredIds.size))
    } else {
      null
    }
  }

  def getModelLoadOptions(fullId:FullId, format:String): ModelLoadOptions = {
    val modelInfo = getModelInfo(fullId.fullid)
    var x = AssetGroups.getModelLoadOptions(fullId.source, format)
    if (modelInfo.isDefined) {
      x = x.copy( unit = Some(modelInfo.get.unit),
                  up = if (modelInfo.get.up != null) Some(modelInfo.get.up) else x.up,
                  front = if (modelInfo.get.front != null) Some(modelInfo.get.front) else x.front )
    }
    x
  }

  def getDefaultModelInfo(fullId:FullId, format:String = null): DefaultModelInfo = {
    AssetGroups.getDefaultModelInfo(fullId.source)
  }
}

class CombinedModelsDb(override val name: String,
                       val defaultModelInfo: DefaultModelInfo = DefaultModelInfo()) extends ModelsDb with Loggable {
  lazy val modelsDbsByName = new scala.collection.mutable.HashMap[String, scala.collection.mutable.ArrayBuffer[ModelsDb]]()
  lazy val modelsDbsBySource = new scala.collection.mutable.HashMap[String, scala.collection.mutable.ArrayBuffer[ModelsDb]]()
  lazy val modelsDbs = new scala.collection.mutable.ArrayBuffer[ModelsDb]
  override def getModelLoadOptions(fullId:FullId, format:String) = {
    var opts: ModelLoadOptions = null
    for (db <- modelsDbs) {
      val dbOpts = db.getModelLoadOptions(fullId, format)
      if (dbOpts != null) {
        if (opts == null || (dbOpts.format != opts.format && dbOpts.format == format)
          || (dbOpts.format == opts.format && dbOpts.path.isDefined && opts.path.isEmpty)) {
          // this one is better
          opts = dbOpts
        }
      }
    }
    if (opts == null) {
      opts = super.getModelLoadOptions(fullId, format)
    }
    opts
  }
{'fraction_non_alphanumeric': 0.11419497376415355, 'fraction_numerical': 0.0013808340237503453, 'mean_word_length': 3.1578645235361655, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3834904', 'n_tokens_mistral': 2340, 'n_tokens_neox': 2328, 'n_words': 486}