<commit_before>package ml.combust.mleap.springboot

import TypeConverters._
import javax.annotation.PostConstruct
import org.slf4j.LoggerFactory
import ml.combust.mleap.pb
import org.springframework.beans.factory.annotation.Value
import org.springframework.stereotype.Component

import java.nio.file.{Paths, Files}
import scalapb.json4s.Parser

@Component
class ModelLoader {
  @Value("${mleap.model.config:#{null}}")
  private val modelConfigPath: String = null

  private val logger = LoggerFactory.getLogger(classOf[ModelLoader])
  private val jsonParser = new Parser()
  private val timeout = 60000

  @PostConstruct
  def loadModel(): Unit = {
    if (modelConfigPath == null) {
      logger.info("Skipping loading model on startup")
      return
    }

    val configPath = Paths.get(modelConfigPath)

    if (!Files.exists(configPath)) {
      logger.warn(s"Model path does not exist: $modelConfigPath")
      return
    }

    logger.info(s"Loading model from $modelConfigPath")

    val request = new String(Files.readAllBytes(configPath))

    StarterConfiguration.getMleapExecutor
      .loadModel(jsonParser.fromJsonString[pb.LoadModelRequest](request))(timeout)
  }
}
<commit_msg>Support loading multiple models from directory of configs<commit_after>package ml.combust.mleap.springboot

import TypeConverters._
import javax.annotation.PostConstruct
import org.slf4j.LoggerFactory
import ml.combust.mleap.pb
import org.springframework.beans.factory.annotation.Value
import org.springframework.stereotype.Component

import scala.collection.JavaConverters._
import java.nio.file.{Paths, Files, Path}
import scalapb.json4s.Parser

@Component
class ModelLoader {
  @Value("${mleap.model.config:#{null}}")
  private val modelConfigPath: String = null

  private val logger = LoggerFactory.getLogger(classOf[ModelLoader])
  private val jsonParser = new Parser()
  private val timeout = 60000

  @PostConstruct
  def loadModel(): Unit = {
    if (modelConfigPath == null) {
      logger.info("Skipping loading model on startup")
      return
    }

    val configPath = Paths.get(modelConfigPath)

    if (!Files.exists(configPath)) {
      logger.warn(s"Model path does not exist: $modelConfigPath")
      return
    }

    val configFiles: List[Path] = if (Files.isDirectory(configPath)) {
      Files.list(configPath).iterator().asScala.toList
    } else {
      List(configPath)
    }

    for (configFile <- configFiles) {
      logger.info(s"Loading model from ${configFile.toString}")

      val request = new String(Files.readAllBytes(configFile))

      StarterConfiguration.getMleapExecutor
        .loadModel(jsonParser.fromJsonString[pb.LoadModelRequest](request))(timeout)
    }
  }
}

{'fraction_non_alphanumeric': 0.10207100591715976, 'fraction_numerical': 0.005177514792899409, 'mean_word_length': 4.252427184466019, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7663168', 'n_tokens_mistral': 822, 'n_tokens_neox': 800, 'n_words': 173}