<commit_before>/*
 * Copyright 2018 Analytics Zoo Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.intel.analytics.zoo.pipeline.inference

import java.lang.{Float => JFloat, Integer => JInt}
import java.util
import java.util.concurrent.LinkedBlockingQueue
import java.util.{List => JList}

import com.intel.analytics.bigdl.nn.abstractnn.Activity

import scala.collection.JavaConverters._

class InferenceModel(private var autoScalingEnabled: Boolean = true,
                     private var concurrentNum: Int = 20,
                     private var originalModel: AbstractModel = null,
                     private[inference] var modelQueue:
                     LinkedBlockingQueue[AbstractModel] = null)
  extends InferenceSupportive with EncryptSupportive with Serializable {

  require(concurrentNum > 0, "concurrentNum should > 0")
  /**
   * default constructor, will create a InferenceModel with auto-scaling enabled.
   *
   * @return an auto-scaling enabled InferenceModel
   */
  def this() = this(true, 20, null, null)

  /**
   * create an auto-scaling disabled InferenceModel with supportedConcurrentNum
   *
   * @param concurrentNum the concurrentNum of the InferenceModel
   * @return an auto-scaling disabled InferenceModel
   */
  def this(concurrentNum: Int) = this(false, concurrentNum, null, null)

  /**
   * create an InferenceModel with specified autoScalingEnabled, supportedConcurrentNum
   * and maxConcurrentNum
   *
   * @param autoScalingEnabled     if auto-scaling is enabled
   * @param concurrentNum          the concurrentNum of the InferenceModel
   * @return a specified InferenceModel
   */
  def this(autoScalingEnabled: Boolean, concurrentNum: Int) =
    this(autoScalingEnabled, concurrentNum, null, null)

  this.modelQueue = new LinkedBlockingQueue[AbstractModel](concurrentNum)

  this.originalModel match {
    case null =>
    case _ => offerModelQueue()
  }

  /**
   * loads a bigdl, analytics-zoo model
   *
   * @param modelPath  the file path of the model
   * @param weightPath the file path of the weights
   * @param blas       whether MKLBLAS or MKLDNN
   */
  def doLoadBigDL(modelPath: String,
             weightPath: String = null,
             blas: Boolean = true): Unit = {
    clearModelQueue()
    this.originalModel = InferenceModelFactory.loadFloatModelForBigDL(modelPath, weightPath, blas)
    offerModelQueue()
  }

  /**
   * loads a bigdl, analytics-zoo model
   *
   * @param modelPath  the file path of the model
   * @param weightPath the file path of the weights
   * @param blas       whether MKLBLAS or MKLDNN
   */
  @deprecated("this method is deprecated, use doLoadBigDL() instead")
  def doLoad(modelPath: String,
             weightPath: String = null,
             blas: Boolean = true): Unit = {
    clearModelQueue()
    this.originalModel = InferenceModelFactory.loadFloatModelForBigDL(modelPath, weightPath, blas)
    offerModelQueue()
  }

  /**
   * loads a caffe model
   *
   * @param modelPath  the path of the prototxt file
   * @param weightPath the path of the caffemodel file
   * @param blas       whether MKLBLAS or MKLDNN
   */
  def doLoadCaffe(modelPath: String,
                  weightPath: String,
                  blas: Boolean = true): Unit = {
    clearModelQueue()
    this.originalModel = InferenceModelFactory.loadFloatModelForCaffe(modelPath, weightPath, blas)
    offerModelQueue()
  }

  /**
   * loads a tensorflow model as TFNet
   *
   * @param modelPath the path of the tensorflow frozen model
   * @param modelType the type of the tensorflow model file: "frozenModel"
   */
  def doLoadTensorflow(modelPath: String, modelType: String): Unit = {
    doLoadTensorflowModel(modelPath, modelType, 1, 1, true)
  }

  /**
   * loads a tensorflow model as TFNet
   *
   * @param modelPath                 the path of the tensorflow frozen model
   * @param modelType                 the type of the tensorflow model file: "frozenModel"
   * @param intraOpParallelismThreads the num of intraOpParallelismThreads
   * @param interOpParallelismThreads the num of interOpParallelismThreads
   * @param usePerSessionThreads      whether to perSessionThreads
   */
  def doLoadTensorflow(modelPath: String,
                       modelType: String,
                       intraOpParallelismThreads: Int,
                       interOpParallelismThreads: Int,
                       usePerSessionThreads: Boolean): Unit = {
    doLoadTensorflowModel(
      modelPath,
      modelType,
      intraOpParallelismThreads,
      interOpParallelismThreads,
      usePerSessionThreads)
  }

  /**
   * loads a tensorflow model as TFNet
   *
   * @param modelPath the path of the tensorflow frozen model file or saved model dir
   * @param modelType the type of the tensorflow model file: "frozenModel" or "savedModel"
   * @param inputs    the inputs of the model
   * @param outputs   the outputs of the model
   */
  def doLoadTensorflow(modelPath: String,
                       modelType: String,
                       inputs: Array[String],
                       outputs: Array[String]): Unit = {
    doLoadTensorflowModel(modelPath, modelType, inputs, outputs, 1, 1, true)
  }

  /**
   * loads a tensorflow model as TFNet
   *
   * @param modelPath                 the path of the tensorflow:
   *                                  frozen model file or saved model dir
   * @param modelType                 the type of the tensorflow model file:
   *                                  "frozenModel" or "savedModel"
   * @param inputs                    the inputs of the model
   * @param outputs                   the outputs of the model
   * @param intraOpParallelismThreads the num of intraOpParallelismThreads
   * @param interOpParallelismThreads the num of interOpParallelismThreads
   * @param usePerSessionThreads      whether to perSessionThreads
   */
  def doLoadTensorflow(modelPath: String,
                       modelType: String,
                       inputs: Array[String],
                       outputs: Array[String],
                       intraOpParallelismThreads: Int,
                       interOpParallelismThreads: Int,
                       usePerSessionThreads: Boolean): Unit = {
    doLoadTensorflowModel(
      modelPath,
      modelType,
      inputs,
      outputs,
      intraOpParallelismThreads,
      interOpParallelismThreads,
      usePerSessionThreads)
  }

  /**
   * loads a tensorflow model as TFNet
   *
   * @param modelBytes the bytes of the tensorflow model tar
   * @param modelType  the type of the tensorflow model file: "frozenModel" or "savedModel"
   * @param inputs     the inputs of the model
   * @param outputs    the outputs of the model
   */
  def doLoadTensorflow(modelBytes: Array[Byte],
                       modelType: String,
                       inputs: Array[String],
                       outputs: Array[String]): Unit = {
    doLoadTensorflowModel(modelBytes, modelType, inputs, outputs, 1, 1, true)
  }

  /**
   * loads a tensorflow model as TFNet
   *
   * @param modelBytes                the bytes of the tensorflow model tar
   * @param modelType                 the type of the tensorflow model file:
   *                                  "frozenModel" or "savedModel"
   * @param inputs                    the inputs of the model
   * @param outputs                   the outputs of the model
   * @param intraOpParallelismThreads the num of intraOpParallelismThreads
   * @param interOpParallelismThreads the num of interOpParallelismThreads
   * @param usePerSessionThreads      whether to perSessionThreads
   */
  def doLoadTensorflow(modelBytes: Array[Byte],
                       modelType: String,
                       inputs: Array[String],
                       outputs: Array[String],
                       intraOpParallelismThreads: Int,
                       interOpParallelismThreads: Int,
                       usePerSessionThreads: Boolean): Unit = {
    doLoadTensorflowModel(
      modelBytes,
      modelType,
      inputs,
      outputs,
      intraOpParallelismThreads,
      interOpParallelismThreads,
      usePerSessionThreads)
  }

  /**
   * load a Torch model as TorchModel
   *
   * @param modelPath the path of the torch script
   */
  def doLoadPyTorch(modelPath: String): Unit = {
    doLoadPyTorchModel(modelPath)
  }

  /**
   * load a Torch model as TorchModel
   *
   * @param modelBytes the bytes of the torch script
   */
  def doLoadPyTorch(modelBytes: Array[Byte]): Unit = {
    doLoadPyTorchModel(modelBytes)
  }

  /**
   * loads a openvino IR
   *
   * @param modelPath  the path of openvino ir xml file
   * @param weightPath the path of openvino ir bin file
   */
  def doLoadOpenVINO(modelPath: String, weightPath: String, batchSize: Int = 0): Unit = {
    if (concurrentNum > 1) {
      InferenceSupportive.logger.warn(s"concurrentNum is $concurrentNum > 1, " +
        s"openvino model does not support shared weights model copies")
    }<commit_msg>update document and unit test (#3037)<commit_after>/*
 * Copyright 2018 Analytics Zoo Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.intel.analytics.zoo.pipeline.inference

import java.lang.{Float => JFloat, Integer => JInt}
import java.util
import java.util.concurrent.LinkedBlockingQueue
import java.util.{List => JList}

import com.intel.analytics.bigdl.nn.abstractnn.Activity

import scala.collection.JavaConverters._

class InferenceModel(private var autoScalingEnabled: Boolean = true,
                     private var concurrentNum: Int = 20,
                     private var originalModel: AbstractModel = null,
                     private[inference] var modelQueue:
                     LinkedBlockingQueue[AbstractModel] = null)
  extends InferenceSupportive with EncryptSupportive with Serializable {

  require(concurrentNum > 0, "concurrentNum should > 0")
  /**
   * default constructor, will create a InferenceModel with auto-scaling enabled.
   *
   * @return an auto-scaling enabled InferenceModel
   */
  def this() = this(true, 20, null, null)

  /**
   * create an auto-scaling disabled InferenceModel with supportedConcurrentNum
   *
   * @param concurrentNum the concurrentNum of the InferenceModel
   * @return an auto-scaling disabled InferenceModel
   */
  def this(concurrentNum: Int) = this(false, concurrentNum, null, null)

  /**
   * create an InferenceModel with specified autoScalingEnabled, supportedConcurrentNum
   * and maxConcurrentNum
   *
   * @param autoScalingEnabled     if auto-scaling is enabled
   * @param concurrentNum          the concurrentNum of the InferenceModel
   * @return a specified InferenceModel
   */
  def this(autoScalingEnabled: Boolean, concurrentNum: Int) =
    this(autoScalingEnabled, concurrentNum, null, null)

  this.modelQueue = new LinkedBlockingQueue[AbstractModel](concurrentNum)

  this.originalModel match {
    case null =>
    case _ => offerModelQueue()
  }

  /**
   * loads a bigdl, analytics-zoo model
   *
   * @param modelPath  the file path of the model
   * @param weightPath the file path of the weights
   * @param blas       whether MKLBLAS or MKLDNN
   */
  def doLoadBigDL(modelPath: String,
             weightPath: String = null,
             blas: Boolean = true): Unit = {
    clearModelQueue()
    this.originalModel = InferenceModelFactory.loadFloatModelForBigDL(modelPath, weightPath, blas)
    offerModelQueue()
  }

  /**
   * loads a bigdl, analytics-zoo model
   *
   * @param modelPath  the file path of the model
   * @param weightPath the file path of the weights
   * @param blas       whether MKLBLAS or MKLDNN
   */
  @deprecated("this method is deprecated, use doLoadBigDL() instead")
  def doLoad(modelPath: String,
             weightPath: String = null,
             blas: Boolean = true): Unit = {
    clearModelQueue()
    this.originalModel = InferenceModelFactory.loadFloatModelForBigDL(modelPath, weightPath, blas)
    offerModelQueue()
  }

  /**
   * loads a caffe model
   *
   * @param modelPath  the path of the prototxt file
   * @param weightPath the path of the caffemodel file
   * @param blas       whether MKLBLAS or MKLDNN
   */
  def doLoadCaffe(modelPath: String,
                  weightPath: String,
                  blas: Boolean = true): Unit = {
    clearModelQueue()
    this.originalModel = InferenceModelFactory.loadFloatModelForCaffe(modelPath, weightPath, blas)
    offerModelQueue()
  }

  /**
   * loads a tensorflow model as TFNet
   *
   * @param modelPath the path of the tensorflow frozen model
   * @param modelType the type of the tensorflow model file: "frozenModel"
   */
  def doLoadTensorflow(modelPath: String, modelType: String): Unit = {
    doLoadTensorflowModel(modelPath, modelType, 1, 1, true)
  }

  /**
   * loads a tensorflow model as TFNet
   *
   * @param modelPath                 the path of the tensorflow frozen model
   * @param modelType                 the type of the tensorflow model file: "frozenModel"
   * @param intraOpParallelismThreads the num of intraOpParallelismThreads
   * @param interOpParallelismThreads the num of interOpParallelismThreads
   * @param usePerSessionThreads      whether to perSessionThreads
   */
  def doLoadTensorflow(modelPath: String,
                       modelType: String,
                       intraOpParallelismThreads: Int,
                       interOpParallelismThreads: Int,
                       usePerSessionThreads: Boolean): Unit = {
    doLoadTensorflowModel(
      modelPath,
      modelType,
      intraOpParallelismThreads,
      interOpParallelismThreads,
      usePerSessionThreads)
  }

  /**
   * loads a tensorflow model as TFNet
   *
   * @param modelPath the path of the tensorflow frozen model file or saved model dir
   * @param modelType the type of the tensorflow model file: "frozenModel" or "savedModel"
   * @param inputs    the inputs of the model
   * @param outputs   the outputs of the model
   */
  def doLoadTensorflow(modelPath: String,
                       modelType: String,
                       inputs: Array[String],
                       outputs: Array[String]): Unit = {
    doLoadTensorflowModel(modelPath, modelType, inputs, outputs, 1, 1, true)
  }

  /**
   * loads a tensorflow model as TFNet
   *
   * @param modelPath                 the path of the tensorflow:
   *                                  frozen model file or saved model dir
   * @param modelType                 the type of the tensorflow model file:
   *                                  "frozenModel" or "savedModel"
   * @param inputs                    the inputs of the model
   * @param outputs                   the outputs of the model
   * @param intraOpParallelismThreads the num of intraOpParallelismThreads
   * @param interOpParallelismThreads the num of interOpParallelismThreads
   * @param usePerSessionThreads      whether to perSessionThreads
   */
  def doLoadTensorflow(modelPath: String,
                       modelType: String,
                       inputs: Array[String],
                       outputs: Array[String],
                       intraOpParallelismThreads: Int,
                       interOpParallelismThreads: Int,
                       usePerSessionThreads: Boolean): Unit = {
    doLoadTensorflowModel(
      modelPath,
      modelType,
      inputs,
      outputs,
      intraOpParallelismThreads,
      interOpParallelismThreads,
      usePerSessionThreads)
  }

  /**
   * loads a tensorflow model as TFNet
   *
   * @param modelBytes the bytes of the tensorflow model tar
   * @param modelType  the type of the tensorflow model file: "frozenModel" or "savedModel"
   * @param inputs     the inputs of the model
   * @param outputs    the outputs of the model
   */
  def doLoadTensorflow(modelBytes: Array[Byte],
                       modelType: String,
                       inputs: Array[String],
                       outputs: Array[String]): Unit = {
    doLoadTensorflowModel(modelBytes, modelType, inputs, outputs, 1, 1, true)
  }

  /**
   * loads a tensorflow model as TFNet
   *
   * @param modelBytes                the bytes of the tensorflow model tar
   * @param modelType                 the type of the tensorflow model file:
   *                                  "frozenModel" or "savedModel"
   * @param inputs                    the inputs of the model
   * @param outputs                   the outputs of the model
   * @param intraOpParallelismThreads the num of intraOpParallelismThreads
   * @param interOpParallelismThreads the num of interOpParallelismThreads
   * @param usePerSessionThreads      whether to perSessionThreads
   */
  def doLoadTensorflow(modelBytes: Array[Byte],
                       modelType: String,
                       inputs: Array[String],
                       outputs: Array[String],
                       intraOpParallelismThreads: Int,
                       interOpParallelismThreads: Int,
                       usePerSessionThreads: Boolean): Unit = {
    doLoadTensorflowModel(
      modelBytes,
      modelType,
      inputs,
      outputs,
      intraOpParallelismThreads,
      interOpParallelismThreads,
      usePerSessionThreads)
  }

  /**
   * load a Torch model as TorchModel
   * The torch model should be saved using zoo_pickle_module in python.
   * {{{
   * from zoo.pipeline.api.torch import zoo_pickle_module
   * model = models.resnet18(pretrained = True)
   * torch.save(model, path, pickle_module=zoo_pickle_module)
   * }}}
   *
   * @param modelPath the path of the torch model
   */
  def doLoadPyTorch(modelPath: String): Unit = {
    if (concurrentNum > 1) {
      InferenceSupportive.logger.warn(s"concurrentNum is $concurrentNum > 1, " +
        s"we recommend use only one copy for Torch Model.")
    }
    doLoadPyTorchModel(modelPath)
  }

  /**
   * load a Torch model as TorchModel
   * The torch model should be saved using zoo_pickle_module in python.
   * {{{
   * from zoo.pipeline.api.torch import zoo_pickle_module
   * model = models.resnet18(pretrained = True)
   * torch.save(model, "$modelPath", pickle_module=zoo_pickle_module)
   * }}}
   *
   * @param modelBytes the bytes of the torch model
   */
  def doLoadPyTorch(modelBytes: Array[Byte]): Unit = {
    if (concurrentNum > 1) {
      InferenceSupportive.logger.warn(s"concurrentNum is $concurrentNum > 1, " +
        s"we recommend use only one copy for Torch Model.")
    }
    doLoadPyTorchModel(modelBytes)
  }

  /**
   * loads a openvino IR
   *
   * @param modelPath  the path of openvino ir xml file
   * @param weightPath the path of openvino ir bin file
   */
  def doLoadOpenVINO(modelPath: String, weightPath: String, batchSize: Int = 0): Unit = {
    if (concurrentNum > 1) {
      InferenceSupportive.logger.warn(s"concurrentNum is $concurrentNum > 1, " +
        s"openvino model does not support shared weights model copies")
    }
{'fraction_non_alphanumeric': 0.07187249398556536, 'fraction_numerical': 0.0029069767441860465, 'mean_word_length': 2.4858490566037736, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6350335', 'n_tokens_mistral': 5596, 'n_tokens_neox': 5418, 'n_words': 1735}