<commit_before>/*
 * Copyright 2016 The BigDL Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.intel.analytics.bigdl.utils.serializer

import com.google.protobuf.ByteString

import scala.collection.JavaConverters._
import scala.reflect.runtime.universe
import com.intel.analytics.bigdl.nn._
import com.intel.analytics.bigdl.nn.abstractnn.DataFormat.{NCHW, NHWC}
import com.intel.analytics.bigdl.nn.abstractnn.{AbstractModule, Activity, DataFormat}
import com.intel.analytics.bigdl.nn.quantized._
import com.intel.analytics.bigdl.optim.{L1L2Regularizer, L1Regularizer, L2Regularizer, Regularizer}
import com.intel.analytics.bigdl.tensor.{DenseType, QuantizedTensor, QuantizedType, Tensor, Storage}
import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric.{NumericBoolean, NumericChar, NumericDouble, NumericFloat, NumericInt, NumericLong, NumericShort, NumericString}
import com.intel.analytics.bigdl.utils.tf.TFTensorNumeric.NumericByteString
import serialization.Bigdl._
import serialization.Bigdl.AttrValue.ArrayValue

import scala.collection.mutable
import scala.reflect.ClassTag

/**
 * Trait which defines get attribute value from saved protobuf data and convert BigDL object to
 * protobuf format attribute data
 */
trait DataConverter {

/**
 * Get attribute value from protobuf attribute data
 * @tparam T data type
 * @param context deserialization context
 * @param attribute  protobuf generated Attribute instance
 * @return BigDL compatible param value
 */
  def getAttributeValue[T : ClassTag](context: DeserializeContext,
                                      attribute: AttrValue)(
    implicit ev: TensorNumeric[T]) : AnyRef

/**
 * Set attribute value to protobuf format
 * @tparam T data type
 * @param context serialization context
 * @param attributeBuilder  the attribute value writable instance
 * @param value the value to be written to protobuf file
 * @param valueType the type of the value to help set the data type
 */
  def setAttributeValue[T : ClassTag](context: SerializeContext[T],
                                      attributeBuilder : AttrValue.Builder, value: Any,
                                      valueType: universe.Type = null)
    (implicit ev: TensorNumeric[T]) : Unit
}

/**
 * General implementation of [[DataConverter]], it provides the conversion entry for all types
 */
object DataConverter extends DataConverter{

  private val typePlaceHolder = universe.typeOf[DataConverter]

  // Customized data converter map, key is the string representation of user defined class type

  private val customizedConverter = new mutable.HashMap[String, DataConverter]

  def registerConverter(tpe : String, converter : DataConverter) : Unit = {
    require(!customizedConverter.contains(tpe), s"converter for $tpe already exists!")
    customizedConverter(tpe) = converter
  }

  private def getRuntimeType[T : ClassTag](value : Any) (implicit ev: TensorNumeric[T])
    : universe.Type = {
    if (value.isInstanceOf[Tensor[_]]) {
      ModuleSerializer.tensorType
    } else if (value.isInstanceOf[AbstractModule[_, _, T]]) {
      ModuleSerializer.abstractModuleType
    } else if (value.isInstanceOf[Regularizer[_]]) {
      ModuleSerializer.regularizerType
    } else if (value.isInstanceOf[InitializationMethod]) {
      universe.typeOf[InitializationMethod]
    } else if (value.isInstanceOf[VariableFormat]) {
      universe.typeOf[VariableFormat]
    } else if (value.isInstanceOf[DataFormat]) {
      universe.typeOf[DataFormat]
    } else {
      val cls = value.getClass
      val runtimeMirror = universe.runtimeMirror(getClass.getClassLoader)
      val clsSymbol = runtimeMirror.classSymbol(cls)
      clsSymbol.toType
    }
  }

  override def getAttributeValue[T : ClassTag](context: DeserializeContext, attribute: AttrValue)
    (implicit ev: TensorNumeric[T]) : AnyRef = {
    attribute.getDataType match {
      case DataType.INT32 => Integer.valueOf(attribute.getInt32Value)
      case DataType.INT64 => Long.box(attribute.getInt64Value)
      case DataType.DOUBLE => Double.box(attribute.getDoubleValue)
      case DataType.FLOAT => Float.box(attribute.getFloatValue)
      case DataType.STRING => attribute.getStringValue
      case DataType.BOOL => Boolean.box(attribute.getBoolValue)
      case DataType.REGULARIZER => RegularizerConverter.getAttributeValue(context, attribute)
      case DataType.TENSOR => TensorConverter.getAttributeValue(context, attribute)
      case DataType.VARIABLE_FORMAT =>
        VariableFormatConverter.getAttributeValue(context, attribute)
      case DataType.INITMETHOD => InitMethodConverter.getAttributeValue(context, attribute)
      case DataType.MODULE => ModuleConverter.getAttributeValue(context, attribute)
      case DataType.NAME_ATTR_LIST => NameListConverter.getAttributeValue(context, attribute)
      case DataType.ARRAY_VALUE => ArrayConverter.getAttributeValue(context, attribute)
      case DataType.DATA_FORMAT => DataFormatConverter.getAttributeValue(context, attribute)
      case DataType.CUSTOM => CustomConverterDelegator.getAttributeValue(context, attribute)
      case _ => throw new IllegalArgumentException
        (s"${attribute.getDataType} can not be recognized")
    }
  }

  override def setAttributeValue[T : ClassTag](
    context: SerializeContext[T], attributeBuilder: AttrValue.Builder,
    value: Any, valueType : universe.Type = typePlaceHolder)
    (implicit ev: TensorNumeric[T]): Unit = {
    // to make it compatible with Java types
    if (valueType =:= universe.typeOf[Int] ||
      valueType =:= universe.typeOf[java.lang.Integer]) {
      attributeBuilder.setDataType(DataType.INT32)
      attributeBuilder.setInt32Value(value.asInstanceOf[Int])
    } else if (valueType =:= universe.typeOf[Long] ||
      valueType =:= universe.typeOf[java.lang.Long]) {
      attributeBuilder.setDataType(DataType.INT64)
      attributeBuilder.setInt64Value(value.asInstanceOf[Long])
    } else if (valueType =:= universe.typeOf[Float] ||
      valueType =:= universe.typeOf[java.lang.Float]) {
      attributeBuilder.setDataType(DataType.FLOAT)
      attributeBuilder.setFloatValue(value.asInstanceOf[Float])
    } else if (valueType =:= universe.typeOf[Double] ||
      valueType =:= universe.typeOf[java.lang.Double]) {
      attributeBuilder.setDataType(DataType.DOUBLE)
      attributeBuilder.setDoubleValue(value.asInstanceOf[Double])
    } else if (valueType =:= universe.typeOf[String] ||
      valueType =:= universe.typeOf[java.lang.String]) {
      attributeBuilder.setDataType(DataType.STRING)
      attributeBuilder.setStringValue(value.asInstanceOf[String])
    } else if (valueType =:= universe.typeOf[Boolean] ||
      valueType =:= universe.typeOf[java.lang.Boolean]) {
      attributeBuilder.setDataType(DataType.BOOL )
      attributeBuilder.setBoolValue(value.asInstanceOf[Boolean])
    } else if (valueType =:= universe.typeOf[VariableFormat]) {
      VariableFormatConverter.setAttributeValue(context, attributeBuilder, value)
    } else if (valueType =:= universe.typeOf[InitializationMethod]) {
      InitMethodConverter.setAttributeValue(context, attributeBuilder, value)
    } else if (valueType.toString == ModuleSerializer.regularizerType.toString) {
      RegularizerConverter.setAttributeValue(context, attributeBuilder, value)
    } else if (valueType <:< universe.typeOf[Tensor[_]]) {
      TensorConverter.setAttributeValue(context, attributeBuilder, value)
    } else if (valueType.toString == ModuleSerializer.tType.toString) {
      if (ev == com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric.NumericDouble) {
        attributeBuilder.setDataType(DataType.DOUBLE)
        attributeBuilder.setDoubleValue(value.asInstanceOf[Double])
      } else {
        attributeBuilder.setDataType(DataType.FLOAT)
        attributeBuilder.setFloatValue(value.asInstanceOf[Float])
      }
    } else if (valueType.toString == ModuleSerializer.abstractModuleType.toString
      || valueType.toString == ModuleSerializer.tensorModuleType.toString
      || valueType.toString == ModuleSerializer.moduleType.toString
      || valueType.toString == ModuleSerializer.boundedModuleType.toString
      ) {
      ModuleConverter.setAttributeValue(context, attributeBuilder, value)
    } else if (value.isInstanceOf[mutable.Map[String, _ <: Any]]) {
      NameListConverter.setAttributeValue(context, attributeBuilder, value)
    } else if (valueType <:< universe.typeOf[Array[_]] ||
      valueType.typeSymbol == universe.typeOf[Array[_ ]].typeSymbol) {
      ArrayConverter.setAttributeValue(context, attributeBuilder, value, valueType)
    } else if (valueType =:= universe.typeOf[DataFormat]) {
      DataFormatConverter.setAttributeValue(context, attributeBuilder, value)
    } else {
      CustomConverterDelegator.setAttributeValue(context, attributeBuilder, value, valueType)
    }
  }

/**
 * DataConverter for [[com.intel.analytics.bigdl.optim.Regularizer]]
 */
  object RegularizerConverter extends DataConverter {

    override def getAttributeValue[T : ClassTag](context: DeserializeContext,
                                                 attribute: AttrValue)
      (implicit ev: TensorNumeric[T]): AnyRef = {<commit_msg>support layer with different type (#2130)<commit_after>/*
 * Copyright 2016 The BigDL Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.intel.analytics.bigdl.utils.serializer

import com.google.protobuf.ByteString

import scala.collection.JavaConverters._
import scala.reflect.runtime.universe
import com.intel.analytics.bigdl.nn._
import com.intel.analytics.bigdl.nn.abstractnn.DataFormat.{NCHW, NHWC}
import com.intel.analytics.bigdl.nn.abstractnn.{AbstractModule, Activity, DataFormat}
import com.intel.analytics.bigdl.nn.quantized._
import com.intel.analytics.bigdl.optim.{L1L2Regularizer, L1Regularizer, L2Regularizer, Regularizer}
import com.intel.analytics.bigdl.tensor.{DenseType, QuantizedTensor, QuantizedType, Tensor, Storage}
import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric.{NumericBoolean, NumericChar, NumericDouble, NumericFloat, NumericInt, NumericLong, NumericShort, NumericString}
import com.intel.analytics.bigdl.utils.tf.TFTensorNumeric.NumericByteString
import serialization.Bigdl._
import serialization.Bigdl.AttrValue.ArrayValue

import scala.collection.mutable
import scala.reflect.ClassTag

/**
 * Trait which defines get attribute value from saved protobuf data and convert BigDL object to
 * protobuf format attribute data
 */
trait DataConverter {

/**
 * Get attribute value from protobuf attribute data
 * @tparam T data type
 * @param context deserialization context
 * @param attribute  protobuf generated Attribute instance
 * @return BigDL compatible param value
 */
  def getAttributeValue[T : ClassTag](context: DeserializeContext,
                                      attribute: AttrValue)(
    implicit ev: TensorNumeric[T]) : AnyRef

/**
 * Set attribute value to protobuf format
 * @tparam T data type
 * @param context serialization context
 * @param attributeBuilder  the attribute value writable instance
 * @param value the value to be written to protobuf file
 * @param valueType the type of the value to help set the data type
 */
  def setAttributeValue[T : ClassTag](context: SerializeContext[T],
                                      attributeBuilder : AttrValue.Builder, value: Any,
                                      valueType: universe.Type = null)
    (implicit ev: TensorNumeric[T]) : Unit
}

/**
 * General implementation of [[DataConverter]], it provides the conversion entry for all types
 */
object DataConverter extends DataConverter{

  private val typePlaceHolder = universe.typeOf[DataConverter]

  // Customized data converter map, key is the string representation of user defined class type

  private val customizedConverter = new mutable.HashMap[String, DataConverter]

  def registerConverter(tpe : String, converter : DataConverter) : Unit = {
    require(!customizedConverter.contains(tpe), s"converter for $tpe already exists!")
    customizedConverter(tpe) = converter
  }

  private def getRuntimeType[T : ClassTag](value : Any) (implicit ev: TensorNumeric[T])
    : universe.Type = {
    if (value.isInstanceOf[Tensor[_]]) {
      ModuleSerializer.tensorType
    } else if (value.isInstanceOf[AbstractModule[_, _, T]]) {
      ModuleSerializer.abstractModuleType
    } else if (value.isInstanceOf[Regularizer[_]]) {
      ModuleSerializer.regularizerType
    } else if (value.isInstanceOf[InitializationMethod]) {
      universe.typeOf[InitializationMethod]
    } else if (value.isInstanceOf[VariableFormat]) {
      universe.typeOf[VariableFormat]
    } else if (value.isInstanceOf[DataFormat]) {
      universe.typeOf[DataFormat]
    } else {
      val cls = value.getClass
      val runtimeMirror = universe.runtimeMirror(getClass.getClassLoader)
      val clsSymbol = runtimeMirror.classSymbol(cls)
      clsSymbol.toType
    }
  }

  override def getAttributeValue[T : ClassTag](context: DeserializeContext, attribute: AttrValue)
    (implicit ev: TensorNumeric[T]) : AnyRef = {
    attribute.getDataType match {
      case DataType.INT32 => Integer.valueOf(attribute.getInt32Value)
      case DataType.INT64 => Long.box(attribute.getInt64Value)
      case DataType.DOUBLE => Double.box(attribute.getDoubleValue)
      case DataType.FLOAT => Float.box(attribute.getFloatValue)
      case DataType.STRING => attribute.getStringValue
      case DataType.BOOL => Boolean.box(attribute.getBoolValue)
      case DataType.REGULARIZER => RegularizerConverter.getAttributeValue(context, attribute)
      case DataType.TENSOR => TensorConverter.getAttributeValue(context, attribute)
      case DataType.VARIABLE_FORMAT =>
        VariableFormatConverter.getAttributeValue(context, attribute)
      case DataType.INITMETHOD => InitMethodConverter.getAttributeValue(context, attribute)
      case DataType.MODULE => ModuleConverter.getAttributeValue(context, attribute)
      case DataType.NAME_ATTR_LIST => NameListConverter.getAttributeValue(context, attribute)
      case DataType.ARRAY_VALUE => ArrayConverter.getAttributeValue(context, attribute)
      case DataType.DATA_FORMAT => DataFormatConverter.getAttributeValue(context, attribute)
      case DataType.CUSTOM => CustomConverterDelegator.getAttributeValue(context, attribute)
      case _ => throw new IllegalArgumentException
        (s"${attribute.getDataType} can not be recognized")
    }
  }

  override def setAttributeValue[T : ClassTag](
    context: SerializeContext[T], attributeBuilder: AttrValue.Builder,
    value: Any, valueType : universe.Type = typePlaceHolder)
    (implicit ev: TensorNumeric[T]): Unit = {
    // to make it compatible with Java types
    if (valueType =:= universe.typeOf[Int] ||
      valueType =:= universe.typeOf[java.lang.Integer]) {
      attributeBuilder.setDataType(DataType.INT32)
      attributeBuilder.setInt32Value(value.asInstanceOf[Int])
    } else if (valueType =:= universe.typeOf[Long] ||
      valueType =:= universe.typeOf[java.lang.Long]) {
      attributeBuilder.setDataType(DataType.INT64)
      attributeBuilder.setInt64Value(value.asInstanceOf[Long])
    } else if (valueType =:= universe.typeOf[Float] ||
      valueType =:= universe.typeOf[java.lang.Float]) {
      attributeBuilder.setDataType(DataType.FLOAT)
      attributeBuilder.setFloatValue(value.asInstanceOf[Float])
    } else if (valueType =:= universe.typeOf[Double] ||
      valueType =:= universe.typeOf[java.lang.Double]) {
      attributeBuilder.setDataType(DataType.DOUBLE)
      attributeBuilder.setDoubleValue(value.asInstanceOf[Double])
    } else if (valueType =:= universe.typeOf[String] ||
      valueType =:= universe.typeOf[java.lang.String]) {
      attributeBuilder.setDataType(DataType.STRING)
      attributeBuilder.setStringValue(value.asInstanceOf[String])
    } else if (valueType =:= universe.typeOf[Boolean] ||
      valueType =:= universe.typeOf[java.lang.Boolean]) {
      attributeBuilder.setDataType(DataType.BOOL )
      attributeBuilder.setBoolValue(value.asInstanceOf[Boolean])
    } else if (valueType =:= universe.typeOf[VariableFormat]) {
      VariableFormatConverter.setAttributeValue(context, attributeBuilder, value)
    } else if (valueType =:= universe.typeOf[InitializationMethod]) {
      InitMethodConverter.setAttributeValue(context, attributeBuilder, value)
    } else if (valueType.toString == ModuleSerializer.regularizerType.toString) {
      RegularizerConverter.setAttributeValue(context, attributeBuilder, value)
    } else if (valueType <:< universe.typeOf[Tensor[_]]) {
      TensorConverter.setAttributeValue(context, attributeBuilder, value)
    } else if (valueType.toString == ModuleSerializer.tType.toString) {
      if (ev == com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric.NumericDouble) {
        attributeBuilder.setDataType(DataType.DOUBLE)
        attributeBuilder.setDoubleValue(value.asInstanceOf[Double])
      } else {
        attributeBuilder.setDataType(DataType.FLOAT)
        attributeBuilder.setFloatValue(value.asInstanceOf[Float])
      }
    } else if (valueType.toString == ModuleSerializer.abstractModuleType.toString
      || valueType.toString == ModuleSerializer.tensorModuleType.toString
      || valueType.toString == ModuleSerializer.moduleType.toString
      || valueType.toString == ModuleSerializer.boundedModuleType.toString
      || valueType <:< universe.typeOf[AbstractModule[_, _, _]]
      ) {
      ModuleConverter.setAttributeValue(context, attributeBuilder, value)
    } else if (value.isInstanceOf[mutable.Map[String, _ <: Any]]) {
      NameListConverter.setAttributeValue(context, attributeBuilder, value)
    } else if (valueType <:< universe.typeOf[Array[_]] ||
      valueType.typeSymbol == universe.typeOf[Array[_ ]].typeSymbol) {
      ArrayConverter.setAttributeValue(context, attributeBuilder, value, valueType)
    } else if (valueType =:= universe.typeOf[DataFormat]) {
      DataFormatConverter.setAttributeValue(context, attributeBuilder, value)
    } else {
      CustomConverterDelegator.setAttributeValue(context, attributeBuilder, value, valueType)
    }
  }

/**
 * DataConverter for [[com.intel.analytics.bigdl.optim.Regularizer]]
 */
  object RegularizerConverter extends DataConverter {

    override def getAttributeValue[T : ClassTag](context: DeserializeContext,
                                                 attribute: AttrValue)
      (implicit ev: TensorNumeric[T]): AnyRef = {
{'fraction_non_alphanumeric': 0.09674310064935066, 'fraction_numerical': 0.003043831168831169, 'mean_word_length': 4.2192216044479744, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3756262', 'n_tokens_mistral': 5324, 'n_tokens_neox': 5321, 'n_words': 1201}