<commit_before>/*
 * Copyright 2018 Analytics Zoo Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.intel.analytics.zoo.tfpark

import java.io.{File, FileInputStream, InputStream}
import java.nio._

import com.intel.analytics.bigdl.dataset.Sample
import com.intel.analytics.bigdl.nn.abstractnn.{AbstractCriterion, AbstractModule, Activity}
import com.intel.analytics.bigdl.optim._
import com.intel.analytics.bigdl.tensor.Tensor
import com.intel.analytics.bigdl.transform.vision.image.ImageFeature
import com.intel.analytics.bigdl.utils.{T, Table}
import com.intel.analytics.zoo.feature.common.Preprocessing
import com.intel.analytics.zoo.feature.image.ImageProcessing
import com.intel.analytics.zoo.pipeline.api.keras.metrics.{Accuracy, BinaryAccuracy, CategoricalAccuracy, SparseCategoricalAccuracy}
import org.tensorflow.framework.GraphDef
import org.tensorflow.{DataType, Tensor => TTensor}

import scala.io.Source
import scala.reflect.io.Path

object TFUtils {

  val defaultSessionConfig = SessionConfig()

  private[zoo] def getTrainMeta(trainMetaPath: Path) = {
    val jsonStr = Source.fromFile(trainMetaPath.jfile).getLines().mkString
    import org.json4s._
    import org.json4s.jackson.JsonMethods._
    implicit val formats = DefaultFormats

    parse(jsonStr).camelizeKeys.extract[TrainMeta]
  }

  private[zoo] def parseGraph(graphProtoTxt: String): GraphDef = {
    var fr: File = null
    var in: InputStream = null
    try {
      fr = new File(graphProtoTxt)
      in = new FileInputStream(fr)

      GraphDef.parseFrom(in)
    } finally {
      if (in != null) in.close()
    }
  }

  private def decodeUVarInt64(bytes: Array[Byte], offset: Int): (Long, Int) = {
    var shift = 0
    var p = offset
    var result: Long = 0
    while (shift  <= 63 && p < bytes.length) {
      val b = bytes(p)
      p += 1
      if ((b & 128) != 0) {
        result |= ((b & 127) << shift)
      } else {
        result |= (b << shift)
        return (result, p)
      }

      shift += 7
    }
    (result, p)
  }

  private def getOffsets(buffer: ByteBuffer, numElem: Int): Array[Int] = {
    val offsetsBuffer = ByteBuffer.wrap(buffer.array()
      .slice(buffer.arrayOffset(), numElem * 8))
      .order(ByteOrder.nativeOrder())
      .asLongBuffer()
    val offsets = new Array[Long](numElem)
    offsetsBuffer.get(offsets)
    offsets.map(_.toInt)
  }

  private[zoo] def tf2bigdl(t: TTensor[_], output: Tensor[_]) = {
    val shape = t.shape().map(_.toInt)
    output.resize(shape)
    val dataType = t.dataType()

    val numericDataTypes = Set(DataType.FLOAT,
      DataType.UINT8, DataType.INT32, DataType.INT64)

    if (dataType == DataType.STRING) {
      val outputTensor = output.asInstanceOf[Tensor[Array[Byte]]]
      require(t.numDimensions() <= 1, "only scalar or Vector string are supported")
      val elements = t.numElements()
      val buffer = ByteBuffer.allocate(t.numBytes())
      t.writeTo(buffer)
      val offsets = getOffsets(buffer, elements)
      val storage = outputTensor.storage().array()
      val strDataOffset = elements * 8
      var i = 0
      while (i < elements) {
        val offset = buffer.arrayOffset() + offsets(i) + strDataOffset
        val (strLen, strStart) = decodeUVarInt64(buffer.array(), offset)
        storage(outputTensor.storageOffset() - 1 + i) = buffer.array()
          .slice(strStart, strStart + strLen.toInt)
        i += 1
      }
    } else if (numericDataTypes(dataType)) {
      dataType match {
        case DataType.FLOAT =>
          val outputTensor = output.asInstanceOf[Tensor[Float]]
          val buffer = FloatBuffer.wrap(
            outputTensor.storage().array(),
            outputTensor.storageOffset() - 1,
            shape.product)
          t.writeTo(buffer)
        case DataType.UINT8 =>
          val outputTensor = output.asInstanceOf[Tensor[Float]]
          val arr = new Array[Byte](shape.product)
          val buffer = ByteBuffer.wrap(arr)
          t.writeTo(buffer)
          byte2float(arr, outputTensor.storage().array(), outputTensor.storageOffset() - 1)
        case DataType.INT32 =>
          val outputTensor = output.asInstanceOf[Tensor[Float]]
          val arr = new Array[Int](shape.product)
          val buffer = IntBuffer.wrap(arr)
          t.writeTo(buffer)
          int2float(arr, outputTensor.storage().array(), outputTensor.storageOffset() - 1)
        case DataType.INT64 =>
          val outputTensor = output.asInstanceOf[Tensor[Float]]
          val arr = new Array[Long](shape.product)
          val buffer = LongBuffer.wrap(arr)
          t.writeTo(buffer)
          long2float(arr, outputTensor.storage().array(), outputTensor.storageOffset() - 1)
      }

    } else {
      throw new Exception(s"data type ${dataType} are not supported")
    }
  }

  private[zoo] def byte2float(src: Array[Byte], dest: Array[Float], offset: Int): Unit = {
    val length = src.length
    var i = 0
    while (i < length) {
      dest(offset + i) = src(i).toFloat
      i += 1
    }
  }

  private[zoo] def int2float(src: Array[Int], dest: Array[Float], offset: Int): Unit = {
    val length = src.length
    var i = 0
    while (i < length) {
      dest(offset + i) = src(i).toFloat
      i += 1
    }
  }

  private[zoo] def long2float(src: Array[Long], dest: Array[Float], offset: Int): Unit = {
    val length = src.length
    var i = 0
    while (i < length) {
      dest(offset + i) = src(i).toFloat
      i += 1
    }
  }



  def tfenum2datatype(enum: Int): DataType = {
    enum match {
      case 1 => DataType.FLOAT
      case 2 => DataType.DOUBLE
      case 3 => DataType.INT32
      case 4 => DataType.UINT8
      case 7 => DataType.STRING
      case 9 => DataType.INT64
      case 10 => DataType.BOOL
      case _ => throw new IllegalArgumentException(s"unsupported tensorflow datatype $enum")

    }
  }

  def tfdatatype2enum(dataType: DataType): Int = {
    dataType match {
      case DataType.FLOAT => 1
      case DataType.DOUBLE => 2
      case DataType.INT32 => 3
      case DataType.UINT8 => 4
      case DataType.STRING => 7
      case DataType.INT64 => 9
      case DataType.BOOL => 10
      case _ => throw new IllegalArgumentException(s"unsupported tensorflow datatype $dataType")

    }
  }

}
<commit_msg>Add pre-made estimator example (#1995)<commit_after>/*
 * Copyright 2018 Analytics Zoo Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.intel.analytics.zoo.tfpark

import java.io.{File, FileInputStream, InputStream}
import java.nio._

import com.intel.analytics.bigdl.dataset.Sample
import com.intel.analytics.bigdl.nn.abstractnn.{AbstractCriterion, AbstractModule, Activity}
import com.intel.analytics.bigdl.optim._
import com.intel.analytics.bigdl.tensor.Tensor
import com.intel.analytics.bigdl.transform.vision.image.ImageFeature
import com.intel.analytics.bigdl.utils.{T, Table}
import com.intel.analytics.zoo.feature.common.Preprocessing
import com.intel.analytics.zoo.feature.image.ImageProcessing
import com.intel.analytics.zoo.pipeline.api.keras.metrics.{Accuracy, BinaryAccuracy, CategoricalAccuracy, SparseCategoricalAccuracy}
import org.tensorflow.framework.GraphDef
import org.tensorflow.{DataType, Tensor => TTensor}

import scala.io.Source
import scala.reflect.io.Path

object TFUtils {

  val defaultSessionConfig = SessionConfig()

  private[zoo] def getTrainMeta(trainMetaPath: Path) = {
    val jsonStr = Source.fromFile(trainMetaPath.jfile).getLines().mkString
    import org.json4s._
    import org.json4s.jackson.JsonMethods._
    implicit val formats = DefaultFormats

    parse(jsonStr).camelizeKeys.extract[TrainMeta]
  }

  private[zoo] def parseGraph(graphProtoTxt: String): GraphDef = {
    var fr: File = null
    var in: InputStream = null
    try {
      fr = new File(graphProtoTxt)
      in = new FileInputStream(fr)

      GraphDef.parseFrom(in)
    } finally {
      if (in != null) in.close()
    }
  }

  private def decodeUVarInt64(bytes: Array[Byte], offset: Int): (Long, Int) = {
    var shift = 0
    var p = offset
    var result: Long = 0
    while (shift  <= 63 && p < bytes.length) {
      val b = bytes(p)
      p += 1
      if ((b & 128) != 0) {
        result |= ((b & 127) << shift)
      } else {
        result |= (b << shift)
        return (result, p)
      }

      shift += 7
    }
    (result, p)
  }

  private def getOffsets(buffer: ByteBuffer, numElem: Int): Array[Int] = {
    val offsetsBuffer = ByteBuffer.wrap(buffer.array()
      .slice(buffer.arrayOffset(), numElem * 8))
      .order(ByteOrder.nativeOrder())
      .asLongBuffer()
    val offsets = new Array[Long](numElem)
    offsetsBuffer.get(offsets)
    offsets.map(_.toInt)
  }

  private[zoo] def tf2bigdl(t: TTensor[_], output: Tensor[_]) = {
    val shape = t.shape().map(_.toInt)
    output.resize(shape)
    val dataType = t.dataType()

    val numericDataTypes = Set(DataType.FLOAT,
      DataType.UINT8, DataType.INT32, DataType.INT64, DataType.DOUBLE)

    if (dataType == DataType.STRING) {
      val outputTensor = output.asInstanceOf[Tensor[Array[Byte]]]
      require(t.numDimensions() <= 1, "only scalar or Vector string are supported")
      val elements = t.numElements()
      val buffer = ByteBuffer.allocate(t.numBytes())
      t.writeTo(buffer)
      val offsets = getOffsets(buffer, elements)
      val storage = outputTensor.storage().array()
      val strDataOffset = elements * 8
      var i = 0
      while (i < elements) {
        val offset = buffer.arrayOffset() + offsets(i) + strDataOffset
        val (strLen, strStart) = decodeUVarInt64(buffer.array(), offset)
        storage(outputTensor.storageOffset() - 1 + i) = buffer.array()
          .slice(strStart, strStart + strLen.toInt)
        i += 1
      }
    } else if (numericDataTypes(dataType)) {
      dataType match {
        case DataType.FLOAT =>
          val outputTensor = output.asInstanceOf[Tensor[Float]]
          val buffer = FloatBuffer.wrap(
            outputTensor.storage().array(),
            outputTensor.storageOffset() - 1,
            shape.product)
          t.writeTo(buffer)
        case DataType.UINT8 =>
          val outputTensor = output.asInstanceOf[Tensor[Float]]
          val arr = new Array[Byte](shape.product)
          val buffer = ByteBuffer.wrap(arr)
          t.writeTo(buffer)
          byte2float(arr, outputTensor.storage().array(), outputTensor.storageOffset() - 1)
        case DataType.INT32 =>
          val outputTensor = output.asInstanceOf[Tensor[Float]]
          val arr = new Array[Int](shape.product)
          val buffer = IntBuffer.wrap(arr)
          t.writeTo(buffer)
          int2float(arr, outputTensor.storage().array(), outputTensor.storageOffset() - 1)
        case DataType.INT64 =>
          val outputTensor = output.asInstanceOf[Tensor[Float]]
          val arr = new Array[Long](shape.product)
          val buffer = LongBuffer.wrap(arr)
          t.writeTo(buffer)
          long2float(arr, outputTensor.storage().array(), outputTensor.storageOffset() - 1)
        case DataType.DOUBLE =>
          val outputTensor = output.asInstanceOf[Tensor[Float]]
          val arr = new Array[Double](shape.product)
          val buffer = DoubleBuffer.wrap(arr)
          t.writeTo(buffer)
          double2float(arr, outputTensor.storage().array(), outputTensor.storageOffset() - 1)
      }

    } else {
      throw new Exception(s"data type ${dataType} are not supported")
    }
  }

  private[zoo] def byte2float(src: Array[Byte], dest: Array[Float], offset: Int): Unit = {
    val length = src.length
    var i = 0
    while (i < length) {
      dest(offset + i) = src(i).toFloat
      i += 1
    }
  }

  private[zoo] def int2float(src: Array[Int], dest: Array[Float], offset: Int): Unit = {
    val length = src.length
    var i = 0
    while (i < length) {
      dest(offset + i) = src(i).toFloat
      i += 1
    }
  }

  private[zoo] def long2float(src: Array[Long], dest: Array[Float], offset: Int): Unit = {
    val length = src.length
    var i = 0
    while (i < length) {
      dest(offset + i) = src(i).toFloat
      i += 1
    }
  }

  private[zoo] def double2float(src: Array[Double], dest: Array[Float], offset: Int): Unit = {
    val length = src.length
    var i = 0
    while (i < length) {
      dest(offset + i) = src(i).toFloat
      i += 1
    }
  }

  def tfenum2datatype(enum: Int): DataType = {
    enum match {
      case 1 => DataType.FLOAT
      case 2 => DataType.DOUBLE
      case 3 => DataType.INT32
      case 4 => DataType.UINT8
      case 7 => DataType.STRING
      case 9 => DataType.INT64
      case 10 => DataType.BOOL
      case _ => throw new IllegalArgumentException(s"unsupported tensorflow datatype $enum")

    }
  }

  def tfdatatype2enum(dataType: DataType): Int = {
    dataType match {
      case DataType.FLOAT => 1
      case DataType.DOUBLE => 2
      case DataType.INT32 => 3
      case DataType.UINT8 => 4
      case DataType.STRING => 7
      case DataType.INT64 => 9
      case DataType.BOOL => 10
      case _ => throw new IllegalArgumentException(s"unsupported tensorflow datatype $dataType")

    }
  }

}

{'fraction_non_alphanumeric': 0.11598502507593417, 'fraction_numerical': 0.013067740340467614, 'mean_word_length': 2.791644349223353, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6892051', 'n_tokens_mistral': 4484, 'n_tokens_neox': 4295, 'n_words': 1111}