<commit_before>/*
 * Copyright 2009-2015 DigitalGlobe, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and limitations under the License.
 */

package org.mrgeo.utils

import java.awt.image._
import java.io._
import java.net.URI
import java.nio._
import java.nio.file.Files
import java.util.zip.GZIPOutputStream
import javax.xml.bind.DatatypeConverter

import org.apache.commons.io.IOUtils
import org.apache.commons.lang3.ArrayUtils
import org.apache.hadoop.fs.Path
import org.apache.spark.Logging
import org.gdal.gdal.{Band, Dataset, Driver, gdal}
import org.gdal.gdalconst.gdalconstConstants
import org.gdal.osr.{CoordinateTransformation, SpatialReference, osr, osrConstants}
import org.mrgeo.core.{MrGeoConstants, MrGeoProperties}
import org.mrgeo.data.raster.RasterUtils
import org.mrgeo.hdfs.utils.HadoopFileUtils
import org.mrgeo.utils.MrGeoImplicits._

import scala.collection.JavaConversions._



class GDALException extends IOException  {
  private var origException: Exception = null

  def this(e: Exception) {
    this()
    origException = e
  }

  def this(msg: String) {
    this()
    origException = new Exception(msg)
  }
  def this(msg: String, e:Exception) {
    this()
    origException = new Exception(msg, e)
  }

  override def printStackTrace() {
    this.origException.printStackTrace()
  }
}

object GDALUtils extends Logging {

  val EPSG4326: String = osrConstants.SRS_WKT_WGS84

  private val VSI_PREFIX: String = "/vsimem/"
  private val GDAL_PAM_ENABLED: String = "GDAL_PAM_ENABLED"

  initializeGDAL()

  // empty method to force static initializer
  def register() = {}

  def isValidDataset(imagename: String): Boolean = {
    try {
      val image: Dataset = GDALUtils.open(imagename)
      GDALUtils.close(image)
      return true
    }
    catch {
      case ignored: IOException =>
    }

    false
  }

  def open(stream: InputStream): Dataset = {
    val imagename: String = "stream" + HadoopUtils.createRandomString(5)
    val bytes: Array[Byte] = IOUtils.toByteArray(stream)

    gdal.FileFromMemBuffer(VSI_PREFIX + imagename, bytes)

    val image = gdal.Open(VSI_PREFIX + imagename)
    if (image != null) {
      logDebug("  Image loaded successfully: " + imagename)
      return image
    }

    logInfo(
      "Image not loaded, but unfortunately no exceptions were thrown, look for a logged explanation somewhere above")
    null
  }

  def createEmptyMemoryRaster(src: Dataset, width: Int, height: Int): Dataset = {

    val bands: Int = src.getRasterCount
    var datatype: Int = -1

    val nodatas = Array.newBuilder[Double]

    val nodata = Array.ofDim[java.lang.Double](1)
    var i: Int = 1
    while (i <= src.GetRasterCount()) {
      val band: Band = src.GetRasterBand(i)
      if (datatype < 0) {
        datatype = band.getDataType
      }

      band.GetNoDataValue(nodata)
      nodatas += (if (nodata(0) == null) { java.lang.Double.NaN } else { nodata(0) })
      i += 1
    }

    createEmptyMemoryRaster(width, height, bands, datatype, nodatas.result())
  }

  def createEmptyMemoryRaster(width: Int, height: Int, bands: Int, datatype: Int, nodatas: Array[Double] = null):Dataset = {
    val driver: Driver = gdal.GetDriverByName("MEM")

    val dataset = driver.Create("InMem", width, height, bands, datatype)

    if (dataset != null) {
      if (nodatas != null) {
        var i: Int = 1
        while (i <= dataset.getRasterCount) {
          val nodata: Double = nodatas(i - 1)
          val band: Band = dataset.GetRasterBand(i)
          band.Fill(nodata)
          band.SetNoDataValue(nodata)
          i += 1
        }
      }
      return dataset
    }

    null
  }

  def toDataset(raster: Raster, nodata: Double = Double.NegativeInfinity,
      bounds:Either[Bounds, TMSUtils.Bounds] = null): Dataset = {
    val nodatas = if (nodata == Double.NegativeInfinity) null else Array.fill[Double](raster.getNumBands)(nodata)
    toDataset(raster, nodatas, bounds)
  }

  def toDataset(raster: Raster, nodatas: Array[Double],
      bounds:Either[Bounds, TMSUtils.Bounds]): Dataset = {
    val datatype = toGDALDataType(raster.getTransferType)

    val ds = GDALUtils.createEmptyMemoryRaster(raster.getWidth, raster.getHeight, raster.getNumBands, datatype, nodatas)

    if (ds != null) {
      copyToDataset(ds, raster)
    }

    val bnds = if (bounds == null) {
      null
    }
    else {
      bounds match {
      case Left(b) => TMSUtils.Bounds.asTMSBounds(b)
      case Right(t) => t
      }
    }

    val xform = new Array[Double](6)
    if (bnds != null) {

      xform(0) = bnds.w
      xform(1) = bnds.width / ds.getRasterXSize
      xform(2) = 0
      xform(3) = bnds.n
      xform(4) = 0
      xform(5) = -bnds.height / ds.getRasterYSize

      ds.SetProjection(GDALUtils.EPSG4326)
    }
    else
    {
      xform(0) = 0
      xform(1) = ds.getRasterXSize
      xform(2) = 0
      xform(3) = 0
      xform(4) = 0
      xform(5) = -ds.getRasterYSize
    }

    ds.SetGeoTransform(xform)

    ds
  }

  def toGDALDataType(rasterType: Int): Int = {
    rasterType match {
    case DataBuffer.TYPE_BYTE => gdalconstConstants.GDT_Byte
    case DataBuffer.TYPE_SHORT => gdalconstConstants.GDT_Int16
    case DataBuffer.TYPE_USHORT => gdalconstConstants.GDT_UInt16
    case DataBuffer.TYPE_INT => gdalconstConstants.GDT_Int32
    case DataBuffer.TYPE_FLOAT => gdalconstConstants.GDT_Float32
    case DataBuffer.TYPE_DOUBLE => gdalconstConstants.GDT_Float64
    case _ => gdalconstConstants.GDT_Unknown
    }
  }

  def toRasterDataBufferType(gdaldatatype: Int): Int = {
    gdaldatatype match {
    case gdalconstConstants.GDT_Byte => DataBuffer.TYPE_BYTE
    case gdalconstConstants.GDT_UInt16 => DataBuffer.TYPE_USHORT
    case gdalconstConstants.GDT_Int16 => DataBuffer.TYPE_SHORT
    case gdalconstConstants.GDT_UInt32 => DataBuffer.TYPE_INT
    case gdalconstConstants.GDT_Int32 => DataBuffer.TYPE_INT
    case gdalconstConstants.GDT_Float32 => DataBuffer.TYPE_FLOAT
    case gdalconstConstants.GDT_Float64 => DataBuffer.TYPE_DOUBLE
    case _ => DataBuffer.TYPE_UNDEFINED
    }
  }

  def toRaster(image: Dataset):Raster = {
    val bands: Int = image.getRasterCount

    val bandlist = Array.range(1, image.getRasterCount + 1)

    val datatype = image.GetRasterBand(1).getDataType
    val pixelsize = gdal.GetDataTypeSize(datatype) / 8

    val width = image.getRasterXSize
    val height = image.getRasterYSize

    val pixelstride = pixelsize * bands
    val linestride = pixelstride * width

    val rastersize = linestride * height

    val data: ByteBuffer = ByteBuffer.allocateDirect(rastersize)
    data.order(ByteOrder.nativeOrder)

    // read the data interleaved (it _should_ be much more efficient reading)
    image.ReadRaster_Direct(0, 0, width, height, width, height, datatype, data,
      bandlist, pixelstride, linestride, pixelsize)

    data.rewind

    toRaster(height, width, bands, datatype, data)
  }

  def toRaster(height: Int, width: Int, bands: Int, gdaldatatype: Int, data: Array[Byte]): Raster = {
    toRaster(height, width, bands, gdaldatatype, ByteBuffer.wrap(data))
  }

  def toRaster(height: Int, width: Int, bands: Int, gdaldatatype: Int, data: ByteBuffer): Raster = {
    val datatype = toRasterDataBufferType(gdaldatatype)
    val bandbytes = height * width * (gdal.GetDataTypeSize(gdaldatatype) / 8)

    val databytes = bandbytes * bands
    val bandoffsets = Array.range(0, bands)

    // keep the raster interleaved
    val sm = new PixelInterleavedSampleModel(datatype, width, height, bands, bands * width, bandoffsets)

    val db =
      datatype match {
      case DataBuffer.TYPE_BYTE =>
        val bytedata: Array[Byte] = new Array[Byte](databytes)
        data.get(bytedata)
        new DataBufferByte(bytedata, bytedata.length)
      case DataBuffer.TYPE_FLOAT =>
        val floatbuff: FloatBuffer = data.asFloatBuffer
        val floatdata: Array[Float] = new Array[Float](databytes / RasterUtils.FLOAT_BYTES)
        floatbuff.get(floatdata)
        new DataBufferFloat(floatdata, floatdata.length)
      case DataBuffer.TYPE_DOUBLE =>
        val doublebuff: DoubleBuffer = data.asDoubleBuffer
        val doubledata: Array[Double] = new Array[Double](databytes / RasterUtils.DOUBLE_BYTES)
        doublebuff.get(doubledata)
        new DataBufferDouble(doubledata, doubledata.length)
      case DataBuffer.TYPE_INT =>
        val intbuff: IntBuffer = data.asIntBuffer
        val intdata: Array[Int] = new Array[Int](databytes / RasterUtils.INT_BYTES)
        intbuff.get(intdata)
        new DataBufferInt(intdata, intdata.length)
      case DataBuffer.TYPE_SHORT =>
        val shortbuff: ShortBuffer = data.asShortBuffer
        val shortdata: Array[Short] = new Array[Short](databytes / RasterUtils.SHORT_BYTES)
        shortbuff.get(shortdata)
        new DataBufferShort(shortdata, shortdata.length)
      case DataBuffer.TYPE_USHORT =>
        val ushortbuff: ShortBuffer = data.asShortBuffer
        val ushortdata: Array[Short] = new Array[Short](databytes / RasterUtils.SHORT_BYTES)
        ushortbuff.get(ushortdata)
        new DataBufferUShort(ushortdata, ushortdata.length)
      case _ =>
        throw new GDALException("Error trying to read raster.  Bad raster data type")
      }

    Raster.createWritableRaster(sm, db, null)
  }

  def swapBytes(bytes: Array[Byte], gdaldatatype: Int) = {

    var tmp: Byte = 0
    var i: Int = 0
    gdaldatatype match {
      // 2 byte value... swap byte 1 with 2
    case gdalconstConstants.GDT_UInt16 | gdalconstConstants.GDT_Int16 =>
      while (i + 1 < bytes.length) {
        tmp = bytes(i)
        bytes(i) = bytes(i + 1)
        bytes(i + 1) = tmp
        i += 2
      }
    // 4 byte value... swap bytes 1 & 4, 2 & 3
    case gdalconstConstants.GDT_UInt32 | gdalconstConstants.GDT_Int32 | gdalconstConstants.GDT_Float32 =>
      while (i + 3 < bytes.length) {
        // swap 0 & 3
        tmp = bytes(i)
        bytes(i) = bytes(i + 3)
        bytes(i + 3) = tmp

        // swap 1 & 2
        tmp = bytes(i + 1)
        bytes(i + 1) = bytes(i + 2)
        bytes(i + 2) = tmp
        i += 4
      }
    // 8 byte value... swap bytes 1 & 8, 2 & 7, 3 & 6, 4 & 5
    case gdalconstConstants.GDT_Float64 =><commit_msg>Modification to allow ingesting imagery of type byte<commit_after>/*
 * Copyright 2009-2015 DigitalGlobe, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and limitations under the License.
 */

package org.mrgeo.utils

import java.awt.image._
import java.io._
import java.net.URI
import java.nio._
import java.nio.file.Files
import java.util.zip.GZIPOutputStream
import javax.xml.bind.DatatypeConverter

import org.apache.commons.io.IOUtils
import org.apache.commons.lang3.ArrayUtils
import org.apache.hadoop.fs.Path
import org.apache.spark.Logging
import org.gdal.gdal.{Band, Dataset, Driver, gdal}
import org.gdal.gdalconst.gdalconstConstants
import org.gdal.osr.{CoordinateTransformation, SpatialReference, osr, osrConstants}
import org.mrgeo.core.{MrGeoConstants, MrGeoProperties}
import org.mrgeo.data.raster.RasterUtils
import org.mrgeo.hdfs.utils.HadoopFileUtils
import org.mrgeo.utils.MrGeoImplicits._

import scala.collection.JavaConversions._



class GDALException extends IOException  {
  private var origException: Exception = null

  def this(e: Exception) {
    this()
    origException = e
  }

  def this(msg: String) {
    this()
    origException = new Exception(msg)
  }
  def this(msg: String, e:Exception) {
    this()
    origException = new Exception(msg, e)
  }

  override def printStackTrace() {
    this.origException.printStackTrace()
  }
}

object GDALUtils extends Logging {

  val EPSG4326: String = osrConstants.SRS_WKT_WGS84

  private val VSI_PREFIX: String = "/vsimem/"
  private val GDAL_PAM_ENABLED: String = "GDAL_PAM_ENABLED"

  initializeGDAL()

  // empty method to force static initializer
  def register() = {}

  def isValidDataset(imagename: String): Boolean = {
    try {
      val image: Dataset = GDALUtils.open(imagename)
      GDALUtils.close(image)
      return true
    }
    catch {
      case ignored: IOException =>
    }

    false
  }

  def open(stream: InputStream): Dataset = {
    val imagename: String = "stream" + HadoopUtils.createRandomString(5)
    val bytes: Array[Byte] = IOUtils.toByteArray(stream)

    gdal.FileFromMemBuffer(VSI_PREFIX + imagename, bytes)

    val image = gdal.Open(VSI_PREFIX + imagename)
    if (image != null) {
      logDebug("  Image loaded successfully: " + imagename)
      return image
    }

    logInfo(
      "Image not loaded, but unfortunately no exceptions were thrown, look for a logged explanation somewhere above")
    null
  }

  def createEmptyMemoryRaster(src: Dataset, width: Int, height: Int): Dataset = {

    val bands: Int = src.getRasterCount
    var datatype: Int = -1

    val nodatas = Array.newBuilder[Double]

    val nodata = Array.ofDim[java.lang.Double](1)
    var i: Int = 1
    while (i <= src.GetRasterCount()) {
      val band: Band = src.GetRasterBand(i)
      if (datatype < 0) {
        datatype = band.getDataType
      }

      band.GetNoDataValue(nodata)
      nodatas += (if (nodata(0) == null) { java.lang.Double.NaN } else { nodata(0) })
      i += 1
    }

    createEmptyMemoryRaster(width, height, bands, datatype, nodatas.result())
  }

  def createEmptyMemoryRaster(width: Int, height: Int, bands: Int, datatype: Int, nodatas: Array[Double] = null):Dataset = {
    val driver: Driver = gdal.GetDriverByName("MEM")

    val dataset = driver.Create("InMem", width, height, bands, datatype)

    if (dataset != null) {
      if (nodatas != null) {
        var i: Int = 1
        while (i <= dataset.getRasterCount) {
          val nodata: Double = nodatas(i - 1)
          val band: Band = dataset.GetRasterBand(i)
          band.Fill(nodata)
          band.SetNoDataValue(nodata)
          i += 1
        }
      }
      return dataset
    }

    null
  }

  def toDataset(raster: Raster, nodata: Double = Double.NegativeInfinity,
      bounds:Either[Bounds, TMSUtils.Bounds] = null): Dataset = {
    val nodatas = if (nodata == Double.NegativeInfinity) null else Array.fill[Double](raster.getNumBands)(nodata)
    toDataset(raster, nodatas, bounds)
  }

  def toDataset(raster: Raster, nodatas: Array[Double],
      bounds:Either[Bounds, TMSUtils.Bounds]): Dataset = {
    val datatype = toGDALDataType(raster.getTransferType)

    val ds = GDALUtils.createEmptyMemoryRaster(raster.getWidth, raster.getHeight, raster.getNumBands, datatype, nodatas)

    if (ds != null) {
      copyToDataset(ds, raster)
    }

    val bnds = if (bounds == null) {
      null
    }
    else {
      bounds match {
      case Left(b) => TMSUtils.Bounds.asTMSBounds(b)
      case Right(t) => t
      }
    }

    val xform = new Array[Double](6)
    if (bnds != null) {

      xform(0) = bnds.w
      xform(1) = bnds.width / ds.getRasterXSize
      xform(2) = 0
      xform(3) = bnds.n
      xform(4) = 0
      xform(5) = -bnds.height / ds.getRasterYSize

      ds.SetProjection(GDALUtils.EPSG4326)
    }
    else
    {
      xform(0) = 0
      xform(1) = ds.getRasterXSize
      xform(2) = 0
      xform(3) = 0
      xform(4) = 0
      xform(5) = -ds.getRasterYSize
    }

    ds.SetGeoTransform(xform)

    ds
  }

  def toGDALDataType(rasterType: Int): Int = {
    rasterType match {
    case DataBuffer.TYPE_BYTE => gdalconstConstants.GDT_Byte
    case DataBuffer.TYPE_SHORT => gdalconstConstants.GDT_Int16
    case DataBuffer.TYPE_USHORT => gdalconstConstants.GDT_UInt16
    case DataBuffer.TYPE_INT => gdalconstConstants.GDT_Int32
    case DataBuffer.TYPE_FLOAT => gdalconstConstants.GDT_Float32
    case DataBuffer.TYPE_DOUBLE => gdalconstConstants.GDT_Float64
    case _ => gdalconstConstants.GDT_Unknown
    }
  }

  def toRasterDataBufferType(gdaldatatype: Int): Int = {
    gdaldatatype match {
    case gdalconstConstants.GDT_Byte => DataBuffer.TYPE_BYTE
    case gdalconstConstants.GDT_UInt16 => DataBuffer.TYPE_USHORT
    case gdalconstConstants.GDT_Int16 => DataBuffer.TYPE_SHORT
    case gdalconstConstants.GDT_UInt32 => DataBuffer.TYPE_INT
    case gdalconstConstants.GDT_Int32 => DataBuffer.TYPE_INT
    case gdalconstConstants.GDT_Float32 => DataBuffer.TYPE_FLOAT
    case gdalconstConstants.GDT_Float64 => DataBuffer.TYPE_DOUBLE
    case _ => DataBuffer.TYPE_UNDEFINED
    }
  }

  def toRaster(image: Dataset):Raster = {
    val bands: Int = image.getRasterCount

    val bandlist = Array.range(1, image.getRasterCount + 1)

    val datatype = image.GetRasterBand(1).getDataType
    val pixelsize = gdal.GetDataTypeSize(datatype) / 8

    val width = image.getRasterXSize
    val height = image.getRasterYSize

    val pixelstride = pixelsize * bands
    val linestride = pixelstride * width

    val rastersize = linestride * height

    val data: ByteBuffer = ByteBuffer.allocateDirect(rastersize)
    data.order(ByteOrder.nativeOrder)

    // read the data interleaved (it _should_ be much more efficient reading)
    image.ReadRaster_Direct(0, 0, width, height, width, height, datatype, data,
      bandlist, pixelstride, linestride, pixelsize)

    data.rewind

    toRaster(height, width, bands, datatype, data)
  }

  def toRaster(height: Int, width: Int, bands: Int, gdaldatatype: Int, data: Array[Byte]): Raster = {
    toRaster(height, width, bands, gdaldatatype, ByteBuffer.wrap(data))
  }

  def toRaster(height: Int, width: Int, bands: Int, gdaldatatype: Int, data: ByteBuffer): Raster = {
    val datatype = toRasterDataBufferType(gdaldatatype)
    val bandbytes = height * width * (gdal.GetDataTypeSize(gdaldatatype) / 8)

    val databytes = bandbytes * bands
    val bandoffsets = Array.range(0, bands)

    // keep the raster interleaved
    val sm = new PixelInterleavedSampleModel(datatype, width, height, bands, bands * width, bandoffsets)

    val db =
      datatype match {
      case DataBuffer.TYPE_BYTE =>
        val bytedata: Array[Byte] = new Array[Byte](databytes)
        data.get(bytedata)
        new DataBufferByte(bytedata, bytedata.length)
      case DataBuffer.TYPE_FLOAT =>
        val floatbuff: FloatBuffer = data.asFloatBuffer
        val floatdata: Array[Float] = new Array[Float](databytes / RasterUtils.FLOAT_BYTES)
        floatbuff.get(floatdata)
        new DataBufferFloat(floatdata, floatdata.length)
      case DataBuffer.TYPE_DOUBLE =>
        val doublebuff: DoubleBuffer = data.asDoubleBuffer
        val doubledata: Array[Double] = new Array[Double](databytes / RasterUtils.DOUBLE_BYTES)
        doublebuff.get(doubledata)
        new DataBufferDouble(doubledata, doubledata.length)
      case DataBuffer.TYPE_INT =>
        val intbuff: IntBuffer = data.asIntBuffer
        val intdata: Array[Int] = new Array[Int](databytes / RasterUtils.INT_BYTES)
        intbuff.get(intdata)
        new DataBufferInt(intdata, intdata.length)
      case DataBuffer.TYPE_SHORT =>
        val shortbuff: ShortBuffer = data.asShortBuffer
        val shortdata: Array[Short] = new Array[Short](databytes / RasterUtils.SHORT_BYTES)
        shortbuff.get(shortdata)
        new DataBufferShort(shortdata, shortdata.length)
      case DataBuffer.TYPE_USHORT =>
        val ushortbuff: ShortBuffer = data.asShortBuffer
        val ushortdata: Array[Short] = new Array[Short](databytes / RasterUtils.SHORT_BYTES)
        ushortbuff.get(ushortdata)
        new DataBufferUShort(ushortdata, ushortdata.length)
      case _ =>
        throw new GDALException("Error trying to read raster.  Bad raster data type")
      }

    Raster.createWritableRaster(sm, db, null)
  }

  def swapBytes(bytes: Array[Byte], gdaldatatype: Int) = {

    var tmp: Byte = 0
    var i: Int = 0
    gdaldatatype match {
    // Since it's byte data, there is nothing to swap - do nothing
    case gdalconstConstants.GDT_Byte => {}
    // 2 byte value... swap byte 1 with 2
    case gdalconstConstants.GDT_UInt16 | gdalconstConstants.GDT_Int16 =>
      while (i + 1 < bytes.length) {
        tmp = bytes(i)
        bytes(i) = bytes(i + 1)
        bytes(i + 1) = tmp
        i += 2
      }
    // 4 byte value... swap bytes 1 & 4, 2 & 3
    case gdalconstConstants.GDT_UInt32 | gdalconstConstants.GDT_Int32 | gdalconstConstants.GDT_Float32 =>
      while (i + 3 < bytes.length) {
        // swap 0 & 3
        tmp = bytes(i)
        bytes(i) = bytes(i + 3)
        bytes(i + 3) = tmp

        // swap 1 & 2
        tmp = bytes(i + 1)
        bytes(i + 1) = bytes(i + 2)
        bytes(i + 2) = tmp
        i += 4
      }
    // 8 byte value... swap bytes 1 & 8, 2 & 7, 3 & 6, 4 & 5
    case gdalconstConstants.GDT_Float64 =>
{'fraction_non_alphanumeric': 0.09989323678224946, 'fraction_numerical': 0.012068885484844265, 'mean_word_length': 3.0519089712243748, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4552328', 'n_tokens_mistral': 7286, 'n_tokens_neox': 6956, 'n_words': 1724}