<commit_before>package io.ddf.jdbc.content

import java.lang.{Integer => JInt}
import java.util
import java.util.{HashMap => JHMap, List => JList, Map => JMap}

import io.ddf.content.Schema.{Column, ColumnType}
import io.ddf.content.{IHandleRepresentations, Schema}
import io.ddf.exception.DDFException
import io.ddf.{DDF, Factor}

import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
import scala.reflect.ClassTag

class SchemaHandler(ddf: DDF) extends io.ddf.content.SchemaHandler(ddf: DDF) {

  @throws(classOf[DDFException])
  override def setFactorLevelsForStringColumns(xCols: Array[String]) {
    xCols.foreach { col =>
      val c = this.getColumn(col)
      if (c.getType eq Schema.ColumnType.STRING) {
        this.setAsFactor(c.getName)
      }
    }
  }

  @throws(classOf[DDFException])
  override def computeFactorLevelsAndLevelCounts = {
    val columnIndexes: util.List[Integer] = new util.ArrayList[Integer]
    val columnTypes: util.List[Schema.ColumnType] = new util.ArrayList[Schema.ColumnType]
    import scala.collection.JavaConversions._
    for (col <- this.getColumns) {
      if (col.getColumnClass eq Schema.ColumnClass.FACTOR) {
        var colFactor: Factor[_] = col.getOptionalFactor
        if (colFactor == null || colFactor.getLevelCounts == null || colFactor.getLevels == null) {
          if (colFactor == null) {
            colFactor = this.setAsFactor(col.getName)
          }
          columnIndexes.add(this.getColumnIndex(col.getName))
          columnTypes.add(col.getType)
        }
      }
    }
    var listLevelCounts: JMap[Integer, JMap[String, Integer]] = null

    //loop through all factors and compute factor

    val table_name = this.getDDF.getTableName
    for (col <- this.getColumns) {
      if (col.getColumnClass eq Schema.ColumnClass.FACTOR) {

        val command = "select " + col.getName() + ", count(" + col.getName() + ") from " + table_name + " group by " + col.getName()
        print(command)

        var sqlResult = this.getDDF.sql(command,"" )
        //JMap[String, Integer]
        var result = sqlResult.getRows()
        val levelCounts: java.util.Map[String, Integer] = new java.util.HashMap[String,Integer]()
        for (item <- result) {
          print(">>>item=")
          println(item)
          levelCounts.put(item, 1)
        }

        if (levelCounts != null) {
          val factor: Factor[_] = col.getOptionalFactor
          val levels: util.List[String] = new util.ArrayList[String](levelCounts.keySet)
          factor.setLevelCounts(levelCounts)
          factor.setLevels(levels, false)
        }
      }
    }

    /*val repHandler: IHandleRepresentations = this.getDDF.getRepresentationHandler
    if (columnIndexes.size > 0) {
      val sqlResult = this.getDDF.getRepresentationHandler.get(Representations.SQL_ARRAY_RESULT).asInstanceOf[SqlArrayResult]
      listLevelCounts = GetMultiFactor.getFactorCounts(sqlResult.result, columnIndexes, columnTypes, classOf[Array[AnyRef]])
      if (listLevelCounts == null) {
        throw new DDFException("Error getting factor levels counts")
      }
      import scala.collection.JavaConversions._
      for (colIndex <- columnIndexes) {
        val column: Column = this.getColumn(this.getColumnName(colIndex))
        val levelCounts: JMap[String, Integer] = listLevelCounts.get(colIndex)
        if (levelCounts != null) {
          val factor: Factor[_] = column.getOptionalFactor
          val levels: util.List[String] = new util.ArrayList[String](levelCounts.keySet)
          factor.setLevelCounts(levelCounts)
          factor.setLevels(levels, false)
        }
      }
    }*/
  }
}

/**
  */
object GetMultiFactor {

  //For Java interoperability
  def getFactorCounts[T](rdd: JList[Array[_]], columnIndexes: JList[JInt], columnTypes: JList[ColumnType], rddUnit: Class[T]):
  JMap[JInt, JMap[String, JInt]] = {
    getFactorCounts(rdd.asScala.toList, columnIndexes, columnTypes)(ClassTag(rddUnit))
  }

  def getFactorCounts[T](rdd: List[Array[_]], columnIndexes: JList[JInt], columnTypes: JList[ColumnType])(implicit tag: ClassTag[T]): JMap[JInt, JMap[String, JInt]] = {
    val columnIndexesWithTypes = (columnIndexes zip columnTypes).toList

    tag.runtimeClass match {
      case arrObj if arrObj == classOf[Array[Object]] =>
        val mapper = new ArrayObjectMultiFactorMapper(columnIndexesWithTypes)
        rdd.map(mapper).reduce(new MultiFactorReducer)<commit_msg>factor works<commit_after>package io.ddf.jdbc.content

import java.lang.{Integer => JInt}
import java.util
import java.util.{HashMap => JHMap, List => JList, Map => JMap}

import io.ddf.content.Schema.{Column, ColumnType}
import io.ddf.content.{IHandleRepresentations, Schema}
import io.ddf.exception.DDFException
import io.ddf.{DDF, Factor}

import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
import scala.reflect.ClassTag

class SchemaHandler(ddf: DDF) extends io.ddf.content.SchemaHandler(ddf: DDF) {

  @throws(classOf[DDFException])
  override def setFactorLevelsForStringColumns(xCols: Array[String]) {
    xCols.foreach { col =>
      val c = this.getColumn(col)
      if (c.getType eq Schema.ColumnType.STRING) {
        this.setAsFactor(c.getName)
      }
    }
  }

  @throws(classOf[DDFException])
  override def computeFactorLevelsAndLevelCounts = {
    val columnIndexes: util.List[Integer] = new util.ArrayList[Integer]
    val columnTypes: util.List[Schema.ColumnType] = new util.ArrayList[Schema.ColumnType]
    import scala.collection.JavaConversions._
    for (col <- this.getColumns) {
      if (col.getColumnClass eq Schema.ColumnClass.FACTOR) {
        var colFactor: Factor[_] = col.getOptionalFactor
        if (colFactor == null || colFactor.getLevelCounts == null || colFactor.getLevels == null) {
          if (colFactor == null) {
            colFactor = this.setAsFactor(col.getName)
          }
          columnIndexes.add(this.getColumnIndex(col.getName))
          columnTypes.add(col.getType)
        }
      }
    }
    var listLevelCounts: JMap[Integer, JMap[String, Integer]] = null

    //loop through all factors and compute factor

    //(select * from hung_test) tmp
    val table_name = this.getDDF.getTableName + ") tmp"
    for (col <- this.getColumns) {
      if (col.getColumnClass eq Schema.ColumnClass.FACTOR) {

        val command = "select " + col.getName() + ", count(" + col.getName() + ") from (" + table_name + " group by " + col.getName()

        var sqlResult = this.getManager.sql(command,"" )
        //JMap[String, Integer]
        var result = sqlResult.getRows()
        val levelCounts: java.util.Map[String, Integer] = new java.util.HashMap[String,Integer]()
        for (item <- result) {
          if(item.split("\t").length > 1)
            levelCounts.put(item.split("\t")(0), Integer.parseInt(item.split("\t")(1)))
          else //todo log this properly
            print(item)
        }

        if (levelCounts != null) {
          val factor: Factor[_] = col.getOptionalFactor
          val levels: util.List[String] = new util.ArrayList[String](levelCounts.keySet)
          factor.setLevelCounts(levelCounts)
          factor.setLevels(levels, false)
        }
      }
    }
  }
}

/**
  */
object GetMultiFactor {

  //For Java interoperability
  def getFactorCounts[T](rdd: JList[Array[_]], columnIndexes: JList[JInt], columnTypes: JList[ColumnType], rddUnit: Class[T]):
  JMap[JInt, JMap[String, JInt]] = {
    getFactorCounts(rdd.asScala.toList, columnIndexes, columnTypes)(ClassTag(rddUnit))
  }

  def getFactorCounts[T](rdd: List[Array[_]], columnIndexes: JList[JInt], columnTypes: JList[ColumnType])(implicit tag: ClassTag[T]): JMap[JInt, JMap[String, JInt]] = {
    val columnIndexesWithTypes = (columnIndexes zip columnTypes).toList

    tag.runtimeClass match {
      case arrObj if arrObj == classOf[Array[Object]] =>
        val mapper = new ArrayObjectMultiFactorMapper(columnIndexesWithTypes)
        rdd.map(mapper).reduce(new MultiFactorReducer)
{'fraction_non_alphanumeric': 0.12177489717063443, 'fraction_numerical': 0.0006232082762059081, 'mean_word_length': 3.3209477652127086, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '825931', 'n_tokens_mistral': 2565, 'n_tokens_neox': 2511, 'n_words': 492}