<commit_before>import org.apache.commons.lang.StringEscapeUtils
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.linalg.Vector
import org.apache.spark.mllib.linalg.distributed.RowMatrix
import org.apache.spark.rdd.RDD
import org.apache.spark.sql._

object utils {

  def plus2(base:Int) : Int = {
    return base + 2
  }

  // See: http://spark.apache.org/docs/latest/mllib-clustering.html
  def schemaRddToVectorRdd(schemaRdd:RDD[Row]) : RDD[Vector] = {
    schemaRdd.map(row => {
      Vectors.dense(row.toSeq.toArray.map({
        case s: String => s.toDouble
        case l: Long => l.toDouble
        case i: Integer => i.toDouble
        case d: Double => d
        case _ => 0.0
      }))
    })
  }

  def readColumnInt(df: DataFrame, colName: String): Array[Integer] = {
    df.select(colName).rdd.map(row => row(0).asInstanceOf[Integer]).collect()
  }

  def readColumnDouble(df: DataFrame, colName: String): Array[Double] = {
    df.select(colName).rdd.map(row => row(0).asInstanceOf[Double]).collect()
  }

  def readColumnBoolean(df: DataFrame, colName: String): Array[Boolean] = {
    df.select(colName).rdd.map(row => row(0).asInstanceOf[Boolean]).collect()
  }

  def readColumnString(df: DataFrame, colName: String): String = {
    val column = df.select(colName).rdd.map(row => row(0).asInstanceOf[String]).collect()
    val escaped = column.map(string => StringEscapeUtils.escapeCsv(string))
    val joined = escaped.mkString("\n")
    return joined + "\n"
  }

  def readColumnDefault(df: DataFrame, colName: String): Array[Any] = {
    df.select(colName).rdd.map(row => row(0)).collect()
  }
}<commit_msg>use 'Array[Int]' to avoid object-type marshalling<commit_after>import org.apache.commons.lang.StringEscapeUtils
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.linalg.Vector
import org.apache.spark.mllib.linalg.distributed.RowMatrix
import org.apache.spark.rdd.RDD
import org.apache.spark.sql._

object utils {

  def plus2(base:Int) : Int = {
    return base + 2
  }

  // See: http://spark.apache.org/docs/latest/mllib-clustering.html
  def schemaRddToVectorRdd(schemaRdd:RDD[Row]) : RDD[Vector] = {
    schemaRdd.map(row => {
      Vectors.dense(row.toSeq.toArray.map({
        case s: String => s.toDouble
        case l: Long => l.toDouble
        case i: Integer => i.toDouble
        case d: Double => d
        case _ => 0.0
      }))
    })
  }

  def readColumnInt(df: DataFrame, colName: String): Array[Int] = {
    df.select(colName).rdd.map(row => row(0).asInstanceOf[Integer]).collect()
  }

  def readColumnDouble(df: DataFrame, colName: String): Array[Double] = {
    df.select(colName).rdd.map(row => row(0).asInstanceOf[Double]).collect()
  }

  def readColumnBoolean(df: DataFrame, colName: String): Array[Boolean] = {
    df.select(colName).rdd.map(row => row(0).asInstanceOf[Boolean]).collect()
  }

  def readColumnString(df: DataFrame, colName: String): String = {
    val column = df.select(colName).rdd.map(row => row(0).asInstanceOf[String]).collect()
    val escaped = column.map(string => StringEscapeUtils.escapeCsv(string))
    val joined = escaped.mkString("\n")
    return joined + "\n"
  }

  def readColumnDefault(df: DataFrame, colName: String): Array[Any] = {
    df.select(colName).rdd.map(row => row(0)).collect()
  }
}
{'fraction_non_alphanumeric': 0.15497601918465229, 'fraction_numerical': 0.00539568345323741, 'mean_word_length': 3.6671328671328673, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3675145', 'n_tokens_mistral': 1205, 'n_tokens_neox': 1165, 'n_words': 199}