<commit_before>package org.apache.spark.examples.h2o

/** Prostate schema definition. */
case class Prostate(ID      :Option[Long]  ,
                    CAPSULE :Option[Int]  ,
                    AGE     :Option[Int]  ,
                    RACE    :Option[Int]  ,
                    DPROS   :Option[Int]  ,
                    DCAPS   :Option[Int]  ,
                    PSA     :Option[Float],
                    VOL     :Option[Float],
                    GLEASON :Option[Int]  ) {
}

/** A dummy csv parser for prostate dataset. */
object ProstateParse extends Serializable {
  def apply(row: Array[String]): Prostate = {
    import org.apache.spark.examples.h2o.SchemaUtils._
    Prostate(long(row(0)), int(row(1)), int(row(2)), int(row(3)), int(row(4)), int(row(5)), float(row(6)), float(row(7)), int(row(8)) )
  }
}

/** Airlines schema definition. */
class Airlines (val Year              :Option[Int],
                val Month             :Option[Int],
                val DayofMonth        :Option[Int],
                val DayOfWeek         :Option[Int],
                val DepTime           :Option[Int],
                val CRSDepTime        :Option[Int],
                val ArrTime           :Option[Int],
                val CRSArrTime        :Option[Int],
                val UniqueCarrier     :Option[String],
                val FlightNum         :Option[Int],
                val TailNum           :Option[Int],
                val ActualElapsedTime :Option[Int],
                val CRSElapsedTime    :Option[Int],
                val AirTime           :Option[Int],
                val ArrDelay          :Option[Int],
                val DepDelay          :Option[Int],
                val Origin            :Option[String],
                val Dest              :Option[String],
                val Distance          :Option[Int],
                val TaxiIn            :Option[Int],
                val TaxiOut           :Option[Int],
                val Cancelled         :Option[Int],
                val CancellationCode  :Option[Int],
                val Diverted          :Option[Int],
                val CarrierDelay      :Option[Int],
                val WeatherDelay      :Option[Int],
                val NASDelay          :Option[Int],
                val SecurityDelay     :Option[Int],
                val LateAircraftDelay :Option[Int],
                val IsArrDelayed      :Option[Boolean],
                val IsDepDelayed      :Option[Boolean]) extends Product {

  /*
  def this() = this(None,None,None,None,None,None,None,None,None,
    None,None,None,None,None,None,None,None,None,
    None,None,None,None,None,None,None,None,None,
    None,None,None,None) */
  override def canEqual(that: Any):Boolean = that.isInstanceOf[Airlines]
  override def productArity: Int = 31
  override def productElement(n: Int) = n match {
    case  0 => Year
    case  1 => Month
    case  2 => DayofMonth<commit_msg>[PUBDEV-304] Product class used as T in RDD[T] has to implement Serializable.<commit_after>package org.apache.spark.examples.h2o

/** Prostate schema definition. */
case class Prostate(ID      :Option[Long]  ,
                    CAPSULE :Option[Int]  ,
                    AGE     :Option[Int]  ,
                    RACE    :Option[Int]  ,
                    DPROS   :Option[Int]  ,
                    DCAPS   :Option[Int]  ,
                    PSA     :Option[Float],
                    VOL     :Option[Float],
                    GLEASON :Option[Int]  ) {
}

/** A dummy csv parser for prostate dataset. */
object ProstateParse extends Serializable {
  def apply(row: Array[String]): Prostate = {
    import org.apache.spark.examples.h2o.SchemaUtils._
    Prostate(long(row(0)), int(row(1)), int(row(2)), int(row(3)), int(row(4)), int(row(5)), float(row(6)), float(row(7)), int(row(8)) )
  }
}

/** Airlines schema definition. */
class Airlines (val Year              :Option[Int],
                val Month             :Option[Int],
                val DayofMonth        :Option[Int],
                val DayOfWeek         :Option[Int],
                val DepTime           :Option[Int],
                val CRSDepTime        :Option[Int],
                val ArrTime           :Option[Int],
                val CRSArrTime        :Option[Int],
                val UniqueCarrier     :Option[String],
                val FlightNum         :Option[Int],
                val TailNum           :Option[Int],
                val ActualElapsedTime :Option[Int],
                val CRSElapsedTime    :Option[Int],
                val AirTime           :Option[Int],
                val ArrDelay          :Option[Int],
                val DepDelay          :Option[Int],
                val Origin            :Option[String],
                val Dest              :Option[String],
                val Distance          :Option[Int],
                val TaxiIn            :Option[Int],
                val TaxiOut           :Option[Int],
                val Cancelled         :Option[Int],
                val CancellationCode  :Option[Int],
                val Diverted          :Option[Int],
                val CarrierDelay      :Option[Int],
                val WeatherDelay      :Option[Int],
                val NASDelay          :Option[Int],
                val SecurityDelay     :Option[Int],
                val LateAircraftDelay :Option[Int],
                val IsArrDelayed      :Option[Boolean],
                val IsDepDelayed      :Option[Boolean]) extends Product with Serializable {

  /*
  def this() = this(None,None,None,None,None,None,None,None,None,
    None,None,None,None,None,None,None,None,None,
    None,None,None,None,None,None,None,None,None,
    None,None,None,None) */
  override def canEqual(that: Any):Boolean = that.isInstanceOf[Airlines]
  override def productArity: Int = 31
  override def productElement(n: Int) = n match {
    case  0 => Year
    case  1 => Month
    case  2 => DayofMonth
{'fraction_non_alphanumeric': 0.10572097691535631, 'fraction_numerical': 0.00585480093676815, 'mean_word_length': 1.4933277731442869, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5936647', 'n_tokens_mistral': 1704, 'n_tokens_neox': 1592, 'n_words': 376}