<commit_before>package demy.util


object util {
    def checkpoint[T : org.apache.spark.sql.Encoder : scala.reflect.runtime.universe.TypeTag] (ds:org.apache.spark.sql.Dataset[T], path:String)
               :org.apache.spark.sql.Dataset[T] = 
    {
        ds.write.mode("overwrite").parquet(path)
        return ds.sparkSession.read.parquet(path).as[T]
    }

    def checkpoint(df:org.apache.spark.sql.DataFrame, path:String) = 
    {
        df.write.mode("overwrite").parquet(path)
        df.sparkSession.read.parquet(path)
    }
/*    def schemaOf[T: scala.reflect.runtime.universe.TypeTag]: StructType = { 
        org.apache.spark.sql.catalyst.ScalaReflection
            .schemaFor[T] // this method requires a TypeTag for T
            .dataType
            .asInstanceOf[StructType] // cast it to a StructType, what spark requires as its Schema
    }*/
}
case class MergedIterator[T, U](a:Iterator[T], b:Iterator[U], defA:T, defB:U) extends Iterator[(T, U)] {
      def hasNext = a.hasNext || b.hasNext
      def next = (if(a.hasNext) a.next else defA, if(b.hasNext) b.next else defB)
}<commit_msg>Undo unwanted deletion of util functions EnumerationFromIterator and iteratorToJava<commit_after>package demy.util

case class MergedIterator[T, U](a:Iterator[T], b:Iterator[U], defA:T, defB:U) extends Iterator[(T, U)] {
  def hasNext = a.hasNext || b.hasNext
  def next = (if(a.hasNext) a.next else defA, if(b.hasNext) b.next else defB)
}


case class EnumerationFromIterator[T](it:Iterator[T]) extends java.util.Enumeration[T] {
  def hasMoreElements() = it.hasNext
  def nextElement() = it.next()
}

object implicits {
  implicit class IteratorToJava[T](it: Iterator[T]) {
    def toJavaEnumeration = EnumerationFromIterator(it)
  }
}

object util {
    def checkpoint[T : org.apache.spark.sql.Encoder : scala.reflect.runtime.universe.TypeTag] (ds:org.apache.spark.sql.Dataset[T], path:String)
               :org.apache.spark.sql.Dataset[T] = 
    {
        ds.write.mode("overwrite").parquet(path)
        return ds.sparkSession.read.parquet(path).as[T]
    }

    def checkpoint(df:org.apache.spark.sql.DataFrame, path:String) = 
    {
        df.write.mode("overwrite").parquet(path)
        df.sparkSession.read.parquet(path)
    }
/*    def schemaOf[T: scala.reflect.runtime.universe.TypeTag]: StructType = { 
        org.apache.spark.sql.catalyst.ScalaReflection
            .schemaFor[T] // this method requires a TypeTag for T
            .dataType
            .asInstanceOf[StructType] // cast it to a StructType, what spark requires as its Schema
    }*/
}
{'fraction_non_alphanumeric': 0.13372093023255813, 'fraction_numerical': 0.0, 'mean_word_length': 3.142857142857143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3380084', 'n_tokens_mistral': 890, 'n_tokens_neox': 859, 'n_words': 166}