<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.zouzias.spark

import org.apache.lucene.document.{DoubleDocValuesField, _}
import org.apache.lucene.facet.FacetField

import scala.reflect.ClassTag

package object lucenerdd {

  private val Stored = Field.Store.YES
  private val DefaultFieldName = "_1"

  private def addTextFacetField(doc: Document, fieldName: String, fieldValue: String): Unit = {
    if ( fieldValue.nonEmpty) { // Issues with empty strings on facets
      doc.add(new FacetField(s"${fieldName}${LuceneRDD.FacetTextFieldSuffix}",
        fieldValue))
    }
  }

  implicit def intToDocument(v: Int): Document = {
    val doc = new Document
    doc.add(new IntField(DefaultFieldName, v, Stored))
    addTextFacetField(doc, DefaultFieldName, v.toString)
    doc
  }

  implicit def longToDocument(v: Long): Document = {
    val doc = new Document
    doc.add(new LongField(DefaultFieldName, v, Stored))
    addTextFacetField(doc, DefaultFieldName, v.toString)
    doc
  }

  implicit def doubleToDocument(v: Double): Document = {
    val doc = new Document
    doc.add(new DoubleField(DefaultFieldName, v, Stored))
    addTextFacetField(doc, DefaultFieldName, v.toString)
    doc
  }

  implicit def floatToDocument(v: Float): Document = {
    val doc = new Document
    doc.add(new FloatField(DefaultFieldName, v, Stored))
    addTextFacetField(doc, DefaultFieldName, v.toString)
    doc
  }

  implicit def stringToDocument(s: String): Document = {
    val doc = new Document
    doc.add(new TextField(DefaultFieldName, s, Stored))
    addTextFacetField(doc, DefaultFieldName, s)
    doc
  }

  private def tupleTypeToDocument[T: ClassTag](doc: Document, index: Int, s: T): Document = {
    typeToDocument(doc, s"_${index}", s)
  }

  private def typeToDocument[T: ClassTag](doc: Document, fieldName: String, s: T): Document = {
    s match {
      case x: String =>
        doc.add(new TextField(fieldName, x, Stored))
        addTextFacetField(doc, fieldName, x)
      case x: Long =>
        doc.add(new LongField(fieldName, x, Stored))
        doc.add(new NumericDocValuesField(s"${fieldName}${LuceneRDD.FacetNumericFieldSuffix}",
          x))
      case x: Int =>
        doc.add(new IntField(fieldName, x, Stored))
        doc.add(new NumericDocValuesField(s"${fieldName}${LuceneRDD.FacetNumericFieldSuffix}",
          x.toLong))
      case x: Float =>
        doc.add(new FloatField(fieldName, x, Stored))
        doc.add(new FloatDocValuesField(s"${fieldName}${LuceneRDD.FacetNumericFieldSuffix}",
          x))
      case x: Double =>
        doc.add(new DoubleField(fieldName, x, Stored))
        doc.add(new DoubleDocValuesField(s"${fieldName}${LuceneRDD.FacetNumericFieldSuffix}",
          x))
    }
    doc
  }

  implicit def iterablePrimitiveToDocument[T: ClassTag](iter: Iterable[T]): Document = {
    val doc = new Document
    iter.foreach( item => tupleTypeToDocument(doc, 1, item))
    doc
  }

  implicit def mapToDocument[T: ClassTag](map: Map[String, T]): Document = {
    val doc = new Document
    map.keys.foreach{ case key =>
      typeToDocument(doc, key, map.get(key).get)
    }
    doc
  }

  implicit def tuple2ToDocument[T1: ClassTag, T2: ClassTag](s: (T1, T2)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    doc
  }

  implicit def tuple3ToDocument[T1: ClassTag,
  T2: ClassTag,
  T3: ClassTag](s: (T1, T2, T3)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    tupleTypeToDocument[T3](doc, 3, s._3)
    doc
  }

  implicit def tuple4ToDocument[T1: ClassTag,
  T2: ClassTag,
  T3: ClassTag,
  T4: ClassTag](s: (T1, T2, T3, T4)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    tupleTypeToDocument[T3](doc, 3, s._3)
    tupleTypeToDocument[T4](doc, 4, s._4)
    doc
  }

  implicit def tuple5ToDocument[T1: ClassTag,
  T2: ClassTag,
  T3: ClassTag,
  T4: ClassTag,
  T5: ClassTag](s: (T1, T2, T3, T4, T5)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    tupleTypeToDocument[T3](doc, 3, s._3)
    tupleTypeToDocument[T4](doc, 4, s._4)
    tupleTypeToDocument[T5](doc, 5, s._5)
    doc
  }

  implicit def tuple6ToDocument[T1: ClassTag,
  T2: ClassTag,
  T3: ClassTag,
  T4: ClassTag,
  T5: ClassTag,
  T6: ClassTag](s: (T1, T2, T3, T4, T5, T6)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    tupleTypeToDocument[T3](doc, 3, s._3)
    tupleTypeToDocument[T4](doc, 4, s._4)
    tupleTypeToDocument[T5](doc, 5, s._5)
    tupleTypeToDocument[T6](doc, 6, s._6)
    doc
  }

  implicit def tuple7ToDocument[T1: ClassTag,
  T2: ClassTag,
  T3: ClassTag,
  T4: ClassTag,
  T5: ClassTag,
  T6: ClassTag,
  T7: ClassTag](s: (T1, T2, T3, T4, T5, T6, T7)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    tupleTypeToDocument[T3](doc, 3, s._3)
    tupleTypeToDocument[T4](doc, 4, s._4)
    tupleTypeToDocument[T5](doc, 5, s._5)
    tupleTypeToDocument[T6](doc, 6, s._6)
    tupleTypeToDocument[T7](doc, 7, s._7)
    doc
  }

  implicit def tuple8ToDocument[T1: ClassTag,
  T2: ClassTag,
  T3: ClassTag,
  T4: ClassTag,
  T5: ClassTag,
  T6: ClassTag,
  T7: ClassTag,
  T8: ClassTag](s: (T1, T2, T3, T4, T5, T6, T7, T8)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    tupleTypeToDocument[T3](doc, 3, s._3)
    tupleTypeToDocument[T4](doc, 4, s._4)
    tupleTypeToDocument[T5](doc, 5, s._5)
    tupleTypeToDocument[T6](doc, 6, s._6)
    tupleTypeToDocument[T7](doc, 7, s._7)
    tupleTypeToDocument[T8](doc, 8, s._8)
    doc
  }


  implicit def tuple9ToDocument[T1: ClassTag,
  T2: ClassTag,
  T3: ClassTag,
  T4: ClassTag,
  T5: ClassTag,
  T6: ClassTag,
  T7: ClassTag,
  T8: ClassTag,
  T9: ClassTag](s: (T1, T2, T3, T4, T5, T6, T7, T8, T9)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    tupleTypeToDocument[T3](doc, 3, s._3)
    tupleTypeToDocument[T4](doc, 4, s._4)
    tupleTypeToDocument[T5](doc, 5, s._5)
    tupleTypeToDocument[T6](doc, 6, s._6)
    tupleTypeToDocument[T7](doc, 7, s._7)
    tupleTypeToDocument[T8](doc, 8, s._8)
    tupleTypeToDocument[T9](doc, 9, s._9)
    doc
  }

  implicit def tuple10ToDocument[T1: ClassTag,
  T2: ClassTag,
  T3: ClassTag,
  T4: ClassTag,
  T5: ClassTag,
  T6: ClassTag,
  T7: ClassTag,
  T8: ClassTag,
  T9: ClassTag,
  T10: ClassTag](s: (T1, T2, T3, T4, T5, T6, T7, T8, T9, T10)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    tupleTypeToDocument[T3](doc, 3, s._3)
    tupleTypeToDocument[T4](doc, 4, s._4)
    tupleTypeToDocument[T5](doc, 5, s._5)
    tupleTypeToDocument[T6](doc, 6, s._6)
    tupleTypeToDocument[T7](doc, 7, s._7)
    tupleTypeToDocument[T8](doc, 8, s._8)
    tupleTypeToDocument[T9](doc, 9, s._9)
    tupleTypeToDocument[T10](doc, 10, s._10)
    doc
  }

}
<commit_msg>typeToDocument public visibility<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.zouzias.spark

import org.apache.lucene.document.{DoubleDocValuesField, _}
import org.apache.lucene.facet.FacetField

import scala.reflect.ClassTag

package object lucenerdd {

  private val Stored = Field.Store.YES
  private val DefaultFieldName = "_1"

  private def addTextFacetField(doc: Document, fieldName: String, fieldValue: String): Unit = {
    if ( fieldValue.nonEmpty) { // Issues with empty strings on facets
      doc.add(new FacetField(s"${fieldName}${LuceneRDD.FacetTextFieldSuffix}",
        fieldValue))
    }
  }

  implicit def intToDocument(v: Int): Document = {
    val doc = new Document
    doc.add(new IntField(DefaultFieldName, v, Stored))
    addTextFacetField(doc, DefaultFieldName, v.toString)
    doc
  }

  implicit def longToDocument(v: Long): Document = {
    val doc = new Document
    doc.add(new LongField(DefaultFieldName, v, Stored))
    addTextFacetField(doc, DefaultFieldName, v.toString)
    doc
  }

  implicit def doubleToDocument(v: Double): Document = {
    val doc = new Document
    doc.add(new DoubleField(DefaultFieldName, v, Stored))
    addTextFacetField(doc, DefaultFieldName, v.toString)
    doc
  }

  implicit def floatToDocument(v: Float): Document = {
    val doc = new Document
    doc.add(new FloatField(DefaultFieldName, v, Stored))
    addTextFacetField(doc, DefaultFieldName, v.toString)
    doc
  }

  implicit def stringToDocument(s: String): Document = {
    val doc = new Document
    doc.add(new TextField(DefaultFieldName, s, Stored))
    addTextFacetField(doc, DefaultFieldName, s)
    doc
  }

  private def tupleTypeToDocument[T: ClassTag](doc: Document, index: Int, s: T): Document = {
    typeToDocument(doc, s"_${index}", s)
  }

  def typeToDocument[T: ClassTag](doc: Document, fieldName: String, s: T): Document = {
    s match {
      case x: String =>
        doc.add(new TextField(fieldName, x, Stored))
        addTextFacetField(doc, fieldName, x)
      case x: Long =>
        doc.add(new LongField(fieldName, x, Stored))
        doc.add(new NumericDocValuesField(s"${fieldName}${LuceneRDD.FacetNumericFieldSuffix}",
          x))
      case x: Int =>
        doc.add(new IntField(fieldName, x, Stored))
        doc.add(new NumericDocValuesField(s"${fieldName}${LuceneRDD.FacetNumericFieldSuffix}",
          x.toLong))
      case x: Float =>
        doc.add(new FloatField(fieldName, x, Stored))
        doc.add(new FloatDocValuesField(s"${fieldName}${LuceneRDD.FacetNumericFieldSuffix}",
          x))
      case x: Double =>
        doc.add(new DoubleField(fieldName, x, Stored))
        doc.add(new DoubleDocValuesField(s"${fieldName}${LuceneRDD.FacetNumericFieldSuffix}",
          x))
    }
    doc
  }

  implicit def iterablePrimitiveToDocument[T: ClassTag](iter: Iterable[T]): Document = {
    val doc = new Document
    iter.foreach( item => tupleTypeToDocument(doc, 1, item))
    doc
  }

  implicit def mapToDocument[T: ClassTag](map: Map[String, T]): Document = {
    val doc = new Document
    map.keys.foreach{ case key =>
      typeToDocument(doc, key, map.get(key).get)
    }
    doc
  }

  implicit def tuple2ToDocument[T1: ClassTag, T2: ClassTag](s: (T1, T2)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    doc
  }

  implicit def tuple3ToDocument[T1: ClassTag,
  T2: ClassTag,
  T3: ClassTag](s: (T1, T2, T3)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    tupleTypeToDocument[T3](doc, 3, s._3)
    doc
  }

  implicit def tuple4ToDocument[T1: ClassTag,
  T2: ClassTag,
  T3: ClassTag,
  T4: ClassTag](s: (T1, T2, T3, T4)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    tupleTypeToDocument[T3](doc, 3, s._3)
    tupleTypeToDocument[T4](doc, 4, s._4)
    doc
  }

  implicit def tuple5ToDocument[T1: ClassTag,
  T2: ClassTag,
  T3: ClassTag,
  T4: ClassTag,
  T5: ClassTag](s: (T1, T2, T3, T4, T5)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    tupleTypeToDocument[T3](doc, 3, s._3)
    tupleTypeToDocument[T4](doc, 4, s._4)
    tupleTypeToDocument[T5](doc, 5, s._5)
    doc
  }

  implicit def tuple6ToDocument[T1: ClassTag,
  T2: ClassTag,
  T3: ClassTag,
  T4: ClassTag,
  T5: ClassTag,
  T6: ClassTag](s: (T1, T2, T3, T4, T5, T6)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    tupleTypeToDocument[T3](doc, 3, s._3)
    tupleTypeToDocument[T4](doc, 4, s._4)
    tupleTypeToDocument[T5](doc, 5, s._5)
    tupleTypeToDocument[T6](doc, 6, s._6)
    doc
  }

  implicit def tuple7ToDocument[T1: ClassTag,
  T2: ClassTag,
  T3: ClassTag,
  T4: ClassTag,
  T5: ClassTag,
  T6: ClassTag,
  T7: ClassTag](s: (T1, T2, T3, T4, T5, T6, T7)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    tupleTypeToDocument[T3](doc, 3, s._3)
    tupleTypeToDocument[T4](doc, 4, s._4)
    tupleTypeToDocument[T5](doc, 5, s._5)
    tupleTypeToDocument[T6](doc, 6, s._6)
    tupleTypeToDocument[T7](doc, 7, s._7)
    doc
  }

  implicit def tuple8ToDocument[T1: ClassTag,
  T2: ClassTag,
  T3: ClassTag,
  T4: ClassTag,
  T5: ClassTag,
  T6: ClassTag,
  T7: ClassTag,
  T8: ClassTag](s: (T1, T2, T3, T4, T5, T6, T7, T8)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    tupleTypeToDocument[T3](doc, 3, s._3)
    tupleTypeToDocument[T4](doc, 4, s._4)
    tupleTypeToDocument[T5](doc, 5, s._5)
    tupleTypeToDocument[T6](doc, 6, s._6)
    tupleTypeToDocument[T7](doc, 7, s._7)
    tupleTypeToDocument[T8](doc, 8, s._8)
    doc
  }


  implicit def tuple9ToDocument[T1: ClassTag,
  T2: ClassTag,
  T3: ClassTag,
  T4: ClassTag,
  T5: ClassTag,
  T6: ClassTag,
  T7: ClassTag,
  T8: ClassTag,
  T9: ClassTag](s: (T1, T2, T3, T4, T5, T6, T7, T8, T9)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    tupleTypeToDocument[T3](doc, 3, s._3)
    tupleTypeToDocument[T4](doc, 4, s._4)
    tupleTypeToDocument[T5](doc, 5, s._5)
    tupleTypeToDocument[T6](doc, 6, s._6)
    tupleTypeToDocument[T7](doc, 7, s._7)
    tupleTypeToDocument[T8](doc, 8, s._8)
    tupleTypeToDocument[T9](doc, 9, s._9)
    doc
  }

  implicit def tuple10ToDocument[T1: ClassTag,
  T2: ClassTag,
  T3: ClassTag,
  T4: ClassTag,
  T5: ClassTag,
  T6: ClassTag,
  T7: ClassTag,
  T8: ClassTag,
  T9: ClassTag,
  T10: ClassTag](s: (T1, T2, T3, T4, T5, T6, T7, T8, T9, T10)): Document = {
    val doc = new Document
    tupleTypeToDocument[T1](doc, 1, s._1)
    tupleTypeToDocument[T2](doc, 2, s._2)
    tupleTypeToDocument[T3](doc, 3, s._3)
    tupleTypeToDocument[T4](doc, 4, s._4)
    tupleTypeToDocument[T5](doc, 5, s._5)
    tupleTypeToDocument[T6](doc, 6, s._6)
    tupleTypeToDocument[T7](doc, 7, s._7)
    tupleTypeToDocument[T8](doc, 8, s._8)
    tupleTypeToDocument[T9](doc, 9, s._9)
    tupleTypeToDocument[T10](doc, 10, s._10)
    doc
  }

}

{'fraction_non_alphanumeric': 0.128203544741555, 'fraction_numerical': 0.035941456184771195, 'mean_word_length': 3.5719932241671373, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7246519', 'n_tokens_mistral': 6166, 'n_tokens_neox': 6015, 'n_words': 1487}