<commit_before>package net.sansa_stack.ml.spark.similarity.run

object Semantic_Similarity_Estimator {

  def main(args: Array[String]): Unit = {
    print("Hello World! I'm yor growing semantic similarity estimator")
  }
}<commit_msg>copied rdf reader to see if the more complex techstack works, and it does after changing project to jdk 8<commit_after>package net.sansa_stack.ml.spark.similarity.run

import net.sansa_stack.rdf.spark.io._
import org.apache.jena.riot.Lang
import org.apache.spark.sql.SparkSession

object Semantic_Similarity_Estimator {

  def main(args: Array[String]): Unit = {
    parser.parse(args, Config()) match {
      case Some(config) =>
        run(config.in)
      case None =>
        println(parser.usage)
    }
  }

  def run(input: String): Unit = {

    val spark = SparkSession.builder
      .appName(s"Triple reader example  $input")
      .master("local[*]")
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .getOrCreate()

    println("======================================")
    println("|        Triple reader example       |")
    println("======================================")

    val lang = Lang.NTRIPLES
    val triples = spark.rdf(lang)(input)

    println("now we are here")

    triples.take(5).foreach(println(_))

    println("and yet we are here")

    // triples.saveAsNTriplesFile(output)

    spark.stop

  }

  case class Config(in: String = "")

  val parser = new scopt.OptionParser[Config]("Triple reader example") {

    head(" Triple reader example")

    opt[String]('i', "input").required().valueName("<path>").
      action((x, c) => c.copy(in = x)).
      text("path to file that contains the data (in N-Triples format)")

    help("help").text("prints this usage text")
  }
}
{'fraction_non_alphanumeric': 0.17115600448933782, 'fraction_numerical': 0.001122334455667789, 'mean_word_length': 3.5139240506329115, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5716440', 'n_tokens_mistral': 590, 'n_tokens_neox': 538, 'n_words': 131}