<commit_before>package net.sansa_stack.examples.spark.ml.outliers.anomalydetection

import scala.collection.mutable
import org.apache.spark.sql.SparkSession
import java.net.{ URI => JavaURI }
import net.sansa_stack.ml.spark.outliers.anomalydetection.{AnomalyDetection=>AlgAnomalyDetection, IQR}
import net.sansa_stack.rdf.spark.io.NTripleReader

object AnomalyDetection {
  def main(args: Array[String]) {
    parser.parse(args, Config()) match {
      case Some(config) =>
        run(config.in, config.out)
      case None =>
        println(parser.usage)
    }
  }

  def run(input: String, outputPath: String): Unit = {

    println("==================================================")
    println("|        Distributed Anomaly Detection           |")
    println("==================================================")

    val spark = SparkSession.builder
      .appName(s"Anomaly Detection example ( $input )")
      .master("local[*]")
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .getOrCreate()

     //N-Triples Reader
    val triplesRDD = NTripleReader.load(spark, JavaURI.create(input))

    //constant parameters defined
    val JSimThreshold = 0.6

    val objList = List("http://www.w3.org/1999/02/22-rdf-syntax-ns#langString",
      "http://www.w3.org/2001/XMLSchema#date")

    //clustering of subjects are on the basis of rdf:type specially object with wikidata and dbpedia.org
    //val triplesType = List("http://www.wikidata.org", "http://dbpedia.org/ontology")
    val triplesType = List("http://dbpedia.org/ontology")
   
    //some of the supertype which are present for most of the subject
    val listSuperType=List(
                           "http://dbpedia.org/ontology/Activity","http://dbpedia.org/ontology/Organisation",
                           "http://dbpedia.org/ontology/Agent","http://dbpedia.org/ontology/SportsLeague",
                           "http://dbpedia.org/ontology/Person","http://dbpedia.org/ontology/Athlete",
                           "http://dbpedia.org/ontology/Event","http://dbpedia.org/ontology/Place",
                           "http://dbpedia.org/ontology/PopulatedPlace","http://dbpedia.org/ontology/Region",
                           "http://dbpedia.org/ontology/Species","http://dbpedia.org/ontology/Eukaryote",
                           "http://dbpedia.org/ontology/Location"
                          )
    //hypernym URI                      
    val hypernym="http://purl.org/linguistics/gold/hypernym"
    
    val outDetection = new AlgAnomalyDetection(triplesRDD, objList, triplesType, JSimThreshold,listSuperType,spark,hypernym)

    val clusterOfSubject = outDetection.run()

    clusterOfSubject.foreach(println)
    
    val setData=clusterOfSubject.map(f=>f._2)
    
    val listofData = clusterOfSubject.map({
      case (a, (b)) => b.map(f => (f._3.toString().toDouble)).toList
    })

    val listofDataArray = listofData.collect()

    for (listofData <- listofDataArray)
      IQR.iqr(listofData,setData)

  }
 

  case class Config(
    in:  String = "",
    out: String = "")

  val parser = new scopt.OptionParser[Config]("Anomaly Detection example") {

    head("Anomaly Detection example")

    opt[String]('i', "input").required().valueName("<path>").
      action((x, c) => c.copy(in = x)).
      text("path to file that contains the data")


    help("help").text("prints this usage text")
  }
}<commit_msg>Update ml layer examples :: Align with SANSA-RDF<commit_after>package net.sansa_stack.examples.spark.ml.outliers.anomalydetection

import scala.collection.mutable
import org.apache.spark.sql.SparkSession
import java.net.{ URI => JavaURI }
import net.sansa_stack.ml.spark.outliers.anomalydetection.{ AnomalyDetection => AlgAnomalyDetection, IQR }
import org.apache.jena.riot.Lang
import net.sansa_stack.rdf.spark.io.rdf._

object AnomalyDetection {
  def main(args: Array[String]) {
    parser.parse(args, Config()) match {
      case Some(config) =>
        run(config.in, config.out)
      case None =>
        println(parser.usage)
    }
  }

  def run(input: String, outputPath: String): Unit = {

    println("==================================================")
    println("|        Distributed Anomaly Detection           |")
    println("==================================================")

    val spark = SparkSession.builder
      .appName(s"Anomaly Detection example ( $input )")
      .master("local[*]")
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .getOrCreate()

    //N-Triples Reader
    val lang = Lang.NTRIPLES
    val triplesRDD = spark.rdf(lang)(input)
    //constant parameters defined
    val JSimThreshold = 0.6

    val objList = List(
      "http://www.w3.org/1999/02/22-rdf-syntax-ns#langString",
      "http://www.w3.org/2001/XMLSchema#date")

    //clustering of subjects are on the basis of rdf:type specially object with wikidata and dbpedia.org
    //val triplesType = List("http://www.wikidata.org", "http://dbpedia.org/ontology")
    val triplesType = List("http://dbpedia.org/ontology")

    //some of the supertype which are present for most of the subject
    val listSuperType = List(
      "http://dbpedia.org/ontology/Activity", "http://dbpedia.org/ontology/Organisation",
      "http://dbpedia.org/ontology/Agent", "http://dbpedia.org/ontology/SportsLeague",
      "http://dbpedia.org/ontology/Person", "http://dbpedia.org/ontology/Athlete",
      "http://dbpedia.org/ontology/Event", "http://dbpedia.org/ontology/Place",
      "http://dbpedia.org/ontology/PopulatedPlace", "http://dbpedia.org/ontology/Region",
      "http://dbpedia.org/ontology/Species", "http://dbpedia.org/ontology/Eukaryote",
      "http://dbpedia.org/ontology/Location")
    //hypernym URI
    val hypernym = "http://purl.org/linguistics/gold/hypernym"

    val outDetection = new AlgAnomalyDetection(triplesRDD, objList, triplesType, JSimThreshold, listSuperType, spark, hypernym)

    val clusterOfSubject = outDetection.run()

    clusterOfSubject.foreach(println)

    val setData = clusterOfSubject.map(f => f._2)

    val listofData = clusterOfSubject.map({
      case (a, (b)) => b.map(f => (f._3.toString().toDouble)).toList
    })

    val listofDataArray = listofData.collect()

    for (listofData <- listofDataArray)
      IQR.iqr(listofData, setData)

  }

  case class Config(
    in:  String = "",
    out: String = "")

  val parser = new scopt.OptionParser[Config]("Anomaly Detection example") {

    head("Anomaly Detection example")

    opt[String]('i', "input").required().valueName("<path>").
      action((x, c) => c.copy(in = x)).
      text("path to file that contains the data")

    help("help").text("prints this usage text")
  }
}
{'fraction_non_alphanumeric': 0.16873889875666073, 'fraction_numerical': 0.0053285968028419185, 'mean_word_length': 3.0901937046004844, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '265854', 'n_tokens_mistral': 2261, 'n_tokens_neox': 2111, 'n_words': 381}