<commit_before>package net.sansa_stack.ml.spark.mining.amieSpark

import java.io.File
import java.net.URI

import net.sansa_stack.ml.spark.mining.amieSpark.KBObject.KB
import net.sansa_stack.ml.spark.mining.amieSpark.Rules.RuleContainer
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{ DataFrame, SQLContext, SparkSession, _ }

import scala.collection.mutable.{ ArrayBuffer, Map }
import scala.util.Try

import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.fs.Path

import net.sansa_stack.rdf.spark.model.JenaSparkRDDOps
import net.sansa_stack.rdf.spark.model.TripleRDD._
import net.sansa_stack.ml.spark.mining.amieSpark.DfLoader.Atom

object MineRules {
  /**
   * 	Algorithm that mines the Rules.
   *
   * @param kb object knowledge base that was created in main
   * @param minHC threshold on head coverage
   * @param maxLen maximal rule length
   * @param threshold on confidence
   * @param sc spark context
   *
   *
   */
  class Algorithm(k: KB, mHC: Double, mL: Int, mCon: Double, hdfsP: String) extends Serializable {

    val kb: KB = k
    val minHC = mHC
    val maxLen = mL
    val minConf = mCon
    val hdfsPath = hdfsP

    def deleteRecursive(path: File): Int = {

      var files = path.listFiles()
      if (files != null) {
        for (f <- files) {
          if (f.isDirectory()) {

            deleteRecursive(f)
            f.delete()
          } else {

            f.delete()

          }
        }

        path.delete()

      }

      return 0

    }

    def ruleMining(sc: SparkContext, sqlContext: SQLContext): ArrayBuffer[RuleContainer] = {

      val fs: FileSystem = FileSystem.get(new URI("hdfs://akswnc5.informatik.uni-leipzig.de:54310/Theresa/"), sc.hadoopConfiguration);
      fs.delete(new Path("/Theresa/permanent0/"), true)
      fs.delete(new Path("/Theresa/permanent1/"), true)

      var predicates = kb.getKbGraph().triples.map { x => x.predicate

      }.distinct
      var z = predicates.collect()

      /**
       * q is a queue with one atom rules
       * is initialized with every distinct relation as head, body is empty
       */
      var q = ArrayBuffer[RuleContainer]()

      for (zz <- z) {
        if (zz != null) {
          var rule = ArrayBuffer(RDFTriple("?a", zz, "?b"))

          var rc = new RuleContainer
          rc.initRule(rule, kb, sc, sqlContext)

          q += rc
        }

      }

      var outMap: Map[String, ArrayBuffer[(ArrayBuffer[RDFTriple], RuleContainer)]] = Map()

      var dataFrameRuleParts: RDD[(RDFTriple, Int, Int)] = null
      var out: ArrayBuffer[RuleContainer] = new ArrayBuffer
      var dublicate: ArrayBuffer[String] = ArrayBuffer("")

      for (i <- 0 to this.maxLen - 1) {
<commit_msg>Remove unused code<commit_after>package net.sansa_stack.ml.spark.mining.amieSpark

import java.io.File
import java.net.URI

import net.sansa_stack.ml.spark.mining.amieSpark.KBObject.KB
import net.sansa_stack.ml.spark.mining.amieSpark.Rules.RuleContainer
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{ DataFrame, SQLContext, SparkSession, _ }

import scala.collection.mutable.{ ArrayBuffer, Map }
import scala.util.Try

import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.fs.Path

import net.sansa_stack.rdf.spark.model.JenaSparkRDDOps
import net.sansa_stack.rdf.spark.model.TripleRDD._
import net.sansa_stack.ml.spark.mining.amieSpark.DfLoader.Atom

object MineRules {
  /**
   * 	Algorithm that mines the Rules.
   *
   * @param kb object knowledge base that was created in main
   * @param minHC threshold on head coverage
   * @param maxLen maximal rule length
   * @param threshold on confidence
   * @param sc spark context
   *
   *
   */
  class Algorithm(k: KB, mHC: Double, mL: Int, mCon: Double, hdfsP: String) extends Serializable {

    val kb: KB = k
    val minHC = mHC
    val maxLen = mL
    val minConf = mCon
    val hdfsPath = hdfsP

    def deleteRecursive(path: File): Int = {

      var files = path.listFiles()
      if (files != null) {
        for (f <- files) {
          if (f.isDirectory()) {

            deleteRecursive(f)
            f.delete()
          } else {

            f.delete()

          }
        }

        path.delete()

      }

      return 0

    }

    def ruleMining(sc: SparkContext, sqlContext: SQLContext): ArrayBuffer[RuleContainer] = {

      var predicates = kb.getKbGraph().triples.map { x => x.predicate

      }.distinct
      var z = predicates.collect()

      /**
       * q is a queue with one atom rules
       * is initialized with every distinct relation as head, body is empty
       */
      var q = ArrayBuffer[RuleContainer]()

      for (zz <- z) {
        if (zz != null) {
          var rule = ArrayBuffer(RDFTriple("?a", zz, "?b"))

          var rc = new RuleContainer
          rc.initRule(rule, kb, sc, sqlContext)

          q += rc
        }

      }

      var outMap: Map[String, ArrayBuffer[(ArrayBuffer[RDFTriple], RuleContainer)]] = Map()

      var dataFrameRuleParts: RDD[(RDFTriple, Int, Int)] = null
      var out: ArrayBuffer[RuleContainer] = new ArrayBuffer
      var dublicate: ArrayBuffer[String] = ArrayBuffer("")

      for (i <- 0 to this.maxLen - 1) {

{'fraction_non_alphanumeric': 0.11163850389215872, 'fraction_numerical': 0.0026580596164799694, 'mean_word_length': 2.561866125760649, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3114261', 'n_tokens_mistral': 1818, 'n_tokens_neox': 1703, 'n_words': 428}