<commit_before>package net.sansa_stack.rdf.spark.io

import java.io.{ByteArrayInputStream, File, FileInputStream, FileOutputStream}
import java.net.{URI, URL}
import java.nio.file.{Files, Path}
import java.util.zip.ZipInputStream

import scala.collection.JavaConverters._

import com.holdenkarau.spark.testing.DataFrameSuiteBase

import org.apache.jena.rdf.model.{ModelFactory, ResourceFactory}
import org.apache.jena.riot.Lang
import org.apache.jena.shared.impl.PrefixMappingImpl
import org.apache.jena.sparql.serializer.SerializationContext
import org.apache.jena.sparql.util.FmtUtils
import org.apache.jena.vocabulary.RDF
import org.scalatest.FunSuite

/**
  * Tests for loading triples from either N-Triples are Turtle files into a DataFrame.
  *
  * @author <NAME>
  */
class RDFLoadingTests
  extends FunSuite
    with DataFrameSuiteBase {

  val logger = com.typesafe.scalalogging.Logger(classOf[RDFLoadingTests].getName)

  import net.sansa_stack.rdf.spark.io._

  test("loading N-Triples file into DataFrame with REGEX parsing mode should result in 10 triples") {

    val path = getClass.getResource("/loader/data.nt").getPath
    val lang: Lang = Lang.NTRIPLES

    val triples = spark.read.rdf(lang)(path)

    val cnt = triples.count()
    assert(cnt == 10)
  }

  test("loading Turtle file into DataFrame should result in 12 triples") {

    val path = getClass.getResource("/loader/data.ttl").getPath
    val lang: Lang = Lang.TURTLE

    val triples = spark.read.rdf(lang)(path)

    val cnt = triples.count()
    assert(cnt == 12)
  }
<commit_msg>round trip tests added<commit_after>package net.sansa_stack.rdf.spark.io

import java.io.{ByteArrayInputStream, File, FileInputStream, FileOutputStream}
import java.net.{URI, URL}
import java.nio.file.{Files, Path}
import java.util.zip.ZipInputStream

import scala.collection.JavaConverters._
import com.holdenkarau.spark.testing.DataFrameSuiteBase
import org.apache.jena.graph.GraphUtil
import org.apache.jena.rdf.model.{ModelFactory, ResourceFactory}
import org.apache.jena.riot.{Lang, RDFDataMgr}
import org.apache.jena.shared.impl.PrefixMappingImpl
import org.apache.jena.sparql.graph.GraphFactory
import org.apache.jena.sparql.serializer.SerializationContext
import org.apache.jena.sparql.util.FmtUtils
import org.apache.jena.vocabulary.RDF
import org.scalatest.FunSuite


import net.sansa_stack.rdf.spark.io._
import net.sansa_stack.rdf.spark.model._

/**
  * Tests for loading triples from either N-Triples are Turtle files into a DataFrame.
  *
  * @author <NAME>
  */
class RDFLoadingTests
  extends FunSuite
    with DataFrameSuiteBase {

  val logger = com.typesafe.scalalogging.Logger(classOf[RDFLoadingTests].getName)

  test("loading N-Triples file into DataFrame with REGEX parsing mode should result in 10 triples") {

    val path = getClass.getResource("/loader/data.nt").getPath
    val lang: Lang = Lang.NTRIPLES

    val triples = spark.read.rdf(lang)(path)

    val cnt = triples.count()
    assert(cnt == 10)
  }

  test("round trip: N-Triples -> DataFrame -> N-Triples") {
    val path = getClass.getResource("/loader/data.nt").getPath
    val lang: Lang = Lang.NTRIPLES

    val graph1 = GraphFactory.createGraphMem()
    RDFDataMgr.read(graph1, path)

    val triplesDF = spark.read.rdf(lang)(path)
    val triples = triplesDF.rdd.map(fromRow).collect()

    val graph2 = GraphFactory.createGraphMem()
    GraphUtil.add(graph2, triples)

    val isomorph = graph1.isIsomorphicWith(graph2)

    assert(isomorph)
  }

  test("round trip: N-Triples -> DataFrame -> Dataset -> N-Triples") {
    val path = getClass.getResource("/loader/data.nt").getPath
    val lang: Lang = Lang.NTRIPLES

    val graph1 = GraphFactory.createGraphMem()
    RDFDataMgr.read(graph1, path)
    graph1.find().asScala.foreach(println)

    val triplesDF = spark.read.rdf(lang)(path)
    val triplesDS = triplesDF.toDS()
    triplesDS.show()
    val triples = triplesDS.collect()
    triples.foreach(println)

    val graph2 = GraphFactory.createGraphMem()
    GraphUtil.add(graph2, triples)

    val isomorph = graph1.isIsomorphicWith(graph2)

    assert(isomorph)
  }

  test("loading Turtle file into DataFrame should result in 12 triples") {

    val path = getClass.getResource("/loader/data.ttl").getPath
    val lang: Lang = Lang.TURTLE

    val triples = spark.read.rdf(lang)(path)

    val cnt = triples.count()
    assert(cnt == 12)
  }

{'fraction_non_alphanumeric': 0.1106325096350034, 'fraction_numerical': 0.006574472908637497, 'mean_word_length': 4.036529680365297, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3113295', 'n_tokens_mistral': 1532, 'n_tokens_neox': 1473, 'n_words': 309}