<commit_before>package org.elinker.core.api.process

import java.io.ByteArrayInputStream
import java.net.URL
import java.util

import akka.actor.SupervisorStrategy.Restart
import akka.actor.{Actor, OneForOneStrategy}
import akka.event.Logging
import com.hp.hpl.jena.query.QueryExecutionFactory
import com.hp.hpl.jena.rdf.model.{Resource, Literal, Model, ModelFactory}
import com.hp.hpl.jena.shared.{JenaException, SyntaxError}
import eu.freme.common.persistence.dao.DatasetSimpleDAO
import eu.freme.common.persistence.model.DatasetSimple
import org.apache.solr.client.solrj.SolrQuery
import org.apache.solr.client.solrj.impl.HttpSolrClient
import org.apache.solr.common.SolrInputDocument
import org.elinker.core.api.process.Rest.{StatusOK, StatusCreated, RestMessage}

import scala.collection.JavaConversions._

/**
 * Dataset management actor
 *
 * @param solrUri SOLR instance URI where entity URIs and labels are indexed
 * @param datasetDAO Hibernate DatasetSimpleDAO instance for manipulating dataset table
 * @author <NAME> <<EMAIL>>
 */
class Datasets(solrUri: String, datasetDAO: DatasetSimpleDAO) extends Actor {

  import Datasets._
  import JsonImplicits._
  import context._

  val log = Logging(system, getClass)

  val solr = new HttpSolrClient(solrUri)

  val defaultIndexProps = Seq("http://www.w3.org/2004/02/skos/core#prefLabel",
    "http://www.w3.org/2004/02/skos/core#altLabel",
    "http://www.w3.org/2000/01/rdf-schema#label")

  def indexData(dataset: String, format: String, body: InputType, defaultLang: String, properties: Seq[String]): Long = {
    def addLabel(resource: Resource, label: Literal) = {
      val uri = resource.getURI
      val document = new SolrInputDocument()
      val language = label.getLanguage
      val partialUpdate = new util.HashMap[String, String]()
      partialUpdate.put("add", label.getString)
      document.addField("label", partialUpdate)
      document.addField("dataset", dataset)
      document.addField("resource", uri)
      document.addField("language", if (language == "") defaultLang else language)
      document.addField("count", 1)
      solr.add("elinker", document)
    }

    def iterateEntityValues(model: Model, property: String): Traversable[(Resource, Literal)] = {
      val iter = model.listStatements(null, model.getProperty(property), null)
      new Traversable[(Resource, Literal)] {
        override def foreach[U](f: ((Resource, Literal)) => U): Unit = {
          for (statement <- iter) {
            f((statement.getSubject.asResource(), statement.getObject.asLiteral()))
          }
        }
      }
    }

    def indexModel(model: Model) = {
      for (property <- properties) {
        for ((resource, label) <- iterateEntityValues(model, property)) {
          addLabel(resource, label)
        }
      }
    }

    body match {
      case TextInput(text) =>
        // RDF data sent in POST body
        val model = ModelFactory.createDefaultModel()
        model.read(new ByteArrayInputStream(text.getBytes), null, format)
        indexModel(model)
      case UrlInput(url) =>
        // Fetch RDF data from given URL
        val model = ModelFactory.createDefaultModel()
        model.read(new URL(url).openStream(), null, format)
        indexModel(model)
      case SparqlInput(query, endpoint) =>
        // Fetch RDF from SPARQL endpoint
        val qe = QueryExecutionFactory.sparqlService(endpoint, query)
        val results = qe.execSelect()
        for(result <- results) {
          val resource = result.get("res").asResource()
          val label = result.get("label").asLiteral()
          addLabel(resource, label)
        }
    }

    solr.commit("elinker", true, true)
    val sizeQuery = new SolrQuery()
    sizeQuery.set("q", s"dataset:$dataset")
    sizeQuery.set("rows", "0")
    val result = solr.query("elinker", sizeQuery)
    result.getResults.getNumFound
  }


  def createDataset(dataset: CreateDataset): (Long, Long) = {
    val d = toDatasetSimple(dataset)
    
    val timeStamp = d.getCreationTime

    println(d.toString)
    datasetDAO.save(d)
    println("Saved")

    (numEntities, timeStamp)
  }
  
  
  def toDatasetSimple (dataset: CreateDataset) :DatasetSimple = {
<commit_msg>Update Datasets.scala<commit_after>package org.elinker.core.api.process

import java.io.ByteArrayInputStream
import java.net.URL
import java.util

import akka.actor.SupervisorStrategy.Restart
import akka.actor.{Actor, OneForOneStrategy}
import akka.event.Logging
import com.hp.hpl.jena.query.QueryExecutionFactory
import com.hp.hpl.jena.rdf.model.{Resource, Literal, Model, ModelFactory}
import com.hp.hpl.jena.shared.{JenaException, SyntaxError}
import eu.freme.common.persistence.dao.DatasetSimpleDAO
import eu.freme.common.persistence.model.DatasetSimple
import org.apache.solr.client.solrj.SolrQuery
import org.apache.solr.client.solrj.impl.HttpSolrClient
import org.apache.solr.common.SolrInputDocument
import org.elinker.core.api.process.Rest.{StatusOK, StatusCreated, RestMessage}

import scala.collection.JavaConversions._

/**
 * Dataset management actor
 *
 * @param solrUri SOLR instance URI where entity URIs and labels are indexed
 * @param datasetDAO Hibernate DatasetSimpleDAO instance for manipulating dataset table
 * @author <NAME> <<EMAIL>>
 */
class Datasets(solrUri: String, datasetDAO: DatasetSimpleDAO) extends Actor {

  import Datasets._
  import JsonImplicits._
  import context._

  val log = Logging(system, getClass)

  val solr = new HttpSolrClient(solrUri)

  val defaultIndexProps = Seq("http://www.w3.org/2004/02/skos/core#prefLabel",
    "http://www.w3.org/2004/02/skos/core#altLabel",
    "http://www.w3.org/2000/01/rdf-schema#label")

  def indexData(dataset: String, format: String, body: InputType, defaultLang: String, properties: Seq[String]): Long = {
    def addLabel(resource: Resource, label: Literal) = {
      val uri = resource.getURI
      val document = new SolrInputDocument()
      val language = label.getLanguage
      val partialUpdate = new util.HashMap[String, String]()
      partialUpdate.put("add", label.getString)
      document.addField("label", partialUpdate)
      document.addField("dataset", dataset)
      document.addField("resource", uri)
      document.addField("language", if (language == "") defaultLang else language)
      document.addField("count", 1)
      solr.add("elinker", document)
    }

    def iterateEntityValues(model: Model, property: String): Traversable[(Resource, Literal)] = {
      val iter = model.listStatements(null, model.getProperty(property), null)
      new Traversable[(Resource, Literal)] {
        override def foreach[U](f: ((Resource, Literal)) => U): Unit = {
          for (statement <- iter) {
            f((statement.getSubject.asResource(), statement.getObject.asLiteral()))
          }
        }
      }
    }

    def indexModel(model: Model) = {
      for (property <- properties) {
        for ((resource, label) <- iterateEntityValues(model, property)) {
          addLabel(resource, label)
        }
      }
    }

    body match {
      case TextInput(text) =>
        // RDF data sent in POST body
        val model = ModelFactory.createDefaultModel()
        model.read(new ByteArrayInputStream(text.getBytes), null, format)
        indexModel(model)
      case UrlInput(url) =>
        // Fetch RDF data from given URL
        val model = ModelFactory.createDefaultModel()
        model.read(new URL(url).openStream(), null, format)
        indexModel(model)
      case SparqlInput(query, endpoint) =>
        // Fetch RDF from SPARQL endpoint
        val qe = QueryExecutionFactory.sparqlService(endpoint, query)
        val results = qe.execSelect()
        for(result <- results) {
          val resource = result.get("res").asResource()
          val label = result.get("label").asLiteral()
          addLabel(resource, label)
        }
    }

    solr.commit("elinker", true, true)
    val sizeQuery = new SolrQuery()
    sizeQuery.set("q", s"dataset:$dataset")
    sizeQuery.set("rows", "0")
    val result = solr.query("elinker", sizeQuery)
    result.getResults.getNumFound
  }


  def createDataset(dataset: CreateDataset): (Long, Long) = {
    val d = toDatasetSimple(dataset)
    
    val timeStamp = d.getCreationTime

    println(d.toString)
    datasetDAO.save(d)
    println("Saved")

    (d.getTotalEntities, timeStamp)
  }
  
  
  def toDatasetSimple (dataset: CreateDataset) :DatasetSimple = {

{'fraction_non_alphanumeric': 0.11812048763167239, 'fraction_numerical': 0.005444431293644218, 'mean_word_length': 3.391891891891892, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '371543', 'n_tokens_mistral': 2637, 'n_tokens_neox': 2562, 'n_words': 561}