<commit_before>package de.fuberlin.wiwiss.silk.util.sparql

import de.fuberlin.wiwiss.silk.util.Uri
import java.util.logging.Logger
import de.fuberlin.wiwiss.silk.entity.{SparqlRestriction, ForwardOperator, Path}

/**
 * Retrieves the most frequent paths of a number of random sample entities.
 *
 * It is typically faster than SparqlAggregatePathsCollector but also less precise.
 *
 * The limitations of the current implementation are:
 * - It does only return forward paths of length 1
 * - It returns a maximum of 100 paths
 */
object SparqlSamplePathsCollector extends SparqlPathsCollector {
  /**Number of sample entities */
  private val maxEntities = 100

  /**The minimum frequency of a property to be considered relevant */
  private val MinFrequency = 0.7

  private implicit val logger = Logger.getLogger(SparqlSamplePathsCollector.getClass.getName)

  def apply(endpoint: SparqlEndpoint, restrictions: SparqlRestriction, limit: Option[Int]): Traversable[(Path, Double)] = {
    val variable = restrictions.toSparql.dropWhile(_ != '?').drop(1).takeWhile(_ != ' ')

    val sampleEntities = {
      if (variable.isEmpty)
        getAllEntities(endpoint)
      else
        getEntities(endpoint, restrictions, variable)
    }

    getEntitiesPaths(endpoint, sampleEntities, variable, limit.getOrElse(100))
  }

  private def getAllEntities(endpoint: SparqlEndpoint): Traversable[String] = {
    val sparql = "SELECT ?s WHERE { ?s ?p ?o } LIMIT " + maxEntities

    val results = endpoint.query(sparql, maxEntities)

    results.map(_("s").value)
  }

  private def getEntities(endpoint: SparqlEndpoint, restrictions: SparqlRestriction, variable: String): Traversable[String] = {
    val sparql = "SELECT ?" + variable + " WHERE { " + restrictions.toSparql + " } LIMIT " + maxEntities

    val results = endpoint.query(sparql, maxEntities)

    results.map(_(variable).value)
  }

  private def getEntitiesPaths(endpoint: SparqlEndpoint, entities: Traversable[String], variable: String, limit: Int): Traversable[(Path, Double)] = {
    logger.info("Searching for relevant properties in " + endpoint)

    val entityArray = entities.toArray

    //Get all properties
    val properties = entityArray.flatMap(entity => getEntityProperties(endpoint, entity, variable, limit))

    //Compute the frequency of each property
    val propertyFrequencies = properties.groupBy(x => x).mapValues(_.size.toDouble / entityArray.size).toList

    //Choose the relevant properties
    val relevantProperties = propertyFrequencies.filter { case (uri, frequency) => frequency > MinFrequency }
                                                .sortWith(_._2 > _._2).take(limit)

    logger.info("Found " + relevantProperties.size + " relevant properties in " + endpoint)

    relevantProperties
  }

  private def getEntityProperties(endpoint: SparqlEndpoint, entityUri: String, variable: String, limit: Int): Traversable[Path] = {
    var sparql = ""
    sparql += "SELECT DISTINCT ?p \n"
    sparql += "WHERE {\n"
    sparql += " <" + entityUri + "> ?p ?o\n"
    sparql += "} LIMIT " + limit

    for (result <- endpoint.query(sparql); binding <- result.values) yield
      Path(variable, ForwardOperator(Uri.fromURI(binding.value)) :: Nil)
  }
}<commit_msg>Fixed issue with duplicated limits<commit_after>package de.fuberlin.wiwiss.silk.util.sparql

import de.fuberlin.wiwiss.silk.util.Uri
import java.util.logging.Logger
import de.fuberlin.wiwiss.silk.entity.{SparqlRestriction, ForwardOperator, Path}

/**
 * Retrieves the most frequent paths of a number of random sample entities.
 *
 * It is typically faster than SparqlAggregatePathsCollector but also less precise.
 *
 * The limitations of the current implementation are:
 * - It does only return forward paths of length 1
 * - It returns a maximum of 100 paths
 */
object SparqlSamplePathsCollector extends SparqlPathsCollector {
  /**Number of sample entities */
  private val maxEntities = 100

  /**The minimum frequency of a property to be considered relevant */
  private val MinFrequency = 0.7

  private implicit val logger = Logger.getLogger(SparqlSamplePathsCollector.getClass.getName)

  def apply(endpoint: SparqlEndpoint, restrictions: SparqlRestriction, limit: Option[Int]): Traversable[(Path, Double)] = {
    val variable = restrictions.toSparql.dropWhile(_ != '?').drop(1).takeWhile(_ != ' ')

    val sampleEntities = {
      if (variable.isEmpty)
        getAllEntities(endpoint)
      else
        getEntities(endpoint, restrictions, variable)
    }

    getEntitiesPaths(endpoint, sampleEntities, variable, limit.getOrElse(100))
  }

  private def getAllEntities(endpoint: SparqlEndpoint): Traversable[String] = {
    val sparql = "SELECT ?s WHERE { ?s ?p ?o }"

    val results = endpoint.query(sparql, maxEntities)

    results.map(_("s").value)
  }

  private def getEntities(endpoint: SparqlEndpoint, restrictions: SparqlRestriction, variable: String): Traversable[String] = {
    val sparql = "SELECT ?" + variable + " WHERE { " + restrictions.toSparql + " }"

    val results = endpoint.query(sparql, maxEntities)

    results.map(_(variable).value)
  }

  private def getEntitiesPaths(endpoint: SparqlEndpoint, entities: Traversable[String], variable: String, limit: Int): Traversable[(Path, Double)] = {
    logger.info("Searching for relevant properties in " + endpoint)

    val entityArray = entities.toArray

    //Get all properties
    val properties = entityArray.flatMap(entity => getEntityProperties(endpoint, entity, variable, limit))

    //Compute the frequency of each property
    val propertyFrequencies = properties.groupBy(x => x).mapValues(_.size.toDouble / entityArray.size).toList

    //Choose the relevant properties
    val relevantProperties = propertyFrequencies.filter { case (uri, frequency) => frequency > MinFrequency }
                                                .sortWith(_._2 > _._2).take(limit)

    logger.info("Found " + relevantProperties.size + " relevant properties in " + endpoint)

    relevantProperties
  }

  private def getEntityProperties(endpoint: SparqlEndpoint, entityUri: String, variable: String, limit: Int): Traversable[Path] = {
    var sparql = ""
    sparql += "SELECT DISTINCT ?p \n"
    sparql += "WHERE {\n"
    sparql += " <" + entityUri + "> ?p ?o\n"
    sparql += "}"

    for (result <- endpoint.query(sparql, limit); binding <- result.values) yield
      Path(variable, ForwardOperator(Uri.fromURI(binding.value)) :: Nil)
  }
}
{'fraction_non_alphanumeric': 0.10875946238220299, 'fraction_numerical': 0.004634636181059787, 'mean_word_length': 3.8241430700447094, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4675217', 'n_tokens_mistral': 1918, 'n_tokens_neox': 1834, 'n_words': 507}