<commit_before>package org.oseraf.bullseye.store

import org.oseraf.bullseye.store.impl.blueprints.BlueprintsGraphStore

import scala.util.{Failure, Success, Try}
import com.thinkaurelius.titan.core.attribute.Text._
import scala.collection.JavaConversions._
/**
 * Created by nhamblet.
 */
trait AttributeBasedNaivelyFuzzySearchPlugin {
  // returns scores from 0 to 1
  def search(key: AttributeContainer.KEY, value: AttributeContainer.VALUE): Iterable[(EntityStore.ID, Double)]
}


trait AttributeBasedSearchAvailableOptionsPlugin {
  def searchableAttributes: Iterable[(AttributeContainer.KEY, String)]
}

trait BruteForceAttributeBasedNaivelyFuzzySearchPlugin
  extends AttributeBasedNaivelyFuzzySearchPlugin
{
  val store: EntityStore with EntityIterationPlugin

  override def search(key: AttributeContainer.KEY, value: AttributeContainer.VALUE): Iterable[(EntityStore.ID, Double)] =
    store.entities.filter( entityId => {
      val attrMatch = attr(entityId, store.entity(entityId), key, "")
      Try(attrMatch.matches("(?i).*" + value + ".*")) match {
        case Success(v) => v
        case Failure(e) => Try(attrMatch.matches(value)) match {
          case Success(v) => v
          case Failure(e) => false
        }
      }
    }).map(eid => (eid, score(attr(eid, store.entity(eid), key, ""), value))).toSeq.sortBy(-_._2)

  def score(actual: AttributeContainer.VALUE, expected: AttributeContainer.VALUE): Double = {
    actual match {
      case s: String if s == expected                                 => 1.0
      case s: String if s.equalsIgnoreCase(expected)                  => 0.95
      case s: String if s.matches(".*\\b" + expected + "\\b.*")       => 0.85
      case s: String if s.matches("(?i).*\\b" + expected + "\\b.*")   => 0.80
      case s: String if s.matches(".*\\b" + expected + ".*")          => 0.75
      case s: String if s.matches("(?i).*\\b" + expected + ".*")      => 0.70
      case _                                                          => 0.50
    }
  }

  private def attr(entityId: EntityStore.ID, entity: Entity, key: AttributeContainer.KEY, default: AttributeContainer.VALUE = "") =
    if (key == AttributeBasedSearch.FAKE_ID_ATTRIBUTE_KEY) entityId
    else entity.attribute(key, default)
}

trait TitanIndexedFuzzyVertexSearchPlugin extends AttributeBasedNaivelyFuzzySearchPlugin {
  val store:BlueprintsGraphStore with EntityIterationPlugin
  override def search(key: AttributeContainer.KEY, value:AttributeContainer.VALUE):Iterable[(EntityStore.ID, Double)] = {
    store.graph.query().has(key, CONTAINS, value).vertices().toList.map(v => (v.getId.toString, 0.0))<commit_msg>adds scoring to TitanIndexed searches<commit_after>package org.oseraf.bullseye.store

import org.oseraf.bullseye.store.impl.blueprints.BlueprintsGraphStore

import scala.util.{Failure, Success, Try}
import com.thinkaurelius.titan.core.attribute.Text._
import scala.collection.JavaConversions._
/**
 * Created by nhamblet.
 */
trait AttributeBasedNaivelyFuzzySearchPlugin {
  // returns scores from 0 to 1
  def search(key: AttributeContainer.KEY, value: AttributeContainer.VALUE): Iterable[(EntityStore.ID, Double)]
}


trait AttributeBasedSearchAvailableOptionsPlugin {
  def searchableAttributes: Iterable[(AttributeContainer.KEY, String)]
}

trait BruteForceAttributeBasedNaivelyFuzzySearchPlugin
  extends AttributeBasedNaivelyFuzzySearchPlugin
{
  val store: EntityStore with EntityIterationPlugin

  override def search(key: AttributeContainer.KEY, value: AttributeContainer.VALUE): Iterable[(EntityStore.ID, Double)] =
    store.entities.filter( entityId => {
      val attrMatch = attr(entityId, store.entity(entityId), key, "")
      Try(attrMatch.matches("(?i).*" + value + ".*")) match {
        case Success(v) => v
        case Failure(e) => Try(attrMatch.matches(value)) match {
          case Success(v) => v
          case Failure(e) => false
        }
      }
    }).map(eid => (eid, score(attr(eid, store.entity(eid), key, ""), value))).toSeq.sortBy(-_._2)

  def score(actual: AttributeContainer.VALUE, expected: AttributeContainer.VALUE): Double = {
    actual match {
      case s: String if s == expected                                 => 1.0
      case s: String if s.equalsIgnoreCase(expected)                  => 0.95
      case s: String if s.matches(".*\\b" + expected + "\\b.*")       => 0.85
      case s: String if s.matches("(?i).*\\b" + expected + "\\b.*")   => 0.80
      case s: String if s.matches(".*\\b" + expected + ".*")          => 0.75
      case s: String if s.matches("(?i).*\\b" + expected + ".*")      => 0.70
      case _                                                          => 0.50
    }
  }

  private def attr(entityId: EntityStore.ID, entity: Entity, key: AttributeContainer.KEY, default: AttributeContainer.VALUE = "") =
    if (key == AttributeBasedSearch.FAKE_ID_ATTRIBUTE_KEY) entityId
    else entity.attribute(key, default)
}

trait TitanIndexedFuzzyVertexSearchPlugin extends AttributeBasedNaivelyFuzzySearchPlugin with BruteForceAttributeBasedNaivelyFuzzySearchPlugin{
  val store:BlueprintsGraphStore with EntityIterationPlugin
  override def search(key: AttributeContainer.KEY, value:AttributeContainer.VALUE):Iterable[(EntityStore.ID, Double)] = {
    store.graph.query().has(key, CONTAINS, value).vertices().toList.map(v => (v.getId.toString, score(v.getProperty(key), value)))
{'fraction_non_alphanumeric': 0.13099219620958752, 'fraction_numerical': 0.008918617614269788, 'mean_word_length': 3.3236947791164657, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4251342', 'n_tokens_mistral': 1626, 'n_tokens_neox': 1566, 'n_words': 348}