<commit_before>package hu.bme.mit.ire

import hu.bme.mit.ire.datatypes.Tuple
import ingraph.ire._
import ingraph.model.expr.{EdgeAttribute, PropertyAttribute, VertexAttribute}
import ingraph.model.fplan.{GetEdges, GetVertices}
import org.apache.spark.sql.catalyst.expressions.Literal

object TupleConstants {
  val ID_KEY = "id"
}

object PropertyTransformer {
  def apply(properties: Map[String, Any], key: String, id: Long): Any = {
    if (key == TupleConstants.ID_KEY)
      id
    else
      properties.getOrElse(key, null)
  }
}

object VertexTransformer {
  def apply(element: IngraphVertex, operator: GetVertices, idParser: IdParser): Tuple = {
    Vector(idParser(element.id)) ++
      operator.internalSchema.map(_.name).drop(1).map(key => PropertyTransformer(element.properties, key, element.id))
  }
}

object EdgeTransformer {
  def apply(edge: IngraphEdge, operator: GetEdges, idParser: IdParser): Tuple = {
    Vector(idParser(edge.sourceVertex.id), idParser(edge.id), idParser(edge.targetVertex.id)) ++
      operator.internalSchema.drop(3)
        .map {
          case PropertyAttribute(name, elementAttribute, _) => elementAttribute match {
            case _: EdgeAttribute => PropertyTransformer(edge.properties, name, edge.id)
            case v: VertexAttribute if v.name == operator.jnode.trg.name =>
              PropertyTransformer(edge.targetVertex.properties, name, edge.targetVertex.id)
            case v: VertexAttribute if v.name == operator.jnode.src.name =>
              PropertyTransformer(edge.sourceVertex.properties, name, edge.sourceVertex.id)
          }
        }
  }
}

class TupleCreator(vertexConverters: Map[Set[String], Set[GetVertices]],
                   edgeConverters: Map[String, Set[GetEdges]],
                   idParser: IdParser = PlainIdParser) extends GraphElementToTupleMapper {

  var transaction: Transaction = _

  def addEdge(edge: IngraphEdge): Unit = {
    for (operators <- edgeConverters.get(edge.`type`); operator <- operators) {
      val sourceLabels = operator.jnode.src.labels.vertexLabels
      val targetLabels = operator.jnode.trg.labels.vertexLabels
      if (sourceLabels.subsetOf(edge.sourceVertex.labels) &&
        targetLabels.subsetOf(edge.targetVertex.labels)) {
        val tuple = EdgeTransformer(edge, operator, idParser)
        transaction.add(operator.toString(), tuple)
        if (!operator.jnode.directed) {
          val rTuple = EdgeTransformer(
            edge.copy(sourceVertex = edge.targetVertex, targetVertex = edge.sourceVertex), operator, idParser)
          transaction.add(operator.toString(), rTuple)
        }
      }
    }
  }

  def removeEdge(edge: IngraphEdge): Unit = {
    for (operators <- edgeConverters.get(edge.`type`); operator <- operators) {
      val sourceLabels = operator.jnode.src.labels.vertexLabels
      val targetLabels = operator.jnode.trg.labels.vertexLabels
      if (sourceLabels.subsetOf(edge.sourceVertex.labels) &&
        targetLabels.subsetOf(edge.targetVertex.labels)) {
        val tuple = EdgeTransformer(edge, operator, idParser)
        transaction.remove(operator.toString(), tuple)
        if (!operator.jnode.directed) {
          val rTuple = EdgeTransformer(
            edge.copy(sourceVertex = edge.targetVertex, targetVertex = edge.sourceVertex), operator, idParser)
          transaction.remove(operator.toString(), rTuple)
        }
      }
    }
  }

  def addVertex(vertex: IngraphVertex): Unit = {
    vertexConverters.filter(p => p._1.subsetOf(vertex.labels)).foreach {
      f =>
        for (operator <- f._2) {
          val tuple = VertexTransformer(vertex, operator, idParser)
          transaction.add(operator.jnode.v.name, tuple)<commit_msg>Cache toStringt that takes up 70% of cpu resources<commit_after>package hu.bme.mit.ire

import hu.bme.mit.ire.datatypes.Tuple
import ingraph.ire._
import ingraph.model.expr.{EdgeAttribute, PropertyAttribute, VertexAttribute}
import ingraph.model.fplan.{GetEdges, GetVertices}
import org.apache.spark.sql.catalyst.expressions.Literal

object TupleConstants {
  val ID_KEY = "id"
}

object PropertyTransformer {
  def apply(properties: Map[String, Any], key: String, id: Long): Any = {
    if (key == TupleConstants.ID_KEY)
      id
    else
      properties.getOrElse(key, null)
  }
}

object VertexTransformer {
  def apply(element: IngraphVertex, operator: GetVertices, idParser: IdParser): Tuple = {
    Vector(idParser(element.id)) ++
      operator.internalSchema.map(_.name).drop(1).map(key => PropertyTransformer(element.properties, key, element.id))
  }
}

object EdgeTransformer {
  def apply(edge: IngraphEdge, operator: GetEdges, idParser: IdParser): Tuple = {
    Vector(idParser(edge.sourceVertex.id), idParser(edge.id), idParser(edge.targetVertex.id)) ++
      operator.internalSchema.drop(3)
        .map {
          case PropertyAttribute(name, elementAttribute, _) => elementAttribute match {
            case _: EdgeAttribute => PropertyTransformer(edge.properties, name, edge.id)
            case v: VertexAttribute if v.name == operator.jnode.trg.name =>
              PropertyTransformer(edge.targetVertex.properties, name, edge.targetVertex.id)
            case v: VertexAttribute if v.name == operator.jnode.src.name =>
              PropertyTransformer(edge.sourceVertex.properties, name, edge.sourceVertex.id)
          }
        }
  }
}

class TupleCreator(vertexConverters: Map[Set[String], Set[GetVertices]],
                   edgeConverters: Map[String, Set[GetEdges]],
                   idParser: IdParser = PlainIdParser) extends GraphElementToTupleMapper {
  val edgeOpString = edgeConverters.values.flatten.map(op => op -> op.toString()).toMap
  var transaction: Transaction = _

  def addEdge(edge: IngraphEdge): Unit = {
    for (operators <- edgeConverters.get(edge.`type`); operator <- operators) {
      val sourceLabels = operator.jnode.src.labels.vertexLabels
      val targetLabels = operator.jnode.trg.labels.vertexLabels
      if (sourceLabels.subsetOf(edge.sourceVertex.labels) &&
        targetLabels.subsetOf(edge.targetVertex.labels)) {
        val tuple = EdgeTransformer(edge, operator, idParser)
        transaction.add(edgeOpString(operator), tuple)
        if (!operator.jnode.directed) {
          val rTuple = EdgeTransformer(
            edge.copy(sourceVertex = edge.targetVertex, targetVertex = edge.sourceVertex), operator, idParser)
          transaction.add(edgeOpString(operator), rTuple)
        }
      }
    }
  }

  def removeEdge(edge: IngraphEdge): Unit = {
    for (operators <- edgeConverters.get(edge.`type`); operator <- operators) {
      val sourceLabels = operator.jnode.src.labels.vertexLabels
      val targetLabels = operator.jnode.trg.labels.vertexLabels
      if (sourceLabels.subsetOf(edge.sourceVertex.labels) &&
        targetLabels.subsetOf(edge.targetVertex.labels)) {
        val tuple = EdgeTransformer(edge, operator, idParser)
        transaction.remove(operator.toString(), tuple)
        if (!operator.jnode.directed) {
          val rTuple = EdgeTransformer(
            edge.copy(sourceVertex = edge.targetVertex, targetVertex = edge.sourceVertex), operator, idParser)
          transaction.remove(operator.toString(), rTuple)
        }
      }
    }
  }

  def addVertex(vertex: IngraphVertex): Unit = {
    vertexConverters.filter(p => p._1.subsetOf(vertex.labels)).foreach {
      f =>
        for (operator <- f._2) {
          val tuple = VertexTransformer(vertex, operator, idParser)
          transaction.add(operator.jnode.v.name, tuple)
{'fraction_non_alphanumeric': 0.11410018552875696, 'fraction_numerical': 0.0013252054068380599, 'mean_word_length': 3.2929465301478955, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1084425', 'n_tokens_mistral': 2221, 'n_tokens_neox': 2187, 'n_words': 420}