<commit_before>/*
 * Copyright (c) 2002-2015 "Neo Technology,"
 * Network Engine for Objects in Lund AB [http://neotechnology.com]
 *
 * This file is part of Neo4j.
 *
 * Neo4j is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

package org.neo4j.cypher.internal.compiler.v2_3.codegen

import java.util

import org.neo4j.codegen
import org.neo4j.codegen.CodeGeneratorOption.BLANK_OPTION
import org.neo4j.codegen.ExpressionTemplate._
import org.neo4j.codegen.MethodReference._
import org.neo4j.codegen.TypeReference.{extending, parameterizedType, typeParameter}
import org.neo4j.codegen._
import org.neo4j.codegen.source.SourceVisitor
import org.neo4j.collection.primitive.hopscotch.LongKeyIntValueTable
import org.neo4j.collection.primitive.{Primitive, PrimitiveLongIntMap, PrimitiveLongIterator, PrimitiveLongObjectMap}
import org.neo4j.cypher.internal.compiler.v2_3._
import org.neo4j.cypher.internal.compiler.v2_3.codegen.CompiledConversionUtils.CompositeKey
import org.neo4j.cypher.internal.compiler.v2_3.executionplan.{GeneratedQuery, GeneratedQueryExecution, SuccessfulCloseable}
import org.neo4j.cypher.internal.compiler.v2_3.helpers.using
import org.neo4j.cypher.internal.compiler.v2_3.planDescription.{Id, InternalPlanDescription}
import org.neo4j.cypher.internal.compiler.v2_3.planner.CantCompileQueryException
import org.neo4j.cypher.internal.compiler.v2_3.symbols.CypherType
import org.neo4j.function.Supplier
import org.neo4j.graphdb.Result.{ResultRow, ResultVisitor}
import org.neo4j.graphdb.{Direction, GraphDatabaseService, Node, Relationship}
import org.neo4j.helpers.collection.MapUtil
import org.neo4j.kernel.api.exceptions.KernelException
import org.neo4j.kernel.api.index.IndexDescriptor
import org.neo4j.kernel.api.properties.Property
import org.neo4j.kernel.api.{ReadOperations, Statement, StatementTokenNameLookup, TokenNameLookup}
import org.neo4j.kernel.impl.api.store.RelationshipIterator
import org.neo4j.kernel.impl.api.{RelationshipDataExtractor, RelationshipVisitor}

import scala.collection.mutable

// <SPI>
trait CodeStructure[T] {
  type SourceSink = (String, String) => Unit
  def generateQuery(packageName: String, className: String, columns: Seq[String], operatorIds: Map[String, Id], sourceSink: Option[SourceSink])
                   (block: MethodStructure[_] => Unit)(implicit codeGenContext: CodeGenContext): T
}

sealed trait JoinTableType
sealed trait CountingJoinTableType extends JoinTableType
sealed trait RecordingJoinTableType extends JoinTableType

case object LongToCountTable extends CountingJoinTableType
case object LongsToCountTable extends CountingJoinTableType
case class LongToListTable(structure: Map[String, CypherType], localMap: Map[String, String]) extends RecordingJoinTableType
case class LongsToListTable(structure: Map[String, CypherType], localMap: Map[String, String]) extends RecordingJoinTableType

trait MethodStructure[E] {

  // misc
  def declareFlag(name: String, initialValue: Boolean)
  def updateFlag(name: String, newValue: Boolean)
  def declarePredicate(name: String): Unit
  def declareProperty(name: String): Unit
  def declareCounter(name: String, initialValue: E): Unit
  def putField(structure: Map[String, CypherType], value: E, fieldType: CypherType, fieldName: String, localVar: String): Unit
  def updateProbeTable(structure: Map[String, CypherType], tableVar: String, tableType: RecordingJoinTableType, keyVars: Seq[String], element: E): Unit
  def probe(tableVar: String, tableType: JoinTableType, keyVars: Seq[String])(block: MethodStructure[E]=>Unit): Unit
  def updateProbeTableCount(tableVar: String, tableType: CountingJoinTableType, keyVar: Seq[String]): Unit
  def allocateProbeTable(tableVar: String, tableType: JoinTableType): Unit
  def method(resultType: JoinTableType, resultVar: String, methodName: String)(block: MethodStructure[E]=>Unit): Unit

  // expressions
  def decreaseCounterAndCheckForZero(name: String): E
  def counterEqualsZero(variableName: String): E
  def newTableValue(targetVar: String, structure: Map[String, CypherType]): E
  def constant(value: Object): E
  def asMap(map: Map[String, E]): E
  def asList(values: Seq[E]): E

  def toSet(value: E): E

  def castToCollection(value: E): E

  def load(varName: String): E

  // arithmetic
  def add(lhs: E, rhs: E): E
  def sub(lhs: E, rhs: E): E
  def mul(lhs: E, rhs: E): E
  def div(lhs: E, rhs: E): E
  def mod(lhs: E, rhs: E): E

  // predicates
  def not(value: E): E

  // null handling
  def markAsNull(varName: String, cypherType: CypherType): Unit
  def nullable(varName: String, cypherType: CypherType, onSuccess: E): E

  // parameters
  def expectParameter(key: String): Unit
  def parameter(key: String): E

  // tracing
  def trace[V](planStepId: String)(block: MethodStructure[E] => V): V
  def incrementDbHits(): Unit
  def incrementRows(): Unit

  // db access
  def labelScan(iterVar: String, labelIdVar: String): Unit
  def hasLabel(nodeVar: String, labelVar: String, predVar: String): E
  def allNodesScan(iterVar: String): Unit
  def lookupLabelId(labelIdVar: String, labelName: String): Unit
  def lookupRelationshipTypeId(typeIdVar: String, typeName: String): Unit
  def nodeGetAllRelationships(iterVar: String, nodeVar: String, direction: Direction): Unit
  def nodeGetRelationships(iterVar: String, nodeVar: String, direction: Direction, typeVars: Seq[String]): Unit
  def connectingRelationships(iterVar: String, fromNode: String, dir: Direction, toNode:String)
  def connectingRelationships(iterVar: String, fromNode: String, dir: Direction, types: Seq[String], toNode: String)
  def nextNode(targetVar: String, iterVar: String): Unit
  def nextRelationshipAndNode(toNodeVar: String, iterVar: String, direction: Direction, fromNodeVar: String, relVar: String): Unit
  def nextRelationship(iterVar: String, direction: Direction, relVar: String): Unit
  def hasNext(iterVar: String): E
  def nodeGetPropertyById(nodeIdVar: String, propId: Int, propValueVar: String): Unit
  def nodeGetPropertyForVar(nodeIdVar: String, propIdVar: String, propValueVar: String): Unit
  def relationshipGetPropertyById(nodeIdVar: String, propId: Int, propValueVar: String): Unit
  def relationshipGetPropertyForVar(nodeIdVar: String, propIdVar: String, propValueVar: String): Unit
  def lookupPropertyKey(propName: String, propVar: String)
  def coerceToBoolean(propertyExpression: E): E

  def newIndexDescriptor(descriptorVar: String, labelVar: String, propKeyVar: String): Unit

  def indexSeek(iterVar: String, descriptorVar: String, value: E): Unit

  // code structure
  def whileLoop(test: E)(block: MethodStructure[E] => Unit): Unit
  def forEach(varName: String, cypherType: CypherType, iterable: E)(block: MethodStructure[E] => Unit): Unit
  def ifStatement(test: E)(block: MethodStructure[E] => Unit): Unit
  def returnSuccessfully(): Unit

  // results
  def materializeNode(nodeIdVar: String): E
  def materializeRelationship(relIdVar: String): E
  def visitRow(): Unit
  def setInRow(column: String, value: E): Unit
}

// </SPI>

// TODO: the rest of this file is implementation of the SPI above, and should move to cypher/cypher

object CodeStructure {

  val __TODO__MOVE_IMPLEMENTATION = new CodeStructure[GeneratedQuery] {

    override def generateQuery(packageName: String, className: String, columns: Seq[String], operatorIds: Map[String, Id], sourceSink: Option[SourceSink])
                              (block: MethodStructure[_] => Unit)(implicit codeGenContext: CodeGenContext) = {
      val generator = try {
        codegen.CodeGenerator.generateCode(CodeStructure.getClass.getClassLoader, sourceSink.map(sink => new SourceVisitor {
          override protected def visitSource(reference: TypeReference, sourceCode: CharSequence): Unit =
            sink.apply(reference.name(), sourceCode.toString)
        }).getOrElse(BLANK_OPTION))
      } catch {
        case e: Exception => throw new CantCompileQueryException(e.getMessage, e)
      }
      val execution = using(generator.generateClass(packageName, className+"Execution", typeRef[GeneratedQueryExecution], typeRef[SuccessfulCloseable])) { clazz =>
        // fields
        val fields = Fields(
          closer = clazz.field(typeRef[TaskCloser], "closer"),
          ro = clazz.field(typeRef[ReadOperations], "ro"),
          db = clazz.field(typeRef[GraphDatabaseService], "db"),
          executionMode = clazz.field(typeRef[ExecutionMode], "executionMode"),
          description = clazz.field(typeRef[Supplier[InternalPlanDescription]], "description"),
          tracer = clazz.field(typeRef[QueryExecutionTracer], "tracer"),
          params = clazz.field(typeRef[util.Map[String, Object]], "params"),
          closeable = clazz.field(typeRef[SuccessfulCloseable], "closeable"),
          success = clazz.generate(Templates.SUCCESS))
        // the "COLUMNS" static field
        clazz.staticField(typeRef[util.List[String]], "COLUMNS", Templates.asList(
          columns.toSeq.map(key => Expression.constant(key))))

        // the operator id fields
        operatorIds.keys.foreach { opId =>
          clazz.staticField(typeRef[Id], opId)
        }

        // simple methods
        clazz.generate(Templates.CONSTRUCTOR)
        clazz.generate(Templates.SET_SUCCESSFUL_CLOSEABLE)
        val close = clazz.generate(Templates.CLOSE)
        clazz.generate(Templates.EXECUTION_MODE)
        clazz.generate(Templates.EXECUTION_PLAN_DESCRIPTION)
        clazz.generate(Templates.JAVA_COLUMNS)

        using(clazz.generate(MethodDeclaration.method(typeRef[Unit], "accept",
          Parameter.param(parameterizedType(classOf[ResultVisitor[_]], typeParameter("E")), "visitor")).
          parameterizedWith("E", extending(typeRef[Exception])).
          throwsException(typeParameter("E")))) { method =>
          using(method.tryBlock()) { body =>
            body.assign(typeRef[ResultRowImpl], "row", Templates.newResultRow)
            body.assign(typeRef[RelationshipDataExtractor], "rel", Templates.newRelationshipDataExtractor)
            block(Method(fields, body, new AuxGenerator(packageName, generator)))
            body.expression(Expression.invoke(body.self(), fields.success))
            using(body.finallyBlock()) { then =>
              then.expression(Expression.invoke(body.self(), close))
            }
          }
        }
        clazz.handle()
      }
      val query = using(generator.generateClass(packageName,className, typeRef[GeneratedQuery])) { clazz =>
        using(clazz.generateMethod(typeRef[GeneratedQueryExecution], "execute",
          param[TaskCloser]("closer"),
          param[Statement]("statement"),
          param[GraphDatabaseService]("db"),
          param[ExecutionMode]("executionMode"),
          param[Supplier[InternalPlanDescription]]("description"),
          param[QueryExecutionTracer]("tracer"),
          param[util.Map[String, Object]]("params"))) { execute =>
          execute.returns(Expression.invoke(Expression.newInstance(execution), MethodReference.constructorReference(execution,
            typeRef[TaskCloser],
            typeRef[Statement],
            typeRef[GraphDatabaseService],
            typeRef[ExecutionMode],
            typeRef[Supplier[InternalPlanDescription]],
            typeRef[QueryExecutionTracer],
            typeRef[util.Map[String, Object]]),
            execute.load("closer"),
            execute.load("statement"),
            execute.load("db"),
            execute.load("executionMode"),
            execute.load("description"),
            execute.load("tracer"),
            execute.load("params")))
        }
        clazz.handle()
      }.newInstance().asInstanceOf[GeneratedQuery]
      val clazz: Class[_] = execution.loadClass()
      operatorIds.foreach {
        case (key, id) => setStaticField(clazz, key, id)
      }
      query
    }
  }

  def method[O <: AnyRef, R](name: String, params: TypeReference*)(implicit owner: Manifest[O], returns: Manifest[R]): MethodReference =
    MethodReference.methodReference(typeReference(owner), typeReference(returns), name, params: _*)

  def staticField[O <: AnyRef, R](name: String)(implicit owner: Manifest[O], fieldType: Manifest[R]): FieldReference =
    FieldReference.staticField(typeReference(owner), typeReference(fieldType), name)

  def param[T <: AnyRef](name: String)(implicit manifest: Manifest[T]): Parameter =
    Parameter.param(typeReference(manifest), name)

  def typeRef[T](implicit manifest: Manifest[T]): TypeReference = typeReference(manifest)

  def typeReference(manifest: Manifest[_]): TypeReference = {
    val arguments = manifest.typeArguments
    val base = TypeReference.typeReference(manifest.runtimeClass)
    if(arguments.nonEmpty) {
      TypeReference.parameterizedType(base, arguments.map(typeReference): _*)
    } else {
      base
    }
  }

  def lowerType(cType:CypherType):TypeReference = cType match {
    case symbols.CTNode => typeRef[Long]
    case symbols.CTRelationship => typeRef[Long]
    case symbols.CTAny => typeRef[Object]
  }

  def nullValue(cType: CypherType) = cType match {
    case symbols.CTNode => Expression.constant(-1L)
    case symbols.CTRelationship => Expression.constant(-1L)
    case symbols.CTAny => Expression.constant(null)
  }
}

private case class Fields(closer: FieldReference,
                          ro: FieldReference,
                          db: FieldReference,
                          executionMode: FieldReference,
                          description: FieldReference,
                          tracer: FieldReference,
                          params: FieldReference,
                          closeable: FieldReference,
                          success: MethodReference)

private class AuxGenerator(val packageName: String, val generator: codegen.CodeGenerator) {
  private val types: mutable.Map[Map[String, CypherType], TypeReference] = mutable.Map.empty
  private var nameId = 0

  def typeReference(structure: Map[String, CypherType]): TypeReference = {
    types.getOrElseUpdate(structure, using(generator.generateClass(packageName, newName())) { clazz =>
      structure.foreach {
        case (fieldName, fieldType) => clazz.field(CodeStructure.lowerType(fieldType), fieldName)
      }
      clazz.handle()
    })
  }
  private def newName() = {
    val name = "ValueType" + nameId
    nameId += 1
    name
  }
}


private case class Method(fields: Fields, generator: CodeBlock, aux:AuxGenerator, tracing: Boolean = true,
                          event: Option[String] = None, var locals:Map[String,LocalVariable]=Map.empty)(implicit context: CodeGenContext)
  extends MethodStructure[Expression] {
  import CodeStructure.typeRef

  override def nextNode(targetVar: String, iterVar: String) =
    generator.assign(typeRef[Long], targetVar, Expression.invoke(generator.load(iterVar), Methods.nextLong))

  override def nextRelationshipAndNode(toNodeVar: String, iterVar: String, direction: Direction, fromNodeVar: String,
                                    relVar: String) = {
    val startNode = Expression.invoke(generator.load("rel"), Methods.startNode)
    val endNode = Expression.invoke(generator.load("rel"), Methods.endNode)
    generator.expression(Expression.invoke(generator.load(iterVar), Methods.relationshipVisit,
                                           Expression.invoke(generator.load(iterVar), Methods.nextLong),
                                           generator.load("rel")))
    generator.assign(typeRef[Long], toNodeVar, direction match {
      case Direction.INCOMING => startNode
      case Direction.OUTGOING => endNode
      case Direction.BOTH => Expression.ternary(Expression.eq(startNode, generator.load(fromNodeVar)), endNode, startNode)
    })
    generator.assign(typeRef[Long], relVar, Expression.invoke(generator.load("rel"), Methods.relationship))
  }

  override def nextRelationship(iterVar: String, direction: Direction, relVar: String) = {
    generator.expression(Expression.invoke(generator.load(iterVar), Methods.relationshipVisit,
                                           Expression.invoke(generator.load(iterVar), Methods.nextLong),
                                           generator.load("rel")))
    generator.assign(typeRef[Long], relVar, Expression.invoke(generator.load("rel"), Methods.relationship))
  }

  override def allNodesScan(iterVar: String) =
    generator.assign(typeRef[PrimitiveLongIterator], iterVar, Expression.invoke(readOperations, Methods.nodesGetAll))

  override def labelScan(iterVar: String, labelIdVar: String) =
    generator.assign(typeRef[PrimitiveLongIterator], iterVar,
                     Expression.invoke(readOperations, Methods.nodesGetForLabel, generator.load(labelIdVar)))


  override def lookupLabelId(labelIdVar: String, labelName: String) =
    generator.assign(typeRef[Int], labelIdVar,
                     Expression.invoke(readOperations, Methods.labelGetForName, Expression.constant(labelName)))

  override def lookupRelationshipTypeId(typeIdVar: String, typeName: String) =
    generator.assign(typeRef[Int], typeIdVar, Expression
      .invoke(readOperations, Methods.relationshipTypeGetForName, Expression.constant(typeName)))

  override def hasNext(iterVar: String) =
    Expression.invoke(generator.load(iterVar), Methods.hasNext)

  override def whileLoop(test: Expression)(block: MethodStructure[Expression] => Unit) =
    using(generator.whileLoop(test)) { body =>
      block(copy(generator = body))
    }

  override def forEach(varName: String, cypherType: CypherType, iterable: Expression)(block: MethodStructure[Expression] => Unit) =
    using(generator.forEach(Parameter.param(CodeStructure.lowerType(cypherType), varName), iterable)) { body =>
      block(copy(generator = body))
    }

  override def ifStatement(test: Expression)(block: (MethodStructure[Expression]) => Unit) = {
    using(generator.ifStatement(test)) { body =>
      block(copy(generator = body))
    }
  }

  override def returnSuccessfully() {
    generator.expression(Expression.invoke(generator.self(), fields.success))
    generator.returns()
  }

  override def declareCounter(name: String, initialValue: Expression): Unit = {
    val variable = generator.declare(typeRef[Int], name)
    locals = locals + (name -> variable)
    generator.assign(variable, Expression.invoke(Methods.mathCastToInt, initialValue))
  }

  override def decreaseCounterAndCheckForZero(name: String): Expression = {
    val local = locals(name)
    generator.assign(local, Expression.sub(local, Expression.constant(1)))
    Expression.eq(Expression.constant(0), local)
  }

  override def counterEqualsZero(name: String): Expression = {
    val local = locals(name)
    Expression.eq(Expression.constant(0), local)
  }

  override def setInRow(column: String, value: Expression) =
    generator.expression(Expression.invoke(resultRow, Methods.set, Expression.constant(column), value))

  override def visitRow() = using(generator.ifStatement(Expression.not(
    Expression.invoke(generator.load("visitor"), Methods.visit, generator.load("row"))))) { body =>
    // NOTE: we are in this if-block if the visitor decided to terminate early (by returning false)
    body.expression(Expression.invoke(body.self(), fields.success))
    body.returns()
  }

  override def materializeNode(nodeIdVar: String) = Expression.invoke(db, Methods.getNodeById, generator.load(nodeIdVar))

  override def nullable(varName: String, cypherType: CypherType, onSuccess: Expression) = {
    Expression.ternary(
      Expression.eq(CodeStructure.nullValue(cypherType), generator.load(varName)),
      Expression.constant(null),
      onSuccess)
  }

  override def materializeRelationship(relIdVar: String) = Expression.invoke(db, Methods.getRelationshipById, generator.load(relIdVar))

  override def trace[V](planStepId: String)(block: MethodStructure[Expression]=>V) = if(!tracing) block(this)
  else {
    val eventName = s"event_$planStepId"
    using(generator.tryBlock(typeRef[QueryExecutionEvent], eventName, traceEvent(planStepId))) { body =>
      block(copy(event = Some(eventName),
        generator = body))
    }
  }

  private def traceEvent(planStepId: String) = Expression.invoke(tracer, Methods.executeOperator,
    Expression.get(FieldReference.staticField(generator.owner(), typeRef[Id], planStepId)))

  override def incrementDbHits() = if(tracing) generator.expression(Expression.invoke(loadEvent, Methods.dbHit))

  override def incrementRows() = if(tracing) generator.expression(Expression.invoke(loadEvent, Methods.row))

  private def loadEvent = generator.load(event.getOrElse(throw new IllegalStateException("no current trace event")))

  override def expectParameter(key: String) =
    using(generator.ifStatement(Expression.not(Expression.invoke(params, Methods.mapContains, Expression.constant(key))))) { block =>
      block.throwException(parameterNotFoundException(key))
    }

  override def parameter(key: String) = Expression.invoke(params, Methods.mapGet, Expression.constant(key))

  override def constant(value: Object) = Expression.constant(value)

  override def not(value: Expression): Expression = Expression.not(value)

  override def markAsNull(varName: String, cypherType: CypherType) =
    generator.assign(CodeStructure.lowerType(cypherType), varName, CodeStructure.nullValue(cypherType))

  override def nodeGetAllRelationships(iterVar: String, nodeVar: String, direction: Direction) = {
    val local = generator.declare(typeRef[RelationshipIterator], iterVar)
    Templates.handleExceptions(generator, fields.ro) { body =>
      body.assign(local, Expression.invoke(readOperations, Methods.nodeGetAllRelationships, body.load(nodeVar), dir(direction)))
    }
  }

  override def nodeGetRelationships(iterVar: String, nodeVar: String, direction: Direction, typeVars: Seq[String]) = {
    val local = generator.declare(typeRef[RelationshipIterator], iterVar)
    Templates.handleExceptions(generator, fields.ro) { body =>
      val args = Seq(body.load(nodeVar), dir(direction)) ++ typeVars.map(body.load)
      body.assign(local, Expression.invoke(readOperations, Methods.nodeGetRelationships, args: _*))
    }
  }

  override def connectingRelationships(iterVar: String, fromNode: String, direction: Direction, toNode: String) = {
    val local = generator.declare(typeRef[RelationshipIterator], iterVar)
    Templates.handleExceptions(generator, fields.ro) { body =>
      body.assign(local, Expression.invoke(Methods.allConnectingRelationships, readOperations, body.load(fromNode), dir(direction), body.load(toNode)))
    }
  }

  override def connectingRelationships(iterVar: String, fromNode: String, direction: Direction, typeVars: Seq[String], toNode: String) = {
    val local = generator.declare(typeRef[RelationshipIterator], iterVar)
    Templates.handleExceptions(generator, fields.ro) { body =>
      val args = Seq(readOperations, body.load(fromNode), dir(direction),  body.load(toNode)) ++ typeVars.map(body.load)
      body.assign(local, Expression.invoke(Methods.connectingRelationships, args:_*))
    }
  }

  override def load(varName: String) = generator.load(varName)

  override def add(lhs: Expression, rhs: Expression) = math(Methods.mathAdd, lhs, rhs)

  override def sub(lhs: Expression, rhs: Expression) = math(Methods.mathSub, lhs, rhs)

  override def mul(lhs: Expression, rhs: Expression) = math(Methods.mathMul, lhs, rhs)

  override def div(lhs: Expression, rhs: Expression) = math(Methods.mathDiv, lhs, rhs)

  override def mod(lhs: Expression, rhs: Expression) = math(Methods.mathMod, lhs, rhs)

  private def math(method: MethodReference, lhs: Expression, rhs: Expression): Expression =
    // TODO: generate specialized versions for specific types
    Expression.invoke(method, lhs, rhs)

  private def readOperations = Expression.get(generator.self(), fields.ro)

  private def db = Expression.get(generator.self(), fields.db)

  private def resultRow = generator.load("row")

  private def tracer = Expression.get(generator.self(), fields.tracer)

  private def params = Expression.get(generator.self(), fields.params)

  private def parameterNotFoundException(key: String) =
    Expression.invoke(Expression.newInstance(typeRef[ParameterNotFoundException]),
      MethodReference.constructorReference(typeRef[ParameterNotFoundException], typeRef[String]),
      Expression.constant(s"Expected a parameter named $key"))

  private def dir(dir: Direction): Expression = dir match {
    case Direction.INCOMING => Templates.incoming
    case Direction.OUTGOING => Templates.outgoing
    case Direction.BOTH => Templates.both
  }

  override def asList(values: Seq[Expression]) = Templates.asList(values)

  override def toSet(value: Expression) =
    Templates.newInstance(typeRef[util.HashSet[Object]], value)

  override def castToCollection(value: Expression) = Expression.invoke(Methods.toCollection, value)

  override def asMap(map: Map[String, Expression]) = {
    Expression.invoke(Methods.arrayAsList, map.flatMap {
      case (key, value) => Seq(Expression.constant(key), value)
    }.toSeq: _*)
  }

  override def method(resultType: JoinTableType, resultVar: String, methodName: String)(block: MethodStructure[Expression]=>Unit) = {
    val returnType: TypeReference = joinTableType(resultType)
    generator.assign(returnType, resultVar, Expression.invoke(generator.self(),MethodReference.methodReference(generator.owner(),returnType, methodName)))
    using(generator.classGenerator().generateMethod(returnType, methodName)) { body =>
      body.assign(typeRef[RelationshipDataExtractor], "rel", Templates.newRelationshipDataExtractor)
      block(copy(generator = body, event = None))
      body.returns(body.load(resultVar))
    }
  }

  override def allocateProbeTable(tableVar: String, tableType: JoinTableType) =
    generator.assign(joinTableType(tableType), tableVar, allocate(tableType))


  private def joinTableType(resultType: JoinTableType): TypeReference = {
    val returnType = resultType match {
      case LongToCountTable => typeRef[PrimitiveLongIntMap]
      case LongsToCountTable => TypeReference.parameterizedType(classOf[util.HashMap[_, _]], classOf[CompositeKey], classOf[java.lang.Integer])
      case LongToListTable(structure,_) => TypeReference.parameterizedType(classOf[PrimitiveLongObjectMap[_]],
        TypeReference.parameterizedType(classOf[util.ArrayList[_]], aux.typeReference(structure)))
      case LongsToListTable(structure,_) => TypeReference.parameterizedType(classOf[util.HashMap[_,_]], typeRef[CompositeKey],
         TypeReference.parameterizedType(classOf[util.ArrayList[_]], aux.typeReference(structure)))
    }
    returnType
  }

  private def allocate(resultType: JoinTableType): Expression = resultType match {
    case LongToCountTable => Templates.newCountingMap
    case LongToListTable(_,_) => Templates.newLongObjectMap
    case LongsToCountTable => Templates.newInstance(joinTableType(LongsToCountTable))
    case typ: LongsToListTable => Templates.newInstance(joinTableType(typ))
  }

  override def updateProbeTableCount(tableVar: String, tableType: CountingJoinTableType,
                                     keyVars: Seq[String]) = tableType match {
    case LongToCountTable =>
      assert(keyVars.size == 1)
      val keyVar = keyVars.head
      val countName = context.namer.newVarName()
      generator.assign(typeRef[Int], countName,
                       Expression.invoke(generator.load(tableVar), Methods.countingTableGet, generator.load(keyVar)))
      generator.expression(Expression.invoke(generator.load(tableVar), Methods.countingTablePut, generator.load(keyVar),
                                             Expression.ternary(
                                               Expression.eq(generator.load(countName), Expression
                                                 .get(CodeStructure.staticField[LongKeyIntValueTable, Int]("NULL"))),
                                               Expression.constant(1),
                                               Expression.add(generator.load(countName), Expression.constant(1)))))
    case LongsToCountTable =>
      val countName = context.namer.newVarName()
      val keyName = context.namer.newVarName()
      generator.assign(typeRef[CompositeKey], keyName, Expression.invoke(Methods.compositeKey, keyVars.map(generator.load): _*))
      generator.assign(typeRef[java.lang.Integer], countName, Expression.invoke(generator.load(tableVar), Methods.countingTableCompositeKeyGet, generator.load(keyName)))
      generator.expression(Expression.invoke(generator.load(tableVar), Methods.countingTableCompositeKeyPut,
                                             generator.load(keyName), Expression.ternary(
          Expression.eq(generator.load(countName), Expression.constant(null)),
          Expression.constant(1),
          Expression.add(generator.load(countName), Expression.constant(1)))))
  }

  override def probe(tableVar: String, tableType: JoinTableType, keyVars: Seq[String])(block: MethodStructure[Expression] => Unit) = tableType match {
    case LongToCountTable =>
      assert(keyVars.size == 1)
      val keyVar = keyVars.head
      val times = generator.declare(typeRef[Int], context.namer.newVarName())
      generator.assign(times, Expression.invoke(generator.load(tableVar), Methods.countingTableGet, generator.load(keyVar)))
      using(generator.whileLoop(Expression.gt(times, Expression.constant(0)))) { body =>
        block(copy(generator=body))
        body.assign(times, Expression.sub(times, Expression.constant(1)))
      }
    case LongsToCountTable =>
      val times = generator.declare(typeRef[Int], context.namer.newVarName())
      val intermediate = generator.declare(typeRef[java.lang.Integer], context.namer.newVarName())
      generator.assign(intermediate, Expression.invoke(generator.load(tableVar),
        Methods.countingTableCompositeKeyGet,
        Expression.invoke(Methods.compositeKey, keyVars.map(generator.load): _*)))
      generator.assign(times,
        Expression.ternary(
          Expression.eq(generator.load(intermediate.name()), Expression.constant(null)),
          Expression.constant(-1), generator.load(intermediate.name())))

      using(generator.whileLoop(Expression.gt(times, Expression.constant(0)))) { body =>
        block(copy(generator=body))
        body.assign(times, Expression.sub(times, Expression.constant(1)))
      }

    case LongToListTable(structure,localVars) =>
      assert(keyVars.size == 1)
      val keyVar = keyVars.head
      // compute the participating types
      val valueType = aux.typeReference(structure)
      val listType = TypeReference.parameterizedType(classOf[util.ArrayList[_]], valueType)
      val tableType = TypeReference.parameterizedType(classOf[PrimitiveLongObjectMap[_]], valueType)
      // the methods we use on those types
      val get = MethodReference.methodReference(tableType, listType, "get", typeRef[Long])
      // generate the code
      val list = generator.declare(listType, context.namer.newVarName())
      val elementName = context.namer.newVarName()
      generator.assign(list, Expression.invoke(generator.load(tableVar), get, generator.load(keyVar)))
      using(generator.ifStatement(Expression.not(Expression.eq(list, Expression.constant(null))))) { onTrue =>
        using(onTrue.forEach(Parameter.param(valueType, elementName), list)) { forEach =>
          localVars.foreach {
            case (local, field) =>
              val fieldType = CodeStructure.lowerType(structure(field))
              forEach.assign(fieldType, local, Expression.get(forEach.load(elementName),FieldReference.field(valueType, fieldType, field)))
          }
          block(copy(generator=forEach))
        }
      }

    case LongsToListTable(structure,localVars) =>
      // compute the participating types
      val valueType = aux.typeReference(structure)
      val listType = TypeReference.parameterizedType(classOf[util.ArrayList[_]], valueType)
      val tableType = TypeReference.parameterizedType(classOf[util.HashMap[_, _]], typeRef[CompositeKey], valueType)
      // the methods we use on those types
      val get = MethodReference.methodReference(tableType, listType, "get", typeRef[CompositeKey])
      // generate the code
      val list = generator.declare(listType, context.namer.newVarName())
      val elementName = context.namer.newVarName()

      generator.assign(list, Expression.invoke(generator.load(tableVar), get, Expression.invoke(Methods.compositeKey, keyVars.map(generator.load): _*)))
      using(generator.ifStatement(Expression.not(Expression.eq(list, Expression.constant(null))))) { onTrue =>
        using(onTrue.forEach(Parameter.param(valueType, elementName), list)) { forEach =>
          localVars.foreach {
            case (local, field) =>
              val fieldType = CodeStructure.lowerType(structure(field))
              forEach.assign(fieldType, local, Expression.get(forEach.load(elementName),FieldReference.field(valueType, fieldType, field)))
          }
          block(copy(generator=forEach))
        }
      }
  }

  override def putField(structure: Map[String, CypherType], value: Expression, fieldType: CypherType, fieldName: String, localVar: String) = {
    generator.put(value, field(structure, fieldType, fieldName), generator.load(localVar))
  }

  override def updateProbeTable(structure: Map[String, CypherType], tableVar: String, tableType: RecordingJoinTableType, keyVars: Seq[String], element: Expression) = tableType match {
    case _: LongToListTable =>
      assert(keyVars.size == 1)
      val keyVar = keyVars.head
      // compute the participating types
      val valueType = aux.typeReference(structure)
      val listType = TypeReference.parameterizedType(classOf[util.ArrayList[_]], valueType)
      val tableType = TypeReference.parameterizedType(classOf[PrimitiveLongObjectMap[_]], valueType)
      // the methods we use on those types
      val get = MethodReference.methodReference(tableType, listType, "get", typeRef[Long])
      val put = MethodReference.methodReference(tableType, listType, "put", typeRef[Long], listType)
      val add = MethodReference.methodReference(listType, typeRef[Boolean], "add", valueType)
      // generate the code
      val listName = context.namer.newVarName()
      val list = generator.declare(listType, listName) // ProbeTable list;
      generator.assign(list, Expression
        .invoke(generator.load(tableVar), get, generator.load(keyVar))) // list = tableVar.get(keyVar);
      using(generator.ifStatement(Expression.eq(Expression.constant(null), generator.load(listName))))
      { onTrue => // if (null == list)
        onTrue.assign(list, Templates.newInstance(listType)) // list = new ListType();
        onTrue.expression(Expression.invoke(generator.load(tableVar), put, generator.load(keyVar),
                                            generator.load(listName))) // tableVar.put(keyVar, list);
      }
      generator.expression(Expression.invoke(list, add, element)) // list.add( element );

    case _: LongsToListTable =>
      // compute the participating types
      val valueType = aux.typeReference(structure)
      val listType = TypeReference.parameterizedType(classOf[util.ArrayList[_]], valueType)
      val tableType = TypeReference.parameterizedType(classOf[util.HashMap[_, _]], typeRef[CompositeKey], valueType)
      // the methods we use on those types
      val get = MethodReference.methodReference(tableType, listType, "get", typeRef[CompositeKey])
      val put = MethodReference.methodReference(tableType, listType, "put", typeRef[CompositeKey], listType)
      val add = MethodReference.methodReference(listType, typeRef[Boolean], "add", valueType)
      // generate the code
      val listName = context.namer.newVarName()
      val keyName = context.namer.newVarName()
      val list = generator.declare(listType, listName) // ProbeTable list;
      generator
        .assign(typeRef[CompositeKey], keyName, Expression.invoke(Methods.compositeKey, keyVars.map(generator.load): _*))
      generator.assign(list, Expression
        .invoke(generator.load(tableVar), get, generator.load(keyName))) // list = tableVar.get(keyVar);
      using(generator.ifStatement(Expression.eq(Expression.constant(null), generator.load(listName))))
      { onTrue => // if (null == list)
        onTrue.assign(list, Templates.newInstance(listType)) // list = new ListType();
        onTrue.expression(Expression.invoke(generator.load(tableVar), put, generator.load(keyName),
                                            generator.load(listName))) // tableVar.put(keyVar, list);
      }
      generator.expression(Expression.invoke(list, add, element)) // list.add( element );


  }

  override def declareProperty(propertyVar: String) = {
    locals = locals + (propertyVar -> generator.declare(typeRef[Object], propertyVar))
  }

  override def hasLabel(nodeVar: String, labelVar: String, predVar: String) =  {<commit_msg>Minor simplification of ListHashTable<commit_after>/*
 * Copyright (c) 2002-2015 "Neo Technology,"
 * Network Engine for Objects in Lund AB [http://neotechnology.com]
 *
 * This file is part of Neo4j.
 *
 * Neo4j is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

package org.neo4j.cypher.internal.compiler.v2_3.codegen

import java.util

import org.neo4j.codegen
import org.neo4j.codegen.CodeGeneratorOption.BLANK_OPTION
import org.neo4j.codegen.ExpressionTemplate._
import org.neo4j.codegen.MethodReference._
import org.neo4j.codegen.TypeReference.{extending, parameterizedType, typeParameter}
import org.neo4j.codegen._
import org.neo4j.codegen.source.SourceVisitor
import org.neo4j.collection.primitive.hopscotch.LongKeyIntValueTable
import org.neo4j.collection.primitive.{Primitive, PrimitiveLongIntMap, PrimitiveLongIterator, PrimitiveLongObjectMap}
import org.neo4j.cypher.internal.compiler.v2_3._
import org.neo4j.cypher.internal.compiler.v2_3.codegen.CompiledConversionUtils.CompositeKey
import org.neo4j.cypher.internal.compiler.v2_3.executionplan.{GeneratedQuery, GeneratedQueryExecution, SuccessfulCloseable}
import org.neo4j.cypher.internal.compiler.v2_3.helpers.using
import org.neo4j.cypher.internal.compiler.v2_3.planDescription.{Id, InternalPlanDescription}
import org.neo4j.cypher.internal.compiler.v2_3.planner.CantCompileQueryException
import org.neo4j.cypher.internal.compiler.v2_3.symbols.CypherType
import org.neo4j.function.Supplier
import org.neo4j.graphdb.Result.{ResultRow, ResultVisitor}
import org.neo4j.graphdb.{Direction, GraphDatabaseService, Node, Relationship}
import org.neo4j.helpers.collection.MapUtil
import org.neo4j.kernel.api.exceptions.KernelException
import org.neo4j.kernel.api.index.IndexDescriptor
import org.neo4j.kernel.api.properties.Property
import org.neo4j.kernel.api.{ReadOperations, Statement, StatementTokenNameLookup, TokenNameLookup}
import org.neo4j.kernel.impl.api.store.RelationshipIterator
import org.neo4j.kernel.impl.api.{RelationshipDataExtractor, RelationshipVisitor}

import scala.collection.mutable

// <SPI>
trait CodeStructure[T] {
  type SourceSink = (String, String) => Unit
  def generateQuery(packageName: String, className: String, columns: Seq[String], operatorIds: Map[String, Id], sourceSink: Option[SourceSink])
                   (block: MethodStructure[_] => Unit)(implicit codeGenContext: CodeGenContext): T
}

sealed trait JoinTableType
sealed trait CountingJoinTableType extends JoinTableType
sealed trait RecordingJoinTableType extends JoinTableType

case object LongToCountTable extends CountingJoinTableType
case object LongsToCountTable extends CountingJoinTableType
case class LongToListTable(structure: Map[String, CypherType], localMap: Map[String, String]) extends RecordingJoinTableType
case class LongsToListTable(structure: Map[String, CypherType], localMap: Map[String, String]) extends RecordingJoinTableType

trait MethodStructure[E] {

  // misc
  def declareFlag(name: String, initialValue: Boolean)
  def updateFlag(name: String, newValue: Boolean)
  def declarePredicate(name: String): Unit
  def declareProperty(name: String): Unit
  def declareCounter(name: String, initialValue: E): Unit
  def putField(structure: Map[String, CypherType], value: E, fieldType: CypherType, fieldName: String, localVar: String): Unit
  def updateProbeTable(structure: Map[String, CypherType], tableVar: String, tableType: RecordingJoinTableType, keyVars: Seq[String], element: E): Unit
  def probe(tableVar: String, tableType: JoinTableType, keyVars: Seq[String])(block: MethodStructure[E]=>Unit): Unit
  def updateProbeTableCount(tableVar: String, tableType: CountingJoinTableType, keyVar: Seq[String]): Unit
  def allocateProbeTable(tableVar: String, tableType: JoinTableType): Unit
  def method(resultType: JoinTableType, resultVar: String, methodName: String)(block: MethodStructure[E]=>Unit): Unit

  // expressions
  def decreaseCounterAndCheckForZero(name: String): E
  def counterEqualsZero(variableName: String): E
  def newTableValue(targetVar: String, structure: Map[String, CypherType]): E
  def constant(value: Object): E
  def asMap(map: Map[String, E]): E
  def asList(values: Seq[E]): E

  def toSet(value: E): E

  def castToCollection(value: E): E

  def load(varName: String): E

  // arithmetic
  def add(lhs: E, rhs: E): E
  def sub(lhs: E, rhs: E): E
  def mul(lhs: E, rhs: E): E
  def div(lhs: E, rhs: E): E
  def mod(lhs: E, rhs: E): E

  // predicates
  def not(value: E): E

  // null handling
  def markAsNull(varName: String, cypherType: CypherType): Unit
  def nullable(varName: String, cypherType: CypherType, onSuccess: E): E

  // parameters
  def expectParameter(key: String): Unit
  def parameter(key: String): E

  // tracing
  def trace[V](planStepId: String)(block: MethodStructure[E] => V): V
  def incrementDbHits(): Unit
  def incrementRows(): Unit

  // db access
  def labelScan(iterVar: String, labelIdVar: String): Unit
  def hasLabel(nodeVar: String, labelVar: String, predVar: String): E
  def allNodesScan(iterVar: String): Unit
  def lookupLabelId(labelIdVar: String, labelName: String): Unit
  def lookupRelationshipTypeId(typeIdVar: String, typeName: String): Unit
  def nodeGetAllRelationships(iterVar: String, nodeVar: String, direction: Direction): Unit
  def nodeGetRelationships(iterVar: String, nodeVar: String, direction: Direction, typeVars: Seq[String]): Unit
  def connectingRelationships(iterVar: String, fromNode: String, dir: Direction, toNode:String)
  def connectingRelationships(iterVar: String, fromNode: String, dir: Direction, types: Seq[String], toNode: String)
  def nextNode(targetVar: String, iterVar: String): Unit
  def nextRelationshipAndNode(toNodeVar: String, iterVar: String, direction: Direction, fromNodeVar: String, relVar: String): Unit
  def nextRelationship(iterVar: String, direction: Direction, relVar: String): Unit
  def hasNext(iterVar: String): E
  def nodeGetPropertyById(nodeIdVar: String, propId: Int, propValueVar: String): Unit
  def nodeGetPropertyForVar(nodeIdVar: String, propIdVar: String, propValueVar: String): Unit
  def relationshipGetPropertyById(nodeIdVar: String, propId: Int, propValueVar: String): Unit
  def relationshipGetPropertyForVar(nodeIdVar: String, propIdVar: String, propValueVar: String): Unit
  def lookupPropertyKey(propName: String, propVar: String)
  def coerceToBoolean(propertyExpression: E): E

  def newIndexDescriptor(descriptorVar: String, labelVar: String, propKeyVar: String): Unit

  def indexSeek(iterVar: String, descriptorVar: String, value: E): Unit

  // code structure
  def whileLoop(test: E)(block: MethodStructure[E] => Unit): Unit
  def forEach(varName: String, cypherType: CypherType, iterable: E)(block: MethodStructure[E] => Unit): Unit
  def ifStatement(test: E)(block: MethodStructure[E] => Unit): Unit
  def returnSuccessfully(): Unit

  // results
  def materializeNode(nodeIdVar: String): E
  def materializeRelationship(relIdVar: String): E
  def visitRow(): Unit
  def setInRow(column: String, value: E): Unit
}

// </SPI>

// TODO: the rest of this file is implementation of the SPI above, and should move to cypher/cypher

object CodeStructure {

  val __TODO__MOVE_IMPLEMENTATION = new CodeStructure[GeneratedQuery] {

    override def generateQuery(packageName: String, className: String, columns: Seq[String], operatorIds: Map[String, Id], sourceSink: Option[SourceSink])
                              (block: MethodStructure[_] => Unit)(implicit codeGenContext: CodeGenContext) = {
      val generator = try {
        codegen.CodeGenerator.generateCode(CodeStructure.getClass.getClassLoader, sourceSink.map(sink => new SourceVisitor {
          override protected def visitSource(reference: TypeReference, sourceCode: CharSequence): Unit =
            sink.apply(reference.name(), sourceCode.toString)
        }).getOrElse(BLANK_OPTION))
      } catch {
        case e: Exception => throw new CantCompileQueryException(e.getMessage, e)
      }
      val execution = using(generator.generateClass(packageName, className+"Execution", typeRef[GeneratedQueryExecution], typeRef[SuccessfulCloseable])) { clazz =>
        // fields
        val fields = Fields(
          closer = clazz.field(typeRef[TaskCloser], "closer"),
          ro = clazz.field(typeRef[ReadOperations], "ro"),
          db = clazz.field(typeRef[GraphDatabaseService], "db"),
          executionMode = clazz.field(typeRef[ExecutionMode], "executionMode"),
          description = clazz.field(typeRef[Supplier[InternalPlanDescription]], "description"),
          tracer = clazz.field(typeRef[QueryExecutionTracer], "tracer"),
          params = clazz.field(typeRef[util.Map[String, Object]], "params"),
          closeable = clazz.field(typeRef[SuccessfulCloseable], "closeable"),
          success = clazz.generate(Templates.SUCCESS))
        // the "COLUMNS" static field
        clazz.staticField(typeRef[util.List[String]], "COLUMNS", Templates.asList(
          columns.toSeq.map(key => Expression.constant(key))))

        // the operator id fields
        operatorIds.keys.foreach { opId =>
          clazz.staticField(typeRef[Id], opId)
        }

        // simple methods
        clazz.generate(Templates.CONSTRUCTOR)
        clazz.generate(Templates.SET_SUCCESSFUL_CLOSEABLE)
        val close = clazz.generate(Templates.CLOSE)
        clazz.generate(Templates.EXECUTION_MODE)
        clazz.generate(Templates.EXECUTION_PLAN_DESCRIPTION)
        clazz.generate(Templates.JAVA_COLUMNS)

        using(clazz.generate(MethodDeclaration.method(typeRef[Unit], "accept",
          Parameter.param(parameterizedType(classOf[ResultVisitor[_]], typeParameter("E")), "visitor")).
          parameterizedWith("E", extending(typeRef[Exception])).
          throwsException(typeParameter("E")))) { method =>
          using(method.tryBlock()) { body =>
            body.assign(typeRef[ResultRowImpl], "row", Templates.newResultRow)
            body.assign(typeRef[RelationshipDataExtractor], "rel", Templates.newRelationshipDataExtractor)
            block(Method(fields, body, new AuxGenerator(packageName, generator)))
            body.expression(Expression.invoke(body.self(), fields.success))
            using(body.finallyBlock()) { then =>
              then.expression(Expression.invoke(body.self(), close))
            }
          }
        }
        clazz.handle()
      }
      val query = using(generator.generateClass(packageName,className, typeRef[GeneratedQuery])) { clazz =>
        using(clazz.generateMethod(typeRef[GeneratedQueryExecution], "execute",
          param[TaskCloser]("closer"),
          param[Statement]("statement"),
          param[GraphDatabaseService]("db"),
          param[ExecutionMode]("executionMode"),
          param[Supplier[InternalPlanDescription]]("description"),
          param[QueryExecutionTracer]("tracer"),
          param[util.Map[String, Object]]("params"))) { execute =>
          execute.returns(Expression.invoke(Expression.newInstance(execution), MethodReference.constructorReference(execution,
            typeRef[TaskCloser],
            typeRef[Statement],
            typeRef[GraphDatabaseService],
            typeRef[ExecutionMode],
            typeRef[Supplier[InternalPlanDescription]],
            typeRef[QueryExecutionTracer],
            typeRef[util.Map[String, Object]]),
            execute.load("closer"),
            execute.load("statement"),
            execute.load("db"),
            execute.load("executionMode"),
            execute.load("description"),
            execute.load("tracer"),
            execute.load("params")))
        }
        clazz.handle()
      }.newInstance().asInstanceOf[GeneratedQuery]
      val clazz: Class[_] = execution.loadClass()
      operatorIds.foreach {
        case (key, id) => setStaticField(clazz, key, id)
      }
      query
    }
  }

  def method[O <: AnyRef, R](name: String, params: TypeReference*)(implicit owner: Manifest[O], returns: Manifest[R]): MethodReference =
    MethodReference.methodReference(typeReference(owner), typeReference(returns), name, params: _*)

  def staticField[O <: AnyRef, R](name: String)(implicit owner: Manifest[O], fieldType: Manifest[R]): FieldReference =
    FieldReference.staticField(typeReference(owner), typeReference(fieldType), name)

  def param[T <: AnyRef](name: String)(implicit manifest: Manifest[T]): Parameter =
    Parameter.param(typeReference(manifest), name)

  def typeRef[T](implicit manifest: Manifest[T]): TypeReference = typeReference(manifest)

  def typeReference(manifest: Manifest[_]): TypeReference = {
    val arguments = manifest.typeArguments
    val base = TypeReference.typeReference(manifest.runtimeClass)
    if(arguments.nonEmpty) {
      TypeReference.parameterizedType(base, arguments.map(typeReference): _*)
    } else {
      base
    }
  }

  def lowerType(cType:CypherType):TypeReference = cType match {
    case symbols.CTNode => typeRef[Long]
    case symbols.CTRelationship => typeRef[Long]
    case symbols.CTAny => typeRef[Object]
  }

  def nullValue(cType: CypherType) = cType match {
    case symbols.CTNode => Expression.constant(-1L)
    case symbols.CTRelationship => Expression.constant(-1L)
    case symbols.CTAny => Expression.constant(null)
  }
}

private case class Fields(closer: FieldReference,
                          ro: FieldReference,
                          db: FieldReference,
                          executionMode: FieldReference,
                          description: FieldReference,
                          tracer: FieldReference,
                          params: FieldReference,
                          closeable: FieldReference,
                          success: MethodReference)

private class AuxGenerator(val packageName: String, val generator: codegen.CodeGenerator) {
  private val types: mutable.Map[Map[String, CypherType], TypeReference] = mutable.Map.empty
  private var nameId = 0

  def typeReference(structure: Map[String, CypherType]): TypeReference = {
    types.getOrElseUpdate(structure, using(generator.generateClass(packageName, newName())) { clazz =>
      structure.foreach {
        case (fieldName, fieldType) => clazz.field(CodeStructure.lowerType(fieldType), fieldName)
      }
      clazz.handle()
    })
  }
  private def newName() = {
    val name = "ValueType" + nameId
    nameId += 1
    name
  }
}

private case class Method(fields: Fields, generator: CodeBlock, aux:AuxGenerator, tracing: Boolean = true,
                          event: Option[String] = None, var locals:Map[String,LocalVariable]=Map.empty)(implicit context: CodeGenContext)
  extends MethodStructure[Expression] {
  import CodeStructure.typeRef

  private case class HashTable(valueType: TypeReference, listType: TypeReference, tableType: TypeReference,
                               get: MethodReference, put: MethodReference, add: MethodReference)
  private implicit class RichTableType(tableType: RecordingJoinTableType) {

    def extractHashTable(): HashTable = tableType match {
      case LongToListTable(structure, localMap) =>
        // compute the participating types
        val valueType = aux.typeReference(structure)
        val listType = TypeReference.parameterizedType(classOf[util.ArrayList[_]], valueType)
        val tableType = TypeReference.parameterizedType(classOf[PrimitiveLongObjectMap[_]], valueType)
        // the methods we use on those types
        val get = MethodReference.methodReference(tableType, listType, "get", typeRef[Long])
        val put = MethodReference.methodReference(tableType, listType, "put", typeRef[Long], listType)
        val add = MethodReference.methodReference(listType, typeRef[Boolean], "add", valueType)

        HashTable(valueType, listType, tableType, get, put, add)

      case LongsToListTable(structure, localMap) =>
        // compute the participating types
        val valueType = aux.typeReference(structure)
        val listType = TypeReference.parameterizedType(classOf[util.ArrayList[_]], valueType)
        val tableType = TypeReference.parameterizedType(classOf[util.HashMap[_, _]], typeRef[CompositeKey], valueType)
        // the methods we use on those types
        val get = MethodReference.methodReference(tableType, listType, "get", typeRef[CompositeKey])
        val put = MethodReference.methodReference(tableType, listType, "put", typeRef[CompositeKey], listType)
        val add = MethodReference.methodReference(listType, typeRef[Boolean], "add", valueType)
        HashTable(valueType, listType, tableType, get, put, add)
    }
  }

  override def nextNode(targetVar: String, iterVar: String) =
    generator.assign(typeRef[Long], targetVar, Expression.invoke(generator.load(iterVar), Methods.nextLong))

  override def nextRelationshipAndNode(toNodeVar: String, iterVar: String, direction: Direction, fromNodeVar: String,
                                    relVar: String) = {
    val startNode = Expression.invoke(generator.load("rel"), Methods.startNode)
    val endNode = Expression.invoke(generator.load("rel"), Methods.endNode)
    generator.expression(Expression.invoke(generator.load(iterVar), Methods.relationshipVisit,
                                           Expression.invoke(generator.load(iterVar), Methods.nextLong),
                                           generator.load("rel")))
    generator.assign(typeRef[Long], toNodeVar, direction match {
      case Direction.INCOMING => startNode
      case Direction.OUTGOING => endNode
      case Direction.BOTH => Expression.ternary(Expression.eq(startNode, generator.load(fromNodeVar)), endNode, startNode)
    })
    generator.assign(typeRef[Long], relVar, Expression.invoke(generator.load("rel"), Methods.relationship))
  }

  override def nextRelationship(iterVar: String, direction: Direction, relVar: String) = {
    generator.expression(Expression.invoke(generator.load(iterVar), Methods.relationshipVisit,
                                           Expression.invoke(generator.load(iterVar), Methods.nextLong),
                                           generator.load("rel")))
    generator.assign(typeRef[Long], relVar, Expression.invoke(generator.load("rel"), Methods.relationship))
  }

  override def allNodesScan(iterVar: String) =
    generator.assign(typeRef[PrimitiveLongIterator], iterVar, Expression.invoke(readOperations, Methods.nodesGetAll))

  override def labelScan(iterVar: String, labelIdVar: String) =
    generator.assign(typeRef[PrimitiveLongIterator], iterVar,
                     Expression.invoke(readOperations, Methods.nodesGetForLabel, generator.load(labelIdVar)))


  override def lookupLabelId(labelIdVar: String, labelName: String) =
    generator.assign(typeRef[Int], labelIdVar,
                     Expression.invoke(readOperations, Methods.labelGetForName, Expression.constant(labelName)))

  override def lookupRelationshipTypeId(typeIdVar: String, typeName: String) =
    generator.assign(typeRef[Int], typeIdVar, Expression
      .invoke(readOperations, Methods.relationshipTypeGetForName, Expression.constant(typeName)))

  override def hasNext(iterVar: String) =
    Expression.invoke(generator.load(iterVar), Methods.hasNext)

  override def whileLoop(test: Expression)(block: MethodStructure[Expression] => Unit) =
    using(generator.whileLoop(test)) { body =>
      block(copy(generator = body))
    }

  override def forEach(varName: String, cypherType: CypherType, iterable: Expression)(block: MethodStructure[Expression] => Unit) =
    using(generator.forEach(Parameter.param(CodeStructure.lowerType(cypherType), varName), iterable)) { body =>
      block(copy(generator = body))
    }

  override def ifStatement(test: Expression)(block: (MethodStructure[Expression]) => Unit) = {
    using(generator.ifStatement(test)) { body =>
      block(copy(generator = body))
    }
  }

  override def returnSuccessfully() {
    generator.expression(Expression.invoke(generator.self(), fields.success))
    generator.returns()
  }

  override def declareCounter(name: String, initialValue: Expression): Unit = {
    val variable = generator.declare(typeRef[Int], name)
    locals = locals + (name -> variable)
    generator.assign(variable, Expression.invoke(Methods.mathCastToInt, initialValue))
  }

  override def decreaseCounterAndCheckForZero(name: String): Expression = {
    val local = locals(name)
    generator.assign(local, Expression.sub(local, Expression.constant(1)))
    Expression.eq(Expression.constant(0), local)
  }

  override def counterEqualsZero(name: String): Expression = {
    val local = locals(name)
    Expression.eq(Expression.constant(0), local)
  }

  override def setInRow(column: String, value: Expression) =
    generator.expression(Expression.invoke(resultRow, Methods.set, Expression.constant(column), value))

  override def visitRow() = using(generator.ifStatement(Expression.not(
    Expression.invoke(generator.load("visitor"), Methods.visit, generator.load("row"))))) { body =>
    // NOTE: we are in this if-block if the visitor decided to terminate early (by returning false)
    body.expression(Expression.invoke(body.self(), fields.success))
    body.returns()
  }

  override def materializeNode(nodeIdVar: String) = Expression.invoke(db, Methods.getNodeById, generator.load(nodeIdVar))

  override def nullable(varName: String, cypherType: CypherType, onSuccess: Expression) = {
    Expression.ternary(
      Expression.eq(CodeStructure.nullValue(cypherType), generator.load(varName)),
      Expression.constant(null),
      onSuccess)
  }

  override def materializeRelationship(relIdVar: String) = Expression.invoke(db, Methods.getRelationshipById, generator.load(relIdVar))

  override def trace[V](planStepId: String)(block: MethodStructure[Expression]=>V) = if(!tracing) block(this)
  else {
    val eventName = s"event_$planStepId"
    using(generator.tryBlock(typeRef[QueryExecutionEvent], eventName, traceEvent(planStepId))) { body =>
      block(copy(event = Some(eventName),
        generator = body))
    }
  }

  private def traceEvent(planStepId: String) = Expression.invoke(tracer, Methods.executeOperator,
    Expression.get(FieldReference.staticField(generator.owner(), typeRef[Id], planStepId)))

  override def incrementDbHits() = if(tracing) generator.expression(Expression.invoke(loadEvent, Methods.dbHit))

  override def incrementRows() = if(tracing) generator.expression(Expression.invoke(loadEvent, Methods.row))

  private def loadEvent = generator.load(event.getOrElse(throw new IllegalStateException("no current trace event")))

  override def expectParameter(key: String) =
    using(generator.ifStatement(Expression.not(Expression.invoke(params, Methods.mapContains, Expression.constant(key))))) { block =>
      block.throwException(parameterNotFoundException(key))
    }

  override def parameter(key: String) = Expression.invoke(params, Methods.mapGet, Expression.constant(key))

  override def constant(value: Object) = Expression.constant(value)

  override def not(value: Expression): Expression = Expression.not(value)

  override def markAsNull(varName: String, cypherType: CypherType) =
    generator.assign(CodeStructure.lowerType(cypherType), varName, CodeStructure.nullValue(cypherType))

  override def nodeGetAllRelationships(iterVar: String, nodeVar: String, direction: Direction) = {
    val local = generator.declare(typeRef[RelationshipIterator], iterVar)
    Templates.handleExceptions(generator, fields.ro) { body =>
      body.assign(local, Expression.invoke(readOperations, Methods.nodeGetAllRelationships, body.load(nodeVar), dir(direction)))
    }
  }

  override def nodeGetRelationships(iterVar: String, nodeVar: String, direction: Direction, typeVars: Seq[String]) = {
    val local = generator.declare(typeRef[RelationshipIterator], iterVar)
    Templates.handleExceptions(generator, fields.ro) { body =>
      val args = Seq(body.load(nodeVar), dir(direction)) ++ typeVars.map(body.load)
      body.assign(local, Expression.invoke(readOperations, Methods.nodeGetRelationships, args: _*))
    }
  }

  override def connectingRelationships(iterVar: String, fromNode: String, direction: Direction, toNode: String) = {
    val local = generator.declare(typeRef[RelationshipIterator], iterVar)
    Templates.handleExceptions(generator, fields.ro) { body =>
      body.assign(local, Expression.invoke(Methods.allConnectingRelationships, readOperations, body.load(fromNode), dir(direction), body.load(toNode)))
    }
  }

  override def connectingRelationships(iterVar: String, fromNode: String, direction: Direction, typeVars: Seq[String], toNode: String) = {
    val local = generator.declare(typeRef[RelationshipIterator], iterVar)
    Templates.handleExceptions(generator, fields.ro) { body =>
      val args = Seq(readOperations, body.load(fromNode), dir(direction),  body.load(toNode)) ++ typeVars.map(body.load)
      body.assign(local, Expression.invoke(Methods.connectingRelationships, args:_*))
    }
  }

  override def load(varName: String) = generator.load(varName)

  override def add(lhs: Expression, rhs: Expression) = math(Methods.mathAdd, lhs, rhs)

  override def sub(lhs: Expression, rhs: Expression) = math(Methods.mathSub, lhs, rhs)

  override def mul(lhs: Expression, rhs: Expression) = math(Methods.mathMul, lhs, rhs)

  override def div(lhs: Expression, rhs: Expression) = math(Methods.mathDiv, lhs, rhs)

  override def mod(lhs: Expression, rhs: Expression) = math(Methods.mathMod, lhs, rhs)

  private def math(method: MethodReference, lhs: Expression, rhs: Expression): Expression =
    // TODO: generate specialized versions for specific types
    Expression.invoke(method, lhs, rhs)

  private def readOperations = Expression.get(generator.self(), fields.ro)

  private def db = Expression.get(generator.self(), fields.db)

  private def resultRow = generator.load("row")

  private def tracer = Expression.get(generator.self(), fields.tracer)

  private def params = Expression.get(generator.self(), fields.params)

  private def parameterNotFoundException(key: String) =
    Expression.invoke(Expression.newInstance(typeRef[ParameterNotFoundException]),
      MethodReference.constructorReference(typeRef[ParameterNotFoundException], typeRef[String]),
      Expression.constant(s"Expected a parameter named $key"))

  private def dir(dir: Direction): Expression = dir match {
    case Direction.INCOMING => Templates.incoming
    case Direction.OUTGOING => Templates.outgoing
    case Direction.BOTH => Templates.both
  }

  override def asList(values: Seq[Expression]) = Templates.asList(values)

  override def toSet(value: Expression) =
    Templates.newInstance(typeRef[util.HashSet[Object]], value)

  override def castToCollection(value: Expression) = Expression.invoke(Methods.toCollection, value)

  override def asMap(map: Map[String, Expression]) = {
    Expression.invoke(Methods.arrayAsList, map.flatMap {
      case (key, value) => Seq(Expression.constant(key), value)
    }.toSeq: _*)
  }

  override def method(resultType: JoinTableType, resultVar: String, methodName: String)(block: MethodStructure[Expression]=>Unit) = {
    val returnType: TypeReference = joinTableType(resultType)
    generator.assign(returnType, resultVar, Expression.invoke(generator.self(),MethodReference.methodReference(generator.owner(),returnType, methodName)))
    using(generator.classGenerator().generateMethod(returnType, methodName)) { body =>
      body.assign(typeRef[RelationshipDataExtractor], "rel", Templates.newRelationshipDataExtractor)
      block(copy(generator = body, event = None))
      body.returns(body.load(resultVar))
    }
  }

  override def allocateProbeTable(tableVar: String, tableType: JoinTableType) =
    generator.assign(joinTableType(tableType), tableVar, allocate(tableType))


  private def joinTableType(resultType: JoinTableType): TypeReference = {
    val returnType = resultType match {
      case LongToCountTable => typeRef[PrimitiveLongIntMap]
      case LongsToCountTable => TypeReference.parameterizedType(classOf[util.HashMap[_, _]], classOf[CompositeKey], classOf[java.lang.Integer])
      case LongToListTable(structure,_) => TypeReference.parameterizedType(classOf[PrimitiveLongObjectMap[_]],
        TypeReference.parameterizedType(classOf[util.ArrayList[_]], aux.typeReference(structure)))
      case LongsToListTable(structure,_) => TypeReference.parameterizedType(classOf[util.HashMap[_,_]], typeRef[CompositeKey],
         TypeReference.parameterizedType(classOf[util.ArrayList[_]], aux.typeReference(structure)))
    }
    returnType
  }

  private def allocate(resultType: JoinTableType): Expression = resultType match {
    case LongToCountTable => Templates.newCountingMap
    case LongToListTable(_,_) => Templates.newLongObjectMap
    case LongsToCountTable => Templates.newInstance(joinTableType(LongsToCountTable))
    case typ: LongsToListTable => Templates.newInstance(joinTableType(typ))
  }

  override def updateProbeTableCount(tableVar: String, tableType: CountingJoinTableType,
                                     keyVars: Seq[String]) = tableType match {
    case LongToCountTable =>
      assert(keyVars.size == 1)
      val keyVar = keyVars.head
      val countName = context.namer.newVarName()
      generator.assign(typeRef[Int], countName,
                       Expression.invoke(generator.load(tableVar), Methods.countingTableGet, generator.load(keyVar)))
      generator.expression(Expression.invoke(generator.load(tableVar), Methods.countingTablePut, generator.load(keyVar),
                                             Expression.ternary(
                                               Expression.eq(generator.load(countName), Expression
                                                 .get(CodeStructure.staticField[LongKeyIntValueTable, Int]("NULL"))),
                                               Expression.constant(1),
                                               Expression.add(generator.load(countName), Expression.constant(1)))))
    case LongsToCountTable =>
      val countName = context.namer.newVarName()
      val keyName = context.namer.newVarName()
      generator.assign(typeRef[CompositeKey], keyName, Expression.invoke(Methods.compositeKey, keyVars.map(generator.load): _*))
      generator.assign(typeRef[java.lang.Integer], countName, Expression.invoke(generator.load(tableVar), Methods.countingTableCompositeKeyGet, generator.load(keyName)))
      generator.expression(Expression.invoke(generator.load(tableVar), Methods.countingTableCompositeKeyPut,
                                             generator.load(keyName), Expression.ternary(
          Expression.eq(generator.load(countName), Expression.constant(null)),
          Expression.constant(1),
          Expression.add(generator.load(countName), Expression.constant(1)))))
  }

  override def probe(tableVar: String, tableType: JoinTableType, keyVars: Seq[String])(block: MethodStructure[Expression] => Unit) = tableType match {
    case LongToCountTable =>
      assert(keyVars.size == 1)
      val keyVar = keyVars.head
      val times = generator.declare(typeRef[Int], context.namer.newVarName())
      generator.assign(times, Expression.invoke(generator.load(tableVar), Methods.countingTableGet, generator.load(keyVar)))
      using(generator.whileLoop(Expression.gt(times, Expression.constant(0)))) { body =>
        block(copy(generator=body))
        body.assign(times, Expression.sub(times, Expression.constant(1)))
      }
    case LongsToCountTable =>
      val times = generator.declare(typeRef[Int], context.namer.newVarName())
      val intermediate = generator.declare(typeRef[java.lang.Integer], context.namer.newVarName())
      generator.assign(intermediate, Expression.invoke(generator.load(tableVar),
        Methods.countingTableCompositeKeyGet,
        Expression.invoke(Methods.compositeKey, keyVars.map(generator.load): _*)))
      generator.assign(times,
        Expression.ternary(
          Expression.eq(generator.load(intermediate.name()), Expression.constant(null)),
          Expression.constant(-1), generator.load(intermediate.name())))

      using(generator.whileLoop(Expression.gt(times, Expression.constant(0)))) { body =>
        block(copy(generator=body))
        body.assign(times, Expression.sub(times, Expression.constant(1)))
      }

    case tableType@LongToListTable(structure,localVars) =>
      assert(keyVars.size == 1)
      val keyVar = keyVars.head

      val hashTable = tableType.extractHashTable()
      // generate the code
      val list = generator.declare(hashTable.listType, context.namer.newVarName())
      val elementName = context.namer.newVarName()
      generator.assign(list, Expression.invoke(generator.load(tableVar), hashTable.get, generator.load(keyVar)))
      using(generator.ifStatement(Expression.not(Expression.eq(list, Expression.constant(null))))) { onTrue =>
        using(onTrue.forEach(Parameter.param(hashTable.valueType, elementName), list)) { forEach =>
          localVars.foreach {
            case (local, field) =>
              val fieldType = CodeStructure.lowerType(structure(field))
              forEach.assign(fieldType, local, Expression.get(forEach.load(elementName),FieldReference.field(hashTable.valueType, fieldType, field)))
          }
          block(copy(generator=forEach))
        }
      }

    case tableType@LongsToListTable(structure,localVars) =>
      val hashTable = tableType.extractHashTable()
      val list = generator.declare(hashTable.listType, context.namer.newVarName())
      val elementName = context.namer.newVarName()

      generator.assign(list, Expression.invoke(generator.load(tableVar),hashTable.get, Expression.invoke(Methods.compositeKey, keyVars.map(generator.load): _*)))
      using(generator.ifStatement(Expression.not(Expression.eq(list, Expression.constant(null))))) { onTrue =>
        using(onTrue.forEach(Parameter.param(hashTable.valueType, elementName), list)) { forEach =>
          localVars.foreach {
            case (local, field) =>
              val fieldType = CodeStructure.lowerType(structure(field))
              forEach.assign(fieldType, local, Expression.get(forEach.load(elementName),FieldReference.field(hashTable.valueType, fieldType, field)))
          }
          block(copy(generator=forEach))
        }
      }
  }



  override def putField(structure: Map[String, CypherType], value: Expression, fieldType: CypherType, fieldName: String, localVar: String) = {
    generator.put(value, field(structure, fieldType, fieldName), generator.load(localVar))
  }

  override def updateProbeTable(structure: Map[String, CypherType], tableVar: String, tableType: RecordingJoinTableType, keyVars: Seq[String], element: Expression) = tableType match {
    case _: LongToListTable =>
      assert(keyVars.size == 1)
      val keyVar = keyVars.head
      val hashTable = tableType.extractHashTable()
      // generate the code
      val listName = context.namer.newVarName()
      val list = generator.declare(hashTable.listType, listName) // ProbeTable list;
      generator.assign(list, Expression
        .invoke(generator.load(tableVar), hashTable.get, generator.load(keyVar))) // list = tableVar.get(keyVar);
      using(generator.ifStatement(Expression.eq(Expression.constant(null), generator.load(listName))))
      { onTrue => // if (null == list)
        onTrue.assign(list, Templates.newInstance(hashTable.listType)) // list = new ListType();
        onTrue.expression(Expression.invoke(generator.load(tableVar), hashTable.put, generator.load(keyVar),
                                            generator.load(listName))) // tableVar.put(keyVar, list);
      }
      generator.expression(Expression.invoke(list, hashTable.add, element)) // list.add( element );

    case _: LongsToListTable =>
      val hashTable = tableType.extractHashTable()
      // generate the code
      val listName = context.namer.newVarName()
      val keyName = context.namer.newVarName()
      val list = generator.declare(hashTable.listType, listName) // ProbeTable list;
      generator
        .assign(typeRef[CompositeKey], keyName, Expression.invoke(Methods.compositeKey, keyVars.map(generator.load): _*))
      generator.assign(list, Expression
        .invoke(generator.load(tableVar), hashTable.get, generator.load(keyName))) // list = tableVar.get(keyVar);
      using(generator.ifStatement(Expression.eq(Expression.constant(null), generator.load(listName))))
      { onTrue => // if (null == list)
        onTrue.assign(list, Templates.newInstance(hashTable.listType)) // list = new ListType();
        onTrue.expression(Expression.invoke(generator.load(tableVar), hashTable.put, generator.load(keyName),
                                            generator.load(listName))) // tableVar.put(keyVar, list);
      }
      generator.expression(Expression.invoke(list, hashTable.add, element)) // list.add( element );


  }

  override def declareProperty(propertyVar: String) = {
    locals = locals + (propertyVar -> generator.declare(typeRef[Object], propertyVar))
  }

  override def hasLabel(nodeVar: String, labelVar: String, predVar: String) =  {
{'fraction_non_alphanumeric': 0.11241401290802369, 'fraction_numerical': 0.001965419245172771, 'mean_word_length': 4.1701338825952625, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 171, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 52, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4675866', 'n_tokens_mistral': 21048, 'n_tokens_neox': 20657, 'n_words': 4443}