# integration code for database usage via CloudGraphs.jl

export
  executeQuery,
  listAllVariables,
  getCloudVert,
  getfnctype,
  usecloudgraphsdatalayer!,
  uselocalmemoryonly!,
  standardcloudgraphsetup,
  consoleaskuserfordb,
  registerGeneralVariableTypes!,
  fullLocalGraphCopy!,
  subLocalGraphCopy!,
  removeGenericMarginals!,
  setBackendWorkingSet!,
  setAllDBSolvable!,
  getExVertFromCloud,
  getAllExVertexNeoIDs,
  getPoseExVertexNeoIDs,
  getFirstPose,
  getfirstpose,
  getLastPose,
  copyAllNodes!,
  copyAllEdges!,
  registerCallback!,
  updateFullCloudVertData!,
  #loading frtend generated fg
  getnewvertdict,
  mergeValuesIntoCloudVert!,
  recoverConstraintType,
  populatenewvariablenodes!,
  populatenewfactornodes!,
  updatenewverts!,
  resetentireremotesession,
  appendvertbigdata!,
  # visualization exports
  getPointCloudFromKinect,
  getPointCloudFromBSON,
  deleteServerSession!


"""
    $(SIGNATURES)

Run Neo4j Cypher queries on the cloudGraph database, and return Tuple with the
unparsed (results, loadresponse).
Throws an error if the query fails.
"""
function executeQuery(connection::Neo4j.Connection,
                      query::AS ) where {AS <: AbstractString}
  #
  loadtx = transaction(connection)
  cph = loadtx(query, submit=true)
  if length(cph.errors) > 0 #Uh oh, return legible error
    err = ""
    for er in cph.errors
      err*="-\r\n"
      for (k,v) in er
        err*=" $k: $v\r\n"
      end
    end
    error("Unable to perform Neo4j query:\r\n$err")
  end
  loadresult = commit(loadtx)
  return cph, loadresult
end
executeQuery(cg::CloudGraph, query::AS) where {AS <:AbstractString} = executeQuery(cg.neo4j.connection, query)

"""
    $(SIGNATURES)

Get cloud vertex from Neo4j using CloudGraphs.jl as identified by `session`, `robot`, `user`, and vertex label `vsym::Symbol`.
"""
function getCloudVert(cg::CloudGraph,
                      session::AbstractString,
                      robot::AbstractString,
                      user::AbstractString,
                      vsym::Symbol;
                      bigdata::Bool=false )
  #
  @warn "getCloudVert(cg, sess, sym) will be deprecated, use getCloudVert(cg, sess, sym=sym) instead."
  # query = " and n.solvable=$(ready) and n.label=$(vsym) "
  # query = reqbackendset ? query*" and n.backendset=$(backendset)" : query
  query = "match (n:$(session):$robot:$user) where n.label='$(vsym)' return id(n)"

  cph, = executeQuery(cg, query)
  # loadtx = transaction(cg.neo4j.connection)
  # cph = loadtx(query, submit=true)
  neoid = cph.results[1]["data"][1]["row"][1]
  CloudGraphs.get_vertex(cg, neoid, bigdata)
end

function getCloudVert(cgl::CloudGraph,
        session::AbstractString,
        robot::AbstractString,
        user::AbstractString;
        exvid::NothingUnion{Int}=nothing,
        neoid::NothingUnion{Int}=nothing,
        sym::NothingUnion{Symbol}=nothing,
        bigdata=false  )
  #
  query = "match (n:$(session):$robot:$user) "

  if sym != nothing
    query = query*" where n.label='$(sym)' "
  elseif exvid != nothing
    query = query*" where n.exVertexId=$(exvid) "
  elseif neoid != nothing
    return CloudGraphs.get_vertex(cgl, neoid, bigdata)
  else
    @show sym, neoid,exvid
    error("Cannot list neighbors if no input reference is given")
  end
  query = query*" return id(n)"

  cph, = executeQuery(cgl.neo4j.connection, query)
  neoid = cph.results[1]["data"][1]["row"][1]
  CloudGraphs.get_vertex(cgl, neoid, bigdata)
end


function listAllVariables(cgl::CloudGraph, session::AbstractString, robot::AbstractString, user::AbstractString)
  #
  query = "match (n:$(session):$robot:$user) where not (n:FACTOR) and exists(n.exVertexId) and n.solvable=1 return n.label, n.exVertexId, id(n), labels(n)"
  cph, = executeQuery(cgl.neo4j.connection, query)

  dd = Dict{Symbol, Tuple{Int, Int, Vector{Symbol}}}()
  for metarows in cph.results[1]["data"]
    data = metarows["row"]
    lbls = setdiff(data[4],String[session])
    enl = (data[2], data[3], Symbol.(lbls))
    dd[Symbol(data[1])] = enl
  end
  return dd
end


function getCloudNeighbors(cgl::CloudGraph, cv::CloudVertex; needdata=false, returntype=false)
  neis = get_neighbors(cgl, cv, needdata=false)
end

"""
    $(SIGNATURES)

List neighbors to node in cgl::CloudGraph by returning Dict{Sym}=(exvid, neoid, Symbol[labels]), and can take
any of the three as input node identifier. Not specifying an identifier will result in all Variable nodes
being returned.
"""
function ls(cgl::CloudGraph, session::AbstractString, robot::AbstractString, user::AbstractString;
      sym::NothingUnion{Symbol}=nothing,
      neoid::NothingUnion{Int64}=nothing,
      exvid::NothingUnion{Int64}=nothing  )
  #

  if sym == nothing && exvid == nothing && neoid == nothing
    # interrupt and just return all variable nodes
    return listAllVariables(cgl, session, robot, user)
  end
  cv = getCloudVert(cgl, session, robot, user, sym=sym, neoid=neoid, exvid=exvid, bigdata=false )

  neis = get_neighbors(cgl, cv, needdata=false)
  dd = Dict{Symbol, Tuple{Int, Int, Vector{Symbol}}}()
  for nei in neis
    lbls = setdiff(nei.labels, String[session])
    dd[Symbol(nei.properties["label"])] = (nei.exVertexId, nei.neo4jNodeId, Symbol.(lbls) )
  end
  return dd
end

# deprecated
function getfnctype(cvl::CloudGraphs.CloudVertex)
  vert = cloudVertex2ExVertex(cvl)
  return getfnctype(vert)
end

function addCloudVert!(fgl::G,
        exvert::Graphs.ExVertex;
        labels::Vector{T}=String[]  ) where {G <: AbstractDFG, T <: AbstractString}
  #
  cv = CloudGraphs.exVertex2CloudVertex(exvert);
  cv.labels = labels
  CloudGraphs.add_vertex!(fgl.cg, cv);
  fgl.cgIDs[exvert.index] = cv.neo4jNodeId
  IncrementalInference.addGraphsVert!(fgl, exvert)
end

# Return Graphs.ExVertex type containing data according to id
function getExVertFromCloud(fgl::G,
                            fgid::Int64;
                            bigdata::Bool=false  ) where G <: AbstractDFG
  #
  neoID = fgl.cgIDs[fgid]
  cvr = CloudGraphs.get_vertex(fgl.cg, neoID, false)
  CloudGraphs.cloudVertex2ExVertex(cvr)
end

function getExVertFromCloud(fgl::G,
                            lbl::Symbol;
                            nt::Symbol=:var,
                            bigdata::Bool=false  ) where G <: AbstractDFG
  # getExVertFromCloud(fgl, fgl.IDs[lbl], bigdata=bigdata)
  getExVertFromCloud(fgl, (nt==:var ? fgl.IDs[lbl] : fgl.fIDs[lbl]), bigdata=bigdata)
end

function updateFullCloudVertData!(fgl::G,
                                  nv::Graphs.ExVertex;
                                  updateMAPest::Bool=false,
                                  bigdata::Bool=false  ) where G <: AbstractDFG
 #
  # TODO -- this get_vertex seems excessive, but we need the CloudVertex

  if(!haskey(fgl.cgIDs, nv.index))
      error("Cannot find $(nv.index) in $(fgl.cgIDs)...")
  end
  neoID = fgl.cgIDs[nv.index]
  # println("updateFullCloudVertData! -- trying to get $(neoID)")
  vert = CloudGraphs.get_vertex(fgl.cg, neoID, false)
  if typeof(getData(nv)) == VariableNodeData && updateMAPest
    mv = getKDEMax(getKDE(nv))
    nv.attributes["MAP_est"] = mv
    # @show nv.attributes["MAP_est"]
  end

  # TODO -- ignoring other properties
  vert.packed = getData(nv) #.attributes["data"]
  for pair in nv.attributes
    if pair[1] != "data"
      vert.properties[pair[1]] = pair[2]
    end
  end

  # also make sure our local copy is updated, need much better refactoring here
  fgl.stateless ? nothing : fgl.g.vertices[nv.index].attributes["data"] = nv.attributes["data"]

  CloudGraphs.update_vertex!(fgl.cg, vert, bigdata)
end

function makeAddCloudEdge!(fgl::G, v1::Graphs.ExVertex, v2::Graphs.ExVertex) where G <: AbstractDFG
  cv1 = CloudGraphs.get_vertex(fgl.cg, fgl.cgIDs[v1.index], false)
  cv2 = CloudGraphs.get_vertex(fgl.cg, fgl.cgIDs[v2.index], false)
  ce = CloudGraphs.CloudEdge(cv1, cv2, "DEPENDENCE");
  retrel = CloudGraphs.add_edge!(fgl.cg, ce);

  # TODO -- keep this edge id in function node data, must refactor
  push!(v2.attributes["data"].edgeIDs, retrel.id) # TODO -- not good way to do this
  updateFullCloudVertData!(fgl, v2)

  IncrementalInference.makeAddEdge!(fgl, v1, v2, saveedgeID=false)
  retrel.id
end


# TODO -- fetching of CloudVertex propably not required, make faster request to @GearsAD
function getCloudOutNeighbors(fgl::G,
                              exVertId::Int64;
                              solvable::Int=1,
                              backendset::Int=1,
                              needdata::Bool=false  ) where G <: AbstractDFG
  #
  # println("Looking for cloud out neighbors")
  cgid = fgl.cgIDs[exVertId]
  cv = CloudGraphs.get_vertex(fgl.cg, cgid, false)
  neighs = CloudGraphs.get_neighbors(fgl.cg, cv)
  neExV = Graphs.ExVertex[]
  for n in neighs
    cgn = CloudGraphs.cloudVertex2ExVertex(n)
    if (
       #cgn.attributes["solvable"] == solvable &&
       #cgn.attributes["backendset"] == backendset &&
       (!needdata || haskey(cgn.attributes, "data") )  )
      push!(neExV, cgn )
    end
  end
  return neExV
end

# return list of neighbors as Graphs.ExVertex type
function getCloudOutNeighbors(fgl::G,
                              vert::Graphs.ExVertex;
                              solvable::Int=1,
                              backendset::Int=1,
                              needdata::Bool=false  ) where G <: AbstractDFG
  # TODO -- test for solvable and backendset here
  getCloudOutNeighbors(fgl, vert.index, solvable=solvable,backendset=backendset, needdata=needdata )
end


function getEdgeFromCloud(fgl::G, id::Int64) where G <: AbstractDFG
  println("getting id=$(id)")
  CloudGraphs.get_edge(fgl.cg, id)
end

function deleteCloudVertex!(fgl::G, vert::Graphs.ExVertex) where G <: AbstractDFG
  neoID = fgl.cgIDs[vert.index]
  cvr = CloudGraphs.get_vertex(fgl.cg, neoID, false)
  CloudGraphs.delete_vertex!(fgl.cg, cvr)
end

function deleteCloudEdge!(fgl::G, edge::CloudEdge) where G <: AbstractDFG
  CloudGraphs.delete_edge!(fgl.cg, edge)
end




function usecloudgraphsdatalayer!()
  IncrementalInference.setdatalayerAPI!(
    addvertex= addCloudVert!,
    getvertex= getExVertFromCloud,
    makeaddedge= makeAddCloudEdge!,
    getedge= getEdgeFromCloud,
    outneighbors= getCloudOutNeighbors,
    updatevertex= updateFullCloudVertData!,
    deletevertex= deleteCloudVertex!,
    deleteedge= deleteCloudEdge!,
    cgEnabled= true )
  nothing
end

function uselocalmemoryonly!()
  IIF.setdatalayerAPI!(
    addvertex= IIF.localapi.addvertex!,
    getvertex= IIF.localapi.getvertex,
    makeaddedge= IIF.localapi.makeaddedge!,
    getedge= IIF.localapi.getedge,
    outneighbors= IIF.localapi.outneighbors,
    updatevertex= IIF.localapi.updatevertex!,
    deletevertex= IIF.localapi.deletevertex!,
    deleteedge= IIF.localapi.deleteedge!,
    cgEnabled= false )
  nothing
end



function getpackedtype(typestring::AS) where {AS <: AbstractString}
  # println("Caesar.getpackedtype($(typestring))")
  eval(Meta.parse(typestring))() # TODO consider caching or better
end



# function should not be necessary, but fixes a minor bug following elimination algorithm
function removeGenericMarginals!(conn)
  loadtx = transaction(conn)
  query = "match (n)-[r]-() where n.packedType = 'IncrementalInference.FunctionNodeData{IncrementalInference.GenericMarginal}' detach delete n,r"
  cph = loadtx(query, submit=true)
  loadresult = commit(loadtx)
  # TODO -- can probably be made better, but should not be necessary in the first place
  loadtx = transaction(conn)
  query = "match (n) where n.packedType = 'IncrementalInference.FunctionNodeData{IncrementalInference.GenericMarginal}' detach delete n"
  cph = loadtx(query, submit=true)
  loadresult = commit(loadtx)
  nothing
end

"""
    $(SIGNATURES)

Get all Neo4j node IDs in current session.
"""
function getAllExVertexNeoIDs(conn::Neo4j.Connection;
        solvable::Int=1,
        backendset::Int=1,

        sessionname::AS="",
        robotname::AS="",
        username::AS="",
        reqbackendset::Bool=true,
        reqSolvable::Bool=true  ) where {AS <: AbstractString}
  #
  sn = length(sessionname) > 0 ? ":"*sessionname : ""
  rn = length(robotname) > 0 ? ":"*robotname : ""
  un = length(username) > 0 ? ":"*username : ""
  query = "match (n$(sn)$(rn)$(un)) where not n:SESSION and not n:MULTISESSION and exists(n.exVertexId)"
  query = reqbackendset || reqSolvable ? query*" and" : query
  query = reqSolvable ? query*" n.solvable=$(solvable)" : query
  query = reqbackendset && reqSolvable ? query*" and" : query
  query = reqbackendset ? query*" n.backendset=$(backendset)" : query
  query = query*" return n.exVertexId, id(n), n.label"

  cph, = executeQuery(conn, query)
  ret = Array{Tuple{Int64,Int64,Symbol},1}()

  # If no results, return empty array...
  if length(cph.results) > 0
      if length(cph.results[1]["data"]) > 0
          @showprogress 1 "Get ExVertex IDs..." for data in cph.results[1]["data"]
            exvid, neoid, sym = data["row"][1], data["row"][2], Symbol(data["row"][3])
            push!(ret, (exvid,neoid,sym)  )
          end
      end
  end

  return ret
end



"""
    $(SIGNATURES)

Build query to fetch sub graph and neighboring nodes.  For example:
FAILS IN SOME CASES
```
match (n0:Hackathon:HexagonalDrive)
where n0.label IN ['x0']
with n0 as m
return m.exVertexId, id(m), m.label
UNION
match (n0:Hackathon:HexagonalDrive)-[:DEPENDENCE]-(n1:HexagonalDrive)
where n0.label IN ['x0']
with n1 as m
return m.exVertexId, id(m), m.label
UNION
match (n0:Hackathon:HexagonalDrive)-[:DEPENDENCE]-(n1:HexagonalDrive)-[:DEPENDENCE]-(n2:HexagonalDrive)
where n0.label IN ['x0']
with n2 as m
return m.exVertexId, id(m), m.label
```
"""
function buildSubGraphIdsQuery(;
            lbls::Vector{AS}=String[""],
            session::AS="",
            robot::AS="",
            user::AS="",
            label::AS="",
            reqSolvable::Bool=true,
            solvable::Int=1,
            reqbackendset::Bool=true,
            backendset::Int=1,
            neighbors::Int=0,
            includeMultisession::Bool=false ) where {AS <: AbstractString}
  #
  sn = length(session) > 0 ? ":"*session : ""
  rn = length(robot) > 0 ? ":"*robot : ""
  un = length(user) > 0 ? ":"*user : ""
  lb = length(label) > 0 ? ":"*label : ""

  query = ""
  for nei in 0:(neighbors)
    query *= "match (n0$(sn)$(rn)$(un)$(lb))"
    outerd = 0
    for d in 1:(nei)
      query *= "-[:DEPENDENCE]-(n$(d)$(sn)$(rn)$(un)$(lb))"
      outerd = d
    end
    query *= " "
    query *= "where n0.label IN ["
    for lbl in lbls
      query *= "'$(lbl)',"
    end
    query = chop(query)*"] "
    if !includeMultisession
      query *= "and not n0:MULTISESSION "
    end
    query *= "with n$(outerd) as m "
    query *= "return m.exVertexId, id(m), m.label"
    query *= nei != (neighbors) ? " UNION " : ""
  end
  return query
end
# qu = buildSubGraphIdsQuery(lbls=["x0";], session="HexagonalDrive", neighbors=3)
# @show qu

"""
    $(SIGNATURES)

Return array of tuples with ExVertex IDs and Neo4j IDs for vertices with label in session.
"""
function getLblExVertexNeoIDs(
        conn::Neo4j.Connection,
        lbls::Vector{AS};
        session::AS="",
        robot::AS="",
        user::AS="",
        label::AS="",
        reqSolvable::Bool=true,
        solvable::Int=1,
        backendset::Int=1,
        reqbackendset::Bool=true,
        neighbors::Int=0,
        includeMultisession::Bool=false) where {AS <: AbstractString}
  #

  query = buildSubGraphIdsQuery(lbls=lbls, session=session, robot=robot, user=user, label=label, neighbors=neighbors, reqSolvable=reqSolvable, solvable=solvable, reqbackendset=reqbackendset, backendset=backendset, includeMultisession=includeMultisession)
  cph, = executeQuery(conn, query)

  ret = Array{Tuple{Int64,Int64,Symbol},1}()
  @showprogress 1 "Get ExVertex IDs..." for data in cph.results[1]["data"]
    exvid, neoid, vsym = data["row"][1], data["row"][2], Symbol(data["row"][3])
    push!(ret, (exvid,neoid,vsym)  )
  end
  return ret
end

"""
    $(SIGNATURES)

Return array of tuples with ExVertex IDs and Neo4j IDs for vertices with label in session.
"""
function getExVertexNeoIDs(
        conn::Neo4j.Connection;
        label::AS="",
        solvable::Int=1,
        backendset::Int=1,
        session::AS="",
        robot::AS="",
        user::AS="",
        reqbackendset::Bool=true  ) where {AS <: AbstractString}
  #
  sn = length(session) > 0 ? ":"*session : ""
  rn = length(robot) > 0 ? ":"*robot : ""
  un = length(user) > 0 ? ":"*user : ""
  lb = length(label) > 0 ? ":"*label : ""
  query = "match (n$(sn)$(rn)$(un)$(lb)) where n.solvable=$(solvable) and exists(n.exVertexId)"
  query = reqbackendset ? query*" and n.backendset=$(backendset)" : query
  query = query*" return n.exVertexId, id(n), n.label"

  cph, = executeQuery(conn, query)
  # loadtx = transaction(conn)
  # cph = loadtx(query, submit=true)

  ret = Array{Tuple{Int64,Int64,Symbol},1}()
  @showprogress 1 "Get ExVertex IDs..." for data in cph.results[1]["data"]
    exvid, neoid, vsym = data["row"][1], data["row"][2], Symbol(data["row"][3])
    push!(ret, (exvid,neoid,vsym)  )
  end
  return ret
end

"""
    $(SIGNATURES)

Return array of tuples with ExVertex IDs and Neo4j IDs for all poses.
"""
function getPoseExVertexNeoIDs(conn::Neo4j.Connection;
        solvable::Int=1,
        backendset::Int=1,
        session::AS="",
        reqbackendset::Bool=true  ) where {AS <: AbstractString}
  #
  getPoseExVertexNeoIDs(conn,
          label="POSE",
          solvable=solvable,
          backendset=backendset,
          session=session,
          reqbackendset=reqbackendset  )
end


function checkandinsertedges!(fgl::G, exvid::Int, nei::CloudVertex; solvable::Int=1, backendset::Int=1) where G <: AbstractDFG
  if nei.properties["solvable"]==solvable &&
     nei.properties["backendset"] == backendset &&
     haskey(fgl.g.vertices, nei.exVertexId)
    #&& nei.exVertexId <= length(fgl.g.vertices)
    alreadythere = false

    # TODO -- error point
    v2 = fgl.g.vertices[nei.exVertexId]
    for graphsnei in Graphs.out_neighbors(v2, fgl.g) # specifically want Graphs function
      # want to ignore if the edge was previously added from the other side, comparing to the out neighbors in the Graphs structure
      if graphsnei.index == exvid #graphsnei.index == nei.exVertexId
        alreadythere = true
        break;
      end
    end
    if !alreadythere
      # add the edge to graph
      v1 = fgl.g.vertices[exvid]
      makeAddEdge!(fgl, v1, v2, saveedgeID=false)
    end
  end
  nothing
end

function copyAllEdges!(fgl::G, cverts::Dict{Int64, CloudVertex}, IDs::Array{Tuple{Int64,Int64, Symbol},1}) where G <: AbstractDFG
  # TODO -- major opportunity for streamlining and improving performance
  # do entire graph, one node at a time
  @showprogress 1 "Copy all edges..." for ids in IDs
    for nei in CloudGraphs.get_neighbors(fgl.cg, cverts[ids[2]], needdata=true)
      checkandinsertedges!(fgl, ids[1], nei, solvable=1, backendset=1)
    end
  end
  nothing
end

function insertnodefromcv!(fgl::G, cvert::CloudGraphs.CloudVertex) where G <: AbstractDFG
  exvid = cvert.exVertexId
  neoid = cvert.neo4jNodeId
  exvert = cloudVertex2ExVertex(cvert)
  # TODO -- change to addNode
  Graphs.add_vertex!(fgl.g, exvert)
  fgl.id < exvert.index ? fgl.id = exvert.index : nothing
  fgl.cgIDs[exvid] = neoid
  if typeof(exvert.attributes["data"]) == VariableNodeData  # variable node
    fgl.IDs[Symbol(exvert.label)] = exvid
    push!(fgl.nodeIDs, exvid)
  else # function node
    fgl.fIDs[Symbol(exvert.label)] = exvid
    push!(fgl.factorIDs, exvid)
  end
  nothing
end

"""
    $(SIGNATURES)

Copy all variable and factor nodes from DB into fgl as listed in IDs vector.
"""
function copyAllNodes!(fgl::G,
                       cverts::Dict{Int64, CloudVertex},
                       IDs::Array{Tuple{Int64,Int64, Symbol},1}  ) where G <: AbstractDFG
                       # conn::Neo4j.Connection  ) # what the?
  #
  @showprogress 1 "Copy all nodes..." for ids in IDs
    cvert = CloudGraphs.get_vertex(fgl.cg, ids[2], false)
    cverts[ids[2]] = cvert
    insertnodefromcv!(fgl, cvert)
  end
  nothing
end

"""
    $(SIGNATURES)

Copy nodes into `fgl` from DB as listed in the `IDs` vector.
"""
function copyGraphNodesEdges!(fgl::G,
                              IDs::Array{Tuple{Int64,Int64, Symbol},1}  ) where G <: AbstractDFG
  #
  if length(IDs) > 0
    cverts = Dict{Int64, CloudVertex}()
    unsorted = Int64[]
    for ids in IDs
      push!(unsorted, ids[1])
    end
    perm = sortperm(unsorted)

    # get and add all the nodes
    sortedIDs = IDs[perm]
    copyAllNodes!(fgl, cverts, sortedIDs)

    # get and insert all edges
    copyAllEdges!(fgl, cverts, sortedIDs)
    return true
  else
    print(".")
    return false
  end
end

"""
    $(SIGNATURES)

Copy sub graph portion defined by, and including a depth of neighbors, from the lbls vector.
"""
function subLocalGraphCopy!(fgl::G,
                            lbls::Union{Vector{AS}, Vector{Symbol}};
                            neighbors::Int=0,
                            reqbackendset::Bool=true,
                            reqSolvable::Bool=true,
                            includeMultisession::Bool=false) where {G <: AbstractDFG, AS <: AbstractString}
  #
  @warn "subGraphCopy! is a work in progress"
  conn = fgl.cg.neo4j.connection
  IDs = getLblExVertexNeoIDs(conn, string.(lbls), session=fgl.sessionname, robot=fgl.robotname, user=fgl.username, reqbackendset=reqbackendset, reqSolvable=reqSolvable, neighbors=neighbors, includeMultisession=includeMultisession)
  println("fullSubGraphCopy: $(length(IDs)) nodes in session $(fgl.sessionname) if reqbackendset=$reqbackendset and reqSolvable=$reqSolvable...")
  copyGraphNodesEdges!(fgl, IDs)
  nothing
end


"""
    $(SIGNATURES)

Fetch a full copy of the DB factor graph under fgl.sessionname.
"""
function fullLocalGraphCopy!(fgl::G;
                             reqbackendset::Bool=true,
                             reqSolvable::Bool=true  ) where G <: AbstractDFG
  #
  conn = fgl.cg.neo4j.connection
  IDs = getAllExVertexNeoIDs(conn, sessionname=fgl.sessionname, robotname=fgl.robotname, username=fgl.username, reqbackendset=reqbackendset, reqSolvable=reqSolvable)
  println("fullLocalGraphCopy: $(length(IDs)) nodes in subgraph for user=$(fgl.username), robot=$(fgl.robotname), session=$(fgl.sessionname) if reqbackendset=$reqbackendset and reqSolvable=$reqSolvable...")
  copyGraphNodesEdges!(fgl, IDs)
end

"""
    $(SIGNATURES)

Set all Neo4j nodes in this session solvable = 1, warning function does not support new GraffSDK data storage formats.
"""
function setAllDBSolvable!(conn::Neo4j.Connection,
                        sessionname::AS,
                        robotname::AS,
                        username::AS) where {AS <: AbstractString}
  #
  @warn "Obsolete setAllDBSolvable! function, see GraffSDK for example solvable function instead."
  sn = length(sessionname) > 0 ? ":"*sessionname : ""
  rn = length(robotname) > 0 ? ":"*robotname : ""
  un = length(username) > 0 ? ":"*username : ""
  query = "match (n$(sn)$(rn)$(un)) set n.solvable=1"
  cph, loadresult = executeQuery(conn, query)
  nothing
end

# TODO --this will only work with DB version, introduces a bug
function setAllDBSolvable!(fgl::G) where G <: AbstractDFG
  setAllDBSolvable!(fgl.cg.neo4j.connection, fgl.sessionname)
end


function setBackendWorkingSet!(
            conn::Neo4j.Connection,
            sessionname::AbstractString,
            robotname::AbstractString,
            username::AbstractString  )
  #
  sn = length(sessionname) > 0 ? ":"*sessionname : ""
  rn = length(robotname) > 0 ? ":"*robotname : ""
  un = length(username) > 0 ? ":"*username : ""
  query = "match (n$(sn)$(rn)$(un)) where not (n:NEWDATA) set n.backendset=1"

  cph, loadresult = executeQuery(conn, query)
  nothing
end

"""
    $(SIGNATURES)

Obtain Neo4j global database address and login credientials from STDIN, then insert and return in the addrdict colletion.
"""
function askneo4jcredentials!(;addrdict=Dict{AbstractString,AbstractString}() )
  need = ["neo4jHost";"neo4jPort";"neo4jUsername";"neo4jPassword";"session";"robotId"]
  info("Please enter information for Neo4j DB:")
  for n in need
    info(n)
    str = readline(stdin)
    addrdict[n] = str
    if length(str) > 0
      if str[end] == "\n"
        addrdict[n] = str[1:(end-1)]
      end
    end
  end
  return addrdict
end

"""
    $(SIGNATURES)

Obtain Mongo database address and login credientials from STDIN, then insert and return in the addrdict colletion.
"""
function askmongocredentials!(;addrdict=Dict{AbstractString,AbstractString}() )
  need = ["mongoHost";"mongoPort";"mongoUsername";"mongoPassword"]
  info("Please enter information for MongoDB:")
  for n in need
    info(n)
    n == "mongoHost" && haskey(addrdict, "neo4jHost") ? print(string("[",addrdict["neo4jHost"],"]: ")) : nothing
    str = readline(stdin)
    addrdict[n] = str
    if length(str) > 0
      if str[end] == "\n"
        addrdict[n] = str[1:(end-1)]
      end
    end
  end
  addrdict["mongoIsUsingCredentials"] = false,
  if addrdict["mongoHost"] == "" && haskey(addrdict, "neo4jHost")
    addrdict["mongoHost"] = addrdict["neo4jHost"]
  elseif addrdict["mongoHost"] != ""
    nothing
  else
    error("Don't know how to get to MongoDB address.")
  end
  return addrdict
end


"""
    $(SIGNATURES)

Obtain database addresses and login credientials from STDIN, as well as a few case dependent options.
"""
function consoleaskuserfordb(;nparticles=false, drawdepth=false, clearslamindb=false, multisession=false, drawedges=false)
  addrdict = Dict{AbstractString, Any}()  #Union{AbstractString, Vector{String}}
  askneo4jcredentials!(addrdict=addrdict)
  askmongocredentials!(addrdict=addrdict)
  need = String[]
  !nparticles ? nothing : push!(need, "num particles")
  !drawdepth ? nothing : push!(need, "draw depth")
  !clearslamindb ? nothing : push!(need, "clearslamindb")
  !user ? nothing : push!(need, "user")
  !robot ? nothing : push!(need, "robot")
  !multisession ? nothing : push!(need, "multisession")
  !drawedges ? nothing : push!(need, "draw edges")


  info("Please also enter information for:")
  for n in need
    info(n)
    n == "draw depth" ? print("[y]/n: ") : nothing
    n == "draw edges" ? print("[y]/n: ") : nothing
    n == "num particles" ? print("[100]: ") : nothing
    n == "clearslamindb" ? print("yes/[no]: ") : nothing
    n == "user" ? print("[]: ") : nothing
    n == "robot" ? print("[]: ") : nothing
    n == "multisession" ? print("comma separated list session names/[n]: ") : nothing
    str = readline(stdin)
    addrdict[n] = str
    if length(str) > 0
      if str[end] == "\n"
        addrdict[n] = str[1:(end-1)]
      end
    end
  end
  if drawdepth
    addrdict["draw depth"] = addrdict["draw depth"]=="" || addrdict["draw depth"]=="y" || addrdict["draw depth"]=="yes" ? "y" : "n"
  end
  if drawedges
    addrdict["draw edges"] = addrdict["draw edges"]=="" || addrdict["draw edges"]=="y" || addrdict["draw edges"]=="yes" ? "y" : "n"
  end
  if nparticles
    addrdict["num particles"] = addrdict["num particles"]!="" ? addrdict["num particles"] : "100"
  end
  if clearslamindb
    addrdict["clearslamindb"] = addrdict["clearslamindb"]=="" || addrdict["clearslamindb"]=="n" || addrdict["clearslamindb"]=="no" ? "n" : addrdict["clearslamindb"]
  end
  if user
      addrdict["user"] = addrdict["user"]!="" ? addrdict["user"] : ""
  end
  if robot
    addrdict["robot"] = addrdict["robot"]!="" ? addrdict["robot"] : ""
  end
  if multisession
    addrdict["multisession"] = strip.(Vector{String}(split(addrdict["multisession"],',')))
  end
  return addrdict
end


"""
    $(SIGNATURES)

Connect to databases via network according to addrdict, or ask user for credentials and return
active cloudGraph object, as well as addrdict.
"""
function standardcloudgraphsetup(;addrdict=nothing,
            nparticles::Bool=false,
            drawdepth::Bool=false,
            drawedges::Bool=false,
            clearslamindb::Bool=false,
            multisession::Bool=false  )
  #
  if addrdict == nothing
    addrdict = consoleaskuserfordb(nparticles=nparticles, drawdepth=drawdepth, clearslamindb=clearslamindb, multisession=multisession, drawedges=drawedges)
  end

  @warn "Not considering field: addrdict[\"mongoIsUsingCredentials\"]"
  # Connect to database
  # addrdict["mongoIsUsingCredentials"]
  configuration = CloudGraphs.CloudGraphConfiguration(
                             addrdict["neo4jHost"], parse(Int, addrdict["neo4jPort"]), addrdict["neo4jUsername"], addrdict["neo4jPassword"],
                             addrdict["mongoHost"], parse(Int, addrdict["mongoPort"]), false, addrdict["mongoUsername"], addrdict["mongoPassword"]);
  cloudGraph = connect(configuration, IncrementalInference.encodePackedType, Caesar.getpackedtype, IncrementalInference.decodePackedType);
  # conn = cloudGraph.neo4j.connection
  # register types of interest in CloudGraphs
  # registerGeneralVariableTypes!(cloudGraph) # TESTING -- using dispatch instead
  Caesar.usecloudgraphsdatalayer!()

  return cloudGraph, addrdict
end

"""
    $(SIGNATURES)

Walk through vertex bigDataElements and return the last matching description.
"""
function getBigDataElement(vertex::CloudVertex, description::AbstractString)
  bde = nothing
  for bDE in vertex.bigData.dataElements
    if bDE.description == description
      bde = bDE
    end
  end
  return bde
end

"""
    $(SIGNATURES)

Return true if vertex has bigDataElements with matching description.
"""
function hasBigDataElement(vertex::CloudVertex, description::AbstractString)
  for bDE in vertex.bigData.dataElements
    if bDE.description == description
      return true
    end
  end
  return false
end

"""
    $(SIGNATURES)

Append big data element into current blob store and update associated global
vertex information.
"""
function appendvertbigdata!(cloudGraph::CloudGraph,
        cv::CloudVertex,
        description,
        data::Vector{UInt8}  )
  #
  bd = CloudGraphs.read_BigData!(cloudGraph, cv)
  bdei = CloudGraphs.BigDataElement(description, data)
  push!(cv.bigData.dataElements, bdei);
  CloudGraphs.save_BigData!(cloudGraph, cv)
end

"""
    $(SIGNATURES)

Append big data element into current blob store and update associated global
vertex information.
"""
function appendvertbigdata!(fgl::G,
      vert::Graphs.ExVertex,
      description::AbstractString,
      data::Vector{UInt8}  ) where G <: AbstractDFG
  #
  # TODO -- improve get/fetch vertex abstraction
  cvid = fgl.cgIDs[vert.index]
  cv = CloudGraphs.get_vertex(fgl.cg, cvid, true)
  appendvertbigdata!(fgl.cg, cv, description, data)
end


"""
    appendvertbigdata!(fg, sym, descr, data)

Append big data element into current blob store using parent appendvertbigdata!,
but here specified by symbol of variable node in the FactorGraph. Note the
default data layer api definition. User must define dlapi to refetching the
 vertex from the data layer. localapi avoids repeated network database fetches.
"""
function appendvertbigdata!(fgl::G,
      sym::Symbol,
      description::AbstractString,
      data  ) where G <: AbstractDFG
  #
  appendvertbigdata!(fgl,
        getVert(fgl, sym),
        description,
        data  )
end


"""
    $(SIGNATURES)

Fetch and insert list of CloudVertices into FactorGraph object, up to neighbor depth.
"""
function fetchsubgraph!(fgl::G,
                        cvs::Vector{CloudGraphs.CloudVertex};
                        numneighbors::Int=0 ) where G <: AbstractDFG
          # overwrite::Bool=false  )
  # recursion termination condition
  numneighbors >= 0 ? nothing : (return nothing)

  for cv in cvs
    # test if these are already in fgl
    if !hasval(fgl.cgIDs, cv.neo4jNodeId) # may have been inserted as previous neighbor
      # add this vert to graph
      insertnodefromcv!(fgl, cv)

      # recursive call on neighbors here
      neicvs = CloudGraphs.get_neighbors(fgl.cg, cv, needdata=true)
      fetchsubgraph!(fgl, neicvs; numneighbors=numneighbors-1 )
      # add edges associated with the neighbors

      if numneighbors-1 >= 0
        for cvn in neicvs
          checkandinsertedges!(fgl, cv.exVertexId, cvn, solvable=1, backendset=1)
          # makeAddEdge!(fgl, fgl.g.vertices[cv.exVertexId], fgl.g.vertices[cvn.exVertexId], saveedgeID=false)
        end
      end
    end

  end
end

"""
    $SIGNATURES

Fetch and insert list of Neo4j IDs into FactorGraph object, up to neighbor depth.
"""
function fetchsubgraph!(fgl::G,
                        neoids::Vector{Int};
                        numneighbors::Int=0 ) where G <: AbstractDFG
                        # overwrite::Bool=false  )
  #
  for nid in neoids
    # test if these are already in fgl
    if !hasval(fgl.cgIDs, nid)
      cv = CloudGraphs.get_vertex(fgl.cg, nid, false)
      fetchsubgraph!(fgl, [cv], numneighbors=numneighbors )
    end
  end
  nothing
end

"""
    getVertNeoIDs!(::CloudGraph, res::Dict{Symbol, Int}; session::AbstractString="NA", robot::AbstractString="NA", user::AbstractString="NA")

Insert into and return dict `res` with Neo4j IDs of ExVertex labels as stored per session in Neo4j database.
"""
function getVertNeoIDs!(cloudGraph::CloudGraph, res::Dict{Symbol, Int}; session::AbstractString="NA", robot::AbstractString="NA", user::AbstractString="NA")
  loadtx = transaction(cloudGraph.neo4j.connection)
  syms = collect(keys(res))
  query = "match (n:$(session):$robot:$user) where "
  for i in 1:length(syms)
    sym = syms[i]
    query =query*"n.label='$(sym)' "
    if i < length(syms)
      query =query*"or "
    end
  end
  query = query*"return id(n), n.label"
  cph = loadtx(query, submit=true)
  for qr in cph.results[1]["data"]
    res[Symbol(qr["row"][2])] = qr["row"][1]
  end
  # Symbol(cph.results[1]["data"][1]["row"][2]) == sym ? nothing : error("Neo4j query not returning $(sym)")
  return res
end

"""
    removeNeo4jID(cg::CloudGraph, neoid=-1)

Remove node from Neo4j according to Neo4j Node ID. Big data elements that may be associated with this
node are not removed.
"""
function removeNeo4jID(cg::CloudGraph; neoid::Int=-1)
  neoid > 0 ? nothing : error("Can't delete negative neoid=$(neoid).")
  loadtx = transaction(cg.neo4j.connection)
  query =  "match (n) where id(n)=$(neoid) detach delete n return count(n)"
  cph = loadtx(query, submit=true)
  commit(loadtx)
  cnt = Int(cph.results[1]["data"][1]["row"][1])
  cnt == 1 ? nothing : error("Did not delete just one entry after running query = $(query)")
  nothing
end



"""
    getfirstpose(cg::CloudGraph, session::AbstractString, robot::AbstractString, user::AbstractString)

Return Tuple{Symbol, Int} of first pose symbol and Neo4j node ID.
"""
function getfirstpose(cg::CloudGraph, session::AbstractString, robot::AbstractString, user::AbstractString)
  query = "match (n:$(session):$(robot):$(user):POSE) with n.label as nlbl, n.exVertexId as exvid, id(n) as neoid order by exvid asc limit 1 return nlbl, neoid"
  cph, = executeQuery(cg, query)
  # loadtx = transaction(cg.neo4j.connection)
  # cph = loadtx(query, submit=true)
  Symbol(cph.results[1]["data"][1]["row"][1]), cph.results[1]["data"][1]["row"][2]
end
getFirstPose(cg::CloudGraph, session::AbstractString, robot::AbstractString, user::AbstractString) = getfirstpose(cg, session, robot, user)

"""
    getLastPose(cg::CloudGraph, session::AbstractString, robot::AbstractString, user::AbstractString)

Return Tuple{Symbol, Int} of first pose symbol and Neo4j node ID.
"""
function getLastPose(cg::CloudGraph, session::AbstractString, robot::AbstractString, user::AbstractString)
  query = "match (n:$(session):$robot:$user:POSE) with n.label as nlbl, n.exVertexId as exvid, id(n) as neoid order by exvid desc limit 1 return nlbl, neoid"
  cph, = executeQuery(cg, query)
  # loadtx = transaction(cg.neo4j.connection)
  # cph = loadtx(query, submit=true)
  Symbol(cph.results[1]["data"][1]["row"][1]), cph.results[1]["data"][1]["row"][2]
end


"""
    insertrobotdatafirstpose!(cg::CloudGraph, session::AbstractString, robot::AbstractString, user::AbstractString, robotdict::Dict)

Saves robotdict via JSON to first pose in a SESSION in the database. Used for
storing general robot specific data in easily accessible manner. Can fetch later
retrieve same dict with counterpart `fetchrobotdatafirstpose` function.
"""
function insertrobotdatafirstpose!(cg::CloudGraph, session::AbstractString, robot::AbstractString, user::AbstractString, robotdict::Dict)
  vsym, neoid = getfirstpose(cg, session, robot, user)
  cv = CloudGraphs.get_vertex(cg, neoid, true)
  appendvertbigdata!(cg, cv, "robot_description", json(robotdict).data  )
end

function tryunpackalltypes!(resp::Dict)
  for (k,v) in resp
    tv = typeof(v)
    if tv == Vector{Any}
      ttv = typeof(v[1])
      if ttv == Vector{Any}
        try resp[k] = hcat(v...) catch end
      else
        try resp[k] = Vector{ttv}(v) catch end
      end
    end
  end
  nothing
end

"""
    fetchrobotdatafirstpose(cg::CloudGraph, session::AbstractString, robot::AbstractString, user::AbstractString)

Return dict of JSON parsed "robot_description" field as was inserted by counterpart
`insertrobotdatafirstpose!` function. Used for storing general robot specific data
in easily accessible manner.
"""
function fetchrobotdatafirstpose(cg::CloudGraph, session::AbstractString, robot::AbstractString, user::AbstractString)
  vsym, neoid = getfirstpose(cg, session, robot, user)
  cv = CloudGraphs.get_vertex(cg, neoid, true)
  bde = Caesar.getBigDataElement(cv, "robot_description")
  resp = JSON.parse(String(take!(IOBuffer(bde.data))))
  tryunpackalltypes!(resp)
  return resp
end


"""
    getRangeKDEMax2D(cgl::CloudGraph, session::AbstractString, vsym1::Symbol, vsym2::Symbol)

Calculate the cartesian distange between two vertices in the graph, by session and symbol names,
and by maximum belief point.
"""
function getRangeKDEMax2D(cgl::CloudGraph, session::AbstractString, vsym1::Symbol, vsym2::Symbol)
  # get the relavent neo4j ids
  syms = Dict(vsym1=>0, vsym2 => 0)
  getVertNeoIDs!(cgl, syms, session=session)

  # build a local subgraph
  sfg = initfg(cloudgraph=cgl, sessionname=session)
  fetchsubgraph!(sfg, collect(values(syms)), numneighbors=1 )

  # calculate distances on local subgraph
  getRangeKDEMax2D(sfg, vsym1, vsym2)
end


"""
    db2jld(cgl::CloudGraph, session::AbstractString, filename::AbstractString)

Fetch and save a FactorGraph session to a jld, using CloudGraph object and session definition.
"""
function db2jld(cgl::CloudGraph, session::AbstractString, filename::AbstractString)
  fg = initfg(sessionname=session, cloudgraph=cgl)
  fullLocalGraphCopy!(fg)
  savejld(fg, file=filename)
  return fg
end

"""
    db2jld(filename::AbstractString; addrdict::NothingUnion{Dict{AbstractString, AbstractString}}=nothing )

Fetch and save a FactorGraph session to a jld, using or asking STDIN for credentials in the addrdict field.
"""
function db2jld(filename::AbstractString; addrdict::NothingUnion{Dict{AbstractString, AbstractString}}=nothing )
  cg, cr = standardcloudgraphsetup(addrdict=addrdict)
  db2jld(cg, cr["session"], filename)
end


function deleteServerSession!(cloudGraph::CloudGraph, session::AbstractString, robot::AbstractString, user::AbstractString)
  query =  "match (n:$(session):$robot:$user)
            detach delete n
            return count(n)"
  return executeQuery(cloudGraph.neo4j.connection, query)
end

# function syncmongos()
#
# end

  #
