text:
const HEADER_BYTES = convert(Vector{UInt8}, codeunits("csearch index 1\n"))
const TRAILER_BYTES = convert(Vector{UInt8}, codeunits("\ncsearch trailr\n"))
const POSTING_LIST_TRAILER_TRIGRAM = UInt8[0xff, 0xff, 0xff]
const POSTING_LIST_TRAILER_DELTA = UInt32[0]

"""
The trailer has the form:
- offset of path list [4]
- offset of name list [4]
- offset of posting lists [4]
- offset of name index [4]
- offset of posting list index [4]
- "\ncsearch trailr\n"
"""
mutable struct IndexTrailerOffsets
    path_list::UInt32
    name_list::UInt32
    posting_list::UInt32
    name_index::UInt32
    posting_list_index::UInt32

    function IndexTrailerOffsets()
        new(0,0,0,0,0)
    end
end

"""
A single posting entry.

The trigram gives the 3 byte trigram that this list describes. The delta list is a sequence of varint-encoded deltas between file
IDs, ending with a zero delta.  For example, the delta list [2,5,1,1,0] encodes the file ID list 1, 6, 7, 8.

The delta list [0] would encode the empty file ID list, but empty posting lists are usually not recorded at all.

The list of posting lists ends with an entry with trigram "\xff\xff\xff" and a delta list consisting a single zero.
"""
struct Posting
    trigram::Vector{UInt8}
    deltas::Vector{UInt32}
end

const Postings = Vector{Posting}

fileids(posting::Posting) = fileids(posting.deltas)
fileids(deltas::Vector{UInt32}) = resize!(convert(Vector{UInt32}, cumsum(deltas)), UInt32(length(deltas)-1)) .- UInt32(1)
function deltas(fileids::Vector{UInt32})
    ids = convert(Vector{Int32}, fileids)
    pushfirst!(ids, -1)
    push!(convert(Vector{UInt32}, diff(ids)), 0)
end
function prune!(posting::Posting, id_map::Dict{UInt32,UInt32})
    fids = fileids(posting)
    filter!(x->x in keys(id_map), fids)
    map!(x->id_map[x], fids, fids)
    empty!(posting.deltas)
    dlts = deltas(fids)
    append!(posting.deltas, dlts)
    posting
end

function prune!(postings::Postings, id_map::Dict{UInt32,UInt32})
    for posting in postings
        prune!(posting, id_map)
    end
    # remove empty postings, except the trailer
    filter!(postings) do posting
        (posting.trigram == POSTING_LIST_TRAILER_TRIGRAM) || (length(posting.deltas) > 1)
    end
    postings
end

"""The name index is a sequence of 4-byte big-endian values listing the byte offset in the name list where each name begins."""
struct NameIndex
    entries::Vector{UInt32}

    function NameIndex()
        new(UInt32[])
    end
end

"""
The posting list index is a sequence of index entries describing each successive posting list.

Index entries are only written for the non-empty posting lists, so finding the posting list for a specific trigram requires a
binary search over the posting list index.  In practice, the majority of the possible trigrams are never seen, so omitting the missing
ones represents a significant storage savings.
"""
struct PostingIndexEntry
    trigram::Vector{UInt8}
    filecount::UInt32
    offset::UInt32
end

struct PostingIndex
    entries::Vector{PostingIndexEntry}

    function PostingIndex()
        new(PostingIndexEntry[])
    end
end

struct Strings
    entries::Vector{String}

    function Strings(entries::Vector{String}=String[])
        new(entries)
    end
end

"""
Stores an index datastructure, which is either read from an existing file or prepared in memory.

An index stored on disk has the format:
- "csearch index 1\n"
- list of paths
- list of names
- list of posting lists
- name index
- posting list index
- trailer

The list of paths is a sorted sequence of NUL-terminated file or directory names. The index covers
the file trees rooted at those paths. The list ends with an empty name ("\x00").

The list of names is a sorted sequence of NUL-terminated file names. The initial entry in the list
corresponds to file #0, the next to file #1, and so on.  The list ends with an empty name ("\x00").

The list of posting lists are a sequence of posting lists. The list of posting lists ends with an
entry with trigram "\xff\xff\xff" and a delta list consisting a single zero.

The indexes enable efficient random access to the lists.

The trailer holds offsets of the various sections of the index.
"""
mutable struct Index
    paths::Strings
    names::Strings
    postings::Postings
    nameindex::NameIndex
    postingindex::PostingIndex
    offsets::IndexTrailerOffsets

    function Index()
        new(Strings(), Strings(), Posting[], NameIndex(), PostingIndex(), IndexTrailerOffsets())
    end
end

function find_exact_paths(idx::Index, paths::AbstractVector{String})
    exact_paths = String[]
    for path in paths
        for entry in idx.paths.entries
            if startswith(entry, path)
                push!(exact_paths, entry)
            end
        end
    end
    exact_paths
end

function find_name_indices(idx::Index, names::AbstractVector{String})
    nameidxs_to_prune = UInt32[]
    for (nameidx,name) in enumerate(idx.names.entries)
        if name in names
            push!(nameidxs_to_prune, nameidx)
        end
    end
    nameidxs_to_prune
end

# -----------------------------------------------
# index manipulation methods
# -----------------------------------------------
"""
Removes the path and all its descendants from the index.
"""
prune_paths!(idx::Index, path::String) = prune_paths!(idx, [path])
prune_paths!(idx::Index, paths::AbstractVector{String}) = prune_exact_paths!(idx, find_exact_paths(idx, paths))
function prune_exact_paths!(idx::Index, paths::Vector{String})
    # do nothing if possible
    isempty(paths) && (return idx)

    filter!(path->!(path in paths), idx.paths.entries)

    # prune names list and create ordered list of names and name indices that were pruned
    nameidxs_to_prune = UInt32[]
    names_to_prune = String[]
    for path in paths
        for (nameidx,name) in enumerate(idx.names.entries)
            if startswith(name, path)
                push!(nameidxs_to_prune, UInt32(nameidx-1))
                push!(names_to_prune, name)
            end
        end
    end
    prune_files!(idx, names_to_prune, nameidxs_to_prune)
end

prune_files!(idx::Index, name_to_prune::String) = prune_files!(idx, [name_to_prune])
prune_files!(idx::Index, names_to_prune::AbstractVector{String}) = prune_files!(idx, names_to_prune, find_name_indices(idx, names_to_prune))
function prune_files!(idx::Index, names_to_prune::AbstractVector{String}, nameidxs_to_prune::Vector{UInt32})
    isempty(nameidxs_to_prune) && (return idx)
    initial_names_count = UInt32(length(idx.names.entries))
    filter!(name->!(name in names_to_prune), idx.names.entries)

    # create a map of old name index to new index
    offset = 0
    namemap = Dict{UInt32,UInt32}()
    for idx in UInt32(0):initial_names_count
        if idx in nameidxs_to_prune
            offset += 1
        else
            namemap[idx] = idx-offset
        end
    end

    prune!(idx.postings, namemap)
    recreate_indices!(idx)
end

function recreate_indices!(idx::Index)
    idx.offsets.path_list = length(HEADER_BYTES)
    idx.offsets.name_list = path_list_size(idx) + idx.offsets.path_list
    idx.offsets.posting_list = recreate_name_index!(idx) + idx.offsets.name_list
    idx.offsets.name_index = recreate_posting_index!(idx) + idx.offsets.posting_list
    idx.offsets.posting_list_index = idx.offsets.name_index + UInt32(4 * (length(idx.names.entries) + 1))
    idx
end

function path_list_size(idx::Index)
    offset = 0
    for path in idx.paths.entries
        offset += UInt32(sizeof(path) + 1)
    end
    offset += UInt32(1) # for the null byte list terminator
    offset
end

function recreate_name_index!(idx::Index)
    empty!(idx.nameindex.entries)
    offset = UInt32(0)
    for name in idx.names.entries
        push!(idx.nameindex.entries, offset)
        offset += UInt32(sizeof(name) + 1)
    end
    push!(idx.nameindex.entries, offset) # for the end marker that we do not store in idx.names
    offset += UInt32(1) # for the null byte list terminator
    offset
end

function recreate_posting_index!(idx::Index)
    empty!(idx.postingindex.entries)
    offset = UInt32(0)
    for posting in idx.postings
        push!(idx.postingindex.entries, PostingIndexEntry(posting.trigram, UInt32(length(posting.deltas)-1), offset))
        offset += UInt32(3)
        for delta in posting.deltas
            offset += UInt32(nbytes_varint_8(delta))
        end
    end
    offset
end

# -----------------------------------------------
# base data reader and writer methods
# -----------------------------------------------
read_big_endian_4(io) = ntoh(read(io, UInt32))
write_big_endian_4(io, v::UInt32) = write(io, hton(v))

function read_varint_8(io)
    res = zero(UInt64)
    n = 0
    byte = UInt8(0x80)
    while (byte & 0x80) != 0
        byte = read(io, UInt8)
        res |= convert(UInt64, byte & 0x7f) << (7*n)
        n += 1
    end
    @assert n > 0
    res
end

function write_varint_8(io::IO, x::UInt32)
    nw = 0
    cont = true
    while cont
        byte = x & 0x7f
        if (x >>>= 7) != 0
            byte |= 0x80
        else
            cont = false
        end
        nw += write(io, UInt8(byte))
    end
    nw
end

function nbytes_varint_8(x::UInt32)
    nw = 0
    cont = true
    while cont
        byte = x & 0x7f
        if (x >>>= 7) != 0
            byte |= 0x80
        else
            cont = false
        end
        nw += 1
    end
    nw
end

# -----------------------------------------------
# reader methods
# -----------------------------------------------
function read(io::IO, idx::Index)
    total_file_size = UInt32(filesize(io))

    # validate header
    header_bytes = read(io, length(HEADER_BYTES))
    (header_bytes != HEADER_BYTES) && error("Not a valid index file")

    # validate trailer
    seek(io, total_file_size - length(TRAILER_BYTES))
    trailer_bytes = read(io, length(TRAILER_BYTES))
    (trailer_bytes != TRAILER_BYTES) && error("Not a valid index file")

    # read trailer offsets
    seek(io, filesize(io) - length(TRAILER_BYTES) - 5*4)
    read(io, idx.offsets)

    idx.paths = read_strings(io, idx.offsets.path_list, idx.offsets.name_list - idx.offsets.path_list)
    idx.names = read_strings(io, idx.offsets.name_list, idx.offsets.posting_list - idx.offsets.name_list)
    idx.postings = read_postings(io, idx.offsets.posting_list, idx.offsets.name_index - idx.offsets.posting_list)
    idx.nameindex = read_name_index(io, idx.offsets.name_index, idx.offsets.posting_list_index - idx.offsets.name_index)
    idx.postingindex = read_posting_index(io, idx.offsets.posting_list_index, UInt32(total_file_size - length(TRAILER_BYTES) - 5*4 - idx.offsets.posting_list_index))

    idx
end

function read_strings(io, pos::UInt32, nbytes::UInt32)
    seek(io, pos)
    Strings(String.(split(String(read(io, nbytes)), '\0'; keepempty=false)))
end

function read_posting(io)
    trigram = read(io, 3)
    deltas = UInt32[]
    while !eof(io)
        delta = UInt32(read_varint_8(io))
        push!(deltas, delta)
        (delta == 0) && break
    end
    Posting(trigram, deltas)
end

function read_name_index(io, pos::UInt32, nbytes::UInt32)
    seek(io, pos)
    endpos = pos + nbytes
    nameindex = NameIndex()
    indices = nameindex.entries
    while !eof(io) && (position(io) < endpos)
        push!(indices, read_big_endian_4(io))
    end
    nameindex
end

function read_posting_index_entry(io)
    trigram = read(io, 3)
    filecount = read_big_endian_4(io)
    offset = read_big_endian_4(io)
    PostingIndexEntry(trigram, filecount, offset)
end

function read_posting_index(io, pos::UInt32, nbytes::UInt32)
    seek(io, pos)
    endpos = pos + nbytes
    postingindex = PostingIndex()
    indices = postingindex.entries
    while !eof(io) && (position(io) < endpos)
        push!(indices, read_posting_index_entry(io))
    end
    postingindex
end

function read_postings(io, pos::UInt32, nbytes::UInt32)
    seek(io, pos)
    endpos = pos + nbytes
    postings = Posting[]
    # need a position check also because in practice some index files may not contain the trailer bytes
    while !eof(io) && (position(io) < endpos)
        posting = read_posting(io)
        push!(postings, posting)
        (posting.trigram == POSTING_LIST_TRAILER_TRIGRAM) && (posting.deltas == POSTING_LIST_TRAILER_DELTA) && break
    end
    postings
end

function read(io::IO, offsets::IndexTrailerOffsets)
    offsets.path_list = read_big_endian_4(io)
    offsets.name_list = read_big_endian_4(io)
    offsets.posting_list = read_big_endian_4(io)
    offsets.name_index = read_big_endian_4(io)
    offsets.posting_list_index = read_big_endian_4(io)

    offsets
end

# -----------------------------------------------
# writer methods
# -----------------------------------------------
function write(io::IO, offsets::IndexTrailerOffsets)
    n = 0
    n += write_big_endian_4(io, offsets.path_list)
    n += write_big_endian_4(io, offsets.name_list)
    n += write_big_endian_4(io, offsets.posting_list)
    n += write_big_endian_4(io, offsets.name_index)
    n += write_big_endian_4(io, offsets.posting_list_index)
    n
end

function write(io::IO, strings::Strings)
    n = 0
    for str in strings.entries
        n += write(io, str)
        n += write(io, 0x00)
    end
    n += write(io, 0x00)
    n
end

function write(io::IO, posting::Posting)
    n = 0
    n += write(io, posting.trigram)
    for delta in posting.deltas
        n += write_varint_8(io, delta)
    end
    n
end

function write(io::IO, postings::Postings)
    n = 0
    for posting in postings
        n += write(io, posting)
    end
    n
end

function write(io::IO, nameindex::NameIndex)
    n = 0
    for entry in nameindex.entries
        n += write_big_endian_4(io, entry)
    end
    n
end

function write(io::IO, entry::PostingIndexEntry)
    n = 0
    n += write(io, entry.trigram)
    n += write_big_endian_4(io, entry.filecount)
    n += write_big_endian_4(io, entry.offset)
    n
end

function write(io::IO, postingindex::PostingIndex)
    n = 0
    for entry in postingindex.entries
        n += write(io, entry)
    end
    n
end

function write(io::IO, idx::Index)
    n = 0
    n += write(io, HEADER_BYTES)
    n += write(io, idx.paths)
    n += write(io, idx.names)
    n += write(io, idx.postings)
    n += write(io, idx.nameindex)
    n += write(io, idx.postingindex)
    n += write(io, idx.offsets)
    n += write(io, TRAILER_BYTES)
    n
end


text:
class Eclair {
  cook() {
    print "Pipe full of creme patissiere.";
    super.cook();
  }
}

ext: jl


text:
# This file was generated by the Julia Swagger Code Generator
# Do not modify this file directly. Modify the swagger specification instead.

struct ChatApi <: SwaggerApi
    client::Swagger.Client
end

function _swaggerinternal_chatDelete(_api::ChatApi; ts=nothing, channel=nothing, as_user=nothing, _mediaType=nothing)
    _ctx = Swagger.Ctx(_api.client, "POST", Dict{String, Any}, "/chat.delete", ["slackAuth"])
    Swagger.set_param(_ctx.form, "ts", ts)  # type Float32
    Swagger.set_param(_ctx.form, "channel", channel)  # type String
    Swagger.set_param(_ctx.form, "as_user", as_user)  # type Bool
    Swagger.set_header_accept(_ctx, ["application/json"])
    Swagger.set_header_content_type(_ctx, (_mediaType === nothing) ? ["application/x-www-form-urlencoded", "application/json"] : [_mediaType])
    return _ctx
end

"""

Deletes a message.
Param: ts::Float32
Param: channel::String
Param: as_user::Bool
Return: Dict{String, Any}
"""
function chatDelete(_api::ChatApi; ts=nothing, channel=nothing, as_user=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatDelete(_api; ts=ts, channel=channel, as_user=as_user, _mediaType=_mediaType)
    Swagger.exec(_ctx)
end

function chatDelete(_api::ChatApi, response_stream::Channel; ts=nothing, channel=nothing, as_user=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatDelete(_api; ts=ts, channel=channel, as_user=as_user, _mediaType=_mediaType)
    Swagger.exec(_ctx, response_stream)
end

function _swaggerinternal_chatDeleteScheduledMessage(_api::ChatApi, channel::String, scheduled_message_id::String; as_user=nothing, _mediaType=nothing)
    _ctx = Swagger.Ctx(_api.client, "POST", Dict{String, Any}, "/chat.deleteScheduledMessage", ["slackAuth"])
    Swagger.set_param(_ctx.form, "as_user", as_user)  # type Bool
    Swagger.set_param(_ctx.form, "channel", channel)  # type String
    Swagger.set_param(_ctx.form, "scheduled_message_id", scheduled_message_id)  # type String
    Swagger.set_header_accept(_ctx, ["application/json"])
    Swagger.set_header_content_type(_ctx, (_mediaType === nothing) ? ["application/x-www-form-urlencoded", "application/json"] : [_mediaType])
    return _ctx
end

"""

Deletes a pending scheduled message from the queue.
Param: channel::String (required)
Param: scheduled_message_id::String (required)
Param: as_user::Bool
Return: Dict{String, Any}
"""
function chatDeleteScheduledMessage(_api::ChatApi, channel::String, scheduled_message_id::String; as_user=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatDeleteScheduledMessage(_api, channel, scheduled_message_id; as_user=as_user, _mediaType=_mediaType)
    Swagger.exec(_ctx)
end

function chatDeleteScheduledMessage(_api::ChatApi, response_stream::Channel, channel::String, scheduled_message_id::String; as_user=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatDeleteScheduledMessage(_api, channel, scheduled_message_id; as_user=as_user, _mediaType=_mediaType)
    Swagger.exec(_ctx, response_stream)
end

function _swaggerinternal_chatGetPermalink(_api::ChatApi, channel::String, message_ts::String; _mediaType=nothing)
    _ctx = Swagger.Ctx(_api.client, "GET", Dict{String, Any}, "/chat.getPermalink", ["slackAuth"])
    Swagger.set_param(_ctx.query, "channel", channel)  # type String
    Swagger.set_param(_ctx.query, "message_ts", message_ts)  # type String
    Swagger.set_header_accept(_ctx, ["application/json"])
    Swagger.set_header_content_type(_ctx, (_mediaType === nothing) ? ["application/x-www-form-urlencoded"] : [_mediaType])
    return _ctx
end

"""

Retrieve a permalink URL for a specific extant message
Param: channel::String (required)
Param: message_ts::String (required)
Return: Dict{String, Any}
"""
function chatGetPermalink(_api::ChatApi, channel::String, message_ts::String; _mediaType=nothing)
    _ctx = _swaggerinternal_chatGetPermalink(_api, channel, message_ts; _mediaType=_mediaType)
    Swagger.exec(_ctx)
end

function chatGetPermalink(_api::ChatApi, response_stream::Channel, channel::String, message_ts::String; _mediaType=nothing)
    _ctx = _swaggerinternal_chatGetPermalink(_api, channel, message_ts; _mediaType=_mediaType)
    Swagger.exec(_ctx, response_stream)
end

function _swaggerinternal_chatMeMessage(_api::ChatApi; channel=nothing, text=nothing, _mediaType=nothing)
    _ctx = Swagger.Ctx(_api.client, "POST", Dict{String, Any}, "/chat.meMessage", ["slackAuth"])
    Swagger.set_param(_ctx.form, "channel", channel)  # type String
    Swagger.set_param(_ctx.form, "text", text)  # type String
    Swagger.set_header_accept(_ctx, ["application/json"])
    Swagger.set_header_content_type(_ctx, (_mediaType === nothing) ? ["application/x-www-form-urlencoded", "application/json"] : [_mediaType])
    return _ctx
end

"""

Share a me message into a channel.
Param: channel::String
Param: text::String
Return: Dict{String, Any}
"""
function chatMeMessage(_api::ChatApi; channel=nothing, text=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatMeMessage(_api; channel=channel, text=text, _mediaType=_mediaType)
    Swagger.exec(_ctx)
end

function chatMeMessage(_api::ChatApi, response_stream::Channel; channel=nothing, text=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatMeMessage(_api; channel=channel, text=text, _mediaType=_mediaType)
    Swagger.exec(_ctx, response_stream)
end

function _swaggerinternal_chatPostEphemeral(_api::ChatApi, channel::String, user::String; as_user=nothing, attachments=nothing, blocks=nothing, icon_emoji=nothing, icon_url=nothing, link_names=nothing, parse=nothing, text=nothing, thread_ts=nothing, username=nothing, _mediaType=nothing)
    _ctx = Swagger.Ctx(_api.client, "POST", Dict{String, Any}, "/chat.postEphemeral", ["slackAuth"])
    Swagger.set_param(_ctx.form, "as_user", as_user)  # type Bool
    Swagger.set_param(_ctx.form, "attachments", attachments)  # type String
    Swagger.set_param(_ctx.form, "blocks", blocks)  # type String
    Swagger.set_param(_ctx.form, "channel", channel)  # type String
    Swagger.set_param(_ctx.form, "icon_emoji", icon_emoji)  # type String
    Swagger.set_param(_ctx.form, "icon_url", icon_url)  # type String
    Swagger.set_param(_ctx.form, "link_names", link_names)  # type Bool
    Swagger.set_param(_ctx.form, "parse", parse)  # type String
    Swagger.set_param(_ctx.form, "text", text)  # type String
    Swagger.set_param(_ctx.form, "thread_ts", thread_ts)  # type String
    Swagger.set_param(_ctx.form, "user", user)  # type String
    Swagger.set_param(_ctx.form, "username", username)  # type String
    Swagger.set_header_accept(_ctx, ["application/json"])
    Swagger.set_header_content_type(_ctx, (_mediaType === nothing) ? ["application/x-www-form-urlencoded", "application/json"] : [_mediaType])
    return _ctx
end

"""

Sends an ephemeral message to a user in a channel.
Param: channel::String (required)
Param: user::String (required)
Param: as_user::Bool
Param: attachments::String
Param: blocks::String
Param: icon_emoji::String
Param: icon_url::String
Param: link_names::Bool
Param: parse::String
Param: text::String
Param: thread_ts::String
Param: username::String
Return: Dict{String, Any}
"""
function chatPostEphemeral(_api::ChatApi, channel::String, user::String; as_user=nothing, attachments=nothing, blocks=nothing, icon_emoji=nothing, icon_url=nothing, link_names=nothing, parse=nothing, text=nothing, thread_ts=nothing, username=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatPostEphemeral(_api, channel, user; as_user=as_user, attachments=attachments, blocks=blocks, icon_emoji=icon_emoji, icon_url=icon_url, link_names=link_names, parse=parse, text=text, thread_ts=thread_ts, username=username, _mediaType=_mediaType)
    Swagger.exec(_ctx)
end

function chatPostEphemeral(_api::ChatApi, response_stream::Channel, channel::String, user::String; as_user=nothing, attachments=nothing, blocks=nothing, icon_emoji=nothing, icon_url=nothing, link_names=nothing, parse=nothing, text=nothing, thread_ts=nothing, username=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatPostEphemeral(_api, channel, user; as_user=as_user, attachments=attachments, blocks=blocks, icon_emoji=icon_emoji, icon_url=icon_url, link_names=link_names, parse=parse, text=text, thread_ts=thread_ts, username=username, _mediaType=_mediaType)
    Swagger.exec(_ctx, response_stream)
end

function _swaggerinternal_chatPostMessage(_api::ChatApi, channel::String; as_user=nothing, attachments=nothing, blocks=nothing, icon_emoji=nothing, icon_url=nothing, link_names=nothing, mrkdwn=nothing, parse=nothing, reply_broadcast=nothing, text=nothing, thread_ts=nothing, unfurl_links=nothing, unfurl_media=nothing, username=nothing, _mediaType=nothing)
    _ctx = Swagger.Ctx(_api.client, "POST", Dict{String, Any}, "/chat.postMessage", ["slackAuth"])
    Swagger.set_param(_ctx.form, "as_user", as_user)  # type String
    Swagger.set_param(_ctx.form, "attachments", attachments)  # type String
    Swagger.set_param(_ctx.form, "blocks", blocks)  # type String
    Swagger.set_param(_ctx.form, "channel", channel)  # type String
    Swagger.set_param(_ctx.form, "icon_emoji", icon_emoji)  # type String
    Swagger.set_param(_ctx.form, "icon_url", icon_url)  # type String
    Swagger.set_param(_ctx.form, "link_names", link_names)  # type Bool
    Swagger.set_param(_ctx.form, "mrkdwn", mrkdwn)  # type Bool
    Swagger.set_param(_ctx.form, "parse", parse)  # type String
    Swagger.set_param(_ctx.form, "reply_broadcast", reply_broadcast)  # type Bool
    Swagger.set_param(_ctx.form, "text", text)  # type String
    Swagger.set_param(_ctx.form, "thread_ts", thread_ts)  # type String
    Swagger.set_param(_ctx.form, "unfurl_links", unfurl_links)  # type Bool
    Swagger.set_param(_ctx.form, "unfurl_media", unfurl_media)  # type Bool
    Swagger.set_param(_ctx.form, "username", username)  # type String
    Swagger.set_header_accept(_ctx, ["application/json"])
    Swagger.set_header_content_type(_ctx, (_mediaType === nothing) ? ["application/x-www-form-urlencoded", "application/json"] : [_mediaType])
    return _ctx
end

"""

Sends a message to a channel.
Param: channel::String (required)
Param: as_user::String
Param: attachments::String
Param: blocks::String
Param: icon_emoji::String
Param: icon_url::String
Param: link_names::Bool
Param: mrkdwn::Bool
Param: parse::String
Param: reply_broadcast::Bool
Param: text::String
Param: thread_ts::String
Param: unfurl_links::Bool
Param: unfurl_media::Bool
Param: username::String
Return: Dict{String, Any}
"""
function chatPostMessage(_api::ChatApi, channel::String; as_user=nothing, attachments=nothing, blocks=nothing, icon_emoji=nothing, icon_url=nothing, link_names=nothing, mrkdwn=nothing, parse=nothing, reply_broadcast=nothing, text=nothing, thread_ts=nothing, unfurl_links=nothing, unfurl_media=nothing, username=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatPostMessage(_api, channel; as_user=as_user, attachments=attachments, blocks=blocks, icon_emoji=icon_emoji, icon_url=icon_url, link_names=link_names, mrkdwn=mrkdwn, parse=parse, reply_broadcast=reply_broadcast, text=text, thread_ts=thread_ts, unfurl_links=unfurl_links, unfurl_media=unfurl_media, username=username, _mediaType=_mediaType)
    Swagger.exec(_ctx)
end

function chatPostMessage(_api::ChatApi, response_stream::Channel, channel::String; as_user=nothing, attachments=nothing, blocks=nothing, icon_emoji=nothing, icon_url=nothing, link_names=nothing, mrkdwn=nothing, parse=nothing, reply_broadcast=nothing, text=nothing, thread_ts=nothing, unfurl_links=nothing, unfurl_media=nothing, username=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatPostMessage(_api, channel; as_user=as_user, attachments=attachments, blocks=blocks, icon_emoji=icon_emoji, icon_url=icon_url, link_names=link_names, mrkdwn=mrkdwn, parse=parse, reply_broadcast=reply_broadcast, text=text, thread_ts=thread_ts, unfurl_links=unfurl_links, unfurl_media=unfurl_media, username=username, _mediaType=_mediaType)
    Swagger.exec(_ctx, response_stream)
end

function _swaggerinternal_chatScheduleMessage(_api::ChatApi; channel=nothing, text=nothing, post_at=nothing, parse=nothing, as_user=nothing, link_names=nothing, attachments=nothing, blocks=nothing, unfurl_links=nothing, unfurl_media=nothing, thread_ts=nothing, reply_broadcast=nothing, _mediaType=nothing)
    _ctx = Swagger.Ctx(_api.client, "POST", Dict{String, Any}, "/chat.scheduleMessage", ["slackAuth"])
    Swagger.set_param(_ctx.form, "channel", channel)  # type String
    Swagger.set_param(_ctx.form, "text", text)  # type String
    Swagger.set_param(_ctx.form, "post_at", post_at)  # type String
    Swagger.set_param(_ctx.form, "parse", parse)  # type String
    Swagger.set_param(_ctx.form, "as_user", as_user)  # type Bool
    Swagger.set_param(_ctx.form, "link_names", link_names)  # type Bool
    Swagger.set_param(_ctx.form, "attachments", attachments)  # type String
    Swagger.set_param(_ctx.form, "blocks", blocks)  # type String
    Swagger.set_param(_ctx.form, "unfurl_links", unfurl_links)  # type Bool
    Swagger.set_param(_ctx.form, "unfurl_media", unfurl_media)  # type Bool
    Swagger.set_param(_ctx.form, "thread_ts", thread_ts)  # type Float32
    Swagger.set_param(_ctx.form, "reply_broadcast", reply_broadcast)  # type Bool
    Swagger.set_header_accept(_ctx, ["application/json"])
    Swagger.set_header_content_type(_ctx, (_mediaType === nothing) ? ["application/x-www-form-urlencoded", "application/json"] : [_mediaType])
    return _ctx
end

"""

Schedules a message to be sent to a channel.
Param: channel::String
Param: text::String
Param: post_at::String
Param: parse::String
Param: as_user::Bool
Param: link_names::Bool
Param: attachments::String
Param: blocks::String
Param: unfurl_links::Bool
Param: unfurl_media::Bool
Param: thread_ts::Float32
Param: reply_broadcast::Bool
Return: Dict{String, Any}
"""
function chatScheduleMessage(_api::ChatApi; channel=nothing, text=nothing, post_at=nothing, parse=nothing, as_user=nothing, link_names=nothing, attachments=nothing, blocks=nothing, unfurl_links=nothing, unfurl_media=nothing, thread_ts=nothing, reply_broadcast=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatScheduleMessage(_api; channel=channel, text=text, post_at=post_at, parse=parse, as_user=as_user, link_names=link_names, attachments=attachments, blocks=blocks, unfurl_links=unfurl_links, unfurl_media=unfurl_media, thread_ts=thread_ts, reply_broadcast=reply_broadcast, _mediaType=_mediaType)
    Swagger.exec(_ctx)
end

function chatScheduleMessage(_api::ChatApi, response_stream::Channel; channel=nothing, text=nothing, post_at=nothing, parse=nothing, as_user=nothing, link_names=nothing, attachments=nothing, blocks=nothing, unfurl_links=nothing, unfurl_media=nothing, thread_ts=nothing, reply_broadcast=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatScheduleMessage(_api; channel=channel, text=text, post_at=post_at, parse=parse, as_user=as_user, link_names=link_names, attachments=attachments, blocks=blocks, unfurl_links=unfurl_links, unfurl_media=unfurl_media, thread_ts=thread_ts, reply_broadcast=reply_broadcast, _mediaType=_mediaType)
    Swagger.exec(_ctx, response_stream)
end

function _swaggerinternal_chatScheduledMessagesList(_api::ChatApi; channel=nothing, latest=nothing, oldest=nothing, limit=nothing, cursor=nothing, _mediaType=nothing)
    _ctx = Swagger.Ctx(_api.client, "GET", Dict{String, Any}, "/chat.scheduledMessages.list", ["slackAuth"])
    Swagger.set_param(_ctx.query, "channel", channel)  # type String
    Swagger.set_param(_ctx.query, "latest", latest)  # type Float32
    Swagger.set_param(_ctx.query, "oldest", oldest)  # type Float32
    Swagger.set_param(_ctx.query, "limit", limit)  # type Int32
    Swagger.set_param(_ctx.query, "cursor", cursor)  # type String
    Swagger.set_header_accept(_ctx, ["application/json"])
    Swagger.set_header_content_type(_ctx, (_mediaType === nothing) ? ["application/x-www-form-urlencoded", "application/json"] : [_mediaType])
    return _ctx
end

"""

Returns a list of scheduled messages.
Param: channel::String
Param: latest::Float32
Param: oldest::Float32
Param: limit::Int32
Param: cursor::String
Return: Dict{String, Any}
"""
function chatScheduledMessagesList(_api::ChatApi; channel=nothing, latest=nothing, oldest=nothing, limit=nothing, cursor=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatScheduledMessagesList(_api; channel=channel, latest=latest, oldest=oldest, limit=limit, cursor=cursor, _mediaType=_mediaType)
    Swagger.exec(_ctx)
end

function chatScheduledMessagesList(_api::ChatApi, response_stream::Channel; channel=nothing, latest=nothing, oldest=nothing, limit=nothing, cursor=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatScheduledMessagesList(_api; channel=channel, latest=latest, oldest=oldest, limit=limit, cursor=cursor, _mediaType=_mediaType)
    Swagger.exec(_ctx, response_stream)
end

function _swaggerinternal_chatUnfurl(_api::ChatApi, channel::String, ts::String; unfurls=nothing, user_auth_message=nothing, user_auth_required=nothing, user_auth_url=nothing, _mediaType=nothing)
    _ctx = Swagger.Ctx(_api.client, "POST", Dict{String, Any}, "/chat.unfurl", ["slackAuth"])
    Swagger.set_param(_ctx.form, "channel", channel)  # type String
    Swagger.set_param(_ctx.form, "ts", ts)  # type String
    Swagger.set_param(_ctx.form, "unfurls", unfurls)  # type String
    Swagger.set_param(_ctx.form, "user_auth_message", user_auth_message)  # type String
    Swagger.set_param(_ctx.form, "user_auth_required", user_auth_required)  # type Bool
    Swagger.set_param(_ctx.form, "user_auth_url", user_auth_url)  # type String
    Swagger.set_header_accept(_ctx, ["application/json"])
    Swagger.set_header_content_type(_ctx, (_mediaType === nothing) ? ["application/x-www-form-urlencoded", "application/json"] : [_mediaType])
    return _ctx
end

"""

Provide custom unfurl behavior for user-posted URLs
Param: channel::String (required)
Param: ts::String (required)
Param: unfurls::String
Param: user_auth_message::String
Param: user_auth_required::Bool
Param: user_auth_url::String
Return: Dict{String, Any}
"""
function chatUnfurl(_api::ChatApi, channel::String, ts::String; unfurls=nothing, user_auth_message=nothing, user_auth_required=nothing, user_auth_url=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatUnfurl(_api, channel, ts; unfurls=unfurls, user_auth_message=user_auth_message, user_auth_required=user_auth_required, user_auth_url=user_auth_url, _mediaType=_mediaType)
    Swagger.exec(_ctx)
end

function chatUnfurl(_api::ChatApi, response_stream::Channel, channel::String, ts::String; unfurls=nothing, user_auth_message=nothing, user_auth_required=nothing, user_auth_url=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatUnfurl(_api, channel, ts; unfurls=unfurls, user_auth_message=user_auth_message, user_auth_required=user_auth_required, user_auth_url=user_auth_url, _mediaType=_mediaType)
    Swagger.exec(_ctx, response_stream)
end

function _swaggerinternal_chatUpdate(_api::ChatApi, channel::String, ts::String; as_user=nothing, attachments=nothing, blocks=nothing, link_names=nothing, parse=nothing, text=nothing, _mediaType=nothing)
    _ctx = Swagger.Ctx(_api.client, "POST", Dict{String, Any}, "/chat.update", ["slackAuth"])
    Swagger.set_param(_ctx.form, "as_user", as_user)  # type String
    Swagger.set_param(_ctx.form, "attachments", attachments)  # type String
    Swagger.set_param(_ctx.form, "blocks", blocks)  # type String
    Swagger.set_param(_ctx.form, "channel", channel)  # type String
    Swagger.set_param(_ctx.form, "link_names", link_names)  # type String
    Swagger.set_param(_ctx.form, "parse", parse)  # type String
    Swagger.set_param(_ctx.form, "text", text)  # type String
    Swagger.set_param(_ctx.form, "ts", ts)  # type String
    Swagger.set_header_accept(_ctx, ["application/json"])
    Swagger.set_header_content_type(_ctx, (_mediaType === nothing) ? ["application/x-www-form-urlencoded", "application/json"] : [_mediaType])
    return _ctx
end

"""

Updates a message.
Param: channel::String (required)
Param: ts::String (required)
Param: as_user::String
Param: attachments::String
Param: blocks::String
Param: link_names::String
Param: parse::String
Param: text::String
Return: Dict{String, Any}
"""
function chatUpdate(_api::ChatApi, channel::String, ts::String; as_user=nothing, attachments=nothing, blocks=nothing, link_names=nothing, parse=nothing, text=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatUpdate(_api, channel, ts; as_user=as_user, attachments=attachments, blocks=blocks, link_names=link_names, parse=parse, text=text, _mediaType=_mediaType)
    Swagger.exec(_ctx)
end

function chatUpdate(_api::ChatApi, response_stream::Channel, channel::String, ts::String; as_user=nothing, attachments=nothing, blocks=nothing, link_names=nothing, parse=nothing, text=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_chatUpdate(_api, channel, ts; as_user=as_user, attachments=attachments, blocks=blocks, link_names=link_names, parse=parse, text=text, _mediaType=_mediaType)
    Swagger.exec(_ctx, response_stream)
end

export chatDelete, chatDeleteScheduledMessage, chatGetPermalink, chatMeMessage, chatPostEphemeral, chatPostMessage, chatScheduleMessage, chatScheduledMessagesList, chatUnfurl, chatUpdate


text:
__precompile__(true)

module Pinecone

using JSON3
using StructTypes

export PineconeContext, PineconeIndex, PineconeVector, whoami, init, list_indexes, query, upsert, delete_index, describe_index_stats, create_index, fetch, delete

struct PineconeContext
    apikey::String
    cloudenvironment::String
    projectname::String
end
Base.show(io::IO, ctx::PineconeContext) = print(io, ctx.apikey, " / ", ctx.cloudenvironment, " / ", ctx.projectname)

struct PineconeIndex
    indexname::String
end
Base.show(io::IO, index::PineconeIndex) = print(io, "PineconeIndex connected to ", index.indexname)

struct PineconeVector
    id::String
    values::Array{<:AbstractFloat}
    metadata::Dict{String, Any}
end
Base.show(io::IO, vec::PineconeVector) = print(io, "PineconeVector is id: ", vec.id, " values: ", vec.values, " meta: ", vec.metadata)
# Make PineconeVector type JSON3 writable per https://quinnj.github.io/JSON3.jl/stable/#Read-JSON-into-a-type
StructTypes.StructType(::Type{PineconeVector}) = StructTypes.UnorderedStruct()

include("networking.jl");

const ENDPOINTWHOAMI = "actions/whoami"
const ENDPOINTLISTINDEXES = "databases"
const ENDPOINTDESCRIBEINDEXSTATS = "describe_index_stats"
const ENDPOINTDELETEINDEX = "databases"
const ENDPOINTQUERYINDEX = "query"
const ENDPOINTUPSERT = "vectors/upsert"
const ENDPOINTCREATEINDEX = "databases"
const ENDPOINTFETCH = "vectors/fetch"
const ENDPOINTDELETE = "vectors/delete"
const ENDPOINTSCALEINDEX = "databases"
const MAX_UPSERT_VECTORS = 1000
const MAX_FETCH = 1000
const MAX_TOPK = 10000
const MAX_TOPK_WITH_META = 1000
const MAX_TOPK_WITH_DATA = 1000
const MAX_DIMS = 10000
const MAX_DELETE = 1000

function __init__()
    nothing
end

"""
    init(apikey::String, environment::String)

Initialize the Pinecone environment using your API Key and a specific environment.

This returns a PineconeContext instance which you'll use for subsequent calls such as query() and upsert().
Your API Key and the cloud environment for your indexes are easily found in the Pinecone console.
On failure returns nothing.

# Example
```julia
using Pinecone
context = Pinecone.init("abcd-123456-zyx", "us-west1-gcp")
```
"""
function init(apikey::String, environment::String)
    response = pineconeHTTPGet(pineconeMakeURLForController(environment, ENDPOINTWHOAMI), apikey)
    if response != nothing && response.status == 200
        rvdict = pineconeGetDict(String(response.body))
        return PineconeContext(apikey, environment, rvdict["project_name"])
    end
end #init

# note no repsonse body from create, derive success from the HTTP 204
"""
   create_index(ctx::PineconeContext, indexname::String, dimension::Int64; metric::String="cosine", indextype::String="approximated", pods::Int64=1, replicas::Int64=1, shards::Int64=1, podtype::String="p1", indexconfig=Dict{String, Any}())

Creates an index with a given PineconeContext, which can be accessed by a call to init(), the name of the index, and the number of dimensions.

Note that there are other parameters for the distance metric, indextype, number of replicas and shards as well as the indexconfig.
This function returns a JSON blob as a string, or nothing if it failed. Do recommend using JSON3 to parse the blob.

# Example
```julia
context = Pinecone.init("abcd-123456-zyx", "us-west1-gcp")
result = Pinecone.create_index(context, testindexname, 10, metric="euclidean", indextype="approximated",replicas=2, shards=1, indexconfig=indexconfig)
```
"""
function create_index(ctx::PineconeContext, indexname::String, dimension::Int64; metric::String="cosine", indextype::String="approximated", pods::Int64=1, replicas::Int64=1, shards::Int64=1, podtype::String="p1", indexconfig=Dict{String, Any}())
    if(dimension > MAX_DIMS)
        throw(ArgumentError("Creating index larger than max dimension size of " * string(MAX_DIMS)))
    end
    if(metric != "cosine" && metric != "dotproduct" && metric != "euclidean")
        throw(ArgumentError("Invalid index type.  Type must be one of 'euclidean', 'cosine', or 'dotproduct'."))
    end
    if(pods <= 0)
        throw(ArgumentError("Number of pods must be > 0."))
    end
    if(shards <= 0)
        throw(ArgumentError("Number of shards must be > 0."))
    end
    if(podtype != "p1" && podtype != "s1")
        throw(ArgumentError("Illegal pod type.  Must be p1 or s1."))
    end
    url = pineconeMakeURLForController(ctx.cloudenvironment, ENDPOINTCREATEINDEX)
    postbody = Dict{String, Any}("name"=>indexname, "dimension"=>dimension, "metric"=>metric, "replicas"=>replicas, "shards"=>shards, "pods"=>pods)
    if indextype !== ""
        postbody["index_type"] = indextype
    end
    if length(indexconfig) > 0
        postbody["index_config"] = indexconfig
    end
    response = pineconeHTTPPost(url, ctx, JSON3.write(postbody))
    response != nothing && (response.status == 200 ||response.status == 204)
end  #create_index

"""
    Index(indexname::String)

Returns a PineconeIndex type which is used for query() and upsert() of data against a specific index.

# Example
```julia-repl
julia> Pinecone.Index("my-index")
PineconeIndex connected to my-index
```
"""
function Index(indexname::String)
    PineconeIndex(indexname);
end #Index

"""
    upsert(ctx::PineconeContext, indexobj::PineconeIndex, vectors::Vector{PineconeVector}, namespace::String="")

upserts a Julia Vector of type PineconeVector using the given PineconeContext and PineconeIndex with an optional namespace (Defaults to not being applied to query if not passed.)
On success returns a JSON blob as a String type, and nothing if it fails.
This function returns a JSON blob as a string, or nothing if it failed. Do recommend using JSON3 to parse the blob.

# Example
```julia-repl
julia> testvector = Pinecone.PineconeVector("testid", [0.3,0.11,0.3,0.3,0.3,0.3,0.3,0.3,0.4,0.3])
context = Pinecone.init("abcd-123456-zyx", "us-west1-gcp")
index = PineconeIndex("myindex")
result = Pinecone.upsert(context, index, [testvector], "testnamespace")
```
"""
function upsert(ctx::PineconeContext, indexobj::PineconeIndex, vectors::Vector{PineconeVector}, namespace::String="")
    if(length(vectors) > MAX_UPSERT_VECTORS)
        throw(ArgumentError("Max number of vectors per upsert is " * string(MAX_UPSERT_VECTORS)))
    end
    url = pineconeMakeURLForIndex(indexobj, ctx, ENDPOINTUPSERT)
    body = Dict{String, Any}("vectors" => vectors)
    if namespace !== nothing && namespace != ""
        body["namespace"] = namespace
    end
    postbody = JSON3.write(body)
    response = pineconeHTTPPost(url, ctx, postbody)
    if response != nothing && response.status == 200
        return String(response.body)
    end
end #upsert

"""
    upsert(ctx::PineconeContext, indexobj::PineconeIndex, ids::Array{String}, vectors::Vector{Vector{T}}, meta::Array{Dict{String,Any}}=Dict{String,Any}[], namespace::String="") where {T<:AbstractFloat}

upserts an array of vectors into PineconeContext and PineconeIndex with an optional metadata
and namespace (Defaults to not being applied to query if not passed.)
On success returns a JSON blob as a String type, and nothing if it fails.
This function returns a JSON blob as a string, or nothing if it failed. Do recommend using JSON3 to parse the blob.
If the length of ids and vectors is not equal, which it must be, ``ArgumentError`` is thrown.
If metadata is passed in, that will be tested for length equality with ids and vectors length, with ``ArgumentError`` also thrown is not aligned on length.
# Example
```julia-repl
julia> context = Pinecone.init("abcd-123456-zyx", "us-west1-gcp")
index = PineconeIndex("myindex")
meta = [Dict{String,Any}("foo"=>"bar"), Dict{String,Any}("bar"=>"baz")]
result = Pinecone.upsert(pinecone_context, pinecone_index, ["zipA", "zipB"], [[0.1, 0.2, 0.3, 0.4, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3],
[0.9, 0.8, 0.7, 0.6, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3]], meta, "mynamespace")
```
"""

function upsert(ctx::PineconeContext, indexobj::PineconeIndex, ids::Array{String}, vectors::Vector{Vector{T}},
    meta::Array{Dict{String,Any}}=Dict{String,Any}[], namespace::String="") where {T<:AbstractFloat}
    numvectors = length(vectors)
    if(numvectors > MAX_UPSERT_VECTORS)
        throw(ArgumentError("Max number of vectors per upsert is " * string(MAX_UPSERT_VECTORS)))
    end
    numids = length(ids)
    nummeta = (meta !== nothing ? length(meta) : 0)
    if length(ids) !== numvectors
        throw(ArgumentError("Length of ids does not match length of vectors: $numids vs $numvectors"))
    end
    if nummeta > 0 && nummeta !== numids
        throw(ArgumentError("Length of ids does not match length of metadata: $numids vs $nummeta"))
    end
    pcvectors = [PineconeVector(ids[i], vectors[i], nummeta > 0 ? meta[i] : Dict{String,Any}()) for i in 1:numvectors]
    upsert(ctx, indexobj, pcvectors, namespace)
end

"""
   query(ctx::PineconeContext, indexobj::PineconeIndex, queries::Vector{PineconeVector}, topk::Int64=10, namespace::String="", includevalues::Bool=false, includemeta::Bool=false, filter::Dict{String, Any}=Dict{String,Any}())

Query an index using the given context that match ``queries`` passed in.  The ``PineconeVector`` that is queries is a simple ``PineconeVector`` as described above.
Note there is an another version of ``query()`` that takes in a ``Vector{Vector{<:AbstractFloat}}`` for the ``queries`` parameter.  Functionally equivalent.
Returns JSON blob as a ``String`` with the results on success, ``nothing`` on failure.

topk is an optional parameter which defaults to 10 if not specified.  Do recommend using JSON3 to parse the blob.

# Example
```julia-repl
julia> testdict = Dict{String, Any}("genre"=>"documentary", "year"=>2019);
julia> pinecone_context = Pinecone.init("abcdef-1234-zyxv", "us-west1-gcp")
abcdef-1234-zyxv / us-west1-gcp / 1c9c2f3
julia> pinecone_index = Pinecone.Index("filter-example");
julia> testvector = Pinecone.PineconeVector("testid", [0.3,0.11,0.3,0.3,0.3,0.3,0.3,0.3,0.4,0.3], testdict)
PineconeVector is id: testid values: [0.3, 0.11, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.4, 0.3]meta: Dict{String, Any}("genre" => "documentary", "year" => 2019)
julia> Pinecone.query(pinecone_context, pinecone_index, [testvector], 4)
"{\"results\":[{\"matches\":[{\"id\":\"testid\",\"score\":3.98966023e-07,\"values\":[0.3,0.11,0.3,0.3,0.3,0.3,0.3,0.3,0.4,0.3]},{\"id\":\"C\",\"score\":0.0461001098,\"values\":[0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3]}
],\"namespace\":\"\"}]}"
```
"""
function query(ctx::PineconeContext, indexobj::PineconeIndex, queries::Vector{PineconeVector}, topk::Int64=10, namespace::String="", includevalues::Bool=false, includemeta::Bool=false, filter::Dict{String, Any}=Dict{String,Any}())
    rawvectors = [queries[i].values for i in 1:length(queries)]
    query(ctx, indexobj, rawvectors, topk, namespace, includevalues, includemeta, filter)
end

"""
   query(ctx::PineconeContext, indexobj::PineconeIndex, queries::Vector{Vector{T}}, topk::Int64=10, namespace::String="", includevalues::Bool=false, includemeta::Bool=false, filter::Dict{String, Any}=Dict{String,Any}()) where {T<:AbstractFloat}

Query an index using the given context that match ``queries`` passed in. Returns JSON blob as a ``String`` with the results on success, ``nothing`` on failure.
Note there is an alternate form for ``query()`` that takes in a ``Vector{PineconeVector}`` instead.  Functionally equivalent.
topk is an optional parameter which defaults to 10 if not specified.  Do recommend using JSON3 to parse the blob.

# Example
```julia-repl
julia> testdict = Dict{String, Any}("genre"=>"documentary", "year"=>2019);
julia> pinecone_context = Pinecone.init("abcdef-1234-zyxv", "us-west1-gcp")
abcdef-1234-zyxv / us-west1-gcp / 1c9c2f3
julia> pinecone_index = Pinecone.Index("filter-example");
julia> testvector = Pinecone.PineconeVector("testid", [0.3,0.11,0.3,0.3,0.3,0.3,0.3,0.3,0.4,0.3], testdict)
PineconeVector is id: testid values: [0.3, 0.11, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.4, 0.3]meta: Dict{String, Any}("genre" => "documentary", "year" => 2019)
julia> Pinecone.query(pinecone_context, pinecone_index,
[[0.2, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3], [0.2, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3]], 4)
"{\"results\":[{\"matches\":[{\"id\":\"testid\",\"score\":3.98966023e-07,\"values\":[0.3,0.11,0.3,0.3,0.3,0.3,0.3,0.3,0.4,0.3]},{\"id\":\"C\",\"score\":0.0461001098,\"values\":[0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3]}
],\"namespace\":\"\"}]}"
```
"""

function query(ctx::PineconeContext, indexobj::PineconeIndex, queries::Vector{Vector{T}}, topk::Int64=10, namespace::String="", includevalues::Bool=false,
        includemeta::Bool=false, filter::Dict{String, Any}=Dict{String,Any}()) where {T<:AbstractFloat}
    if topk > MAX_TOPK
        throw(ArgumentError("topk larger than largest topk available of " * string(MAX_TOPK)))
    end
    if includevalues == true && topk > MAX_TOPK_WITH_DATA
        throw(ArgumentError("topk larger than largest topk available of " * string(MAX_TOPK_WITH_DATA) * " when including data in results"))
    end
    if includemeta == true && topk > MAX_TOPK_WITH_META
        throw(ArgumentError("topk larger than largest topk available of " * string(MAX_TOPK_WITH_META) * " when including meatadata in results"))
    end
    url = pineconeMakeURLForIndex(indexobj, ctx, ENDPOINTQUERYINDEX)
    body = Dict{String, Any}("topK"=>topk, "includeValues"=>includevalues, "includeMetadata"=>includemeta, "namespace"=>namespace)
    if(length(filter) > 0)
        body["filter"] = filter;
    end
    body["queries"] =  []
    for vec in queries
        push!(body["queries"], Dict{String, Any}("values"=>vec))
    end
    postbody = JSON3.write(body)
    response = pineconeHTTPPost(url, ctx, postbody)
    if response != nothing && (response.status == 200 || response.status == 400)
        return String(response.body)
    end
end #query

"""
    fetch(ctx::PineconeContext, indexobj::PineconeIndex, ids::Array{String}, namespace::String)

Fetches vectors based on the vector ids for each vector, provided as the ``ids`` array for a given namespace.  .
Note that namespace is optional, and if not provided defaults to not using it to filter query results by namespace.
Returns JSON blob as ``String`` show below, or ``nothing`` on failure.
# Example
```julia-repl
julia> context = Pinecone.init("asdf-1234-zyxv", "us-west1-gcp")
index = PineconeIndex("my-index")
Pinecone.fetch(context, index, ["testid", "testid2"], "testnamespace")
{"vectors":{"testid":{"id":"testid","values":[0.3,0.11,0.3,0.3,0.3,0.3,0.3,0.3,0.4,0.3],"metadata":{"year":2019,"genre":"documentary"}},
"testid2":{"id":"testid2","values":[0.3,0.11,0.3,0.3,0.3,0.3,0.3,0.3,0.4,0.3],"metadata":{"genre":"documentary","year":2019}}},"namespace":"testnamespace"}
```
"""
function fetch(ctx::PineconeContext, indexobj::PineconeIndex, ids::Array{String}, namespace::String="")
    if(length(ids) > MAX_FETCH)
        throw(ArgumentError("Max number of vectors per fetch is " * string(MAX_FETCH)))
    end
    renamedids = ["ids=$row" for row in ids]
    urlargs = "?" * join(renamedids, "&") * "&namespace=$namespace"
    url = pineconeMakeURLForIndex(indexobj, ctx, ENDPOINTFETCH) * urlargs
    response = pineconeHTTPGet(url, ctx)
    if response != nothing && (response.status == 200 || response.status == 400)
        return String(response.body)
    end
end

"""
    delete(ctx::PineconeContext, indexobj::PineconeIndex, ids::Array{String}, deleteall::Bool, namespace::String)

Deletes vectors indexed by ``ids``.  **Warning:** The entire namespace for the vector will be deleted if deleteall=true

Returns JSON blob as ``String`` of empty object or nothing if fails.
# Example
```julia-repl
julia> context = Pinecone.init("asdf-1234-zyxv", "us-west1-gcp")
index = PineconeIndex("my-index")
Pinecone.delete(context, index, ["deleteme1", "deleteme2"], false, "timnamespace")
{}
```
"""
function delete(ctx::PineconeContext, indexobj::PineconeIndex, ids::Array{String}, deleteall::Bool=false, namespace::String="")
    if(length(ids) > MAX_DELETE)
        throw(ArgumentError("Max number of vectors per delete is " * string(MAX_DELETE)))
    end
    renamedids = ["ids=$row" for row in ids]
    urlargs = "?" * join(renamedids, "&") * "&deleteAll=" * string(deleteall) * "&namespace=$namespace"
    url = pineconeMakeURLForIndex(indexobj, ctx, ENDPOINTDELETE) * urlargs
    response = pineconeHTTPDelete(url, ctx)
    if response != nothing && (response.status == 200 || response.status == 400)
        return String(response.body)
    end
end

"""
    list_indexes(context::PineconeContext)

Returns a JSON array listing indexes for a given account, which is indicated by the PineconeContext instance passed in.

# Example
```julia-repl
julia> context = Pinecone.init("asdf-1234-zyxv", "us-west1-gcp")
Pinecone.list_indexes(context)
["example-index", "filter-example"]
```
"""
function list_indexes(context::PineconeContext)
    response = pineconeHTTPGet(pineconeMakeURLForController(context.cloudenvironment, ENDPOINTLISTINDEXES), context)
    if response != nothing && response.status == 200
        return String(response.body)
    end
end

"""
    whoami(context::PineconeContext)

Returns JSON blob with information about your connection to Pinecone.

# Example
```julia-repl
julia> context = Pinecone.init("abcd-1234-zyxv", "us-west1-gcp")
Pinecone.whoami(context)
```
"""
function whoami(context::PineconeContext)
    response = pineconeHTTPGet(pineconeMakeURLForController(context.cloudenvironment, ENDPOINTWHOAMI), context)
    if response != nothing && response.status == 200
        return String(response.body)
    end
end

"""
    delete_index(ctx::PineconeContext, indexobj::PineconeIndex)

Deletes an index, returns true on successful response from Pinecone backend.

This delets a given Pinecone index, note that this is an asynch call and doesn't guarantee that on return that the index is actually deleted (yet).

# Example
```julia
Pinecone.delete_index(context, Pinecone.Index("index-to-delete"))
```
"""
function delete_index(ctx::PineconeContext, indexobj::PineconeIndex)
    url = pineconeMakeURLForController(ctx.cloudenvironment, ENDPOINTDELETEINDEX * "/" * indexobj.indexname)
    response = pineconeHTTPDelete(url, ctx)
    response !== nothing && response.status == 204
end

"""
    describe_index_stats(ctx::PineconeContext, indexobj::PineconeIndex)

Returns JSON blob as a String type describing a particular index.  Returns ``nothing`` on failure.

# Example
```julia
context = Pinecone.init("abcde-1234-zyxv", "us-west1-gcp")
index = PineconeIndex("my-index")
Pinecone.describe_index_stats(context, index)
"namespaces":{"test_namespace":{"vectorCount":1},"testnamespace":{"vectorCount":1},"":{"vectorCount":5}},"dimension":10}
```
"""
function describe_index_stats(ctx::PineconeContext, indexobj::PineconeIndex)
    url = pineconeMakeURLForIndex(indexobj, ctx, ENDPOINTDESCRIBEINDEXSTATS)
    response = pineconeHTTPGet(url, ctx)
    if response != nothing && response.status == 200
        return String(response.body)
    end
end

"""
    scale_index(ctx::PineconeContext, indexobj::PineconeIndex, replicas::Int64)

Scales out the number of replicas for a given index.

# Example
```julia
context = Pinecone.init("abcde-1234-zyxv", "us-west1-gcp")
index = PineconeIndex("my-index")
result = Pinecone.scale_index(context, index, 2)
```
"""
function scale_index(ctx::PineconeContext, indexobj::PineconeIndex, replicas::Int64)
    if replicas < 0
        throw(ArgumentError("Number of replicas should be greater than 0"))
    end
    postbody = Dict{String, Any}("replicas"=>replicas)
    url = pineconeMakeURLForController(ctx.cloudenvironment, ENDPOINTSCALEINDEX) * "/" * indexobj.indexname
    response = pineconeHTTPPatch(url, ctx, JSON3.write(postbody))
    # we get back 204, docs say 200 but they're wrong
    # https://www.pinecone.io/docs/api/operation/scale_index/
    response != nothing && (response.status == 200 || response.status == 204)
end

end # module


text:
using Plots
using Plots.Measures
using DataFrames
using LinearAlgebra
using JSON

using ACEtb
using ACEtb.FitTools
using ACEtb.AnalysisTools
using LaTeXStrings

##

const DFTB_PATH = "$(@__DIR__)/../dftbplus/build/install/bin/"
const OUTPUT_DIR = joinpath(@__DIR__, "Results_fits/")
const PARAM_FILE = "Al-Al.acetb.json"

fc = FitContext(@__DIR__, DFTB_PATH, OUTPUT_DIR, PARAM_FILE)

results = tabulate_results(fc)
filter!(x -> !startswith(x.param_hash, "order"), results)

subresults = dropmissing(results)
subresults = subresults[(subresults.rcut .== 10.0) .& (subresults.N_train .<= 2) .& (subresults.rtol .== 1e-8), :]

gdf = groupby(subresults, :config)

degplots = [plot(; yscale=:log10, legend=:bottomleft, xlabel="Degree",
                ylabel="Training error")
            plot(; yscale=:log10, legend=:false, xlabel="Degree",
                ylabel="Band error FCC", ylim=(10^0.2, 10^0.4))
            plot(; yscale=:log10, legend=:false, xlabel="Degree",
                ylabel="Band error BCC")] #ylim=(10^0.2, 10^0.4))]
for df in gdf
    for order in (2, 3)
        nrow(df) <= 1 && continue
        degree = df[df.order .== order, :degree]
        train_err = df[df.order .== order, :error_train]
        band_err_fcc = df[df.order .== order, :band_error_FCC]
        band_err_bcc = df[df.order .== order, :band_error_BCC]
        config = df.config[df.order .== order][1]
        sortorder = sortperm(degree)
        plot!(degplots[1], degree[sortorder],
              train_err[sortorder], linewidth=2,
              label="Order $order $config")
        plot!(degplots[2], degree[sortorder],
              band_err_fcc[sortorder], linewidth=2,
              label="Order $order $config")
        plot!(degplots[3], degree[sortorder],
              band_err_bcc[sortorder], linewidth=2,
              label="Order $order $config")
    end
end

rcut_plot = [plot(; xlabel="Cutoff radius", ylabel="Training Error", legend=:bottomleft, yscale=:log10),
             plot(; xlabel="Cutoff radius", ylabel="FCC Band Error", legend=false, yscale=:log10),
             plot(; xlabel="Cutoff radius", ylabel="BCC Band Error", legend=false, yscale=:log10)]

order = 2
for (color, config_type) in enumerate(["highT", "lowT", "mixT"])
    rcut_mask =  ((results.config .== config_type) .&
                    (results.rcut .<= 11.0) .&
                    (results.order .== order) .&
                    (results.rcut .> 6.0) .&
                    (results.N_train .<= 2) .&
                    (results.pcut .== 1))
    if config_type == "highT"
        rcut_mask .&= results.degree .== 10
    else
        rcut_mask .&= results.degree .== 10
    end
    rcut_mask = findall(rcut_mask)
    rcut_mask = rcut_mask[sortperm(results[rcut_mask, :rcut])]
    @show order config_type rcut_mask, results[rcut_mask, [:rcut, :pot_file]]

    plot!(rcut_plot[1], results[rcut_mask, :rcut], marker=:dot,
            markersize=3, markerstrokecolor=color, color=color,
            results[rcut_mask, :error_train], linewidth=2,
            label="Order $order $config_type")

    plot!(rcut_plot[2], results[rcut_mask, :rcut], marker=:dot,
            markersize=3, markerstrokecolor=color, color=color,
            results[rcut_mask, :band_error_FCC], linewidth=2,
            label="Order $order $config_type")

    plot!(rcut_plot[3], results[rcut_mask, :rcut], marker=:dot,
            markersize=3, markerstrokecolor=color, color=color,
            results[rcut_mask, :band_error_BCC], linewidth=2,
            label="Order $order $config_type")

end

rp = plot(rcut_plot..., layout=(1, 3), size=(1200, 400))
savefig("rcut.pdf")

dp = plot(degplots..., layout=(1, 3), size=(1200, 400))
savefig("degree.pdf")

# bp = comp_plot(fc, "order2_deg10_envdeg4_rcut11_renv3.0_conf15_pcut1_Ntrain1",
#                label="mixT Order 2 Deg 10 rcut 11")
# savefig("bands.pdf")

# bp

##

N_train = 100 * 321

p1 = scatter(results.N_bond, results.error_train ./ sqrt.(results.N_bond), xscale=:log10, yscale=:log10, label="Train", legend=:bottomright)
xlabel!(L"N_\mathrm{bonds}")
ylabel!("RMSE Hamiltonian")

scatter!(results.N_bond, results.error_test ./ sqrt(N_train), xscale=:log10, yscale=:log10, label="Test")
# xlabel!(L"N_\mathrm{bonds}")
# ylabel!("RMSE Hamiltonian")

p3 = scatter(results.N_bond, results.band_error_FCC, xscale=:log10, yscale=:log10, label="FCC")
scatter!(results.N_bond, results.band_error_BCC, xscale=:log10, yscale=:log10, label="BCC")

xlabel!(L"N_\mathrm{bonds}")
ylabel!(L"\mathrm{Band\; error}")

p = plot(p1, p3, size=(600, 300), layout=(1, 2), leftmargin=5Plots.mm, bottommargin=5Plots.mm)

##

println("Loading test data...")
param_hash = results[argmin(results.N_train), :param_hash] # pick smallest model
pot_dict, _, Bint_table, cutf_func = load_model(fc, param_hash)
test_dataset = pot_dict["test_datasets"]
test_data = ACEtb.Utils.get_data(test_dataset, cutf_func, ACEtb.Bonds.get_env)
println("done")

##

test_errors = Dict()
train_errors = Dict()

for param_hash in sort(results, :N_train)[1:5, :param_hash]
    @show param_hash
    pot_dict, _, Bint_table, cutf_func = load_model(fc, param_hash)
    test_errors[param_hash] = hcat(test_error(test_data, Bint_table)...)
    println("Test error: 2-norm $(norm(test_errors[param_hash])), inf-norm $(norm(test_errors[param_hash],Inf)), rms $(mean(sqrt(test_errors[param_hash].^2)))")

    train_dataset = pot_dict["training_datasets"]
    train_data = ACEtb.Utils.get_data(test_dataset, cutf_func, ACEtb.Bonds.get_env)
    train_errors[param_hash] = hcat(test_error(train_data, Bint_table)...)
    println("Test error: 2-norm $(norm(train_errors[param_hash])), inf-norm $(norm(train_errors[param_hash],Inf)), rms $(mean(sqrt(train_errors[param_hash].^2)))")
end


text:
using ReplMaker
using REPL
using Serialization
using Sockets

struct RemoteException <: Exception
    msg::String
end

function Base.showerror(io::IO, e::RemoteException)
    # Here, we presume that e.msg is fully formatted.
    indented_msg = join("  " .* split(e.msg, '\n'), '\n')
    print(io, "RemoteException:\n", indented_msg)
end

# Super dumb macro expander which expands calls to only a single macro
# `macro_name` which is implemented as `func` taking the expressions passed to
# the macro. Complexities which are ignored:
#   * Hygiene / scoping - all symbols are non-hygenic
#   * Nesting of macro expansions - `macro_name` is expanded first regardless
#     of how it's nested within other macros in the source.
function simple_macro_expand!(func, ex, macro_name)
    if Meta.isexpr(ex, :macrocall) && ex.args[1] == macro_name
        return func(ex.args[3:end]...)
    elseif Meta.isexpr(ex, (:quote, :inert, :meta))
        # pass
    elseif ex isa Expr
        map!(ex.args, ex.args) do x
            simple_macro_expand!(func, x, macro_name)
        end
    end
    return ex
end

# Read and verify header bytes on initializing the connection
function verify_header(io, ser_version=Serialization.ser_version)
    magic = String(read(io, length(PROTOCOL_MAGIC)))
    if magic != PROTOCOL_MAGIC
        if !isopen(io)
            error("RemoteREPL stream was closed while reading header")
        else
            error("RemoteREPL protocol magic number mismatch: $(repr(magic)) != $(repr(PROTOCOL_MAGIC))")
        end
    end
    version = read(io, typeof(PROTOCOL_VERSION))
    if version != PROTOCOL_VERSION
        error("RemoteREPL protocol version number mismatch: $version != $PROTOCOL_VERSION")
    end
    # Version 1: We rely on the standard Serialization library for simplicity;
    # that's backward but not forward compatible depending on
    # Serialization.ser_version, so we check for an exact match
    remote_ser_version = read(io, UInt32)
    if remote_ser_version != ser_version
        error("""
              RemoteREPL Julia Serialization format version mismatch: $remote_ser_version != $ser_version
              Try using the same version of Julia on the server and client.
              """)
    end
    return true
end

#-------------------------------------------------------------------------------
# Client side connection / session handling
mutable struct Connection
    host
    port
    tunnel
    ssh_opts
    region
    namespace
    socket
    in_module
end

function Connection(; host=Sockets.localhost, port::Integer=DEFAULT_PORT,
                    tunnel::Symbol = host!=Sockets.localhost ? :ssh : :none,
                    ssh_opts=``, region=nothing, namespace=nothing)
    conn = Connection(host, port, tunnel, ssh_opts, region, namespace, nothing, :Main)
    setup_connection!(conn)
    finalizer(close, conn)
end

Base.isopen(conn::Connection) = !isnothing(conn.socket) && isopen(conn.socket)

function setup_connection!(conn::Connection)
    socket = if conn.tunnel == :none
        connect(conn.host, conn.port)
    else
        connect_via_tunnel(conn.host, conn.port; retry_timeout=5,
            tunnel=conn.tunnel, ssh_opts=conn.ssh_opts, region=conn.region,
            namespace=conn.namespace)
    end
    try
        verify_header(socket)
    catch exc
        close(socket)
        rethrow()
    end
    conn.socket = socket
    if conn.in_module != :Main
        send_and_receive(conn, (:in_module, conn.in_module))
    end
    conn
end

function ensure_connected!(conn)
    if !isopen(conn)
        @info "Connection dropped, attempting reconnect"
        setup_connection!(conn)
    end
end

function Base.close(conn::Connection)
    try
        if !isnothing(conn.socket) && isopen(conn.socket)
            serialize(conn.socket, (:exit,nothing))
            close(conn.socket)
        end
    finally
        conn.socket = nothing
    end
end

function ensure_connected!(f::Function, conn::Connection; retries=1)
    n_try = 1
    while true
        try
            ensure_connected!(conn)
            return f()
        catch exc
            if exc isa RemoteException
                # This is an expected error - do not retry
                rethrow()
            end
            try
                close(conn)
            catch
                exc isa Base.IOError || rethrow()
            end
            if n_try == retries+1
                @error "Network or internal error running remote repl" exception=exc,catch_backtrace()
                return nothing
            end
            n_try += 1
        end
    end
end

function send_and_receive(conn::Connection, request; read_response=true)
    serialize(conn.socket, request)
    flush(conn.socket)
    if !read_response
        return (:nothing, nothing)
    end
    max_tries = 3
    for i = 1:max_tries
        try
            return deserialize(conn.socket)
        catch exc
            # If an InterruptException is caught here while waiting for a
            # response from the server, we assume the user was trying to
            # interrupt the remote evaluation.
            #
            # This isn't perfect. For example, for large messages the user
            # might interrupt the deserialization while waiting on socket IO,
            # and corrupt the stream. However, there's not much we can do about
            # this without improving the Julia runtime.
            if exc isa InterruptException
                if i < max_tries
                    send_interrupt(conn)
                    continue
                else
                    close(conn)
                    return (:error, "ERROR: Failed to interrupt server, connection closed!")
                end
            end
            rethrow()
        end
    end
end

function send_interrupt(conn::Connection)
    serialize(conn.socket, (:interrupt, nothing))
end

#-------------------------------------------------------------------------------
# REPL integration
function parse_input(str)
    Base.parse_input_line(str)
end

function match_magic_syntax(str)
    m = match(r"(%module|\?) *(.*)", str)
    if !isnothing(m)
        return (m[1], m[2])
    else
        return nothing
    end
end

function valid_input_checker(prompt_state)
    cmdstr = String(take!(copy(REPL.LineEdit.buffer(prompt_state))))
    magic = match_magic_syntax(cmdstr)
    if !isnothing(magic)
        cmdstr = magic[2]
    end
    ex = parse_input(cmdstr)
    return !Meta.isexpr(ex, :incomplete)
end

struct RemoteCompletionProvider <: REPL.LineEdit.CompletionProvider
    connection
end

function REPL.complete_line(provider::RemoteCompletionProvider,
                            state::REPL.LineEdit.PromptState)::
                            Tuple{Vector{String},String,Bool}
    # See REPL.jl complete_line(c::REPLCompletionProvider, s::PromptState)
    partial = REPL.beforecursor(state.input_buffer)
    full = REPL.LineEdit.input_string(state)
    if !isempty(full) && startswith("%modul", full)
        return (["%module"], full, true)
    end
    result = ensure_connected!(provider.connection) do
        send_and_receive(provider.connection, (:repl_completion, (partial, full)))
    end
    if isnothing(result) || result[1] != :completion_result
        return ([], "", false)
    end
    return result[2]
end

function run_remote_repl_command(conn, out_stream, cmdstr)
    ensure_connected!(conn) do
        # Set terminal properties for formatting result
        display_props = Dict(
            :displaysize=>displaysize(out_stream),
            :color=>get(out_stream, :color, false),
            :limit=>true
        )
        # TODO breaking change - send these as part of :eval, perhaps ?
        send_and_receive(conn, (:display_properties, display_props), read_response=false)

        # Send actual command
        magic = match_magic_syntax(cmdstr)
        if isnothing(magic)
            # Normal remote evaluation
            ex = parse_input(cmdstr)

            ex = simple_macro_expand!(ex, Symbol("@remote")) do clientside_ex
                try
                    x = Main.eval(clientside_ex)
                    if x === Base.stdout
                        # The local stdout cannot be serialized in any sensible way,
                        # but we store a placeholder for it which will be transformed
                        # into a serverside approximation of the client stream.
                        return STDOUT_PLACEHOLDER
                    else
                        # Any expressions wrapped in `@remote` need to be executed
                        # on the client and wrapped in a QuoteNode to prevent them
                        # being eval'd again in the expression on the server side.
                        QuoteNode(x)
                    end
                catch _
                    error("Error while evaluating `@remote($clientside_ex)` before passing to the server")
                end
            end

            cmd = (:eval, ex)
        else
            # Magic prefixes
            if magic[1] == "?"
                # Help mode
                cmd = (:help, magic[2])
            elseif magic[1] == "%module"
                mod_ex = parse_input(magic[2])
                cmd = (:in_module, mod_ex)
            end
        end
        messageid, value = send_and_receive(conn, cmd)
        result_for_display = nothing
        if messageid in (:in_module, :eval_result, :help_result, :error)
            if !isnothing(value)
                if messageid != :eval_result || !REPL.ends_with_semicolon(cmdstr)
                    result_for_display = Text(value)
                end
            end
        else
            @error "Unexpected response from server" messageid
        end
        return result_for_display
    end
end

remote_eval_and_fetch(::Nothing, ex) = error("No remote connection is active")

function remote_eval_and_fetch(conn::Connection, ex)
    ensure_connected!(conn) do
        cmd = (:eval_and_get, ex)
        messageid, value = send_and_receive(conn, cmd)
        if messageid == :eval_and_get_result
            logstring = value[2]
            if !isempty(logstring)
                # TODO: Improve this with an async remote logger
                @info Text("Remote logs\n" * logstring)
            end
            return value[1]
        elseif messageid == :error
            throw(RemoteException(value))
        else
            error("Unexpected response message id $messageid from server")
        end
    end
end

function repl_prompt_text(conn::Connection)
    host = conn.host == Sockets.localhost ? "localhost" : conn.host
    port = conn.port == DEFAULT_PORT ? "" : ":$(conn.port)"
    disconnected = isopen(conn) ? "" : " [disconnected]"
    return "julia@$host$port$disconnected> "
end

#-------------------------------------------------------------------------------
# Public client APIs

# Connection which is currently attached to the REPL mode.
_repl_client_connection = nothing

"""
    connect_repl([host=localhost,] port::Integer=$DEFAULT_PORT;
                 use_ssh_tunnel = (host != localhost) ? :ssh : :none,
                 ssh_opts = ``)

Connect client REPL to a remote `host` on `port`. This is then accessible as a
remote sub-repl of the current Julia session.

For security, `connect_repl()` uses an ssh tunnel for remote hosts. This means
that `host` needs to be running an ssh server and you need ssh credentials set
up for use on that host. For secure networks this can be disabled by setting
`tunnel=:none`.

To provide extra options to SSH, you may use the `ssh_opts` keyword, for
example an identity file may be set with ```ssh_opts = `-i /path/to/identity.pem` ```.
Alternatively, you may want to set this up permanently using a `Host` section
in your ssh config file.

You can also use the following technologies for tunneling in place of SSH:
1) AWS Session Manager: set `tunnel=:aws`. The optional `region` keyword
   argument can be used to specify the AWS Region of your server.
2) kubectl: set `tunnel=:k8s`. The optional `namespace` keyword argument can be
   used to specify the namespace of your Kubernetes resource.

See README.md for more information.
"""
function connect_repl(host=Sockets.localhost, port::Integer=DEFAULT_PORT;
                      tunnel::Symbol = host!=Sockets.localhost ? :ssh : :none,
                      ssh_opts=``, region=nothing, namespace=nothing,
                      startup_text=true)
    global _repl_client_connection

    if !isnothing(_repl_client_connection)
        try
            close(_repl_client_connection)
        catch exc
            @warn "Exception closing connection" exception=(exc,catch_backtrace())
        end
    end

    conn = Connection(host=host, port=port, tunnel=tunnel,
                      ssh_opts=ssh_opts, region=region, namespace=namespace)
    out_stream = stdout
    ReplMaker.initrepl(c->run_remote_repl_command(conn, out_stream, c),
                       repl         = Base.active_repl,
                       valid_input_checker = valid_input_checker,
                       prompt_text  = ()->repl_prompt_text(conn),
                       prompt_color = :magenta,
                       start_key    = '>',
                       sticky_mode  = true,
                       mode_name    = "remote_repl",
                       completion_provider = RemoteCompletionProvider(conn),
                       startup_text = startup_text
                       )
    # Record the connection which is attached to the REPL
    _repl_client_connection = conn

    nothing
end

connect_repl(port::Integer) = connect_repl(Sockets.localhost, port)

"""
    @remote ex

Execute expression `ex` on the other side of the current RemoteREPL connection
and return the value.

This can be used in both directions:
1. From the normal `julia>` prompt, execute `ex` on the remote server and
   return the value to the client.
2. From a remote prompt, execute `ex` on the *client* and push the
   resulting value to the remote server.

# Examples

Push a value from the client to the server:

```
julia> client_val = 1:100;

julia@localhost> server_val = @remote client_val
1:100
```

Fetch a pair of variables `(x,y)` from the server, and plot them on the client
with a single line:
```
# In two lines
julia> x,y = @remote (x, y)
       plot(x, y)

# Or as a single expression
julia> plot(@remote((x, y))...)
```
"""
macro remote(ex)
    _remote_expr(:_repl_client_connection, ex)
end

macro remote(conn, ex)
    _remote_expr(esc(conn), ex)
end

_remote_expr(conn, ex) = :(remote_eval_and_fetch($conn, $(QuoteNode(ex))))

#--------------------------------------------------
"""
    remote_eval(cmdstr)
    remote_eval(host, port, cmdstr)

Parse a string `cmdstr`, evaluate it in the remote REPL server's `Main` module,
then close the connection. Returns the result which the REPL would normally
pass to `show()` (likely a `Text` object).

For example, to cause the remote Julia instance to exit, you could use

```
using RemoteREPL
RemoteREPL.remote_eval("exit()")
```
"""
function remote_eval(host, port::Integer, cmdstr::AbstractString;
                     tunnel::Symbol = host!=Sockets.localhost ? :ssh : :none)
    conn = Connection(; host=host, port=port, tunnel=tunnel)
    local result
    try
        setup_connection!(conn)
        result = run_remote_repl_command(conn, IOBuffer(), cmdstr)
    finally
        close(conn)
    end
    return result
end

function remote_eval(cmdstr::AbstractString)
    remote_eval(Sockets.localhost, DEFAULT_PORT, cmdstr)
end


text:
module io

using HDF5

"""
Load segment size list
"""
function load_seg_list(fn)
    return readdlm(fn, ';', Int64)
end

function load_semmap(fname)
    table = readdlm(fname, ';', Int)
    Dict( table[i,1] => table[i,2] for i in 1:size(table,1) )
end

"""
Load Synaptor edge csv & parse columns appropriately
"""
function load_edges(fn)
    tbl = readdlm(fn, ';')[:,1:4]
    tbl[:,2] = [map(parse, split(tbl[i,2][2:end-1],",")) for i in 1:size(tbl,1)]
    tbl[:,3] = [map(parse, split(tbl[i,3][2:end-1],",")) for i in 1:size(tbl,1)]
    return tbl
end

"""
Load python parsed edge csv & parse columns appropriately
"""
function load_edges_from_python(fn)
    tbl = readdlm(fn, ',', Int64)
    new_tbl = Array(Any, size(tbl,1), 4)
    new_tbl[:,1] = tbl[:,1]
    new_tbl[:,2] = [[tbl[i,2:3]...] for i in 1:size(tbl,1)]
    new_tbl[:,3] = [[tbl[i,4:6]...] for i in 1:size(tbl,1)]
    new_tbl[:,4] = tbl[:,7]
    return new_tbl
end

function read_h5( fname, read_whole_dset=true, h5_dset_name="/main" )

  if read_whole_dset
    d = h5read( fname, h5_dset_name );
  else
    f = h5open( fname );
    d = f[h5_dset_name];
  end

  d
end

function write_map_file( output_fname, dicts... )

  open(output_fname, "w+") do f
    if length(dicts) > 0
    for k in keys(dicts[1])

      vals = ["$(d[k]);" for d in dicts ];
      write(f, "$k;$(vals...)\n")

    end #for
    end #if
  end#open

end

"""
Filter edges & save in easy python format with flat arrays

Inputs:
  fn: filename to write to
  edges: mixed array (list of lists)
  filter_ids: list of segment ids to include

Output:
  file with all lists expanded

"""
function python_write_edges(fn, edges, filter_ids)
  e = []
  for i in 1:size(edges,1)
    if edges[i,2][1] in filter_ids && edges[i,2][2] in filter_ids
      push!(e, [edges[i,1], edges[i,2]..., edges[i,3]..., edges[i,4]])
    end
  end
  writedlm(fn, hcat(e...)', ",")
end

end #module end


text:
import Random
import Base.Iterators

abstract type MarkovChain end

function accept(::MarkovChain)
    nothing
end

function reject(::MarkovChain)
    nothing
end

propose(old_state, chain::MarkovChain) = propose!(copy(old_state), old_state, chain)

mutable struct MetropolisSampler{S,Sys}
    burn_in::Int
    subsample::Int
    current_energy::Float64
    state::S
    chain::Sys
end

MetropolisSampler(state, chain; burn_in::Int=0, subsample::Int=0) = MetropolisSampler(burn_in, subsample, energy(state, chain), state, chain)

function set_state(sampler::MetropolisSampler{S}, state::S) where {S}
    sampler.current_energy = energy(state, sampler.chain)
    sampler.state = state
end

Base.IteratorSize(::Type{<:MetropolisSampler}) = Base.IsInfinite()

Base.iterate(sampler::MetropolisSampler) = iterate(sampler, copy(sampler.state))

function Base.iterate(sampler::MetropolisSampler{S,Sys}, new_state::S) where {S,Sys}
    accepted = 0
    rejected = 0

    while true
        propose!(new_state, sampler.state, sampler.chain)
        new_energy = energy(new_state, sampler.chain)

        # metropolis acceptance criterion
        if Random.randexp() >= new_energy - sampler.current_energy
            accept(sampler.chain)
            accepted += 1
            sampler.current_energy = new_energy
            # simple variable swap (sampler.state <--> new_state)
            tmp = new_state
            new_state = sampler.state
            sampler.state = tmp
        else
            reject(sampler.chain)
            rejected += 1
        end

        if (accepted + rejected) > max(sampler.burn_in, sampler.subsample)
            sampler.burn_in = 0
            return accepted, new_state
        end
    end
end

function sample(sampler::MetropolisSampler)
    iterate(sampler)
    copy(sampler.state)
end

function sample(sampler::MetropolisSampler, num_samples::Integer)
    map(Iterators.take(sampler, num_samples)) do _
        copy(sampler.state)
    end
end

function sample(f, sampler::MetropolisSampler, num_samples::Integer)
    map(Iterators.take(sampler, num_samples)) do _
        f(sampler.state)
    end
end


text:
using Mimi
using Test
using DataFrames
using VegaLite
using Electron

import Mimi:
    dataframe_or_scalar, createspec_singlevalue,
    createspec_lineplot, createspec_multilineplot, createspec_barplot,
    getmultiline, getline, getbar, _spec_for_item, menu_item_list, explore,
    getdataframe, reset_compdefs

reset_compdefs()

@defcomp MyComp begin
    a = Parameter(index=[time, regions])
    b = Parameter(index=[time])
    c = Parameter(index=[regions])
    d = Parameter()
    e = Parameter(index=[four])
    f::Array{Float64, 2} = Parameter()

    x = Variable(index=[time, regions])

    function run_timestep(p, v, d, t)
        for r in d.regions
            v.x[t, r] = 0
        end
    end
end

m = Model()
set_dimension!(m, :time, 2000:2100)
set_dimension!(m, :regions, 3)
set_dimension!(m, :four, 4)

add_comp!(m, MyComp)
set_param!(m, :MyComp, :a, ones(101,3))
set_param!(m, :MyComp, :b, 1:101)
set_param!(m, :MyComp, :c, [4,5,6])
set_param!(m, :MyComp, :d, .5)
set_param!(m, :MyComp, :e, [1,2,3,4])
set_param!(m, :MyComp, :f, [1.0 2.0; 3.0 4.0])

run(m)

# 1.  dataframe helper functions
@test typeof(dataframe_or_scalar(m, :MyComp, :a)) == DataFrame
@test typeof(dataframe_or_scalar(m, :MyComp, :d)) == Float64

#2.  JSON strings for the spec "values" key
# TODO:  getmultiline, getline, getbar, getdatapart

#3.  full specs for VegaLit

#TODO:  createspec_singlevalue, createspec_multilineplot,
#createspec_lineplot, createspec_barplot, _spec_for_item

s = menu_item_list(m)
@test typeof(s) == Array{Any, 1}
@test length(s) == 7

#4.  explore(m::Model, title = "Electron")
w = explore(m, title = "Testing Window")
@test typeof(w) == Electron.Window

#5.  explore(m::Model, comp_name::Symbol, datum_name::Symbol;
#       dim_name::Union{Nothing, Symbol} = nothing)

p = explore(m, :MyComp, :a)
@test typeof(p) == VegaLite.VLSpec{:plot}

#6.  errors and warnings
@defcomp MyComp2 begin

    a = Parameter(index = [time, regions, four])
    x = Variable(index=[time, regions])

    function run_timestep(p, v, d, t)
        for r in d.regions
            v.x[t, r] = rand(10)[1]
        end
    end
end

m2 = Model()
set_dimension!(m2, :time, 2000:2100)
set_dimension!(m2, :regions, 3)
set_dimension!(m2, :four, 4)

add_comp!(m2, MyComp2)
set_param!(m2, :MyComp2, :a, ones(101, 3, 4))

run(m2)

#spec creation for MyComp.a should fail and error, haven't handled case of > 3 dims yet
# TODO Reenable
# @test_warn "has >2 graphing dims" explore(m2)


text:
using Mill, JSON, Flux, JsonGrinder, Test, HierarchicalUtils, Setfield

using JsonGrinder: DictEntry, suggestextractor, schema
using Mill: reflectinmodel

@testset "testing pipeline mixing different types of arrays" begin
	j1 = JSON.parse("""{"a": 1, "b": "hello works", "c":{ "a":1 ,"b": "hello world"}}""")
	j2 = JSON.parse("""{"a": 2, "b": "hello world", "c":{ "a":2 ,"b": "hello"}}""")

	sch = schema([j1,j2])
	extractor = suggestextractor(sch)
	ds = map(s-> extractor(s), [j1,j2])
	dss = reduce(catobs, ds)

	m = reflectinmodel(dss, k -> Dense(k,10, relu));
	o = m(dss).data
	for i in 1:length(ds)
		@test o[:,i] ≈ m(ds[i]).data
	end
end

@testset "testing pipeline with simple arrays and missing values" begin
	j1 = JSON.parse("""{"a": 4, "b": {"a":[1,2,3],"b": 1},"c": { "a": {"a":[1,2,3],"b":[4,5,6]}}}""",inttype=Float64)
	j2 = JSON.parse("""{"a": 4, "c": {"a": {"a":[2,3],"b":[5,6]}}}""")
	j3 = JSON.parse("""{"a": 4, "b": {"a":[1,2,3], "b": 1}}""")
	j4 = JSON.parse("""{"a": 4, "b": {}}""")
	j5 = JSON.parse("""{"b": {}}""")
	j6 = JSON.parse("""{}""")

	sch = JsonGrinder.schema([j1,j2,j3])
	extractor = suggestextractor(sch)
	dss = map(extractor, [j1,j2,j3,j4,j5,j6])
	ds = reduce(catobs, dss)
	m = reflectinmodel(ds, k -> Dense(k,10, relu))
	o = m(ds).data

	for i in 1:length(dss)
		@test o[:,i] ≈ m(dss[i]).data
	end
end

# todo: dodělat test pro navrhnutý model a zkontrolovat, že je ok, i s missingy a jedním klíčem v dictu a se scalary
@testset "testing pipeline with arrays of dicts and missing values" begin
	j1 = JSON.parse("""{"a": [{"a":1},{"b":2}]}""")
	j2 = JSON.parse("""{"a": [{"a":1,"b":3},{"b":2,"a" : 1}]}""")
	j3 = JSON.parse("""{"a": [{"a":2,"b":3}]}""")
	j4 = JSON.parse("""{"a": []}""")
	j5 = JSON.parse("""{}""")

	sch = JsonGrinder.schema([j1,j2,j3])
	extractor = suggestextractor(sch)
	dss = map(s-> extractor(s), [j1,j2,j3,j4,j5])
	ds = reduce(catobs, dss)
	m = reflectinmodel(ds, k -> Dense(k,10, relu));
	o = m(ds).data
	for i in 1:length(dss)
		@test o[:,i] ≈ m(dss[i]).data
	end
end

@testset "testing pipeline with single-keyed dicts and merging scalars" begin
	j1 = JSON.parse("""{"a": [{"a":1},{"b":2,"c":"oh"}]}""")
	j2 = JSON.parse("""{"a": [{"a":1,"b":3,"c":"hi"},{"b":2,"a":1,"c":"Mark"}]}""")
	j3 = JSON.parse("""{"a": [{"a":2,"b":3}]}""")
	j4 = JSON.parse("""{"a": []}""")
	j5 = JSON.parse("""{}""")

	sch = JsonGrinder.schema([j1,j2,j3,j4, j5])
	ext = suggestextractor(sch)
	dss = ext.([j1,j2,j3,j4,j5])
	ds = reduce(catobs, dss)
	m = reflectinmodel(ds, k -> Dense(k, 10, relu))

	@test isequal(NodeIterator(m, ds) |> collect, [
		(m[""], ds[""]),
		(m["U"], ds["U"]),
		(m["k"], ds["k"]),
		(m["o"], ds["o"]),
		(m["s"], ds["s"]),
		(m["w"], ds["w"]),
	])
	@test NodeIterator(sch, ext) |> collect == [
		(sch[""], ext[""]),
		(sch["U"], ext["U"]),
		(sch["k"], ext["k"]),
		(sch["o"], ext["o"]),
		(sch["s"], ext["s"]),
		(sch["w"], ext["w"]),
	]
	@test NodeIterator(sch, m) |> collect == [
		(sch[""], m[""]),
		(sch["U"], m["U"]),
		(sch["k"], m["k"]),
		(sch["o"], m["o"]),
		(sch["s"], m["s"]),
		(sch["w"], m["w"]),
	]
	@test NodeIterator(ext, m) |> collect == [
		(ext[""], m[""]),
		(ext["U"], m["U"]),
		(ext["k"], m["k"]),
		(ext["o"], m["o"]),
		(ext["s"], m["s"]),
		(ext["w"], m["w"]),
	]
	@test isequal(NodeIterator(ext, ds) |> collect, [
		(ext[""], ds[""]),
		(ext["U"], ds["U"]),
		(ext["k"], ds["k"]),
		(ext["o"], ds["o"]),
		(ext["s"], ds["s"]),
		(ext["w"], ds["w"]),
	])
	@test buf_printtree(sch) == """
	[Dict] (updated = 5)
	  └── a: [List] (updated = 4)
	           └── [Dict] (updated = 5)
	                 ├── a: [Scalar - Int64], 2 unique values, updated = 4
	                 ├── b: [Scalar - Int64], 2 unique values, updated = 4
	                 └── c: [Scalar - String], 3 unique values, updated = 3"""
	@test buf_printtree(ext) == """
	Dict
	  └── a: Array of
	           └── Dict
	                 ├── a: Float32
	                 ├── b: Float32
	                 └── c: String"""
	@test buf_printtree(m) == """
	ProductModel … ↦ ArrayModel(identity)
	  └── a: BagModel … ↦ ⟨SegmentedMean(10)⟩ ↦ ArrayModel(Dense(11, 10, relu))
	           └── ProductModel … ↦ ArrayModel(Dense(12, 10, relu))
	                 ├── a: ArrayModel([pre_imputing]Dense(1, 1))
	                 ├── b: ArrayModel([pre_imputing]Dense(1, 1))
	                 └── c: ArrayModel([post_imputing]Dense(2053, 10, relu))"""
	@test buf_printtree(ds) == """
	ProductNode with 5 obs
	  └── a: BagNode with 5 obs
	           └── ProductNode with 5 obs
	                 ├── a: ArrayNode(1×5 Array with Union{Missing, Float32} elements) with 5 obs
	                 ├── b: ArrayNode(1×5 Array with Union{Missing, Float32} elements) with 5 obs
	                 └── c: ArrayNode(2053×5 NGramMatrix with Union{Missing, Int64} elements) with 5 obs"""

	@test m[""].m.m == identity

	@test nobs(dss[5]["s"].data) == 0
	@test nobs(dss[5]) == 1
end


text:
include("SolveBalances_Flow.jl")
include("DataFile_Flow.jl")
using PyPlot

TSTART = 0
TSTOP = 24
Ts = 0.01

data_dictionary = DataFile_Flow(TSTART,TSTOP,Ts)

(T,X) = SolveBalances_Flow(TSTART,TSTOP,Ts,data_dictionary)


text:
## Not finished!#

#=
Conway's Game of Life takes place on an infinite two-dimensional board of square cells. Each cell is either dead or alive, and at each tick, the following rules apply:

    Any live cell with less than two live neighbours dies.
    Any live cell with two or three live neighbours remains living.
    Any live cell with more than three live neighbours dies.
    Any dead cell with exactly three live neighbours becomes a live cell.

A cell neighbours another cell if it is horizontally, vertically, or diagonally adjacent.

Implement Conway's Game of Life. It should be able to be initialized with a starting list of live cell coordinates and the number of steps it should run for. Once initialized, it should print out the board state at each step. Since it's an infinite board, print out only the relevant coordinates, i.e. from the top-leftmost live cell to bottom-rightmost live cell.

You can represent a live cell with an asterisk (*) and a dead cell with a dot (.).
=#

function ConwayGameOfLife(livecells, steps::Int)
    board = initialize(livecells)
    for step in 0:steps
        println("Step: $step")
        display(board)
        for cell_i in 1:size(board, 1)
            for cell_j in 1:size(board, 2)
                board[cell_index] = nextstatus(cell_i, cell_j, board)
            end
        end
    end
end

function initialize(livecells)
    board = fill(".", maximum(livecells[:,1]), maximum(livecells[:,2]))
    for living_cell in 1:size(livecells, 1)
        board[livecells[living_cell, 1], livecells[living_cell, 2]] = "*"
    end
    return board
end

function nextstatus(cell_i, cell_j, board)
    previous_status = board[cell_i, cell_j]
    livingneighbour = livingneighbours(cell_i, cell_j, board)
    if previous_status == "*" && livingneighbour < 2
        new_status = "."
    elseif previous_status == "*" && livingneighbour >= 2 && livingneighbour <= 3
        new_status = "*"
    elseif previous_status == "*" && livingneighbour > 3
        new_status = "."
    elseif previous_status == "." && livingneighbour == 3
        new_status = "*"
    else
        new_status = previous_status
    end
    return new_status
end

function livingneighbours(cell_i, cell_j, board)

end

# Create random board
livecells = []
for i in 1:10
    push!(livecells, rand(collect(1:4)),rand(collect(1:4)))
end
livecells = reshape(livecells, 10, 2)

max_stars_repo_name: DominiqueCaron/daily-coding-problem

max_issues_repo_name: DominiqueCaron/daily-coding-problem

max_forks_repo_name: DominiqueCaron/daily-coding-problem


text:
f(x) = begin
  x = x + 3
  return x
end

print(f(3))

max_stars_repo_name: moldray/juliaTodo

max_issues_repo_name: moldray/juliaTodo

max_forks_repo_name: moldray/juliaTodo


text:
__precompile__()

module MLJTime

using DecisionTree
using Statistics

export InvFeatureGen, RandomForestClassifierTS, predict_f, proba_predict

include("IntervalBasedForest.jl")

end #module