text:
# Ergänze Tabellen mit Personendaten um Angaben aus dem digitalen Personenregister
# des Projektes Germania Sacra
#
# Projektseite
# https://adw-goe.de/forschung/forschungsprojekte-akademienprogramm/germania-sacra/
#
# Digitales Personenregister
# http://germania-sacra-datenbank.uni-goettingen.de/
#
# API
# http://personendatenbank.germania-sacra.de/api/v1.0/person
# https://adw-goe.de/forschung/forschungsprojekte-akademienprogramm/germania-sacra/schnittstellen-und-linked-data/
#
# Dokumentation
# ../README.md
#
"""
    module GSquery

Finde die GS-Nummer und weitere Daten zu Personen im digitalen Personenregister
der Germania Sacra
"""
module GSquery

using DataFrames
using JSON
using HTTP
using Dates
mod_dev = true
if mod_dev
    using Infiltrator
end
using Logging

include("./Util.jl")
include("./GSOcc.jl")

"""
    QRecord

Element im Verzeichnis "records" in einer Anfrage an die Personendatenbank.
"""
struct QRecord
    data::Dict{String, Any}
end

import Base.getindex
getindex(r::QRecord, key::AbstractString) = getindex(r.data, key)

# Parameter

"""
    URLGS

URL des API der Germania Sacra
"""
const URLGS="http://personendatenbank.germania-sacra.de/api/v1.0/person"

"""
    URLGSINDEX

URL des Index API der Germania Sacra
"""
const URLGSINDEX="http://germania-sacra-datenbank.uni-goettingen.de/persons/index"

"""
    minscore

Ein Datensatz mit mindestens diesen Übereinstimmungen wird übernommen.
"""
global minscore = QRecord(Dict("muster" => "vn ae ab ao", "zeitguete" => 0))

"""
    setminmatchkey(matchkey::AbstractString)

Setze den Parameter für den Mindestwert an Übereinstimmung.

`matchkey`: Muster, das noch als Treffer ausgegeben wird.

Treffer mit einem schlechterem Muster als `matchkey` werden nicht berücksichtigt.
`setminmatchkey()` gibt das aktuelle Muster aus.

Beispiel

`"fn vn"`: Mindestens Familienname und Vorname müssen übereinstimmen.
"""
function setminmatchkey(matchkey::AbstractString)
    global minscore = QRecord(Dict("muster" => matchkey, "zeitguete" => 0))
end

setminmatchkey() = minscore

"""
    LIMITN

Zahl der Datensätze, die pro Anfrage gelesen werden sollen.
"""
const LIMITN = 300

"""
    logpath

Wenn der Pfad nicht leer ist, werden hierhin die Log-Mitteilungen geschrieben.
"""
global logpath = ""

"""
    setlogpath(logfile::AbstractString)

Setze den Namen der Datei für Logdaten

Wenn der Pfad der Logdatei leer ist (""), werden keine Logdaten geschrieben.
`setlogpath()`: gib den aktuellen Pfad der Logdatei aus.
"""
function setlogpath(logfile::AbstractString)
    path = splitdir(logfile)[1]
    path == "" && (path = ".")
    if ispath(path)
        global logpath = logfile
    else
        @error "Pfad nicht gefunden"
    end
end

setlogpath() = logpath

global filelog = Logging.NullLogger()

"""
    GSSTRINGCOLS

Liste der Spalten, die aus dem digitalen Personenregister ausgelesen werden.
"""
const GSSTRINGCOLS = [:GSN1_GS,
                      :GSN_GS,
                      :ID_GND_GS,
                      :Qualitaet_GS,
                      :Vorname_GS,
                      :Vornamenvarianten_GS,
                      :Namenspraefix_GS,
                      :Familienname_GS,
                      :Familiennamenvarianten_GS,
                      :Namenszusatz_GS,
                      :Geburtsdatum_GS,
                      :Sterbedatum_GS,
                      :Amt_GS,
                      :Amtsbeginn_GS,
                      :Amtsende_GS,
                      :Dioezese_GS,
                      :Aemter_GS,
                      :Dioezesen_GS]

const RANKMAX = 199

"""
    Rank

Verwalte Ranklisten.
"""
module Rank

# Parameter
"""
    fileranks

Dateiname der Liste mit Übereinstimmungsmustern. Wenn die entsprechende Datei nicht vorhanden ist, wird automatisch eine Rangliste von Mustern mit `makeranklist`erstellt.

Beispiel:
fn sd vn ae ab ao at
...
vn ab ao
...
ab
ao
at

"""
fileranks = "GSQueryranks.txt"

"""
    setfileofranks(fileranks::AbstractString)

Setze den Dateinamen für die Liste mit Übereinstimmungsmustern.

Wenn die entsprechende Datei nicht vorhanden ist, wird für die Abfrage
automatisch eine Rangliste von Mustern erstellt.

`setfileofranks()`: Gib den aktuellen Dateinamen aus.
"""
setfileofranks(file::AbstractString) = (global fileranks = file)
setfileofranks() = return fileranks

const SLISTKEY = ["fn", "sd", "vn", "ae", "ab", "ao", "at"]
const DXLISTKEY = Dict((k => i) for (i, k) in enumerate(SLISTKEY))
kpless(a, b) = DXLISTKEY[a] < DXLISTKEY[b]

"""
    makeranklist(le = SLISTKEY)

Erzeuge eine Rangliste. Sie wird in der Regel manuell nachjustiert
und in eine Textdatei geschrieben. Siehe `drank`
"""
function makeranklist(le = SLISTKEY)
    rv = [String[]]
    isshorter(a, b) = length(a) < length(b)
    for e in Iterators.reverse(le)
        rvn = copy(rv)
        for q in rv
            push!(rvn, vcat(e, q))
        end
        rv = rvn
    end
    sort!(rv, alg=MergeSort, lt = isshorter)
    (join(q, " ") for q in Iterators.reverse(rv))
end

"""
    drank

Verzeichnis von Mustern zu Rängen.
"""
drank = Dict{String, Int}()

"""
    setdrank(fileranks)

Lies eine Liste mit Übereinstimmungsmustern aus `fileranks`, falls vorhanden.
Erzeuge anderenfalls eine Standardversion der Liste von Übereinstimmungsmustern.
"""
function setdrank(fileranks)
    global drank
    if isfile(fileranks)
        @info "Lies Übereinstimmungsmuster aus " fileranks
        drank = Rank.readdrank(fileranks)
    else
        @info "Datei nicht gefunden " fileranks
        @info "Liste der Übereinstimmungsmuster wird erzeugt."
        setdrank()
    end
    return nothing
end

function setdrank()
    global drank
    drank = Dict(p => rk for (rk, p) in enumerate(makeranklist()))
end

"""
    readdrank(file::AbstractString)

Lies Liste von Übereinstimmungsmustern aus `file`.
"""
function readdrank(file::AbstractString)
    local drank = Dict(p => rk for (rk, p) in enumerate(eachline(file)))
    # Der letzte Rang entspricht dem leeren Muster
    if !haskey(drank, "")
        drank[""] = maximum(values(drank)) + 1
    end
    drank
end

"""
    islessinset(a, b; drank=drank)

Rückgabewert `drank[b] < drank[a]`

Prüfe, ob `a` und `b` als Schlüssel vorhanden sind. Beachte: Je kleiner der Rang, desto besser ist das Muster.
"""
function islessinset(a, b; drank=drank)
    pa = get(drank, a, 0)
    pb = get(drank, b, 0)
    if pa == 0
        error("Ungültiger Schlüssel: " * a)
    end
    if pb == 0
        error("Ungültiger Schlüssel: " * b)
    end

    return pb < pa
end

getrank(key) = drank[key]

isvalidrank(key) = haskey(drank, key)

"""
    ranklist()

Gib die Liste mit Übereinstimmungsmustern aus.
"""
ranklist() = sort(collect(drank), lt=(a, b) -> a.second < b.second)

end # module Rank

# globale Variablen

inputcols = [:ID,
             :Praefix,
             :Vorname,
             :Familienname,
             :Familiennamenvarianten,
             :Sterbedatum,
             :Amtsart,
             :Amtsbeginn,
             :Amtsende,
             :Amtsort,
             :GSN_ID,
             :GND_ID,]

"""
    setinputcols(cols)

Setze die Namen der Spalten der Eingabetabelle, die in die Ausgabetabelle übernommen werden sollen

`setinputcols()`: Gib die Spaltennamen aus.
"""
function setinputcols(cols)
    global inputcols
    inputcols = Symbol.(cols)
end

setinputcols() = inputcols

"""
    setcolnameid(id)

Setze den Namen der Spalte, welche die ID enthält

`setcolnameid()`: Gib den aktuellen Wert aus

Der Spaltenname muss der gleiche für Personen und Ämter sein, z.B. 'ID_Domherr'.

Voreinstellung: `:ID`
"""
setcolnameid(id) = (global inputcols[1] = Symbol(id))
setcolnameid() = inputcols[1]

colid() = (global inputcols; getindex(inputcols::Array{Symbol, 1}, 1))

# Spalten, die vom Programm bewertet werden können.
const MATCHCOLS = [:Vorname, :Vornamenvarianten, :Familienname, :Familiennamenvarianten, :Sterbedatum]
const MATCHOCCCOLS = [:Amtsart, :Amtsbeginn, :Amtsende, :Amtsort, :ID_Kloster]

# Funktionen
import Base.isless

function isless(a::QRecord, b::QRecord)
    (Rank.islessinset(a.data["muster"],  b.data["muster"])
     || (a.data["muster"] == b.data["muster"]
         && a.data["zeitguete"] < b.data["zeitguete"]))
end

import Base.isequal

function isequal(a::QRecord, b::QRecord)
    (isequal(a.data["muster"],  b.data["muster"])
     && isequal(a.data["zeitguete"], b.data["zeitguete"]))
end

"""
    getGS(url, params::Dict{String, String}; offset = 0, limit = LIMITN, format = "json")
       -> Verzeichnis von Datensätzen; Dict{String, Any}

Stelle eine Anfrage an die Schnittstelle der Germania Sacra.
`params`: Paare der Form "Feldname" => "Wert"

## Beispiele
getGS(URLGS, Dict("person.gndnummer" => "138018383"))
getGS(URLGS, Dict("person.vorname" => "Christoph", "person.nachname" => "Berg"))
gstGS(URLGSINDEX, Dict("name" => "Georg Anton Rodenstein")
"""
function getGS(url, params; offset = 0, limit = LIMITN, format = "json")
    qparams = copy(params)
    qparams["format"] = format
    qparams["offset"] = string(0)
    qparams["limit"] = string(limit)

    rq = HTTP.request("GET", url, query=qparams, readtimeout = 30, retries = 5);

    rds = String(rq.body);
    rds4parser = replace(rds, r"\n *" => "");
    return rdt = JSON.parse(rds4parser)
end

"""
    reconcile!(df,
               querycols,
               dfocc;
               nmsg = 40,
               toldateofdeath = 2,
               toloccupation = 2)

Frage das digitale Personenregister nach den Spalten in `querycols` ab.
Vergleiche die gefundenen Datensätze mit Name und Amtsdaten aus dem Abfragedatensatz.
Ergänze `df` für jeden Datensatz mit den Daten aus dem besten Treffer.
Gib nach einer Zahl von `nmsg` Datensätzen eine Fortschrittsmeldung aus.
Verwende `toldateofdeath` als Toleranz für das Sterbedatum und
`toloccupation` als Toleranz für Amtsdaten.

    reconcile!(df, fields; nmsg = 40)

Frage das digitale Personenregister nach den Feldern in `fields` ab.
Ergänze `df` für jeden Datensatz mit den Daten aus dem erstbesten Treffer.
Gib nach einer Zahl von `nmsg` Datensätzen eine Fortschrittsmeldung aus.

`fields` ordnet jeder Spalte in `df` einen `Feldnamen` im Personenregister zu.

``` julia
fields = Dict(:Vorname => "person.vorname", :Familienname => "person.nachname")
fields = ((:ID_GND => "person.gndnummer"),)
```
"""
function reconcile!(df,
                    querycols,
                    dfocc;
                    nmsg = 40,
                    toldateofdeath = 2,
                    toloccupation = 2)

    global filelog
    global logpath

    if length(Rank.drank) == 0
        Rank.setdrank()
    end

    dfcols = Symbol.(names(df))
    if !(colid() in dfcols)
        error("Die Eingabetabelle enthält keine passende Spalte für die ID (ist: `$(colid())`). "
              * "Siehe `setcolnameid`")
    end

    # Prüfe Vergleichsspalten
    mcols = intersect(MATCHCOLS, dfcols)
    occmcols = intersect(MATCHOCCCOLS, Symbol.(names(dfocc)))

    startid = df[1, colid()]
    minkey = minscore["muster"]
    minrank = Rank.getrank(minkey)

    if logpath != ""
        logio = open(logpath, "a")
        filelog = SimpleLogger(logio)
        with_logger(filelog) do
            @info now() startid
            @info "Level:" minkey minrank
            @info "Gültige Ämter:" GSOcc.occupations
        end
    end

    @info "Level:" minkey minrank
    @info "Gültige Ämter" GSOcc.occupations
    @info "Abfragespalten" querycols

    dictquery = Dict("name" => "",
                     "ort" => "")

    # dbest: Histogramm über die Zahl der jeweils besten Treffer.
    dbest = Dict{Int, Int}()
    irow = 1
    for row in eachrow(df)
        irow % nmsg == 0 && println("Datensatz: ", irow)
        records = QRecord[]
        nbest = 0
        ffound = false
        currentid = row[colid()]

        if ismissing(row[:Vorname]) || row[:Vorname] == ""
            # kein gültiger Eintrag
            @warn "Kein Vorname für (keine Abfrage): " currentid
        else
            try

                # Die Abfrage nach Amt müsste mehrfach durchgeführt werden,
                # weil ein Amt unter verschiedenen Bezeichnungen in der Personendatenbank
                # erfasst sein kann.
                # Schränke nur nach dem Ort ein und werte die allfällige größere Trefferliste aus.

                # Abfrage nach Name und Amtsort/Bistum

                if :Familienname in querycols && !ismissing(row[:Familienname])
                    c_gn = split(row[:Vorname])
                    c_fn = split(row[:Familienname], [' ', '-'])
                    dictquery["name"] = strip(c_gn[1]) * " " * strip(c_fn[1])
                else
                    dictquery["name"] = strip(row[:Vorname])
                end

                # Ämter für row
                rowsocc = Util.rowselect(dfocc, currentid, colid())

                places = String[]
                if :Amtsort in querycols
                    places = filter(!ismissing, rowsocc[!, :Amtsort])
                end

                # Wir fragen für einen Datensatz in `df` mehrmals ab.
                # Ein Referenzdatensatz kann mehrmals auftauchen.
                # Eine mehrmalige Prüfung in `evaluate!` wird vermieden.
                listchecked = String[]
                # Argumente für evalutate!
                evalargs = (listchecked, row, rowsocc, toldateofdeath, toloccupation, mcols, occmcols)

                if length(places) > 0
                    for place in places
                        dictquery["ort"] = place
                        gsres = getGS(URLGSINDEX, dictquery)
                        @debug listchecked
                        append!(records, evaluate!(gsres, evalargs...))
                    end
                else
                    delete!(dictquery, "ort")
                    gsres = getGS(URLGSINDEX, dictquery)
                    records = evaluate!(gsres, evalargs...)
                end

                bestkey = ""
                if length(records) > 0
                    bestrec, posbest = findmax(records)
                    bestkey = bestrec["muster"]
                    if !isless(bestrec, minscore)
                        # nbest: Zahl der Treffer in GS, die zu einer gleichen besten
                        # Bewertung führen.
                        nbest = writematch!(row, bestrec, records)
                    else
                        with_logger(filelog) do
                            @info "abgelehnt" currentid  bestkey
                        end
                    end
                end

                dbest[nbest] = get(dbest, nbest, 0) + 1

        catch
            @warn "Fehler bei Datensatz: " currentid
            rethrow()
            end
        end
        irow += 1
    end

    if logpath != ""
        flush(logio)
        close(logio)
    end
    vbest = sort(map(identity, (p for p in dbest)))
    return vbest
end

# reconcilebyname!
# obsolet: Kann über das Argument querycols in reconcile erreicht werden.
# gelöscht 2020-07-09

reconcile!(df::AbstractDataFrame, field::Pair{Symbol, String}; nmsg = 40) = reconcile!(df, tuple(field), nmsg)

"""
    reconcile!(df::AbstractDataFrame, fields; limit = 20, nmsg = 40)

Finde Treffer in der Personendatenbank, schreibe maximal `limit` Ergebnisse pro angefragtem Datensatz
"""

function reconcile(df::AbstractDataFrame, fields; limit = 20, nmsg = 40)
    coltokey = Dict(col => kgs for (col, kgs) in fields)

    dictquery = Dict(p.second => "" for p in coltokey)
    println(dictquery);

    dfout = similar(df, 0)
    irow = 0
    for row in eachrow(df)
        (irow += 1) % nmsg == 0 && println("Datensatz: ", irow)

        for (col, kgs) in coltokey
            qv = row[col]
            if !Util.hasdata(qv)
                delete!(dictquery, kgs)
            else
                dictquery[kgs] = qv
            end
        end
        if isempty(dictquery) continue end

        gsres = getGS(URLGS, dictquery, limit = limit)

        records = gsres["records"]
        n = length(records)
        if n > 0
            if limit == 1
                theone = records[1]
                lastocc = GSOcc.getlastocc(theone)
                if !isnothing(lastocc)
                    theone["amt"] = lastocc
                end
                rowout = push!(dfout, rowout) |> last
                writerow!(rowout, theone, n)
            else
                for i in 1:min(limit, n)
                    rowout = push!(dfout, row) |> last
                    writerow!(rowout, records[i], n)
                end
            end
        end
    end
    return dfout
end

"""
    reconcilebyGND!(df::AbstractDataFrame;
                    colgnd = :ID_GND,
                    nmsg = 40,
                    toldateofdeath = 6,
                    tolendofoccupation = 11))

Frage GS anhand des Werts in der Spalte `colgnd` ab.

Vergleichswert im digitalen Personenregister:
"""
function reconcilebyGND!(df::AbstractDataFrame;
                         colgnd = :ID_GND,
                         nmsg = 40,
                         toldateofdeath = 6, tolendofoccupation = 11)
    irow = 1
    for row in eachrow(df)
        irow % nmsg == 0 && println("Datensatz: ", irow)
        queryGSbyGND!(row, colgnd, toldateofdeath, tolendofoccupation)
        irow += 1
    end
end

function getGNDnumber(sgndnummer)
    rgx = r"[^/]+$"
    rgm = match(rgx, sgndnummer)
    if rgm == nothing
        @warn ("Keine GND-Nummer gefunden in: " * sgndnummer)
        return ""
    else
        return rgm.match
    end
end

"""
    queryGSbyGND!(row, colgnd, toldateofdeath, tolendofoccupation)

Ergänze `row` um Daten aus dem besten Eintrag in der Personendatenbank.
"""
function queryGSbyGND!(row, colgnd, toldateofdeath, tolendofoccupation)
    records = QRecord[]

    gndin = row[colgnd]
    if gndin != ""
        gndnummer = getGNDnumber(gndin)
        gsres = getGS(URLGS, Dict("person.gndnummer" => gndnumber))
        records = evaluate!(gsres, row, toldateofdeath, tolendofoccupation)
    end

    if length(records) > 0
        writerow!(row, records[1])
    end
end

"""
    queryGSNbyGND!(row, colsrc::Symbol, coldst::Symbol)

Frage GSN aufgrund der GND-Nummer ab. Schreibe die GSN in row[coldst]
"""
function queryGSNbyGND!(row, colsrc::Symbol, coldst::Symbol)
    gndnumber = row[colsrc]
    gsn = ""
    if gndnumber != ""
        gsres = getGS(URLGS, Dict("person.gndnummer" => gndnumber))
        ares = gsres["records"]
        nres = length(ares)
        if nres > 0
            row[coldst] = gsn = ares[1]["item.gsn"][1]["nummer"]
        elseif nres > 1
            @warn "Mehrere Treffer für" gndnumber
        end
    end
    gsn
end

"""
     evaluate!(gsres::Dict{String, Any}, listchecked, row, rowsocc, toldod, tolocc, mcols, occmcols)

Bewerte die Datensätze in `gsres` anhand der Werte in `row` und in den Zeilen `rowsocc`.
`listchecked`: Liste von IDs für Personen, die für `row` schon einmal geprüft wurden.
`toldod`: Toleranz Sterbedatum
`tolocc`: Toleranz Amtsbeginn, Amtsende
`mcols`: Vergleichspalten zur Person in den Abfragedaten
`occmcols`: Vergleichsspalten zum Amt in den Abfragedaten
"""
function evaluate!(gsres::Dict{String, Any}, listchecked, row, rowsocc, toldod, tolocc, mcols, occmcols)
    records = QRecord[]
    nvalid = 0
    matchkey = ""
    sreject = String[]

    for record in gsres["records"]
        idgs = record["person"]["id"]
        if idgs in listchecked
            continue
        else
            push!(listchecked, idgs)
        end
        # println(record["preferredName"], ": mk: ", mk, " scdod: ", scdod)

        # Die Bewertung wird in `record` in den Feldern `muster` und `zeitguete`
        # abgelegt.
        record["amuster"] = String[]
        record["zeitguete"] = 0
        evaluategnfn!(record, row, mcols)
        evaluatedod!(record, row, toldod, mcols)

        # Wenn die Sterbedaten vorhanden sind, aber außerhalb der Toleranz
        # liegen, lehne den Datensatz ab.
        if record["zeitguete"] == -1
            push!(sreject, join(record["amuster"], " "))
            continue
        end

        GSOcc.evaluate!(record, rowsocc, tolocc, occmcols)

        # Sortiere die Elemente des Musters entsprechend der Reihenfolge in
        # SLISTKEY
        sort!(record["amuster"], lt = Rank.kpless)
        matchkey = record["muster"] = join(record["amuster"], " ")

        if Rank.isvalidrank(matchkey)
            push!(records, QRecord(record))
        else
            matchkey != "" && push!(sreject, matchkey)
        end
    end

    if length(sreject) > 0
        with_logger(filelog) do
            @info "verworfen:" row[colid()] unique(sreject)
        end
    end

    return records
end

# Umgebung für evaltuatedod!
let

    ryear = "[0-9]?[0-9]?[0-9]{2}"
    rgxyear = Regex(ryear)
    rgxdobad = Regex("(" * ryear * ")"
                     * " *((-|bis)[^1-9]*"
                     * "(" * ryear * "))?")

    function matchdod(sdodqd::Union{<:AbstractString, Missing}, sdodcand, toldod)
        matchkey = ""
        score = 0
        if ismissing(sdodqd) || sdodqd in ("", "(?)", "?")
            return matchkey, score
        end

        rgm = match(rgxyear, sdodqd)
        if rgm == nothing
            @warn "Ungültiges Sterbedatum '$(sdodqd)'"
            return matchkey, score
        end
        dodqd = parse(Int, rgm.match)

        sdodcand == "" && return matchkey, score
        rgm = match(rgxyear, sdodcand)
        if rgm == nothing
            @warn ("Kein Datum gefunden in '" * sdodcand * "' für " * string(row[col(id)]))
            return matchkey, score
        end
        dodcand = parse(Int, rgm.match)
        delta = abs(dodcand - dodqd)
        if delta <= toldod
            score = toldod - delta
            matchkey = "sd"
        else
            score = -1
            matchkey = "dsd"
        end
        return matchkey, score
    end

    function matchdod(dodqd::T, sdodcand, toldod) where T<:Real
        matchkey = ""
        score = 0

        sdodcand == "" && return matchkey, score
        rgm = match(rgxyear, sdodcand)
        if rgm == nothing
            @warn ("Kein Datum gefunden in '" * sdodcand * "' für " * string(row[col(id)]))
            return matchkey, score
        end
        dodcand = parse(Int, rgm.match)
        delta = abs(dodcand - dodqd)
        if delta <= toldod
            score = toldod - delta
            matchkey = "sd"
        else
            score = -1
            matchkey = "dsd"
        end
        return matchkey, score
    end

    """
        evaluatedod!(record, row, toldod, mcols)

    Verwende `toldod` für die Bewertung des Wertes in `:Sterbedatum`.
    Wenn beide Daten vorhanden sind, aber außerhalb der Toleranz liegen,
    schreibe in das Feld "zeitguete" der Wert -1.
    `mcols`: Spalten in den Abfragedaten
    """
    global evaluatedod!
    function evaluatedod!(record, row, toldod, mcols)::Nothing
        # Für die Bischöfe vor 1198 gibt es oft nur eine Angabe für das
        # Jahrhundert. "[4. Jh.]"

        # rgxjh = r"([1-9][0-9]?)\. Jh" # wird im Moment nicht weiter verfolgt

        matchkey, score = "", 0
        if :Sterbedatum in mcols
            matchkey, score = matchdod(row[:Sterbedatum],
                                       record["person"]["sterbedatum"],
                                       toldod)

            if matchkey != ""
                push!(record["amuster"], matchkey)
                record["zeitguete"] = score
            end
        end
        return nothing
    end

end

"""
    evaluategnfn!(record, row, mcols)

Suche nach Familienname und oder Vorname
`qcol`: Spalten in den Abfragedaten
"""
function evaluategnfn!(record, row, mcols)

    stripsplit(a) = strip.(split(String(a), r",|;"))

    fnqd = [row[:Familienname]]
    if :Familiennamenvarianten in mcols
        fnqdalt = row[:Familiennamenvarianten]
        if Util.hasdata(fnqdalt)
            append!(fnqd, stripsplit(fnqdalt))
        end
    end

    fngs = [record["person"]["familienname"]]
    fngsalt = record["person"]["familiennamenvarianten"]
    if Util.hasdata(fngsalt)
        append!(fngs, stripsplit(fngsalt))
    end

    gnqd = [row[:Vorname]]
    if :Vornamenvarianten in mcols
        gnqdalt = row[:Vornamenvarianten]
        if Util.hasdata(gnqdalt)
            append!(gnqd, stripsplit(gsqdalt))
        end
    end

    gngs = [record["person"]["vorname"]]
    gngsalt = record["person"]["vornamenvarianten"]
    if Util.hasdata(gngsalt)
        append!(gngs, stripsplit(gngsalt))
    end

    ffn = findcommonelement(Util.checkname, fnqd, fngs)
    fgn = findcommonelement(Util.checkname, gnqd, gngs)

    matchkey = ["fn", "vn"][[ffn, fgn]]

    if matchkey != String[]
        append!(record["amuster"], matchkey)
    end
    return matchkey
end

"""
    findcommenelement(f, nqd, ngs)

Prüfe, ob `nqd` und `ngs` mindestens ein übereinstimmendes Elementepaar haben.
"""
function findcommonelement(f, nqd, ngs)::Bool
    fcommon = false
    for cnqd in nqd, cngs in ngs
        if f(cnqd, cngs)
            fcommon = true
            break
        end
    end
    return fcommon
end

function writematch!(row, bestrec, records)

    nbest = count(isequal(bestrec), records)
    writerow!(row, bestrec, nbest)
    if nbest > 1
        with_logger(filelog) do
            @info "mehrere Treffer:" row[colid()] bestrec["muster"]
        end
    end
    nbest
end

writerow!(row, record::QRecord, nbest = 1) = writerow!(row, record.data, nbest)

"""
    writerow!(row, node, nbest = 1)

`node` ist ein Datensatz aus der Liste "records" einer GS-Abfrage.
"""
function writerow!(row, node::Dict{String, Any}, nbest = 1)
    agsn = String[]
    for item in node["item.gsn"]
        push!(agsn, item["nummer"])
    end

    # Finde die erste GSN
    gsn1 = (pdelim = (findfirst(',', agsn[1]))) == nothing ? agsn[1] : agsn[1][1:(pdelim - 1)]

    row[:GSN1_GS] = gsn1
    row[:GSN_GS] = join(agsn, ", ")
    row[:ID_GND_GS] = node["person"]["gndnummer"]
    if haskey(node, "muster")
        matchkey = node["muster"]
        row[:Qualitaet_GS] = matchkey
        row[:QRang_GS] = Rank.getrank(matchkey)
    end
    row[:nTreffer_GS] = nbest
    row[:Vorname_GS] = node["person"]["vorname"]
    row[:Vornamenvarianten_GS] = node["person"]["vornamenvarianten"]
    row[:Namenspraefix_GS] = node["person"]["namenspraefix"]
    row[:Familienname_GS] = node["person"]["familienname"]
    row[:Familiennamenvarianten_GS] = node["person"]["familiennamenvarianten"]
    row[:Namenszusatz_GS] = node["person"]["namenszusatz"]
    row[:Geburtsdatum_GS] = node["person"]["geburtsdatum"]
    row[:Sterbedatum_GS] = node["person"]["sterbedatum"]
    if haskey(node, "amt") # eines der gesuchten Ämter
        row[:Amt_GS] = node["amt"]["bezeichnung"]
        row[:Amtsbeginn_GS] = node["amt"]["von"]
        row[:Amtsende_GS] = node["amt"]["bis"]
        row[:Dioezese_GS] = node["amt"]["dioezese"]
    end
    row[:Aemter_GS] = getocc(node)[1]
    # Ämter gibt es in GS mehrere
end

function getocc(node)
    aocc = String[]
    aoccplace = String[]
    if haskey(node, "aemter")
        for occrec in node["aemter"]
            push!(aocc, occrec["bezeichnung"])
            push!(aoccplace, occrec["dioezese"])
        end
    end
    return join(aocc, ", "), join(filter(!isequal(""), aoccplace), ", ")
end

function makeGSDataFrame(df::AbstractDataFrame)
    dfcols = Symbol.(names(df))
    if !(colid() in dfcols)
        error("Die Eingabetabelle enthält keine passende Spalte für die ID (ist: `$(colid())`). "
              * "Siehe `setcolnameid`")
    end

    global inputcols
    cpcols = intersect(inputcols, dfcols)
    dfgs = copy(df[!, cpcols])
    # `insertcols!` scheint nicht einfacher
    for col in GSSTRINGCOLS
        dfgs[!, col] .= ""
    end
    dfgs[!, :nTreffer_GS] .= 0
    dfgs[!, :QRang_GS] .= RANKMAX
    dfgs
end

"""
    clearbylevel!(df::AbstractDataFrame, matchkey::AbstractString, level::Int)

Lösche Datensätze, deren Qualität unter `matchkey`, bzw. `rank` liegt. Die Angaben
sind zur Sicherheit redundant.
"""
function clearbylevel!(df::AbstractDataFrame, matchkey::AbstractString, rank::Int)
    if rank != getrank(matchkey)
        error("Muster und Rang stimmen nicht überein")
    end
    colrank = df[!, :QRang_GS]
    ixr = colrank .> rank
    df[ixr, GSSTRINGCOLS] .= "" # zulässig!
    df[ixr, :QRang_GS] .= RANKMAX
    count(df[!, :QRang_GS] .< RANKMAX)
end

"""
    infolevel!(df::AbstractDataFrame, matchkey::AbstractString)

"""
function infolevel(df::AbstractDataFrame, matchkey::AbstractString)
    rank = getrank(matchkey)
    println("Das Muster '", matchkey, "' hat die Position: ", rank, ".")
    println("Datensätze insgesamt: ", size(df, 1))
    colrank = df[!, :QRang_GS]
    println("Rang größer als '", matchkey, "': ", count(colrank .> rank))
    rankmax = maximum(colrank)
    nvalid = count(colrank .< rankmax)
    ndel = count((colrank .< rankmax) .& (colrank .> rank))
    println("Davon kleiner maximaler Rang : ", ndel)
    println("Es bleiben erhalten: ", nvalid - ndel)
end

"""

Lege ein Verzeichnis von `colsrc zu `coldst` an.
"""
function iddictionary(df::AbstractDataFrame, colsrc::Symbol, coldst::Symbol)
    tsrc = eltype(df[!, colsrc])
    tdst = eltype(df[!, coldst])

    odict = Dict{tsrc, Set{tdst}}()
    for row in eachrow(df)
        vsrc, vdst = row[[colsrc, coldst]]
        if vsrc != "" && vdst != ""
            odict[vsrc] = push!(get(odict, vsrc, Set{tdst}()), vdst)
        end
    end
    odict
end

function compareyear(df::AbstractDataFrame, colA, colB, coldst)
    rgx = r"[0-9]?[0-9]?[0-9][0-9]"
    df[!, coldst] .= ""
    for row in eachrow(df)
        row[:nTreffer_GS] == "" && continue
        a = row[colA]
        b = row[colB]
        a == b == "" && continue
        (a == "" || b == "") && (row[coldst] = "v"; continue)
        rgma = match(rgx, a)
        rgmb = match(rgx, b)
        (rgma == nothing || rgmb == nothing) && (row[coldst] = "?"; continue)
        row[coldst] = string(abs(parse(Int, rgma.match) - parse(Int, rgmb.match)))
    end
end

function comparename(df::AbstractDataFrame, colA, colB, coldst)
    df[!, coldst] .= ""
    for row in eachrow(df)
        row[:nTreffer_GS] == "" && continue
        a = row[colA]
        b = row[colB]
        a == b == "" && continue
        (a == "" || b == "") && (row[coldst] = "v"; continue)
        seta = split(a)
        setb = split(b)
        ni = length(intersect(seta, setb))
        nu = length(union(seta, setb))
        row[coldst] = string(round(Int, (nu - ni)/nu * 100))
    end
end

"""
    deltaID(ref, gs)

Vergleiche IDs: `ref` ist z.B. manuell bestätigt.
"""
function deltaID(ref, gs)
    if gs == ""
        ref == "" ? (return "vv") : (return "xv")
    else
        ref == gs ? (return "t") : (return "f")
    end
end

end # GSquery


text:
using DataFrames

Base.convert(::Type{Float64}, ::NAtype) = NaN
using Gadfly

mad(X) = mad(X, median(X))
mad(X, med) = median([abs(x - med) for x in X])
spread(X) = maximum(x) - minimum(X)
radius(x, y) = √((x)^2 + (y)^2)

function v_a(df)
  vx = DataArray(Float64, size(df, 1))
  vy = DataArray(Float64, size(df, 1))
  ax = DataArray(Float64, size(df, 1))
  ay = DataArray(Float64, size(df, 1))

  vx[1] = NA
  vy[1] = NA
  ax[1] = NA
  ay[1] = NA

  for i in 2:size(df, 1)
    # check that frames directly follow each other
    if df[i-1, :Frame] == df[i, :Frame]-1
      Δy = df[i, :y] - df[i-1, :y]
      Δx = df[i, :x] - df[i-1, :x]
      Δt = df[i, :Time] - df[i-1, :Time]

      vx[i] = Δx/Δt
      vy[i] = Δy/Δt

      Δvx = vx[i] - vx[i-1]
      Δvy = vy[i] - vy[i-1]

      ax[i] = Δvx/Δt
      ay[i] = Δvy/Δt
    else #  Discontinuity
      vx[i] = NA
      vy[i] = NA
      ax[i] = NA
      ay[i] = NA
    end
  end

  r = DataFrame(t = df[:, :Time], vx = vx, vy = vy, ax = ax, ay = ay)
  r[:s] = √(r[:vx].^2 .+ r[:vy].^2)
  r[:a] = √(r[:ax].^2 .+ r[:ay].^2)
  r[:s_avg] = moving_avg(r[:s], 120) #at 30fps  => 4s
  r[:a_avg] = moving_avg(r[:a], 120)
  return r
end

function processData(folder, fileN)
  df = readtable("$folder/$fileN")
  dfOld = df

  median_r = median(head(df[:Radius]))

  # petri dish
  pR, px, py = readdlm("$folder/petriDish.txt")[1, 2:end]
  pDcm = 25 # cm
  cboatD = 0.3 #cm
  ratioCboat = median_r/(0.5*cboatD)

  rInfluence = 3.5 #cm
  rValid = pR/ratioCboat - rInfluence

  checkValid(x, y) = radius(x, y) < rValid

  df[:x] = (df[:x] - px)/ratioCboat
  df[:y] = (df[:y] - py)/ratioCboat

  #polar coordinates
  # df[:r] = map(radius, df[:x], df[:y])
  # df[:ϕ] = map((x, y) -> atan2(x,y), df[:x], df[:y])

  print("Recorded $(size(df, 1)) data points")
  # Remove all points that are probably bogus
  # cond = median_r - 2σ_r .<= df[:Radius] .<= median_r + 2σ_r

  duplicates = Int64[]
  for subdf in groupby(df, :Frame)
      if size(subdf, 1) > 1
          push!(duplicates, subdf[1,:Frame])
      end
  end

  cond = Bool[x ∉ duplicates for x in df[:Frame]]

  dfDiscarded = sub(df, !cond)
  df = sub(df, cond)
  print(" $(size(df, 1)) of them are valid and")
  # remove all points that are in the influence of the border
  dfValid = sub(df, map(checkValid, df[:x], df[:y]))
  println(" $(size(dfValid, 1)) are outside the influence of the boundary.")
  # plot(df, x = :x, y = :y)

  dfVA = v_a(df)
  dfValidVA = v_a(dfValid)
  writetable("$folder/va.csv", dfVA)
  writetable("$folder/vaFiltered.csv", dfValidVA)

  df, dfOld, dfVA, dfValidVA
end

function moving_avg(X :: DataArray, window :: Integer)
  avg = similar(X, Float64)
  if isodd(window)
    step = div(window - 1, 2)
  else
    step = div(window, 2)
  end

  for i in 1:size(X,1)
    if isna(X[i])
      avg[i] = NA
    else
      sum = 0.0
      n = 0
      for j in i-step:i+step
        if (1 <= j <= size(X,1)) && (!isna(X[j]))
          sum += X[j]
          n += 1
        end
      end
      avg[i] = sum / n
    end
  end
  return avg
end

function mean_windowed_frames(df, id, c, Δ)
  r = 1:Δ:size(df, 1)
  idx = Array(Float64, length(r))
  vals = Array(Float64, length(r))
  j = 1
  for i in r
    if i+Δ < size(df, 1)
      idx[j] = mean(df[i:i+Δ, id])
      vals[j] = mean(df[i:i+Δ, c])
    else
      idx[j] = mean(df[i:end, id])
      vals[j] = mean(df[i:end, c])
    end
    j += 1
  end
  idx, vals
end

function mean_windowed_time(df, id, c, Δt)
  r = 0:Δt:maximum(df[id])
  idx = Array(Float64, length(r)-1)
  vals = Array(Float64, length(r)-1)

  j = 1
  for t0 in r
    if j < length(r)
      t1 = t0 + Δt
      idx[j] = t0 + 0.5Δt

      cond = t0 .<= df[id] .< t1
      subdf = sub(df, cond, c)

      vals[j] = mean(subdf[c])
    end
    j += 1
  end
  idx, vals
end

# plot(vs, x = :t, y = :v, Geom.line)

ext: jl

max_stars_repo_name: vchuravy/TrackingCBoat

max_issues_repo_name: vchuravy/TrackingCBoat

max_forks_repo_name: vchuravy/TrackingCBoat


text:
import CImGuiExtensions.enable!
import CImGuiExtensions.disable!
import CImGuiExtensions.isenabled
import CImGuiExtensions.isrunning

Base.@kwdef mutable struct TimeZoneModel{T₁ <: AbstractString, T₂ <: Vector{String}} <: AbstractModel
    zone::T₁
    possible_zones::T₂
end

function get_zone(model::TimeZoneModel)
    model.zone
end

function get_possible_zones(model::TimeZoneModel)
    model.possible_zones
end

function set_zone!(model::TimeZoneModel, zone::AbstractString)
    model.zone = zone
end

Base.@kwdef struct TimeZoneDisplayProperties <: AbstractDisplayProperties
end

mutable struct TimeZoneControl <: AbstractControl
    isenabled::Bool
end

mutable struct SelectTimeZone <: AbstractOperation
    isenabled::Bool
end

struct TimeZoneContext{T₁ <: AbstractControl,   T₂ <: AbstractModel,  T₃ <: AbstractDisplayProperties, T₄ <: SelectTimeZone} <: AbstractPlotContext
    control::T₁
    model::T₂
    display_properties::T₃
    action::T₄
end

function isenabled(ctrl::SelectTimeZone)
    return ctrl.isenabled
end

function isenabled(ctrl::TimeZoneControl)
    return ctrl.isenabled
end

function enable!(ctrl::SelectTimeZone)
    ctrl.isenabled = true
end

function enable!(ctrl::TimeZoneControl)
    ctrl.isenabled = true
end

function disable!(ctrl::TimeZoneControl)
    ctrl.isenabled  = false
end

function disable!(ctrl::SelectTimeZone)
    ctrl.isenabled  = false
end

function (context::TimeZoneContext)()
        control = context.control
        model = context.model
        display_properties = context.display_properties
        action = context.action
        isenabled(control) ? control(model, display_properties, action) : nothing
        data = isenabled(action) ? action(get_zone(model)) : nothing
end

function (control::TimeZoneControl)(model::TimeZoneModel, properties::TimeZoneDisplayProperties, action::SelectTimeZone)
    if CImGui.BeginPopupModal("Select Time Zone", C_NULL, CImGui.ImGuiWindowFlags_AlwaysAutoResize)
        items = get_possible_zones(model)
        item_current = get_zone(model)
        # here our selection is a single pointer stored outside the object.
        if CImGui.BeginCombo("", item_current) # the second parameter is the label previewed before opening the combo.
            for n = 0:length(items)-1
                is_selected = item_current == items[n+1]
                CImGui.Selectable(items[n+1], is_selected) && (item_current = items[n+1];)
                is_selected && CImGui.SetItemDefaultFocus() # set the initial focus when opening the combo (scrolling + for keyboard navigation support in the upcoming navigation branch)
            end
            CImGui.EndCombo()
        end
        set_zone!(model, item_current)
        CImGui.Separator()
        CImGui.Button("OK", (120, 0)) && enable!(action) && disable!(control) && CImGui.CloseCurrentPopup()
        CImGui.SetItemDefaultFocus()
        CImGui.EndPopup()
    end
end

function (select::SelectTimeZone)(zone::AbstractString)
    disable!(select)
    zone
end

function isrunning(context::TimeZoneContext)
    isenabled(context.control) || isenabled(context.action)
end

function enable!(context::TimeZoneContext)
    enable!(context.control)
    CImGui.OpenPopup("Select Time Zone")
end


text:
# Knitro model

##################################################
# Model definition
##################################################
mutable struct Model
    # KNITRO context environment
    env::Env
    userdata::Dict

    # special callbacks (undefined by default)
    # (this functions do not depend on callback environments)
    ms_process::Function
    mip_callback::Function
    user_callback::Function
    ms_initpt_callback::Function
    puts_callback::Function

    # constructor
    function Model()
        model = new(Env(), Dict())
        # add a destructor to properly delete model
        finalizer(KN_free, model)
        return model
    end
    Model(env::Env, options::Dict) = new(env, options)
end

"Free solver object."
KN_free(m::Model) = free_env(m.env)

"Create solver object."
KN_new() = Model()

is_valid(m::Model) = is_valid(m.env)

##################################################
# Basic model manipulation
##################################################

"Set all parameters specified in the given file"
function KN_load_param_file(m::Model, filename::AbstractString)
    ret = @kn_ccall(load_param_file, Cint, (Ptr{Cvoid}, Ptr{Cchar}),
                    m.env, filename)
    _checkraise(ret)
end

"Save current parameters in the given file"
function KN_save_param_file(m::Model, filename::AbstractString)
    ret = @kn_ccall(save_param_file, Cint, (Ptr{Cvoid}, Ptr{Cchar}),
                    m.env, filename)
    _checkraise(ret)
end

"Reset all parameters to default values"
function KN_reset_params_to_defaults(m::Model)
    ret = @kn_ccall(reset_params_to_defaults, Cint, (Ptr{Cvoid}, ),
                    m.env)
    _checkraise(ret)
end

"Set tuner file."
function KN_load_tuner_file(m::Model, filename::AbstractString)
    ret = @kn_ccall(load_tuner_file, Cint, (Ptr{Cvoid}, Ptr{Cchar}),
                    m.env, filename)
    _checkraise(ret)
end

function KN_load_mps_file(m::Model, filename::AbstractString)
    @assert KNITRO_VERSION >= v"12.0"
    ret = @kn_ccall(load_mps_file, Cint, (Ptr{Cvoid}, Ptr{Cchar}),
                    m.env, filename)
    _checkraise(ret)
end

##################################################
# LM license manager
##################################################
"""
Type declaration for the Artelys License Manager context object.
Applications must not modify any part of the context.
"""
mutable struct LMcontext
    ptr_lmcontext::Ptr{Cvoid}
    # Keep a pointer to instantiated models in order to free
    # memory properly.
    linked_models::Vector{Model}

    function LMcontext()
        ptrref = Ref{Ptr{Cvoid}}()
        res = @kn_ccall(checkout_license, Cint, (Ptr{Cvoid},), ptrref)
        if res != 0
            error("KNITRO: Error checkout license")
        end
        lm = new(ptrref[], Model[])
        finalizer(KN_release_license, lm)
        return lm
    end
end

function Env(lm::LMcontext)
    ptrptr_env = Ref{Ptr{Cvoid}}()
    res = @kn_ccall(new_lm, Cint, (Ptr{Cvoid},Ptr{Cvoid}), lm.ptr_lmcontext, ptrptr_env)
    if res != 0
        error("Fail to retrieve a valid KNITRO KN_context. Error $res")
    end
    Env(ptrptr_env[])
end

function attach!(lm::LMcontext, model::Model)
    push!(lm.linked_models, model)
    return
end

# create Model with license manager
function Model(lm::LMcontext)
    model = Model(Env(lm), Dict())
    attach!(lm, model)
    return model
end
KN_new_lm(lm::LMcontext) = Model(lm)

function KN_release_license(lm::LMcontext)
    # First, ensure that all linked models are properly freed
    # before releasing license manager!
    KN_free.(lm.linked_models)

    if lm.ptr_lmcontext != C_NULL
        refptr = Ref{Ptr{Cvoid}}(lm.ptr_lmcontext)
        @kn_ccall(release_license, Cint, (Ptr{Cvoid},), refptr)
        lm.ptr_lmcontext = C_NULL
    end
end


text:
#!/usr/bin/julia

#cont-convert.jl
#continuous conversion tests on conversions

include("../../unum.jl")
using Unums

#performs a conversion wherin we try every possible bit permutation of the unum
#and then run the following steps:
# 1) random float
# 2) convert to unum
# 3) direct calculate float from unum
# 4) check to make sure 1 & 3 are the same
# 4) convert back to float from unum
# 5) check to make sure 4 & 1 are the same

#test random, exact addition in the forward direction.
while true
  x1 = exp(100 * randn()) * (rand() > 0.5 ? -1 : 1)
  x2 = 0.0
  try
    u1 = convert(Unum{4,6}, x1)
    x2 = convert(Float64, u1)
  catch
    println("error converting $(bits(x1))")
  end

  if (x1 != x2)
    println("$x1 => $x2 converts funny, diff: $(x1 - x2)")
  end
end

max_stars_repo_name: REX-Computing/unumjl

max_issues_repo_name: REX-Computing/unumjl

max_forks_repo_name: REX-Computing/unumjl


text:
#    Copyright (c) 2021 J.A. Duffek
#    Copyright (c) 2000 D.M. Spink
#
#    This program is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program; if not, see <http://www.gnu.org/licenses/>.

"""
    nrbderiv(nurbs::NURBS{I,F}) where {I<:Integer,F<:AbstractFloat}

Construct the first derivative representation of a NURBS curve or surface.

The derivatives of a B-Spline are themselves a B-Spline of lower degree, giving
an efficient means of evaluating multiple derivatives. However, although the
same approach can be applied to NURBS, the situation for NURBS is more complex.
I have at present restricted the derivatives to just the first. I don't claim
that this implentation is the best approach, but it will have to do for now.
The function returns a data struture that for a NURBS curve contains the first
derivatives of the B-Spline representation. Remember that a NURBS curve is
represented by a univariate B-Spline using the homogeneous coordinates. The
derivative data structure can be evaluated later with the function
[`nrbdeval`](@ref).

# Arguments:
- `nurbs`: NURBS data structure, see [`nrbmak`](@ref).

# Output:
- `dnurbs`: A data structure that represents the first derivatives of a NURBS
  curve or surface.

# Examples:
```julia
julia> dnurbs = nrbderiv(nurbs);
```

See the function [`nrbdeval`](@ref) for an example.

# Reference
- see also: [`bspderiv`](@ref), [`nrbdeval`](@ref)
"""
function nrbderiv(nurbs::NURBS{I,F}) where {I<:Integer,F<:AbstractFloat}
degree = nurbs.order .- 1;

if typeof(nurbs)<:NURBS2D
  # NURBS represents a surface
  num1 = nurbs.number[1];
  num2 = nurbs.number[2];
  dnurbs = Vector{NURBS2D{I,F}}(undef,2)

  dknots = Vector{Vector{F}}(undef,2);
  # taking derivatives along the u direction
  dknots[2] = nurbs.knots[2];
  dcoefs = permutedims(nurbs.coefs,[1,3,2]);
  dcoefs = reshape(dcoefs,(4*num2,num1));
  dcoefs, dknots[1] = bspderiv(degree[1],dcoefs,nurbs.knots[1]);
  dcoefs = permutedims(reshape(dcoefs,(4,num2,size(dcoefs,2))),[1,3,2]);
  dnurbs[1] = nrbmak(dcoefs, dknots);

  dknots = Vector{Vector{F}}(undef,2);
  # taking derivatives along the v direction
  dknots[1] = nurbs.knots[1];
  dcoefs = reshape(nurbs.coefs,(4*num1,num2));
  dcoefs, dknots[2] = bspderiv(degree[2],dcoefs,nurbs.knots[2]);
  dcoefs = reshape(dcoefs,(4,num1,size(dcoefs,2)));
  dnurbs[2] = nrbmak(dcoefs, dknots);

  return dnurbs;
else
  # NURBS represents a curve

  dcoefs, dknots = bspderiv(degree,nurbs.coefs,nurbs.knots);
  dnurbs = nrbmak(dcoefs, dknots);

  return dnurbs;
end # if
end # nrbderiv


text:
# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

_sample(::Variogram, p::Point) = [p]

function _sample(γ::Variogram, g::Geometry)
  α = _spacing(γ, g)
  rng = MersenneTwister(123)
  sample(rng, g, MinDistanceSampling(α))
end

function _sample(γ::Variogram, s::Segment)
  s′ = Segment(s(.05), s(.95))
  n  = _dims(γ, s)
  sample(s′, RegularSampling(n...))
end

function _sample(γ::Variogram, q::Quadrangle)
  q′ = Quadrangle(q(.05,.05), q(.95,.05), q(.95,.95), q(.05,.95))
  n  = _dims(γ, q)
  sample(q′, RegularSampling(n...))
end

function _sample(γ::Variogram, h::Hexahedron)
  h′ = Hexahedron(h(.05,.05,.05), h(.95,.05,.05), h(.95,.95,.05), h(.05,.95,.05),
                  h(.05,.05,.95), h(.95,.05,.95), h(.95,.95,.95), h(.05,.95,.95))
  n  = _dims(γ, h)
  sample(h′, RegularSampling(n...))
end

function _spacing(γ::Variogram, g::Geometry)
  r = range(γ)
  s = sides(boundingbox(g))
  l = minimum(filter(>(0), s))
  r > 0 ? min(r, l) / 3 : l / 3
end

function _dims(γ::Variogram, g::Geometry)
  s = sides(boundingbox(g))
  α = _spacing(γ, g)
  n = ceil.(Int, s ./ α)
  replace(n, 0 => 1)
end


text:
__precompile__()

module AutomotiveDrivingModels

using Compat
using Reexport
using Discretizers
using Parameters

@reexport using DataFrames
@reexport using Distributions
@reexport using Vec

include("core/AutoCore.jl")
@reexport using .AutoCore

include(Pkg.dir("AutomotiveDrivingModels", "src", "utils", "minkowski.jl"))
include(Pkg.dir("AutomotiveDrivingModels", "src", "features", "features.jl"))
include(Pkg.dir("AutomotiveDrivingModels", "src", "features", "feature_extractors.jl"))
include(Pkg.dir("AutomotiveDrivingModels", "src", "features", "aggregate.jl"))

include(Pkg.dir("AutomotiveDrivingModels", "src", "simulation", "actions.jl"))
include(Pkg.dir("AutomotiveDrivingModels", "src", "behaviors", "driver_models.jl"))
include(Pkg.dir("AutomotiveDrivingModels", "src", "simulation", "simulation.jl"))

include(Pkg.dir("AutomotiveDrivingModels", "src", "behaviors", "behaviors.jl"))

include(Pkg.dir("AutomotiveDrivingModels", "src", "utils", "roadway_generation.jl"))
include(Pkg.dir("AutomotiveDrivingModels", "src", "utils", "trajdata_cleaning.jl"))

include(Pkg.dir("AutomotiveDrivingModels", "src", "evaluation", "metrics.jl"))
include(Pkg.dir("AutomotiveDrivingModels", "src", "evaluation", "foldsets.jl"))
include(Pkg.dir("AutomotiveDrivingModels", "src", "evaluation", "fold_assigners.jl"))
include(Pkg.dir("AutomotiveDrivingModels", "src", "evaluation", "trajdata_segments.jl"))
include(Pkg.dir("AutomotiveDrivingModels", "src", "evaluation", "evaluation_data.jl"))

end # module


text:
Xcords, Ycords, Zcords = readSRF()
Plots.unicodeplots()
plotSRF(Xcords, Ycords, Zcords)


text:
module JMLi
using DataStructures
using ReplMaker
using JML
using REPL.LineEdit

# a bug of julia? workaround:
Base.convert(::Type{Symbol}, a::Int) = Symbol(a)

const repl_scope = global_scope()
const repl_main = new_module_spec("main", "<repl>")
const repl_modules = OrderedDict("main" => repl_main)

function valid_expr(s)
    try
        str = String(take!(copy(LineEdit.buffer(s))))
        runparser(str, Val(:rexp), Val(:stmt)) !== nothing
    catch e
        if e isa JML.ParserFailed
            return false
        end
        rethrow(e)
    end
end

function parse_jml_expr(s)
    try
        to_julia(
            s;
            mode=:stmt,
            scope=repl_scope,
            modules=repl_modules
        )
    catch e
        println(string(e))
    end
end

initrepl(
    parse_jml_expr,
    valid_input_checker = valid_expr,
    prompt_text="jml> ",
    prompt_color = :light_blue,
    start_key="\\",
    mode_name="jml_mode"
)
end

max_forks_repo_name: thautwarm/Rupy


text:
export LLVMType, issized, context, show

abstract type LLVMType end
reftype(::Type{T}) where {T<:LLVMType} = API.LLVMTypeRef

identify(::Type{LLVMType}, ref::API.LLVMTypeRef) =
    identify(LLVMType, Val{API.LLVMGetTypeKind(ref)}())
identify(::Type{LLVMType}, ::Val{K}) where {K} = error("Unknown type kind $K")

@inline function check(::Type{T}, ref::API.LLVMTypeRef) where T<:LLVMType
    ref==C_NULL && throw(UndefRefError())
    if Base.JLOptions().debug_level >= 2
        T′ = identify(LLVMType, ref)
        if T != T′
            error("invalid conversion of $T′ type reference to $T")
        end
    end
end

# Construct a concretely typed type object from an abstract type ref
function LLVMType(ref::API.LLVMTypeRef)
    ref == C_NULL && throw(UndefRefError())
    T = identify(LLVMType, ref)
    return T(ref)::LLVMType
end

issized(typ::LLVMType) =
    convert(Core.Bool, API.LLVMTypeIsSized(ref(typ)))
context(typ::LLVMType) = Context(API.LLVMGetTypeContext(ref(typ)))

function Base.show(io::IO, typ::LLVMType)
    output = unsafe_message(API.LLVMPrintTypeToString(ref(typ)))
    print(io, output)
end

Base.isempty(@nospecialize(T::LLVMType)) = false

## integer

export width

@checked struct IntegerType <: LLVMType
    ref::reftype(LLVMType)
end
identify(::Type{LLVMType}, ::Val{API.LLVMIntegerTypeKind}) = IntegerType

for T in [:Int1, :Int8, :Int16, :Int32, :Int64, :Int128]
    jl_fname = Symbol(T, :Type)
    api_fname = Symbol(:LLVM, jl_fname)
    @eval begin
        $jl_fname() = IntegerType(API.$api_fname())
        $jl_fname(ctx::Context) =
            IntegerType(
                      API.$(Symbol(api_fname, :InContext))(ref(ctx)))
    end
end

IntType(bits::Integer) = IntegerType(API.LLVMIntType(bits))
IntType(bits::Integer, ctx::Context) =
    IntegerType(API.LLVMIntTypeInContext(ref(ctx), bits))

width(inttyp::IntegerType) = API.LLVMGetIntTypeWidth(ref(inttyp))

## floating-point

# NOTE: this type doesn't exist in the LLVM API,
#       we add it for convenience of typechecking generic values (see execution.jl)
 abstract type FloatingPointType <: LLVMType end

# NOTE: we don't handle the obscure types here (:X86FP80, :FP128, :PPCFP128),
#       they would also need special casing as LLVMPPCFP128Type != LLVMPPC_FP128TypeKind
for T in [:Half, :Float, :Double]
    jl_fname = Symbol(T, :Type)
    api_typename = Symbol(:LLVM, T)
    api_fname = Symbol(:LLVM, jl_fname)
    enumkind = Symbol(:LLVM, T, :TypeKind)
    @eval begin
        @checked struct $api_typename <: FloatingPointType
            ref::reftype(FloatingPointType)
        end
        identify(::Type{LLVMType}, ::Val{API.$enumkind}) = $api_typename

        $jl_fname() = $api_typename(API.$api_fname())
        $jl_fname(ctx::Context) =
            $api_typename(API.$(Symbol(api_fname, :InContext))(ref(ctx)))
    end
end

## function types

export isvararg, return_type, parameters

@checked struct FunctionType <: LLVMType
    ref::reftype(LLVMType)
end
identify(::Type{LLVMType}, ::Val{API.LLVMFunctionTypeKind}) = FunctionType

FunctionType(rettyp::LLVMType, params::Vector{T}=LLVMType[];
             vararg::Core.Bool=false) where {T<:LLVMType} =
    FunctionType(API.LLVMFunctionType(ref(rettyp), ref.(params),
                                      length(params), convert(Bool, vararg)))

isvararg(ft::FunctionType) =
    convert(Core.Bool, API.LLVMIsFunctionVarArg(ref(ft)))

return_type(ft::FunctionType) =
    LLVMType(API.LLVMGetReturnType(ref(ft)))

function parameters(ft::FunctionType)
    nparams = API.LLVMCountParamTypes(ref(ft))
    params = Vector{API.LLVMTypeRef}(undef, nparams)
    API.LLVMGetParamTypes(ref(ft), params)
    return LLVMType[LLVMType(param) for param in params]
end

## composite types

abstract type CompositeType <: LLVMType end

## sequential types

export addrspace

abstract type SequentialType <: CompositeType end

Base.eltype(typ::SequentialType) = LLVMType(API.LLVMGetElementType(ref(typ)))

@checked struct PointerType <: SequentialType
    ref::reftype(SequentialType)
end
identify(::Type{LLVMType}, ::Val{API.LLVMPointerTypeKind}) = PointerType

function PointerType(eltyp::LLVMType, addrspace=0)
    return PointerType(API.LLVMPointerType(ref(eltyp), addrspace))
end

addrspace(ptrtyp::PointerType) =
    API.LLVMGetPointerAddressSpace(ref(ptrtyp))

@checked struct ArrayType <: SequentialType
    ref::reftype(SequentialType)
end
identify(::Type{LLVMType}, ::Val{API.LLVMArrayTypeKind}) = ArrayType

function ArrayType(eltyp::LLVMType, count)
    return ArrayType(API.LLVMArrayType(ref(eltyp), count))
end

Base.length(arrtyp::ArrayType) = API.LLVMGetArrayLength(ref(arrtyp))

Base.isempty(@nospecialize(T::ArrayType)) = length(T) == 0 || isempty(eltype(T))

@checked struct VectorType <: SequentialType
    ref::reftype(SequentialType)
end
identify(::Type{LLVMType}, ::Val{API.LLVMVectorTypeKind}) = VectorType

function VectorType(eltyp::LLVMType, count)
    return VectorType(API.LLVMVectorType(ref(eltyp), count))
end

Base.size(vectyp::VectorType) = API.LLVMGetVectorSize(ref(vectyp))

## structure types

export name, ispacked, isopaque, elements!

@checked struct StructType <: SequentialType
    ref::reftype(SequentialType)
end
identify(::Type{LLVMType}, ::Val{API.LLVMStructTypeKind}) = StructType

function StructType(name::String, ctx::Context=GlobalContext())
    return StructType(API.LLVMStructCreateNamed(ref(ctx), name))
end

StructType(elems::Vector{T}, packed::Core.Bool=false) where {T<:LLVMType} =
    StructType(API.LLVMStructType(ref.(elems), length(elems),
                                  convert(Bool, packed)))

StructType(elems::Vector{T}, ctx::Context, packed::Core.Bool=false) where {T<:LLVMType} =
    StructType(API.LLVMStructTypeInContext(ref(ctx), ref.(elems),
                                           length(elems), convert(Bool, packed)))

name(structtyp::StructType) =
    unsafe_string(API.LLVMGetStructName(ref(structtyp)))
ispacked(structtyp::StructType) =
    convert(Core.Bool, API.LLVMIsPackedStruct(ref(structtyp)))
isopaque(structtyp::StructType) =
    convert(Core.Bool, API.LLVMIsOpaqueStruct(ref(structtyp)))

elements!(structtyp::StructType, elems::Vector{T}, packed::Core.Bool=false) where {T<:LLVMType} =
    API.LLVMStructSetBody(ref(structtyp), ref.(elems),
                          length(elems), convert(Bool, packed))

Base.isempty(@nospecialize(T::StructType)) =
    isempty(elements(T)) || all(isempty, elements(T))

# element iteration

export elements

struct StructTypeElementSet
    typ::StructType
end

elements(typ::StructType) = StructTypeElementSet(typ)

Base.eltype(::StructTypeElementSet) = LLVMType

function Base.getindex(iter::StructTypeElementSet, i)
    @boundscheck 1 <= i <= length(iter) || throw(BoundsError(iter, i))
    return LLVMType(API.LLVMStructGetTypeAtIndex(ref(iter.typ), i-1))
end

function Base.iterate(iter::StructTypeElementSet, i=1)
    i >= length(iter) + 1 ? nothing : (iter[i], i+1)
end

Base.length(iter::StructTypeElementSet) = API.LLVMCountStructElementTypes(ref(iter.typ))

Base.lastindex(iter::StructTypeElementSet) = length(iter)

# NOTE: optimized `collect`
function Base.collect(iter::StructTypeElementSet)
    elems = Vector{API.LLVMTypeRef}(undef, length(iter))
    API.LLVMGetStructElementTypes(ref(iter.typ), elems)
    return LLVMType[LLVMType(elem) for elem in elems]
end

## other

@checked struct VoidType <: LLVMType
    ref::reftype(LLVMType)
end
identify(::Type{LLVMType}, ::Val{API.LLVMVoidTypeKind}) = VoidType

VoidType() = VoidType(API.LLVMVoidType())
VoidType(ctx::Context) =
    VoidType(API.LLVMVoidTypeInContext(ref(ctx)))

@checked struct LabelType <: LLVMType
    ref::reftype(LLVMType)
end
identify(::Type{LLVMType}, ::Val{API.LLVMLabelTypeKind}) = LabelType

LabelType() = LabelType(API.LLVMLabelType())
LabelType(ctx::Context) =
    LabelType(API.LLVMLabelTypeInContext(ref(ctx)))

@checked struct MetadataType <: LLVMType
    ref::reftype(LLVMType)
end
identify(::Type{LLVMType}, ::Val{API.LLVMMetadataTypeKind}) = MetadataType

MetadataType(ctx::Context) =
    MetadataType(API.LLVMMetadataTypeInContext(ref(ctx)))

@checked struct TokenType <: LLVMType
    ref::reftype(LLVMType)
end
identify(::Type{LLVMType}, ::Val{API.LLVMTokenTypeKind}) = TokenType

TokenType(ctx::Context) =
    TokenType(API.LLVMTokenTypeInContext(ref(ctx)))


text:
immutable EdgeIterState
    s::Int  # src vertex
    di::Int # index into adj of dest vertex
    fin::Bool
end

type EdgeIter
    m::Int
    adj::Vector{Vector{Int}}
    directed::Bool
end

eltype(::Type{EdgeIter}) = Edge

EdgeIter(g::Graph) = EdgeIter(ne(g), g.fadjlist, false)
EdgeIter(g::DiGraph) = EdgeIter(ne(g), g.fadjlist, true)

function _next(eit::EdgeIter, state::EdgeIterState = EdgeIterState(1,1,false), first::Bool = true)
    s = state.s
    di = state.di
    if !first
        di += 1
    end
    fin = state.fin
    while s <= length(eit.adj)
        arr = eit.adj[s]
        while di <= length(arr)
            if eit.directed || s <= arr[di]
                return EdgeIterState(s, di, fin)
            end
            di += 1
        end
        s += 1
        di = 1
    end
    fin = true
    return EdgeIterState(s, di, fin)
end

start(eit::EdgeIter) = _next(eit)
done(eit::EdgeIter, state::EdgeIterState) = state.fin
length(eit::EdgeIter) = eit.m

function next(eit::EdgeIter, state)
    edge = Edge(state.s, eit.adj[state.s][state.di])
    return(edge, _next(eit, state, false))
end

function _isequal(e1::EdgeIter, e2)
    for e in e2
        s, d = e
        found = length(searchsorted(e1.adj[s], d)) > 0
        if !e1.directed
            found = found || length(searchsorted(e1.adj[d],s)) > 0
        end
        !found && return false
    end
    return true
end
==(e1::EdgeIter, e2::AbstractArray{Edge,1}) = _isequal(e1, e2)
==(e1::AbstractArray{Edge,1}, e2::EdgeIter) = _isequal(e2, e1)
==(e1::EdgeIter, e2::Set{Edge}) = _isequal(e1, e2)
==(e1::Set{Edge}, e2::EdgeIter) = _isequal(e2, e1)

function ==(e1::EdgeIter, e2::EdgeIter)
    length(e1.adj) == length(e2.adj) || return false
    e1.directed == e2.directed || return false
    for i in 1:length(e1.adj)
        e1.adj[i] == e2.adj[i] || return false
    end
    return true
end

show(io::IO, eit::EdgeIter) = write(io, "EdgeIter $(eit.m)")
show(io::IO, s::EdgeIterState) = write(io, "EdgeIterState [$(s.s), $(s.di), $(s.fin)]")


text:
using Dash
using DashHtmlComponents
using DashCoreComponents
js_files = [
    "https://www.google-analytics.com/analytics.js",
    Dict("src"=> "https://cdn.polyfill.io/v2/polyfill.min.js"),
    Dict(
        "src"=> "https://cdnjs.cloudflare.com/ajax/libs/ramda/0.26.1/ramda.min.js",
        "integrity"=> "sha256-43x9r7YRdZpZqTjDT5E0Vfrxn1ajIZLyYWtfAXsargA=",
        "crossorigin"=> "anonymous",
    ),
    Dict(
        "src"=> "https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.11/lodash.min.js",
        "integrity"=> "sha256-7/yoZS3548fXSRXqc/xYzjsmuW3sFKzuvOCHd06Pmps=",
        "crossorigin"=> "anonymous",
    ),
]

css_files = [
    "https://codepen.io/chriddyp/pen/bWLwgP.css",
    Dict(
        "href"=> "https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css",
        "rel"=> "stylesheet",
        "integrity"=> "sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO",
        "crossorigin"=> "anonymous",
    )
]

app = dash(external_scripts=js_files, external_stylesheets=css_files)
app.index_string = """<!DOCTYPE html>

        {%metas%}
        <title>{%title%}</title>
        {%css%}

        <button type="button" id="btn">Btn</button>
        {%app_entry%}

            {%config%}
            {%scripts%}
            {%renderer%}

"""

app.layout = html_div()

run_server(app)


text:
# AUTO GENERATED FILE - DO NOT EDIT

export prism

"""
    prism(;kwargs...)

A Prism component.
Code highlight with Mantine theme colors and styles. For more information, see: https://mantine.dev/others/prism/
Keyword arguments:
- `id` (String; optional): The ID of this component, used to identify dash components in callbacks
- `className` (String; optional): Often used with CSS to style elements with common properties
- `code` (String; required): Code which will be highlighted
- `colorScheme` (a value equal to: "light", "dark"; optional): Force color scheme, defaults to theme.colorScheme
- `copiedLabel` (String; optional): Copy button tooltip in copied state
- `copyLabel` (String; optional): Copy button tooltip
- `highlightLines` (optional): Highlight line at given line number with color from theme.colors. highlightLines has the following type: Dict with Strings as keys and values of type lists containing elements 'color', 'label'.
Those elements have the following types:
  - `color` (a value equal to: "dark", "gray", "red", "pink", "grape", "violet", "indigo", "blue", "cyan", "teal", "green", "lime", "yellow", "orange"; optional)
  - `label` (String; optional)
- `language` (a value equal to: "markup", "bash", "clike", "c", "cpp", "css", "javascript", "jsx", "coffeescript", "actionscript", "css-extr", "diff", "git", "go", "graphql", "handlebars", "json", "less", "makefile", "markdown", "objectivec", "ocaml", "python", "reason", "sass", "scss", "sql", "stylus", "tsx", "typescript", "wasm", "yaml"; required): Programming language that should be highlighted
- `noCopy` (Bool; optional): True to remove copy to clipboard button
- `style` (Dict; optional): Inline style override
- `withLineNumbers` (Bool; optional): Display line numbers
"""
function prism(; kwargs...)
        available_props = Symbol[:id, :className, :code, :colorScheme, :copiedLabel, :copyLabel, :highlightLines, :language, :noCopy, :style, :withLineNumbers]
        wild_props = Symbol[]
        return Component("prism", "Prism", "dash_mantine_components", available_props, wild_props; kwargs...)
end


text:
"""
ДАНО: Робот - в произвольной клетке ограниченного прямоугольного поля, на котором могут находиться также
внутренние прямоугольные перегородки (все перегородки изолированы друг от друга, прямоугольники могут вырождаться в отрезки)

РЕЗУЛЬТАТ: Робот - в исходном положении и в углах поля стоят маркеры
"""

using HorizonSideRobots
originRobot = Robot(animate=true, "5example.sit")

# moving_in_angle! передвинет робота в Юго-Западный угол поля и вернёт в переменную back_bath ОБРАТНЫЙ путь робота
function moving_in_angle!(robot::Robot, sides)
    back_path = []
    while (!(isborder(robot, sides[1])) || !(isborder(robot, sides[2])))
        for side in (sides[1], sides[2])
            steps = 0
            while (!(isborder(robot,side)))
                move!(robot,side)
                steps += 1
            end
            push!(back_path, steps)
        end
    end
    return reverse!(back_path)
end

# moving_back_to_start! вернёт робота в начальное положение через обратный путь робота
function moving_back_to_start!(robot::Robot,sides,back_path::Vector)
    for (index,value) in enumerate(back_path)
        if (index % 2 == 1)
            for step in 1:value
                move!(robot, anti_side(sides[2]))
            end
        else
            for step in 1:value
                move!(robot, anti_side(sides[1]))
            end
        end
    end
end

function master5!(robot::Robot)::Nothing
    for sides in [(West,Sud), (West,Nord), (Ost,Sud), (Ost,Nord)]
        back_path = moving_in_angle!(robot,sides)
        putmarker!(robot)
        moving_back_to_start!(robot,sides,back_path)
    end
end

master5!(originRobot)

max_stars_repo_name: Igetsuga/AlgorithmicsHomework

max_issues_repo_name: Igetsuga/AlgorithmicsHomework

max_forks_repo_name: Igetsuga/AlgorithmicsHomework


text:
module TrussTopOptProblems

using Ferrite, StaticArrays, LinearAlgebra
using SparseArrays
using ..TopOpt
using ..TopOpt.Utilities
using Setfield
import Ferrite: assemble!
using LinearAlgebra: I, norm
using NearestNeighbors

abstract type AbstractFEAMaterial end
struct TrussFEAMaterial{T} <: AbstractFEAMaterial
    E::T # Young's modulus
    ν::T # Poisson's ratio
end

abstract type AbstractFEACrossSec end
struct TrussFEACrossSec{T} <: AbstractFEACrossSec
    A::T # cross section area
end

include("grids.jl")
include("problem_types.jl")
include("matrices_and_vectors.jl")
include("elementinfo.jl")
include(joinpath("TrussIO", "TrussIO.jl"))
using .TrussIO
include(joinpath("TrussVisualization", "TrussVisualization.jl"))
using .TrussVisualization

export TrussGrid, TrussProblem, TrussFEACrossSec, TrussFEAMaterial
export PointLoadCantileverTruss
export load_truss_geo, load_truss_json

end # module


text:
#=
Handles models defined as named tuples.

* Developer: Hilding Elmqvist, Mogram AB
* First version: January 2021
* License: MIT (expat)

=#
export mergeModels, recursiveMerge, Redeclare, showModel, @showModel, Model, Map, Par, Var, setLogMerge,
    constant, parameter, input, output, potential, flow, interval, @info_str

using Base.Meta: isexpr
using DataStructures: OrderedDict
using Unitful
using ModiaBase.Symbolic: removeBlock, prepend

function stringifyDefinition(v)
    if typeof(v) in [Symbol, Expr]
        v = removeBlock(v)
        v = ":(" * string(v) * ")"
    end
    return v
end

function showModel(m, level=0)
    level += 1
#    print("  "^(level-1))
    if typeof(m) <: NamedTuple && haskey(m, :class)
        print(m.class)
    end
    println("(")
    for (k, v) in zip(keys(m), m)
        if typeof(v) <: NamedTuple
            print("  "^level, k, " = ")
            showModel(v, level)
        elseif k != :class
            println("  "^level, k, " = ", stringifyDefinition(v), ",")
        end
    end
    println("  "^(level-1), "),")
end

macro showModel(model)
    modelName = string(model)
    mod = :( print($modelName, " = "); showModel($model); println() )
    return esc(mod)
end

global logMerge = false

function setLogMerge(val)
    global logMerge
    logMerge = val
end

function mergeModels(m1::NamedTuple, m2::NamedTuple, env=Symbol())
    mergedModels = OrderedDict{Symbol,Any}(pairs(m1))
    for (k,v) in collect(pairs(m2))
        if typeof(v) <: NamedTuple
            if k in keys(mergedModels) && ! (:_redeclare in keys(v))
                if logMerge; print("In $k: ") end
                m = mergeModels(mergedModels[k], v, k)
                mergedModels[k] = m
            elseif :_redeclare in keys(v)
                if logMerge; println("Redeclaring: $k = $v") end
                mergedModels[k] = v
            elseif nothing in values(v) # TODO: Refine

            else
                if !(:_redeclare in keys(mergedModels))
                    if logMerge; print("Adding: $k = "); showModel(v, 2) end
                end
                mergedModels[k] = v
            end
        elseif v === nothing
            if logMerge; println("Deleting: $k") end
            delete!(mergedModels, k)
        else
            if logMerge
                if k in keys(mergedModels)
                    if mergedModels[k] != v
                        println("Changing: $k = $(stringifyDefinition(mergedModels[k])) to $k = $(stringifyDefinition(v))")
                    end
                elseif !(:_redeclare in keys(mergedModels))
                    println("Adding: $k = $(stringifyDefinition(v))")
                end
            end
            mergedModels[k] = v
        end
    end
#    delete!(mergedModels, :class)
    return (; mergedModels...) # Transform OrderedDict to named tuple
end

Model(; kwargs...) = (; class = :Model, kwargs...)

Map(; kwargs...) = (; class = :Map, kwargs...)

Par(; kwargs...) = Map(; class = :Par, kwargs...)

Var(;kwargs...) = (; class = :Var, kwargs...)
Var(value; kwargs...) = (; class = :Var, value=value, kwargs...)

Redeclare = ( _redeclare = true, )

constant = Var(constant = true)
parameter = Var(parameter = true)
input = Var(input = true)
output = Var(output = true)
potential = Var(potential = true)
flow = Var(flow = true)

interval(min, max) = Var(min=min, max=max)

macro info_str(text)
    Var(info=text)
end

Base.:|(m::NamedTuple, n::NamedTuple) =  mergeModels(m, n) # TODO: Chane to updated recursiveMerge
Base.:|(m, n) = if !(typeof(n) <: NamedTuple); recursiveMerge(m, (; value=n)) else recursiveMerge(n, (value=m,)) end

recursiveMerge(x, ::Nothing) = x
recursiveMerge(x, y) = y
#recursiveMerge(x::Expr, y::Expr) = begin dump(x); dump(y); Expr(x.head, x.args..., y.args...) end
recursiveMerge(x::Expr, y::Tuple) = begin x = copy(x); xargs = x.args; xargs[y[2]] = y[3]; Expr(x.head, xargs...) end

function recursiveMerge(nt1::NamedTuple, nt2::NamedTuple)
    all_keys = union(keys(nt1), keys(nt2))
#    all_keys = setdiff(all_keys, [:class])
    gen = Base.Generator(all_keys) do key
        v1 = get(nt1, key, nothing)
        v2 = get(nt2, key, nothing)
        key => recursiveMerge(v1, v2)
    end
    return (; gen...)
end

function unpackPath(path, sequence)
    if typeof(path) == Symbol
        push!(sequence, path)
    elseif isexpr(path, :.)
        unpackPath(path.args[1], sequence)
        push!(sequence, path.args[2].value)
    elseif isexpr(path, :ref)
        unpackPath(path.args[1], sequence)
        push!(sequence, path.args[2:end]...)
    end
end

function collectConnector(model)
    potentials = []
    flows = []
    for (k,v) in zip(keys(model), model)
        if typeof(v) <: NamedTuple && :class in keys(v) && v.class == :Var ||
            typeof(v) <: NamedTuple && :variable in keys(v) && v.variable
            if :potential in keys(v) && v.potential
                push!(potentials, k)
            end
            if :flow in keys(v) && v.flow
                push!(flows, k)
            end
        end
    end
    return potentials, flows
end

function convertConnections!(connections, model, modelName, logging=false)
#    println("\nconvertConnections")
#    showModel(model)
#    @show connections
#    println()
    connectEquations = []
    alreadyConnected = []
    for i in 1:length(connections)
        c = connections[i]

        if c.head == :tuple
            connected = c.args

            potential1 = nothing
            inflow = 0
            outflow = 0
            signalFlow1 = nothing
            connectedOutput = nothing
            potentials1 = []
            flows1 = []
            for con in connected
                if con in alreadyConnected
                    error("Already connected: $con, found in connection: $connected")
                end
                push!(alreadyConnected, con)

                sequence = []
                unpackPath(con, sequence)
                mod = model
                for s in sequence[1:end-1]
                    if s in keys(mod)
                        mod = mod[s]
                    else
                        error("Invalid path $con: $s not found in $(keys(mod))")
                    end
                end

                if sequence[end] in keys(mod)
                    mod = mod[sequence[end]]

                    if :input in keys(mod) && mod.input || :output in keys(mod) && mod.output
                        signalFlow = con
                        if signalFlow1 !== nothing
                            push!(connectEquations, :($signalFlow1 = $signalFlow))
                        end
                        signalFlow1 = signalFlow

                        if :output in keys(mod) && mod.output
                            if connectedOutput != nothing # TODO: refine logic concerning inner and outer inputs and outputs
#                                error("It is not allowed to connect two outputs: ", connectedOutput, ", ", con)
                            end
                            connectedOutput = con
                        end
                    else
                        potentials, flows = collectConnector(mod)
                        # Deprecated
                        if :potentials in keys(mod)
                            potentials = vcat(potentials, mod.potentials.args)
                        end
                        if :flows in keys(mod)
                            flows = vcat(flows, mod.flows.args)
                        end

                        if length(potentials1) > 0 && potentials != potentials1
                            error("Not compatible potential variables: $potentials1 != $potentials, found in connection: $connected")
                        end
                        for p in potentials
                            potential = append(con, p)
                            if potential1 != nothing
                                push!(connectEquations, :($potential1 = $potential))
                            end
                            potential1 = potential
                        end
                        potentials1 = potentials

                        if length(flows1) > 0 && flows != flows1
                            error("Not compatible flow variables: $flows1 != $flows, found in connection: $connected")
                        end
                        for f in flows
                            flowVar = append(con, f)
                            if length(sequence) == 1
                                if inflow == 0
                                    inflow = flowVar
                                else
                                    inflow = :($inflow + $flowVar)
                                end
                            else
                                if outflow == 0
                                    outflow = flowVar
                                else
                                    outflow = :($outflow + $flowVar)
                                end
                            end
                        end
                        flows1 = flows
                    end
                else
                    # Deprecated
                    signalFlow = con
                    if signalFlow1 !== nothing
                        push!(connectEquations, :($signalFlow1 = $signalFlow))
                    end
                    signalFlow1 = signalFlow
                end
            end
            if inflow != 0 || outflow != 0
                push!(connectEquations, :($inflow = $outflow))
            end
        end
    end
    if length(connectEquations) > 0 && logging
        println("Connect equations in $modelName:")
        for e in connectEquations
            println("  ", e)
        end
    end
    return connectEquations
end

function flattenModelTuple!(model, modelStructure, modelName; unitless = false, log=false)
    connections = []
    extendedModel = merge(model, NamedTuple())
    for (k,v) in zip(keys(model), model)
#        @show k v typeof(v)
        # Deprecated
        if k in [:inputs, :outputs, :potentials, :flows, :class]
            if k in [:inputs, :outputs, :potentials, :flows]
                printstyled("  Deprecated construct in model $modelName: ", color=:yellow)
                println("$k = $v\n    Use: ... = $(string(k)[1:end-1]) ...")
            end
        elseif k == :init
            printstyled("  Deprecated construct in model $modelName: ", color=:yellow)
            println("$k = $v\n    Use: ... = Var(init=...) ...")
            for (x,x0) in zip(keys(v), v)
                if x != :class
                    if unitless && typeof(x0) != Expr
                        x0 = ustrip(x0)
                    end
                    modelStructure.init[x] = x0
                    modelStructure.mappedParameters = (;modelStructure.mappedParameters..., x => x0)
                end
            end
        elseif k == :start
            printstyled("  Deprecated construct in model $modelName: ", color=:yellow)
            println("$k = $v\n    Use: ... = Var(start=...) ...")
            for (s,s0) in zip(keys(v), v)
                if s != :class
                    if unitless
                        s0 = ustrip(s0)
                    end
                    modelStructure.start[s] = s0
                    modelStructure.mappedParameters = (;modelStructure.mappedParameters...,  s => s0)
                end
            end

        elseif typeof(v) in [Int64, Float64] || typeof(v) <: Unitful.Quantity ||
                typeof(v) in [Array{Int64,1}, Array{Int64,2}, Array{Float64,1}, Array{Float64,2}] ||
				typeof(v) <: NamedTuple && :class in keys(v) && v.class == :Par ||
				typeof(v) <: NamedTuple && :parameter in keys(v) && v.parameter
            if unitless && !(typeof(v) <: NamedTuple)
                v = ustrip(v)
            end
            modelStructure.parameters[k] = v
            modelStructure.mappedParameters = (;modelStructure.mappedParameters..., k => v)
        elseif (typeof(v) <: NamedTuple && :class in keys(v) && v.class in [:Par, :Var] ||
            typeof(v) <: NamedTuple && :parameter in keys(v) && v.parameter) &&
            :value in keys(v)
            v = v.value
            if typeof(v) in [Expr, Symbol]
                push!(modelStructure.equations, removeBlock(:($k = $v)))
            else
                modelStructure.parameters[k] = v
                modelStructure.mappedParameters = (;modelStructure.mappedParameters..., k => v)
            end
         elseif typeof(v) <: NamedTuple && :class in keys(v) && v.class in [:Var] ||
            typeof(v) <: NamedTuple && :variable in keys(v) && v.variable
            if :init in keys(v)
                x0 = v.init
                if unitless && typeof(x0) != Expr
                    x0 = ustrip(x0)
                end
                modelStructure.init[k] = x0
                modelStructure.mappedParameters = (;modelStructure.mappedParameters..., k => x0)
            end
            if :start in keys(v)
                s0 = v.start
                if unitless && typeof(s0) != Expr
                    s0 = ustrip(s0)
                end
                modelStructure.start[k] = s0
                modelStructure.mappedParameters = (;modelStructure.mappedParameters..., k => s0)
            end
        elseif typeof(v) <: NamedTuple # instantiate
                subModelStructure = ModelStructure()
                flattenModelTuple!(v, subModelStructure, k; unitless, log)
#                println("subModelStructure")
#                printModelStructure(subModelStructure, k)
                mergeModelStructures(modelStructure, subModelStructure, k)
        elseif typeof(v) <:Array && length(v) > 0 && typeof(v[1]) <: NamedTuple # array of instances
            i = 0
            for a in v
                i += 1
                subModelStructure = ModelStructure()
                flattenModelTuple!(a, subModelStructure, k; unitless, log)
                mergeModelStructures(modelStructure, subModelStructure, Symbol(string(k)*"_"*string(i)) )
            end
        elseif isexpr(v, :vect) || isexpr(v, :vcat) || isexpr(v, :hcat)
            arrayEquation = false
            for e in v.args
                if isexpr(e, :(=))
                    if unitless
                        e = removeUnits(e)
                    end
                    push!(modelStructure.equations, removeBlock(e))
                elseif isexpr(e, :tuple)
                    push!(connections, e)
                else
                    arrayEquation = true
                end
            end
            if arrayEquation
                push!(modelStructure.equations, removeBlock(:($k = $(prepend(v, :up)))))
#                 push!(modelStructure.equations, removeBlock(:($k = $v)))
            end
        elseif isexpr(v, :(=)) # Single equation
            if unitless
                v = removeUnits(v)
            end
            push!(modelStructure.equations, removeBlock(v))
        elseif v !== nothing # Binding expression
#            println("Binding expression")
#            @show k v typeof(v)

            if unitless
                v = removeUnits(v)
            end
            push!(modelStructure.equations, :($k = $(prepend(v, :up))))
#            push!(modelStructure.equations, :($k = $v))
#            @show modelStructure.equations
        end
    end
#    printModelStructure(modelStructure, "flattened")
    connectEquations = convertConnections!(connections, extendedModel, modelName, log)
    push!(modelStructure.equations, connectEquations...)
end


text:
export Planar, PlanarNN

struct PlanarNN
    u::AbstractVector
    w::AbstractVector
    b::AbstractVector

    h::Function
end

function PlanarNN(nvars::Integer, h::Function=tanh; cond=false, rng::Union{AbstractRNG, Nothing}=nothing)
    u = isnothing(rng) ? randn(nvars) : randn(rng, nvars)
    w = isnothing(rng) ? randn(cond ? nvars*2 : nvars) : randn(rng, cond ? nvars*2 : nvars)
    b = isnothing(rng) ? randn(1) : randn(rng, 1)
    PlanarNN(u, w, b, h)
end

Flux.@functor PlanarNN (u, w, b)

function (m::PlanarNN)(z::AbstractVecOrMat)::AbstractVecOrMat
    u, w, b = m.u, m.w, only(m.b)
    h = NNlib.fast_act(m.h, z)
    u * h.(transpose(w) * z .+ b)
end

"""
Implementation of Planar Flows from

[Chen, Ricky TQ, Yulia Rubanova, Jesse Bettencourt, and David Duvenaud. "Neural Ordinary Differential Equations." arXiv preprint arXiv:1806.07366 (2018).](https://arxiv.org/abs/1806.07366)
"""
struct Planar{T <: AbstractFloat} <: AbstractICNF{T}
    re::Optimisers.Restructure
    p::AbstractVector{T}

    nvars::Integer
    basedist::Distribution
    tspan::Tuple{T, T}

    solver_test::SciMLBase.AbstractODEAlgorithm
    solver_train::SciMLBase.AbstractODEAlgorithm

    sensealg_test::SciMLBase.AbstractSensitivityAlgorithm
    sensealg_train::SciMLBase.AbstractSensitivityAlgorithm

    acceleration::AbstractResource
    array_mover::Function

    # trace_test
    # trace_train
end

function Planar{T}(
        nn::PlanarNN,
        nvars::Integer,
        ;
        basedist::Distribution=MvNormal(zeros(T, nvars), Diagonal(ones(T, nvars))),
        tspan::Tuple{T, T}=convert(Tuple{T, T}, (0, 1)),

        solver_test::SciMLBase.AbstractODEAlgorithm=default_solver_test,
        solver_train::SciMLBase.AbstractODEAlgorithm=default_solver_train,

        sensealg_test::SciMLBase.AbstractSensitivityAlgorithm=default_sensealg,
        sensealg_train::SciMLBase.AbstractSensitivityAlgorithm=default_sensealg,

        acceleration::AbstractResource=default_acceleration,
        ) where {T <: AbstractFloat}
    array_mover = make_mover(acceleration, T)
    p, re = destructure(nn)
    Planar{T}(
        re, p |> array_mover, nvars, basedist, tspan,
        solver_test, solver_train,
        sensealg_test, sensealg_train,
        acceleration, array_mover,
    )
end

function augmented_f(icnf::Planar{T}, sz::Tuple{T2, T2})::Function where {T <: AbstractFloat, T2 <: Integer}
    o_ = ones(T, sz) |> icnf.array_mover

    function f_aug(u, p, t)
        m = icnf.re(p)
        z = u[1:end - 1, :]
        mz, back = Zygote.pullback(m, z)
        J = only(back(o_))
        trace_J = transpose(m.u) * J
        vcat(mz, -trace_J)
    end
    f_aug
end

function inference(icnf::Planar{T}, mode::TestMode, xs::AbstractMatrix{T}, p::AbstractVector=icnf.p; rng::Union{AbstractRNG, Nothing}=nothing)::AbstractVector where {T <: AbstractFloat}
    xs = xs |> icnf.array_mover
    zrs = zeros(T, 1, size(xs, 2)) |> icnf.array_mover
    f_aug = augmented_f(icnf, size(xs))
    prob = ODEProblem{false}(f_aug, vcat(xs, zrs), icnf.tspan, p)
    sol = solve(prob, icnf.solver_test; sensealg=icnf.sensealg_test)
    fsol = sol[:, :, end]
    z = fsol[1:end - 1, :]
    Δlogp = fsol[end, :]
    logp̂x = logpdf(icnf.basedist, z) - Δlogp
    logp̂x
end

function inference(icnf::Planar{T}, mode::TrainMode, xs::AbstractMatrix{T}, p::AbstractVector=icnf.p; rng::Union{AbstractRNG, Nothing}=nothing)::AbstractVector where {T <: AbstractFloat}
    xs = xs |> icnf.array_mover
    zrs = zeros(T, 1, size(xs, 2)) |> icnf.array_mover
    f_aug = augmented_f(icnf, size(xs))
    prob = ODEProblem{false}(f_aug, vcat(xs, zrs), icnf.tspan, p)
    sol = solve(prob, icnf.solver_train; sensealg=icnf.sensealg_train)
    fsol = sol[:, :, end]
    z = fsol[1:end - 1, :]
    Δlogp = fsol[end, :]
    logp̂x = logpdf(icnf.basedist, z) - Δlogp
    logp̂x
end

function generate(icnf::Planar{T}, mode::TestMode, n::Integer, p::AbstractVector=icnf.p; rng::Union{AbstractRNG, Nothing}=nothing)::AbstractMatrix{T} where {T <: AbstractFloat}
    new_xs = isnothing(rng) ? rand(icnf.basedist, n) : rand(rng, icnf.basedist, n)
    new_xs = new_xs |> icnf.array_mover
    zrs = zeros(T, 1, size(new_xs, 2)) |> icnf.array_mover
    f_aug = augmented_f(icnf, size(new_xs))
    prob = ODEProblem{false}(f_aug, vcat(new_xs, zrs), reverse(icnf.tspan), p)
    sol = solve(prob, icnf.solver_test; sensealg=icnf.sensealg_test)
    fsol = sol[:, :, end]
    z = fsol[1:end - 1, :]
    z
end

function generate(icnf::Planar{T}, mode::TrainMode, n::Integer, p::AbstractVector=icnf.p; rng::Union{AbstractRNG, Nothing}=nothing)::AbstractMatrix{T} where {T <: AbstractFloat}
    new_xs = isnothing(rng) ? rand(icnf.basedist, n) : rand(rng, icnf.basedist, n)
    new_xs = new_xs |> icnf.array_mover
    zrs = zeros(T, 1, size(new_xs, 2)) |> icnf.array_mover
    f_aug = augmented_f(icnf, size(new_xs))
    prob = ODEProblem{false}(f_aug, vcat(new_xs, zrs), reverse(icnf.tspan), p)
    sol = solve(prob, icnf.solver_train; sensealg=icnf.sensealg_train)
    fsol = sol[:, :, end]
    z = fsol[1:end - 1, :]
    z
end

Flux.@functor Planar (p,)

function loss(icnf::Planar{T}, xs::AbstractMatrix{T}, p::AbstractVector=icnf.p; agg::Function=mean) where {T <: AbstractFloat}
    logp̂x = inference(icnf, TrainMode(), xs, p)
    agg(-logp̂x)
end


text:
# RETURN FUNCTIONS
function mountaincar_return_based_model_search_main(num_episodes_offline::Int64)
    rng = MersenneTwister(0)
    model_search_steps = []
    model = MountainCar(0.0)
    horizon = 500
    for rock_c in range_of_values
        mountaincar = MountainCar(rock_c)
        data = generate_batch_data(mountaincar, true_params, num_episodes_offline, horizon; rng = rng)
        agent =
            MountainCarModelSearchAgent(mountaincar, model, horizon, data)
        n_steps = run_return_based_model_search(agent; debug = true)
        push!(model_search_steps, n_steps)
    end
    model_search_steps
end

function mountaincar_return_based_model_search(rock_c::Float64, num_episodes_offline::Int64)
    rng = MersenneTwister(0)
    model = MountainCar(0.0)
    mountaincar = MountainCar(rock_c)
    horizon = 500
    data = generate_batch_data(mountaincar, true_params, num_episodes_offline, horizon; rng = rng)
    agent = MountainCarModelSearchAgent(mountaincar, model, horizon, data)
    n_steps = run_return_based_model_search(agent; debug = true)
    println("Reached goal in ", n_steps)
end

# BELLMAN ERROR FUNCTIONS
function mountaincar_bellman_based_model_search(rock_c::Float64, num_episodes_offline::Int64)
    rng = MersenneTwister(0)
    model = MountainCar(0.0)
    mountaincar = MountainCar(rock_c)
    horizon = 500
    data = generate_batch_data(mountaincar, true_params, num_episodes_offline, horizon; rng = rng)
    agent = MountainCarModelSearchAgent(mountaincar, model, horizon, data)
    n_steps = run_bellman_based_model_search(agent; debug = true)
    println(n_steps)
end

function mountaincar_bellman_based_model_search_main(num_episodes_offline::Int64)
    rng = MersenneTwister(0)
    model_search_steps = []
    model = MountainCar(0.0)
    horizon = 500
    for rock_c in range_of_values
        mountaincar = MountainCar(rock_c)
        data = generate_batch_data(mountaincar, true_params, num_episodes_offline, horizon; rng = rng)
        agent =
            MountainCarModelSearchAgent(mountaincar, model, horizon, data)
        n_steps = run_bellman_based_model_search(agent, max_steps = 1e4)
        push!(model_search_steps, n_steps)
    end
    model_search_steps
end

function bellman_experiment(rock_c::Float64, num_episodes_offline::Int64; seed::Int64 = 0)
    rng = MersenneTwister(seed)
    model = MountainCar(0.0)
    mountaincar = MountainCar(rock_c)
    horizon = 500
    data = generate_batch_data(mountaincar, true_params, num_episodes_offline, horizon, rng = rng)
    println("Generated ", length(data), " transitions")
    agent = MountainCarModelSearchAgent(mountaincar, model, horizon, data)
    n_steps = run_return_based_model_search(agent; debug = true)
    n_bellman_steps = run_bellman_based_model_search(agent; debug = true)
    println("Return ", n_steps)
    println("Bellman ", n_bellman_steps)
    n_steps, n_bellman_steps
end

function bellman_experiment(
    rock_c::Float64,
    data::Array{MountainCarContTransition},
)
    model = MountainCar(0.0)
    mountaincar = MountainCar(rock_c)
    horizon = 500
    agent = MountainCarModelSearchAgent(mountaincar, model, horizon, data)
    n_steps = run_return_based_model_search(agent; debug=false)
    n_bellman_steps = run_bellman_based_model_search(agent; debug=false)
    #println("Return ", n_steps)
    #println("Bellman ", n_bellman_steps)
    n_steps, n_bellman_steps
end

function bellman_experiment_episodes(rock_c::Float64)
    println()
    println("Experiment with rock_c ", rock_c)
    horizon = 500
    # num_episodes = [250, 500, 1000, 1500, 2000]
    num_episodes = [5, 10, 20]
    parent_rng = MersenneTwister(0)
    seeds = rand(parent_rng, UInt32, 10)
    experiment_data = get_experiment_data(
        MountainCar(rock_c),
        horizon,
        seeds,
        num_episodes[end],
    )
    n_steps = []
    n_bellman_steps = []
    for episodes in num_episodes
        n_sub_steps = []
        n_bellman_sub_steps = []
        for data in experiment_data
            sliced_data = data[1:min(episodes*horizon, length(data))]
            result = bellman_experiment(rock_c, sliced_data)
            push!(n_sub_steps, result[1])
            push!(n_bellman_sub_steps, result[2])
        end
        push!(n_steps, n_sub_steps)
        push!(n_bellman_steps, n_bellman_sub_steps)
    end
    n_matrix_steps = hcat(n_steps...)
    n_matrix_bellman_steps = hcat(n_bellman_steps...)
    mean_n_steps = mean(n_matrix_steps, dims=1)
    std_n_steps = std(n_matrix_steps, dims=1)
    mean_n_bellman_steps = mean(n_matrix_bellman_steps, dims=1)
    std_n_bellman_steps = std(n_matrix_bellman_steps, dims=1)
    println(num_episodes)
    println("Mean return steps ", mean_n_steps)
    println("Std return steps ", std_n_steps)
    println("Mean bellman steps ", mean_n_bellman_steps)
    println("Std bellman steps ", std_n_bellman_steps)
    println(n_steps)
    println(n_bellman_steps)
    n_steps, n_bellman_steps
end

function all_bellman_experiments()
    rock_c_values = [0, 0.005, 0.01, 0.015, 0.02, 0.025, 0.03, 0.035, 0.04]
    n_return_steps = []
    n_bellman_steps = []
    for rock_c in rock_c_values
        result = bellman_experiment_episodes(rock_c)
        push!(n_return_steps, result[1])
        push!(n_bellman_steps, result[2])
    end
    println()
    n_return_steps, n_bellman_steps
end

function bellman_experiment_rock_c()
    num_episodes = 1000
    rock_cs = [0, 0.01, 0.02, 0.03, 0.04]
    n_steps = []
    n_bellman_steps = []
    for rock_c in rock_cs
        result = bellman_experiment(rock_c, num_episodes; seed = 0)
        push!(n_steps, result[1])
        push!(n_bellman_steps, result[2])
    end
    println(num_episodes)
    println(n_steps)
    println(n_bellman_steps)
end

function get_experiment_data(
    mountaincar::MountainCar,
    horizon::Int64,
    seeds::Array{UInt32},
    max_episodes::Int64,
)::Array{Array{MountainCarContTransition}}
    experiment_data = []
    for seed in seeds
        rng = MersenneTwister(seed)
        push!(experiment_data, generate_batch_data(
            mountaincar,
            true_params,
            max_episodes,
            horizon;
            rng = rng
        ))
    end
    experiment_data
end

# PROFILING FUNCTIONS
function mfmc_evaluation_profile()
    mountaincar = MountainCar(0.0)
    data = generate_batch_data(mountaincar, true_params, 1000, 500)
    x_array, xnext_array, cost_array = preprocess_data(data)
    policy, values, _ = value_iteration(mountaincar, true_params)
    mfmc_evaluation(mountaincar, policy, values, 500, x_array, xnext_array, cost_array, 1)
end

function hill_climb_profile()
    mountaincar = MountainCar(0.0)
    data = generate_batch_data(mountaincar, true_params, 10, 10)
    optimization_params =
        MountainCarOptimizationParameters([0.0024, 1], [-0.0025, 3], 20, false)
    return_based_model_search(mountaincar, data, optimization_params, 10)
end

# HARDCODED DISTANCE FUNCTIONS
function hardcoded_experiment(rock_c::Float64, num_episodes_offline::Int64;
                              seed = 0)
    rng = MersenneTwister(seed)
    model = MountainCar(0.0)
    mountaincar = MountainCar(rock_c)
    horizon = 500
    data = generate_batch_data(mountaincar, true_params, num_episodes_offline, horizon; rng = rng)
    println("Generated ", length(data), " transitions")
    agent = MountainCarModelSearchAgent(mountaincar, model, horizon, data)
    n_steps = run_return_based_model_search(
        agent; debug = false, eval_distance = true
    )
    n_ensemble_steps = run_return_based_model_search(
        agent; hardcoded=true, debug = false, eval_distance = true
    )
    println("Without hardcoded distance ", n_steps)
    println("With hardcoded distance ", n_ensemble_steps)
    n_steps, n_ensemble_steps
end

function hardcoded_experiment_episodes(rock_c::Float64)
    # num_episodes = [250, 500, 750, 1000]
    num_episodes = [10, 25, 50]
    for episodes in num_episodes
        println("Running with ", episodes, " episodes")
        hardcoded_experiment(rock_c, episodes)
    end
end

function hardcoded_experiment_seeds(rock_c::Float64, num_episodes::Int64)
    parent_rng = MersenneTwister(0)
    seeds = rand(parent_rng, UInt32, 5)
    # num_episodes = 10
    n_steps = []
    n_hardcoded_steps = []
    for seed in seeds
        println()
        result = hardcoded_experiment(rock_c, num_episodes; seed=seed)
        push!(n_steps, result[1])
        push!(n_hardcoded_steps, result[2])
    end
    n_steps, n_hardcoded_steps
end

# OPTIMISTIC EXPERIMENTS
function optimistic_experiment(rock_c::Float64, num_episodes_offline::Int64;
                               seed = 0)
    rng = MersenneTwister(seed)
    model = MountainCar(0.0)
    mountaincar = MountainCar(rock_c)
    horizon = 500
    data = generate_batch_data(mountaincar, true_params, num_episodes_offline,
                               horizon; rng = rng)
    println("Generated ", length(data), " transitions")
    agent = MountainCarModelSearchAgent(mountaincar, model, horizon, data)
    n_steps = run_return_based_model_search(
        agent; debug = true,
    )
    n_optimistic_steps = run_return_based_model_search(
        agent; debug = true, optimistic = true,
    )
    println("Without optimism ", n_steps)
    println("With optimism ", n_optimistic_steps)
    n_steps, n_optimistic_steps
end


text:
## Test names.jl retrieve and set dimension names
## (c) 2013--2020 David A. van Leeuwen

include("init-namedarrays.jl")

@test names(n) == Array[["one", "two"], ["a", "b", "c", "d"]]
@test defaultnames(n.array) == Array[["1", "2"], ["1", "2", "3", "4"]]

@test dimnames(n) == [:A, :B]
@test dimnames(n.array) == [:A, :B]

@test dimnames(n.array, 1) == :A
@test dimnames(n.array, 2) == :B

dn1 = ["一", "二"]
dn2 = ["一", "二", "三", "四"]

@inferred setnames!(n, dn1, 1)
for (i, zh) in enumerate(dn2)
	@inferred setnames!(n, zh, 2, i)
end

@test names(n) == Array[dn1, dn2]

@inferred setdimnames!(n, ("thing1", :thing2))
@test dimnames(n) == ["thing1", :thing2]
setdimnames!(n, ["magnificent", 7]) ## can't infer mixed types
@test dimnames(n) == ["magnificent", 7]

println(n)

## replacing a single index name, and then accessing it was never tested, #83
for (dim, numerals) in enumerate(defaultnames(n.array))
	for (index, name) in enumerate(numerals)
		setnames!(n, name, dim, index)
		@test names(n, dim)[index] == name
	end
end

## I am sure this can be done more nicely with CartesianIndices
for i in 1:2, j in 1:4
	@test n[string(i), string(j)] == n[i, j]
end

println(n)

@test_throws TypeError setnames!(n, [:a, :v], 1)
@test_throws TypeError setnames!(n, :a, 1, 1)
@test_throws DimensionMismatch setnames!(n, ["a"], 1)
for i in [0,5], j in 1:2
	@test_throws BoundsError setnames!(n, "a", j, i)
end