text:
function splice(a,idx,rep=nothing)
      res=a[1:idx-1]
      !isnothing(rep) && append!(res,rep)
      append!(res,a[idx+1:end])
      res
end

max_stars_repo_name: yeruoforever/SimpleGrammar

max_issues_repo_name: yeruoforever/SimpleGrammar

max_forks_repo_name: yeruoforever/SimpleGrammar


text:
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# Description
#
#   Private functions and macros.
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

# Convert the alignment symbol into a LaTeX alignment string.
function _latex_alignment(s::Symbol)
    if s == :l
        return "l"
    elseif s == :c
        return "c"
    elseif s == :r
        return "r"
    else
        error("Invalid LaTeX alignment symbol: $s.")
    end
end

# Get the LaTeX table description (alignment and vertical columns).
function _latex_table_desc(alignment::Vector{Symbol},
                           vlines::AbstractVector{Int},
                           left_vline::AbstractString,
                           mid_vline::AbstractString,
                           right_vline::AbstractString)
    str = "{"

    0 ∈ vlines && (str *= left_vline)

    for i = 1:length(alignment)
        str *= _latex_alignment(alignment[i])

        if i ∈ vlines
            if i != length(alignment)
                str *= mid_vline
            else
                str *= right_vline
            end
        end
    end

    str *= "}"

    return str
end

# Wrap the `text` into LaTeX environment(s).
_latex_envs(text::AbstractString, envs::Vector{String}) = _latex_envs(text, envs, length(envs))
_latex_envs(text::AbstractString, env::String) = "\\" * string(env) * "{" * text * "}"

function _latex_envs(text::AbstractString, envs::Vector{String}, i::Int)
    @inbounds if i > 0
        str = _latex_envs(text, envs[i])
        return _latex_envs(str, envs, i -= 1)
    end

    return text
end

# This was adapted from Julia `escape_string` function. In case of LaTeX, we
# should not escape the sequence `\\`, i.e. the backslash that is already
# escaped.
function _str_latex_escaped(io::IO, s::AbstractString, esc="")
    a = Iterators.Stateful(s)
    for c in a
        if c in esc
            print(io, '\\', c)
        elseif isascii(c)
            c == '\0'          ? print(io, escape_nul(peek(a))) :
            c == '\e'          ? print(io, "\\e") :
          # c == '\\'          ? print(io, "\\\\") :
            '\a' <= c <= '\r'  ? print(io, '\\', "abtnvfr"[Int(c)-6]) :
            isprint(c)         ? print(io, c) :
                                 print(io, "\\x", string(UInt32(c), base = 16, pad = 2))
        elseif !isoverlong(c) && !ismalformed(c)
            isprint(c)         ? print(io, c) :
            c <= '\x7f'        ? print(io, "\\x", string(UInt32(c), base = 16, pad = 2)) :
            c <= '\uffff'      ? print(io, "\\u", string(UInt32(c), base = 16, pad = need_full_hex(peek(a)) ? 4 : 2)) :
                                 print(io, "\\U", string(UInt32(c), base = 16, pad = need_full_hex(peek(a)) ? 8 : 4))
        else # malformed or overlong
            u = bswap(reinterpret(UInt32, c))
            while true
                print(io, "\\x", string(u % UInt8, base = 16, pad = 2))
                (u >>= 8) == 0 && break
            end
        end
    end
end

_str_latex_escaped(s::AbstractString, esc="") =
    sprint(_str_latex_escaped, s, esc, sizehint=lastindex(s))

ext: jl


text:
s = open("input") do file
    readlines(file);
end

nums = map(x->parse(Int, x), s);

sort!(nums);

function GetPathCount(nums)
    if nums[1] != 0
        pushfirst!(nums, 0);
    end

    combos = ones(Int128, length(nums));
    for i = 2:length(nums)
        count::Int128 = combos[i-1];
        for j = i-3:i-2
            if j > 0
                if nums[i] - nums[j] <= 3
                    count += combos[j];
                end
            end
        end

        combos[i] = max(1, count);
    end

    return combos[end];
end

println(GetPathCount(nums))


text:
module GLFW

using GLFW_jll

function GetVersion()
	major, minor, rev = Ref{Cint}(), Ref{Cint}(), Ref{Cint}()
	ccall((:glfwGetVersion, libglfw), Cvoid, (Ref{Cint}, Ref{Cint}, Ref{Cint}), major, minor, rev)
	VersionNumber(major[], minor[], rev[])
end

include("callback.jl")
include("glfw3.jl")
include("vulkan.jl")
include("monitor_properties.jl")

function __init__()
	libversion = GetVersion()

	if libversion.major != 3
		error("GLFW version $libversion not supported")
	end
	# Save errors that occur during initialization
	errors = Vector{Exception}()
	SetErrorCallback(err -> push!(errors, err))

	try
		Init()
	catch err
		push!(errors, err)
	finally
		SetErrorCallback(throw)
	end

	if is_initialized()
		atexit(Terminate)
		for err in errors
			@warn err  # Warn about any non-fatal errors that may have occurred during initialization
		end
	else
		throw(errors)  # Throw fatal errors
	end
end

end


text:
# ---
# title: 180. Consecutive Numbers
# id: problem180
# author: Tian Jun
# date: 2020-10-31
# difficulty: Medium
# categories:
# link: <https://leetcode.com/problems/consecutive-numbers/description/>
# hidden: true
# ---
#
# Write a SQL query to find all numbers that appear at least three times
# consecutively.
#
#
#
#     +----+-----+
#     | Id | Num |
#     +----+-----+
#     | 1  |  1  |
#     | 2  |  1  |
#     | 3  |  1  |
#     | 4  |  2  |
#     | 5  |  1  |
#     | 6  |  2  |
#     | 7  |  2  |
#     +----+-----+
#
#
# For example, given the above `Logs` table, `1` is the only number that appears
# consecutively for at least three times.
#
#
#
#     +-----------------+
#     | ConsecutiveNums |
#     +-----------------+
#     | 1               |
#     +-----------------+
#
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


text:
###############################################
## Reading and writing Julia data .jld files ##
###############################################

module JLD
using HDF5
# Add methods to...
import HDF5: close, dump, exists, file, getindex, setindex!, g_create, g_open, o_delete, name, names, read, size, write,
             HDF5ReferenceObj, HDF5BitsKind, ismmappable, readmmap
import Base: length, endof, show, done, next, start, delete!

# .jld files written before v"0.4.0-dev+1419" might have Uint32 instead of UInt32 as the typename string.
# See julia issue #8907
if VERSION >= v"0.4.0-dev+1419"
    julia_type(s::String) = _julia_type(replace(s, r"Uint(?=\d{1,3})", "UInt"))
else
    julia_type(s::String) = _julia_type(s)
end

const magic_base = "Julia data file (HDF5), version "
const version_current = v"0.1"
const pathrefs = "/_refs"
const pathtypes = "/_types"
const pathrequire = "/_require"
const name_type_attr = "julia type"

typealias BitsKindOrByteString Union(HDF5BitsKind, ByteString)

### Dummy types used for converting attribute strings to Julia types
type UnsupportedType; end
type UnconvertedType; end
type CompositeKind; end   # here this means "a type with fields"

immutable JldDatatype
    dtype::HDF5Datatype
    index::Int
end

immutable JldWriteSession
    persist::Vector{Any} # To hold objects that should not be garbage-collected
    h5ref::ObjectIdDict  # To hold mapping from Object/Array -> HDF5ReferenceObject

    JldWriteSession() = new(Any[], ObjectIdDict())
end

# The Julia Data file type
# Purpose of the nrefs field:
# length(group) only returns the number of _completed_ items in a group. Since
# we'll write recursively, we need to keep track of the number of reference
# objects _started_.
type JldFile <: HDF5.DataFile
    plain::HDF5File
    version::VersionNumber
    toclose::Bool
    writeheader::Bool
    mmaparrays::Bool
    compress::Bool
    h5jltype::Dict{Int,Type}
    jlh5type::Dict{Type,JldDatatype}
    jlref::Dict{HDF5ReferenceObj,WeakRef}
    truncatemodules::Vector{ByteString}
    gref # Group references; can't annotate type here due to circularity
    nrefs::Int

    function JldFile(plain::HDF5File, version::VersionNumber=version_current, toclose::Bool=true,
                     writeheader::Bool=false, mmaparrays::Bool=false,
                     compress::Bool=false)
        f = new(plain, version, toclose, writeheader, mmaparrays, compress,
                Dict{HDF5Datatype,Type}(), Dict{Type,HDF5Datatype}(),
                Dict{HDF5ReferenceObj,WeakRef}(), ByteString[])
        if toclose
            finalizer(f, close)
        end
        f
    end
end

immutable JldGroup
    plain::HDF5Group
    file::JldFile
end

immutable JldDataset
    plain::HDF5Dataset
    file::JldFile
end

iscompressed(f::JldFile) = f.compress
iscompressed(g::JldGroup) = g.file.compress
iscompressed(d::JldGroup) = d.file.compress

immutable PointerException <: Exception; end
show(io::IO, ::PointerException) = print(io, "cannot write a pointer to JLD file")

immutable TypeMismatchException <: Exception
    typename::ByteString
end
show(io::IO, e::TypeMismatchException) =
    print(io, "stored type $(e.typename) does not match currently loaded type")

# Wrapper for associative keys
# We write this instead of the associative to avoid dependence on the
# Julia hash function
immutable AssociativeWrapper{K,V,T<:Associative}
    keys::Vector{K}
    values::Vector{V}
end

include("jld_types.jl")

file(x::JldFile) = x
file(x::Union(JldGroup, JldDataset)) = x.file

function close(f::JldFile)
    if f.toclose
        close(f.plain)
        if f.writeheader
            magic = zeros(Uint8, 512)
            tmp = string(magic_base, f.version)
            magic[1:length(tmp)] = tmp.data
            rawfid = open(f.plain.filename, "r+")
            write(rawfid, magic)
            close(rawfid)
        end
        f.toclose = false
    end
    nothing
end
close(g::Union(JldGroup, JldDataset)) = close(g.plain)
show(io::IO, fid::JldFile) = isvalid(fid.plain) ? print(io, "Julia data file version ", fid.version, ": ", fid.plain.filename) : print(io, "Julia data file (closed): ", fid.plain.filename)

function jldopen(filename::String, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool; mmaparrays::Bool=false, compress::Bool=false)
    local fj
    if ff && !wr
        error("Cannot append to a write-only file")
    end
    if !cr && !isfile(filename)
        error("File ", filename, " cannot be found")
    end
    version = version_current
    pa = p_create(HDF5.H5P_FILE_ACCESS)
    # HDF5.h5p_set_libver_bounds(pa, HDF5.H5F_LIBVER_18, HDF5.H5F_LIBVER_18)
    try
        pa["fclose_degree"] = HDF5.H5F_CLOSE_STRONG
        if cr && (tr || !isfile(filename))
            # We're truncating, so we don't have to check the format of an existing file
            # Set the user block to 512 bytes, to save room for the header
            p = p_create(HDF5.H5P_FILE_CREATE)
            local f
            try
                p["userblock"] = 512
                f = HDF5.h5f_create(filename, HDF5.H5F_ACC_TRUNC, p.id, pa.id)
            finally
                close(p)
            end
            fj = JldFile(HDF5File(f, filename), version, true, true, mmaparrays, compress)
            # initialize empty require list
            write(fj, pathrequire, ByteString[])
        else
            # Test whether this is a jld file
            sz = filesize(filename)
            if sz < 512
                error("File size indicates $filename cannot be a Julia data file")
            end
            magic = Array(Uint8, 512)
            rawfid = open(filename, "r")
            try
                magic = read!(rawfid, magic)
            finally
                close(rawfid)
            end
            if beginswith(magic, magic_base.data)
                version = convert(VersionNumber, bytestring(convert(Ptr{Uint8}, magic) + length(magic_base)))
                if version < v"0.1.0"
                    if !isdefined(JLD, :JLD00)
                        eval(:(include(joinpath($(dirname(@__FILE__)), "JLD00.jl"))))
                    end
                    fj = JLD00.jldopen(filename, rd, wr, cr, tr, ff; mmaparrays=mmaparrays)
                else
                    f = HDF5.h5f_open(filename, wr ? HDF5.H5F_ACC_RDWR : HDF5.H5F_ACC_RDONLY, pa.id)
                    fj = JldFile(HDF5File(f, filename), version, true, true, mmaparrays, compress)
                    # Load any required files/packages
                    if exists(fj, pathrequire)
                        r = read(fj, pathrequire)
                        for fn in r
                            require(fn)
                        end
                    end
                end
            else
                if ishdf5(filename)
                    println("$filename is an HDF5 file, but it is not a recognized Julia data file. Opening anyway.")
                    fj = JldFile(h5open(filename, rd, wr, cr, tr, ff), version_current, true, false, mmaparrays, compress)
                else
                    error("$filename does not seem to be a Julia data or HDF5 file")
                end
            end
        end
    finally
        close(pa)
    end
    return fj
end

function jldopen(fname::String, mode::String="r"; mmaparrays::Bool=false, compress::Bool=false)
    mode == "r"  ? jldopen(fname, true , false, false, false, false, mmaparrays=mmaparrays, compress=compress) :
    mode == "r+" ? jldopen(fname, true , true , false, false, false, mmaparrays=mmaparrays, compress=compress) :
    mode == "w"  ? jldopen(fname, false, true , true , true , false, mmaparrays=mmaparrays, compress=compress) :
#     mode == "w+" ? jldopen(fname, true , true , true , true , false) :
#     mode == "a"  ? jldopen(fname, false, true , true , false, true ) :
#     mode == "a+" ? jldopen(fname, true , true , true , false, true ) :
    error("invalid open mode: ", mode)
end

function jldopen(f::Function, args...; kws...)
    jld = jldopen(args...; kws...)
    try
        f(jld)
    finally
        close(jld)
    end
end

function jldobject(obj_id::HDF5.Hid, parent)
    obj_type = HDF5.h5i_get_type(obj_id)
    obj_type == HDF5.H5I_GROUP ? JldGroup(HDF5Group(obj_id, file(parent.plain)), file(parent)) :
    obj_type == HDF5.H5I_DATATYPE ? HDF5Datatype(obj_id) :
    obj_type == HDF5.H5I_DATASET ? JldDataset(HDF5Dataset(obj_id, file(parent.plain)), file(parent)) :
    error("Invalid object type for path ", path)
end

getindex(parent::Union(JldFile, JldGroup), path::ByteString) =
    jldobject(HDF5.h5o_open(parent.plain.id, path), parent)

function getindex(parent::Union(JldFile, JldGroup, JldDataset), r::HDF5ReferenceObj)
    if r == HDF5.HDF5ReferenceObj_NULL; error("Reference is null"); end
    obj_id = HDF5.h5r_dereference(parent.plain.id, HDF5.H5R_OBJECT, r)
    jldobject(obj_id, parent)
end

### "Inherited" behaviors
g_create(parent::Union(JldFile, JldGroup), args...) = JldGroup(g_create(parent.plain, args...), file(parent))
function g_create(f::Function, parent::Union(JldFile, JldGroup), args...)
    g = JldGroup(g_create(parent.plain, args...), file(parent))
    try
        f(g)
    finally
        close(g)
    end
end
g_open(parent::Union(JldFile, JldGroup), args...) = JldGroup(g_open(parent.plain, args...), file(parent))
name(p::Union(JldFile, JldGroup, JldDataset)) = name(p.plain)
exists(p::Union(JldFile, JldGroup, JldDataset), path::ByteString) = exists(p.plain, path)
root(p::Union(JldFile, JldGroup, JldDataset)) = g_open(file(p), "/")
o_delete(parent::Union(JldFile, JldGroup), args...) = o_delete(parent.plain, args...)
function ensurepathsafe(path::ByteString)
    if any([beginswith(path, s) for s in (pathrefs,pathtypes,pathrequire)])
        error("$name is internal to the JLD format, use o_delete if you really want to delete it")
    end
end
function delete!(o::JldDataset)
    fullpath = name(o)
    ensurepathsafe(fullpath)
    o_delete(o.file, fullpath)
    refspath = joinpath(pathrefs, fullpath[2:end])
    exists(o.file, refspath) && o_delete(o.file, refspath)
end
function delete!(g::JldGroup)
    fullpath = name(g)
    ensurepathsafe(fullpath)
    for o in g typeof(o) == JldDataset && delete!(o) end
    o_delete(g.file,name(g))
end
function delete!(parent::Union(JldFile, JldGroup), path::ByteString)
    exists(parent, path) || error("$path does not exist in $parent")
    delete!(parent[path])
end
delete!(parent::Union(JldFile, JldGroup), args::(ByteString...)) = for a in args delete!(parent,a) end
ismmappable(obj::JldDataset) = ismmappable(obj.plain)
readmmap(obj::JldDataset, args...) = readmmap(obj.plain, args...)
setindex!(parent::Union(JldFile, JldGroup), val, path::ASCIIString) = write(parent, path, val)

start(parent::Union(JldFile, JldGroup)) = (names(parent), 1)
done(parent::Union(JldFile, JldGroup), state) = state[2] > length(state[1])
next(parent::Union(JldFile, JldGroup), state) = parent[state[1][state[2]]], (state[1], state[2]+1)

### Julia data file format implementation ###

### Read ###

function read(parent::Union(JldFile, JldGroup), name::ByteString)
    local val
    obj = parent[name]
    try
        val = read(obj)
    finally
        close(obj)
    end
    val
end
read(parent::Union(JldFile,JldGroup), name::Symbol) = read(parent,bytestring(string(name)))

function read(obj::JldDataset)
    dtype = datatype(obj.plain)
    dspace_id = HDF5.h5d_get_space(obj.plain)
    extent_type = HDF5.h5s_get_simple_extent_type(dspace_id)
    try
        if extent_type == HDF5.H5S_SCALAR
            # Scalar value
            return read_scalar(obj, dtype, jldatatype(file(obj), dtype))
        elseif extent_type == HDF5.H5S_SIMPLE
            return read_array(obj, dtype, dspace_id, HDF5.H5S_ALL)
        elseif extent_type == HDF5.H5S_NULL
            # Empty array
            if HDF5.h5t_get_class(dtype) == HDF5.H5T_REFERENCE
                T = refarray_eltype(obj)
            else
                T = jldatatype(file(obj), dtype)
            end
            if exists(obj, "dims")
                dims = a_read(obj.plain, "dims")
                return Array(T, dims...)
            else
                return T[]
            end
        end
    finally
        HDF5.h5s_close(dspace_id)
    end
end

## Scalars
read_scalar{T<:BitsKindOrByteString}(obj::JldDataset, dtype::HDF5Datatype, ::Type{T}) =
    read(obj.plain, T)
function read_scalar(obj::JldDataset, dtype::HDF5Datatype, T::Type)
    buf = Array(Uint8, sizeof(dtype))
    HDF5.readarray(obj.plain, dtype.id, buf)
    return after_read(jlconvert(T, file(obj), pointer(buf)))
end

after_read(x) = x

# Special case for associative, to rehash keys
function after_read{K,V,T}(x::AssociativeWrapper{K,V,T})
    ret = T()
    keys = x.keys
    values = x.values
    n = length(keys)
    if applicable(sizehint, ret, n)
        sizehint(ret, n)
    end
    for i = 1:n
        ret[keys[i]] = values[i]
    end
    ret
end

## Arrays

# Read an array
function read_array(obj::JldDataset, dtype::HDF5Datatype, dspace_id::HDF5.Hid, dsel_id::HDF5.Hid,
                    dims::(Int...)=convert((Int...), HDF5.h5s_get_simple_extent_dims(dspace_id)[1]))
    if HDF5.h5t_get_class(dtype) == HDF5.H5T_REFERENCE
        return read_refs(obj, refarray_eltype(obj), dspace_id, dsel_id, dims)
    else
        return read_vals(obj, dtype, jldatatype(file(obj), dtype), dspace_id, dsel_id, dims)
    end
end

# Arrays of basic HDF5 kinds
function read_vals{S<:HDF5BitsKind}(obj::JldDataset, dtype::HDF5Datatype, T::Union(Type{S}, Type{Complex{S}}),
                                    dspace_id::HDF5.Hid, dsel_id::HDF5.Hid, dims::(Int...))
    if obj.file.mmaparrays && HDF5.iscontiguous(obj.plain) && dsel_id == HDF5.H5S_ALL
        readmmap(obj.plain, Array{T})
    else
        out = Array(T, dims)
        HDF5.h5d_read(obj.plain.id, dtype.id, dspace_id, dsel_id, HDF5.H5P_DEFAULT, out)
        out
    end
end

# Arrays of immutables/bitstypes
function read_vals(obj::JldDataset, dtype::HDF5Datatype, T::Type, dspace_id::HDF5.Hid,
                   dsel_id::HDF5.Hid, dims::(Int...))
    n = prod(dims)
    h5sz = sizeof(dtype)
    out = Array(T, dims)

    # Read from file
    buf = Array(Uint8, h5sz*n)
    HDF5.h5d_read(obj.plain.id, dtype.id, dspace_id, dsel_id, HDF5.H5P_DEFAULT, buf)

    f = file(obj)
    h5offset = pointer(buf)
    if T.pointerfree && !T.mutable
        jloffset = pointer(out)
        jlsz = T.size

        # Perform conversion in buffer
        for i = 1:n
            jlconvert!(jloffset, T, f, h5offset)
            jloffset += jlsz
            h5offset += h5sz
        end
    else
        # Convert each item individually
        for i = 1:n
            out[i] = jlconvert(T, f, h5offset)
            h5offset += h5sz
        end
    end
    out
end

# Arrays of references
function read_refs{T}(obj::JldDataset, ::Type{T}, dspace_id::HDF5.Hid, dsel_id::HDF5.Hid,
                      dims::(Int...))
    refs = Array(HDF5ReferenceObj, dims)
    HDF5.h5d_read(obj.plain.id, HDF5.H5T_STD_REF_OBJ, dspace_id, dsel_id, HDF5.H5P_DEFAULT, refs)

    out = Array(T, dims)
    f = file(obj)
    for i = 1:length(refs)
        if refs[i] != HDF5.HDF5ReferenceObj_NULL
            out[i] = read_ref(f, refs[i])
        end
    end
    out
end

# Get element type of a reference array
function refarray_eltype(obj::JldDataset)
    typename = a_read(obj.plain, "julia eltype")
    T = julia_type(typename)
    if T == UnsupportedType
        warn("type $typename not present in workspace; interpreting array as Array{Any}")
        return Any
    end
    return T
end

## Reference
function read_ref(f::JldFile, ref::HDF5ReferenceObj)
    if haskey(f.jlref, ref)
        # Stored as WeakRefs and may no longer exist
        val = f.jlref[ref].value
        val != nothing && return val
    end

    dset = f[ref]
    data = try
        read(dset)
    finally
        close(dset)
    end

    f.jlref[ref] = WeakRef(data)
    data
end

### Writing ###

write(parent::Union(JldFile, JldGroup), name::ByteString,
      data, wsession::JldWriteSession=JldWriteSession()) =
    close(_write(parent, name, data, wsession))

# Pick whether to use compact or default storage based on data size
const COMPACT_PROPERTIES = p_create(HDF5.H5P_DATASET_CREATE)
HDF5.h5p_set_layout(COMPACT_PROPERTIES.id, HDF5.H5D_COMPACT)
function dset_create_properties(parent, sz::Int, obj)
    sz <= 8192 && return COMPACT_PROPERTIES
    if iscompressed(parent)
        chunk = HDF5.heuristic_chunk(obj)
        if !isempty(chunk)
            p = p_create(HDF5.H5P_DATASET_CREATE)
            p["chunk"] = chunk
            p["blosc"] = 5
            return p
        end
    end
    return HDF5.DEFAULT_PROPERTIES
end

# Write "basic" types
function _write{T<:Union(HDF5BitsKind, ByteString)}(parent::Union(JldFile, JldGroup),
                                                    name::ByteString,
                                                    data::Union(T, Array{T}),
                                                    wsession::JldWriteSession)
    dset, dtype = d_create(parent.plain, bytestring(name), data, HDF5._link_properties(name), dset_create_properties(parent, sizeof(data), data))
    try
        # Write the attribute
        isa(data, Array) && isempty(data) && a_write(dset, "dims", [size(data)...])
        # Write the data
        HDF5.writearray(dset, dtype.id, data)
    finally
        close(dtype)
    end
    dset
end

# General array types
function _write{T}(parent::Union(JldFile, JldGroup),
                   path::ByteString, data::Array{T},
                   wsession::JldWriteSession)
    f = file(parent)
    dtype = h5fieldtype(f, T, true)
    buf = h5convert_array(f, data, dtype, wsession)
    dims = convert(Array{HDF5.Hsize, 1}, [reverse(size(data))...])
    dspace = dataspace(data)
    try
        dset = d_create(parent.plain, path, dtype.dtype, dspace, HDF5._link_properties(path), dset_create_properties(parent, sizeof(buf), buf))
        if dtype == JLD_REF_TYPE
            a_write(dset, "julia eltype", full_typename(f, T))
        end
        if isempty(data) && ndims(data) != 1
            a_write(dset, "dims", [size(data)...])
        else
            HDF5.writearray(dset, dtype.dtype.id, buf)
        end
        return dset
    finally
        close(dspace)
    end
end

# Dispatch correct method for Array{Union()}
_write(parent::Union(JldFile, JldGroup), path::ByteString, data::Array{Union()},
       wsession::JldWriteSession) =
    invoke(_write, (Union(JldFile, JldGroup), ByteString, Array, JldWriteSession), parent,
           path, data, wsession)

# Convert an array to the format to be written to the HDF5 file, either
# references or values
function h5convert_array(f::JldFile, data::Array,
                         dtype::JldDatatype, wsession::JldWriteSession)
    if dtype == JLD_REF_TYPE
        refs = Array(HDF5ReferenceObj, length(data))
        for i = 1:length(data)
            if isdefined(data, i)
                refs[i] = write_ref(f, data[i], wsession)
            else
                refs[i] = HDF5.HDF5ReferenceObj_NULL
            end
        end
        reinterpret(Uint8, refs) # for type stability
    else
        gen_h5convert(f, eltype(data))
        h5convert_vals(f, data, dtype, wsession)
    end
end

# Hack to ensure that _h5convert_vals isn't compiled before h5convert!
function h5convert_vals(f::JldFile, data::ANY, dtype::JldDatatype,
                        wsession::JldWriteSession)
    for i = true; end # prevents inlining
    _h5convert_vals(f, data, dtype, wsession)
end

# Convert an array of immutables or bitstypes to a buffer representing
# HDF5 compound objects. A separate function so that it is specialized.
function _h5convert_vals(f::JldFile, data::Array,
                         dtype::JldDatatype, wsession::JldWriteSession)
    sz = HDF5.h5t_get_size(dtype)
    n = length(data)
    buf = Array(Uint8, sz*n)
    offset = pointer(buf)
    for i = 1:n
        h5convert!(offset, f, data[i], wsession)
        offset += sz
    end
    buf
end

# Get reference group, creating a new one if necessary
function get_gref(f::JldFile)
    isdefined(f, :gref) && return f.gref::JldGroup

    if !exists(f, pathrefs)
        gref = f.gref = g_create(f, pathrefs)
    else
        gref = f.gref = f[pathrefs]
    end
    f.nrefs = length(gref)
    gref
end

# Write a reference
function write_ref(parent::JldFile, data, wsession::JldWriteSession)
    # Check whether we have already written this object
    ref = get(wsession.h5ref, data, HDF5.HDF5ReferenceObj_NULL)
    ref != HDF5.HDF5ReferenceObj_NULL && return ref

    # Write an new reference
    gref = get_gref(parent)
    name = @sprintf "%08d" (parent.nrefs += 1)
    dset = _write(gref, name, data, wsession)

    # Add reference to reference list
    ref = HDF5ReferenceObj(HDF5.objinfo(dset).addr)
    close(dset)
    if !isa(data, Tuple) && typeof(data).mutable
        wsession.h5ref[data] = ref
    end
    ref
end
write_ref(parent::JldGroup, data, wsession::JldWriteSession) =
    write_ref(file(parent), data, wsession)

# Special case for associative, to rehash keys
function _write(parent::Union(JldFile, JldGroup), name::ByteString,
                d::Associative, wsession::JldWriteSession)
    n = length(d)
    K, V = eltype(d)
    ks = Array(K, n)
    vs = Array(V, n)
    i = 0
    for (k,v) in d
        ks[i+=1] = k
        vs[i] = v
    end
    write_compound(parent, name, AssociativeWrapper{K,V,typeof(d)}(ks, vs), wsession)
end

# Expressions, drop line numbers
function _write(parent::Union(JldFile, JldGroup),
                name::ByteString, ex::Expr,
                wsession::JldWriteSession)
    args = ex.args
    # Discard "line" expressions
    keep = trues(length(args))
    for i = 1:length(args)
        if isa(args[i], Expr) && args[i].head == :line
            keep[i] = false
        end
    end
    newex = Expr(ex.head)
    newex.args = args[keep]
    write_compound(parent, name, newex, wsession)
end

# Generic (tuples, immutables, and compound types)
_write(parent::Union(JldFile, JldGroup), name::ByteString, s,
      wsession::JldWriteSession) =
    write_compound(parent, name, s, wsession)
function write_compound(parent::Union(JldFile, JldGroup), name::ByteString,
                        s, wsession::JldWriteSession)
    T = typeof(s)
    f = file(parent)
    dtype = h5type(f, T, true)
    gen_h5convert(f, T)

    buf = Array(Uint8, HDF5.h5t_get_size(dtype))
    h5convert!(pointer(buf), file(parent), s, wsession)

    dspace = HDF5Dataspace(HDF5.h5s_create(HDF5.H5S_SCALAR))
    try
        dset = HDF5.d_create(parent.plain, name, dtype.dtype, dspace, HDF5._link_properties(name), dset_create_properties(parent, length(buf), buf))
        HDF5.writearray(dset, dtype.dtype.id, buf)
        return dset
    finally
        close(dspace)
    end
end

### Size, length, etc ###
size(dset::JldDataset) = size(dset.plain)
length(dset::JldDataset) = prod(size(dset))
endof(dset::JldDataset) = length(dset)

### Read/write via getindex/setindex! ###
function getindex(dset::JldDataset, indices::Union(Integer, RangeIndex)...)
    sz = map(length, indices)
    dsel_id = HDF5.hyperslab(dset.plain, indices...)
    try
        dspace = HDF5._dataspace(sz)
        try
            return read_array(dset, datatype(dset.plain), dspace.id, dsel_id, sz)
        finally
            close(dspace)
        end
    finally
        HDF5.h5s_close(dsel_id)
    end
end

function setindex!{T,N}(dset::JldDataset, X::AbstractArray{T,N}, indices::RangeIndex...)
    f = file(dset)
    sz = map(length, indices)
    dsel_id = HDF5.hyperslab(dset.plain, indices...)
    try
        dtype = datatype(dset.plain)
        try
            # Convert array to writeable buffer
            if HDF5.h5t_get_class(dtype) == HDF5.H5T_REFERENCE
                written_eltype = refarray_eltype(dset)
                jldtype = JLD_REF_TYPE
            else
                written_eltype = jldatatype(f, dtype)
                jldtype = JldDatatype(dtype, -1)
            end

            buf = h5convert_array(f, convert(Array{written_eltype,N}, X), jldtype,
                                  JldWriteSession())

            dspace = HDF5._dataspace(sz)
            try
                HDF5.h5d_write(dset.plain.id, dtype, dspace, dsel_id, HDF5.H5P_DEFAULT, buf)
            finally
                close(dspace)
            end
        finally
            close(dtype)
        end
    finally
        HDF5.h5s_close(dsel_id)
    end
end

length(x::Union(JldFile, JldGroup)) = length(names(x))

### Converting attribute strings to Julia types

is_valid_type_ex(s::Symbol) = true
is_valid_type_ex(s::QuoteNode) = true
is_valid_type_ex(x::Int) = true
is_valid_type_ex(e::Expr) = ((e.head == :curly || e.head == :tuple || e.head == :.) && all(map(is_valid_type_ex, e.args))) ||
                            (e.head == :call && (e.args[1] == :Union || e.args[1] == :TypeVar))

# Work around https://github.com/JuliaLang/julia/issues/8226
const _typedict = Dict{String,Type}()
_typedict["Core.Type{TypeVar(:T,Union(Core.Any,Core.Undef))}"] = Type

function _julia_type(s::String)
    typ = get(_typedict, s, UnconvertedType)
    if typ == UnconvertedType
        typ = julia_type(parse(s))
        if typ != UnsupportedType
            _typedict[s] = typ
        end
    end
    typ
end

function julia_type(e::Union(Symbol, Expr))
    if is_valid_type_ex(e)
        try     # try needed to catch undefined symbols
            typ = eval(Main, e)
            isa(typ, Type) && return typ
        end
    end
    return UnsupportedType
end

### Converting Julia types to fully qualified names
function full_typename(io::IO, file::JldFile, jltype::UnionType)
    print(io, "Union(")
    if !isempty(jltype.types)
        full_typename(io, file, jltype.types[1])
        for i = 2:length(jltype.types)
            print(io, ',')
            full_typename(io, file, jltype.types[i])
        end
    end
    print(io, ')')
end
function full_typename(io::IO, file::JldFile, tv::TypeVar)
    if is(tv.lb, None) && is(tv.ub, Any)
        print(io, "TypeVar(:", tv.name, ")")
    elseif is(tv.lb, None)
        print(io, "TypeVar(:", tv.name, ",")
        full_typename(io, file, tv.ub)
        print(io, ')')
    else
        print(io, "TypeVar(:")
        print(io, tv.name)
        print(io, ',')
        full_typename(io, file, tv.lb)
        print(io, ',')
        full_typename(io, file, tv.ub)
        print(io, ')')
    end
end
function full_typename(io::IO, file::JldFile, jltype::(Type...))
    print(io, '(')
    for t in jltype
        full_typename(io, file, t)
        print(io, ',')
    end
    print(io, ')')
end
full_typename(io::IO, ::JldFile, x) = print(io, x)
function full_typename(io::IO, file::JldFile, jltype::DataType)
    mod = jltype.name.module
    if mod != Main
        mname = string(mod)
        for x in file.truncatemodules
            if beginswith(mname, x)
                mname = length(x) == length(mname) ? "" : mname[sizeof(x)+1:end]
                break
            end
        end

        if !isempty(mname)
            print(io, mname)
            print(io, '.')
        end
    end

    print(io, jltype.name.name)
    if !isempty(jltype.parameters)
        print(io, '{')
        full_typename(io, file, jltype.parameters[1])
        for i = 2:length(jltype.parameters)
            print(io, ',')
            full_typename(io, file, jltype.parameters[i])
        end
        print(io, '}')
    end
end
function full_typename(file::JldFile, x)
    io = IOBuffer(Array(Uint8, 64), true, true)
    truncate(io, 0)
    full_typename(io, file, x)
    takebuf_string(io)
end

function truncate_module_path(file::JldFile, mod::Module)
    push!(file.truncatemodules, string(mod))
end

function names(parent::Union(JldFile, JldGroup))
    n = names(parent.plain)
    keep = trues(length(n))
    const reserved = [pathrefs[2:end], pathtypes[2:end], pathrequire[2:end]]
    for i = 1:length(n)
        if in(n[i], reserved)
            keep[i] = false
        end
    end
    n[keep]
end

function save_write(f, s, vname, wsession::JldWriteSession)
    if !isa(vname, Function)
        try
            write(f, s, vname)
        catch e
            if isa(e, PointerException)
                warn("Skipping $vname because it contains a pointer")
            end
        end
    end
end

macro save(filename, vars...)
    if isempty(vars)
        # Save all variables in the current module
        writeexprs = Array(Expr, 0)
        m = current_module()
        for vname in names(m)
            s = string(vname)
            if !ismatch(r"^_+[0-9]*$", s) # skip IJulia history vars
                v = eval(m, vname)
                if !isa(v, Module)
                    push!(writeexprs, :(save_write(f, $s, $(esc(vname)), wsession)))
                end
            end
        end
    else
        writeexprs = Array(Expr, length(vars))
        for i = 1:length(vars)
            writeexprs[i] = :(write(f, $(string(vars[i])), $(esc(vars[i])), wsession))
        end
    end

    quote
        local f = jldopen($(esc(filename)), "w")
        wsession = JldWriteSession()
        try
            $(Expr(:block, writeexprs...))
        finally
            close(f)
        end
    end
end

macro load(filename, vars...)
    if isempty(vars)
        if isa(filename, Expr)
            filename = eval(current_module(), filename)
        end
        # Load all variables in the top level of the file
        readexprs = Array(Expr, 0)
        vars = Array(Expr, 0)
        f = jldopen(filename)
        nms = names(f)
        for n in nms
            obj = f[n]
            if isa(obj, JldDataset)
                sym = esc(symbol(n))
                push!(readexprs, :($sym = read($f, $n)))
                push!(vars, sym)
            end
        end
        return Expr(:block,
                    Expr(:global, vars...),
                    Expr(:try,  Expr(:block, readexprs...), false, false,
                         :(close($f))),
                    Symbol[v.args[1] for v in vars]) # "unescape" vars
    else
        readexprs = Array(Expr, length(vars))
        for i = 1:length(vars)
            readexprs[i] = :($(esc(vars[i])) = read(f, $(string(vars[i]))))
        end
        return Expr(:block,
                    Expr(:global, map(esc, vars)...),
                    :(local f = jldopen($(esc(filename)))),
                    Expr(:try,  Expr(:block, readexprs...), false, false,
                         :(close(f))),
                    Symbol[v for v in vars]) # vars is a tuple
    end
end

# Save all the key-value pairs in the dict as top-level variables of the JLD
function save(filename::String, dict::Associative; compress::Bool=false)
    jldopen(filename, "w"; compress=compress) do file
        wsession = JldWriteSession()
        for (k,v) in dict
            write(file, bytestring(k), v, wsession)
        end
    end
end
# Or the names and values may be specified as alternating pairs
function save(filename::String, name::String, value, pairs...; compress::Bool=false)
    if isodd(length(pairs)) || !isa(pairs[1:2:end], (String...))
        throw(ArgumentError("arguments must be in name-value pairs"))
    end
    jldopen(filename, "w"; compress=compress) do file
        wsession = JldWriteSession()
        write(file, bytestring(name), value, wsession)
        for i=1:2:length(pairs)
            write(file, bytestring(pairs[i]), pairs[i+1], wsession)
        end
    end
end

# load with just a filename returns a dictionary containing all the variables
function load(filename::String)
    jldopen(filename, "r") do file
        (ByteString => Any)[var => read(file, var) for var in names(file)]
    end
end
# When called with explicitly requested variable names, return each one
function load(filename::String, varname::String)
    jldopen(filename, "r") do file
        read(file, varname)
    end
end
load(filename::String, varnames::String...) = load(filename, varnames)
function load(filename::String, varnames::(String...))
    jldopen(filename, "r") do file
        map((var)->read(file, var), varnames)
    end
end

function addrequire(file::JldFile, filename::String)
    files = read(file, pathrequire)
    push!(files, filename)
    o_delete(file, pathrequire)
    write(file, pathrequire, files)
end

export
    addrequire,
    ismmappable,
    jldopen,
    o_delete,
    plain,
    readmmap,
    readsafely,
    @load,
    @save,
    load,
    save,
    truncate_module_path

end


text:
# This file is part of the ValidatedNumerics.jl package; MIT licensed

doc"""An `IntervalBox` is an $N$-dimensional rectangular box, given
by a Cartesian product of $N$ `Interval`s.
"""

immutable IntervalBox{N, T} <: FixedVector{N, Interval{T}}  # uses FixedSizeArrays.jl package
    _ :: NTuple{N, Interval{T}}
end

IntervalBox(x::Interval) = IntervalBox( (x,) )  # single interval treated as tuple with one element

## arithmetic operations
# Note that standard arithmetic operations are implemented automatically by FixedSizeArrays.jl

mid(X::IntervalBox) = [mid(x) for x in X]

## set operations

⊆(X::IntervalBox, Y::IntervalBox) = all([x ⊆ y for (x,y) in zip(X, Y)])

∩(X::IntervalBox, Y::IntervalBox) = IntervalBox([x ∩ y for (x,y) in zip(X, Y)]...)

isempty(X::IntervalBox) = any(isempty, X)

diam(X::IntervalBox) = maximum([diam(x) for x in X])

emptyinterval(X::IntervalBox) = IntervalBox(map(emptyinterval, X))


text:
@testset "Multivalued" begin
    # 1d
    A0 = rand(20)
    A = reinterpret(MyPair{Float64}, A0)
    a1, a2 = A0[1:2:end], A0[2:2:end]
    @test length(A) == 10
    itp = interpolate(A, BSpline(Constant()))
    @test itp(3.2) ≈ MyPair(A0[5],A0[6])
    itp = interpolate(A, BSpline(Linear()))
    @test itp(3.2) ≈ 0.8*MyPair(A0[5],A0[6]) + 0.2*MyPair(A0[7],A0[8])
    it = BSpline(Quadratic(Flat(OnGrid())))
    itp = interpolate(A, it)
    @test itp(3.2) ≈ MyPair(interpolate(a1, it)(3.2), interpolate(a2, it)(3.2))

    # 2d
    A0 = rand(100)
    A = reshape(reinterpret(MyPair{Float64}, A0), (10,5))
    a1, a2 = reshape(A0[1:2:end], (10,5)), reshape(A0[2:2:end], (10,5))
    for it in (BSpline(Constant()),
               BSpline(Linear()),
               BSpline(Quadratic(Flat(OnGrid()))))
        itp = interpolate(A, it)
        @test itp(3.2,1.8) ≈ MyPair(interpolate(a1, it)(3.2,1.8), interpolate(a2, it)(3.2,1.8))
    end
end


text: Dict{String, Any}("people" => Any[Dict{String, Any}("first_name" => Dict{String, Any}("value" => "Bruce", "type" => "string"), "last_name" => Dict{String, Any}("value" => "Springsteen", "type" => "string"))])


text:
# Test of Min-Cut and maximum adjacency visit

using Graphs
using Base.Test

g = simple_inclist(8, is_directed=false)

# Original example by Stoer

wedges = [
    (1, 2, 2.),
    (1, 5, 3.),
    (2, 3, 3.),
    (2, 5, 2.),
    (2, 6, 2.),
    (3, 4, 4.),
    (3, 7, 2.),
    (4, 7, 2.),
    (4, 8, 2.),
    (5, 6, 3.),
    (6, 7, 1.),
    (7, 8, 3.) ]

m = length(wedges)
eweights = zeros(m)

for i = 1 : m
    we = wedges[i]
    add_edge!(g, we[1], we[2])
    eweights[i] = we[3]
end

@assert num_vertices(g) == 8
@assert num_edges(g) == m

parity, bestcut = min_cut(g, eweights)

@test length(parity) == 8
@test parity == Bool[ true, true, false, false, true, true, false, false ]
@test bestcut == 4.0

parity, bestcut = min_cut(g)

@test length(parity) == 8
@test parity == Bool[ true, false, false, false, false, false, false, false ]
@test bestcut == 2.0

vertices = maximum_adjacency_visit(g)

@test vertices == Int64[1, 2, 5, 6, 3, 7, 4, 8]


text:
function replicate(;
  refreshcomp::Bool = true,
  refreshtretcrsp::Bool = true,
  refreshtvolmcapcrsp::Bool = true,
  refreshdataseries::Bool = true,
  refreshportfolios::Bool = true,
  parallel::NBool = nothing,
  trialrunonly::Bool = false,
  table1::Bool=true,
  table2::Bool=true,
  table3::Bool=true,
  table4::Bool=true,
  table5::Bool=true,
  table6::Bool=true,
  yearrange::Union{UnitRange{Int},Nothing} = nothing,
  outsuffix::Union{Nothing,String} = nothing,

  disttype::Symbol=:full,
  table7::Bool=true,
  table8::Bool=true,
  refreshseocrsp::Bool=true,
  refreshdistcrsp::Bool=true,
  refreshevent::Bool=true,
  refreshcumex::Bool= true,
  figure1::Bool = true,
  figure2::Bool = true,
  figure3::Bool = true,
  figure4::Bool = true,
  table9::Bool=true,
  table10::Bool= true,

  tablea1::Bool = true,
  tablea2::Bool = true,
  writepaneltocsv::Bool = false)::Nothing

  local panel::DataFrame
  local bb::DataFrame
  local dist::DataFrame
  local cumex::DataFrame

  (!isnothing(parallel)) && (PARALLEL[] = parallel)
  REPLICATION_TYPE[] = trialrunonly ? :monthly : :daily
  (!isnothing(yearrange)) && (YEAR_RANGE[] = yearrange)

  (disttype ∈ [:primaryfull, :primaryshort, :oldfull, :oldshort]) || error("disttype must be ∈ [:primaryfull, :primaryshort, :oldfull, :oldshort]")
  DIST_TYPE[] = disttype

  (!isnothing(outsuffix)) && (OUT_SUFFIX[] = outsuffix)

  #not strictly necessary, but helps stop doubling up on the multi-threading
  @info "Beginning replication.
    type: $(REPLICATION_TYPE[]); parallel: $(PARALLEL[]); years: $(YEAR_RANGE[]); disttype: $(disttype)"

  #this encompasses all pre-processing activities
  if sum([refreshcomp, refreshtretcrsp, refreshtvolmcapcrsp,
    refreshdataseries, refreshportfolios]) > 0
    @info "Beginning portfolio construction"
    panel = constructportfolios(
      refreshcomp = refreshcomp,
      refreshtvolmcapcrsp = refreshtvolmcapcrsp,
      refreshtretcrsp = refreshtretcrsp,
      refreshdataseries=refreshdataseries,
      refreshportfolios=refreshportfolios,
      trialrunonly = trialrunonly)
  elseif table1 + table2 + table3 + table4 + table5 + table6 + tablea1 > 0
    panel = constructportfolios(refreshportfolios=false, trialrunonly=trialrunonly)
  end

  writepaneltocsv && panel |> CSV.write("$DATA_PATH\\PANEL-$(REPLICATION_TYPE[]).csv")
  #######################panel study
  table1 && maketable1(panel, trialrunonly=trialrunonly)
  table2 && maketable2(panel, trialrunonly=trialrunonly)
  table3 && maketable3(panel, trialrunonly=trialrunonly)
  table4 && maketable4(panel, trialrunonly=trialrunonly)
  table5 && maketable5(panel, trialrunonly=trialrunonly)
  table6 && maketable6(panel, trialrunonly=trialrunonly)

  ######the following is just some code for comparing to Capacity
  panel.lbm = ((x)->isfinite(x) ? x : missing).(log.(panel.bkequity .* 1_000_000) ./ (panel.Llmkequity))
  panel.lLbm = log.(panel.Lbm)
  println(DataFrames.describe(panel, :all))
  ##################################event study
  println("Beginning Event Analysis")
  (bb,dist) = formevent(refreshevent=refreshevent,
    refreshseocrsp=refreshseocrsp,
    refreshdistcrsp=refreshdistcrsp)
  cumex = formcumex(dist, refreshcumex = refreshcumex)
  table7 && maketable7(bb)
  table8 && maketable8(dist)
  figure1 && makefigure1(dist)
  figure2 && makefigure2(dist)
  table9 && maketable9(cumex)
  figure3 && makefigure3(cumex)
  table10 && maketable10(cumex)
  figure4 && makefigure4(dist)

  #####################################appendix
  @info "Begining appendix tables"
  tablea1 && maketablea1(panel, trialrunonly=trialrunonly)
  tablea2 && maketablea2(dist)

  return nothing
end

max_stars_repo_name: clintonTE/CCA

max_issues_repo_name: clintonTE/CCA

max_forks_repo_name: clintonTE/CCA


text:
module MLJ

## EXPORTS

export MLJ_VERSION

# defined in include files:
export @curve, @pcurve, pretty,                   # utilities.jl
    coerce, supervised, unsupervised,             # tasks.jl
    report,                                       # machines.jl
    Holdout, CV, evaluate!, Resampler,            # resampling.jl
    Params, params, set_params!,                  # parameters.jl
    strange, iterator,                            # parameters.jl
    Grid, TunedModel, learning_curve!,            # tuning.jl
    EnsembleModel,                                # ensembles.jl
    rebind!,                                      # networks.jl
    machines, sources, anonymize!,                # composites.jl
    @from_network,                                # composites.jl
    fitresults,                                   # composites.jl
    @pipeline,                                    # pipelines.jl
    matching                                      # matching.jl

# defined in include files "machines.jl and "networks.jl":
export Machine, NodalMachine, machine, AbstractNode,
        source, node, fit!, freeze!, thaw!, Node, sources, origins

# re-export from Random, Statistics, Distributions, CategoricalArrays:
export pdf, mode, median, mean, shuffle!, categorical, shuffle, levels, levels!
export std, support

# re-export from MLJBase and ScientificTypes:
export nrows, nfeatures, color_off, color_on,
    selectrows, selectcols,
    SupervisedTask, UnsupervisedTask, MLJTask,
    Deterministic, Probabilistic, Unsupervised, Supervised,
    DeterministicNetwork, ProbabilisticNetwork,
    GrayImage, ColorImage, Image,
    Found, Continuous, Finite, Infinite,
    OrderedFactor, Unknown,
    Count, Multiclass, Binary, Scientific,
    scitype, scitype_union, schema, scitypes,
    target_scitype, input_scitype, output_scitype,
    predict, predict_mean, predict_median, predict_mode,
    transform, inverse_transform, se, evaluate, fitted_params,
    @constant, @more, HANDLE_GIVEN_ID, UnivariateFinite,
    classes,
    partition, unpack,
    mav, mae, rms, rmsl, rmslp1, rmsp, l1, l2,
    misclassification_rate, cross_entropy,
    default_measure,
    @load_boston, @load_ames, @load_iris, @load_reduced_ames,
    @load_crabs

# re-export from MLJModels:
export models, localmodels, @load, load, info,
    ConstantRegressor, ConstantClassifier,     # builtins/Constant.jl
    StaticTransformer, FeatureSelector,        # builtins/Transformers.jl
    UnivariateStandardizer, Standardizer,
    UnivariateBoxCoxTransformer,
    OneHotEncoder, UnivariateDiscretizer,
    FillImputer

## IMPORTS

using MLJBase
using MLJModels

# these are defined in MLJBase
export load_boston, load_ames, load_iris
export load_reduced_ames
export load_crabs

# to be extended:
import MLJBase: fit, update, clean!,
    predict, predict_mean, predict_median, predict_mode,
    transform, inverse_transform, se, evaluate, fitted_params,
    show_as_constructed, params
import MLJModels: models

import Pkg, Pkg.TOML
using Tables, OrderedCollections

using  CategoricalArrays
import Distributions
import Distributions: pdf, mode
import Statistics, StatsBase, LinearAlgebra, Random
import Random: AbstractRNG, MersenneTwister

using ProgressMeter

import PrettyTables
using ScientificTypes

using ComputationalResources
using ComputationalResources: CPUProcesses

const DEFAULT_RESOURCE = Ref{AbstractResource}(CPU1())

# convenience packages
using DocStringExtensions: SIGNATURES, TYPEDEF

# to be extended:
import Base: ==, getindex, setindex!
import StatsBase.fit!

# from Standard Library:
import Distributed: @distributed, nworkers, pmap
using RecipesBase # for plotting

const srcdir = dirname(@__FILE__) # the directory containing this file:
const CategoricalElement = Union{CategoricalString,CategoricalValue}

# FIXME replace with either Pkg.installed()["MLJ"] or
# uuid = Pkg.project().dependencies["MLJ"]
# version = Pkg.dependencies()[uuid].version
# ---
# this is currently messy because it's been enacted then reverted
# see https://github.com/JuliaLang/julia/pull/33410
# and https://github.com/JuliaLang/Pkg.jl/pull/1086/commits/996c6b9b69ef0c058e0105427983622b7cc8cb1d
toml = Pkg.TOML.parsefile(joinpath(dirname(dirname(pathof(MLJ))), "Project.toml"))
const MLJ_VERSION = toml["version"]

## INCLUDES

include("utilities.jl")     # general purpose utilities
include("machines.jl")
include("networks.jl")      # for building learning networks
include("composites.jl")    # composite models & exporting learning networks
include("pipelines.jl")     # pipelines (exported linear learning networks)
include("operations.jl")    # syntactic sugar for operations (predict, etc)

if VERSION ≥ v"1.3.0-"
    include("arrows.jl")
end

include("resampling.jl")    # resampling strategies and model evaluation
include("parameters.jl")    # hyperparameter ranges and grid generation
include("tuning.jl")
include("ensembles.jl")     # homogeneous ensembles
include("model_matching.jl")# inferring model search criterion from data
include("tasks.jl")         # enhancements to MLJBase task interface
include("scitypes.jl")      # extensions to ScientificTypes.sictype
include("plotrecipes.jl")

end # module


text:
module SubPkgA

println("EVALUATING SubPkgA!")

fnA() = println("SubPkgA.fnA()")

function __init__()
	println("Initializing SubPkgA!")
end

end # module


text:
using DelimitedFiles

puzzleinput = readdlm("inputs/day15.txt", ',', Int)

function play(startingnums, numofturns)

    spoken = fill(-1, numofturns)

    for i in eachindex(startingnums[1:end - 1])
        spoken[startingnums[i] + 1] = i
    end
    # Preparation of the game: every elf says his number that is the puzzleinput

    lstspkn = last(startingnums)
    turn = length(startingnums)
    stop = numofturns - 1
    while true
        # Game loop

        newnumber = spoken[lstspkn + 1] > 0 ? turn - spoken[lstspkn + 1] : 0
        # Had the last number spoken also been spoken before?

        # YES: then the next number to speak is the difference between
        # the turn number when it was last spoken (actual turn - 1) and
        # the turn number the time it was most recently spoken before then

        # NO: is a new numbers then the elf says "0"

        spoken[lstspkn + 1] = turn
        lstspkn = newnumber
        # Now the elf says the number

        if turn < stop
            turn += 1
        else
            return lstspkn
        end
    end

end

# First Half

@time play(puzzleinput, 2020)

# Second Half

@time play(puzzleinput, 30000000)


text:
"""
    AnalogOutChannelCount(dev::DWFDevice)

Return the number of analog out channels.
"""
function AnalogOutChannelCount(dev::DWFDevice)
    cChannel = Ref(Cint(0))
    FDwfAnalogInChannelCount(dev.hdwf[], cChannel)
    return cChannel[]
end

"""
    AnalogOutEnableChannel!(dev::DWFDevice, channel::Int; state::Bool=true)

Enable analog out channel (enable by default)
"""
function AnalogOutEnableChannel!(dev::DWFDevice, channel::Int; state::Bool=true)
    FDwfAnalogOutNodeEnableSet(dev.hdwf[], channel, AnalogOutNodeCarrier, state)
    return nothing
end

"""
    AnalogOutStart!(dev::DWFDevice, channel::Int)

Start signal generation on given analog output channel
"""
function AnalogOutStart!(dev::DWFDevice, channel::Int)
    FDwfAnalogOutConfigure(dev.hdwf[], channel, true)
end

"""
    AnalogOutStop!(dev::DWFDevice, channel::Int)

Start signal generation on given analog output channel
"""
function AnalogOutStop!(dev::DWFDevice, channel::Int)
    FDwfAnalogOutConfigure(dev.hdwf[], channel, false)
end

"""
    AnalogOutDCMode!(dev::DWFDevice, channel::Int)

Enable DC signal generation mode on given analog output channel
"""
function AnalogOutDCMode!(dev::DWFDevice, channel::Int)
    FDwfAnalogOutNodeFunctionSet(dev.hdwf[], channel, AnalogOutNodeCarrier, funcDC)
end

"""
    AnalogOutDCVal!(dev::DWFDevice, channel::Int, v::T) where {T<:Real}

Enable DC offset value for signal generation mode on given analog output channel
"""
function AnalogOutDCVal!(dev::DWFDevice, channel::Int, v::T) where {T<:Real}
    FDwfAnalogOutNodeOffsetSet(dev.hdwf[], channel, AnalogOutNodeCarrier, v)
    #FDwfAnalogOutNodeAmplitudeSet(dev.hdwf[], channel, AnalogOutNodeCarrier, v)
end


text:
#
# Correctness Tests
#

using kNN

my_tests = ["majority_vote.jl",
            "classifier.jl",
            "regress.jl"]

println("Running tests:")

for my_test in my_tests
    println(" * $(my_test)")
    include(my_test)
end


text:
using JuMP
import HiGHS
lp_solver = optimizer_with_attributes(HiGHS.Optimizer, MOI.Silent() => true)

max_stars_repo_name: blegat/Polyhedra

max_issues_repo_name: blegat/Polyhedra

max_forks_repo_name: blegat/Polyhedra


text:
@testset "ardtransform" begin
    rng = MersenneTwister(123456)

    @testset "Real input" begin
        v = randn(rng, 1)
        t = ARDTransform(v)

        x = randn(rng, 4)
        x′ = map(t, x)

        @test all([t(x[n]) == v[1] * x[n] for n in eachindex(x)])
        @test all([t(x[n]) ≈ x′[n] for n in eachindex(x)])
    end

    @testset "Vector input" begin
        D = 3
        v = randn(rng, D)
        t = ARDTransform(v)

        XV = [randn(rng, D) for _ in 1:5]
        XC = ColVecs(randn(rng, D, 7))
        XR = RowVecs(randn(rng, 2, D))

        @testset "$(typeof(x))" for x in [XV, XC, XR]
            x′ = map(t, x)
            @test all([t(x[n]) == v .* x[n] for n in eachindex(x)])
            @test all([t(x[n]) ≈ x′[n] for n in eachindex(x)])
        end
    end

    # Check that setting the parameters works.
    D = 4
    t = ARDTransform(randn(rng, D))
    v = randn(rng, D)
    KernelFunctions.set!(t, v)
    @test all(t.v .== v)

    # Check ARD constructor with constant fill.
    s = randn(rng)
    @test ARDTransform(s, D).v == ARDTransform(s * ones(D)).v

    @test_throws DimensionMismatch map(t, ColVecs(randn(rng, D + 1, 3)))

    @test repr(t) == "ARD Transform (dims: $D)"
    test_ADs(x->transform(SEKernel(), exp.(x)), randn(rng, 3))
end


text:
using Test
using MersenneTwister

@testset "MT19937" begin
    mt = MtRandom()

    # https://oeis.org/A221557
    N = 624
    seq = [extract_number!(mt) for i in 1:(N * 2 + 1)]
    @test seq[1:3] == [0xD091BB5C, 0x22AE9EF6, 0xE7E1FAEE]
    @test seq[N] == 0xEFA14DFF
    @test seq[N + 1] == 0xF914DC58
    @test seq[N * 2 + 1] == 0x155F212F
end

max_stars_repo_name: gyk/TrivialSolutions

max_issues_repo_name: gyk/TrivialSolutions

max_forks_repo_name: gyk/TrivialSolutions


text:
include("./fetch.jl")
include("./parsedata.jl")
choice=0; s_start=1; s_end=-1; s_name=""
function setseries(name::AbstractString, ch::AbstractString="single", st::Int64=1,  en::Int64=-1)
    global choice, s_start, s_end, s_name
    s_name=name
    if (ch=="series")
        choice=1
    elseif (ch=="single")
        choice=2
    else
        error("Pl. set type as \"single\" or \"series\" only.")
    end
    s_start=st
    s_end=en
    return 0
end

function segdata(name::AbstractString, typedata::AbstractString, dir::AbstractString)
    s_start=1; s_end=-1; s_name=""; files=[]
    if (choice==1)
        i=0; flag=0
        for i in 1:length(s_name)
            if s_name[i]=='*'
                flag=1
                break
            end
        end
        if (flag==0) i+=1 end
        for j in (s_start:s_end)
            file="$(dir)/$(s_name[1:i-1])"
            file*="$(j)"
            file*="$(s_name[i+1:end])"
            push!(files,file)
        end
    elseif (choice==2)
        files=["$(s_name)"]
    else
        error("Please call setseries() first.")
        return
    end
    fileflag=0
    for file in readdir(dir)
        if (isfile(joinpath(dir,file)))
            fileflag=1
            break
        end
    end
    if(fileflag==0)
        error("Please call add(<url>) to download dataset first.")
    end
    data = UInt8[]
    for file in files
        append!(data, open(read,file))
    end
    if (typedata=="matrix")
      readMat(data)
    else
      readLine(dir, files)
    end
end

function traindata(name::AbstractString, typedata::AbstractString="matrix", dir::AbstractString="")
    if(dir=="")
      dir=joinpath(Pkg.dir("DataDeps"), "datasets/$(name)")
    end
    try
        f=open(joinpath(Pkg.dir("DataDeps"),"datasets/downloads.json"), "r")
        dicttxt = readstring(f)
        close(f)
        dictall=JSON.parse(dicttxt)
        dir=dictall[name]
    end
    segdata(name,typedata,dir)
end
#=
function testdata(name::AbstractString, dir="")
    segdata(name,dir)
end
=#


text:
#=
   Logistic regression - Challenger

   See ~/jags/logistic_regression_challenger.R
   """
   model {
     for (i in 1:N){
       y[i] ~ dbern(p[i])
       p[i] <- 1 / (1 + exp(-z[i]))
       z[i] <- w0 + w1 * x[i]
     }
     w0 ~ dnorm(0, .001)
     w1 ~ dnorm(0, .0001)

   Output:
      Mean     SD  Naive SE Time-series SE
w0 16.8900 7.7265 0.0122167       0.290180
w1 -0.2602 0.1135 0.0001794       0.004232

2. Quantiles for each variable:

     2.5%     25%     50%     75%    97.5%
w0  4.131 11.3621 16.1240 21.5029 34.27954
w1 -0.516 -0.3279 -0.2488 -0.1789 -0.07332
   """

  From https://www.zinkov.com/posts/2012-06-27-why-prob-programming-matters/
  "Logistic Regression"
  """
  Logistic Regression can be seen as a generalization of Linear Regression where the output is
  transformed
  to lie between 0 and 1. This model only differs from the previous one by a single line, illustrating that
  adding this complexity does not require starting from scratch. The point with probabilistic programming
  is you are able to explore slightly more complex models very easily.
  """

  From https://www.stat.ubc.ca/~bouchard/courses/stat520-sp2014-15/lecture/2015/02/27/notes-lecture3.html
  x = 66,70,69,68,67,72,73,70,57,63,70,78,67,53,67,75,70,81,76,79,75,76,58
  y = 1,0,1,1,1,1,1,1,0,0,0,1,1,0,1,1,1,1,1,1,0,1,0

Summary Statistics
  parameters       mean       std   naive_se      mcse        ess      rhat   ess_per_sec
      Symbol    Float64   Float64    Float64   Float64    Float64   Float64       Float64

          w0   -33.3287   27.6137     0.2761    2.7552    22.0381    1.0003        0.3419
          w1     0.5239    0.4022     0.0040    0.0401    22.0718    0.9999        0.3424
   y_post[1]     0.6917    0.4618     0.0046    0.0230    80.9484    1.0391        1.2558
   y_post[2]     0.9368    0.2433     0.0024    0.0089   159.7127    1.0322        2.4777
   y_post[3]     0.9186    0.2735     0.0027    0.0074   304.2265    1.0235        4.7195
   y_post[4]     0.8776    0.3278     0.0033    0.0077   380.5956    1.0229        5.9043
   y_post[5]     0.8053    0.3960     0.0040    0.0130   193.3661    1.0251        2.9997
   y_post[6]     0.9399    0.2377     0.0024    0.0114    93.7454    1.0461        1.4543
   y_post[7]     0.9394    0.2386     0.0024    0.0116    90.6911    1.0474        1.4069
   y_post[8]     0.9356    0.2455     0.0025    0.0088   166.6864    1.0323        2.5858
   y_post[9]     0.3533    0.4780     0.0048    0.0369    36.9957    0.9999        0.5739
  y_post[10]     0.4858    0.4998     0.0050    0.0415    30.6274    1.0526        0.4751
  y_post[11]     0.9357    0.2453     0.0025    0.0083   193.5760    1.0295        3.0030
  y_post[12]     0.9158    0.2777     0.0028    0.0172    52.1763    1.0862        0.8094
  y_post[13]     0.7979    0.4016     0.0040    0.0139   171.6327    1.0287        2.6626
  y_post[14]     0.2547    0.4357     0.0044    0.0354    32.1260    1.0606        0.4984
  y_post[15]     0.7999    0.4001     0.0040    0.0133   190.0694    1.0279        2.9486
  y_post[16]     0.9272    0.2598     0.0026    0.0148    64.1946    1.0679        0.9959
  y_post[17]     0.9384    0.2404     0.0024    0.0081   203.3125    1.0277        3.1540
  y_post[18]     0.8981    0.3025     0.0030    0.0208    42.3442    1.1074        0.6569
  y_post[19]     0.9268    0.2605     0.0026    0.0148    63.1168    1.0693        0.9791
  y_post[20]     0.9092    0.2873     0.0029    0.0184    48.8488    1.0916        0.7578
  y_post[21]     0.9295    0.2560     0.0026    0.0142    65.7403    1.0655        1.0198
  y_post[22]     0.9262    0.2615     0.0026    0.0149    61.2013    1.0709        0.9494
  y_post[23]     0.3722    0.4834     0.0048    0.0377    36.3094    1.0021        0.5633

  Cf ~/blog/logistic_regression-challenger.blog
     ~/webppl/logistic_regression-challenger.wppl

  Compare with logistic_regression.jl for another approach.

=#

using Turing, StatsPlots

include("jl_utils.jl")

@model function logistic_regression_challenger(xs,ys)
    n = length(xs)
    w0 ~ Normal(0, sqrt(1/.001)) # JAGS has dnorm(mu,1/variance)
    w1 ~ Normal(0, sqrt(1/.0001))
    # w0 ~ Normal(0, 10) # JAGS has dnorm(mu,1/variance)
    # w1 ~ Normal(0, 10)

    p = tzeros(n)
    z = tzeros(n)
    for i in 1:n
        z[i] = w0 + w1 * xs[i]
        p[i] = 1 / (1 + exp(-z[i]))
        ys[i] ~ Bernoulli(p[i])

        # ys[i] ~ Bernoulli(1 / (1 + exp(-w0 + w1 * xs[i])))
    end

    y_post = tzeros(n)
    for i in 1:n
        y_post[i] ~ Bernoulli(p[i])
    end
end

xs = [66,70,69,68,67,72,73,70,57,63,70,78,67,53,67,75,70,81,76,79,75,76,58];
# ys = [true,false,true,true,true,true,true,true,false,false,false,true,true,false,true,true,true,true,true,true,false,true,false];
ys = [1,0,1,1,1,1,1,1,0,0,0,1,1,0,1,1,1,1,1,1,0,1,0];
#     1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3

model = logistic_regression_challenger(xs,ys)

# chns = sample(model, Prior(), 10_000)
# chns = sample(model, MH(), 10_000)
chns = sample(model, PG(15), 1_000)
# chns = sample(model, SMC(), 10_000)
# chns = sample(model, IS(), 10_000)

display(chns)
# display(plot(chns))

# display(plot(chns[[:w0,:w1]]) # quite bad!

println("Posterior predictive:")
num_correct = 0
n = length(ys)
for i in 1:n
    pred1 = mean(chns["y_post[$i]"])
    pred2 = round(Int64,pred1)
    c = "!"
    if pred2 == ys[i]
        global num_correct += 1
        c = ""
    end
    println("$i: true: $(ys[i]) pred: $pred2 ($pred1)$c" )
end
println("num_correct: $num_correct / $n ($(100*num_correct/n)%)")

max_stars_repo_name: tias/hakank

max_issues_repo_name: tias/hakank

max_forks_repo_name: tias/hakank


text:
const DEFAULT_SAMPLE_SIZE = 1000

export sourceParticles

foreach([<=, >=, <, >]) do cmp
    MonteCarloMeasurements.register_primitive(cmp, eval)
end

export particles

particles(v::Vector{<:Real}) = Particles(v)
particles(d, N::Int=DEFAULT_SAMPLE_SIZE) = Particles(N, d)

# particles(n :: NUTS_result) = particles(n.samples)
using IterTools

function particles(v::Vector{Vector{T} }) where {T}
    map(eachindex(v[1])) do j particles([x[j] for x in v]) end
end

function particles(v::Vector{Array{T,N} }) where {T,N}
    map(CartesianIndices(v[1])) do j particles([x[j] for x in v]) end
end

function particles(s::Vector{NamedTuple{vs, T}}) where {vs, T}
    nt = NamedTuple()
    for k in keys(s[1])
        nt = merge(nt, [k => particles(getproperty.(s,k))])
    end
    nt
end

export parts

# Just a little helper function for particles
# https://github.com/baggepinnen/MonteCarloMeasurements.jl/issues/22
parts(d, N::Int=DEFAULT_SAMPLE_SIZE) = particles(d, N)
parts(x::Normal{P} where {P <: AbstractParticles}, N::Int=DEFAULT_SAMPLE_SIZE) = Particles(length(x.μ), Normal()) * x.σ + x.μ
parts(x::Sampleable{F,S}, N::Int=DEFAULT_SAMPLE_SIZE) where {F,S} = Particles(N,x)
parts(x::Integer, N::Int=DEFAULT_SAMPLE_SIZE) = parts(float(x))
parts(x::Real, N::Int=DEFAULT_SAMPLE_SIZE) = parts(repeat([x],N))
parts(x::AbstractArray, N::Int=DEFAULT_SAMPLE_SIZE) = Particles(x)
parts(p::Particles, N::Int=DEFAULT_SAMPLE_SIZE) = p
parts(d::For, N::Int=DEFAULT_SAMPLE_SIZE) = parts.(d.f.(d.θ...), N)
function parts(d::For{F,Tuple{I}}, N::Int=DEFAULT_SAMPLE_SIZE) where {F <: Function, I <: Integer}
    parts.(d.f.(Base.OneTo.(d.θ)...), N)
end

parts(d::iid, N::Int=DEFAULT_SAMPLE_SIZE) = map(1:d.size) do j parts(d.dist, N) end
# size
# dist

# MonteCarloMeasurements.Particles(n::Int, d::For) =
# -(a::Particles{Float64,1000}, b::Array{Float64,1}) = [a-bj for bj in b]

# Base.promote(a::Particles{T,N}, b)  where {T,N} = (a,parts(b))
# Base.promote(a, b::Particles{T,N})  where {T,N} = (parts(a),b)

# promote_rule(::Type{A}, ::Type{B}) where {A <: Real, B <: AbstractParticles{T,N}} where {T} = AbsractParticles{promote_type(A,T),N} where {N}
# promote_rule(::Type{B}, ::Type{A}) where {A <: Real, B <: AbstractParticles{T,N}} where {T} = AbsractParticles{promote_type(A,T),N} where {N}

@inline function particles(m::JointDistribution, N::Int=DEFAULT_SAMPLE_SIZE)
    return _particles(getmoduletypencoding(m.model), m.model, m.args, Val(N))
end

@gg M function _particles(_::Type{M}, _m::Model, _args, _n::Val{_N}) where {M <: TypeLevel{Module},_N}
    Expr(:let,
        Expr(:(=), :M, from_type(M)),
        sourceParticles()(type2model(_m), _n) |> loadvals(_args, NamedTuple()))
end

@gg M function _particles(_::Type{M}, _m::Model, _args::NamedTuple{()}, _n::Val{_N}) where {M <: TypeLevel{Module},_N}
    Expr(:let,
        Expr(:(=), :M, from_type(M)),
        sourceParticles()(type2model(_m), _n))
end

export sourceParticles
function sourceParticles()

    function(m::Model, ::Type{Val{_N}}) where {_N}
        _m = canonical(m)
        proc(_m, st::Assign)  = :($(st.x) = $(st.rhs))
        proc(_m, st::Sample)  = :($(st.x) = parts($(st.rhs), $_N))
        proc(_m, st::Return)  = :(return $(st.rhs))
        proc(_m, st::LineNumber) = nothing

        vals = map(x -> Expr(:(=), x,x),variables(_m))

        wrap(kernel) = @q begin
            $kernel
            $(Expr(:tuple, vals...))
        end

        buildSource(_m, proc, wrap) |> flatten
    end
end


text:
module Arbiter

export ArbiterTask

include("graph.jl")
include("task.jl")
include("scheduler.jl")
include("base.jl")
include("sync.jl")

import .ArbiterTasks: ArbiterTask

end # module


text:
#                    host        dbname      table       col    type
columns_types = Dict{String, # host
                     Dict{String, # dbname
                          Dict{String, # schema
                               Dict{String, # table
                                    Dict{String,String} # column, type
                                    }
                               }
                          }
                     }()
function util_get_column_type(dbconn::LibPQ.Connection,
                              tablename::String,
                              schema::String,
                              colname::String,
                              refresh = false)

     dbname = util_getdbname(dbconn)
     host = util_getdbhost(dbconn)

     if !haskey(columns_types, host)
         columns_types[host] = Dict{String, # dbname
                                    Dict{String, # schema
                                         Dict{String, # table
                                              Dict{String,String} # column, type
                                              }
                                         }
                                    }()
     end
     if !haskey(columns_types[host], dbname)
         columns_types[host][dbname] = Dict{String, # schema
                                            Dict{String, # table
                                                 Dict{String,String} # column, type
                                                 }
                                            }()
     end
     if !haskey(columns_types[host][dbname], schema)
         columns_types[host][dbname][schema] = Dict{String, # table
                                                        Dict{String,String} # column, type
                                                        }()
     end
     if !haskey(columns_types[host][dbname][schema], tablename)
         columns_types[host][dbname][schema][tablename] = Dict{String,String}()
     end
     if (!haskey(columns_types[host][dbname][schema][tablename], colname)
          || refresh == true)
         # Get the type from database
         query_string = "SELECT data_type from information_schema.columns
                            WHERE table_schema = \$1
                              AND table_name = \$2
                              AND column_name = \$3"

         query_result = execute_plain_query(query_string,
                              [schema, tablename,colname],
                              dbconn)
         if nrow(query_result) == 0
             error("Unable to retrieve the type of column[$colname] "
             * "in table[$tablename] of schema[$schema]")
         end

         coltype::String = query_result[1,1]
         columns_types[host][dbname][schema][tablename][colname] = coltype
     else
         coltype = columns_types[host][dbname][schema][tablename][colname]
     end

     return coltype

end

function util_is_column_numeric(dbconn::LibPQ.Connection,
                                tablename::String,
                                schema::String,
                                colname::String)

    coltype = util_get_column_type(dbconn,
                                   tablename,
                                   schema,
                                   colname)

    if coltype in (["smallint","integer","bigint","decimal","numeric","real",
                    "double precision","smallserial","serial","bigserial"])
       return true
    end

   return false

end


text:
#
# Denoise an SPD Example with the linearized Riemannian Chambolle-Pock algorithm
#
# L2-TV functional with anisotropic TV
#
# This example is used in Section 6.2 of
#
# > R. Bergmann, R. Herzog, M. Silva Louzeiro, D. Tenbrinck, J. Vidal Núñez:
# > _Fenchel Duality Theory and a Primal-Dual Algorithm on Riemannian Manifolds_,
# > arXiv: [1908.02022](https://arxiv.org/abs/1908.02022)
#
using Manopt, Manifolds
using Images, CSV, DataFrames, LinearAlgebra, JLD2, Dates
#
# Settings
experiment_name = "SPD_Image_CP"
export_original = true
export_result = true
export_table = true
use_debug = true
asy_render_detail = 2
results_folder = joinpath(@__DIR__, "Image_TV")
comparison_data = joinpath(
    @__DIR__, "..", "CyclicProximalPoint", "Image_TV", "SPD_Image_CPPA-cost.jld2"
)
!isdir(results_folder) && mkdir(results_folder)
#
# Manifold & Data
f = artificial_SPD_image2(32)
if export_original
    fn = joinpath(results_folder, experiment_name * "-orig.asy")
    asymptote_export_SPD(fn; data=f, scale_axes=(7.5, 7.5, 7.5))
    render_asymptote(fn; render=asy_render_detail)
end
sC = StopAfterIteration(400)
try
    cppa_data = load(comparison_data)
    cost_threshold = cppa_data["cost_function_value"]
    cppa_iter = cppa_data["iterations"]
    global sC = StopWhenAny(StopAfterIteration(400), StopWhenCostLess(cost_threshold))
    @info "Comparison to CPPA (`SPDImage_CPPA.jl`) and its cost after $(cppa_iter) iterations of $cost_threshold."
catch e
    msg = sprint(showerror, e)
    @info "Comparison to CPPA (`CyclicProximalPoint/SPDImage_CPPA.jl`) not possible, data file $(comparison_data) either missing or corrupted.\n Error: $(msg).\n\n Starting with a default stopping criterion."
end
#
# Parameters
L = sqrt(8)
α = 6.0
σ = 0.40
τ = 0.40
θ = 1.0
γ = 0.2
pixelM = SymmetricPositiveDefinite(3)
#
# load TV model
#
include("Image_TV_commons.jl")

m = fill(Matrix{Float64}(I, 3, 3), size(f))
n = Λ(m)
x0 = f
ξ0 = ProductRepr(zero_tangent_vector(M2, m2(m)), zero_tangent_vector(M2, m2(m)))

@info "with parameters σ: $σ | τ: $τ | θ: $θ | γ: $γ."
@time o = ChambollePock(
    M,
    N,
    cost,
    x0,
    ξ0,
    m,
    n,
    proxFidelity,
    proxPriorDual,
    DΛ,
    AdjDΛ;
    primal_stepsize=σ,
    dual_stepsize=τ,
    relaxation=θ,
    acceleration=γ,
    relax=:dual,
    debug=use_debug ? [:Iteration, " | ", :Cost, "\n", 10, :Stop] : missing,
    record=export_table ? [:Iteration, :Cost] : missing,
    stopping_criterion=sC,
    variant=:linearized,
    return_options=true,
)
y = get_solver_result(o)
export_table && (r = get_record(o))

numIter = length(r)
if export_result
    fn = joinpath(
        results_folder,
        experiment_name * "img-result-$(numIter)-α$(replace(string(α), "." => "-")).asy",
    )
    asymptote_export_SPD(fn; data=y, scale_axes=(7.5, 7.5, 7.5))
    render_asymptote(fn; render=asy_render_detail)
end
if export_table
    A = cat([ri[1] for ri in r], [ri[2] for ri in r]; dims=2)
    CSV.write(
        joinpath(results_folder, experiment_name * "-Cost.csv"),
        DataFrame(A);
        writeheader=false,
    )
end


text:
module os

end

max_stars_repo_name: tmaone/xsh

max_issues_repo_name: turing-complete/xsh

max_forks_repo_name: turing-complete/xsh


text:
module SpkTuning

using SpkCore, Statistics, Optim#,NLopt,

include("./TuningUtils.jl")

export TuningData, fit, f0, f1, interpolate
export Contrast, LogisticContrast, Area, SSE, WeightedSSE

# ============================================================================ #
f0(x::AbstractVector{<:Real}, tf::Real=0.0) = mean(x)
f1(x::AbstractVector{<:Real}, tf::Real) = abs(f1xfm(cycle_mean(x, floor(Int, 1.0 / tf))))
# ============================================================================ #
function trial_reduce(op::Function, data::TuningData, bin_size::Real=0.001, calc_error::Bool=false)
    kbin = 1:round(Int, data.dur / bin_size)
    ulab = sort(unique(data.lab))
    mn = zeros(length(ulab))
    err = ones(length(ulab))
    for k in eachindex(ulab)
        kevt = findall(isequal(ulab[k]), data.lab)
        if calc_error
            d = psth(data.ts, data.evt[kevt], kbin, bin_size)[1] ./ bin_size
            tmp = zeros(size(d, 2))
            kc = 1
            for col in eachcol(tmp)
                tmp[kc] = op(col, data.tf * bin_size)
                kc += 1
            end
            # tmp = mapslices(x->op(x, data.tf), d, dims=2)
            mn[k] = mean(tmp)
            err[k] = std(tmp)
        else
            d = vec(mean(psth(data.ts, data.evt[kevt], kbin, bin_size)[1], dims=2)) ./ bin_size
            mn[k] = op(d, data.tf * bin_size)
        end
    end
    return FitData(ulab, mn, err)
end
# ============================================================================ #
function fit(::Type{T}, data::TuningData, op::Function, bin_size::Real=0.001) where {T<:TuningFunction{SSE}}
    return fit(T, trial_reduce(op, data, bin_size))
end
# ---------------------------------------------------------------------------- #
function fit(::Type{T}, data::TuningData, op::Function, bin_size::Real=0.001) where {T<:TuningFunction{WeightedSSE}}
    return fit(T, trial_reduce(op, data, bin_size, true))
end
# ---------------------------------------------------------------------------- #
function fit(::Type{T}, fd::FitData) where {T<:TuningFunction}
    p0, lb, ub = parameters(T, fd)
    count = 0
    fobj(p::Vector) = begin
        count += 1
        return evaluate(T, fd, p)
    end

    # Optim.jl version using LBFGS
    res = optimize(fobj, lb, ub, p0, Fminbox(LBFGS()); autodiff = :forward)
    flag = Optim.x_converged(res) || Optim.f_converged(res) || Optim.g_converged(res) ?
        :converged : :converge_fail
    return TuningResult(T, fd, Optim.minimizer(res), Optim.minimum(res), flag)

    # opt = Opt(:LN_PRAXIS, length(p0))
    #
    # lower_bounds!(opt, lb)
    # upper_bounds!(opt, ub)
    # min_objective!(opt, fobj)
    # minf, minx, ret = optimize(opt, p0)
    #
    # return TuningResult(T, fd, minx, minf, ret)
end
# ============================================================================ #
function objective(::Type{T}, fd::FitData, model::Vector) where {T<:TuningFunction{SSE}}
    return sum((model - fd.y).^2)
end
# ---------------------------------------------------------------------------- #
function objective(::Type{T}, fd::FitData, model::Vector) where {T<:TuningFunction{WeightedSSE}}
    return sum(((model - fd.y) .^ 2.0) ./ fd.err)
end
# ---------------------------------------------------------------------------- #
function evaluate(::Type{T}, fd::FitData, p::Vector) where {T<:TuningFunction}
    return objective(T, fd, generate(T, fd.x, p))
end
# ---------------------------------------------------------------------------- #
function interpolate(tr::TuningResult{T}, n::Int=100, log::Bool=false) where {T<:TuningFunction}

    xmn, xmx = extrema(tr.data.x)

    if log
        x = 10 .^ range(log10(xmn), log10(xmx), length=n)
    else
        x = range(xmn, xmx, length=n)
    end
    y = generate(T, collect(x), tr.p)

    return x, y
end
# ============================================================================ #
# function test()
#     db = get_database("contrast", id->id>100);
#     ret, lgn, evt, lab = get_data(db, 1);
#     dur = get_uniform_param(db[1], "stimulus_duration");
#     tf = get_uniform_param(db[1], "temporal_frequency");
#     td = TuningData(ret, evt, lab, dur, tf);
#     tr = fit(Contrast{SSE}, td, f0, 0.001);
#     x, y = interpolate(tr, 100);
#     plot(tr.data.x, tr.data.y, ".")
#     plot(x, y)
# end
# ============================================================================ #
end


text:
data =XLSX.readxlsx("data.xlsx")
#=
source_data = readxlsheet("data.xlsx","Source_ESDMI")
sink_data = readxlsheet("data.xlsx","Sink_2")
cost_data = readxlsheet("data.xlsx", "Cost")
=#

source_data = data["Source_ESDMI"][:]
sink_data = data["Sink_2"][:]
cost_data = data["Cost"][:]

sources = [string(source_data[i,3]) for i in 2:size(source_data,1)]
sinks = [string(sink_data[i,2]) for i in 2:size(sink_data,1)]
sink_type = unique(sink_data[2:end,1])

# New sets
domestic_sites = [string(sink_data[i,2]) for i in 2:size(sink_data,1) if sink_data[i,1] == "Inland"]
domestic_EOR_sites = [string(sink_data[i,2]) for i in 2:size(sink_data,1) if sink_data[i,6] == 1]
domestic_non_EOR_sites = [string(sink_data[i,2]) for i in 2:size(sink_data,1) if sink_data[i,6] == 0 && sink_data[i,1] == "Inland"]

set_type_sink_dict = Dict([sink_data[i,1],sink_data[i,2]] => 1 for i in 2:size(sink_data,1))
set_domestic_EOR_dict = Dict([sink_data[i,2]] => 1 for i in 2:size(sink_data,1) if sink_data[i,6] == 1)

param_source_lat = Dict([source_data[i,3] => source_data[i,4] for i in 2:size(source_data,1)])
param_source_long = Dict([source_data[i,3] => source_data[i,5] for i in 2:size(source_data,1)])
param_source_region = Dict([source_data[i,3] => source_data[i,1] for i in 2:size(source_data,1)])
param_source_fuel = Dict([source_data[i,3] => source_data[i,2] for i in 2:size(source_data,1)])

param_sink_lat = Dict([sink_data[i,2] => sink_data[i,3] for i in 2:size(sink_data,1)])
param_sink_long = Dict([sink_data[i,2] => sink_data[i,4] for i in 2:size(sink_data,1)])
param_sink_region = Dict(sink_data[i,2] => sink_data[i,7] for i in 2:size(sink_data,1))

param_source_sink_dist = Dict()
for s in sources
    for k in sinks
            if !haskey(set_type_sink_dict,["Offshore",k])
                param_source_sink_dist[s,k] = haversine((param_source_long[s],param_source_lat[s]), (param_sink_long[k], param_sink_lat[k]), 6372.8)
            end
    end
end

for k in sinks
    for l in sinks
        if haskey(set_type_sink_dict,["Hub",k]) && haskey(set_type_sink_dict,["Offshore",l])
            param_source_sink_dist[k,l] = haversine((param_sink_long[k],param_sink_lat[k]), (param_sink_long[l], param_sink_lat[l]), 6372.8)
        end
    end
end

source_sink_dist = DataFrame(source = String[], Sink = String[], Distance_km = Float64[])

param_source_size = Dict([source_data[i,3] => source_data[i,6] for i in 2:size(source_data,1)])
param_sink_limit = Dict([sink_data[i,2] => sink_data[i,5] for i in 2:size(sink_data,1)])

param_capture_cost = Dict([source_data[i,3] => source_data[i,8] for i in 2:size(source_data,1)])
param_transport_cost = Dict([source_data[i,3] => source_data[i,10] for i in 2:size(source_data,1)])

#transportation cost per tonne CO2 per km = .18 from APEN paper; .01 from the review survey
#unit_transport_cost_dict = Dict()
#unit_transport_cost_dict["Inland"] = 0.01

transport_cost_offshore_dict = Dict([[cost_data[i,1],cost_data[i,2]] => cost_data[i,4] for i in 2:size(cost_data,1)])

unit_utilization_price_dict = Dict()
unit_utilization_price_dict["EOR"] = 5

#capture_cost = 46 # $/ton CO2
storage_cost = 15 # $/ton CO2
oil_per_ton_CO2 = 0.25 # ton of oils/ton of CO2
Oil_price = 328.5 # $/ton of oil = 45 $ per barrel. 1 ton = 7.3 barrels
CO2_price = 15 # $/ton
cost_sharing_factor = 0.7

max_stars_repo_name: CUHK-IITR-EnergyTransition/CCUS

max_issues_repo_name: CUHK-IITR-EnergyTransition/CCUS

max_forks_repo_name: CUHK-IITR-EnergyTransition/CCUS


text:
## methods for core
core_object_methods = (:as_poly, :atoms,
                       :compare, :compare_pretty,
                       :doit, :dummy_eq, # :count,
                       :has,
                       :sort_key,
                       :args_cnc,
                       :as_coeff_Add, :as_coeff_Mul, :as_coeff_add,
                       :as_coeff_exponent, :as_coeff_factors, :as_coeff_mul,
                       :as_coeff_terms, :as_coefficient, :as_coefficient_dict,
                       :as_content_primitive, :as_expr, :as_independent,
                       :as_leading_term, :as_numer_denom, :as_ordered_factors,
                       :as_ordered_terms, :as_powers_dicts, :as_real_imag,
                       :as_terms,
                       :coeff, :compute_leading_term, :could_extract_minus_sign,
                       :equals,  :extract_additively,
                       :extract_branch_factor, :extract_multiplicatively,
                       :getO, :getn,
                       ## :is_integer called as x[:is_integer]
                       :is_constant, :is_polynomial, :is_rational_function,
                       :integer_nthroot,
                       :leadterm,
                       :primitive,
                       :removeO
                       )

core_object_properties = (:assumptions0,
                          :is_even, :is_odd,
                          :is_number, :is_integer, :is_real, :is_complex, :is_rational,
                          :is_commutative
                          )

## From relational
core_sympy_methods = (:sympify, :flatten,
                      :Dummy,
                      :Mod, :Rel,
                      :PoleError,
                      :count_ops,
                      :gcdex, :half_gcdex,
                      :igcd, :ilcm
                      )

"""
Extract left and right hand side of a relation, parts of a relation.

(These are properties in SymPy, functions in SymPy.jl)

Examples:
```
x = Sym("x")
Eq(x, sin(x)) |> rhs  ## sin(x)
```
"""
rhs(ex::Sym, args...; kwargs...) = PyObject(ex)[:rhs]
lhs(ex::Sym, args...; kwargs...) = PyObject(ex)[:lhs]

"""
Return a vector of free symbols in an expression
"""
function free_symbols(ex::Sym)
    fs = PyObject(ex)[:free_symbols]
    ## are these a set?
    if fs[:__class__][:__name__] == "set"
        convert(Vector{Sym}, collect(fs))
    else
        Sym[]
    end
end
function free_symbols{T<:SymbolicObject}(exs::Vector{T})
    as = map(free_symbols, exs)
    out = as[1]
    if length(as) > 1
        for j in 2:length(as)
            for u in as[j]
                u in out || push!(out, u)
            end
        end
    end
    out
end
function free_symbols(exs::Tuple)
    as = map(free_symbols, exs)
    out = as[1]
    if length(as) > 1
        for j in 2:length(as)
            for u in as[j]
                u in out || push!(out, u)
            end
        end
    end
    out
end
export free_symbols


text:
struct OutcomeSample
    ϵ::Float32
    OutcomeSample(x=0.60) = new(x)
end

function (outcome_traverse::OutcomeSample)(sol::DeepCFRSolver, h, p, t, π_i=1.0f0, π_ni=1.0f0, s=1.0f0)
    game = sol.game
    current_player = player(game, h)
    ϵ = outcome_traverse.ϵ

    if isterminal(game, h)
        return Float32(utility(game, p, h)/s) , 1.0f0

    elseif iszero(current_player)
        a = chance_action(game, h)
        h′ = next_hist(game, h, a)
        return outcome_traverse(sol, h′, p, t, π_i, π_ni, s)

    elseif current_player == p
        I = vectorized(game, infokey(game, h)) # info KEY here, not infostate
        A = actions(game, h)
        σ = regret_match_strategy(sol, I, p)

        σ′ = ϵ*inv(length(A)) .+ (1-ϵ) .* σ # TODO: can probably get around allocation here
        a_idx = weighted_sample(σ′)
        a = A[a_idx]
        h′ = next_hist(game, h, a)
        u, π_tail = outcome_traverse(sol, h′, p, t, π_i*σ[a_idx], π_ni, s*σ′[a_idx])

        W = u*π_ni
        r̃ = Vector{Float32}(undef, length(A))
        for (k, a′) in enumerate(A)
            r̃[k] = if k == a_idx
                W*π_tail*(1 - σ[a_idx])
            else
                -W*σ[a_idx]
            end
        end

        push!(sol.Mv[p], I,t,r̃)

        return u, π_tail*σ[a_idx]
    else
        I = vectorized(game, infokey(game, h)) # info KEY here, not infostate
        A = actions(game, h)
        σ = regret_match_strategy(sol, I, current_player)

        a_idx = weighted_sample(σ)
        a = A[a_idx]
        h′ = next_hist(game, h, a)
        u, π_tail = outcome_traverse(sol, h′, p, t, π_i, π_ni*σ[a_idx], s*σ[a_idx])

        push!(sol.Mπ, I,t,σ)
        return u, π_tail*σ[a_idx]
    end
end


text:
function compute_p(x::Vector{Float64}, omega::Vector{Float64})::Float64
  z = sum(x .* omega)
  return MCMC.sigmoid(z)
end

function compute_p(x::Matrix{Float64}, omega::Vector{Float64})
  z = x * omega
  return MCMC.sigmoid.(z)
end


text:
mutable struct TrainingContext
    system::ParsingSystem
    settings::Settings
    train_connlu_file::String
    connlu_sentences::Vector{ConnluSentence}
    test_connlu_sentences::Vector{ConnluSentence}
    test_connlu_file::String
    training_results_file::String
    model_file::String
    optimizer
    best_uas
    best_las
    best_loss
    test_dataset
    beam_coef

    TrainingContext(
        system::ParsingSystem,
        settings::Settings,
        train_connlu_file::String,
        connlu_sentences::Vector{ConnluSentence},
        test_connlu_sentences::Vector{ConnluSentence},
        test_connlu_file::String,
        training_results_file::String,
        model_file::String;
        beam_coef = 0.5
    ) =
    new(
        system,
        settings,
        train_connlu_file,
        connlu_sentences,
        test_connlu_sentences,
        test_connlu_file,
        training_results_file,
        model_file,
        undef,
        0,
        0,
        Inf,
        undef,
        beam_coef
    )
end

max_stars_repo_name: LarsWl/DependencyParser

max_issues_repo_name: LarsWl/DependencyParser

max_forks_repo_name: LarsWl/DependencyParser


text:
export read_con_headers

"""
read_con_headers(con::SeisCon, keys::Array{String,1}, blocks::Array{Int,1};
                                prealloc_traces::Int = 50000)

Read `keys` from `blocks` in `con`. Skips all data and only reads and returns headers.

# Examples

h = read_con_headers(s, ["SourceX"; "SourceY"], Array(1:11))
"""
function read_con_headers(con::SeisCon, keys::Array{String,1}, blocks::Array{Int,1};
                                prealloc_traces::Int = 50000)
    nblocks = length(blocks)

    # Check dsf
    datatype = Float32
    if con.dsf == 1
        datatype = IBMFloat32
    elseif con.dsf != 5
        @error "Data type not supported ($(fh.bfh.DataSampleFormat))"
    end

    # Check for RecSrcScalar
    in("RecSourceScalar", keys) ? nothing : push!(keys, "RecSourceScalar")
    in("ElevationScalar", keys) ? nothing : push!(keys, "ElevationScalar")

    # Pre-allocate
    headers = Array{BinaryTraceHeader,1}(undef, prealloc_traces)
    fh = FileHeader(); set_fileheader!(fh.bfh, :ns, con.ns)
    set_fileheader!(fh.bfh, :DataSampleFormat, con.dsf)

    # Read all blocks
    trace_count = 0
    for block in blocks

        # Check size of next block and pass view to pre-alloc
        brange = con.blocks[block].endbyte - con.blocks[block].startbyte
        ntraces = Int((brange)/(240 + con.ns*4))

        # Check if there is room in pre-alloc'd mem
        isroom = (trace_count + ntraces) <= length(headers)
        if ~isroom
            println("Expanding preallocated memory")
            prealloc_traces *= 2
            prealloc_traces += ntraces
            append!(headers, Array{BinaryTraceHeader,1}(undef, ntraces+prealloc_traces))
        end
        tmp_headers = view(headers, (trace_count+1):(trace_count+ntraces))

        # Read the next block
        read_block_headers!(con.blocks[block], keys, con.ns, con.dsf, tmp_headers)
        trace_count += ntraces
    end

    return SeisBlock{datatype}(fh, headers[1:trace_count], Array{datatype,2}(undef,0,0))

end

function read_con_headers(con::SeisCon, blocks::Array{Int,1};
                                prealloc_traces::Int = 50000)
    nblocks = length(blocks)

    # Check dsf
    datatype = Float32
    if con.dsf == 1
        datatype = IBMFloat32
    elseif con.dsf != 5
        @error "Data type not supported ($(fh.bfh.DataSampleFormat))"
    end

    # Pre-allocate
    headers = Array{BinaryTraceHeader,1}(undef, prealloc_traces)
    fh = FileHeader(); set_fileheader!(fh.bfh, :ns, con.ns)
    set_fileheader!(fh.bfh, :DataSampleFormat, con.dsf)

    # Read all blocks
    trace_count = 0
    for block in blocks

        # Check size of next block and pass view to pre-alloc
        brange = con.blocks[block].endbyte - con.blocks[block].startbyte
        ntraces = Int((brange)/(240 + con.ns*4))

        # Check if there is room in pre-alloc'd mem
        isroom = (trace_count + ntraces) <= length(headers)
        if ~isroom
            println("Expanding preallocated memory")
            prealloc_traces *= 2
            prealloc_traces += ntraces
            append!(headers, Array{BinaryTraceHeader,1}(undef, ntraces+prealloc_traces))
        end
        tmp_headers = view(headers, (trace_count+1):(trace_count+ntraces))

        # Read the next block
        read_block_headers!(con.blocks[block], con.ns, con.dsf, tmp_headers)
        trace_count += ntraces
    end

    return SeisBlock{datatype}(fh, headers[1:trace_count], Array{datatype,2}(undef,0,0))

end

function read_con_headers(con::SeisCon, blocks::TR;
                          prealloc_traces::Int = 50000) where {TR<:AbstractRange}
    read_con_headers(con, Array(blocks), prealloc_traces = prealloc_traces)
end
function read_con_headers(con::SeisCon, blocks::Integer;
                                prealloc_traces::Int = 50000)
    read_con_headers(con, [blocks], prealloc_traces = prealloc_traces)
end
function read_con_headers(con::SeisCon, keys::Array{String,1}, blocks::TR;
                          prealloc_traces::Int = 50000) where {TR<:AbstractRange}
    read_con_headers(con, keys, Array(blocks), prealloc_traces = prealloc_traces)
end
function read_con_headers(con::SeisCon, keys::Array{String,1}, blocks::Integer;
                                prealloc_traces::Int = 50000)
    read_con_headers(con, keys, [blocks], prealloc_traces = prealloc_traces)
end


text:
using Metatheory
using Metatheory.EGraphs
using TermInterface
using Test

struct MyExpr
  head::Any
  # NOTE! this will not work, when replacing
  # with z in the theory defined below, the arg type
  # will be EGraphs.EClass! Additional manipulation
  # is needed for custom term types with stricter arg types
  # args::Vector{Union{Int, MyExpr}}
  args::Vector{Any}
  # additional metadata
  foo::String
  bar::Vector{Complex}
  baz::Set{Int}
end

import Base.(==)
(==)(a::MyExpr, b::MyExpr) = a.head == b.head && a.args == b.args && a.foo == b.foo && a.bar == b.bar && a.baz == b.baz

MyExpr(head, args) = MyExpr(head, args, "", Complex[], Set{Int}())
MyExpr(head) = MyExpr(head, [])

# Methods needed by `src/TermInterface.jl`
TermInterface.exprhead(e::MyExpr) = :call
TermInterface.operation(e::MyExpr) = e.head
TermInterface.arguments(e::MyExpr) = e.args
TermInterface.istree(e::MyExpr) = true
# NamedTuple
TermInterface.metadata(e::MyExpr) = (foo = e.foo, bar = e.bar, baz = e.baz)
EGraphs.preprocess(e::MyExpr) = MyExpr(e.head, e.args, uppercase(e.foo), e.bar, e.baz)

# f(g(2), h(4)) with some metadata in h
hcall = MyExpr(:h, [4], "hello", [2 + 3im, 4 + 2im], Set{Int}([4, 5, 6]))
ex = MyExpr(:f, [MyExpr(:g, [2]), hcall])

function TermInterface.similarterm(x::MyExpr, head, args; metadata = nothing, exprhead = :call)
  meta = isnothing(metadata) ? ("", Complex[], Set{Int64}()) : metadata
  MyExpr(head, args, meta...)
end

function EGraphs.egraph_reconstruct_expression(
  T::Type{MyExpr},
  op,
  args;
  metadata = ("", Complex[], Set{Int64}()),
  exprhead = nothing,
)
  meta = isnothing(metadata) ? ("", Complex[], Set{Int64}()) : metadata
  MyExpr(op, args, meta...)
end

# let's create an egraph
g = EGraph(ex; keepmeta = true)

# ========== !!! ============= !!! ===============
# ========== !!! ============= !!! ===============
# ========== !!! ============= !!! ===============

settermtype!(g, :f, 2, MyExpr)
settermtype!(g, :f, 1, MyExpr)
settermtype!(g, :g, 1, MyExpr)

# ========== !!! ============= !!! ===============
# ========== !!! ============= !!! ===============
# ========== !!! ============= !!! ===============

# let's create an example theory
t = @theory a begin
  # this way, z will be a regular expr
  # f(g(2), a) => z(a)
  # we can use dynamic rules to construct values of type MyExpr
  # f(g(2), a) |> MyExpr(:z, [a])

  # terms in the RHS inherit the type of terms in the lhs
  f(g(2), a) --> f(a)
end

saturate!(g, t)

# TODO test metadata

expected =
  MyExpr(:f, [MyExpr(:h, [4], "HELLO", Complex[2 + 3im, 4 + 2im], Set([5, 4, 6]))], "", Complex[], Set{Int64}())

extracted = extract!(g, astsize)

@test expected == extracted


text:
using IteratedProcessSimulations
using Test
using DataFrames
using DataFrameMacros

using Soss
using MeasureTheory
using StatsModels
using Chain

import Distributions: Binomial
import GLM: glm, LogitLink, fit, coef, predict, @formula

@testset "IteratedProcessSimulations.jl" begin
    test_dgp = @model params begin
        income_eur ~ Normal(params[:mu_income], 0.1)
        individual_attributes ~ Normal(1, 0.1)
        default_prob = logistic(individual_attributes + income_eur)
        default ~ Bernoulli(default_prob)
    end

    n_epochs = 3

    test_simulation_description = DataFrame(
        "n_datapoints" => [200, 40, 30],
        "mu_income" => fill(1, n_epochs),
        "epoch" => 0:(n_epochs-1),
    )

    epoch_parameters = eachrow(test_simulation_description)[1]

    test_data = generate_data(test_dgp, epoch_parameters)
    @test nrow(test_data) == 200
    @test ncol(test_data) == 8

    epoch_parameters = eachrow(test_simulation_description)[2]
    test_data_1 = generate_data(test_dgp, epoch_parameters)
    @test nrow(test_data_1) == 40
    @test ncol(test_data_1) == 8

    test_data_2 = generate_data(test_dgp, test_simulation_description)
    @test nrow(test_data_2) == 270
    @test ncol(test_data_2) == 8

    function fit_model(epoch_parameters::DataFrameRow, training_data::DataFrame, new_data::DataFrame)
        # Drop unobserved outcomes
        training_data = @chain training_data @subset(:observed == true)

        # TODO: add example where model is dynamic, chosen via simulation parameters!
        glm(@formula(default ~ 1 + income_eur), training_data, Binomial(), LogitLink())
    end

    test_data = @chain test_data @transform(:observed = true)
    m = fit_model(epoch_parameters, test_data, test_data_1)
    # @test coef(m) ≈ [0.18764107801174243, 0.06135920993917277]

    function transform_data(df::DataFrame)
        return df
    end

    function summarize_model(epoch_parameters::DataFrameRow, model::StatsModels.TableRegressionModel, simulation_data::DataFrame, new_data::DataFrame)
        DataFrame(:epoch => [epoch_parameters.epoch], :income_coef => coef(values(model))[1])
    end

    # @test summarize_model(epoch_parameters, m, test_data, test_data)[1, :income_coef] ≈ 0.019

    function choose_observations(epoch_parameters::DataFrameRow, model::Union{StatsModels.TableRegressionModel, Nothing}, new_data::DataFrame, simulation_data::DataFrame)
        # Select 'unobserved' datapoints

        # All 'historical' data points are considered observed
        if epoch_parameters.epoch < 1
            new_data = @chain new_data begin
                @transform(:observed = true)
            end
        else
            new_data[!, :predicted_labels] = predict(model, new_data)

            new_data = @chain new_data begin
                @transform(:observed = true)
            end
        end

        # Add new data to dataset
        append!(simulation_data, new_data, promote=true)

        return simulation_data
    end

    df = choose_observations(epoch_parameters, m, test_data_1, test_data_1)
    @test nrow(df) == 80
    @test (@chain df @combine(sum(:observed)) _[!, :observed_sum][1]) == 40

    ips = IteratedProcessSimulation(test_dgp, test_simulation_description, transform_data, fit_model, summarize_model, choose_observations)
    @test isa(ips, IteratedProcessSimulation)

    sim_data, model_summary, model_objects = run_simulation(ips)

    @test ncol(sim_data) == 9
    @test nrow(sim_data) == 270

    sim_data, model_summary, model_objects = run_simulation(ips, 5)

    @test ncol(sim_data) == 9
    @test nrow(sim_data) == 1350
end