text:
#=
Copyright (c) 2016, Intel Corporation
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
- Redistributions of source code must retain the above copyright notice,
  this list of conditions and the following disclaimer.
- Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
THE POSSIBILITY OF SUCH DAMAGE.
=#

module CaptureAPI

using CompilerTools
using CompilerTools.AstWalker
import ..API
import HPAT
using HPAT.Partitioning
using HPAT.SEQ
using HPAT.TWO_D
using HPAT.ONE_D

import CompilerTools.DebugMsg
DebugMsg.init()
#using Debug

""" At macro level, translate DataSource into function calls so that type inference
    and ParallelAccelerator compilation works with knowledge of calls and allocations for arrays.
"""
function process_node(node::Expr, state, top_level_number, is_top_level, read)
    @dprintln(3,"translating expr, head: ",node.head," node: ",node)
    # rename symbols before translation
    AstWalk(node, rename_symbols,  state.rename_map)

    if node.head == :(=)
        return process_assignment(node, state, node.args[1], node.args[2])
    elseif node.head==:call && node.args[1]==:DataSink
        return translate_data_sink(node.args[2], state, node.args[3], node.args[4:end])
    elseif node.head==:ref
        t1 = node.args[1]
        # table column ref like: t1[:c1]
        # TODO: column assignment like t1[:c2] =... should be processed in assignment
        if haskey(state.tableCols,t1)
            c1 = node.args[2]
            @assert isQuote(c1) " $node = $c1 invalid table ref"
            return getColName(t1, getQuoteValue(c1))
        end
        CompilerTools.AstWalker.ASTWALK_RECURSE
    elseif node.head==:macrocall
        return process_macros(node, state,node.args[1])
    end
    CompilerTools.AstWalker.ASTWALK_RECURSE
end

function process_node(node::Any, state, top_level_number, is_top_level, read)
    CompilerTools.AstWalker.ASTWALK_RECURSE
end

function process_assignment(node, state, lhs::Symbol, rhs::Expr)
    @dprintln(3,"assignment: ", lhs)

    if rhs.head ==:call && rhs.args[1]==:DataSource
        @dprintln(3,"datasource: ", lhs)
        arr_var_expr = rhs.args[2]
        @assert arr_var_expr.head==:curly "curly syntax expected for DataSource"
        if arr_var_expr.args[1]==:DataTable
            return translate_data_table(lhs, state, arr_var_expr, rhs.args[3], rhs.args[4:end])
        else
            return translate_data_source(lhs, state, arr_var_expr, rhs.args[3], rhs.args[4:end])
        end
    elseif rhs.head==:call && rhs.args[1]==:join
        return translate_join(lhs, rhs, state)
    elseif rhs.head==:call && rhs.args[1]==:aggregate
        return translate_aggregate(lhs,rhs,state)
    elseif rhs.head==:vcat
        return translate_table_vcat(lhs, rhs, state)
    elseif rhs.head==:ref
        t1 = rhs.args[1]
        # table filter like t1 = t1[:c1>=2]
        if haskey(state.tableCols, t1)
            return translate_filter(lhs, t1, rhs.args[2], state)
        end
        # set type of output of DAAL calls
    elseif rhs.head==:call && (rhs.args[1]==:Kmeans || rhs.args[1]==:LinearRegression || rhs.args[1]==:NaiveBayes)
        in_matrix = rhs.args[2]
        node.args[1] = :($lhs::typeof($in_matrix))
    elseif rhs.head==:call && rhs.args[1]==:stencil
        return translate_stencil(lhs, rhs, state)
    end
    CompilerTools.AstWalker.ASTWALK_RECURSE
end

function process_assignment(node, state, lhs::ANY, rhs::ANY)
    CompilerTools.AstWalker.ASTWALK_RECURSE
end

""" desugar stencil() and construct runStencl call, example:
      avg = stencil(x->(x[-1]+2*x[0]+x[1])/4.0, t1[:val2])
        :runStencil
        :((b,a)->begin  # REPL[5], line 5:
            b[0] = (a[-1] + 2*a[0] + a[1])/4.0 # REPL[5], line 6:
            return (b,a)
        end)
        :avg
        :t1_val2
        1
        :(:oob_skip)
 format: do_node(in_tuple, do_block)  out in 1 :(:oob_skip)
"""
function translate_stencil(lhs::Symbol, rhs::Expr, state)
    @dprintln(3, "translating stencil: ", lhs," = ",rhs)

    in_lambda = rhs.args[2]
    @assert in_lambda.head==:-> "invalid stencil call $rhs"
    # t1[:val2] to t1_val2
    in_arr = process_node(rhs.args[3], state, 0, false, true)

    in_sym = in_lambda.args[1]
    in_expr = in_lambda.args[2].args[end]
    var_tuple = Expr(:tuple, :b, in_sym)
    stencil_expr = Expr(:(=), Expr(:ref, :b, 0), in_expr)
    do_block = Expr(:block, stencil_expr, Expr(:return, var_tuple))
    do_node = Expr(:->, var_tuple, do_block)

    run_stencil_call = Expr(:call, :runStencil, do_node, lhs, in_arr, 1, Expr(:quote, :oob_skip))
    alloc_assign = Expr(:(=), lhs, :(Array(eltype($in_arr), size($in_arr, 1))) )
    ret = Expr(:block, alloc_assign, run_stencil_call)
    @dprintln(3, "translating stencil returns: ", ret)
    return ret
end

function translate_table_vcat(t_out, rhs::Expr, state)
    @assert rhs.head==:vcat "invalid vcat node"
    t1 = rhs.args[1]
    t2 = rhs.args[2]
    # ignore if not a table vcat
    if !haskey(state.tableCols, t1) && !haskey(state.tableCols, t2)
        return CompilerTools.AstWalker.ASTWALK_RECURSE
    end
    # if one arg is table, both should be table
    @assert haskey(state.tableCols, t1) && haskey(state.tableCols, t2) "both vcat inputs should be tables"
    @assert Set(state.tableCols[t1])==Set(state.tableCols[t2]) "vcat inputs should have same schema"

    if t_out==t1 || t_out==t2
      id = get_unique_id(state)
      new_t_out = Symbol("$(t_out)_v_$(id)")
      @dprintln(3, "renaming vcat output table: ", t_out, " -> ", new_t_out)
      # symbol will be renamed for future AST nodes
      add_symbol_rename(t_out, new_t_out, state)
      t_out = new_t_out
    end

    # Adding new output table to state.tableCols
    state.tableCols[t_out] = state.tableCols[t1]
    state.tableTypes[t_out] = state.tableTypes[t1]
    #t_in_id = getTableId(t_in,state)
    t_out_id = get_unique_id(state)
    state.tableIds[t_out_id] = t_out
    res = []
    for c in state.tableCols[t1]
        t1_c_arr = getColName(t1,c)
        t2_c_arr = getColName(t2,c)
        t_out_c_arr = getColName(t_out,c)
        #push!(res, :($t_out_c_arr = [$t1_c_arr; $t2_c_arr]))
        push!(res, :($t_out_c_arr = vcat($t1_c_arr, $t2_c_arr)))
    end
    ret = Expr(:block, res...)
    @dprintln(3,"vcat returns: ", ret)
    return ret
end

function process_macros(node, state, func)
    if func==Symbol("@partitioned")
        state.array_partitioning[node.args[2]] = convert_partitioning(node.args[3])
        return CompilerTools.AstWalker.ASTWALK_REMOVE
    end
    return CompilerTools.AstWalker.ASTWALK_RECURSE
end

function convert_partitioning(p::Symbol)
    if p==:HPAT_2D
        return TWO_D
    elseif p==:HPAT_1D
        return ONE_D
    elseif p==:HPAT_SEQ
        return SEQ
    else
        error("unknown partitioning $p")
        return SEQ
    end
    return SEQ
end

""" Translate filter out_t = t1[cond]

    We create an array of arrays to pass the columns to table_filter since
    arrays are passed by value with tuples.
                _t1_cond_e = cond
                _filter_t1 = Array(Vector,n)
                _filter_t1[1] = _t1_c1
                ...
                (HPAT.API.table_filter!)(_t1_e,_filter_t1)
                _t1_c1 = _filter_t1[1]
                ...
    """

function translate_filter(t_out::Symbol, t_in::Symbol, cond::Expr, state)
    @dprintln(3, "translating filter: ",t_in," ",cond," -> ", t_out)

    # @assert t_out!=t_in "Output table of filter must have different name"
    # rename output if it is the same name
    if t_out==t_in
      id = get_unique_id(state)
      new_t_out = Symbol("$(t_out)_f_$(id)")
      @dprintln(3, "renaming filter output table: ", t_out, " -> ", new_t_out)
      # symbol will be renamed for future AST nodes
      add_symbol_rename(t_out, new_t_out, state)
      t_out = new_t_out
    end

    # Adding new output table to state.tableCols
    state.tableCols[t_out] = state.tableCols[t_in]
    state.tableTypes[t_out] = state.tableTypes[t_in]
    t_in_id = getTableId(t_in,state)
    t_out_id = get_unique_id(state)
    state.tableIds[t_out_id] = t_out

    # convert math operations to element-wise versions to work with arrays
    cond = AstWalk(cond, convert_oprs_to_elementwise,  (t_in, state.tableCols[t_in]))
    # replace column name with actual array in expression
    cond = AstWalk(cond, replace_col_with_array,  (t_in, state.tableCols[t_in]))
    # evaluate the condition into a BitArray
    cond_arr = Symbol("@$(t_in)@cond_e")
    cond_assign = :( $cond_arr = $cond )

    t_in_num_cols = length(state.tableCols[t_in])
    arg_arr_in = Symbol("_filter_in_$t_in")
    t_in_col_arr = :($arg_arr_in = Array(Vector,$(t_in_num_cols)))

    # Output table has same number of columns
    arg_arr_out = Symbol("_filter_out_$t_out")
    #t_out_col_arr = :($arg_arr_out = Array(Vector,$(t_in_num_cols)))
    # assign column arrays
    # e.g. t1[1] = _t1_c1
    t_in_col_arrs = map(x->getColName(t_in,x), state.tableCols[t_in])
    # Output table has same column names as in input table
    t_out_col_arrs = map(x->getColName(t_out,x), state.tableCols[t_out])

    assigns_in = [ Expr(:(=),:($arg_arr_in[$i]),:($(t_in_col_arrs[i]))) for i in 1:length(state.tableCols[t_in]) ]
    assigns_out = [ Expr(:(=), :($(t_out_col_arrs[i])::Vector{$(state.tableTypes[t_out][i])}) ,:($arg_arr_out[$i])) for i in 1:length(state.tableCols[t_out]) ]
    #assigns_out = [ Expr(:(=), :($(t_out_col_arrs[i])) ,:($arg_arr_out[$i])) for i in 1:length(state.tableCols[t_out]) ]

    mod_call = GlobalRef(HPAT.API, :table_filter!)
    filter_call = :( $arg_arr_out = ($mod_call)($t_in_id,$t_out_id,$cond_arr,($arg_arr_in)) )
    ret = Expr(:block, cond_assign, t_in_col_arr, assigns_in..., filter_call, assigns_out...)
    @dprintln(3,"filter returns: ", ret)
    return ret
end

"""
    get table id from table name
"""
function getTableId(t_in::Symbol, state)
    for (id,t) in state.tableIds
        if t==t_in
            return id
        end
    end
    return -1
end

"""
    Basic join will match first column of each column array
      t3 = join(t1, t2, :c1==:c1, :c2)
                    ->  t3_c1, t3_c2,... = join([t1_c1,t1_c2,...], [t2_c1,t2_c2,...])
                        assertEqShape(t3_c1, t3_c2,...)
                        newTableMeta(:t3, [:c1,:c2,...])
    """
function translate_join(lhs, rhs, state)
    @dprintln(3,"join: ", lhs)
    # 1st and 2nd args are tables to join
    t1 = rhs.args[2]
    t2 = rhs.args[3]

    # 0.5 uses (call == a b) for comparison
    if VERSION > v"0.5.0-d"
        @assert rhs.args[4].head==:call "invalid join key head"
        @assert rhs.args[4].args[1]==:(==) "invalid join key"
        # get key columns
        key1 = getQuoteValue(rhs.args[4].args[2])
        key2 = getQuoteValue(rhs.args[4].args[3])
    else
        # 0.4 uses (comparison a == b) for comparison
        @assert rhs.args[4].head==:comparison "invalid join key head"
        @assert rhs.args[4].args[2]==:(==) "invalid join key"
        # get key columns
        key1 = getQuoteValue(rhs.args[4].args[1])
        key2 = getQuoteValue(rhs.args[4].args[3])
    end

    key1_arr = getColName(t1, key1)
    key2_arr = getColName(t2, key2)

    new_key = getQuoteValue(rhs.args[5])
    new_key_arr = getColName(lhs, new_key)

    # Make key column first column and then append to it
    t1_col_arrs_sorted = [key1_arr]
    t2_col_arrs_sorted = [key2_arr]
    # get rest of the columns
    rest_cols1 = filter(x->x!=key1, state.tableCols[t1])
    rest_cols2 = filter(x->x!=key2, state.tableCols[t2])
    rest_cols1_arrs = map(x->getColName(t1,x),rest_cols1)
    rest_cols2_arrs = map(x->getColName(t2,x),rest_cols2)
    append!(t1_col_arrs_sorted, rest_cols1_arrs)
    append!(t2_col_arrs_sorted, rest_cols2_arrs)
    rest_cols3_arrs = map(x->getColName(lhs,x),[rest_cols1;rest_cols2])
    # save new table
    state.tableCols[lhs] = [new_key;rest_cols1;rest_cols2]
    t_out_id = get_unique_id(state)
    state.tableIds[t_out_id] = lhs
    t1_id = getTableId(t1,state)
    t2_id = getTableId(t2,state)

    #@assert key1==t1_col_arrs_sorted[1] "Join key $key1 of table $t1 is not column 1"
    #@assert key2==t2_col_arrs_sorted[1] "Join key $key2 of table $t2 is not column 1"
    @dprintln(3, "new table join output: ",lhs," ", state.tableCols[lhs])
    # pass tables as array of columns since [t1_c1,t1_c2...] flattens to single array instead of array of arrays
    # eg. t1 = Array(Vector,n)

    _join_t1 = Symbol("_join_$t1")
    _join_t2 = Symbol("_join_$t2")
    t1_num_cols = length(state.tableCols[t1])
    t1_col_arr = :($_join_t1 = Array(Vector,$(t1_num_cols)))
    t2_num_cols = length(state.tableCols[t2])
    t2_col_arr = :($_join_t2 = Array(Vector,$(t2_num_cols)))
    # assign column arrays
    # e.g. t1[1] = _t1_c1
    assign1 = [ Expr(:(=),:($_join_t1[$i]), t1_col_arrs_sorted[i]) for i in 1:length(t1_col_arrs_sorted) ]
    assign2 = [ Expr(:(=),:($_join_t2[$i]), t2_col_arrs_sorted[i]) for i in 1:length(t2_col_arrs_sorted) ]

    #out = [t1_col_arr;t2_col_arr]
    # TODO: assign types
    #ret = :( ($new_key_arr,$(rest_cols3_arrs...)) = HPAT.API.join([$key1_arr;$(rest_cols1_arrs...)], [$key2_arr;$(rest_cols2_arrs...)]) )
    # GlobalRef since Julia doesn't resolve the module! why does GlobalRef work in surface AST??
    g_call = GlobalRef(HPAT.API,:join)
    _j_out = Symbol("_join_out_$lhs")
    join_call = :( $_j_out = $(g_call)($t_out_id,$t1_id,$t2_id,$_join_t1, $_join_t2) )

    col_types = [ get_column_type(state, t1, key1) ]
    col_types1 = [ get_column_type(state, t1, i) for i in rest_cols1]
    col_types2 = [ get_column_type(state, t2, i) for i in rest_cols2]
    col_types = [col_types;col_types1;col_types2]
    # save new table types
    state.tableTypes[lhs] = col_types

    typ_assigns = [ :($new_key_arr::Vector{$(col_types[1])} = $_j_out[1]) ]
    typ_assigns1 = [ :($(rest_cols3_arrs[i])::Vector{$(col_types[i+1])} = $_j_out[$(i+1)]) for i in 1:length(rest_cols3_arrs)]
    typ_assigns = [typ_assigns;typ_assigns1]

    ret = Expr(:block,t1_col_arr,assign1...,t2_col_arr,assign2...,join_call,typ_assigns...)
    @dprintln(3,"join returns: ",ret)
    return ret
end

# Return type for given table and column
function get_column_type(state, t, col)
    index_col = findfirst(state.tableCols[t], col)
    return state.tableTypes[t][index_col]
end

"""
    example: t4 = aggregate(t3, :userid, :sumo2 = sum(:val2==1.1), :size_val3 = size(:val3))

    f is a reduction function on grouped data, e is closure for filtering column elements
     t2 = aggregate(t1, :c1, :c3=f(e(:c2,...)),...)
                    ->  t2_c3_e = e(t1_c2,...)
                        ...
                        t2_c1, t2_c3,... = aggregate(t1_c1, (t2_c3_e,f),...)
                        assertEqShape(t3_c1, t3_c3,...)
                        newTableMeta(:t3, [:c1,:c3,...])

    """
function translate_aggregate(lhs, rhs, state)
    @dprintln(3,"aggregate: ", lhs)
    t1 = rhs.args[2]
    c1, c1_out = get_aggregate_key_names(rhs.args[3])
    c1_arr = getColName(t1, c1)
    c1_out_arr = getColName(lhs, c1_out)
    out_e = []
    out_dummies = []
    out_aggs = []
    out_arrs = [c1_out_arr]
    out_cols = [c1_out]

    for col_expr in rhs.args[4:end]
        @assert col_expr.head==:kw "expected assignment for new aggregate column"
        # output column name
        out_col = getQuoteValue(col_expr.args[1])
        out_col_arr = getColName(lhs, out_col)
        push!(out_cols, out_col)
        push!(out_arrs, out_col_arr)
        out_e_arr = Symbol("@$(lhs)@$(out_col)@e")
        @assert col_expr.args[2].head==:call "expected aggregation function"
        # aggregation function
        func = col_expr.args[2].args[1]
        # aggregation expression
        e = col_expr.args[2].args[2]
        # to add types to aggregate output
        # make a dummy call to get the type with user's function
        # then use it in type assertion. Julia type inference can infer the type and use it for common functions
        # example: t2_c1, t2_c2 = aggregate(t1_c1, (t1_c2_e,sum))
        #          _T_c2 = typeof(sum(t1_c2_e))
        #          t2_c2::_T_c2 = t2_c2
        typ_name = Symbol("_T_$(out_col)")
        dummy_reduce = :( $typ_name = typeof($(func)($out_e_arr)) )
        push!(out_dummies, dummy_reduce)

        # handle length(unique(e))
        if func==:length && e.head==:call && e.args[1]==:unique
            func = :length_unique
            e = e.args[2]
        end
        # convert math operations to element-wise versions to work with arrays
        e = AstWalk(e, convert_oprs_to_elementwise,  (t1, state.tableCols[t1]))
        # replace column name with actual array in expression
        e = AstWalk(e, replace_col_with_array,  (t1, state.tableCols[t1]))

        push!(out_aggs, :(($out_e_arr, $func)))
        push!(out_e,:($out_e_arr=$e))

        # typ_assigns = [ :($new_key_arr::Vector{$(col_types[1])} = _j_out[1]) ]

    end
    append!(out_e,out_dummies)

    out_var = Symbol("_agg_out_$(lhs)_@_$(t1)")
    t_in_id = getTableId(t1,state)
    t_out_id = get_unique_id(state)
    state.tableIds[t_out_id] = lhs

    # assign types of output columns
    # we know the type of key column already
    out_type_assigns = [ :($c1_out_arr::Vector{$(state.tableTypes[t1][1])} = $(out_var)[1]) ]
    # create type variables for other types
    out_typs = [ Symbol("_T_$(out_cols[i])") for i in 1:length(out_cols) ]
    out_typs[1] = state.tableTypes[t1][1]
    out_type_assigns1 = [ :($(out_arrs[i])::Vector{$(out_typs[i])} = $(out_var)[$i])  for i in 2:length(out_arrs) ]
    out_type_assigns = [out_type_assigns;out_type_assigns1]

    # GlobalRef since Julia doesn't resolve the module! why does GlobalRef work in surface AST??
    agg_call = GlobalRef(HPAT.API,:aggregate)

    out_call = Expr(:(=), out_var, :($(agg_call)($t_in_id,$t_out_id,$c1_arr,[$(out_aggs...)])) )
    push!(out_e, out_call)
    push!(out_e, out_type_assigns...)
    state.tableCols[lhs] = out_cols
    state.tableTypes[lhs] = out_typs
    # TODO: save new table types
    #ret = quote $(out_e...) end
    ret = Expr(:block, out_e...)
    @dprintln(3,"aggregate returns: ",ret)
    return ret
end

function get_aggregate_key_names(key_expr)
    c1 = :null
    c1_out = :null
    if isQuote(key_expr)
        c1 = getQuoteValue(key_expr)
        c1_out = c1
    else
        @assert key_expr.head==:kw "expected assignment for aggregate key"
        c1_out = getQuoteValue(key_expr.args[1])
        c1 = getQuoteValue(key_expr.args[2])
    end
    return c1, c1_out
end

agg_oprs_map = Dict{Symbol, Symbol}(
                                    # comparison operators
                                    :(>) => :(.>),
                                    :(<) => :(.<),
                                    :(<=) => :(.<=),
                                    :(>=) => :(.>=),
                                    :(==) => :(.==),

                                    # binary operators
                                    :(+) => :(.+),
                                    :(-) => :(.-),
                                    :(*) => :(.*),
                                    :(/) => :(./),
                                    :(%) => :(.%),
                                    :(^) => :(.^),
                                    :(<<) => :(.<<),
                                    :(>>) => :(.>>)
                                    )

""" Convert math operations to element-wise versions to work with arrays

    example: user can write 't4 = aggregate(t3, :userid, :sumo2 = sum(:val2==1.1), :size_val3 = size(:val3))'
    the aggregate expression ':val2==1.1' should be translated to '_t4_val2.==1.1' to be valid for arrays.

    For simplicity, we assume we can convert all operations with element-wise versions.
    This is wrong for rare use cases such as comparison of two arrays([1,2]==[1,3] is not the same as [1,2].==[1,3]),
    TODO: make it fully general.
    """
function convert_oprs_to_elementwise(node::Symbol, table::Tuple{Symbol,Vector{Symbol}}, top_level_number, is_top_level, read)
    if haskey(agg_oprs_map,node)
        return agg_oprs_map[node]
    end
    return CompilerTools.AstWalker.ASTWALK_RECURSE
end

function convert_oprs_to_elementwise(node::ANY, table::Tuple{Symbol,Vector{Symbol}}, top_level_number, is_top_level, read)
    return CompilerTools.AstWalker.ASTWALK_RECURSE
end

"""
    Replace column symbols with translated array names in aggregate expressions
    """
function replace_col_with_array(node::QuoteNode, table::Tuple{Symbol,Vector{Symbol}}, top_level_number, is_top_level, read)
    col_sym = getQuoteValue(node)
    if col_sym in table[2]
        return getColName(table[1], col_sym)
    end
    return CompilerTools.AstWalker.ASTWALK_RECURSE
end

function replace_col_with_array(node::Expr, table::Tuple{Symbol,Vector{Symbol}}, top_level_number, is_top_level, read)
    if node.head!=:quote
        return CompilerTools.AstWalker.ASTWALK_RECURSE
    end
    col_sym = getQuoteValue(node)
    if col_sym in table[2]
        return getColName(table[1], col_sym)
    end
    return CompilerTools.AstWalker.ASTWALK_RECURSE
end

function replace_col_with_array(node::ANY, table::Tuple{Symbol,Vector{Symbol}}, top_level_number, is_top_level, read)
    return CompilerTools.AstWalker.ASTWALK_RECURSE
end

"""
    rename symbols of the node according to rename_map
"""
function rename_symbols(node::Symbol, rename_map::Dict{Symbol,Symbol}, top_level_number, is_top_level, read)
    new_sym = node
    while haskey(rename_map, new_sym)
      new_sym = rename_map[new_sym]
    end
    return new_sym
end

function rename_symbols(node::ANY, rename_map::Dict{Symbol,Symbol}, top_level_number, is_top_level, read)
    return CompilerTools.AstWalker.ASTWALK_RECURSE
end

function translate_data_sink(var, state, source_typ, other_args)
    @assert source_typ==:HDF5 || source_typ==:TXT "Only HDF5 and TXT (text) data sources supported for now."

    # desugar call
    call_name = Symbol("data_sink_$source_typ")
    # GlobalRef since Julia doesn't resolve the module!
    api_call = GlobalRef(HPAT.API, call_name)
    rhs = Expr(:call, api_call, var, other_args...)
    return rhs
end

function translate_data_source(lhs, state, arr_var_expr, source_typ, other_args)
    @assert arr_var_expr.args[1]==:Array || arr_var_expr.args[1]==:Matrix || arr_var_expr.args[1]==:Vector "Data sources need Vector or Array or Matrix as type"
    @assert source_typ==:HDF5 || source_typ==:TXT "Only HDF5 and TXT (text) data sources supported for now."

    # desugar call
    call_name = Symbol("data_source_$source_typ")
    # GlobalRef since Julia doesn't resolve the module!
    api_call = GlobalRef(HPAT.API, call_name)
    rhs = Expr(:call, api_call, arr_var_expr, other_args...)
    return Expr(:(=), lhs, rhs)
end

""" Data tables are broken down to individual column arrays, table meta data is saved
    table_name = DataSource(DataTable{:column1=<typeof_column1>, :column2=<typeof_column2>, ...}, HDF5, file_name)
                    ->  table_name_column1 = DataSource(...)
                        table_name_column2 = DataSource(...)
                        assertEqShape([table_name_column1, table_name_column2])
                        newTableMeta(:table_name, [:column1,:column2])

    Example table to translate:
            t1 = DataSource(DataTable{:userid = Int64,:val2 = Float64},HDF5,file_name)
    """
function translate_data_table(lhs, state, arr_var_expr, source_typ, other_args)
    @assert arr_var_expr.args[1]==:DataTable "expected :DataTable"
    # arr_var_expr has the form: :(DataTable{:userid = Int64,:val2 = Float64})
    @dprintln(3,"translating data table: ",arr_var_expr)
    out = []
    col_names = Symbol[]
    col_types = Symbol[]
    for column in arr_var_expr.args[2:end]
        @dprintln(3,"table column: ", column)
        @assert column.head==:(=)
        col_name = getQuoteValue(column.args[1])
        push!(col_names, col_name)
        col_type = column.args[2]
        push!(col_types,col_type)
        col_lhs = getColName(lhs,col_name)
        col_source = translate_data_source(col_lhs, state, :(Vector{$(col_type)}), source_typ, ["/"*string(col_name);other_args])
        push!(out, col_source)
    end
    # save table info in state
    state.tableCols[lhs] = col_names
    state.tableTypes[lhs] = col_types
    t_out_id = get_unique_id(state)
    state.tableIds[t_out_id] = lhs
    ret = quote $(out...) end
    @dprintln(3, "data table returns: ",ret)
    return ret
end

function isQuote(e::Expr)
    return e.head==:quote
end

function isQuote(e::QuoteNode)
    return true
end

function isQuote(e::ANY)
    return false
end

"""
    Julia sometimes returns a QuoteNode for :column1
"""
function getQuoteValue(exp::QuoteNode)
    return exp.value
end

"""
    Julia sometimes returns an Expr for :column1
"""
function getQuoteValue(exp::Expr)
    @assert exp.head==:quote "expected :quote expression"
    return exp.args[1]
end

"""
    Convert a table column to an array name
"""
function getColName(t::Symbol, c::Symbol)
    return Symbol("@$(t)@$(c)")
end

"""
    reverse of getColName()
    get column name from array name
"""
function revColName(t::Symbol, c_arr::Symbol)
    t_str = string(t)
    c_arr_str = string(c_arr)
    before_col_len = 2+length(t_str) # two underscores + length of table name
    return c_arr_str[before_col_len+1:end]
end

"""
    get a unique id for renaming
"""
function get_unique_id(state)
  state.unique_id += 1
  return state.unique_id
end

"""
    add symbol to rename map, to be applied from next AST node
"""
function add_symbol_rename(t::Symbol, new_t::Symbol, state)
  state.rename_map[t] = new_t
end

end # module


text:
export dbc_icons

dbc_icons = (
    BOOTSTRAP = "https://cdn.jsdelivr.net/npm/bootstrap-icons@1.5.0/font/bootstrap-icons.css",
    FONT_AWESOME = "https://use.fontawesome.com/releases/v5.15.4/css/all.css",
)

ext: jl

max_stars_repo_name: glsdown/dash-bootstrap-components

max_issues_repo_name: glsdown/dash-bootstrap-components

max_forks_repo_name: glsdown/dash-bootstrap-components


text:
# A form is something that ends up as geometry in the graphic.

abstract FormPrimitive

immutable Form{P <: FormPrimitive} <: ComposeNode
    primitives::Vector{P}
end

function isempty(f::Form)
    return isempty(f.primitives)
end

function isscalar(f::Form)
    return length(f.primitives) == 1
end

function draw{P}(backend::Backend, t::Transform, units::UnitBox,
                 box::AbsoluteBoundingBox, form::Form{P})
    draw(backend, Form(P[absolute_units(primitive, t, units, box)
                         for primitive in form.primitives]))
end

# Polygon
# -------

immutable PolygonPrimitive <: FormPrimitive
    points::Vector{Point}
end

typealias Polygon Form{PolygonPrimitive}

function polygon()
    return PolygonPrimitive([PolygonPrimitive(Point[])])
end

function polygon{T <: XYTupleOrPoint}(points::AbstractArray{T})
    return Polygon([PolygonPrimitive([convert(Point, point)
                                      for point in points])])
end

function polygon(point_arrays::AbstractArray)
    polyprims = Array(PolygonPrimitive, length(point_arrays))
    for (i, point_array) in enumerate(point_arrays)
        polyprims[i] = PolygonPrimitive([convert(Point, point)
                                         for point in point_array])
    end
    return Polygon(polyprims)
end

function absolute_units(p::PolygonPrimitive, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return PolygonPrimitive(Point[absolute_units(point, t, units, box)
                                  for point in p.points])
end

# Rectangle
# ---------

immutable RectanglePrimitive <: FormPrimitive
    corner::Point
    width::Measure
    height::Measure
end

typealias Rectangle Form{RectanglePrimitive}

function rectangle()
    return Rectangle([RectanglePrimitive(Point(0.0w, 0.0h), 1.0w, 1.0h)])
end

function rectangle(x0, y0, width, height)
    corner = Point(x0, y0)
    width = x_measure(width)
    height = y_measure(height)
    return Rectangle([RectanglePrimitive(corner, width, height)])
end

function rectangle(x0s::AbstractArray, y0s::AbstractArray,
                   widths::AbstractArray, heights::AbstractArray)
    return Rectangle([RectanglePrimitive(
                         Point(x0, y0),
                         x_measure(width), y_measure(height))
                      for (x0, y0, width, height) in cyclezip(x0s, y0s, widths, heights)])
end

function absolute_units(p::RectanglePrimitive, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    # SVG doesn't support negative width/height to indicate flipped axis,
    # so we have to adjust manually.
    corner = absolute_units(p.corner, t, units, box)
    width = absolute_units(p.width, t, units, box)
    height = absolute_units(p.height, t, units, box)

    return RectanglePrimitive(
        Point(Measure(abs=width < 0 ? corner.x.abs + width : corner.x.abs),
              Measure(abs=height < 0 ? corner.y.abs + height : corner.y.abs)),
        Measure(abs=abs(width)),
        Measure(abs=abs(height)))
end

# Circle
# ------

immutable CirclePrimitive <: FormPrimitive
    center::Point
    radius::Measure
end

typealias Circle Form{CirclePrimitive}

function circle()
    return Circle([CirclePrimitive(Point(0.5w, 0.5h), 0.5w)])
end

function circle(x, y, r)
    return Circle([CirclePrimitive(Point(x, y), x_measure(r))])
end

function circle(xs::AbstractArray, ys::AbstractArray, rs::AbstractArray)
    return Circle([CirclePrimitive(Point(x, y), x_measure(r))
                   for (x, y, r) in cyclezip(xs, ys, rs)])
end

function absolute_units(p::CirclePrimitive, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return CirclePrimitive(absolute_units(p.center, t, units, box),
                           Measure(abs=absolute_units(p.radius, t, units, box)))
end

# Ellipse
# -------

immutable EllipsePrimitive <: FormPrimitive
    center::Point
    x_point::Point
    y_point::Point
end

typealias Ellipse Form{EllipsePrimitive}

function ellipse()
    return Ellipse([EllipsePrimitive(Point(0.5w, 0.5h),
                                     Point(1.0w, 0.5h),
                                     Point(0.5w, 1.0h))])
end

function ellipse(x, y, x_radius, y_radius)
    return Ellipse([EllipsePrimitive(Point(x, y),
                                     Point(x + x_measure(x_radius), y),
                                     Point(x, y + y_measure(y_radius)))])
end

function ellipse(xs::AbstractArray, ys::AbstractArray,
                 x_radiuses::AbstractArray, y_radiuses::AbstractArray)
    return Ellipse[EllipsePrimitive(Point(x, y),
                                    Point(x_measure(x) + x_measure(x_radius), y),
                                    Point(x, y_measure(y) + y_measure(y_radius)))
                   for (x, y, x_radius, y_radius) in cyclezip(xs, ys, x_radiuses, y_radiuses)]
end

function absolute_units(p::EllipsePrimitive, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return EllipsePrimitive(absolute_units(p.center, t, units, box),
                            absolute_units(p.x_point, t, units, box),
                            absolute_units(p.y_point, t, units, box))
end

# Text
# ----

abstract HAlignment
immutable HLeft   <: HAlignment end
immutable HCenter <: HAlignment end
immutable HRight  <: HAlignment end

const hleft   = HLeft()
const hcenter = HCenter()
const hright  = HRight()

abstract VAlignment
immutable VTop    <: VAlignment end
immutable VCenter <: VAlignment end
immutable VBottom <: VAlignment end

const vtop    = VTop()
const vcenter = VCenter()
const vbottom = VBottom()

immutable TextPrimitive <: FormPrimitive
    position::Point
    value::String
    halign::HAlignment
    valign::VAlignment

    # Text forms need their own rotation field unfortunately, since there is no
    # way to give orientation with just a position point.
    rot::Rotation
end

typealias Text Form{TextPrimitive}

function text(x, y, value::String,
              halign::HAlignment=hleft, valign::VAlignment=vbottom,
              rot=Rotation())
    return Text([TextPrimitive(Point(x, y), value, halign, valign, rot)])
end

function text(x, y, value,
              halign::HAlignment=hleft, valign::VAlignment=vbottom,
              rot=Rotation())
    return Text([TextPrimitive(Point(x, y), string(value), halign, valign, rot)])
end

function text(xs::AbstractArray, ys::AbstractArray, values::AbstractArray{String},
              haligns::AbstractArray=[hleft], valigns::AbstractArray=[vbottom],
              rots::AbstractArray=[Rotation()])
    return Text([TextPrimitive(Point(x, y), value, halign, valign, rot)
                 for (x, y, value, halign, valign, rot) in cyclezip(xs, ys, values, haligns, valigns, rots)])
end

function text(xs::AbstractArray, ys::AbstractArray, values::AbstractArray,
              haligns::AbstractArray=[hleft], valigns::AbstractArray=[vbottom],
              rots::AbstractArray=[Rotation()])
    return Text([TextPrimitive(Point(x, y), string(value), halign, valign, rot)
                 for (x, y, value, halign, valign, rot) in cyclezip(xs, ys, values, haligns, valigns, rots)])
end

function absolute_units(p::TextPrimitive, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return TextPrimitive(absolute_units(p.position, t, units, box),
                         p.value, p.halign, p.valign,
                         absolute_units(p.rot, t, units, box))
end

# Line
# ----

immutable LinePrimitive <: FormPrimitive
    points::Vector{Point}
end

typealias Line Form{LinePrimitive}

function line()
    return Line([LinePrimitive(Point[])])
end

function line{T <: XYTupleOrPoint}(points::AbstractArray{T})
    return Line([LinePrimitive([convert(Point, point) for point in points])])
end

function line(point_arrays::AbstractArray)
    lineprims = Array(LinePrimitive, length(point_arrays))
    for (i, point_array) in enumerate(point_arrays)
        lineprims[i] = LinePrimitive([convert(Point, point)
                                      for point in point_array])
    end
    return Line(lineprims)
end

function absolute_units(p::LinePrimitive, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return LinePrimitive([absolute_units(point, t, units, box) for point in p.points])
end

# Curve
# -----

immutable CurvePrimitive <: FormPrimitive
    anchor0::Point
    ctrl0::Point
    ctrl1::Point
    anchor1::Point
end

typealias Curve Form{CurvePrimitive}

function curve(anchor0::XYTupleOrPoint, ctrl0::XYTupleOrPoint,
               ctrl1::XYTupleOrPoint, anchor1::XYTupleOrPoint)
    return Curve([CurvePrimitive(convert(Point, ancho0), convert(Point, ctrl0),
                                 convert(Point, ctrl1), convert(Point, anchor1))])
end

function curve(anchor0s::AbstractArray, ctrl0s::AbstractArray,
               ctrl1s::AbstractArray, anchor1s::AbstractArray)
    return Curve([CurvePrimitive(convert(Point, anchor0), convert(Point, ctrl0),
                                 convert(Point, ctrl1), convert(Point, anchor1))
                  for (anchor0, ctrl0, ctrl1, anchor1) in cyclezip(anchor0s, ctrl0s, ctrl1s, anchor1s)])
end

function absolute_units(p::CurvePrimitive, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return CurvePrimitive(absolute_units(p.anchor0, t, units, box),
                          absolute_units(p.ctrl0, t, units, box),
                          absolute_units(p.ctrl1, t, units, box),
                          absolute_units(p.anchor1, t, units, box))
end

# Bitmap
# ------

immutable BitmapPrimitive <: FormPrimitive
    mime::String
    data::Vector{Uint8}
    corner::Point
    width::Measure
    height::Measure
end

typealias Bitmap Form{BitmapPrimitive}

function bitmap(mime::String, data::Vector{Uint8}, x0, y0, width, height)
    corner = Point(x0, y0)
    width = x_measure(width)
    height = y_measure(height)
    return Bitmap([BitmapPrimitive(mime, data, corner, width, height)])
end

function bitmap(mimes::AbstractArray, datas::AbstractArray,
                x0s::AbstractArray, y0s::AbstractArray,
                widths::AbstractArray, heights::AbstractArray)
    return Bitmap([BitmapPrimitive(mime, data, x0, y0, x_measure(width), y_measure(height))
                   for (mime, data, x0, y0, width, height) in cyclezip(mimes, datas, x0s, y0s, widths, heights)])
end

function absolute_units(p::BitmapPrimitive, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return BitmapPrimitive(p.mime, p.data,
                           absolute_units(p.corner, t, units, box),
                           Measure(abs=absolute_units(p.width, t, units, box)),
                           Measure(abs=absolute_units(p.width, t, units, box)))
end

# Path
# ----

# An implementation of the SVG path mini-language.

abstract PathOp

immutable MoveAbsPathOp <: PathOp
    to::Point
end

function assert_pathop_tokens_len(op_type, tokens, i, needed)
    provided = length(tokens) - i + 1
    if provided < needed
        error("In path $(op_type) requires $(needed) argumens but only $(provided) provided.")
    end
end

function parsepathop(::Type{MoveAbsPathOp}, tokens::AbstractArray, i)
    assert_pathop_tokens_len(MoveAbsPathOp, tokens, i, 2)
    op = MoveAbsPathOp(Point(tokens[i], tokens[i + 1]))
    return (op, i + 2)
end

function absolute_units(p::MoveAbsPathOp, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return MoveAbsPathOp(absolute_units(p.to, t, units, box))
end

immutable MoveRelPathOp <: PathOp
    to::Point
end

function parsepathop(::Type{MoveRelPathOp}, tokens::AbstractArray, i)
    assert_pathop_tokens_len(MoveRelPathOp, tokens, i, 2)
    op = MoveRelPathOp(Point(tokens[i], tokens[i + 1]))
    return (op, i + 2)
end

function absolute_units(p::MoveRelPathOp, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return MoveRelPathOp(Point(Measure(abs=absolute_units(p.to.x, t, units, box)),
                               Measure(abs=absolute_units(p.to.y, t, units, box))))
end

immutable ClosePathOp <: PathOp
end

function parsepathop(::Type{ClosePathOp}, tokens::AbstractArray, i)
    return (ClosePathOp(), i)
end

function absolute_units(p::ClosePathOp, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return p
end

immutable LineAbsPathOp <: PathOp
    to::Point
end

function parsepathop(::Type{LineAbsPathOp}, tokens::AbstractArray, i)
    assert_pathop_tokens_len(LineAbsPathOp, tokens, i, 2)
    op = LineAbsPathOp(Point(tokens[i], tokens[i + 1]))
    return (op, i + 2)
end

function absolute_units(p::LineAbsPathOp, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return LineAbsPathOp(absolute_units(p.to, t, units, box))
end

immutable LineRelPathOp <: PathOp
    to::Point
end

function parsepathop(::Type{LineRelPathOp}, tokens::AbstractArray, i)
    assert_pathop_tokens_len(LineRelPathOp, tokens, i, 2)
    op = LineRelPathOp(Point(tokens[i], tokens[i + 1]))
    return (op, i + 2)
end

function absolute_units(p::LineRelPathOp, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return LineRelPathOp(Point(Measure(abs=absolute_units(p.to.x, t, units, box)),
                               Measure(abs=absolute_units(p.to.y, t, units, box))))
end

immutable HorLineAbsPathOp <: PathOp
    x::Measure
end

function parsepathop(::Type{HorLineAbsPathOp}, tokens::AbstractArray, i)
    assert_pathop_tokens_len(HorLineAbsPathOp, tokens, i, 1)
    op = HorLineAbsPathOp(x_measure(tokens[i]))
    return (op, i + 1)
end

function absolute_units(p::HorLineAbsPathOp, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return HorLineAbsPathOp(Measure(abs=absolute_x_position(p.x, t, units, box)))
end

immutable HorLineRelPathOp <: PathOp
    Δx::Measure
end

function parsepathop(::Type{HorLineRelPathOp}, tokens::AbstractArray, i)
    assert_pathop_tokens_len(HorLineRelPathOp, tokens, i, 1)
    op = HorLineRelPathOp(x_measure(tokens[i]))
    return (op, i + 1)
end

function absolute_units(p::HorLineRelPathOp, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return HorLineRelPathOp(Measure(abs=absolute_units(p.Δx, t, units, box)))
end

immutable VertLineAbsPathOp <: PathOp
    y::Measure
end

function parsepathop(::Type{VertLineAbsPathOp}, tokens::AbstractArray, i)
    assert_pathop_tokens_len(VertLineAbsPathOp, tokens, i, 1)
    op = VertLineAbsPathOp(y_measure(tokens[i]))
    return (op, i + 1)
end

function absolute_units(p::VertLineAbsPathOp, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return VertLineAbsPathOp(Measure(abs=absolute_y_position(p.x, t, units, box)))
end

immutable VertLineRelPathOp <: PathOp
    Δy::Measure
end

function parsepathop(::Type{VertLineRelPathOp}, tokens::AbstractArray, i)
    assert_pathop_tokens_len(VertLineRelPathOp, tokens, i, 1)
    op = VertLineRelPathOp(y_measure(tokens[i]))
    return (op, i + 1)
end

function absolute_units(p::VertLineRelPathOp, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return VertLineRelPathOp(Measure(abs=absolute_units(p.Δy, t, units, box)))
end

immutable CubicCurveAbsPathOp <: PathOp
    ctrl1::Point
    ctrl2::Point
    to::Point
end

function parsepathop(::Type{CubicCurveAbsPathOp}, tokens::AbstractArray, i)
    assert_pathop_tokens_len(CubicCurveAbsPathOp, tokens, i, 6)
    op = CubicCurveAbsPathOp(Point(tokens[i],     tokens[i + 1]),
                             Point(tokens[i + 2], tokens[i + 3]),
                             Point(tokens[i + 4], tokens[i + 5]))
    return (op, i + 6)
end

function absolute_units(p::CubicCurveAbsPathOp, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return CubicCurveAbsPathOp(
            absolute_units(p.ctrl1, t, units, box),
            absolute_units(p.ctrl2, t, units, box),
            absolute_units(p.to, t, units, box))
end

immutable CubicCurveRelPathOp <: PathOp
    ctrl1::Point
    ctrl2::Point
    to::Point
end

function parsepathop(::Type{CubicCurveRelPathOp}, tokens::AbstractArray, i)
    assert_pathop_tokens_len(CubicCurveRelPathOp, tokens, i, 6)
    op = CubicCurveRelPathOp(Point(tokens[i],     tokens[i + 1]),
                             Point(tokens[i + 2], tokens[i + 3]),
                             Point(tokens[i + 4], tokens[i + 5]))
    return (op, i + 6)
end

function absolute_units(p::CubicCurveRelPathOp, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return CubicCurveRelPathOp(
            Point(Measure(abs=absolute_units(p.ctrl1.x, t, units, box)),
                  Measure(abs=absolute_units(p.ctrl1.y, t, units, box))),
            Point(Measure(abs=absolute_units(p.ctrl2.x, t, units, box)),
                  Measure(abs=absolute_units(p.ctrl2.y, t, units, box))),
            Point(Measure(abs=absolute_units(p.to.x, t, units, box)),
                  Measure(abs=absolute_units(p.to.y, t, units, box))))
end

immutable QuadCurveAbsPathOp <: PathOp
    ctrl1::Point
    to::Point
end

function parsepathop(::Type{QuadCurveAbsPathOp}, tokens::AbstractArray, i)
    assert_pathop_tokens_len(QuadCurveAbsPathOp, tokens, i, 4)
    op = QuadCurveAbsPathOp(Point(tokens[i],     tokens[i + 1]),
                            Point(tokens[i + 2], tokens[i + 3]))
    return (op, i + 4)
end

function absolute_units(p::QuadCurveAbsPathOp, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return QuadCurveAbsPathOp(
            absolute_units(p.ctrl1, t, units, box),
            absolute_units(p.to, t, units, box))
end

immutable QuadCurveRelPathOp <: PathOp
    ctrl1::Point
    to::Point
end

function parsepathop(::Type{QuadCurveRelPathOp}, tokens::AbstractArray, i)
    assert_pathop_tokens_len(QuadCurveRelPathOp, tokens, i, 4)
    op = QuadCurveRelPathOp(Point(tokens[i],     tokens[i + 1]),
                            Point(tokens[i + 2], tokens[i + 3]))
    return (op, i + 4)
end

function absolute_units(p::QuadCurveRelPathOp, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return QuadCurveRelPathOp(
            Point(Measure(abs=absolute_units(p.ctrl1.x, t, units, box)),
                  Measure(abs=absolute_units(p.ctrl1.y, t, units, box))),
            Point(Measure(abs=absolute_units(p.to.x, t, units, box)),
                  Measure(abs=absolute_units(p.to.y, t, units, box))))
end

immutable QuadCurveShortAbsPathOp <: PathOp
    to::Point
end

function parsepathop(::Type{QuadCurveShortAbsPathOp}, tokens::AbstractArray, i)
    assert_pathop_tokens_len(QuadCurveShortAbsPathOp, tokens, i, 2)
    op = QuadCurveShortAbsPathOp(Point(tokens[i], tokens[i + 1]))
    return (op, i + 2)
end

function absolute_units(p::QuadCurveShortAbsPathOp, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return QuadCurveShortAbsPathOp(
            absolute_units(p.to, t, units, box))
end

immutable QuadCurveShortRelPathOp <: PathOp
    to::Point
end

function parsepathop(::Type{QuadCurveShortRelPathOp}, tokens::AbstractArray, i)
    assert_pathop_tokens_len(QuadCurveShortRelPathOp, tokens, i, 2)
    op = QuadCurveShortRelPathOp(Point(tokens[i], tokens[i + 1]))
    return (op, i + 2)
end

function absolute_units(p::QuadCurveRelPathOp, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return QuadCurveRelPathOp(
            Point(Measure(abs=absolute_units(p.to.x, t, units, box)),
                  Measure(abs=absolute_units(p.to.y, t, units, box))))
end

immutable ArcAbsPathOp <: PathOp
    rx::Measure
    ry::Measure
    rotation::Float64
    largearc::Bool
    sweep::Bool
    to::Point
end

function absolute_units(p::ArcAbsPathOp, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return ArcAbsPathOp(
        Measure(abs=absolute_units(p.rx, t, units, box)),
        Measure(abs=absolute_units(p.ry, t, units, box)),
        p.rotation,
        p.largearc,
        p.sweep,
        absolute_units(p.to, t, units, box))
end

immutable ArcRelPathOp <: PathOp
    rx::Measure
    ry::Measure
    rotation::Float64
    largearc::Bool
    sweep::Bool
    to::Point
end

function parsepathop{T <: Union(ArcAbsPathOp, ArcRelPathOp)}(::Type{T}, tokens::AbstractArray, i)
    assert_pathop_tokens_len(T, tokens, i, 7)

    if isa(tokens[i + 3], Bool)
        largearc = tokens[i + 3]
    elseif tokens[i + 3] == 0
        largearc = false
    elseif tokens[i + 3] == 1
        largearc = true
    else
        error("largearc argument to the arc path operation must be boolean")
    end

    if isa(tokens[i + 4], Bool)
        sweep = tokens[i + 4]
    elseif tokens[i + 4] == 0
        sweep = false
    elseif tokens[i + 4] == 1
        sweep = true
    else
        error("sweep argument to the arc path operation must be boolean")
    end

    if !isa(tokens[i + 2], Number)
        error("path arc operation requires a numerical rotation")
    end

    op = T(x_measure(tokens[i]),
           y_measure(tokens[i + 1]),
           convert(Float64, tokens[i + 2]),
           largearc, sweep,
           Point(tokens[i + 5], tokens[i + 6]))

    return (op, i + 7)
end

function absolute_units(p::ArcRelPathOp, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return ArcRelPathOp(
        Measure(abs=absolute_units(p.rx, t, units, box)),
        Measure(abs=absolute_units(p.ry, t, units, box)),
        p.rotation,
        p.largearc,
        p.sweep,
        Point(Measure(abs=absolute_units(to.x, t, units, box)),
              Measure(abs=absolute_units(to.y, t, units, box))))
end

const path_ops = [
     :M => MoveAbsPathOp,
     :m => MoveRelPathOp,
     :Z => ClosePathOp,
     :z => ClosePathOp,
     :L => LineAbsPathOp,
     :l => LineRelPathOp,
     :H => HorLineAbsPathOp,
     :h => HorLineAbsPathOp,
     :V => VertLineAbsPathOp,
     :v => VertLineRelPathOp,
     :C => CubicCurveAbsPathOp,
     :c => CubicCurveRelPathOp,
     :Q => QuadCurveAbsPathOp,
     :q => QuadCurveRelPathOp,
     :T => QuadCurveShortAbsPathOp,
     :t => QuadCurveShortRelPathOp,
     :A => ArcAbsPathOp,
     :a => ArcRelPathOp
]

# TODO: Actually, I don't think I'll need this.
const path_op_type_string = [
     MoveAbsPathOp           => "M",
     MoveRelPathOp           => "m",
     ClosePathOp             => "Z",
     ClosePathOp             => "z",
     LineAbsPathOp           => "L",
     LineRelPathOp           => "l",
     HorLineAbsPathOp        => "H",
     HorLineAbsPathOp        => "h",
     VertLineAbsPathOp       => "V",
     VertLineRelPathOp       => "v",
     CubicCurveAbsPathOp     => "C",
     CubicCurveRelPathOp     => "c",
     QuadCurveAbsPathOp      => "Q",
     QuadCurveRelPathOp      => "q",
     QuadCurveShortAbsPathOp => "T",
     QuadCurveShortRelPathOp => "t",
     ArcAbsPathOp            => "A",
     ArcRelPathOp            => "a"
]

# A path is an array of symbols, numbers, and measures following SVGs path
# mini-language.
function parsepath(tokens::AbstractArray)
    ops = PathOp[]
    last_op_type = nothing
    i = 1
    while i <= length(tokens)
        tok = tokens[i]
        if isa(tok, Symbol)
            if !haskey(path_ops, tok)
                error("$(tok) is not a valid path operation")
            else
                op_type = path_ops[tok]
                i += 1
                op, i = parsepathop(op_type, tokens, i)
                push!(ops, op)
                last_op_type
            end
        else
            i += 1
            op, i = parsepathop(last_op_type, tokens, i)
        end
    end

    return ops
end

immutable PathPrimitive <: FormPrimitive
    ops::Vector{PathOp}
end

typealias Path Form{PathPrimitive}

function path(tokens::AbstractArray)
    return Path([PathPrimitive(parsepath(tokens))])
end

function path{T <: AbstractArray}(tokens::AbstractArray{T})
    return Path([PathPrimitive(parsepath(ts)) for ts in tokens])
end

function absolute_units(p::PathPrimitive, t::Transform, units::UnitBox,
                        box::AbsoluteBoundingBox)
    return PathPrimitive([absolute_units(op, t, units, box) for op in p.ops])
end


text:
using Random, Test
using TupleLabXD

using Base: tail, front

n = 10

p = randperm(n)
ip = invperm(p)

t = (p...,)

@test @inferred(TupleLabXD.argtail2(1, 2, 3, 4)) == (3, 4)
@test @inferred(TupleLabXD.tail2(t)) == t[3:n]
@test @inferred(TupleLabXD.unsafe_tail(t)) == t[2:n]
@test @inferred(TupleLabXD.unsafe_front(t)) == t[1:n-1]
@test @inferred(TupleLabXD.unsafe_tail(())) == ()
@test @inferred(TupleLabXD.unsafe_front(())) == ()
@test @inferred(TupleLabXD.vcat()) == ()
@test @inferred(TupleLabXD.vcat((1, 2))) == (1, 2)
@test @inferred(TupleLabXD.getindices(t, ())) == ()
@test @inferred(TupleLabXD.getindices(t, (1,2,3))) == t[1:3]

@test @inferred(TupleLabXD.deleteat((1, 2), (1, ))) == (2, )
for i = 1:n
    @test @inferred(TupleLabXD.deleteat(t, i)) == (deleteat!(copy(p), i)...,)
    @test @inferred(TupleLabXD.insertat(t, i, (1,2,3))) == (vcat(p[1:i-1], [1,2,3], p[i+1:n])...,)
end
@test @inferred(TupleLabXD.deleteat((1,2,3,4,5,6), (3,1,5))) == (2,4,6)
@test_throws BoundsError TupleLabXD.deleteat(t, 0)
@test_throws BoundsError TupleLabXD.deleteat(t, n+1)
@test_throws BoundsError TupleLabXD.insertat(t, 0, (1,2,3))
@test_throws BoundsError TupleLabXD.insertat(t, n+1, (1,2,3))
for i = 0:n
    @test @inferred(TupleLabXD.insertafter(t, i, (1,2,3))) == (vcat(p[1:i], [1,2,3], p[i+1:n])...,)
end
@test_throws BoundsError TupleLabXD.insertafter(t, -1, (1,2,3))
@test_throws BoundsError TupleLabXD.insertafter(t, n+1, (1,2,3))
@test @inferred(TupleLabXD.vcat((1,2,3),4,(5,),(),(6,7,8))) == (1,2,3,4,5,6,7,8)
@test @inferred(TupleLabXD.flatten((1,2,3),4,(5,),(),(6,7,8))) == (1,2,3,4,5,6,7,8)
@test @inferred(TupleLabXD.flatten((1,(2,3)),4,(5,),(),((6,),(7,(8,))))) == (1,2,3,4,5,6,7,8)

@test @inferred(TupleLabXD.sum(())) == 0
@test @inferred(TupleLabXD.sum((1, ))) == 1
@test @inferred(TupleLabXD.sum(t)) == sum(t)
@test @inferred(TupleLabXD.cumsum(())) == ()
@test @inferred(TupleLabXD.cumsum((1, ))) == (1, )
@test @inferred(TupleLabXD.cumsum(t)) == (cumsum(p)...,)
@test @inferred(TupleLabXD.prod(())) == 1
@test @inferred(TupleLabXD.prod((2, ))) == 2
@test @inferred(TupleLabXD.prod(t)) == prod(t)
@test @inferred(TupleLabXD.cumprod(())) == ()
@test @inferred(TupleLabXD.cumprod((1, ))) == (1, )
@test @inferred(TupleLabXD.cumprod(t)) == (cumprod(p)...,)

for a in (t, (1, ))
    @test @inferred(TupleLabXD.findmin(a)) == findmin(a)
    @test @inferred(TupleLabXD.findmax(a)) == findmax(a)
    @test @inferred(TupleLabXD.minimum(a)) == minimum(a)
    @test @inferred(TupleLabXD.maximum(a)) == maximum(a)
    @test @inferred(TupleLabXD.argmin(a)) == argmin(a)
    @test @inferred(TupleLabXD.argmax(a)) == argmax(a)
end

@test @inferred(TupleLabXD.sort((1, ))) == (1, )
@test @inferred(TupleLabXD.sort(())) == ()
@test @inferred(TupleLabXD.sort(t; rev = true)) == (sort(p; rev = true)...,)
@test @inferred(TupleLabXD.sort(t; rev = false)) == (sort(p; rev = false)..., )
@test @inferred(TupleLabXD.sortperm(t)) == (sortperm(p)...,)
@test @inferred(TupleLabXD.sortperm(t; rev=true)) == (sortperm(p; rev=true)..., )
@test @inferred(TupleLabXD.invperm(t)) == (ip...,)
@test @inferred(TupleLabXD.isperm(t)) == true
@test @inferred(TupleLabXD.isperm((1,2,1))) == false
@test @inferred(TupleLabXD.permute(t, t)) == (p[p]...,)

@test @inferred(TupleLabXD.vcat()) == ()
@test @inferred(TupleLabXD.diff(())) == ()
@test @inferred(TupleLabXD.diff((1, ))) == ()
@test @inferred(TupleLabXD.diff((1, 2, 3))) == (1, 1)


text:
using JLD2, Plots, Printf, LinearAlgebra
include(pwd() * "/analysis_scripts/" * "post_analysis.jl")

searchdir(path, key) = filter(x -> occursin(key, x), readdir(path))
mesoscale_dir = pwd()
checkpoints = searchdir(mesoscale_dir, "iteration")
tke_list = []
b_list = []
wb_list = []
ub_list = []
vb_list = []
uu_list = []
vv_list = []
ww_list = []

for i in 2:length(checkpoints)
    file = jldopen( mesoscale_dir * "/" * checkpoints[i])
    b = file["tracers"]["b"]["data"][2:end-1, 2:end-1, 2:end-1]
    u = file["velocities"]["u"]["data"][2:end-1, 2:end-1, 2:end-1]
    v = file["velocities"]["v"]["data"][2:end-1, 2:end-1, 2:end-1]
    w = file["velocities"]["w"]["data"][2:end-1, 2:end-1, 2:end-1]
    v = (v[:, 1:end-1, :] + v[:, 2:end, :]) .* 0.5 # put everything on cell centers
    w = (w[:, :, 1:end-1] + w[:,:, 2:end]) .* 0.5  # put everything on cell centers
    close(file)
    n = length(b)
    tke = @. u^2 + v^2 + w^2
    ub = u .* b
    vb = v .* b
    wb = w .* b
    uu = u .* u
    vv = v .* v
    ww = w .* w
    avg_b = sum(b) / n
    avg_tke = sum(tke) / n
    avg_wb = sum(wb) / n
    avg_ub = sum(ub) / n
    avg_vb = sum(vb) / n
    avg_uu = sum(uu) / n
    avg_vv = sum(vv) / n
    avg_ww = sum(ww) / n
    push!(b_list, avg_b)
    push!(tke_list, avg_tke)
    push!(ub_list, avg_ub)
    push!(vb_list, avg_vb)
    push!(wb_list, avg_wb)
    push!(uu_list, avg_uu)
    push!(vv_list, avg_vv)
    push!(ww_list, avg_ww)
    println("the average buoyancy is " * string(avg_b))
    println("the average tke is " * string(avg_tke))
    println("the average wb is " * string(avg_wb))
end

plot(vb_list, label = "vb")
plot!(ub_list, label = "ub")
plot!(wb_list, label = "wb")

##
plot(uu_list[1:end-2], label = "uu")
plot!(vv_list[1:end-2], label = "vv")
plot!(ww_list[1:end-2], label = "ww")

max_stars_repo_name: sandreza/Mesoscale

max_issues_repo_name: sandreza/Mesoscale

max_forks_repo_name: sandreza/Mesoscale


text:
#=  In real life, after students are assigned a school by the DA mechanism, they may receive an admission
    offer from a private school or choose to move out of district, both of which are equivalent to the
    student assigning increased preference to the "outside option" of not attending any of the schools in
    the primary market. This frees up seats, and running a second lottery allows us to offer better
    schools to some of the students while guaranteeing them their first round assignment. An important
    design consideration is which tiebreaking mechanism to use in each round. Feigenbaum et al. (2020)
    recommend using STB in the both rounds but reversing the STB lottery numbers in the second round,
    which minimizes reassignment. (Note that this is not the same as running the DA algorithm in reverse;
    we still use student-proposing DA). An arbitrary permutation of the first-round lottery numbers can
    instead be used in the second round to allow the market designed to optimize for distributional goals.
    The code below offers a sketch of a two-round reassignment mechanism with a reverse lottery.       =#

using DeferredAcceptance, Random

function argsort(vec::AbstractArray{<:Real, 1})::AbstractArray{<:Real, 1}
    return invperm(sortperm(vec))
end

n = 100         # Number of students
m = 70          # Number of schools with unit capacities
γ = m / 10      # Average number of ranks by which outside options improve between rounds.

#=  School m + 1 is the outside option and has unlimited capacity. Note that DA() implements its own
    "reject bin" automatically, so specifying a new outside option is redundant if the students
    uniformly prefer schools in the market to nonassignment. The dynamic-matching case relaxes this
    assumption, allowing that some students actually prefer nonassignment to a subset of the schools;
    we need to encode that preference explicitly.                       =#

students = hcat((randperm(m + 1) for i in 1:n)...)
schools = ones(Int64, n, m + 1)
capacities = [ones(Int64, m)..., n]

# Break ties using STB (which is equiv. to HTB with blend 0) and store the lottery numbers
schools_r1, lottery_r1 = hybridtiebreaking(schools, 0., return_add=true)

assn_r1 = deferredacceptance(students, schools_r1, capacities; verbose=true)   # Compute first-round assignments

# Reveal outside options and update student preference lists.
students_float = convert(Array{Float16}, students)              # Rank of student's outside options
students_float[m + 1, :] -= γ * randexp(Float16, n)             # improves by an exponential random var,
students_r2 = mapslices(argsort, students_float, dims=1)        # whose mean is γ.

schools_r2 = copy(schools)                          # Compute second-round school preferences:
lowest_school_rank = minimum(schools)                     # For robustness; in my data this will always be 1
for (s, c) in enumerate(assn_r1[1])                       # Guarantee r1 assignments
    schools_r2[s, c] = lowest_school_rank - 1
end
schools_r2 += (1. .- lottery_r1)                          # Then break ties using reverse lottery numbers
schools_r2 = mapslices(argsort, schools_r2, dims=1)

assn_r2 = deferredacceptance(students_r2, schools_r2, capacities; verbose=true)  # Compute r2 assignments

for i in zip(assn_r1[2], assn_r2[2])        # Compare the assignments. r1 should dominate rankwise.
    println("$(i[1]) ≥ $(i[2])")
end

max_stars_repo_name: maxkapur/DeferredAcceptance

max_issues_repo_name: maxkapur/DeferredAcceptance

max_forks_repo_name: maxkapur/DeferredAcceptance


text:
module CommunalHelperDreamMoveBlock

using ..Ahorn, Maple
using Ahorn.CommunalHelper
using Ahorn.CommunalHelperEntityPresets: CustomDreamBlockData

const entityData = appendkwargs(CustomDreamBlockData, :(
    direction::String="Right",
    moveSpeed::Number=60.0,
    noCollide::Bool=false,
))
@mapdefdata Entity "CommunalHelper/DreamMoveBlock" DreamMoveBlock entityData

const placements = Ahorn.PlacementDict(
    "Dream Move Block ($direction) (Communal Helper)" => Ahorn.EntityPlacement(
        DreamMoveBlock,
        "rectangle",
        Dict{String,Any}(
            "direction" => direction,
        ),
    ) for direction in Maple.move_block_directions
)

Ahorn.editingOptions(entity::DreamMoveBlock) = Dict{String,Any}(
    "direction" => Maple.move_block_directions,
    "moveSpeed" => Dict{String,Number}(
        "Slow" => 60.0,
        "Fast" => 75.0,
    ),
)
Ahorn.minimumSize(entity::DreamMoveBlock) = 16, 16
Ahorn.resizable(entity::DreamMoveBlock) = true, true

Ahorn.selection(entity::DreamMoveBlock) = Ahorn.getEntityRectangle(entity)

const arrows = Dict{String,String}(
    "up" => "objects/CommunalHelper/dreamMoveBlock/arrow02",
    "left" => "objects/CommunalHelper/dreamMoveBlock/arrow04",
    "right" => "objects/CommunalHelper/dreamMoveBlock/arrow00",
    "down" => "objects/CommunalHelper/dreamMoveBlock/arrow06",
)

function Ahorn.render(ctx::Ahorn.Cairo.CairoContext, entity::DreamMoveBlock)
    width = Int(get(entity.data, "width", 8))
    height = Int(get(entity.data, "height", 8))

    renderDreamBlock(ctx, 0, 0, width, height, entity.data)

    direction = lowercase(get(entity.data, "direction", "up"))
    arrowSprite = Ahorn.getSprite(arrows[direction], "Gameplay")
    Ahorn.drawImage(ctx, arrowSprite, div(width - arrowSprite.width, 2), div(height - arrowSprite.height, 2))
end

end


text:
#
# A Derivative mod of the Luxor.jl example strange(). See https://github.com/JuliaGraphics/Luxor.jl
# Rob Baruch, 2018
#

using Luxor, Colors, ProgressMeter

struct StrangeSet
	ctrl_params::Dict{Symbol,Float64}
	dot_size::Float64
	res::Int64
	color_profile::Dict{Symbol,Float64}
	filename::String
	flip::Bool
	cmode::String
end

struct CMode{s} end

function cmode_symbol(::CMode{s}) where s
	     (s ∈ [:rgb, :clamp, :grey, :gray] ? s : throw(error("wrong mode, $s")))

	     if s == :rgb
		     "rgb"
	     elseif s == :clamp
		     "clamp"
	     elseif s == :grey || s == :gray
		     "grey"
	     end
end

function cmode_string(mode)
	if ! in(mode, ["rgb", "clamp", "grey", "gray"])
		throw(error("invalid mode string: $mode"))
	else
		if mode == "rgb"
			:rgb
		elseif mode == "clamp"
			:clamp
		elseif mode == "grey" || mode == "gray"
			:grey
		end
	end
end

function rs()
	(-1.0)^convert(Int64,floor((rand()*1000.0))) % 2
end

function strange(fn, flip, cmode, dotsize, w=800.0,
		 ctrl=Dict([:a=>2.24 + rs()*rand()*0.5, :b=>0.43, :c => -0.65, :d=>-2.43 + rs()*rand()*0.25, :e=> 1.0]),
		 colors=Dict([:r=>.7, :g=>0.5, :b=>.8]),
		 mode=:fill)
           xmin = -2.0; xmax = 2.0; ymin= -2.0; ymax = 2.0
	   Drawing(w, w, fn)
           origin()
           background("white")
           xinc = w/(xmax - xmin)
           yinc = w/(ymax - ymin)
           # control parameters
           x = y = z = 0.0
           wover2 = w/2
	   p = Progress(w*w, 1)
	   count = 0

	   if flip
		f1 = cos
	   	f2 = sin
	   else
	   	f1 = sin
	   	f2 = cos
	   end

	   # crank...
           for j in 1:w
               for i in 1:w

		   count += 1
		   update!(p, count)

		   xx = f1(ctrl[:a] * y) - z  *  f2(ctrl[:b] * x)
		   yy = z * f1(ctrl[:c] * x) - f2(ctrl[:d] * y)
                   zz = ctrl[:e] * f1(x)
                   x = xx; y = yy; z = zz
                   if xx < xmax && xx > xmin && yy < ymax && yy > ymin
                       xpos = rescale(xx, xmin, xmax, -wover2, wover2) # scale to range
                       ypos = rescale(yy, ymin, ymax, -wover2, wover2) # scale to range

		       if cmode_string(cmode) == :rgb
			       rcolor = rescale(xx, -1, 1, 0.0, colors[:r])
			       gcolor = rescale(yy, -1, 1, 0.0, colors[:g])
			       bcolor = rescale(zz, -1, 1, 0.0, colors[:b])
			       setcolor(convert(Colors.HSV, Colors.RGB(rcolor, gcolor, bcolor)))
		       elseif cmode_string(cmode) == :clamp
			       rcolor = rescale(xx, -1, 1, 0.0, colors[:r])
			       gcolor = rescale(yy, -1, 1, 0.0, colors[:g])
			       setcolor(convert(Colors.HSV, Colors.RGB(rcolor, gcolor, gcolor)))
		       elseif cmode_string(cmode) == :grey
			       rcolor = rescale(yy, -1, 1, 0.0, colors[:r])
			       setcolor(convert(Colors.HSV, Colors.RGB(rcolor, rcolor, rcolor)))
		       end
                       circle(Point(xpos, ypos), dotsize, mode)
                   end
               end
           end
	   println("cleaning up...")
           finish()
       end


text:
module Blosc
export compress, compress!, decompress, decompress!

const libblosc = Pkg.dir("Blosc", "deps", "libblosc")

function __init__()
    ccall((:blosc_init,libblosc), Void, ())
    atexit() do
        ccall((:blosc_destroy,libblosc), Void, ())
    end
end

# The following constants should match those in blosc.h
const VERSION_FORMAT = 2
const MAX_OVERHEAD = 16
const DOSHUFFLE = 0x1
const MEMCPYED = 0x2
const MAX_THREADS = 256

# Blosc is currently limited to 32-bit buffer sizes (Blosc/c-blosc#67)
const MAX_BUFFERSIZE = typemax(Cint) - MAX_OVERHEAD
const MAX_TYPESIZE = 255

# low-level functions:
blosc_compress(level, shuffle, itemsize, srcsize, src, dest, destsize) =
    ccall((:blosc_compress,libblosc), Cint,
          (Cint,Cint,Csize_t, Csize_t, Ptr{Void}, Ptr{Void}, Csize_t),
          level, shuffle, itemsize, srcsize, src, dest, destsize)

blosc_decompress(src, dest, destsize) =
    ccall((:blosc_decompress,libblosc), Cint,
          (Ptr{Void},Ptr{Void},Csize_t), src, dest, destsize)

# check whether the strides of A correspond to contiguous data
iscontiguous(::Array) = true
iscontiguous(::Vector) = true
iscontiguous(A::DenseVector) = stride(A,1) == 1
function iscontiguous(A::DenseArray)
    p = sortperm([strides(A)...])
    s = 1
    for k = 1:ndims(A)
        if stride(A,p[k]) != s
            return false
        end
        s *= size(A,p[k])
    end
    return true
end

# Returns the size of compressed data inside dest
function compress!{T}(dest::DenseVector{Uint8},
                      src::Ptr{T},
                      src_size::Integer;
	                  level::Integer=5,
                      shuffle::Bool=true,
                      itemsize::Integer=sizeof(T))
    iscontiguous(dest) || throw(ArgumentError("dest must be contiguous array"))
    if !isbits(T)
        throw(ArgumentError("buffer eltype must be `isbits` type"))
    end
    if itemsize <= 0
        throw(ArgumentError("itemsize must be positive"))
    end
    if level < 0 || level > 9
        throw(ArgumentError("invalid compression level $level not in [0,9]"))
    end
    if src_size > MAX_BUFFERSIZE
        throw(ArgumentError("data > $MAX_BUFFERSIZE bytes is not supported by Blosc"))
    end
    sz = blosc_compress(level, shuffle, itemsize, src_size, src, dest, sizeof(dest))
    sz < 0 && error("Blosc internal error when compressing data (errorcode: $sz)")
    return convert(Int, sz)
end

compress!(dest::DenseVector{Uint8}, src::String; kws...) =
    compress!(dest, pointer(src), sizeof(src); kws...)

function compress!(dest::DenseVector{Uint8}, src::DenseArray; kws...)
    iscontiguous(src) || throw(ArgumentError("src must be a contiguous array"))
    return compress!(dest, pointer(src), sizeof(src); kws...)
end

function compress{T}(src::Ptr{T}, src_size::Integer; kws...)
    dest = Array(Uint8, src_size + MAX_OVERHEAD)
    sz = compress!(dest,src,src_size; kws...)
    assert(sz > 0 || src_size == 0)
    return resize!(dest, sz)
end
function compress(src::DenseArray; kws...)
    iscontiguous(src) || throw(ArgumentError("src must be a contiguous array"))
    compress(pointer(src), sizeof(src); kws...)
end
compress(src::String; kws...) = compress(pointer(src), sizeof(src); kws...)

# given a compressed buffer, return the (uncompressed, compressed, block) size
const _sizes_vals = Array(Csize_t, 3)
function cbuffer_sizes(buf::Ptr)
    ccall((:blosc_cbuffer_sizes,libblosc), Void,
          (Ptr{Void}, Ptr{Csize_t}, Ptr{Csize_t}, Ptr{Csize_t}),
          buf,
          pointer(_sizes_vals, 1),
          pointer(_sizes_vals, 2),
          pointer(_sizes_vals, 3))
    return (_sizes_vals[1], _sizes_vals[2], _sizes_vals[3])
end
sizes(buf::Vector{Uint8}) = cbuffer_sizes(pointer(buf))

function decompress!{T}(dest::DenseVector{T}, src::DenseVector{Uint8})
    if !iscontiguous(dest) || !iscontiguous(src)
        throw(ArgumentError("src and dest must be contiguous arrays"))
    end
    if !isbits(T)
        throw(ArgumentError("dest must be a DenseVector of `isbits` element types"))
    end
    uncompressed, = sizes(src)
    if uncompressed == 0
        return resize!(dest, 0)
    end
    sizeT = sizeof(T)
    len = div(uncompressed, sizeT)
    if len * sizeT != uncompressed
        error("uncompressed data is not a multiple of sizeof($T)")
    end
    resize!(dest, len)
    sz = blosc_decompress(src, dest, sizeof(dest))
    sz == 0 && error("Blosc decompress error, compressed data is corrupted")
    sz <  0 && error("Blosc decompress error, output buffer is not large enough")
    return dest
end
decompress{T}(::Type{T}, src::DenseVector{Uint8}) =
    decompress!(Array(T,0), src)

# Initialize a pool of threads for compression / decompression.
# If `nthreads` is 1, the the serial version is chosen and a possible previous existing pool is ended.
# If this function is not callled, `nthreads` is set to 1 internally.
function set_num_threads(n::Integer=CPU_CORES)
    1 <= n <= MAX_THREADS || throw(ArgumentError("must have 1 ≤ nthreads ≤ $MAX_THREADS"))
    return ccall((:blosc_set_nthreads,libblosc), Cint, (Cint,), n)
end

# Select the compressor to be used.
# Supported ones are "blosclz", "lz4", "lz4hc", "snappy", and "zlib".
# If this function is not called, "blosclz" will be used.
# Throws an ArgumentError if the given compressor is not supported
function set_compressor(s::String)
    compcode = ccall((:blosc_set_compressor,libblosc), Cint, (Ptr{Uint8},), s)
    compcode == -1 && throw(ArgumentError("unrecognized compressor $s"))
    return compcode
end

# Force the use of a specific blocksize
function set_blocksize(blocksize::Integer)
    blocksize >= 0 || throw(ArgumentError("n must be ≥ 0 (default)"))
    ccall((:blosc_set_blocksize,libblosc), Void, (Csize_t,), blocksize)
end

# Allow Blosc to set the optimal blocksize (default)
set_default_blocksize() = set_blocksize(0)

function compression_library(src::DenseVector{Uint8})
    iscontiguous(src) || throw(ArgumentError("src must be a contiguous array"))
    nptr = ccall((:blosc_cbuffer_complib,libblosc), Ptr{Cchar}, (Ptr{Void},), convert(Ptr{Void}, src))
    nptr == convert(Ptr{Cchar}, 0) && error("unknown compression library")
    name = bytestring(nptr)
    return name
end

immutable CompressionInfo
    library::String
    typesize::Int
    pure_memcopy::Bool
    shuffled::Bool
end

# return compressor information for a compressed buffer
function compressor_info(cbuf::DenseVector{Uint8})
    iscontiguous(cbuf) || throw(ArgumentError("cbuf must be contiguous array"))
    flag, typesize = Cint[0], Csize_t[0]
    ccall((:blosc_cbuffer_metainfo, libblosc), Void,
          (Ptr{Void},Ptr{Csize_t},Ptr{Cint}), cbuf, typesize, flag)
    pure_memcopy = bool(flag[1] & MEMCPYED)
    shuffled = bool(flag[1] & DOSHUFFLE)
    return CompressionInfo(compression_library(cbuf),
                           typesize[1],
                           pure_memcopy,
                           shuffled)
end

# list of compression libraries in the Blosc library build (list of strings)
compressors() = split(bytestring(ccall((:blosc_list_compressors, libblosc), Ptr{Cchar}, ())), ',')

# given a compressor in the Blosc library, return (compressor name, library name, version number) tuple
function compressor_info(name::String)
    lib, ver = Array(Ptr{Cchar},1), Array(Ptr{Cchar},1)
    ret = ccall((:blosc_get_complib_info, libblosc), Cint,
                (Ptr{Cchar},Ptr{Ptr{Cchar}},Ptr{Ptr{Cchar}}),
                name, lib, ver)
    ret < 0 && error("Error retrieving compressor info for $name")
    lib_str = bytestring(lib[1]); c_free(lib[1])
    ver_str = bytestring(ver[1]); c_free(ver[1])
    return (name, lib_str, convert(VersionNumber, ver_str))
end

# Get info from compression libraries included in the `Blosc` library build.
# Returns an array of tuples, (library name, version number).
compressors_info() = map(compressor_info, compressors())

# Free possible memory temporaries and thread resources.
# Use this when you are not going to use `Blosc` for a long while.
# In case of problems releasing resources, it returns false, else returns true.
free_resources!() = ccall((:blosc_free_resources,libblosc), Cint, ()) == 0

end # module


text:
# File iteration
Tables.rowaccess(::Type{<:File}) = true
Tables.rows(f::File) = f

Base.@propagate_inbounds function Base.getindex(f::File{false}, row::Int)
    @boundscheck checkbounds(f, row)
    return Row{false}(getnames(f), getcolumns(f), getlookup(f), row, 0, 0)
end

Base.@propagate_inbounds function Base.getindex(f::File{true}, row::Int)
    @boundscheck checkbounds(f, row)
    c = getcolumn(f, 1)
    i = row
    for (j, A) in enumerate(c.args)
        n = length(A)
        i <= n && return Row{true}(getnames(f), getcolumns(f), getlookup(f), row, j, i)
        i -= n
    end
    return Row{true}(getnames(f), getcolumns(f), getlookup(f), row, 1, 1)
end

# non-threaded file
@inline function Base.iterate(f::File{false}, st::Int=1)
    st > length(f) && return nothing
    return Row{false}(getnames(f), getcolumns(f), getlookup(f), st, 0, 0), st + 1
end

# threaded file
mutable struct RowIterationState
    row::Int64
    array_index::Int64
    array_i::Int64
    array_len::Int64
    array_lens::Vector{Int64}
end

@inline function Base.iterate(f::File{true})
    cols = getcols(f)
    (cols == 0 || getrows(f) == 0) && return nothing
    c = getcolumn(f, 1)
    array_lens = [length(x) for x in c.args]
    st = RowIterationState(2, 1, 2, array_lens[1], array_lens)
    return Row{true}(getnames(f), getcolumns(f), getlookup(f), 1, 1, 1), st
end

@inline function Base.iterate(f::File{true}, st)
    row = st.row
    array_index = st.array_index
    array_i = st.array_i
    row > length(f) && return nothing
    if array_i + 1 > st.array_len
        st.array_index += 1
        st.array_i = 1
        st.array_len = st.array_lens[min(end, st.array_index)]
    else
        st.array_i += 1
    end
    st.row += 1
    return Row{true}(getnames(f), getcolumns(f), getlookup(f), row, array_index, array_i), st
end

@noinline badcolumnerror(name) = throw(ArgumentError("`$name` is not a valid column name"))

@inline function Base.getproperty(row::Row{false}, col::Symbol)
    column = getcolumn(row, col)
    @inbounds x = column[getrow(row)]
    return x
end

@inline function Base.getindex(row::Row{false}, col::Int)
    column = getcolumn(row, col)
    @inbounds x = column[getrow(row)]
    return x
end

@inline function Base.getindex(row::Row{false}, col::Symbol)
    column = getcolumn(row, col)
    @inbounds x = column[getrow(row)]
    return x
end

@inline function Base.getproperty(row::Row{true}, col::Symbol)
    column = getcolumn(row, col)
    @inbounds x = column.args[getarrayindex(row)][getarrayi(row)]
    return x
end

@inline function Base.getindex(row::Row{true}, col::Int)
    column = getcolumn(row, col)
    @inbounds x = column.args[getarrayindex(row)][getarrayi(row)]
    return x
end

@inline function Base.getindex(row::Row{true}, col::Symbol)
    column = getcolumn(row, col)
    @inbounds x = column.args[getarrayindex(row)][getarrayi(row)]
    return x
end


text:
#############################################################################
# log.jl
# natural logarithm of an logression
# All expressions and atoms are subtpyes of AbstractExpr.
# Please read expressions.jl first.
#############################################################################

import Base.log
export log
export sign, curvature, monotonicity, evaluate

### Logarithm

mutable struct LogAtom <: AbstractExpr
    head::Symbol
    id_hash::UInt64
    children::Tuple{AbstractExpr}
    size::Tuple{Int, Int}

    function LogAtom(x::AbstractExpr)
        if sign(x)==ComplexSign()
            error("The argument should be real but it's instead complex")
        else
            children = (x,)
            return new(:log, hash(children), children, x.size)
        end
    end
end

function sign(x::LogAtom)
    return NoSign()
end

function monotonicity(x::LogAtom)
    return (Nondecreasing(),)
end

function curvature(x::LogAtom)
    return ConcaveVexity()
end

function evaluate(x::LogAtom)
    return log.(evaluate(x.children[1]))
end

log(x::AbstractExpr) = LogAtom(x)

function conic_form!(e::LogAtom, unique_conic_forms::UniqueConicForms)
    if !has_conic_form(unique_conic_forms, e)
        # log(z) \geq x  <=>    (x,ones(),z) \in ExpCone
        z = e.children[1]
        y = Constant(ones(size(z)))
        x = Variable(size(z))
        objective = conic_form!(x, unique_conic_forms)
        conic_form!(ExpConstraint(x, y, z), unique_conic_forms)

        cache_conic_form!(unique_conic_forms, e, objective)
    end
    return get_conic_form(unique_conic_forms, e)
end


text:
using DrWatson
quickactivate(@__DIR__, "AccuracyAtTopPrimal_oms")

using Experiments
using Experiments: AccuracyAtTopPrimal, DatasetProvider

using DatasetProvider: Ionosphere, Spambase, Gisette, MNIST, FashionMNIST, CIFAR10, CIFAR20, CIFAR100, SVHN2, HEPMASS

# ------------------------------------------------------------------------------------------
# Model settings
# ------------------------------------------------------------------------------------------
τ = [0.01, 0.05]
K = [1, 3, 5, 10, 15, 20]
β = [10, 1, 0.1, 0.01, 0.001, 0.0001]
λ = [0, 1e-1, 1e-2, 1e-3, 1e-4, 1e-5]
surrogate = hinge

models = vcat(
    Model(PatMat; τ, β, surrogate),
    Model(PatMatNP; τ, β, surrogate),
    Model(TopPush; λ),
    Model(TopPushK; K),
    Model(τFPL; τ, λ, surrogate),
    Model(TopMean; τ, λ, surrogate),
    Model(Grill; τ, λ, surrogate),
    Model(GrillNP; τ, λ, surrogate),
)

# ------------------------------------------------------------------------------------------
# Datasets with 10 seeds
# ------------------------------------------------------------------------------------------
datasets = [
    Data(Ionosphere),
    Data(Spambase),
    Data(Gisette; batchsize = 512),
    Data(MNIST; poslabels = 1, batchsize = 512),
    Data(FashionMNIST; poslabels = 1, batchsize = 512),
    Data(CIFAR10; poslabels = 1, batchsize = 512),
    Data(CIFAR20; poslabels = 1, batchsize = 512),
    Data(CIFAR100; poslabels = 1, batchsize = 512),
    Data(SVHN2; poslabels = 1, batchsize = 512),
]

train = [Train(;
    seed = seed,
    iters = 10000,
    saveat = 500,
    optimiser = ADAM,
    step = 0.01,
) for seed in 1:10]

run_simulations(datasets, train, models)

# HEPMASS
train_hep = [Train(;
    seed = seed,
    iters = 10000,
    saveat = 10000,
    optimiser = ADAM,
    step = 0.01,
) for seed in 1:10]

run_simulations(Data(HEPMASS; batchsize = 512), train_hep, models)

# ------------------------------------------------------------------------------------------
# Datasets with 1 seeds
# ------------------------------------------------------------------------------------------
datasets_1 = reduce(vcat, [[
    Data(MNIST; poslabels = l, batchsize = 512),
    Data(FashionMNIST; poslabels = l, batchsize = 512),
    Data(CIFAR10; poslabels = l, batchsize = 512),
    Data(CIFAR20; poslabels = l, batchsize = 512),
    Data(CIFAR100; poslabels = l, batchsize = 512),
    Data(SVHN2; poslabels = l + 1, batchsize = 512),
] for l in 0:9])

train_1 = Train(;
    seed = 1,
    iters = 10000,
    saveat = 500,
    optimiser = ADAM,
    step = 0.01,
)

run_simulations(datasets_1, train_1, models)

# ------------------------------------------------------------------------------------------
# Benchmarks
# ------------------------------------------------------------------------------------------
datasets_bench = [
    Data(Ionosphere),
    Data(Spambase),
    Data(Gisette; batchsize = 512),
    Data(MNIST; poslabels = 1, batchsize = 512),
    Data(FashionMNIST; poslabels = 1, batchsize = 512),
    Data(CIFAR10; poslabels = 1, batchsize = 512),
    Data(SVHN2; poslabels = 1, batchsize = 512),
    Data(HEPMASS; batchsize = 512),
]

table = run_benchmarks(datasets_bench; force = true)


text:
@testset "utils" begin
    @testset "topk_index" begin
        X = [8,7,6,5,4,3,2,1]
        @test topk_index(X, 4) == [1,2,3,4]
        @test topk_index(X', 4) == [1,2,3,4]
    end
end


text:
mutable struct Font
  family::AbstractString
  pointsize::Int
  halign::Symbol
  valign::Symbol
  rotation::Float64
  color::Union{Nothing,AbstractString}
end

"""
    font(args...)
Create a Font from a list of features. Values may be specified either as
arguments (which are distinguished by type/value) or as keyword arguments.
# Arguments
- `family`: AbstractString. "serif" or "sans-serif" or "monospace"
- `pointsize`: Integer. Size of font in points
- `halign`: Symbol. Horizontal alignment (:hcenter, :left, or :right)
- `valign`: Symbol. Vertical aligment (:vcenter, :top, or :bottom)
- `rotation`: Real. Angle of rotation for text in degrees (use a non-integer type)
# Examples
```julia-repl
julia> font(8)
julia> font(family="serif",halign=:center,rotation=45.0)
```
"""
function font(args...;kw...)

  # defaults
  family = ""
  pointsize = -1
  halign = :hcenter
  valign = :vcenter
  rotation = 0.0
  color = nothing

  for arg in args
    T = typeof(arg)

    if T == Font
      family = arg.family
      pointsize = arg.pointsize
      halign = arg.halign
      valign = arg.valign
      rotation = arg.rotation
      color = arg.color
    elseif arg == :center
      halign = :hcenter
      valign = :vcenter
    elseif arg in (:hcenter, :left, :right)
      halign = arg
    elseif arg in (:vcenter, :top, :bottom)
      valign = arg
    elseif T <: Symbol || T <: AbstractString
      if startswith(string(arg), "#")
        color = string(arg)
      else
        family = string(arg)
      end
    elseif typeof(arg) <: Integer
      pointsize = arg
    elseif typeof(arg) <: Real
      rotation = convert(Float64, arg)
    else
      @warn("Unused font arg: $arg ($(typeof(arg)))")
    end
  end

  for symbol in keys(kw)
    if symbol == :family
      family = kw[:family]
    elseif symbol == :pointsize
      pointsize = kw[:pointsize]
    elseif symbol == :halign
      halign = kw[:halign]
      if halign == :center
        halign = :hcenter
      end
      @assert halign in (:hcenter, :left, :right)
    elseif symbol == :valign
      valign = kw[:valign]
      if valign == :center
        valign = :vcenter
      end
      @assert valign in (:vcenter, :top, :bottom)
    elseif symbol == :rotation
      rotation = kw[:rotation]
    elseif symbol == :color
      color = kw[:color]
    else
      @warn("Unused font kwarg: $symbol")
    end
  end

  Font(family, pointsize, halign, valign, rotation, color)
end

"Wrap a string with font info"
struct PlotText
  str::AbstractString
  font::Font
end
PlotText(str) = PlotText(string(str), font())

"""
    text(string, args...; kw...)
Create a PlotText object wrapping a string with font info, for plot annotations.
`args` and `kw` are passed to `font`.
"""
text(t::PlotText) = t
text(t::PlotText, font::Font) = PlotText(t.str, font)
text(str::AbstractString, f::Font) = PlotText(str, f)
function text(str, args...;kw...)
  PlotText(string(str), font(args...;kw...))
end

Base.length(t::PlotText) = length(t.str)

export text


text:
module Filters

export AbstractCTFilter, RamLak, CTFilter, CTFilterOrFunc
export ram_lak

using FFTW: fftfreq

abstract type AbstractCTFilter end

const CTFilterOrFunc = Union{Function,AbstractCTFilter}

struct RamLak <: AbstractCTFilter end

(f::RamLak)(args...) = ram_lak(args...)

function ram_lak(::Type{T}, nd::Int, nϕ::Int) where {T<:Real}
    freqs = abs.(fftfreq(nd, one(T)))
    repeat(freqs, outer = (1, nϕ))
end

ram_lak(nd::Int, nϕ::Int) = ram_lak(Float32, nd, nϕ)

struct CTFilter{F<:Function} <: AbstractCTFilter
    filter::F
end

end # module


text:
import ProgressMeter

function testfunc(n, dt, tsleep)
    p = ProgressMeter.Progress(n, dt)
    for i = 1:n
        sleep(tsleep)
        ProgressMeter.next!(p)
    end
end
println("Testing original interface...")
testfunc(107, 0.01, 0.01)

function testfunc2(n, dt, tsleep, desc, barlen)
    p = ProgressMeter.Progress(n, dt, desc, barlen)
    for i = 1:n
        sleep(tsleep)
        ProgressMeter.next!(p)
    end
end
println("Testing desc and progress bar")
testfunc2(107, 0.01, 0.01, "Computing...", 50)
println("Testing no desc and no progress bar")
testfunc2(107, 0.01, 0.01, "", 0)

function testfunc3(n, tsleep, desc)
    p = ProgressMeter.Progress(n, desc)
    for i = 1:n
        sleep(tsleep)
        ProgressMeter.next!(p)
    end
end
println("Testing tty width...")
testfunc3(107, 0.02, "Computing (use tty width)...")
println("Testing no description...")
testfunc3(107, 0.02, "")

function testfunc4()  # test "days" format
    p = ProgressMeter.Progress(10000000, "Test...")
    for i = 1:105
        sleep(0.02)
        ProgressMeter.next!(p)
    end
end

println("Testing that not even 1% required...")
testfunc4()

function testfunc5(n, dt, tsleep, desc, barlen)
    p = ProgressMeter.Progress(n, dt, desc, barlen)
    for i = 1:int(floor(n/2))
        sleep(tsleep)
        ProgressMeter.next!(p)
    end
    for i = int(ceil(n/2)):n
        sleep(tsleep)
        ProgressMeter.next!(p, :red)
    end
end

println("Testing changing the bar color")
testfunc5(107, 0.01, 0.01, "Computing...", 50)
println("")
println("All tests complete")


text:
# # Visualizing configurations with Makie.jl
import InteratomicPotentials as Potentials

# Let's load as test data a `GaN` configuration
file_path = joinpath(dirname(pathof(Potentials)), "..", "test", "DATA", "1", "DATA")
radii = 1.5
configuration = Potentials.load_lammps(
    file_path;
    atom_names = [:Ga, :N],
    radii = [radii, radii],
    weights = [1.0, 0.5],
    boundary_type = ["p", "p", "p"],
    units = "metal"
)

# In order to use Makie we define a recipe
import Makie

# The recipe! - will get called for plot(!)(x::Configuration)
function Makie.plot!(p::Makie.Plot(Potentials.Configuration))
    configuration = to_value(p[1]) # first argument is the Configuration

    colors = [:red, :blue]
    radii = configuration.masses ./ maximum(configuration.masses)

    for atom_kind in 1:configuration.num_atom_types
        name = configuration.atom_names[atom_kind]
        radius = radii[atom_kind]

        atoms = filter(Atom->Atom.Type == name, configuration.Atoms)
        positions = map(Atom->Point(Atom.Position...), atoms)

        meshscatter!(p, positions, markersize = radius, label=string(name), color=colors[atom_kind])
    end
end

# Now we can plot our configuration

using GLMakie
fig = Figure()
axes = Axis3(fig[1,1], aspect=:data)
plot!(axes, configuration)
fig

# Makie allows us to quickly create recordings

function load_configuration(i)
    file_path = joinpath(dirname(pathof(Potentials)), "..", "test", "DATA", string(i), "DATA")
    radii = 1.5
    return Potentials.load_lammps(
        file_path;
        atom_names = [:Ga, :N],
        radii = [radii, radii],
        weights = [1.0, 0.5],
        boundary_type = ["p", "p", "p"],
        units = "metal"
    )
end

# Create a idx `Node`

idx = Node(1)
configuration = @lift load_configuration($idx)

fig = Figure()
axes = Axis3(fig[1,1], aspect=:data)
plot!(axes, configuration)

record(fig, "GaN.mp4", 1:61, framerate=1) do frame
    idx[] = frame
end


text:
"""
    lognormal(x, μ, σ)

```math
\\frac{(x-μ)^2}{2σ^2}
```
"""
function lognormal(x, μ, σ)
  if σ == Inf
    return 0
  end
  if σ ≈ 0
    return Inf
  end
  (x-μ)^2/2/σ^2
end

"""
    logpoisson(n, x...)

```math
  λ > 0 \\
  λ = sum([x...])
  λ - n*log(λ) + n*log(n) - n
```math
"""
function logpoisson(n, x...)
  λ = sum([x...])
  if λ <= 0
    return Inf
  end
  λ - n*log(λ) + n*log(n) - n
end

"""
    batlog(x)

Compute the natural log, but create bounds to ignore very large
numbers / infinities.
```jldoctest
julia> batlog.([12.0, 1e6, 0])
3-element Array{Float64, 1}:
  2.4849
  13.8155
  -100000.0
```
"""
function batlog(x::Float64; up::Float64=1000.0, down::Float64=-1000.0)
  if x <= 0
    return -1e8
  end
  y = log(x)
  if y < down
    @debug "Batlog hit floor" y
  end
  if y > up
    @debug "Batlog hit ceiling" y
  end
  minimum([up, maximum([down, y])])
end


text:
const SolvedVariable = Union{AbstractFloat, SymEngine.Basic}
const InputVariable = Union{AbstractFloat}
const OutputVariable = Union{AbstractFloat, Nothing}

mutable struct Reactor
  solved::Bool
  steady::Bool
  valid::Bool

  B_0::SolvedVariable
  R_0::SolvedVariable
  I_M::SolvedVariable
  n_20::SolvedVariable
  T_k::SolvedVariable
  H::SolvedVariable

  N_G::InputVariable
  kappa::InputVariable
  epsilon::InputVariable
  beta_N::InputVariable
  Q::InputVariable
  f_D::InputVariable
  eta_CD::InputVariable
  B_max::InputVariable
  P_F::InputVariable
  b::InputVariable
  nu_n::InputVariable
  nu_T::InputVariable
  nu_J::Union{InputVariable,SolvedVariable}
  eta_T::InputVariable
  eta_RF::InputVariable
  sigma_max::InputVariable
  B_max_hat::InputVariable
  sigma_max_hat::InputVariable
  tau_P::InputVariable

  q_star::OutputVariable
  P_W::OutputVariable
  h_para::OutputVariable
  cost::OutputVariable
  B_C::OutputVariable
end

function Reactor(;
    solved=false, valid=true, steady=nothing,
    B_0=symbols(:B_0), R_0=symbols(:R_0), I_M=symbols(:I_M),
    n_20=symbols(:n_20), T_k=symbols(:T_k), H=symbols(:H),
    N_G=NaN, kappa=NaN, epsilon=NaN, beta_N=NaN, Q=NaN, f_D=NaN,
    eta_CD=NaN, B_max=NaN, P_F=NaN, b=NaN, nu_n=NaN, nu_T=NaN,
    nu_J=NaN, eta_T=NaN, eta_RF=NaN, sigma_max=NaN, B_max_hat=NaN,
    sigma_max_hat=NaN, tau_P=NaN
  )

  if isnothing(steady)
    if isnan(tau_P)
      @assert !isnan(eta_CD)
      steady = true
    else
      steady = false
    end
  end

  if !steady && isnan(nu_J)
    nu_J = symbols(:nu_J)
  end

  Reactor(
    solved, steady, valid,
    B_0, R_0, I_M, n_20, T_k, H,
    N_G, kappa, epsilon, beta_N,
    Q, f_D, eta_CD, B_max, P_F, b,
    nu_n, nu_T, nu_J, eta_T, eta_RF,
    sigma_max, B_max_hat, sigma_max_hat,
    tau_P, repeat([nothing], 5)...
  )
end

export Reactor


text:
"
    onesite dmrg
"
@with_kw struct Dmrg{F} <: Algorithm
    tol::Float64 = Defaults.tol
    maxiter::Int = Defaults.maxiter
    verbose::Bool = Defaults.verbose
    finalize::F = Defaults._finalize
end

find_groundstate(state,H,alg::Dmrg,envs...) = find_groundstate!(copy(state),H,alg,envs...)
function find_groundstate!(state::Union{FiniteMPS,MPSComoving}, H::Hamiltonian,alg::Dmrg,envs = environments(state,H))
    tol=alg.tol;maxiter=alg.maxiter
    iter = 0; delta::Float64 = 2*tol

    while iter < maxiter && delta > tol
        delta=0.0

        for pos = [1:(length(state)-1);length(state):-1:2]
            (eigvals,vecs) = @closure eigsolve(state.AC[pos],1,:SR,Lanczos()) do x
                ac_prime(x,pos,state,envs)
            end
            delta = max(delta,calc_galerkin(state,pos,envs))

            state.AC[pos] = vecs[1]
        end

        alg.verbose && @info "Iteraton $(iter) error $(delta)"
        flush(stdout)

        iter += 1

        #finalize
        (state,envs) = alg.finalize(iter,state,H,envs)::Tuple{typeof(state),typeof(envs)};
    end

    return state,envs,delta
end

"twosite dmrg"
@with_kw struct Dmrg2{F} <: Algorithm
    tol = Defaults.tol;
    maxiter = Defaults.maxiter;
    trscheme = truncerr(1e-6);
    verbose = Defaults.verbose
    finalize::F = Defaults._finalize
end

find_groundstate(state,H,alg::Dmrg2,envs...) = find_groundstate!(copy(state),H,alg,envs...)
function find_groundstate!(state::Union{FiniteMPS,MPSComoving}, H::Hamiltonian,alg::Dmrg2,envs = environments(state,H))
    tol=alg.tol;maxiter=alg.maxiter
    iter = 0; delta::Float64 = 2*tol

    while iter < maxiter && delta > tol
        delta=0.0

        ealg = Lanczos()

        #left to right sweep
        for pos= 1:(length(state)-1)
            @plansor ac2[-1 -2; -3 -4]:=state.AC[pos][-1 -2;1]*state.AR[pos+1][1 -4;-3]
            (eigvals,vecs) = @closure eigsolve(ac2,1,:SR,ealg) do x
                ac2_prime(x,pos,state,envs)
            end
            newA2center = vecs[1]

            (al,c,ar,ϵ) = tsvd(newA2center,trunc=alg.trscheme,alg=TensorKit.SVD())
            normalize!(c);
            v = @plansor ac2[1 2;3 4]*conj(al[1 2;5])*conj(c[5;6])*conj(ar[6;3 4])
            delta = max(delta,abs(1-abs(v)));

            state.AC[pos] = (al,complex(c))
            state.AC[pos+1] = (complex(c),_transpose_front(ar))
        end

        for pos = length(state)-2:-1:1
            @plansor ac2[-1 -2; -3 -4]:=state.AL[pos][-1 -2;1]*state.AC[pos+1][1 -4;-3]
            (eigvals,vecs) = @closure eigsolve(ac2,1,:SR,ealg) do x
                ac2_prime(x,pos,state,envs)
            end
            newA2center = vecs[1]

            (al,c,ar,ϵ) = tsvd(newA2center,trunc=alg.trscheme,alg=TensorKit.SVD())
            normalize!(c);
            v = @plansor ac2[1 2;3 4]*conj(al[1 2;5])*conj(c[5;6])*conj(ar[6;3 4])
            delta = max(delta,abs(1-abs(v)));

            state.AC[pos+1] = (complex(c),_transpose_front(ar))
            state.AC[pos] = (al,complex(c))
        end

        alg.verbose && @info "Iteraton $(iter) error $(delta)"
        flush(stdout)
        #finalize
        (state,envs) = alg.finalize(iter,state,H,envs)::Tuple{typeof(state),typeof(envs)};
        iter += 1
    end

    return state,envs,delta
end


text:
module AlphaVantageResponseTest
using AlphaVantage
using Test

@testset "Response" begin
    names = Matrix{AbstractString}(["a" "b"])

    data = Matrix{Any}(rand(2, 2))
    response = AlphaVantageResponse(data, names)
    @test isa(response, AlphaVantageResponse)

    response = AlphaVantageResponse(tuple(data, names))
    @test isa(response, AlphaVantageResponse)
end

end


text:
abstract type AbstractVar end
abstract type AbstractDomain end

abstract type AbstractSetVar <: AbstractVar end

"""
    abstract type AbstractIntVar end

Abstract type for integer variables.
"""
abstract type AbstractIntVar <: AbstractVar end

"""
    abstract type AbstractBoolVar <: AbstractVar end

Abstract type for all boolean variables.
"""
abstract type AbstractBoolVar <: AbstractVar end

"""
    function id(x::AbstractVar)

Return the `string` identifier of `x`. Every variable must be assigned a unique identifier upon creation, that
will be used as a key to identify the variable in the `CPModel` object.
"""
id(x::AbstractVar) = x.id
parentId(x::AbstractVar) = rootVariable(x).id

include("IntDomain.jl")
include("IntVar.jl")

include("BoolDomain.jl")
include("BoolVar.jl")

abstract type IntVarView <: AbstractIntVar end
abstract type IntDomainView <: AbstractIntDomain end

include("IntVarView.jl")

abstract type BoolVarView <: AbstractBoolVar end
abstract type BoolDomainView <: AbstractBoolDomain end

include("BoolVarView.jl")

include("IntSetDomain.jl")
include("IntSetVar.jl")