text:
function set_expression_names_and_values(request_map, refs)
    if length(refs.attrs) != 0
       request_map["ExpressionAttributeNames"] = refs.attrs
    end
    if length(refs.vals) != 0
        request_map["ExpressionAttributeValues"] = refs.vals
    end
end

load_row(table, attr_dict; is_old=false) =
    after_load(table.extension, table, value_from_attributes(table.ty, attr_dict);
               is_old=is_old)

attributes_to_write(table, item) =
    transform_attrs_for_write(table.extension, table, attribute_value(item)["M"])

updates_to_send(table, updates) =
    transform_update_expression(table.extension, table, updates)

#     _    ____ ___      ____      _   ___ _
#    / \  |  _ \_ _|_   / ___| ___| |_|_ _| |_ ___ _ __ ___
#   / _ \ | |_) | |(_) | |  _ / _ \ __|| || __/ _ \ '_ ` _ \
#  / ___ \|  __/| | _  | |_| |  __/ |_ | || ||  __/ | | | | |
# /_/   \_\_|  |___(_)  \____|\___|\__|___|\__\___|_| |_| |_|

# https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_GetItem.html

function get_item_query_dict(table :: DynamoTable, key, range, consistent_read, only_returning)
    request_map = Dict{Any, Any}("TableName" => table.name,
                                 "ConsistentRead" => consistent_read,
                                 "Key" => keydict(table, key, range))

    # TODO: "ReturnConsumedCapacity"

    if only_returning != nothing
        refs = refs_tracker()
        request_map["ProjectionExpression"] = join([write_expression(refs, e) for e=only_returning], ", ")
        request_map["ExpressionAttributeNames"] = refs.attrs
    end

    request_map
end

function get_item(table :: DynamoTable, key, range=nothing;
                  consistant_read=true, only_returning=nothing :: Union{Void, Array{DynamoReference}})
    request_map = get_item_query_dict(table, key, range, consistant_read, only_returning)

    (status, res) = dynamo_execute(table.aws_env, "GetItem", request_map)
    check_status(status, res)

    if haskey(res, "Item")
        return load_row(table, res["Item"])
    else
        return nothing
    end
end

#     _    ____ ___
#    / \  |  _ \_ _|_
#   / _ \ | |_) | |(_)
#  / ___ \|  __/| | _
# /_/   \_\_|  |___(_)
#  ____        _       _      ____      _   ___ _
# | __ )  __ _| |_ ___| |__  / ___| ___| |_|_ _| |_ ___ _ __ ___
# |  _ \ / _` | __/ __| '_ \| |  _ / _ \ __|| || __/ _ \ '_ ` _ \
# | |_) | (_| | || (__| | | | |_| |  __/ |_ | || ||  __/ | | | | |
# |____/ \__,_|\__\___|_| |_|\____|\___|\__|___|\__\___|_| |_| |_|

# https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchGetItem.html

# note: you can only fetch 100 items at a time with this API
# TODO: multiplex calls to fix this client side?

type BatchGetItemPart
    table :: DynamoTable
    keys
    only_returning
    consistant_read
end

batch_get_item_part(table :: DynamoTable, keys...;
                    only_returning=nothing :: Union{Void, Array{DynamoReference}}, consistant_read=true) =
    BatchGetItemPart(table, keys, only_returning, consistant_read)

function batch_get_item_dict(arr :: Array{BatchGetItemPart})
    m = Dict()

    # TODO: "ReturnConsumedCapacity"

    for part=arr
        request_map = Dict{Any, Any}("ConsistentRead" => part.consistant_read,
                                     "Keys" => [keydict(part.table, e...) for e=part.keys])

        if part.only_returning != nothing
            refs = refs_tracker()
            request_map["ProjectionExpression"] = join([write_expression(refs, e) for e=part.only_returning], ", ")
            request_map["ExpressionAttributeNames"] = refs.attrs
        end

        m[part.table.name] = request_map
    end

    Dict("RequestItems" => m)
end

function batch_get_item(arr :: Array{BatchGetItemPart})
    request_map = batch_get_item_dict(arr)

    (status, res) = dynamo_execute(arr[1].table.aws_env, "BatchGetItem", request_map)
    check_status(status, res)

    table_lookup = Dict()
    for e = arr
        table_lookup[e.table.name] = e.table
    end

    result = []
    for (name, list)=res["Responses"]
        table = table_lookup[name]
        for e=list
            item = load_row(table, e)
            if item != nothing
                push!(result, item)
            end
        end
    end

    result
end

# helper function for the single table case
batch_get_item(table :: DynamoTable, keys...;
               only_returning=nothing :: Union{Void, Array{DynamoReference}}, consistant_read=true) =
    batch_get_item([batch_get_item_part(table :: DynamoTable, keys...; only_returning=only_returning, consistant_read=consistant_read)])

#     _    ____ ___
#    / \  |  _ \_ _|_
#   / _ \ | |_) | |(_)
#  / ___ \|  __/| | _
# /_/   \_\_|  |___(_)
#  ____        _   ___ _
# |  _ \ _   _| |_|_ _| |_ ___ _ __ ___
# | |_) | | | | __|| || __/ _ \ '_ ` _ \
# |  __/| |_| | |_ | || ||  __/ | | | | |
# |_|    \__,_|\__|___|\__\___|_| |_| |_|

function put_item_dict(table :: DynamoTable, item;
                       conditions=no_conditions() :: CEBoolean, return_old=false)
    request_map = Dict("TableName" => table.name,
                       "Item" => attributes_to_write(table, item))

    conditional_expression = and(conditions, row_write_conditions(table.extension, table, item))
    if can_write_expression(conditional_expression)
        refs = refs_tracker()
        request_map["ConditionExpression"] = write_expression(refs, conditional_expression)
        set_expression_names_and_values(request_map, refs)
    end

    if return_old
        request_map["ReturnValues"] = "ALL_OLD"
    end

    # TODO: ReturnConsumedCapacity ?
    # TODO: ReturnItemCollectionMetrics ?

    request_map
end

function put_item(table :: DynamoTable, item; conditions=no_conditions() :: CEBoolean, return_old=false)
    request_map = put_item_dict(table, item; conditions=conditions, return_old=return_old)

    (status, res) = dynamo_execute(table.aws_env, "PutItem", request_map)
    check_status(status, res)

    if return_old && haskey(res, "Attributes")
        return load_row(table, res["Attributes"]; is_old=true)
    else
        return nothing
    end
end

#     _    ____ ___
#    / \  |  _ \_ _|_
#   / _ \ | |_) | |(_)
#  / ___ \|  __/| | _
# /_/   \_\_|  |___(_)
#  ____        _       _  __        __    _ _       ___ _
# | __ )  __ _| |_ ___| |_\ \      / / __(_) |_ ___|_ _| |_ ___ _ __ ___
# |  _ \ / _` | __/ __| '_ \ \ /\ / / '__| | __/ _ \| || __/ _ \ '_ ` _ \
# | |_) | (_| | || (__| | | \ V  V /| |  | | ||  __/| || ||  __/ | | | | |
# |____/ \__,_|\__\___|_| |_|\_/\_/ |_|  |_|\__\___|___|\__\___|_| |_| |_|

# https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html

# NOTE: you can't update items using this API call... use update_item instead

# TODO: ReturnConsumedCapacity
# TODO: ReturnItemCollectionMetrics

type BatchWriteItemPart
    table :: DynamoTable
    keys_to_delete
    items_to_write
end
batch_write_item_part(table :: DynamoTable, keys_to_delete, items_to_write) =
    BatchWriteItemPart(table, keys_to_delete, items_to_write)
batch_delete_part(table :: DynamoTable, keys...) =
    BatchWriteItemPart(table, keys, [])
batch_put_part(table :: DynamoTable, items...) =
    BatchWriteItemPart(table, [], items)

function batch_write_item_dict(parts :: Array{BatchWriteItemPart})
    dicts = []

    current_ct = 0
    current_dict = Dict()

    function add_op(table_name, op)
        if current_ct == 25
            push!(dicts, current_dict)

            current_ct = 0
            current_dict = Dict()
        end

        if !haskey(current_dict, table_name)
            current_dict[table_name] = []
        end
        ops = current_dict[table_name]

        push!(ops, op)
        current_ct += 1
    end

    for p=parts
        for e=p.keys_to_delete
            if !can_batch_delete(p.table.extension, p.table)
                error("Tried to batch delete from table which doesn't support it. (The table extension probably requires writing conditions)")
            end
            add_op(p.table.name, Dict("DeleteRequest" => Dict("Key" => keydict(p.table, e...))))
        end
        for e=p.items_to_write
            if !can_batch_write(p.table.extension, p.table)
                error("Tried to batch write to table which doesn't support it. (The table extension probably requires writing conditions)")
            end
            add_op(p.table.name, Dict("PutRequest" => Dict("Item" => attributes_to_write(p.table, e))))
        end
    end

    push!(dicts, current_dict)
    dicts
end

function batch_write_item(parts :: Array{BatchWriteItemPart})
    dicts = batch_write_item_dict(parts)

    # TODO: ReturnConsumedCapacity
    # TODO: ReturnItemCollectionMetrics

    function process_dict(d, current_retry=0)
        (status, res) = dynamo_execute(parts[1].table.aws_env, "BatchWriteItem", Dict("RequestItems" => d))
        check_status(status, res)

        # TODO -- log with capacity stuffs
        # exponential backoff per the dynamodb docs
        if length(res["UnprocessedItems"]) != 0
            if current_retry > 9
                error("Request failed after 10 retries")
            end
            sleep(2^current_retry * 0.05)

            process_dict(res["UnprocessedItems"], current_retry+1)
        end
    end

    for d=dicts
        process_dict(d)
    end
end

# helper/simpler methods
batch_put_item(table :: DynamoTable, items...) =
    batch_write_item([batch_put_part(table, items...)])
batch_delete_item(table :: DynamoTable, keys...) =
    batch_write_item([batch_delete_part(table, keys...)])

#     _    ____ ___
#    / \  |  _ \_ _|_
#   / _ \ | |_) | |(_)
#  / ___ \|  __/| | _
# /_/   \_\_|  |___(_)
#  _   _           _       _       ___ _
# | | | |_ __   __| | __ _| |_ ___|_ _| |_ ___ _ __ ___
# | | | | '_ \ / _` |/ _` | __/ _ \| || __/ _ \ '_ ` _ \
# | |_| | |_) | (_| | (_| | ||  __/| || ||  __/ | | | | |
#  \___/| .__/ \__,_|\__,_|\__\___|___|\__\___|_| |_| |_|
#       |_|

# TODO -- enum of something? Julia doesn't support that though...
const RETURN_NONE = "NONE"
const RETURN_ALL_OLD = "ALL_OLD"
const RETURN_UPDATED_OLD = "UPDATED_OLD"
const RETURN_ALL_NEW = "ALL_NEW"
const RETURN_UPDATED_NEW = "UPDATED_NEW"

function update_item_dict(table :: DynamoTable, key, range, update_expression;
                          conditions=no_conditions() :: CEBoolean, returning=RETURN_NONE)
    request_map = Dict("TableName" => table.name,
                       "Key" => keydict(table, key, range),
                       "ReturnValues" => returning)

    # ReturnConsumedCapacity == INDEXES | TOTAL | NONE

    refs = refs_tracker()
    request_map["UpdateExpression"] = serialize_updates(updates_to_send(table, update_expression), refs)

    if can_write_expression(conditions)
        request_map["ConditionExpression"] = serialize_expression(conditions, refs)
    end

    set_expression_names_and_values(request_map, refs)
    request_map
end

function update_item(table :: DynamoTable, key, range, update_expression :: Array;
                     conditions=no_conditions() :: CEBoolean, returning=RETURN_NONE)
    request_map = update_item_dict(table, key, range, update_expression;
                                   conditions=conditions, returning=returning)

    # TODO: run it
    resp = Dict()

    (status, res) = dynamo_execute(table.aws_env, "UpdateItem", request_map)
    check_status(status, res)

    if returning == RETURN_ALL_NEW
        return load_row(table, res["Attributes"])
    elseif returning == RETURN_ALL_OLD
        return load_row(table, res["Attributes"]; is_old=true)
    elseif returning == RETURN_UPDATED_OLD || returning == RETURN_UPDATED_NEW
        value_from_attributes(Dict, res["Attributes"])
    end
end

update_item(table :: DynamoTable, key, update_expression :: Array;
            conditions=no_conditions() :: CEBoolean, returning=RETURN_NONE) =
    update_item(table, key, nothing, update_expression; conditions=conditions, returning=returning)

update_item{T <: DynamoUpdateExpression}(table :: DynamoTable, key, range, update_expression :: T;
            conditions=no_conditions() :: CEBoolean, returning=RETURN_NONE) =
    update_item(table, key, range, [update_expression]; conditions=conditions, returning=returning)

update_item{T <: DynamoUpdateExpression}(table :: DynamoTable, key, update_expression :: T;
            conditions=no_conditions() :: CEBoolean, returning=RETURN_NONE) =
    update_item(table, key, nothing, [update_expression]; conditions=conditions, returning=returning)

#     _    ____ ___
#    / \  |  _ \_ _|_
#   / _ \ | |_) | |(_)
#  / ___ \|  __/| | _
# /_/   \_\_|  |___(_)
#  ____       _      _       ___ _
# |  _ \  ___| | ___| |_ ___|_ _| |_ ___ _ __ ___
# | | | |/ _ \ |/ _ \ __/ _ \| || __/ _ \ '_ ` _ \
# | |_| |  __/ |  __/ ||  __/| || ||  __/ | | | | |
# |____/ \___|_|\___|\__\___|___|\__\___|_| |_| |_|

# https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DeleteItem.html

function delete_item_dict(table :: DynamoTable, key, range=nothing;
                          conditions=no_conditions() :: CEBoolean, return_old=false)
    request_map = Dict("TableName" => table.name,
                       "Key" => keydict(table, key, range))

    if return_old
        request_map["ReturnValues"] = "ALL_OLD"
    end

    refs = refs_tracker()

    conditional_expression = and(row_delete_conditions(table.extension, table), conditions)
    if can_write_expression(conditional_expression)
        request_map["ConditionExpression"] = serialize_expression(conditional_expression, refs)
        set_expression_names_and_values(request_map, refs)
    end

    request_map
end

function delete_item(table :: DynamoTable, key, range=nothing;
                     conditions=no_conditions() :: CEBoolean, return_old=false)
    request_map = delete_item_dict(table, key, range; conditions=conditions, return_old=return_old)

    (status, res) = dynamo_execute(table.aws_env, "DeleteItem", request_map)
    check_status(status, res)

    if return_old
        return load_row(table, res["Attributes"]; is_old=true)
    end
end

#     _    ____ ___
#    / \  |  _ \_ _|_
#   / _ \ | |_) | |(_)
#  / ___ \|  __/| | _
# /_/   \_\_|  |___(_)
#   ___
#  / _ \ _   _  ___ _ __ _   _
# | | | | | | |/ _ \ '__| | | |
# | |_| | |_| |  __/ |  | |_| |
#  \__\_\\__,_|\___|_|   \__, |
#                        |___/

# TODO: ReturnConsumedCapacity

const SELECT_ALL_ATTRIBUTES = "ALL_ATTRIBUTES"
const SELECT_ALL_PROJECTED_ATTRIBUTES = "ALL_PROJECTED_ATTRIBUTES"
const SELECT_SPECIFIC_ATTRIBUTES = "SPECIFIC_ATTRIBUTES"
const SELECT_COUNT = "COUNT"

function query_dict(table :: DynamoTable, hash_val, range_condition;
               filter=nothing :: Union{Void, CEBoolean}, projection=DynamoReference[] :: Array{DynamoReference},
               consistant_read=true, scan_index_forward=true, limit=nothing, index_name=nothing,
               select_type=nothing, start_key=nothing)
    refs = refs_tracker()
    request_map = Dict{AbstractString, Any}("TableName" => table.name,
                       "KeyConditionExpression" => serialize_expression(and(attr(table.hash_key_name) == hash_val,
                                                                            range_condition), refs))

    if start_key != nothing
        request_map["ExclusiveStartKey"] = start_key
    end
    # only write if value isn't the default value
    if consistant_read == false
        request_map["ConsistentRead"] = consistant_read
    end
    if scan_index_forward == false
        request_map["ScanIndexForward"] = scan_index_forward
    end

    if index_name != nothing
        request_map["IndexName"] = index_name
    end
    if length(projection) != 0
        request_map["ProjectionExpression"] = join([write_expression(refs, e) for e=projection], ", ")
    elseif select_type != nothing
        request_map["Select"] = select_type
    end
    if filter != nothing
        request_map["FilterExpression"] = serialize_expression(filter, refs)
    end
    if limit != nothing
        request_map["Limit"] = limit
    end
    set_expression_names_and_values(request_map, refs)

    request_map
end

function query(table :: DynamoTable, hash_val, range_condition = no_conditions() :: CEBoolean;
               filter=nothing :: Union{Void, CEBoolean}, projection=DynamoReference[] :: Array{DynamoReference},
               consistant_read=true, scan_index_forward=true, limit=nothing, index_name=nothing,
               select_type=nothing)

    returned_ct = 0
    function run_query_part(start_key)
        request_map = query_dict(table, hash_val, range_condition; filter=filter, projection=projection,
                                 consistant_read=consistant_read, scan_index_forward=scan_index_forward,
                                 limit=limit, index_name=index_name, select_type=select_type,
                                 start_key=start_key)

        (status, res) = dynamo_execute(table.aws_env, "Query", request_map)
        check_status(status, res)

        # TODO: potentially interesting return values?
        # Count -- number of items returned
        # ScannedCount -- number of items accessed

        for e=res["Items"]
            produce(value_from_attributes(table.ty, e))
            returned_ct += 1
            if returned_ct == limit
                return
            end
        end

        if haskey(res, "LastEvaluatedKey")
            res["Items"] = nothing
            run_query_part(res["LastEvaluatedKey"])
        end
    end

    @task run_query_part(nothing)
end

query(table :: DynamoLocalIndex, range_condition;
      filter=nothing :: Union{Void, CEBoolean}, projection=[] :: Array{DynamoReference},
      consistant_read=true, scan_index_forward=true, limit=nothing, select_type=nothing) =
   query(table.parent, hash_val, range_condition; filter=filter, projection=projection,
         consistant_read=consistant_read, scan_index_forward=scan_index_forward,
         limit=limit, index_name=table.index_name, select_type=select_type)

# NOTE: consistant_reads aren't possible on global secondary indexes (hence the missing param)
query(table :: DynamoGlobalIndex, range_condition;
      filter=nothing :: Union{Void, CEBoolean}, projection=[] :: Array{DynamoReference},
      scan_index_forward=true, limit=nothing, select_type=nothing) =
   query(table.parent, hash_val, range_condition; filter=filter, projection=projection,
         consistant_read=false, scan_index_forward=scan_index_forward,
         limit=limit, index_name=table.index_name, select_type=select_type)

#     _    ____ ___
#    / \  |  _ \_ _|_
#   / _ \ | |_) | |(_)
#  / ___ \|  __/| | _
# /_/   \_\_|  |___(_)
#  ____
# / ___|  ___ __ _ _ __
# \___ \ / __/ _` | '_ \
#  ___) | (_| (_| | | | |
# |____/ \___\__,_|_| |_|

function scan_dict(table :: DynamoTable, filter = no_conditions() :: CEBoolean;
                   projection=DynamoReference[] :: Array{DynamoReference}, consistant_read=false, scan_index_forward=true,
                   limit=nothing, select_type=nothing, count=false, segment=nothing, total_segments=nothing,
                   index_name=nothing, start_key=nothing)

    # TODO: ReturnConsumedCapacity

    refs = refs_tracker()
    request_map = Dict{AbstractString, Any}("TableName" => table.name)

    if start_key != nothing
        request_map["ExclusiveStartKey"] = start_key
    end
    # only write if value isn't the default value (false)
    if consistant_read == true
        request_map["ConsistentRead"] = consistant_read
    end
    if scan_index_forward == false
        request_map["ScanIndexForward"] = scan_index_forward
    end
    if segment != nothing || total_segments != nothing
        if segment == nothing || total_segments == nothing
            error("you must specify BOTH segment and total_segments or neither")
        end
        request_map["Segment"] = segment
        request_map["TotalSegments"] = total_segments
    end

    if can_write_expression(filter)
        request_map["FilterExpression"] = serialize_expression(filter, refs)
    end
    if index_name != nothing
        request_map["IndexName"] = index_name
    end
    if length(projection) != 0
        request_map["ProjectionExpression"] = join([write_expression(refs, e) for e=projection], ", ")
    elseif count
        request_map["Select"] = "Count"
    elseif select_type != nothing
        request_map["Select"] = select_type
    end
    if limit != nothing
        request_map["Limit"] = limit
    end
    set_expression_names_and_values(request_map, refs)

    request_map
end

function scan(table :: DynamoTable, filter = no_conditions() :: CEBoolean;
              projection=DynamoReference[] :: Array{DynamoReference}, consistant_read=false, scan_index_forward=true,
              limit=nothing, select_type=nothing, count=false, segment=nothing, total_segments=nothing,
              index_name=nothing)

    returned_ct = 0
    function run_scan_part(start_key)
        request_map = scan_dict(table, filter;
                        projection=projection, consistant_read=consistant_read,
                        scan_index_forward=scan_index_forward, limit=limit, select_type=select_type,
                        count=count, segment=segment, total_segments=total_segments,
                        index_name=index_name, start_key=start_key)

        (status, res) = dynamo_execute(table.aws_env, "Scan", request_map)
        check_status(status, res)

        # TODO: potentially interesting return values?
        # Count -- number of items returned
        # ScannedCount -- number of items accessed

        for e=res["Items"]
            produce(value_from_attributes(table.ty, e))
            returned_ct += 1
            if returned_ct == limit
                return
            end
        end

        if haskey(res, "LastEvaluatedKey")
            run_scan_part(res["LastEvaluatedKey"])
        end
    end

    @task run_scan_part(nothing)
end

scan(table :: DynamoLocalIndex, filter = no_conditions() :: CEBoolean;
     projection=DynamoReference[] :: Array{DynamoReference}, scan_index_forward=true,
     limit=nothing, select_type=nothing, count=false, segment=nothing, total_segments=nothing) =
    scan(table.parent, filter=filter;
         projection=projection, consistant_read=false, scan_index_forward=scan_index_forward,
         limit=limit, select_type=select_type, count=count, segment=segment, total_segments=total_segments,
         index_name=table.index_name)

scan(table :: DynamoGlobalIndex, filter = no_conditions() :: CEBoolean;
     projection=DynamoReference[] :: Array{DynamoReference}, scan_index_forward=true,
     limit=nothing, select_type=nothing, count=false, segment=nothing, total_segments=nothing) =
    scan(table.parent, filter=filter;
         projection=projection, consistant_read=false, scan_index_forward=scan_index_forward,
         limit=limit, select_type=select_type, count=count, segment=segment, total_segments=total_segments,
         index_name=table.index_name)


text:
export Policy, solve

struct Policy
    policy::Matrix{Real}
    actions::Vector{String}
    function Policy(p::Matrix; a::Vector{String} = actions)::Policy
        return new(p, a);
    end
end

function Base.show(io::IO, p::Policy)::Nothing
    # println(io, "\033[2J") # clear repl
    # println(io, "\033[$(displaysize(stdout)[1])A")
    rows, cols = size(p.policy)
    border = "\t" * " "^7 * "-"^(cols * 7)
    println(io, border)
    for i in rows:-1:1
        print(io, "\t")
        for j in 1:cols
            if j == 1
                @printf "%-5i  | %s   |" i p.actions[p.policy[j,i]]
            else
                @printf "  %s   |" p.actions[p.policy[j,i]]
            end
        end
        println(io, "\n" * border)
    end
    print(io, "\t   ")
    for k in 1:cols
        @printf "    %3.0f" k
    end
end

function solve(g::Grid; iterations::Int = 100, belres::Float64 = 1e-3, discount_factor::Float64 = 0.90)::Policy
    ns = length(statespace(g)) # number of states
    na = length(actionspace)   # number of actions

    # initialize the policy matrix
    pol = zeros(Int64, ns)
    util = zeros(ns)

    # time solver
    total_time = 0.0
    iter_time = 0.0

    # main loop
    for i in 1:iterations
        residual = 0.0
        iter_time = @elapsed begin
            # state loop
            for (istate, s) in enumerate(statespace(g))
                if terminal(g, s)
                    pol[istate] = 1
                else
                    old_util = util[istate] # for residual
                    max_util = -Inf
                    # action loop
                    # util(s) = max_a( R(s,a) + discount_factor * sum(T(s'|s,a)util(s') )
                    for (iaction, a) in enumerate(actionspace)
                        dist = transition(g, s, a)
                        u = 0.0
                        for (sp, p) in weighted_iterator(dist)
                            p == 0.0 ? continue : nothing # skip if zero prob
                            r = reward(g, sp)
                            isp = stateindex(g, sp)
                            u += p * (r + discount_factor * util[isp])
                        end
                        new_util = u
                        if new_util > max_util
                            max_util = new_util
                            pol[istate] = iaction
                        end
                    end # action
                    # update the value array
                    util[istate] = max_util
                    diff = abs(max_util - old_util)
                    diff > residual ? (residual = diff) : nothing
                end
            end # state
        end # time
        total_time += iter_time
        @printf("[Iteration %-4d] residual: %10.3G | iteration runtime: %10.3f ms, (%10.3G s total)\n", i, residual, iter_time*1000.0, total_time)
        residual < belres ? break : nothing
    end # main
    rows, cols = g.dims
    policy = reshape(pol[1:end-1], rows, cols)
    return Policy(policy);
end

ext: jl

max_stars_repo_name: Joshua-Robison/GridWorld

max_issues_repo_name: Joshua-Robison/GridWorld

max_forks_repo_name: Joshua-Robison/GridWorld


text:
using StatsBase
using Test

n = 5000

# 1D integer counts

x = rand(1:5, n)
w = weights(rand(n))

c = counts(x, 5)
@test size(c) == (5,)
c0 = Int[count(v->v == i, x) for i in 1:5]
@test c == c0
@test counts(x .+ 1, 2:6) == c0
@test proportions(x, 1:5) ≈ (c0 ./ n)

c = counts(x)
@test size(c) == (5,)
c0 = Int[count(v->v == i, x) for i in 1:5]
@test c == c0
@test counts(x .+ 1, 2:6) == c0
@test proportions(x) ≈ (c0 ./ n)

c = counts(x, 5, w)
@test size(c) == (5,)
c0 = Float64[sum(w.values[x .== i]) for i in 1:5]
@test c                      ≈ c0
@test counts(x .+ 1, 2:6, w) ≈ c0
@test proportions(x, 1:5, w) ≈ (c0 ./ sum(w))

c = counts(x, w)
@test size(c) == (5,)
c0 = Float64[sum(w.values[x .== i]) for i in 1:5]
@test c                      ≈ c0
@test counts(x .+ 1, 2:6, w) ≈ c0
@test proportions(x, w)      ≈ (c0 ./ sum(w))

# 2D integer counts

x = rand(1:4, n)
y = rand(1:5, n)
w = weights(rand(n))

c = counts(x, y, (4, 5))
@test size(c) == (4, 5)
c0 = Int[count(t->t != 0,  (x .== i) .& (y .== j)) for i in 1:4, j in 1:5]
@test c == c0
@test counts(x .+ 2, y .+ 3, (3:6, 4:8)) == c0
@test proportions(x, y, (1:4, 1:5)) ≈ (c0 ./ n)

c = counts(x, y)
@test size(c) == (4, 5)
c0 = Int[count(t->t != 0, (x .== i) .& (y .== j)) for i in 1:4, j in 1:5]
@test c == c0
@test counts(x .+ 2, y .+ 3, (3:6, 4:8)) == c0
@test proportions(x, y,) ≈ (c0 ./ n)

c = counts(x, y, (4, 5), w)
@test size(c) == (4, 5)
c0 = Float64[sum(w.values[(x .== i) .& (y .== j)]) for i in 1:4, j in 1:5]
@test c                                     ≈ c0
@test counts(x .+ 2, y .+ 3, (3:6, 4:8), w) ≈ c0
@test proportions(x, y, (1:4, 1:5), w)      ≈ (c0 ./ sum(w))

c = counts(x, y, w)
@test size(c) == (4, 5)
c0 = Float64[sum(w.values[(x .== i) .& (y .== j)]) for i in 1:4, j in 1:5]
@test c                                     ≈ c0
@test counts(x .+ 2, y .+ 3, (3:6, 4:8), w) ≈ c0
@test proportions(x, y, w)                  ≈ (c0 ./ sum(w))

# count map

x = ["a", "b", "a", "a", "b", "c"]
w = [1.0, 1.5, 2.0, 2.5, 3.0, 3.5]

cm = countmap(x)
@test cm["a"] == 3
@test cm["b"] == 2
@test cm["c"] == 1

# iterator, non-radixsort
cm_missing = countmap(skipmissing(x))
cm_any_itr = countmap((i for i in x))
@test cm_missing == cm_any_itr == cm
@test cm_missing isa Dict{String, Int}
@test cm_any_itr isa Dict{Any, Int}

pm = proportionmap(x)
@test pm["a"] ≈ (1/2)
@test pm["b"] ≈ (1/3)
@test pm["c"] ≈ (1/6)

# testing the radixsort branch of countmap
xx = repeat([6, 1, 3, 1], outer=100_000)
cm = countmap(xx)
@test cm == Dict(1 => 200_000, 3 => 100_000, 6 => 100_000)

# with iterator
cm_missing = countmap(skipmissing(xx))
@test cm_missing isa Dict{Int, Int}
@test cm_missing == cm

cm_any_itr = countmap((i for i in xx))
@test cm_any_itr isa Dict{Any,Int} # no knowledge about type
@test cm_missing == cm

# testing the radixsort-based addcounts
xx = repeat([6, 1, 3, 1], outer=100_000)
cm = Dict{Int, Int}()
StatsBase.addcounts_radixsort!(cm,xx)
@test cm == Dict(1 => 200_000, 3 => 100_000, 6 => 100_000)
xx2 = repeat([7, 1, 3, 1], outer=100_000)
StatsBase.addcounts_radixsort!(cm,xx2)
@test cm == Dict(1 => 400_000, 3 => 200_000, 6 => 100_000, 7 => 100_000)
# with iterator
cm_missing = Dict{Int, Int}()
StatsBase.addcounts_radixsort!(cm_missing,skipmissing(xx))
@test cm_missing == Dict(1 => 200_000, 3 => 100_000, 6 => 100_000)
StatsBase.addcounts_radixsort!(cm_missing,skipmissing(xx2))
@test cm_missing == Dict(1 => 400_000, 3 => 200_000, 6 => 100_000, 7 => 100_000)

# testing the Dict-based addcounts
cm = Dict{Int, Int}()
cm_itr = Dict{Int, Int}()
StatsBase.addcounts_dict!(cm,xx)
StatsBase.addcounts_dict!(cm_itr,skipmissing(xx))
@test cm_itr == cm == Dict(1 => 200_000, 3 => 100_000, 6 => 100_000)
@test cm_itr isa Dict{Int, Int}

cm = countmap(x, weights(w))
@test cm["a"] == 5.5
@test cm["b"] == 4.5
@test cm["c"] == 3.5

@test cm == countmap(x, w)

pm = proportionmap(x, weights(w))
@test pm["a"] ≈ (5.5 / 13.5)
@test pm["b"] ≈ (4.5 / 13.5)
@test pm["c"] ≈ (3.5 / 13.5)

# testing small bits type
bx = [true, false, true, true, false]
cm_bx_missing = countmap(skipmissing(bx))
@test cm_bx_missing == countmap(bx) == Dict(true => 3, false => 2)
@test cm_bx_missing isa Dict{Bool, Int}

for T in [UInt8, UInt16, Int8, Int16]
    tx = T[typemin(T), 8, typemax(T), 19, 8]
    tx_missing = skipmissing(T[typemin(T), 8, typemax(T), 19, 8])
    cm_tx_missing = countmap(tx_missing)
    @test cm_tx_missing == countmap(tx) == Dict(typemin(T) => 1, typemax(T) => 1, 8 => 2, 19 => 1)
    @test cm_tx_missing isa Dict{T, Int}
end

@testset "views" begin
    X = view([1,1,1,2,2], 1:5)
    @test countmap(X) == countmap(copy(X))
end


text:
using LinearAlgebra, Distributions
using Statistics: mean
export predictability

"""
    predictability(ds::DynamicalSystem; kwargs...) -> chaos_type, ν, C

Determine whether `ds` displays strongly chaotic, partially-predictable chaotic
or regular behaviour, using the method by Wernecke et al. described in[^Wernecke2017].

Return the type of the behavior, the cross-distance scaling coefficient `ν`
and the correlation coefficient `C`.
Typical values for `ν`, `C` and `chaos_type` are given in Table 2 of[^Wernecke2017]:

| `chaos_type` | `ν` | `C` |
|--------------|-----|-----|
|    `:SC`     |  0  |  0  |
|    `:PPC`    |  0  |  1  |
|    `:REG`    |  1  |  1  |

## Keyword Arguments
* `Ttr = 200` : Extra "transient" time to evolve the system before sampling from
   the trajectory. Should be `Int` for discrete systems.
* `T_sample = 1e4` : Time to evolve the system for taking samples. Should be
  `Int` for discrete systems.
* `n_samples = 500` : Number of samples to take for use in calculating statistics.
* `λ_max = lyapunov(ds, 5000)` : Value to use for largest Lyapunov exponent
  for finding the Lyapunov prediction time. If it is less than zero a regular
  result is returned immediatelly.
* `d_tol = 1e-3` : tolerance distance to use for calculating Lyapunov prediction time.
* `T_multiplier = 10` : Multiplier from the Lyapunov prediction time to the evaluation time.
* `T_max = Inf` : Maximum time at which to evaluate trajectory distance. If the internally
   computed evaluation time is larger than `T_max`, stop at `T_max` instead.
* `δ_range = 10.0 .^ (-9:-6)` : Range of initial condition perturbation distances
   to use to determine scaling `ν`.
* `diffeq...` : Keyword arguments propagated into `init` of DifferentialEquations.jl.
  See [`trajectory`](@ref) for examples. Only valid for continuous systems.

## Description
Samples points from a trajectory of the system to be used as initial conditions. Each of
these initial conditions is randomly perturbed by a distance `δ`, and the trajectories for
both the original and perturbed initial conditions are computed to the 'evaluation time'
`T`.

The average (over the samples) distance and cross-correlation coefficient
of the state at time `T` is
computed. This is repeated for a range of `δ` (defined by `δ_range`), and linear
regression is used to determine how the distance and cross-correlation scale with `δ`,
allowing for identification of chaos type.

The evaluation time `T` is calculated as `T = T_multiplier*Tλ`, where the Lyapunov
prediction time `Tλ = log(d_tol/δ)/λ_max`. This may be very large if the `λ_max` is small,
e.g. when the system is regular, so this internally computed time `T` can be overridden by
a smaller `T_max` set by the user.

## Performance Notes
For continuous systems, it is likely that the `maxiters` used by the integrators needs to
be increased, e.g. to 1e9. This is part of the `diffeq` kwargs.
In addition, be aware that this function does a *lot* of internal computations.
It is operating in a different speed than e.g. [`lyapunov`](@ref).

[^Wernecke2017]: Wernecke, H., Sándor, B. & Gros, C. *How to test for partially predictable chaos*. [Scientific Reports **7**, (2017)](https://www.nature.com/articles/s41598-017-01083-x).
"""
function predictability(ds::DynamicalSystem;
                        Ttr::Real = 200,
                        T_sample::Real = 1e4,
                        n_samples::Integer = 500,
                        λ_max::Real = lyapunov(ds, 5000),
                        d_tol::Real = 1e-3,
                        T_multiplier::Real = 10,
                        T_max::Real = Inf,
                        δ_range::AbstractArray = 10.0 .^ (-9:-6),
                        diffeq...
                        )

    λ_max < 0 && return :REG, 1.0, 1.0
    # Internal Constants
    ν_threshold = 0.5
    C_threshold = 0.5

    # Sample points from a single trajectory of the system
    samples = sample_trajectory(ds, Ttr, T_sample, n_samples; diffeq...)

    # Calculate the mean position and variance of the trajectory. ([1] pg. 5)
    # Using samples 'Monte Carlo' approach instead of direct integration
    μ = mean(samples)
    s² = mean(map(x->(x-μ)⋅(x-μ), samples))

    # Calculate cross-distance scaling and correlation scaling
    distances = Float64[] # Mean distances at time T for different δ
    correlations = Float64[] # Cross-correlation at time T for different δ
    p_integ = parallel_integrator(ds, samples[1:2]; diffeq...)
    for δ in δ_range
        # TODO: some kind of warning should be thrown for very large Tλ
        Tλ = log(d_tol/δ)/λ_max
        T = min(T_multiplier * Tλ, T_max)
        Σd = 0
        Σd² = 0
        for u in samples
            # Sample perturbation from surface of sphere
            n = rand(Normal(), size(u))
            n /= norm(n)
            û = u + δ*n
            # Update integrator with new initial conditions
            reinit!(p_integ, [u, û])
            # Simulate trajectory until T
            while p_integ.t < T
                step!(p_integ)
            end
            # Accumulate distance and square-distance
            d = norm(p_integ.u[1]-p_integ.u[2], 2)
            Σd  += d
            Σd² += d^2
        end
        # Calculate mean distance and square-distance
        d = Σd/length(samples)
        D = Σd²/length(samples)
        # Convert mean square-distance into cross-correlation
        C = 1 - D/2s²
        # Store mean distance and cross-correlation
        push!(distances, d)
        push!(correlations, C)
    end

    # Perform regression to check cross-distance scaling
    ν = slope(log.(δ_range), log.(distances))
    C = mean(correlations)

    # Determine chaotic nature of the system
    if ν > ν_threshold && C > C_threshold
        chaos_type = :REG
    elseif ν <= ν_threshold && C > C_threshold
        chaos_type = :PPC
    elseif ν <= ν_threshold && C ≤ C_threshold
        chaos_type = :SC
    else
        # Covers the case when ν > ν_threshold but C <= C_threshold
        chaos_type = :INDETERMINATE
    end

    return chaos_type, ν, C
end

function sample_trajectory(ds::ContinuousDynamicalSystem,
                           Ttr::Real, T_sample::Real,
                           n_samples::Real;
                           diffeq...)
    # Samples *approximately* `n_samples` points.
    β = T_sample/n_samples
    D_sample = Exponential(β)
    sample_trajectory(ds, Ttr, T_sample, D_sample; diffeq...)
end

function sample_trajectory(ds::DiscreteDynamicalSystem,
                           Ttr::Real, T_sample::Real,
                           n_samples::Real;
                           diffeq...)
    @assert n_samples < T_sample "discrete systems must satisfy n_samples < T_sample"
    # Samples *approximately* `n_samples` points.
    p = n_samples/T_sample
    D_sample = Geometric(p)
    sample_trajectory(ds, Ttr, T_sample, D_sample; diffeq...)
end

function sample_trajectory(ds::DynamicalSystem,
                           Ttr::Real, T_sample::Real,
                           D_sample::UnivariateDistribution;
                           diffeq...)
    # Simulate initial transient
    integ = integrator(ds; diffeq...)
    while integ.t < Ttr
        step!(integ)
    end

    # Time to the next sample is sampled from the distribution D_sample
    # e.g. Continuous systems: D_sample is Exponential distribution
    samples = typeof(integ.u)[]
    while integ.t < Ttr + T_sample
        step!(integ, rand(D_sample))
        push!(samples, integ.u)
    end
    samples
end


text:
export rand_controller

function rand_controller(cfg::Dict)
    Random.seed!(cfg["seed"])
    nouts = output_lengths(cfg)
    cell_division(x::Array{Float64}) = rand(Bool)
    new_cell_params(x::Array{Float64}) = rand(nouts[2])
    cell_death(x::Array{Float64}) = rand(Bool)
    cell_state_update(x::Array{Float64}) = rand(nouts[4])
    cell_param_update(x::Array{Float64}) = rand(nouts[5])
    connect(x::Array{Float64}) = rand(Bool)
    new_conn_params(x::Array{Float64}) = rand(nouts[7])
    disconnect(x::Array{Float64}) = rand(Bool)
    conn_state_update(x::Array{Float64}) = rand(nouts[9])
    conn_param_update(x::Array{Float64}) = rand(nouts[10])

    Controller(cell_division, new_cell_params, cell_death,
               cell_state_update, cell_param_update,
               connect, new_conn_params, disconnect,
               conn_state_update, conn_param_update)
end


text:
using Test

using Distributions, LinearAlgebra, Statistics
using EnKF

@testset "Initialize" begin

A = MvNormal(zeros(4), I)
ENS = initialize(10, A)

@test size(ENS) == (10,(4,))

end

max_stars_repo_name: mleprovost/EnKF

max_issues_repo_name: mleprovost/EnKF

max_forks_repo_name: mleprovost/EnKF


text:
using PiecewiseDeterministicMarkovProcesses, LinearAlgebra, Random, DifferentialEquations, Sundials
	const PDMP = PiecewiseDeterministicMarkovProcesses

function AnalyticalSampleCHV(xc0, xd0, ti, nj::Int64)
	xch = [xc0[1]]
	xdh = [xd0[1]]
	th  = [ti]
	t = ti
	while length(th)<nj
		xc = xch[end]
		xd = xdh[end]
		S = -log(rand())
		if mod(xd,2) == 0
			t += 1/10*log(1+10*S/xc)
			push!(xch,xc + 10 * S )
		else
			t += 1/(3xc)*(exp(3S)-1)
			push!(xch,xc * exp(-3S) )
		end
		push!(xdh,xd + 1 )
		push!(th,t)

		S = -log(rand())
	end
	return th, xch, xdh
end

function F!(ẋ, xc, xd, parms, t)
	if mod(xd[1], 2)==0
		ẋ[1] = 10xc[1]
	else
		ẋ[1] = -3xc[1]^2
	end
end

R(x) = x

function R!(rate, xc, xd, parms, t, issum::Bool)
	# rate function
	if issum == false
		rate[1] = R(xc[1])
		rate[2] = parms[1]
		return 0., parms[1] + 50.
	else
		return R(xc[1]) + parms[1], parms[1] + 50.
	end
end

xc0 = [1.0]
	xd0 = [0, 0]

	nu = [1 0;0 -1]
	parms = [.0]
	ti = 0.322156
	tf = 100000.
	nj = 50

errors = Float64[]

Random.seed!(8)
	res_a_chv = AnalyticalSampleCHV(xc0,xd0,ti,nj)

problem = PDMP.PDMPProblem(F!, R!, nu, xc0, xd0, parms, (ti, tf))
println("\n\nComparison of solvers")
	for ode in [(:cvode,"cvode"),(:lsoda,"lsoda"),(CVODE_BDF(),"CVODEBDF"),(CVODE_Adams(),"CVODEAdams"),(Tsit5(),"tsit5"),(Rodas4P(autodiff=false),"rodas4P-noAutoDiff"),(Rodas4P(),"rodas4P-AutoDiff"),(Rosenbrock23(),"RS23"),(AutoTsit5(Rosenbrock23(autodiff=true)),"AutoTsit5-RS23")]
	Random.seed!(8)
	res =  PDMP.solve(problem, CHV(ode[1]); n_jumps = nj)
	println("--> norm difference = ", norm(res.time - res_a_chv[1], Inf64), "  - solver = ",ode[2])
	push!(errors,norm(res.time - res_a_chv[1],Inf64))
end


text:
using mFEM
using Test

@testset "mFEM.jl" begin
    # Write your tests here.
end


text:
module Fields

export
    Face, Cell,
    AbstractField, Field, CellField, XFaceField, YFaceField, ZFaceField,
    ReducedField, AveragedField, ComputedField,
    interior, interiorparent, data,
    xnode, ynode, znode, location,
    set!, compute!, @compute,
    VelocityFields, TracerFields, tracernames, PressureFields, TendencyFields

using Oceananigans.Architectures
using Oceananigans.Grids
using Oceananigans.BoundaryConditions

Base.zeros(FT, ::CPU, Nx, Ny, Nz) = zeros(FT, Nx, Ny, Nz)
Base.zeros(FT, ::GPU, Nx, Ny, Nz) = zeros(FT, Nx, Ny, Nz) |> CuArray
Base.zeros(arch, grid, Nx, Ny, Nz) = zeros(eltype(grid), arch, Nx, Ny, Nz)

include("new_data.jl")
include("abstract_field.jl")
include("field.jl")
include("reduced_field.jl")
include("averaged_field.jl")
include("computed_field.jl")
include("pressure_field.jl")
include("function_field.jl")
include("set!.jl")
include("field_tuples.jl")
include("show_fields.jl")

end


text:
"""
abstract ParameterState{S<:ValueSupport, F<:VariateForm} <: VariableState{F}

Root of parameter state type hierarchy
"""
abstract ParameterState{S<:ValueSupport, F<:VariateForm} <: VariableState{F}

typealias DiscreteParameterState{F<:VariateForm} ParameterState{Discrete, F}
typealias ContinuousParameterState{F<:VariateForm} ParameterState{Continuous, F}

typealias UnivariateParameterState{S<:ValueSupport} ParameterState{S, Univariate}
typealias MultivariateParameterState{S<:ValueSupport} ParameterState{S, Multivariate}
typealias MatrixvariateParameterState{S<:ValueSupport} ParameterState{S, Matrixvariate}

typealias ParameterStateVector{S<:ParameterState} Vector{S}

value_support{S<:ValueSupport, F<:VariateForm}(::Type{ParameterState{S, F}}) = S
variate_form{S<:ValueSupport, F<:VariateForm}(::Type{ParameterState{S, F}}) = F

diagnostics(state::ParameterState) = Dict(zip(state.diagnostickeys, state.diagnosticvalues))

Base.(:(==)){S<:ParameterState}(z::S, w::S) = reduce(&, [getfield(z, n) == getfield(w, n) for n in fieldnames(S)])

Base.isequal{S<:ParameterState}(z::S, w::S) = reduce(&, [isequal(getfield(z, n), getfield(w, n)) for n in fieldnames(S)])


text:
# compression detection
const ZIP = UInt8[0x50, 0x4b, 0x03, 0x04]
const GZIP = UInt8[0x1f, 0x8b, 0x08]

iscompressed(bytes::CodeUnits) = length(bytes) > 3 && (all(bytes[1:4] .== ZIP) || all(bytes[1:3] .== GZIP))
iscompressed(str::String) = iscompressed(bytes(str))
iscompressed(d::Dict) = false
iscompressed(d) = false

# Based on the net/http/sniff.go implementation of DetectContentType
# sniff implements the algorithm described
# at http://mimesniff.spec.whatwg.org/ to determine the
# Content-Type of the given data. It considers at most the
# first 512 bytes of data. sniff always returns
# a valid MIME type: if it cannot determine a more specific one, it
# returns "application/octet-stream".
const MAXSNIFFLENGTH = 512
const WHITESPACE = Set{UInt8}([UInt8('\t'),UInt8('\n'),UInt8('\u0c'),UInt8('\r'),UInt8(' ')])

"""
`HTTP.sniff(content::Union{Vector{UInt8}, String, IO})` => `String` (mimetype)

`HTTP.sniff` will look at the first 512 bytes of `content` to try and determine a valid mimetype.
If a mimetype can't be determined appropriately, `"application/octet-stream"` is returned.

Supports JSON detection through the `HTTP.isjson(content)` function.

## Examples
```julia
julia> HTTP.sniff("Hello world!!")
"text/plain; charset=utf-8"

julia> HTTP.sniff("<html><body>Hello world!!</body></html>")
"text/html; charset=utf-8"

julia> HTTP.sniff("{\"a\": -1.0}")
"application/json; charset=utf-8"
```
"""
function sniff end

function sniff(body::IO)
    alreadymarked = ismarked(body)
    mark(body)
    data = bytes(read(body, MAXSNIFFLENGTH))
    reset(body)
    alreadymarked && mark(body)
    return sniff(data)
end

sniff(str::String) = sniff(bytes(str)[1:min(length(bytes(str)), MAXSNIFFLENGTH)])

function sniff(data::CodeUnits)
    firstnonws = 1
    while firstnonws < length(data) && data[firstnonws] in WHITESPACE
        firstnonws += 1
    end

    for sig in SNIFF_SIGNATURES
        ismatch(sig, data, firstnonws) && return contenttype(sig)
    end
    return "application/octet-stream" # fallback
end

struct Exact
    sig::CodeUnits
    contenttype::String
end
contenttype(e::Exact) = e.contenttype

function ismatch(e::Exact, data::CodeUnits, firstnonws)
    length(data) < length(e.sig) && return false
    for i = 1:length(e.sig)
        e.sig[i] == data[i] || return false
    end
    return true
end

struct Masked
    mask::CodeUnits
    pat::CodeUnits
    skipws::Bool
    contenttype::String
end
Masked(mask::CodeUnits, pat::CodeUnits, contenttype::String) = Masked(mask, pat, false, contenttype)

contenttype(m::Masked) = m.contenttype

function ismatch(m::Masked, data::CodeUnits, firstnonws)
    # pattern matching algorithm section 6
    # https://mimesniff.spec.whatwg.org/#pattern-matching-algorithm
    sk = (m.skipws ? firstnonws : 1) - 1
    length(m.pat) != length(m.mask) && return false
    length(data) < length(m.mask) && return false
    for (i, mask) in enumerate(m.mask)
        (data[i+sk] & mask) != m.pat[i] && return false
    end
    return true
end

struct HTMLSig
    html::CodeUnits
    HTMLSig(str::String) = new(bytes(str))
end

contenttype(h::HTMLSig) = "text/html; charset=utf-8"

function ismatch(h::HTMLSig, data::CodeUnits, firstnonws)
    length(data) < length(h.html)+1 && return false
    for (i, b) in enumerate(h.html)
        db = data[i+firstnonws-1]
        (UInt8('A') <= b && b <= UInt8('Z')) && (db &= 0xDF)
        b != db && return false
    end
    data[length(h.html)+firstnonws] in (UInt8(' '), UInt8('>')) || return false
    return true
end

struct MP4Sig end
contenttype(::Type{MP4Sig}) = "video/mp4"

function byteequal(data1, ind, data2)
    for i = 1:length(data2)
        @inbounds data1[ind+i-1] == data2[i] || return false
    end
    return true
end

const mp4ftype = bytes("ftyp")
const mp4 = bytes("mp4")

# Byte swap int
bigend(b) = UInt32(b[4]) | UInt32(b[3])<<8 | UInt32(b[2])<<16 | UInt32(b[1])<<24

function ismatch(::Type{MP4Sig}, data::CodeUnits, firstnonws)
    # https://mimesniff.spec.whatwg.org/#signature-for-mp4
    # c.f. section 6.2.1
    length(data) < 12 && return false
    boxsize = Int(bigend(data))
    (boxsize % 4 != 0 || length(data) < boxsize) && return false
    byteequal(data, 5, mp4ftype) || return false
    for st = 9:4:boxsize+1
        st == 13 && continue
        byteequal(data, st, mp4) && return true
    end
    return false
end

struct TextSig end
contenttype(::Type{TextSig}) = "text/plain; charset=utf-8"

function ismatch(::Type{TextSig}, data::CodeUnits, firstnonws)
    # c.f. section 5, step 4.
    for i = firstnonws:min(length(data),MAXSNIFFLENGTH)
        b = data[i]
        (b <= 0x08 || b == 0x0B ||
        0x0E <= b <= 0x1A ||
        0x1C <= b <= 0x1F) && return false
    end
    return true
end

struct JSONSig end
contenttype(::Type{JSONSig}) = "application/json; charset=utf-8"

ismatch(::Type{JSONSig}, data::CodeUnits, firstnonws) = isjson(data)[1]

const DISPLAYABLE_TYPES = ["text/html; charset=utf-8",
                    "text/plain; charset=utf-8",
                    "application/json; charset=utf-8",
                    "text/xml; charset=utf-8",
                    "text/plain; charset=utf-16be",
                    "text/plain; charset=utf-16le"]

# Data matching the table in section 6.
const SNIFF_SIGNATURES = [
    HTMLSig("<!DOCTYPE HTML"),
    HTMLSig("<HTML"),
    HTMLSig("<HEAD"),
    HTMLSig("<SCRIPT"),
    HTMLSig("<IFRAME"),
    HTMLSig("<H1"),
    HTMLSig("<DIV"),
    HTMLSig("<FONT"),
    HTMLSig("<TABLE"),
    HTMLSig("<A"),
    HTMLSig("<STYLE"),
    HTMLSig("<TITLE"),
    HTMLSig("<B"),
    HTMLSig("<BODY"),
    HTMLSig("<BR"),
    HTMLSig("<P"),
    HTMLSig("<!--"),
    Masked([0xff,0xff,0xff,0xff,0xff], bytes("<?xml"), true, "text/xml; charset=utf-8"),
    Exact(bytes("%PDF-"), "application/pdf"),
    Exact(bytes("%!PS-Adobe-"), "application/postscript"),

    # UTF BOMs.
    Masked([0xFF,0xFF,0x00,0x00], [0xFE,0xFF,0x00,0x00], "text/plain; charset=utf-16be"),
    Masked([0xFF,0xFF,0x00,0x00], [0xFF,0xFE,0x00,0x00], "text/plain; charset=utf-16le"),
    Masked([0xFF,0xFF,0xFF,0x00], [0xEF,0xBB,0xBF,0x00], "text/plain; charset=utf-8"),

    Exact(bytes("GIF87a"), "image/gif"),
    Exact(bytes("GIF89a"), "image/gif"),
    Exact([0x89,0x50,0x4E,0x47,0x0D,0x0A,0x1A,0x0A], "image/png"),
    Exact([0xFF,0xD8,0xFF], "image/jpeg"),
    Exact(bytes("BM"), "image/bmp"),
    Masked([0xFF,0xFF,0xFF,0xFF,0x00,0x00,0x00,0x00,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF],
           UInt8['R','I','F','F',0x00,0x00,0x00,0x00,'W','E','B','P','V','P'],
           "image/webp"),
    Exact([0x00,0x00,0x01,0x00], "image/vnd.microsoft.icon"),
    Masked([0xFF,0xFF,0xFF,0xFF,0x00,0x00,0x00,0x00,0xFF,0xFF,0xFF,0xFF],
           UInt8['R','I','F','F',0x00,0x00,0x00,0x00,'W','A','V','E'],
           "audio/wave"),
    Masked([0xFF,0xFF,0xFF,0xFF,0x00,0x00,0x00,0x00,0xFF,0xFF,0xFF,0xFF],
           UInt8['F','O','R','M',0x00,0x00,0x00,0x00,'A','I','F','F'],
           "audio/aiff"),
    Masked([0xFF,0xFF,0xFF,0xFF],
           bytes(".snd"),
           "audio/basic"),
    Masked(UInt8['O','g','g','S',0x00],
           UInt8[0x4F,0x67,0x67,0x53,0x00],
           "application/ogg"),
    Masked([0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF],
           UInt8['M','T','h','d',0x00,0x00,0x00,0x06],
           "audio/midi"),
    Masked([0xFF,0xFF,0xFF],
            bytes("ID3"),
           "audio/mpeg"),
    Masked([0xFF,0xFF,0xFF,0xFF,0x00,0x00,0x00,0x00,0xFF,0xFF,0xFF,0xFF],
           UInt8['R','I','F','F',0x00,0x00,0x00,0x00,'A','V','I',' '],
        "video/avi"),
    Exact([0x1A,0x45,0xDF,0xA3], "video/webm"),
    Exact([0x52,0x61,0x72,0x20,0x1A,0x07,0x00], "application/x-rar-compressed"),
    Exact([0x50,0x4B,0x03,0x04], "application/zip"),
    Exact([0x1F,0x8B,0x08], "application/x-gzip"),
    MP4Sig,
    JSONSig,
    TextSig, # should be last
]

function ignorewhitespace(bytes, i, maxlen)
    while true
        eof, b, i = nextbyte(bytes, i, maxlen)
        eof && return true, b, i
        b in WHITESPACE || return false, b, i
    end
end

function nextbyte(bytes, i, maxlen)
    i += 1
    i >= maxlen && return true, 0x00, i
    @inbounds b = bytes[i]
    return false, b, i
end

function restofstring(bytes, i, maxlen)
    while true
        eof, b, i = nextbyte(bytes, i, maxlen)
        eof && return i
        b == DOUBLE_QUOTE && return i
        if b == ESCAPE
            eof, b, i = nextbyte(bytes, i, maxlen)
        end
    end
end

macro expect(ch)
    return esc(quote
        eof, b, i = ignorewhitespace(bytes, i, maxlen)
        eof && return true, i
        b == $ch || return false, i
    end)
end

const OPEN_CURLY_BRACE  = UInt8('{')
const CLOSE_CURLY_BRACE  = UInt8('}')
const OPEN_SQUARE_BRACE  = UInt8('[')
const CLOSE_SQUARE_BRACE  = UInt8(']')
const DOUBLE_QUOTE  = UInt8('"')
const ESCAPE        = UInt8('\\')
const COMMA = UInt8(',')
const COLON = UInt8(':')
const MINUS = UInt8('-')
const ZERO = UInt8('0')
const NINE = UInt8('9')
const LITTLE_N = UInt8('n')
const LITTLE_U = UInt8('u')
const LITTLE_L = UInt8('l')
const LITTLE_T = UInt8('t')
const LITTLE_R = UInt8('r')
const LITTLE_E = UInt8('e')
const LITTLE_F = UInt8('f')
const LITTLE_A = UInt8('a')
const LITTLE_S = UInt8('s')
const PERIOD = UInt8('.')

const REF = Vector{Ptr{UInt8}}(undef, 1)

function isjson(bytes, i=0, maxlen=min(length(bytes), MAXSNIFFLENGTH))
    # ignore leading whitespace
    isempty(bytes) && return false, 0
    eof, b, i = ignorewhitespace(bytes, i, maxlen)
    eof && return true, i
    # must start with:
    if b == OPEN_CURLY_BRACE
        # '{' start of object
        # must then read a string key, potential whitespace, then colon, potential whitespace then recursively check `isjson`
        while true
            @expect DOUBLE_QUOTE
            i = restofstring(bytes, i, maxlen)
            @expect COLON
            ret, i = isjson(bytes, i, maxlen)
            ret || return false, i
            eof, b, i = ignorewhitespace(bytes, i, maxlen)
            (eof || b == CLOSE_CURLY_BRACE) && return true, i
            b != COMMA && return false, i
        end
    elseif b == OPEN_SQUARE_BRACE
        # '[' start of array
        # peek at next byte to check for empty array
        ia = i
        eof, b, i = nextbyte(bytes, i, maxlen)
        if b != CLOSE_SQUARE_BRACE
            i = ia
            # recursively check `isjson`, then potential whitespace, then ',' or ']'
            while true
                ret, i = isjson(bytes, i, maxlen)
                ret || return false, i
                eof, b, i = ignorewhitespace(bytes, i, maxlen)
                (eof || b == CLOSE_SQUARE_BRACE) && return true, i
                b != COMMA && return false, i
            end
        end
    elseif b == DOUBLE_QUOTE
        # '"' start of string
        # must read until end of string w/ potential escaped '"'
        i = restofstring(bytes, i, maxlen)
    elseif (ZERO <= b <= NINE) || (b == MINUS)
        # must read until end of number
        v = zero(Float64)
        ptr = pointer(bytes) + i - 1
        v = ccall(:jl_strtod_c, Float64, (Ptr{UInt8}, Ptr{Ptr{UInt8}}), ptr, REF)
        i += REF[1] - ptr - 1
    elseif b == LITTLE_N
        # null
        @expect LITTLE_U
        @expect LITTLE_L
        @expect LITTLE_L
    elseif b == LITTLE_T
        # true
        @expect LITTLE_R
        @expect LITTLE_U
        @expect LITTLE_E
    elseif b == LITTLE_F
        # false
        @expect LITTLE_A
        @expect LITTLE_L
        @expect LITTLE_S
        @expect LITTLE_E
    else
        return false, i
    end
    return true, i
end


text:
function Mass2DMatrix(mesh)
    return assembleScalar(mesh,localMass2D!,0.0)
end
function Mass2DMatrix(mesh,scalarFunc)
    return assembleScalar(mesh,localMassWithScalar2D!,scalarFunc)
end

function Laplace2DMatrix(mesh,farr,param::T) where T<:AbstractDummyParameter
    D = assembleScalar(mesh,localLaplace2D!,0.0)
    F = WeakScalar2D(mesh,farr)
    return D,F
end

function Darcy2DMatrix(mesh,farr,param::T) where T<:AbstractConstantParameter
    D = assembleScalar(mesh,localLaplace2D!,0.0)
    D = param.α*D
    F = WeakScalar2D(mesh,farr)
    return D,F
end
function Darcy2DMatrix(mesh,farr,param::T) where T<:AbstractVariableParameter
    D = assembleScalar(mesh,localDarcy2D!,param.α)
    F = WeakScalar2D(mesh,farr)
    return D,F
end
function Darcy2DMatrix_2b(mesh,farr,param::T,prob) where T<:AbstractVariableParameter
    D, F = assembleScalarandRHS_2b(mesh,localDarcy2D!,param.α,prob)
    return D, F
end

function AdvDiff2DMatrix(mesh,farr,param::T) where T<:AbstractConstantParameter
    D = assembleScalar(mesh,localLaplace2D!,0.0)
    A = assembleScalar(mesh,localAdvDiff2D!,param)
    F = WeakScalar2D(mesh,farr)
    Stiff = A + D/param.Pe
    return Stiff, F
end
function AdvDiff2DMatrix(mesh,farr,param::T) where T<:AbstractVariableParameter
    D = assembleScalar(mesh,localDarcy2D!,param.Pe)
    A = assembleScalar(mesh,localAdvDiff2D!,param)
    F = WeakScalar2D(mesh,farr)
    Stiff = A + D
    return Stiff, F
end

function Stokes2DMatrix(mesh::FluidMesh,prob,param::T) where T<:AbstractConstantParameter
    Stiff = assembleHalfFluid(mesh,localStokesConst2D!,param.μ)
    Fx    = WeakScalar2D(mesh,prob.bcval[:forcingX])
    Fy    = WeakScalar2D(mesh,prob.bcval[:forcingY])
    G     = zeros(length(mesh.xyp))
    F     = vcat(Fx,Fy,G)

    return Stiff, F
end
function Stokes2DMatrix(mesh::FluidMesh,prob,param::T) where T<:AbstractVariableParameter
    error("Stokes 2D variable viscosity not working right now")

    # old original -- THIS ONE IS WORKING (? need to test this)
    Stiff = assembleFullFluid_WRONG(mesh,localStokesVar2D_WRONG!,param.μ)
    Fx    = WeakScalar2D(mesh,prob.bcval[:forcingX])
    Fy    = WeakScalar2D(mesh,prob.bcval[:forcingY])
    G     = zeros(length(mesh.xyp))
    F     = vcat(Fx,Fy,G)
    return Stiff, F
end

function Brinkman2DMatrix(mesh::FluidMesh,prob,param::T) where T<:AbstractConstantParameter
    Stiff = assembleHalfFluid(mesh,localBrinkmanConst2D!,param.μ,param.α)
    Fx    = WeakScalar2D(mesh,prob.bcval[:forcingX])
    Fy    = WeakScalar2D(mesh,prob.bcval[:forcingY])
    G     = zeros(length(mesh.xyp))
    F     = vcat(Fx,Fy,G)

    return Stiff, F
end
function Brinkman2DMatrix(mesh::FluidMesh,prob,param::T) where T<:AbstractVariableParameter
    Stiff = assembleFullFluid(mesh,localBrinkmanVar2D!,param.μ,param.α)
    Fx    = WeakScalar2D(mesh,prob.bcval[:forcingX])
    Fy    = WeakScalar2D(mesh,prob.bcval[:forcingY])
    G     = zeros(length(mesh.xyp))
    F     = vcat(Fx,Fy,G)

    return Stiff, F
end

"""
Generates matrix operator to be used to solve brinkman-like equations
  -α1*lap(u) + α2*u + α3*grad(p) = f
"""
function BrinkmanMP2DMatrix(mesh::FluidMesh,prob,param::T) where T<:AbstractConstantParameter
	Stiff = assembleHalfFluid(mesh,localBrinkmanMPConst2D!,param.α1,param.α2,param.α3)
	Fx    = WeakScalar2D(mesh,prob.bcval[:forcingX])
	Fy    = WeakScalar2D(mesh,prob.bcval[:forcingY])
	G     = zeros(length(mesh.xyp))
	F     = vcat(Fx,Fy,G)

	return Stiff, F
end
function BrinkmanMP2DMatrix(mesh::FluidMesh,prob,param::T) where T<:AbstractVariableParameter
    Stiff = assembleMPB(mesh,localMPBVar2D!,param.α1,param.α2,param.α3)
	Fx    = WeakScalar2D(mesh,prob.bcval[:forcingX])
	Fy    = WeakScalar2D(mesh,prob.bcval[:forcingY])
	G     = zeros(length(mesh.xyp))
	F     = vcat(Fx,Fy,G)

	return Stiff, F
end

function LaplaceASMatrix(mesh,farr,param)
    D     = assembleScalar(mesh,localLaplaceConstAS!,param.κ)
    Stiff = D
    F     = WeakScalarAS(mesh, farr)
    return Stiff, F
end

function AdvDiffASMatrix(mesh,farr,param::T) where T<:AbstractVariableParameter
    D     = assembleScalar(mesh,localLaplaceVarAS!,param.Pe)
    A     = assembleScalar(mesh,localAdvDiffAS!,param)
    Stiff = A + D
    F     = WeakScalarAS(mesh,farr)
    return Stiff, F
end

function StokesASMatrix(mesh,prob,param::T) where T<:AbstractConstantParameter
    Stiff = assembleHalfFluid(mesh,localStokesConstAS!,param.μ)
    Fx    = WeakScalarAS(mesh,prob.bcval[:forcingX])
    Fy    = WeakScalarAS(mesh,prob.bcval[:forcingY])
    G     = zeros(Float64,length(mesh.xyp))
    F     = vcat(Fx,Fy,G)
    return Stiff, F
end
function StokesASMatrix(mesh,prob,param::T) where T<:AbstractVariableParameter
    Stiff = assembleFullFluid_WRONG(mesh,localStokesVarAS!,param.μ)
    Fx    = WeakScalarAS(mesh,prob.bcval[:forcingX])
    Fy    = WeakScalarAS(mesh,prob.bcval[:forcingY])
    G     = zeros(Float64,length(mesh.xyp))
    F     = vcat(Fx,Fy,G)
    return Stiff, F
end


text:
include("Sparse/Sparse.jl")
include("Sparse/Matrix.jl")
include("Sparse/Row.jl")
include("Sparse/Module.jl")
include("Sparse/Trafo.jl")
include("Sparse/HNF.jl")
include("Sparse/Solve.jl")
include("Sparse/UpperTriangular.jl")


text:
# ---
# title: 1187. Make Array Strictly Increasing
# id: problem1187
# author: Tian Jun
# date: 2020-10-31
# difficulty: Hard
# categories: Dynamic Programming
# link: <https://leetcode.com/problems/make-array-strictly-increasing/description/>
# hidden: true
# ---
#
# Given two integer arrays `arr1` and `arr2`, return the minimum number of
# operations (possibly zero) needed to make `arr1` strictly increasing.
#
# In one operation, you can choose two indices `0 <= i < arr1.length` and `0 <=
# j < arr2.length` and do the assignment `arr1[i] = arr2[j]`.
#
# If there is no way to make `arr1` strictly increasing, return `-1`.
#
#
#
# **Example 1:**
#
#
#
#     Input: arr1 = [1,5,3,6,7], arr2 = [1,3,2,4]
#     Output: 1
#     Explanation: Replace 5 with 2, then arr1 = [1, 2, 3, 6, 7].
#
#
# **Example 2:**
#
#
#
#     Input: arr1 = [1,5,3,6,7], arr2 = [4,3,1]
#     Output: 2
#     Explanation: Replace 5 with 3 and then replace 3 with 4. arr1 = [1, 3, 4, 6, 7].
#
#
# **Example 3:**
#
#
#
#     Input: arr1 = [1,5,3,6,7], arr2 = [1,6,3,3]
#     Output: -1
#     Explanation: You can't make arr1 strictly increasing.
#
#
#
# **Constraints:**
#
#   * `1 <= arr1.length, arr2.length <= 2000`
#   * `0 <= arr1[i], arr2[i] <= 10^9`
#
#
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


text:
using CompScienceMeshes, BEAST
using LinearAlgebra

Γ = meshcuboid(1.0, 1.0, 1.0, 0.15)
X = raviartthomas(Γ)

κ,  η  = 1.0, 1.0
κ′, η′ = 1.4κ, η/1.4

T  = Maxwell3D.singlelayer(wavenumber=κ)
T′ = Maxwell3D.singlelayer(wavenumber=κ′)
K  = Maxwell3D.doublelayer(wavenumber=κ)
K′ = Maxwell3D.doublelayer(wavenumber=κ′)

# d = normalize(x̂+ŷ+ẑ)
d = ẑ
# p = normalize(d × ẑ)
p = x̂
E = Maxwell3D.planewave(direction=d, polarization=p, wavenumber=κ)
H = -1/(im*κ*η)*curl(E)

e, h = (n × E) × n, (n × H) × n

@hilbertspace j m
@hilbertspace k l

α, α′ = 1/η, 1/η′
pmchwt = @discretise(
    (η*T+η′*T′)[k,j] +      (K+K′)[k,m] -
         (K+K′)[l,j] + (α*T+α′*T′)[l,m] == h[k] + e[l],
    j∈X, m∈X, k∈X, l∈X)

u = solve(pmchwt)

nX = numfunctions(X)
uj = u[1:nX]
um = u[nX+1:end]

Θ, Φ = range(0.0,stop=2π,length=100), 0.0
ffpoints = [point(cos(ϕ)*sin(θ), sin(ϕ)*sin(θ), cos(θ)) for θ in Θ for ϕ in Φ]

# Don't forgt the far field comprises two contributions
ffm = potential(MWFarField3D(κ*im), ffpoints, um, X)
ffj = potential(MWFarField3D(κ*im), ffpoints, uj, X)
ff = η*im*κ*ffj + im*κ*cross.(ffpoints, ffm)

using Plots
plot(xlabel="theta")
plot!(Θ,norm.(ff),label="far field")

import PlotlyJS
using LinearAlgebra
fcrj, _ = facecurrents(uj,X)
PlotlyJS.plot(patch(Γ, norm.(fcrj)))


text:
####
#### automatically generate a list of integrations
####

###
### Usage
###
### 1. ensure a development version of Tables.jl (`pkg> dev Tables`) in `DEPOT_PATH[1]/dev/Tables`
### 2. make sure the General registry is up to date (`pkg> up`)
### 3. run this script, which uses the first depot from DEPOT_PATH

DEPOT = first(DEPOT_PATH)
REGISTRIES = joinpath(DEPOT, "registries")

# find each package w/ a direct dependency on Tables.jl
pkgs = cd(REGISTRIES) do
    dirname.(readlines(`grep -rl ^Tables General/. --include=Deps.toml`))
end

pkgnames = sort([splitpath(x)[end] for x in pkgs])

function parseurl(file)
    repo = readlines(file)[end]
    return strip(split(repo, " = ")[end], '"')
end

urls = [parseurl(joinpath(REGISTRIES, "General", string(first(nm)), nm, "Package.toml"))
        for nm in pkgnames]

open(joinpath(DEPOT, "dev", "Tables", "INTEGRATIONS.md"), "w+") do io
    println(io, "Packages currently integrating with Tables.jl:")
    for (nm, url) in zip(pkgnames, urls)
        println(io, "* [$nm]($url)")
    end
end


text:
export visualize

using DifferentiableNAS
using Flux
using CUDA
using LightGraphs
using GraphPlot
using Cairo
using Compose
using BSON
using LaTeXStrings
using PyCall
graphviz = pyimport("graphviz")
include("training_utils.jl")

function visualize(αs, filename::String)
    nodelabels = ["c_{k-2}", "c_{k-1}", "0", "1", "2", "3", "c_{k}"]
    inputindices, _, opnames = discretize(αs, 1, false, 4; plotting = true)
    g = graphviz.Digraph(
          format="png",
          edge_attr=Dict("fontsize"=>"20", "fontname"=>"times"),
          node_attr=Dict("style"=>"filled", "shape"=>"rect", "align"=>"center", "fontsize"=>"20", "height"=>"0.5", "width"=>"0.5", "penwidth"=>"2", "fontname"=>"times"),
          engine="dot")
    g.body = ["rankdir=LR"]

    g.node(nodelabels[1], fillcolor="darkseagreen2")
    g.node(nodelabels[2], fillcolor="darkseagreen2")

    for i in 3:6
      g.node(nodelabels[i], fillcolor="lightblue")
    end

    for i in 1:length(inputindices)
        g.edge(nodelabels[inputindices[i][1]], nodelabels[i+2], label=opnames[i][1], fillcolor="gray")
        g.edge(nodelabels[inputindices[i][2]], nodelabels[i+2], label=opnames[i][2], fillcolor="gray")
    end

    g.node(nodelabels[7], fillcolor="palegoldenrod")
    for i in 3:6
      g.edge(nodelabels[i], nodelabels[7], fillcolor="gray")
    end
    g.render(filename, view=false)
end

function visualize(filename::String)
    nodelabels = ["c_{k-2}", "c_{k-1}", "0", "1", "2", "3", "c_{k}"]
    inputindices, _, opnames = parse_genotype(channels=1, reduce=false, steps=4)
    g = graphviz.Digraph(
          format="png",
          edge_attr=Dict("fontsize"=>"20", "fontname"=>"times"),
          node_attr=Dict("style"=>"filled", "shape"=>"rect", "align"=>"center", "fontsize"=>"20", "height"=>"0.5", "width"=>"0.5", "penwidth"=>"2", "fontname"=>"times"),
          engine="dot")
    g.body = ["rankdir=LR"]

    g.node(nodelabels[1], fillcolor="darkseagreen2")
    g.node(nodelabels[2], fillcolor="darkseagreen2")

    for i in 3:6
      g.node(nodelabels[i], fillcolor="lightblue")
    end

    for i in 1:length(inputindices)
        g.edge(nodelabels[inputindices[i][1]], nodelabels[i+2], label=opnames[i][1], fillcolor="gray")
        g.edge(nodelabels[inputindices[i][2]], nodelabels[i+2], label=opnames[i][2], fillcolor="gray")
    end

    g.node(nodelabels[7], fillcolor="palegoldenrod")
    for i in 3:6
      g.edge(nodelabels[i], nodelabels[7], fillcolor="gray")
    end
    g.render(filename, view=false)
end

function visualize_supercell(filename::String)
    nodelabels = ["c_{k-2}", "c_{k-1}", "0", "1", "2", "3", "c_{k}"]
    g = graphviz.Digraph(
          format="png",
          edge_attr=Dict("fontsize"=>"20", "fontname"=>"times"),
          node_attr=Dict("style"=>"filled", "shape"=>"rect", "align"=>"center", "fontsize"=>"20", "height"=>"0.5", "width"=>"0.5", "penwidth"=>"2", "fontname"=>"times"),
          engine="dot")
    g.body = ["rankdir=LR"]

    g.node(nodelabels[1], fillcolor="darkseagreen2")
    g.node(nodelabels[2], fillcolor="darkseagreen2")

    for i in 3:6
      g.node(nodelabels[i], fillcolor="lightblue")
    end

    connects = vcat([[(j,i) for j = 1:i-1] for i = 3:6]...)

    for connect in connects
        g.edge(nodelabels[connect[1]], nodelabels[connect[2]], fillcolor="black", penwidth="3")
    end

    g.node(nodelabels[7], fillcolor="palegoldenrod")
    for i in 3:6
      g.edge(nodelabels[i], nodelabels[7], fillcolor="gray")
    end
    g.render(filename, view=false)
end

max_stars_repo_name: huangyxi/DifferentiableNAS

max_issues_repo_name: huangyxi/DifferentiableNAS

max_forks_repo_name: huangyxi/DifferentiableNAS


text:
using KNITRO, Test
using MathOptInterface
const MOI = MathOptInterface
const MOIT = MOI.Test
const MOIU = MOI.Utilities
const MOIB = MOI.Bridges

MOIU.@model(KnitroModelData,
            (),
            (MOI.EqualTo, MOI.GreaterThan, MOI.LessThan, MOI.Interval),
            (MOI.Zeros, MOI.Nonnegatives, MOI.Nonpositives, MOI.ZeroOne, MOI.SecondOrderCone),
            (),
            (MOI.SingleVariable,),
            (MOI.ScalarAffineFunction, MOI.ScalarQuadraticFunction),
            (MOI.VectorOfVariables,),
            (MOI.VectorAffineFunction, ))

const config = MOIT.TestConfig(atol=1e-4, rtol=1e-4,
                               optimal_status=MOI.LOCALLY_SOLVED)

@testset "MOI Linear tests" begin
    # To check unbounded problem, change the default algorithm used by KNITRO.
    optimizer = KNITRO.Optimizer(outlev=0, algorithm=3)

    exclude = ["linear8a", # Behavior in infeasible case doesn't match test.
               "linear12", # Same as above.
               "linear14", # TODO
               "linear8c", # Problem catching infeasibility ray
               ]
    model_for_knitro = MOIU.UniversalFallback(KnitroModelData{Float64}())
    linear_optimizer = MOI.Bridges.SplitInterval{Float64}(
                        MOIU.CachingOptimizer(model_for_knitro, optimizer))
    MOIT.contlineartest(linear_optimizer, config, exclude)
end

@testset "MOI QP/QCQP tests" begin
    optimizer = KNITRO.Optimizer(outlev=0)
    qp_optimizer = MOIU.CachingOptimizer(KnitroModelData{Float64}(), optimizer)
    MOIT.contquadratictest(qp_optimizer, config)
end

@testset "MOI SOCP tests" begin
    # Presolve must be switch off to get proper dual variables.
    optimizer = KNITRO.Optimizer(outlev=0, presolve=0)
    socp_optimizer = MOIU.CachingOptimizer(KnitroModelData{Float64}(), optimizer)
    # Behavior in infeasible case doesn't match test.
    exclude = ["lin3", "lin4"]
    MOIT.lintest(socp_optimizer, config, exclude)
end

@testset "MOI NLP tests" begin
    optimizer = KNITRO.Optimizer(outlev=0)
    MOIT.nlptest(optimizer, config)
end

@testset "MOI MILP test" begin
    optimizer = KNITRO.Optimizer(outlev=0)
    MOIT.knapsacktest(optimizer, config)
end


text:
# Use baremodule to shave off a few KB from the serialized `.ji` file
baremodule libpsl_jll
using Base
using Base: UUID
import JLLWrappers

JLLWrappers.@generate_main_file_header("libpsl")
JLLWrappers.@generate_main_file("libpsl", UUID("c4fadf96-db99-5883-8c55-41cf0165eeb0"))
end  # module libpsl_jll


text:
using GeoData, Test
using GeoData: cleankeys

@testset "cleankeys" begin
    @test cleankeys(["a", "b", "c"]) == (:a, :b, :c)
    @test cleankeys(("a", "b", "c")) == (:a, :b, :c)
end


text:
@test isadmtup([0,2,4]) == false
@test isadmtup([0,2,6,8,12]) == true
@test isadmtup([3, 5, 11, 17, 29]) == false
@test isadmtup([5, 7, 13, 19, 31]) == false
@test isadmtup([5,7,11,13,17]) == true


text:
module GridWorlds

using POMDPs, POMDPModels, POMDPToolbox, Distributions, DiscreteValueIteration, AdaptiveStressTesting2

export
    GWParams,
    GWSim,
    GWState

const RS = [GridWorldState(10,3),
            GridWorldState(3,10),
            GridWorldState(3,9),
            GridWorldState(3,8),
            GridWorldState(1,3),
            GridWorldState(10,6),
            GridWorldState(9,6),
            GridWorldState(6,5),
            GridWorldState(6,4),
            GridWorldState(6,3),
            GridWorldState(5,3)]
const RV = [10.0, -10.0, -10.0, -10.0, -10.0, -10.0, -10.0, -10.0, -10.0, -10.0, -10.0]

const EVENTSTATES = RS[2:end]
const EVENTSTATES_VEC = [[s.x,s.y] for s in EVENTSTATES]
struct GWState
    t::Int
    x::GridWorldState
end
struct GWParams
    t_max::Int
end
GWParams() = GWParams(10)
mutable struct GWSim <: MDP{GWState,Void}
    p::GWParams
    x0::GridWorldState
    mdp::MDP
    policy::Policy
    t::Int
    x::GridWorldState
    md::Float64
end
GWSim(p::GWParams, mdp::MDP, policy::Policy, x0::GridWorldState) = GWSim(p, x0, mdp, policy, 0, x0, Inf)
function GWSim(x0=GridWorldState(3,1))
    mdp = GridWorld(rs=RS, rv=RV, terminals=Set(RS), discount_factor=0.95, tp=0.91)
    solver = ValueIterationSolver(max_iterations=100, belres=1e-3)
    policy = solve(solver, mdp)
    return GWSim(GWParams(), mdp, policy, x0)
end

POMDPs.actions(gw::GWSim) = [nothing]
function POMDPs.initial_state(sim::GWSim, rng::AbstractRNG)
    sim.t, sim.x, sim.md = 0, sim.x0, Inf
    GWState(sim.t, sim.x)
end
function POMDPs.generate_s(sim::GWSim, s::GWState, a::Void, rng::AbstractRNG)
    sim.t += 1
    sim.x = generate_s(sim.mdp, sim.x, action(sim.policy, sim.x), rng)
    ss = [sim.x.x, sim.x.y]
    md = Float64(minimum(manhattan(ss,e) for e in EVENTSTATES_VEC))
    sim.md = min(sim.md, md)
    GWState(sim.t, sim.x)
end
POMDPs.isterminal(sim::GWSim, s::GWState) = sim.x in RS || sim.t >= sim.p.t_max
POMDPs.reward(sim::GWSim, s::GWState, a::Void, sp::GWState) = 0.0

function Distributions.pdf(sim::GWSim, s::GWState, a::Void, sp::GWState)
    if s.x == sp.x && sp.done
        return 1.0
    end
    sp_dist = transition(sim.mdp, s.x, action(sim.policy,s.x))
    prob = pdf(sp_dist, sp.x)
    prob
end

Base.hash(sim::GWSim) = hash(sim.t, hash(sim.x))

AdaptiveStressTesting2.isevent(sim::GWSim, s::GWState) = s.x in EVENTSTATES
AdaptiveStressTesting2.miss_distance(sim::GWSim, s::GWState) = sim.md
function AdaptiveStressTesting2.state_distance(sim::GWSim, s1::GWState, s2::GWState)
    sqrt((s1.x.x-s2.x.x)^2+(s1.x.y-s2.x.y)^2)
end

manhattan(x,y) = sum(abs, x - y)

end #module


text:
module Unums

export
  AbstractUnum,

  FixedUnum64,
  BinaryUnum64,
  Unum64,
  mask64,

  FixedUnum16,
  BinaryUnum16,
  Unum16,
  mask16,

  isexact,
  isinexact,

  createmask

import Base: isequal, isfinite, isinf, isinteger,
             isless, isnan, isnull, isnumber, isreal, issubnormal,
             typemin, typemax, box, unbox

import Base: +, -, *, /, ==, <, >, <=, >=, !, &, |, <<, ~

# ---------------------------------------------------------------------------------------

include("types.jl")
include("utils.jl")
include("ops.jl")
include("convert/float.jl")
# include("convert/int.jl")

# ---------------------------------------------------------------------------------------

end # module


text:
module TestIteration

using Test, DataFrames

df = DataFrame(A = Vector{Union{Int, Missing}}(1:2), B = Vector{Union{Int, Missing}}(2:3))

@test size(eachrow(df)) == (size(df, 1),)
@test IndexStyle(eachrow(df)) == IndexLinear()
@test sprint(summary, eachrow(df)) == "2-element DataFrameRows"
@test Base.IndexStyle(eachrow(df)) == IndexLinear()
@test eachrow(df)[1] == DataFrameRow(df, 1, :)
@test collect(eachrow(df)) isa Vector{<:DataFrameRow}
@test eltype(eachrow(df)) <: DataFrameRow
for row in eachrow(df)
    @test isa(row, DataFrameRow)
    @test (row[:B] - row[:A]) == 1
    # issue #683 (https://github.com/JuliaData/DataFrames.jl/pull/683)
    @test collect(pairs(row)) isa Vector{Pair{Symbol, Int}}
end

@test size(eachcol(df)) == (size(df, 2),)
@test IndexStyle(eachcol(df)) == IndexLinear()
@test Base.IndexStyle(eachcol(df)) == IndexLinear()
@test size(eachcol(df, true)) == (size(df, 2),)
@test IndexStyle(eachcol(df, true)) == IndexLinear()
@test size(eachcol(df, false)) == (size(df, 2),)
@test IndexStyle(eachcol(df, false)) == IndexLinear()
@test length(eachcol(df)) == size(df, 2)
@test length(eachcol(df, true)) == size(df, 2)
@test length(eachcol(df, false)) == size(df, 2)
@test eachcol(df)[1] == df[:, 1]
@test eachcol(df, true)[1] == (:A => df[:, 1])
@test eachcol(df, false)[1] == df[:, 1]
@test collect(eachcol(df, true)) isa Vector{Pair{Symbol, AbstractVector}}
@test collect(eachcol(df, true)) == [:A => [1, 2], :B => [2, 3]]
@test collect(eachcol(df)) isa Vector{AbstractVector}
@test collect(eachcol(df)) == [[1, 2], [2, 3]]
@test collect(eachcol(df, false)) isa Vector{AbstractVector}
@test collect(eachcol(df, false)) == [[1, 2], [2, 3]]
@test eltype(eachcol(df, true)) == Pair{Symbol, AbstractVector}
@test eltype(eachcol(df, false)) == AbstractVector
@test eltype(eachcol(df)) == AbstractVector
for col in eachcol(df, true)
    @test typeof(col) <: Pair{Symbol, <:AbstractVector}
end
for col in eachcol(df)
    @test isa(col, AbstractVector)
end
for col in eachcol(df, false)
    @test isa(col, AbstractVector)
end

@test map(x -> minimum(convert(Vector, x)), eachrow(df)) == [1,2]
@test map(Vector, eachrow(df)) == [[1, 2], [2, 3]]
@test mapcols(minimum, df) == DataFrame(A = [1], B = [2])
@test map(minimum, eachcol(df, false)) == [1, 2]
@test map(minimum, eachcol(df)) == [1, 2]
@test eltype.(eachcol(mapcols(Vector{Float64}, df))) == [Float64, Float64]
@test eltype(map(Vector{Float64}, eachcol(df, false))) == Vector{Float64}
@test eltype(map(Vector{Float64}, eachcol(df))) == Vector{Float64}

row = DataFrameRow(df, 1, :)

row[:A] = 100
@test df[1, :A] == 100

row[1] = 101
@test df[1, :A] == 101

df = DataFrame(A = Vector{Union{Int, Missing}}(1:4), B = Union{String, Missing}["M", "F", "F", "M"])

s1 = view(df, 1:3, :)
s1[2,:A] = 4
@test df[2, :A] == 4
@test view(s1, 1:2, :) == view(df, 1:2, :)

s2 = view(df, 1:2:3, :)
s2[2, :B] = "M"
@test df[3, :B] == "M"
@test view(s2, 1:1:2, :) == view(df, [1,3], :)

@test_throws MethodError for x in df; end

@testset "mapcols" begin
    df_mapcols = DataFrame(a=1:10, b=11:20)
    @test mapcols(sum, df_mapcols) == DataFrame(a=55, b=155)
    @test mapcols(x -> 1, df_mapcols) == DataFrame(a=1, b=1)
    @test_throws ArgumentError mapcols(x -> x[1] == 1 ? 0 : [0], df_mapcols)
    @test_throws ArgumentError mapcols(x -> x[1] == 1 ? x : 0, df_mapcols)
    @test_throws ArgumentError mapcols(x -> x[1] != 1 ? x : 0, df_mapcols)
    df_mapcols2 = mapcols(x -> x, df_mapcols)
    @test df_mapcols2 == df_mapcols
    @test df_mapcols2.a !== df_mapcols.a
    @test df_mapcols2.b !== df_mapcols.b
end

@testset "SubDataFrame" begin
    df = DataFrame([11:16 21:26 31:36 41:46])
    sdf = view(df, [3,1,4], [3,1,4])
    @test sdf == df[[3,1,4], [3,1,4]]
    @test eachrow(sdf) == eachrow(df[[3,1,4], [3,1,4]])
    @test eachcol(sdf, true) == eachcol(df[[3,1,4], [3,1,4]], true)
    @test eachcol(sdf, false) == eachcol(df[[3,1,4], [3,1,4]], false)
    @test size(eachrow(sdf)) == (3,)
    @test size(eachcol(sdf, true)) == (3,)
    @test size(eachcol(sdf, false)) == (3,)
end

@testset "parent mutation" begin
    df = DataFrame([11:16 21:26 31:36 41:46])
    sdf = view(df, [3,1,4], [3,1,4])
    erd = eachrow(df)
    erv = eachrow(sdf)
    names!(df, Symbol.(string.("y", 1:4)))
    df[!, 1] = 51:56
    @test df[1, :] == erd[1]
    @test copy(erv[1]) == (y3=33, y1=53, y4=43)
    df[!, :z] .= 1
    @test length(erd[1]) == 5 # the added column is reflected
    select!(df, Not([4,5]))
    @test copy(erd[1]) == (y1 = 51, y2 = 21, y3 = 31) # the removed columns are reflected
end

@testset "getproperty and propertynames" begin
    df_base = DataFrame([11:16 21:26 31:36 41:46])
    for df in (df_base, view(df_base, 1:3, 1:3))
        for x in (eachcol(df), eachcol(df, true), eachrow(df))
            @test propertynames(x) == propertynames(df)
            for n in names(df)
                @test getproperty(x, n) === getproperty(df, n)
            end
            @test_throws ArgumentError x.a
        end
    end
end

end # module


text:
cd(@__DIR__); import Pkg; Pkg.activate(".")
coords = (left = -145.0, right = -50.0, bottom = 20.0, top = 75.0)
Pkg.develop(path="$(homedir())/github/SimpleSDMLayers.jl")
using Revise
using SimpleSDMLayers
using Test

## Step 7: No replace! in geotiff ####
import SimpleSDMLayers._prepare_layer_for_burnin
coords = (left = -145.0, right = -50.0, bottom = 20.0, top = 75.0)
layer = SimpleSDMPredictor(WorldClim, BioClim, 1)

array = layer.grid
replace!(array, nothing => NaN)
layer.grid # NaN

array = replace(layer.grid, nothing => NaN)
layer.grid # NaN


text:
module MacrosUtils

# create Singleton Types
struct MacrosType{T} end

MacrosType(s::String) = MacrosType{Symbol(s)}();

MacrosType("All");

end


text:
"""
Abstract Vulkan object accessible via a handle.
"""
abstract type AbstractHandle{H} end

Base.broadcastable(x::AbstractHandle) = Ref(x)

handle_type(::Type{<:AbstractHandle{H}}) where {H} = handle_type(H)
handle_type(T::Type{Handle}) = T

handle(r::AbstractHandle) = handle(r.handle)

device(x::AbstractHandle) = device(handle(x))
instance(x::AbstractHandle) = instance(handle(x))

Base.unsafe_convert(T::Type{Ptr{Cvoid}}, h::AbstractHandle) = Base.unsafe_convert(T, handle(h))
Base.convert(::Type{H}, x::AbstractHandle{H}) where {H<:Handle} = handle(x)

"""
Application-owned resource hosted in memory on the GPU.

Typical instances represent a buffer or an image `handle`
bound to `memory`.
"""
struct Allocated{H,M} <: AbstractHandle{H}
    handle::H
    memory::M
end

memory(a::Allocated) = a.memory

struct Created{H<:Handle,I} <: AbstractHandle{H}
    handle::H
    info::I
end

info(c::Created) = c.info

function require_feature(device::Created{Device,DeviceCreateInfo}, feature::Symbol)
    getproperty(info(device).enabled_features, feature) || error("Feature '$feature' required but not enabled.")
end

function require_extension(inst_or_device::Union{Created{Instance,InstanceCreateInfo},Created{Device,DeviceCreateInfo}}, extension)
    string(extension) in info(inst_or_device).enabled_extension_names || error("Extension '$extension' required but not enabled.")
end

function require_layer(instance::Created{Instance,InstanceCreateInfo}, layer)
    string(layer) in info(instance).enabled_layer_names || error("Layer '$layer' required but not enabled.")
end


text:
using ADI
using BenchmarkTools
using CSV
using DataFrames
using InvertedIndices
using PyCall

results = []

frames = [
    randn(Float32, 51, 51) .+ 100,
    randn(Float32, 101, 101) .+ 100,
    randn(Float32, 201, 201) .+ 100,
    randn(Float32, 301, 301) .+ 100,
    randn(Float32, 401, 401) .+ 100
]

# parameters
fwhm = 5

# julia benchmarks

for frame in frames
    N = length(frame)
    @info "Benchmarking - Julia" N=N

    time_elapsed = @belapsed detectionmap($frame, fwhm)

    @info "Benchmarking - Julia" N=N time=time_elapsed
    push!(results, (framework="ADI.jl", N=N, time=time_elapsed))
end

# python benchmarks

vip = pyimport("vip_hci")

for frame in frames[begin:3] # vip way too slow
    N = length(frame)
    @info "Benchmarking - Python" N=N

    time_elapsed = @belapsed vip.metrics.snrmap($frame, fwhm, verbose=false)

    @info "Benchmarking - Python" N=N time=time_elapsed
    push!(results, (framework="VIP", N=N, time=time_elapsed))
end

# load results file and update
path = joinpath(@__DIR__, "snrmap_benchmarks.csv")
df = isfile(path) ? CSV.File(path) |> DataFrame : DataFrame()
out = append!(DataFrame(results), df)
unique!(out, Not(:time)) |> CSV.write(path)