text:
################################################################################
# Visuals
################################################################################
function visualize(env::Environment, traj::Vector{Vector{T}}) where T
	@assert size(traj[1]) == size(env.state)
    storage = generate_storage(env.mechanism, [env.representation == :minimal ? minimal_to_maximal(env.mechanism, x) : x for x in traj])
    visualize(env.mechanism, storage,
        vis=env.vis)
end

################################################################################
# Miscellaneous
################################################################################
type2symbol(H) = Symbol(lowercase(String(H.name.name)))

function get_control_mask(n_inputs, indices)
    n_outputs = length(indices)
    m = zeros(n_outputs, n_inputs)
    for (i,ind) in enumerate(indices)
        m[i,ind] = 1.0
    end
    return m
end


text:
@doc raw"""
    trust_regions(M, F, ∇F, x, H)

evaluate the Riemannian trust-regions solver for optimization on manifolds.
It will attempt to minimize the cost function F on the Manifold M.
If no Hessian H is provided, a standard approximation of the Hessian based on
the gradient ∇F will be computed.
For solving the the inner trust-region subproblem of finding an update-vector,
it uses the Steihaug-Toint truncated conjugate-gradient method.
For a description of the algorithm and more details see

* P.-A. Absil, C.G. Baker, K.A. Gallivan,
    Trust-region methods on Riemannian manifolds, FoCM, 2007.
    doi: [10.1007/s10208-005-0179-9](https://doi.org/10.1007/s10208-005-0179-9)
* A. R. Conn, N. I. M. Gould, P. L. Toint, Trust-region methods, SIAM,
    MPS, 2000. doi: [10.1137/1.9780898719857](https://doi.org/10.1137/1.9780898719857)

# Input
* `M` – a manifold $\mathcal M$
* `F` – a cost function $F \colon \mathcal M \to \mathbb R$ to minimize
* `∇F`- the gradient $\nabla F \colon \mathcal M \to T \mathcal M$ of $F$
* `x` – an initial value $x  ∈  \mathcal M$
* `H` – the hessian $H( \mathcal M, x, \xi)$ of $F$

# Optional
* `retraction` – approximation of the exponential map
* `preconditioner` – a preconditioner (a symmetric, positive definite operator
  that should approximate the inverse of the Hessian)
* `stopping_criterion` – ([`StopWhenAny`](@ref)([`StopAfterIteration`](@ref)`(1000)`,
  [`StopWhenGradientNormLess`](@ref)`(10^(-6))`) a functor inheriting
  from [`StoppingCriterion`](@ref) indicating when to stop.
* `Δ_bar` – the maximum trust-region radius
* `Δ` - the (initial) trust-region radius
* `useRandom` – set to true if the trust-region solve is to be initiated with a
  random tangent vector. If set to true, no preconditioner will be
  used. This option is set to true in some scenarios to escape saddle
  points, but is otherwise seldom activated.
* `ρ_prime` – Accept/reject threshold: if ρ (the performance ratio for the
  iterate) is at least ρ', the outer iteration is accepted.
  Otherwise, it is rejected. In case it is rejected, the trust-region
  radius will have been decreased. To ensure this, ρ' >= 0 must be
  strictly smaller than 1/4. If ρ_prime is negative, the algorithm is not
  guaranteed to produce monotonically decreasing cost values. It is
  strongly recommended to set ρ' > 0, to aid convergence.
* `ρ_regularization` – Close to convergence, evaluating the performance ratio ρ
  is numerically challenging. Meanwhile, close to convergence, the
  quadratic model should be a good fit and the steps should be
  accepted. Regularization lets ρ go to 1 as the model decrease and
  the actual decrease go to zero. Set this option to zero to disable
  regularization (not recommended). When this is not zero, it may happen
  that the iterates produced are not monotonically improving the cost
  when very close to convergence. This is because the corrected cost
  improvement could change sign if it is negative but very small.
* `return_options` – (`false`) – if actiavated, the extended result, i.e. the
  complete [`Options`](@ref) are returned. This can be used to access recorded values.
  If set to false (default) just the optimal value `x_opt` is returned

# Output
* `x` – the last reached point on the manifold

# see also
[`truncated_conjugate_gradient_descent`](@ref)
"""
function trust_regions(M::Manifold, F::TF, ∇F::TdF, x, H::TH; kwargs...) where {TF,TdF,TH}
    x_res = allocate(x)
    copyto!(x_res, x)
    return trust_regions!(M, F, ∇F, x_res, H; kwargs...)
end
@doc raw"""
    trust_regions!(M, F, ∇F, x, H; kwargs...)

evaluate the Riemannian trust-regions solver for optimization on manifolds in place of `x`.

# Input
* `M` – a manifold $\mathcal M$
* `F` – a cost function $F \colon \mathcal M \to \mathbb R$ to minimize
* `∇F`- the gradient $\nabla F \colon \mathcal M \to T \mathcal M$ of $F$
* `x` – an initial value $x  ∈  \mathcal M$
* `H` – the hessian $H( \mathcal M, x, \xi)$ of $F$

for more details and all options, see [`trust_regions`](@ref)
"""
function trust_regions!(
    M::Manifold,
    F::TF,
    ∇F::TdF,
    x,
    H::TH;
    retraction_method::AbstractRetractionMethod=ExponentialRetraction(),
    preconditioner::Tprec=(M, x, ξ) -> ξ,
    stopping_criterion::StoppingCriterion=StopWhenAny(
        StopAfterIteration(1000), StopWhenGradientNormLess(10^(-6))
    ),
    Δ_bar=sqrt(manifold_dimension(M)),
    Δ=Δ_bar / 8,
    useRandom::Bool=false,
    ρ_prime::Float64=0.1,
    ρ_regularization=1000.0,
    return_options=false,
    kwargs..., #collect rest
) where {TF,TdF,TH,Tprec}
    (ρ_prime >= 0.25) && throw(
        ErrorException("ρ_prime must be strictly smaller than 0.25 but it is $ρ_prime.")
    )
    (Δ_bar <= 0) && throw(ErrorException("Δ_bar must be positive but it is $Δ_bar."))
    (Δ <= 0 || Δ > Δ_bar) && throw(
        ErrorException("Δ must be positive and smaller than Δ_bar (=$Δ_bar) but it is $Δ."),
    )
    p = HessianProblem(M, F, ∇F, H, preconditioner)
    o = TrustRegionsOptions(
        x,
        stopping_criterion,
        Δ,
        Δ_bar,
        retraction_method,
        useRandom,
        ρ_prime,
        ρ_regularization,
    )
    o = decorate_options(o; kwargs...)
    resultO = solve(p, o)
    if return_options
        return resultO
    else
        return get_solver_result(resultO)
    end
end

initialize_solver!(p::P, o::O) where {P<:HessianProblem,O<:TrustRegionsOptions} = nothing

function step_solver!(p::P, o::O, iter) where {P<:HessianProblem,O<:TrustRegionsOptions}
    # Determine eta0
    if o.useRand == false
        # Pick the zero vector
        eta = zero_tangent_vector(p.M, o.x)
    else
        # Random vector in T_x M (this has to be very small)
        eta = 10.0^(-6) * random_tangent(p.M, o.x)
        while norm(p.M, o.x, eta) > o.Δ
            # Must be inside trust-region
            eta = sqrt(sqrt(eps(Float64))) * eta
        end
    end
    # Solve TR subproblem approximately
    opt = truncated_conjugate_gradient_descent(
        p.M,
        p.cost,
        p.gradient,
        o.x,
        eta,
        p.hessian,
        o.Δ;
        preconditioner=p.precon,
        useRandom=o.useRand,
        #debug = [:Iteration," ",:Stop],
        return_options=true,
    )
    option = get_options(opt) # remove decorators
    η = get_solver_result(option)
    SR = get_active_stopping_criteria(option.stop)
    Hη = getHessian(p, o.x, η)
    # Initialize the cost function F und the gradient of the cost function
    # ∇F at the point x
    grad = get_gradient(p, o.x)
    fx = get_cost(p, o.x)
    norm_grad = norm(p.M, o.x, grad)
    # If using randomized approach, compare result with the Cauchy point.
    if o.useRand
        # Check the curvature,
        Hgrad = getHessian(p, o.x, grad)
        gradHgrad = inner(p.M, o.x, grad, Hgrad)
        if gradHgrad <= 0
            tau_c = 1
        else
            tau_c = min(norm_grad^3 / (o.Δ * gradHgrad), 1)
        end
        # and generate the Cauchy point.
        η_c = (-tau_c * o.Δ / norm_grad) * grad
        Hη_c = (-tau_c * o.Δ / norm_grad) * Hgrad
        # Now that we have computed the Cauchy point in addition to the
        # returned eta, we might as well keep the best of them.
        mdle = fx + inner(p.M, o.x, grad, η) + 0.5 * inner(p.M, o.x, Hη, η)
        mdlec = fx + inner(p.M, o.x, grad, η_c) + 0.5 * inner(p.M, o.x, Hη_c, η_c)
        if mdlec < mdle
            η = η_c
            Hη = Hη_c
        end
    end
    # Compute the tentative next iterate (the proposal)
    x_prop = retract(p.M, o.x, η, o.retraction_method)
    # Compute the function value of the proposal
    fx_prop = get_cost(p, x_prop)
    # Check the performance of the quadratic model against the actual cost.
    ρnum = fx - fx_prop
    ρden = -inner(p.M, o.x, η, grad) - 0.5 * inner(p.M, o.x, η, Hη)
    # Since, at convergence, both ρnum and ρden become extremely small,
    # computing ρ is numerically challenging. The break with ρnum and ρden
    # can thus lead to a large error in rho, making the
    # acceptance / rejection erratic. Meanwhile, close to convergence,
    # steps are usually trustworthy and we should transition to a Newton-
    # like method, with rho=1 consistently. The heuristic thus shifts both
    # rhonum and rhoden by a small amount such that far from convergence,
    # the shift is irrelevant and close to convergence, the ratio rho goes
    # to 1, effectively promoting acceptance of the step.
    ρ_reg = max(1, abs(fx)) * eps(Float64) * o.ρ_regularization
    ρnum = ρnum + ρ_reg
    ρden = ρden + ρ_reg
    model_decreased = (ρden >= 0)
    ρ = ρnum / ρden
    # Choose the new TR radius based on the model performance.
    # If the actual decrease is smaller than 1/4 of the predicted decrease,
    # then reduce the TR radius.
    if ρ < 1 / 4 || model_decreased == false || isnan(ρ)
        o.Δ = o.Δ / 4
    elseif ρ > 3 / 4 && any([
        typeof(s) in [StopWhenTrustRegionIsExceeded, StopWhenCurvatureIsNegative] for
        s in SR
    ])
        o.Δ = min(2 * o.Δ, o.Δ_bar)
    else
        o.Δ = o.Δ
    end
    # Choose to accept or reject the proposed step based on the model
    # performance. Note the strict inequality.
    if model_decreased && ρ > o.ρ_prime
        o.x = x_prop
    end
    return nothing
end
get_solver_result(o::O) where {O<:TrustRegionsOptions} = o.x

ext: jl


text:
function generate_well_conditioned_matrix(rng, N)
    A = randn(rng, N, N)
    return A * A' + I
end

@testset "linalg" begin
    @testset "dot" begin
        @testset "Vector" begin
            rng, M = MersenneTwister(123456), 3
            x, y = randn(rng, M), randn(rng, M)
            ẋ, ẏ = randn(rng, M), randn(rng, M)
            x̄, ȳ = randn(rng, M), randn(rng, M)
            frule_test(dot, (x, ẋ), (y, ẏ))
            rrule_test(dot, randn(rng), (x, x̄), (y, ȳ))
        end
        @testset "Matrix" begin
            rng, M, N = MersenneTwister(123456), 3, 4
            x, y = randn(rng, M, N), randn(rng, M, N)
            ẋ, ẏ = randn(rng, M, N), randn(rng, M, N)
            x̄, ȳ = randn(rng, M, N), randn(rng, M, N)
            frule_test(dot, (x, ẋ), (y, ẏ))
            rrule_test(dot, randn(rng), (x, x̄), (y, ȳ))
        end
        @testset "Array{T, 3}" begin
            rng, M, N, P = MersenneTwister(123456), 3, 4, 5
            x, y = randn(rng, M, N, P), randn(rng, M, N, P)
            ẋ, ẏ = randn(rng, M, N, P), randn(rng, M, N, P)
            x̄, ȳ = randn(rng, M, N, P), randn(rng, M, N, P)
            frule_test(dot, (x, ẋ), (y, ẏ))
            rrule_test(dot, randn(rng), (x, x̄), (y, ȳ))
        end
    end
    @testset "inv" begin
        rng, N = MersenneTwister(123456), 3
        B = generate_well_conditioned_matrix(rng, N)
        frule_test(inv, (B, randn(rng, N, N)))
        rrule_test(inv, randn(rng, N, N), (B, randn(rng, N, N)))
    end
    @testset "det" begin
        rng, N = MersenneTwister(123456), 3
        B = generate_well_conditioned_matrix(rng, N)
        frule_test(det, (B, randn(rng, N, N)))
        rrule_test(det, randn(rng), (B, randn(rng, N, N)))
    end
    @testset "logdet" begin
        rng, N = MersenneTwister(123456), 3
        B = generate_well_conditioned_matrix(rng, N)
        frule_test(logdet, (B, randn(rng, N, N)))
        rrule_test(logdet, randn(rng), (B, randn(rng, N, N)))
    end
    @testset "tr" begin
        rng, N = MersenneTwister(123456), 4
        frule_test(tr, (randn(rng, N, N), randn(rng, N, N)))
        rrule_test(tr, randn(rng), (randn(rng, N, N), randn(rng, N, N)))
    end
    @testset "*" begin
        rng = MersenneTwister(123456)
        dims = [3,4,5]
        for n in dims, m in dims, p in dims
            n > 3 && n == m == p && continue  # don't need to test square case multiple times
            A = randn(rng, m, n)
            B = randn(rng, n, p)
            Ȳ = randn(rng, m, p)
            rrule_test(*, Ȳ, (A, randn(rng, m, n)), (B, randn(rng, n, p)))
        end
    end
    @testset "$f" for f in [/, \]
        rng = MersenneTwister(42)
        for n in 3:5, m in 3:5
            A = randn(rng, m, n)
            B = randn(rng, m, n)
            Ȳ = randn(rng, size(f(A, B)))
            rrule_test(f, Ȳ, (A, randn(rng, m, n)), (B, randn(rng, m, n)))
        end
        # Vectors
        x = randn(rng, 10)
        y = randn(rng, 10)
        ȳ = randn(rng, size(f(x, y))...)
        rrule_test(f, ȳ, (x, randn(rng, 10)), (y, randn(rng, 10)))
        if f == (/)
            @testset "$T on the RHS" for T in (Diagonal, UpperTriangular, LowerTriangular)
                RHS = T(randn(rng, T == Diagonal ? 10 : (10, 10)))
                Y = randn(rng, 5, 10)
                Ȳ = randn(rng, size(f(Y, RHS))...)
                rrule_test(f, Ȳ, (Y, randn(rng, size(Y))), (RHS, randn(rng, size(RHS))))
            end
        else
            @testset "$T on LHS" for T in (Diagonal, UpperTriangular, LowerTriangular)
                LHS = T(randn(rng, T == Diagonal ? 10 : (10, 10)))
                y = randn(rng, 10)
                ȳ = randn(rng, size(f(LHS, y))...)
                rrule_test(f, ȳ, (LHS, randn(rng, size(LHS))), (y, randn(rng, 10)))
                Y = randn(rng, 10, 10)
                Ȳ = randn(rng, 10, 10)
                rrule_test(f, Ȳ, (LHS, randn(rng, size(LHS))), (Y, randn(rng, size(Y))))
            end
            @testset "Matrix $f Vector" begin
                X = randn(rng, 10, 4)
                y = randn(rng, 10)
                ȳ = randn(rng, size(f(X, y))...)
                rrule_test(f, ȳ, (X, randn(rng, size(X))), (y, randn(rng, 10)))
            end
            @testset "Vector $f Matrix" begin
                x = randn(rng, 10)
                Y = randn(rng, 10, 4)
                ȳ = randn(rng, size(f(x, Y))...)
                rrule_test(f, ȳ, (x, randn(rng, size(x))), (Y, randn(rng, size(Y))))
            end
        end
    end
    @testset "norm" begin
        rng = MersenneTwister(3)
        for dims in [(), (5,), (3, 2), (7, 3, 2)]
            A = randn(rng, dims...)
            p = randn(rng)
            ȳ = randn(rng)
            rrule_test(norm, ȳ, (A, randn(rng, dims...)), (p, randn(rng)))
        end
    end
end


text:
# This file is auto-generated by AWSMetadata.jl
using AWS
using AWS.AWSServices: iotanalytics
using AWS.Compat
using AWS.UUIDs

"""
    batch_put_message(channel_name, messages)
    batch_put_message(channel_name, messages, params::Dict{String,<:Any})

Sends messages to a channel.

# Arguments
- `channel_name`: The name of the channel where the messages are sent.
- `messages`: The list of messages to be sent. Each message has the format: {
  \"messageId\": \"string\", \"payload\": \"string\"}. The field names of message payloads
  (data) that you send to IoT Analytics:   Must contain only alphanumeric characters and
  undescores (_). No other special characters are allowed.   Must begin with an alphabetic
  character or single underscore (_).   Cannot contain hyphens (-).   In regular expression
  terms: \"^[A-Za-z_]([A-Za-z0-9]*|[A-Za-z0-9][A-Za-z0-9_]*)\".    Cannot be more than 255
  characters.   Are case insensitive. (Fields named foo and FOO in the same payload are
  considered duplicates.)   For example, {\"temp_01\": 29} or {\"_temp_01\": 29} are valid,
  but {\"temp-01\": 29}, {\"01_temp\": 29} or {\"__temp_01\": 29} are invalid in message
  payloads.

"""
function batch_put_message(
    channelName, messages; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "POST",
        "/messages/batch",
        Dict{String,Any}("channelName" => channelName, "messages" => messages);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function batch_put_message(
    channelName,
    messages,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "POST",
        "/messages/batch",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}("channelName" => channelName, "messages" => messages),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    cancel_pipeline_reprocessing(pipeline_name, reprocessing_id)
    cancel_pipeline_reprocessing(pipeline_name, reprocessing_id, params::Dict{String,<:Any})

Cancels the reprocessing of data through the pipeline.

# Arguments
- `pipeline_name`: The name of pipeline for which data reprocessing is canceled.
- `reprocessing_id`: The ID of the reprocessing task (returned by
  StartPipelineReprocessing).

"""
function cancel_pipeline_reprocessing(
    pipelineName, reprocessingId; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "DELETE",
        "/pipelines/$(pipelineName)/reprocessing/$(reprocessingId)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function cancel_pipeline_reprocessing(
    pipelineName,
    reprocessingId,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "DELETE",
        "/pipelines/$(pipelineName)/reprocessing/$(reprocessingId)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_channel(channel_name)
    create_channel(channel_name, params::Dict{String,<:Any})

Used to create a channel. A channel collects data from an MQTT topic and archives the raw,
unprocessed messages before publishing the data to a pipeline.

# Arguments
- `channel_name`: The name of the channel.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"channelStorage"`: Where channel data is stored. You can choose one of serviceManagedS3
  or customerManagedS3 storage. If not specified, the default is serviceManagedS3. You can't
  change this storage option after the channel is created.
- `"retentionPeriod"`: How long, in days, message data is kept for the channel. When
  customerManagedS3 storage is selected, this parameter is ignored.
- `"tags"`: Metadata which can be used to manage the channel.
"""
function create_channel(channelName; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "POST",
        "/channels",
        Dict{String,Any}("channelName" => channelName);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_channel(
    channelName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "POST",
        "/channels",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("channelName" => channelName), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_dataset(actions, dataset_name)
    create_dataset(actions, dataset_name, params::Dict{String,<:Any})

Used to create a dataset. A dataset stores data retrieved from a data store by applying a
queryAction (a SQL query) or a containerAction (executing a containerized application).
This operation creates the skeleton of a dataset. The dataset can be populated manually by
calling CreateDatasetContent or automatically according to a trigger you specify.

# Arguments
- `actions`: A list of actions that create the dataset contents.
- `dataset_name`: The name of the dataset.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"contentDeliveryRules"`: When dataset contents are created, they are delivered to
  destinations specified here.
- `"lateDataRules"`: A list of data rules that send notifications to CloudWatch, when data
  arrives late. To specify lateDataRules, the dataset must use a DeltaTimer filter.
- `"retentionPeriod"`: Optional. How long, in days, versions of dataset contents are kept
  for the dataset. If not specified or set to null, versions of dataset contents are retained
  for at most 90 days. The number of versions of dataset contents retained is determined by
  the versioningConfiguration parameter. For more information, see  Keeping Multiple Versions
  of IoT Analytics datasets in the IoT Analytics User Guide.
- `"tags"`: Metadata which can be used to manage the dataset.
- `"triggers"`: A list of triggers. A trigger causes dataset contents to be populated at a
  specified time interval or when another dataset's contents are created. The list of
  triggers can be empty or contain up to five DataSetTrigger objects.
- `"versioningConfiguration"`: Optional. How many versions of dataset contents are kept. If
  not specified or set to null, only the latest version plus the latest succeeded version (if
  they are different) are kept for the time period specified by the retentionPeriod
  parameter. For more information, see Keeping Multiple Versions of IoT Analytics datasets in
  the IoT Analytics User Guide.
"""
function create_dataset(
    actions, datasetName; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "POST",
        "/datasets",
        Dict{String,Any}("actions" => actions, "datasetName" => datasetName);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_dataset(
    actions,
    datasetName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "POST",
        "/datasets",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}("actions" => actions, "datasetName" => datasetName),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_dataset_content(dataset_name)
    create_dataset_content(dataset_name, params::Dict{String,<:Any})

Creates the content of a dataset by applying a queryAction (a SQL query) or a
containerAction (executing a containerized application).

# Arguments
- `dataset_name`: The name of the dataset.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"versionId"`: The version ID of the dataset content. To specify versionId for a dataset
  content, the dataset must use a DeltaTimer filter.
"""
function create_dataset_content(
    datasetName; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "POST",
        "/datasets/$(datasetName)/content";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_dataset_content(
    datasetName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "POST",
        "/datasets/$(datasetName)/content",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_datastore(datastore_name)
    create_datastore(datastore_name, params::Dict{String,<:Any})

Creates a data store, which is a repository for messages.

# Arguments
- `datastore_name`: The name of the data store.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"datastorePartitions"`:  Contains information about the partition dimensions in a data
  store.
- `"datastoreStorage"`: Where data in a data store is stored.. You can choose
  serviceManagedS3 storage, customerManagedS3 storage, or iotSiteWiseMultiLayerStorage
  storage. The default is serviceManagedS3. You can't change the choice of Amazon S3 storage
  after your data store is created.
- `"fileFormatConfiguration"`: Contains the configuration information of file formats. IoT
  Analytics data stores support JSON and Parquet. The default file format is JSON. You can
  specify only one format. You can't change the file format after you create the data store.
- `"retentionPeriod"`: How long, in days, message data is kept for the data store. When
  customerManagedS3 storage is selected, this parameter is ignored.
- `"tags"`: Metadata which can be used to manage the data store.
"""
function create_datastore(datastoreName; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "POST",
        "/datastores",
        Dict{String,Any}("datastoreName" => datastoreName);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_datastore(
    datastoreName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "POST",
        "/datastores",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("datastoreName" => datastoreName), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_pipeline(pipeline_activities, pipeline_name)
    create_pipeline(pipeline_activities, pipeline_name, params::Dict{String,<:Any})

Creates a pipeline. A pipeline consumes messages from a channel and allows you to process
the messages before storing them in a data store. You must specify both a channel and a
datastore activity and, optionally, as many as 23 additional activities in the
pipelineActivities array.

# Arguments
- `pipeline_activities`: A list of PipelineActivity objects. Activities perform
  transformations on your messages, such as removing, renaming or adding message attributes;
  filtering messages based on attribute values; invoking your Lambda unctions on messages for
  advanced processing; or performing mathematical transformations to normalize device data.
  The list can be 2-25 PipelineActivity objects and must contain both a channel and a
  datastore activity. Each entry in the list must contain only one activity. For example:
  pipelineActivities = [ { \"channel\": { ... } }, { \"lambda\": { ... } }, ... ]
- `pipeline_name`: The name of the pipeline.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"tags"`: Metadata which can be used to manage the pipeline.
"""
function create_pipeline(
    pipelineActivities, pipelineName; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "POST",
        "/pipelines",
        Dict{String,Any}(
            "pipelineActivities" => pipelineActivities, "pipelineName" => pipelineName
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_pipeline(
    pipelineActivities,
    pipelineName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "POST",
        "/pipelines",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "pipelineActivities" => pipelineActivities,
                    "pipelineName" => pipelineName,
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    delete_channel(channel_name)
    delete_channel(channel_name, params::Dict{String,<:Any})

Deletes the specified channel.

# Arguments
- `channel_name`: The name of the channel to delete.

"""
function delete_channel(channelName; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "DELETE",
        "/channels/$(channelName)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function delete_channel(
    channelName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "DELETE",
        "/channels/$(channelName)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    delete_dataset(dataset_name)
    delete_dataset(dataset_name, params::Dict{String,<:Any})

Deletes the specified dataset. You do not have to delete the content of the dataset before
you perform this operation.

# Arguments
- `dataset_name`: The name of the dataset to delete.

"""
function delete_dataset(datasetName; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "DELETE",
        "/datasets/$(datasetName)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function delete_dataset(
    datasetName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "DELETE",
        "/datasets/$(datasetName)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    delete_dataset_content(dataset_name)
    delete_dataset_content(dataset_name, params::Dict{String,<:Any})

Deletes the content of the specified dataset.

# Arguments
- `dataset_name`: The name of the dataset whose content is deleted.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"versionId"`: The version of the dataset whose content is deleted. You can also use the
  strings \"LATEST\" or \"LATEST_SUCCEEDED\" to delete the latest or latest successfully
  completed data set. If not specified, \"LATEST_SUCCEEDED\" is the default.
"""
function delete_dataset_content(
    datasetName; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "DELETE",
        "/datasets/$(datasetName)/content";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function delete_dataset_content(
    datasetName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "DELETE",
        "/datasets/$(datasetName)/content",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    delete_datastore(datastore_name)
    delete_datastore(datastore_name, params::Dict{String,<:Any})

Deletes the specified data store.

# Arguments
- `datastore_name`: The name of the data store to delete.

"""
function delete_datastore(datastoreName; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "DELETE",
        "/datastores/$(datastoreName)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function delete_datastore(
    datastoreName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "DELETE",
        "/datastores/$(datastoreName)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    delete_pipeline(pipeline_name)
    delete_pipeline(pipeline_name, params::Dict{String,<:Any})

Deletes the specified pipeline.

# Arguments
- `pipeline_name`: The name of the pipeline to delete.

"""
function delete_pipeline(pipelineName; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "DELETE",
        "/pipelines/$(pipelineName)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function delete_pipeline(
    pipelineName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "DELETE",
        "/pipelines/$(pipelineName)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_channel(channel_name)
    describe_channel(channel_name, params::Dict{String,<:Any})

Retrieves information about a channel.

# Arguments
- `channel_name`: The name of the channel whose information is retrieved.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"includeStatistics"`: If true, additional statistical information about the channel is
  included in the response. This feature can't be used with a channel whose S3 storage is
  customer-managed.
"""
function describe_channel(channelName; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "GET",
        "/channels/$(channelName)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_channel(
    channelName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "GET",
        "/channels/$(channelName)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_dataset(dataset_name)
    describe_dataset(dataset_name, params::Dict{String,<:Any})

Retrieves information about a dataset.

# Arguments
- `dataset_name`: The name of the dataset whose information is retrieved.

"""
function describe_dataset(datasetName; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "GET",
        "/datasets/$(datasetName)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_dataset(
    datasetName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "GET",
        "/datasets/$(datasetName)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_datastore(datastore_name)
    describe_datastore(datastore_name, params::Dict{String,<:Any})

Retrieves information about a data store.

# Arguments
- `datastore_name`: The name of the data store

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"includeStatistics"`: If true, additional statistical information about the data store
  is included in the response. This feature can't be used with a data store whose S3 storage
  is customer-managed.
"""
function describe_datastore(
    datastoreName; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "GET",
        "/datastores/$(datastoreName)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_datastore(
    datastoreName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "GET",
        "/datastores/$(datastoreName)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_logging_options()
    describe_logging_options(params::Dict{String,<:Any})

Retrieves the current settings of the IoT Analytics logging options.

"""
function describe_logging_options(; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "GET", "/logging"; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET
    )
end
function describe_logging_options(
    params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "GET", "/logging", params; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET
    )
end

"""
    describe_pipeline(pipeline_name)
    describe_pipeline(pipeline_name, params::Dict{String,<:Any})

Retrieves information about a pipeline.

# Arguments
- `pipeline_name`: The name of the pipeline whose information is retrieved.

"""
function describe_pipeline(pipelineName; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "GET",
        "/pipelines/$(pipelineName)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_pipeline(
    pipelineName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "GET",
        "/pipelines/$(pipelineName)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    get_dataset_content(dataset_name)
    get_dataset_content(dataset_name, params::Dict{String,<:Any})

Retrieves the contents of a dataset as presigned URIs.

# Arguments
- `dataset_name`: The name of the dataset whose contents are retrieved.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"versionId"`: The version of the dataset whose contents are retrieved. You can also use
  the strings \"LATEST\" or \"LATEST_SUCCEEDED\" to retrieve the contents of the latest or
  latest successfully completed dataset. If not specified, \"LATEST_SUCCEEDED\" is the
  default.
"""
function get_dataset_content(datasetName; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "GET",
        "/datasets/$(datasetName)/content";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function get_dataset_content(
    datasetName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "GET",
        "/datasets/$(datasetName)/content",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    list_channels()
    list_channels(params::Dict{String,<:Any})

Retrieves a list of channels.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"maxResults"`: The maximum number of results to return in this request. The default
  value is 100.
- `"nextToken"`: The token for the next set of results.
"""
function list_channels(; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "GET", "/channels"; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET
    )
end
function list_channels(
    params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "GET", "/channels", params; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET
    )
end

"""
    list_dataset_contents(dataset_name)
    list_dataset_contents(dataset_name, params::Dict{String,<:Any})

Lists information about dataset contents that have been created.

# Arguments
- `dataset_name`: The name of the dataset whose contents information you want to list.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"maxResults"`: The maximum number of results to return in this request.
- `"nextToken"`: The token for the next set of results.
- `"scheduledBefore"`: A filter to limit results to those dataset contents whose creation
  is scheduled before the given time. See the field triggers.schedule in the CreateDataset
  request. (timestamp)
- `"scheduledOnOrAfter"`: A filter to limit results to those dataset contents whose
  creation is scheduled on or after the given time. See the field triggers.schedule in the
  CreateDataset request. (timestamp)
"""
function list_dataset_contents(
    datasetName; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "GET",
        "/datasets/$(datasetName)/contents";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function list_dataset_contents(
    datasetName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "GET",
        "/datasets/$(datasetName)/contents",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    list_datasets()
    list_datasets(params::Dict{String,<:Any})

Retrieves information about datasets.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"maxResults"`: The maximum number of results to return in this request. The default
  value is 100.
- `"nextToken"`: The token for the next set of results.
"""
function list_datasets(; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "GET", "/datasets"; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET
    )
end
function list_datasets(
    params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "GET", "/datasets", params; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET
    )
end

"""
    list_datastores()
    list_datastores(params::Dict{String,<:Any})

Retrieves a list of data stores.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"maxResults"`: The maximum number of results to return in this request. The default
  value is 100.
- `"nextToken"`: The token for the next set of results.
"""
function list_datastores(; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "GET", "/datastores"; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET
    )
end
function list_datastores(
    params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "GET", "/datastores", params; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET
    )
end

"""
    list_pipelines()
    list_pipelines(params::Dict{String,<:Any})

Retrieves a list of pipelines.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"maxResults"`: The maximum number of results to return in this request. The default
  value is 100.
- `"nextToken"`: The token for the next set of results.
"""
function list_pipelines(; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "GET", "/pipelines"; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET
    )
end
function list_pipelines(
    params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "GET", "/pipelines", params; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET
    )
end

"""
    list_tags_for_resource(resource_arn)
    list_tags_for_resource(resource_arn, params::Dict{String,<:Any})

Lists the tags (metadata) that you have assigned to the resource.

# Arguments
- `resource_arn`: The ARN of the resource whose tags you want to list.

"""
function list_tags_for_resource(
    resourceArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "GET",
        "/tags",
        Dict{String,Any}("resourceArn" => resourceArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function list_tags_for_resource(
    resourceArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "GET",
        "/tags",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("resourceArn" => resourceArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    put_logging_options(logging_options)
    put_logging_options(logging_options, params::Dict{String,<:Any})

Sets or updates the IoT Analytics logging options. If you update the value of any
loggingOptions field, it takes up to one minute for the change to take effect. Also, if you
change the policy attached to the role you specified in the roleArn field (for example, to
correct an invalid policy), it takes up to five minutes for that change to take effect.

# Arguments
- `logging_options`: The new values of the IoT Analytics logging options.

"""
function put_logging_options(
    loggingOptions; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "PUT",
        "/logging",
        Dict{String,Any}("loggingOptions" => loggingOptions);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function put_logging_options(
    loggingOptions,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "PUT",
        "/logging",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("loggingOptions" => loggingOptions), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    run_pipeline_activity(payloads, pipeline_activity)
    run_pipeline_activity(payloads, pipeline_activity, params::Dict{String,<:Any})

Simulates the results of running a pipeline activity on a message payload.

# Arguments
- `payloads`: The sample message payloads on which the pipeline activity is run.
- `pipeline_activity`: The pipeline activity that is run. This must not be a channel
  activity or a data store activity because these activities are used in a pipeline only to
  load the original message and to store the (possibly) transformed message. If a Lambda
  activity is specified, only short-running Lambda functions (those with a timeout of less
  than 30 seconds or less) can be used.

"""
function run_pipeline_activity(
    payloads, pipelineActivity; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "POST",
        "/pipelineactivities/run",
        Dict{String,Any}("payloads" => payloads, "pipelineActivity" => pipelineActivity);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function run_pipeline_activity(
    payloads,
    pipelineActivity,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "POST",
        "/pipelineactivities/run",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "payloads" => payloads, "pipelineActivity" => pipelineActivity
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    sample_channel_data(channel_name)
    sample_channel_data(channel_name, params::Dict{String,<:Any})

Retrieves a sample of messages from the specified channel ingested during the specified
timeframe. Up to 10 messages can be retrieved.

# Arguments
- `channel_name`: The name of the channel whose message samples are retrieved.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"endTime"`: The end of the time window from which sample messages are retrieved.
- `"maxMessages"`: The number of sample messages to be retrieved. The limit is 10. The
  default is also 10.
- `"startTime"`: The start of the time window from which sample messages are retrieved.
"""
function sample_channel_data(channelName; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "GET",
        "/channels/$(channelName)/sample";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function sample_channel_data(
    channelName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "GET",
        "/channels/$(channelName)/sample",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    start_pipeline_reprocessing(pipeline_name)
    start_pipeline_reprocessing(pipeline_name, params::Dict{String,<:Any})

Starts the reprocessing of raw message data through the pipeline.

# Arguments
- `pipeline_name`: The name of the pipeline on which to start reprocessing.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"channelMessages"`: Specifies one or more sets of channel messages that you want to
  reprocess. If you use the channelMessages object, you must not specify a value for
  startTime and endTime.
- `"endTime"`: The end time (exclusive) of raw message data that is reprocessed. If you
  specify a value for the endTime parameter, you must not use the channelMessages object.
- `"startTime"`: The start time (inclusive) of raw message data that is reprocessed. If you
  specify a value for the startTime parameter, you must not use the channelMessages object.
"""
function start_pipeline_reprocessing(
    pipelineName; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "POST",
        "/pipelines/$(pipelineName)/reprocessing";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function start_pipeline_reprocessing(
    pipelineName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "POST",
        "/pipelines/$(pipelineName)/reprocessing",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    tag_resource(resource_arn, tags)
    tag_resource(resource_arn, tags, params::Dict{String,<:Any})

Adds to or modifies the tags of the given resource. Tags are metadata that can be used to
manage a resource.

# Arguments
- `resource_arn`: The ARN of the resource whose tags you want to modify.
- `tags`: The new or modified tags for the resource.

"""
function tag_resource(resourceArn, tags; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "POST",
        "/tags",
        Dict{String,Any}("resourceArn" => resourceArn, "tags" => tags);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function tag_resource(
    resourceArn,
    tags,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "POST",
        "/tags",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}("resourceArn" => resourceArn, "tags" => tags),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    untag_resource(resource_arn, tag_keys)
    untag_resource(resource_arn, tag_keys, params::Dict{String,<:Any})

Removes the given tags (metadata) from the resource.

# Arguments
- `resource_arn`: The ARN of the resource whose tags you want to remove.
- `tag_keys`: The keys of those tags which you want to remove.

"""
function untag_resource(
    resourceArn, tagKeys; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "DELETE",
        "/tags",
        Dict{String,Any}("resourceArn" => resourceArn, "tagKeys" => tagKeys);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function untag_resource(
    resourceArn,
    tagKeys,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "DELETE",
        "/tags",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}("resourceArn" => resourceArn, "tagKeys" => tagKeys),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    update_channel(channel_name)
    update_channel(channel_name, params::Dict{String,<:Any})

Used to update the settings of a channel.

# Arguments
- `channel_name`: The name of the channel to be updated.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"channelStorage"`: Where channel data is stored. You can choose one of serviceManagedS3
  or customerManagedS3 storage. If not specified, the default is serviceManagedS3. You can't
  change this storage option after the channel is created.
- `"retentionPeriod"`: How long, in days, message data is kept for the channel. The
  retention period can't be updated if the channel's Amazon S3 storage is customer-managed.
"""
function update_channel(channelName; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "PUT",
        "/channels/$(channelName)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function update_channel(
    channelName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "PUT",
        "/channels/$(channelName)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    update_dataset(actions, dataset_name)
    update_dataset(actions, dataset_name, params::Dict{String,<:Any})

Updates the settings of a dataset.

# Arguments
- `actions`: A list of DatasetAction objects.
- `dataset_name`: The name of the dataset to update.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"contentDeliveryRules"`: When dataset contents are created, they are delivered to
  destinations specified here.
- `"lateDataRules"`: A list of data rules that send notifications to CloudWatch, when data
  arrives late. To specify lateDataRules, the dataset must use a DeltaTimer filter.
- `"retentionPeriod"`: How long, in days, dataset contents are kept for the dataset.
- `"triggers"`: A list of DatasetTrigger objects. The list can be empty or can contain up
  to five DatasetTrigger objects.
- `"versioningConfiguration"`: Optional. How many versions of dataset contents are kept. If
  not specified or set to null, only the latest version plus the latest succeeded version (if
  they are different) are kept for the time period specified by the retentionPeriod
  parameter. For more information, see Keeping Multiple Versions of IoT Analytics datasets in
  the IoT Analytics User Guide.
"""
function update_dataset(
    actions, datasetName; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "PUT",
        "/datasets/$(datasetName)",
        Dict{String,Any}("actions" => actions);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function update_dataset(
    actions,
    datasetName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "PUT",
        "/datasets/$(datasetName)",
        Dict{String,Any}(mergewith(_merge, Dict{String,Any}("actions" => actions), params));
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    update_datastore(datastore_name)
    update_datastore(datastore_name, params::Dict{String,<:Any})

Used to update the settings of a data store.

# Arguments
- `datastore_name`: The name of the data store to be updated.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"datastoreStorage"`: Where data in a data store is stored.. You can choose
  serviceManagedS3 storage, customerManagedS3 storage, or iotSiteWiseMultiLayerStorage
  storage. The default is serviceManagedS3. You can't change the choice of Amazon S3 storage
  after your data store is created.
- `"fileFormatConfiguration"`: Contains the configuration information of file formats. IoT
  Analytics data stores support JSON and Parquet. The default file format is JSON. You can
  specify only one format. You can't change the file format after you create the data store.
- `"retentionPeriod"`: How long, in days, message data is kept for the data store. The
  retention period can't be updated if the data store's Amazon S3 storage is customer-managed.
"""
function update_datastore(datastoreName; aws_config::AbstractAWSConfig=global_aws_config())
    return iotanalytics(
        "PUT",
        "/datastores/$(datastoreName)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function update_datastore(
    datastoreName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "PUT",
        "/datastores/$(datastoreName)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    update_pipeline(pipeline_activities, pipeline_name)
    update_pipeline(pipeline_activities, pipeline_name, params::Dict{String,<:Any})

Updates the settings of a pipeline. You must specify both a channel and a datastore
activity and, optionally, as many as 23 additional activities in the pipelineActivities
array.

# Arguments
- `pipeline_activities`: A list of PipelineActivity objects. Activities perform
  transformations on your messages, such as removing, renaming or adding message attributes;
  filtering messages based on attribute values; invoking your Lambda functions on messages
  for advanced processing; or performing mathematical transformations to normalize device
  data. The list can be 2-25 PipelineActivity objects and must contain both a channel and a
  datastore activity. Each entry in the list must contain only one activity. For example:
  pipelineActivities = [ { \"channel\": { ... } }, { \"lambda\": { ... } }, ... ]
- `pipeline_name`: The name of the pipeline to update.

"""
function update_pipeline(
    pipelineActivities, pipelineName; aws_config::AbstractAWSConfig=global_aws_config()
)
    return iotanalytics(
        "PUT",
        "/pipelines/$(pipelineName)",
        Dict{String,Any}("pipelineActivities" => pipelineActivities);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function update_pipeline(
    pipelineActivities,
    pipelineName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return iotanalytics(
        "PUT",
        "/pipelines/$(pipelineName)",
        Dict{String,Any}(
            mergewith(
                _merge, Dict{String,Any}("pipelineActivities" => pipelineActivities), params
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end


text:
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# Description
# ==============================================================================
#
#   Pre-defined formats for HTML tables.
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

export tf_html_default, tf_html_dark, tf_html_minimalist, tf_html_simple
export tf_html_matrix

const tf_html_default = HTMLTableFormat()

const tf_html_dark = HTMLTableFormat(
    css = """
    table, td, th {
        border-collapse: collapse;
        font-family: sans-serif;
    }

    td, th {
        border-bottom: 0;
        padding: 4px
    }

    tr:nth-child(even) {
        background: #242627 !important;
        color: white;
    }

    tr:nth-child(odd) {
        background: #363b3d !important;
        color: white;
    }

    tr.header {
        background: #000 !important;
        color: #5fa6c2 !important;
        font-weight: bold;
    }

    tr.subheader {
        background: #242627 !important;
        color: white;
    }

    tr.headerLastRow {
        border-bottom: 2px solid black;
    }

    th.rowNumber, td.rowNumber {
        text-align: right;
    }
    """
)

const tf_html_minimalist = HTMLTableFormat(
    css = """
    table, td, th {
        border-collapse: collapse;
        font-family: sans-serif;
    }

    td, th {
        border-bottom: 0;
        background: #fff !important;
        padding: 4px
    }

    tr.header {
        background: #fff !important;
        font-weight: bold;
    }

    tr.subheader {
        background: #fff !important;
        color: dimgray;
    }

    tr.headerLastRow {
        border-bottom: 2px solid black;
    }

    th.rowNumber, td.rowNumber {
        text-align: right;
    }
    """
)

const tf_html_simple = HTMLTableFormat(
    css = """
    table, td, th {
        border-collapse: collapse;
        font-family: sans-serif;
    }

    td, th {
        border-bottom: 0;
        padding: 4px
    }

    tr:nth-child(odd) {
        background: #eee;
    }

    tr:nth-child(even) {
        background: #fff;
    }

    tr.header {
        background: #fff !important;
        font-weight: bold;
    }

    tr.subheader {
        background: #fff !important;
        color: dimgray;
    }

    tr.headerLastRow {
        border-bottom: 2px solid black;
    }

    th.rowNumber, td.rowNumber {
        text-align: right;
    }
    """
)

const tf_html_matrix = HTMLTableFormat(
    css = """
    table {
        position: relative;
    }

    table::before,
    table::after {
        border: 1px solid #000;
        content: "";
        height: 100%;
        position: absolute;
        top: 0;
        width: 6px;
    }

    table::before {
        border-right: 0px;
        left: -6px;
    }

    table::after {
        border-left: 0px;
        right: -6px;
    }

    td {
        padding: 5px;
        text-align: center;
    }
    """
)


text:
function simulate(b::Beta, n::Vector{<:Integer})
    @assert all(n .> 0) "All nᵢ must be strictly positive for i = 1,...,N"
    N = length(n)

    x = zeros(Integer, N)
    θ = zeros(N)
    for i in 1:N
        θ[i] = rand(b)
        x[i] = rand(Binomial(n[i], θ[i]))
    end
    return (x = x, n = n, θ = θ)
end

n   = rand(1:100, 100)
sim = simulate(Beta(10, 30), n)


text:
using Trixi, OrdinaryDiffEq

dg = DGMulti(polydeg = 3, element_type = Tri(), approximation_type = Polynomial(),
             surface_integral = SurfaceIntegralWeakForm(FluxHLL()),
             volume_integral = VolumeIntegralWeakForm())

equations = CompressibleEulerEquations2D(1.4)
initial_condition = initial_condition_convergence_test
source_terms = source_terms_convergence_test

mesh = DGMultiMesh(dg, cells_per_dimension = (4, 4), periodicity=true)
semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition, dg,
                                    source_terms = source_terms)

tspan = (0.0, 0.4)
ode = semidiscretize(semi, tspan)

summary_callback = SummaryCallback()
alive_callback = AliveCallback(alive_interval=10)
analysis_interval = 100
analysis_callback = AnalysisCallback(semi, interval=analysis_interval, uEltype=real(dg))
callbacks = CallbackSet(summary_callback, alive_callback, analysis_callback)

###############################################################################
# run the simulation

sol = solve(ode, CarpenterKennedy2N54(williamson_condition=false),
            dt = 0.5 * estimate_dt(mesh, dg), save_everystep=false, callback=callbacks);
summary_callback() # print the timer summary


text:
#!/usr/bin/env julia

# Package load for jupyter notebook container install
Pkg.init()
Pkg.update()
Pkg.add("HDF5")
Pkg.add("Gadfly")
Pkg.add("RDatasets")
Pkg.add("IJulia")
# Precompile Julia packages
using IJulia

max_stars_repo_name: taqtiqa-mark/bob

max_issues_repo_name: taqtiqa-mark/bob

max_forks_repo_name: taqtiqa-mark/bob


text:
using Swizzles.ExtrudedArrays
using Swizzles.NamedArrays
using Swizzles.ValArrays
using Swizzles.ArrayifiedArrays
using Swizzles.Virtuals

using Base.Broadcast: broadcastable, Broadcasted, materialize

using Swizzles

@testset "Virtuals" begin
    for x in [@_((1, 2, 3) .+ [1, 2, 3]'),
              @_(0 .+ [1, 2, 3] .+ [1 2 3; 4 5 6; 7 8 9]),
              Swizzle(+)(@_(0 .+ [1, 2, 3] .+ [1 2 3; 4 5 6; 7 8 9])),
              Swizzle(+, 1)(@_(0 .+ [1, 2, 3] .+ [1 2 3; 4 5 6; 7 8 9]))]
        @test kept.(keeps(virtualize(:root, typeof(lift_keeps(x))))) == kept.(keeps(x))
        @test eltype(virtualize(:root, typeof(lift_keeps(x)))) == eltype(x)
    end
end


text:
module STBIncomes
#
# This module implements a list of incomes, roughly based on FRS incomes,
# implemented with constants and
# fixed length arrays. It's used by the calculation and results modules for
# fast computation. In the household module, incomes are modelled as a Dictionary.
# [Results.jl] has a routine to map between the two.
#
# TODO Investigate Named Arrays
#

# using Base: String
using StaticArrays
using DataFrames
using ScottishTaxBenefitModel
using .RunSettings: Settings
using .Utils

    # declarations  ----------------
@exported_enum Incomes begin
    WAGES
    SELF_EMPLOYMENT_INCOME
    ODD_JOBS
    PRIVATE_PENSIONS
    NATIONAL_SAVINGS
    BANK_INTEREST
    STOCKS_SHARES
    INDIVIDUAL_SAVINGS_ACCOUNT
    PROPERTY
    ROYALTIES
    BONDS_AND_GILTS
    OTHER_INVESTMENT_INCOME
    OTHER_INCOME
    ALIMONY_AND_CHILD_SUPPORT_RECEIVED
    PRIVATE_SICKNESS_SCHEME_BENEFITS
    ACCIDENT_INSURANCE_SCHEME_BENEFITS
    HOSPITAL_SAVINGS_SCHEME_BENEFITS
    UNEMPLOYMENT_OR_REDUNDANCY_INSURANCE
    PERMANENT_HEALTH_INSURANCE
    ANY_OTHER_SICKNESS_INSURANCE
    CRITICAL_ILLNESS_COVER
    TRADE_UNION_SICK_OR_STRIKE_PAY

    HEALTH_INSURANCE
    ALIMONY_AND_CHILD_SUPPORT_PAID
    TRADE_UNIONS_ETC
    FRIENDLY_SOCIETIES
    WORK_EXPENSES
    AVCS
    OTHER_DEDUCTIONS
    LOAN_REPAYMENTS
    PENSION_CONTRIBUTIONS_EMPLOYEE
    PENSION_CONTRIBUTIONS_EMPLOYER

    INCOME_TAX
    NATIONAL_INSURANCE
    LOCAL_TAXES
    SOCIAL_FUND_LOAN_REPAYMENT
    STUDENT_LOAN_REPAYMENTS
    CARE_INSURANCE

    CHILD_BENEFIT
    STATE_PENSION
    BEREAVEMENT_ALLOWANCE
    ARMED_FORCES_COMPENSATION_SCHEME
    WAR_WIDOWS_PENSION
    SEVERE_DISABILITY_ALLOWANCE
    ATTENDANCE_ALLOWANCE
    CARERS_ALLOWANCE
    INDUSTRIAL_INJURY_BENEFIT
    INCAPACITY_BENEFIT
    PERSONAL_INDEPENDENCE_PAYMENT_DAILY_LIVING
    PERSONAL_INDEPENDENCE_PAYMENT_MOBILITY
    DLA_SELF_CARE
    DLA_MOBILITY
    EDUCATION_ALLOWANCES
    FOSTER_CARE_PAYMENTS
    MATERNITY_ALLOWANCE
    MATERNITY_GRANT
    FUNERAL_GRANT
    ANY_OTHER_NI_OR_STATE_BENEFIT
    FRIENDLY_SOCIETY_BENEFITS
    GOVERNMENT_TRAINING_ALLOWANCES
    CONTRIB_JOBSEEKERS_ALLOWANCE
    GUARDIANS_ALLOWANCE
    WIDOWS_PAYMENT
    WINTER_FUEL_PAYMENTS
    # legacy mt benefits
    WORKING_TAX_CREDIT
    CHILD_TAX_CREDIT
    NON_CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE
    INCOME_SUPPORT
    PENSION_CREDIT
    SAVINGS_CREDIT
    NON_CONTRIB_JOBSEEKERS_ALLOWANCE
    HOUSING_BENEFIT

    FREE_SCHOOL_MEALS
    UNIVERSAL_CREDIT
    OTHER_BENEFITS
    STUDENT_GRANTS
    STUDENT_LOANS
    COUNCIL_TAX_BENEFIT
    CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE

    SCOTTISH_CHILD_PAYMENT
    SCOTTISH_CARERS_SUPPLEMENT

    DISCRESIONARY_HOUSING_PAYMENT # not just Scottish, but, hey..

    SCOTTISH_DISABILITY_ASSISTANCE_CHILDREN_DAILY_LIVING
    SCOTTISH_DISABILITY_ASSISTANCE_CHILDREN_MOBILITY
    SCOTTISH_DISABILITY_ASSISTANCE_OLDER_PEOPLE
    SCOTTISH_DISABILITY_ASSISTANCE_WORKING_AGE_DAILY_LIVING
    SCOTTISH_DISABILITY_ASSISTANCE_WORKING_AGE_MOBILITY

    BASIC_INCOME

end

const FIRST_INCOME = instances(Incomes)[1]
const LAST_INCOME = instances(Incomes)[end]
const NUM_INCOMES = length(instances(Incomes))[1]

# FIXME the export_enum macro should take care of these ..
export WAGES
export SELF_EMPLOYMENT_INCOME
export ODD_JOBS
export PRIVATE_PENSIONS
export NATIONAL_SAVINGS
export BANK_INTEREST
export STOCKS_SHARES
export INDIVIDUAL_SAVINGS_ACCOUNT
export PROPERTY
export ROYALTIES
export BONDS_AND_GILTS
export OTHER_INVESTMENT_INCOME
export OTHER_INCOME
export ALIMONY_AND_CHILD_SUPPORT_RECEIVED
export PRIVATE_SICKNESS_SCHEME_BENEFITS
export ACCIDENT_INSURANCE_SCHEME_BENEFITS
export HOSPITAL_SAVINGS_SCHEME_BENEFITS
export UNEMPLOYMENT_OR_REDUNDANCY_INSURANCE
export PERMANENT_HEALTH_INSURANCE
export ANY_OTHER_SICKNESS_INSURANCE
export CRITICAL_ILLNESS_COVER
export TRADE_UNION_SICK_OR_STRIKE_PAY
export HEALTH_INSURANCE
export ALIMONY_AND_CHILD_SUPPORT_PAID
export TRADE_UNIONS_ETC
export FRIENDLY_SOCIETIES
export WORK_EXPENSES
export AVCS
export OTHER_DEDUCTIONS
export LOAN_REPAYMENTS
export PENSION_CONTRIBUTIONS_EMPLOYEE
export PENSION_CONTRIBUTIONS_EMPLOYER
export INCOME_TAX
export NATIONAL_INSURANCE
export LOCAL_TAXES
export SOCIAL_FUND_LOAN_REPAYMENT
export STUDENT_LOAN_REPAYMENTS
export CARE_INSURANCE
export CHILD_BENEFIT
export STATE_PENSION
export BEREAVEMENT_ALLOWANCE
export ARMED_FORCES_COMPENSATION_SCHEME
export WAR_WIDOWS_PENSION
export SEVERE_DISABILITY_ALLOWANCE
export ATTENDANCE_ALLOWANCE
export CARERS_ALLOWANCE
export INDUSTRIAL_INJURY_BENEFIT
export INCAPACITY_BENEFIT
export PERSONAL_INDEPENDENCE_PAYMENT_DAILY_LIVING
export PERSONAL_INDEPENDENCE_PAYMENT_MOBILITY
export DLA_SELF_CARE
export DLA_MOBILITY
export EDUCATION_ALLOWANCES
export FOSTER_CARE_PAYMENTS
export MATERNITY_ALLOWANCE
export MATERNITY_GRANT
export FUNERAL_GRANT
export ANY_OTHER_NI_OR_STATE_BENEFIT
export FRIENDLY_SOCIETY_BENEFITS
export GOVERNMENT_TRAINING_ALLOWANCES
export CONTRIB_JOBSEEKERS_ALLOWANCE
export GUARDIANS_ALLOWANCE
export WIDOWS_PAYMENT
export WINTER_FUEL_PAYMENTS
# legacy mt benefits
export WORKING_TAX_CREDIT
export CHILD_TAX_CREDIT
export NON_CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE
export INCOME_SUPPORT
export PENSION_CREDIT
export SAVINGS_CREDIT
export NON_CONTRIB_JOBSEEKERS_ALLOWANCE
export HOUSING_BENEFIT

export FREE_SCHOOL_MEALS
export UNIVERSAL_CREDIT
export OTHER_BENEFITS
export STUDENT_GRANTS
export STUDENT_LOANS
export COUNCIL_TAX_BENEFIT
export CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE
export SCOTTISH_CHILD_PAYMENT
export SCOTTISH_CARERS_SUPPLEMENT
export BASIC_INCOME
export DISCRESIONARY_HOUSING_PAYMENT
export SCOTTISH_DISABILITY_ASSISTANCE_CHILDREN_DAILY_LIVING
export SCOTTISH_DISABILITY_ASSISTANCE_CHILDREN_MOBILITY
export SCOTTISH_DISABILITY_ASSISTANCE_OLDER_PEOPLE
export SCOTTISH_DISABILITY_ASSISTANCE_WORKING_AGE_DAILY_LIVING
export SCOTTISH_DISABILITY_ASSISTANCE_WORKING_AGE_MOBILITY

function sz(thing)::Int
    t = 0
    for i in instances(thing)
      t += 1
     end
     t
end

const INC_ARRAY_SIZE = sz(Incomes)
const IncomesArray = MVector{INC_ARRAY_SIZE,T} where T

Base.getindex( X::IncomesArray, s::Incomes ) = getindex(X,Int(s)+1) # enums without explicit numbers start at 0...
Base.setindex!( X::IncomesArray, x, s::Incomes) = setindex!(X,x,Int(s)+1)

const IncomesSet = Vector{Incomes}

function fill( starti::Incomes, stopi :: Incomes) :: IncomesSet
    is = IncomesSet()
    for i in Int(starti):Int(stopi)
        push!(is, Incomes(i))
    end
    return is
end

const NON_CALCULATED_INCOMES = fill( WAGES, TRADE_UNION_SICK_OR_STRIKE_PAY )
const NON_CALCULATED_ITEMS = fill( WAGES, PENSION_CONTRIBUTIONS_EMPLOYER )
const BENEFITS = fill(CHILD_BENEFIT, LAST_INCOME ) # careful!
const LEGACY_MTBS =fill(WORKING_TAX_CREDIT,HOUSING_BENEFIT)

const MEANS_TESTED_BENS = union(fill(WORKING_TAX_CREDIT,UNIVERSAL_CREDIT), [SCOTTISH_CHILD_PAYMENT] )
const NON_MEANS_TESTED_BENS = setdiff( BENEFITS, MEANS_TESTED_BENS )
const INCOME_TAXES = setdiff(fill(INCOME_TAX,NATIONAL_INSURANCE),[LOCAL_TAXES]) # CT treated seperately

const CALCULATED = fill(INCOME_TAX,SCOTTISH_CARERS_SUPPLEMENT)
const SCOTTISH_SICKNESS_BENEFITS = [
        SCOTTISH_CARERS_SUPPLEMENT,
        SCOTTISH_DISABILITY_ASSISTANCE_CHILDREN_DAILY_LIVING,
        SCOTTISH_DISABILITY_ASSISTANCE_WORKING_AGE_MOBILITY ]
const UK_SICKNESS_ILLNESS = fill(SEVERE_DISABILITY_ALLOWANCE,DLA_MOBILITY)
const SICKNESS_ILLNESS = union(UK_SICKNESS_ILLNESS, SCOTTISH_SICKNESS_BENEFITS )
const DEDUCTIONS = fill(HEALTH_INSURANCE,PENSION_CONTRIBUTIONS_EMPLOYEE) # not employer since wages are net of this

const PASSED_THROUGH_BENEFITS = [
        ARMED_FORCES_COMPENSATION_SCHEME,
        WAR_WIDOWS_PENSION,
        SEVERE_DISABILITY_ALLOWANCE,
        INDUSTRIAL_INJURY_BENEFIT,
        INCAPACITY_BENEFIT,
        EDUCATION_ALLOWANCES,
        FOSTER_CARE_PAYMENTS,
        MATERNITY_GRANT,
        FUNERAL_GRANT,
        ANY_OTHER_NI_OR_STATE_BENEFIT,
        GUARDIANS_ALLOWANCE,
        FREE_SCHOOL_MEALS,
        OTHER_BENEFITS,
        STUDENT_GRANTS,
        STUDENT_LOANS ]
const ALL_INCOMES = union( NON_CALCULATED_INCOMES, BENEFITS )
const ALL_INCOMES_EXCEPT_HOUSING_BENEFITS = setdiff(ALL_INCOMES,[COUNCIL_TAX_BENEFIT,HOUSING_BENEFIT])

const DIRECT_TAXES_AND_DEDUCTIONS = union(INCOME_TAXES,DEDUCTIONS)

# exports ----------------
export GROSS_INCOME
export BENEFITS
export LEGACY_MTBS
export MEANS_TESTED_BENS
export NON_MEANS_TESTED_BENS
export INCOME_TAXES
export CALCULATED
export DEDUCTIONS
export DIRECT_TAXES_AND_DEDUCTIONS
export SICKNESS_ILLNESS
export INC_ARRAY_SIZE
export ALL_INCOMES
export ALL_INCOMES_EXCEPT_HOUSING_BENEFITS
export PASSED_THROUGH_BENEFITS

export iname
export make_static_incs
export make_mutable_incs
export IncludedItems
export isum
export any_positive

struct IncludedItems
    included :: IncomesSet
    deducted :: Union{Nothing,IncomesSet}
end

function make_a( T :: Type ) :: IncomesArray
    return IncomesArray{T}( zeros(T, INC_ARRAY_SIZE))
end

function isum( a :: IncomesArray{T},
    included :: IncomesSet;
    deducted :: Union{IncomesSet,Nothing} = nothing ) :: T where T
    s = zero(T)
    for i in included
        s += a[i]
    end
    if ! isnothing(deducted)
        for i in deducted
            s -= a[i]
        end
    end
    return s
end

function i2sym( i :: Incomes ) :: Symbol
    return Symbol(lowercase( string(i)))
end

function isum( a :: AbstractArray,
    which :: IncludedItems )
    return isum( a, which.included, deducted=which.deducted)
end

function any_positive( a :: AbstractArray,
    which ) :: Bool
    for i in which
        if a[i] > 0
            return true
        end
    end
    return false
end

function make_static_incs(
    T         :: Type;
    ones,
    minusones ) :: IncomesArray
    v = IncomesArray{T}(zeros(T,INC_ARRAY_SIZE))
    for i in ones
        v[i] = one(T)
    end
    for i in minusones
        v[i] = -one(T)
    end
    return IncomesArray(v)
end

function iname(i::Incomes)::String
    return pretty(i)
end

# FIXME what if we din't export these ones, deleted '_INCOME' and instead forced Incomes.SAVINGS ...
export LEGACY_MT_INCOME
export GROSS_INCOME
export LEGACY_HB_INCOME
export LEGACY_PC_INCOME
export LEGACY_SAVINGS_CREDIT_INCOME
export DIVIDEND_INCOME
export EXEMPT_INCOME
export ALL_TAXABLE_INCOME
export NON_SAVINGS_INCOME
export SAVINGS_INCOME
export DEFAULT_PASSPORTED_BENS
export UC_OTHER_INCOME
export UC_EARNED_INCOME

const SAVINGS_INCOME = [BANK_INTEREST, BONDS_AND_GILTS,OTHER_INVESTMENT_INCOME]

"""
TODO check this carefully against WTC,PC and IS chapters
note this doesn't include wages and TaxBenefitSystem
which are handled in the `calc_incomes` function.
poss. have 2nd complete version for WTC/CTC
"""
const UC_EARNED_INCOME = IncludedItems(
    [
        WAGES
        # se treated seperately
    ],
    [
        INCOME_TAX,
        NATIONAL_INSURANCE,
        PENSION_CONTRIBUTIONS_EMPLOYEE # !! NOTE THIS IS NOT GROSSED UP, see: CPAG 118
        ## ? student loan repayments?
    ]
)

export
    LEGACY_CAP_BENEFITS,
    UC_CAP_BENEFITS

const COMMON_CAP_BENEFITS = [
    CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE,
    CONTRIB_JOBSEEKERS_ALLOWANCE,
    WIDOWS_PAYMENT,
    MATERNITY_ALLOWANCE,
    INCAPACITY_BENEFIT,
    CHILD_BENEFIT,
    SEVERE_DISABILITY_ALLOWANCE
]

const UC_CAP_BENEFITS = union([UNIVERSAL_CREDIT], COMMON_CAP_BENEFITS )

const LEGACY_CAP_BENEFITS = union(
    [HOUSING_BENEFIT,
        CHILD_TAX_CREDIT,
        NON_CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE,
        NON_CONTRIB_JOBSEEKERS_ALLOWANCE,
        INCOME_SUPPORT],
        COMMON_CAP_BENEFITS
)

# FIXME check these lists again very carefully indeed.
# FIXME the CPAG guide describes a very convoluted way
# of calculating, which requires 2 lists of earned and unearned income
# I think that's important just for
const UC_OTHER_INCOME = [
        # WAGES,
        # SELF_EMPLOYMENT_INCOME, # counts, but see minimum level of earnings regs 19/20 p119
        OTHER_INCOME,
        CARERS_ALLOWANCE,
        ALIMONY_AND_CHILD_SUPPORT_RECEIVED, # FIXME THERE IS A 15 DISREGARD SEE PP 438
        EDUCATION_ALLOWANCES,
        FOSTER_CARE_PAYMENTS,
        STATE_PENSION,
        PRIVATE_PENSIONS,
        BEREAVEMENT_ALLOWANCE,
        WAR_WIDOWS_PENSION,
        CONTRIB_JOBSEEKERS_ALLOWANCE, ## CONTRIBUTION BASED Only
        CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE,
        INDUSTRIAL_INJURY_BENEFIT,
        INCAPACITY_BENEFIT,
        MATERNITY_ALLOWANCE,
        MATERNITY_GRANT,
        FUNERAL_GRANT,
        ANY_OTHER_NI_OR_STATE_BENEFIT,
        TRADE_UNION_SICK_OR_STRIKE_PAY,
        FRIENDLY_SOCIETY_BENEFITS,
        # WORKING_TAX_CREDIT,
        PRIVATE_SICKNESS_SCHEME_BENEFITS,
        ACCIDENT_INSURANCE_SCHEME_BENEFITS,
        HOSPITAL_SAVINGS_SCHEME_BENEFITS,
        GOVERNMENT_TRAINING_ALLOWANCES,
        GUARDIANS_ALLOWANCE,
        WIDOWS_PAYMENT,
        UNEMPLOYMENT_OR_REDUNDANCY_INSURANCE ]
## FIXME CHECK THIS list
## NOTE wages, se are treated seperately
const LEGACY_MT_INCOME = IncludedItems(
    [
        OTHER_INCOME,
        CARERS_ALLOWANCE,
        ALIMONY_AND_CHILD_SUPPORT_RECEIVED, # FIXME THERE IS A 15 DISREGARD SEE PP 438
        EDUCATION_ALLOWANCES,
        FOSTER_CARE_PAYMENTS,
        STATE_PENSION,
        PRIVATE_PENSIONS,
        BEREAVEMENT_ALLOWANCE,
        WAR_WIDOWS_PENSION,
        CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE,
        CONTRIB_JOBSEEKERS_ALLOWANCE, ## CONTRIBUTION BASED Only
        INDUSTRIAL_INJURY_BENEFIT,
        INCAPACITY_BENEFIT,
        MATERNITY_ALLOWANCE,
        MATERNITY_GRANT,
        FUNERAL_GRANT,
        ANY_OTHER_NI_OR_STATE_BENEFIT,
        TRADE_UNION_SICK_OR_STRIKE_PAY,
        FRIENDLY_SOCIETY_BENEFITS,
        WORKING_TAX_CREDIT ,
        PRIVATE_SICKNESS_SCHEME_BENEFITS,
        ACCIDENT_INSURANCE_SCHEME_BENEFITS,
        HOSPITAL_SAVINGS_SCHEME_BENEFITS,
        GOVERNMENT_TRAINING_ALLOWANCES,
        GUARDIANS_ALLOWANCE,
        WIDOWS_PAYMENT,
        UNEMPLOYMENT_OR_REDUNDANCY_INSURANCE
    ],
    [
        STUDENT_LOAN_REPAYMENTS,
        ALIMONY_AND_CHILD_SUPPORT_PAID
    ]
)

const GROSS_INCOME = IncludedItems(
    [
        WAGES,
        SELF_EMPLOYMENT_INCOME,
        ODD_JOBS,
        PRIVATE_PENSIONS,
        NATIONAL_SAVINGS,
        BANK_INTEREST,
        STOCKS_SHARES,
        INDIVIDUAL_SAVINGS_ACCOUNT,
        PROPERTY,
        ROYALTIES,
        BONDS_AND_GILTS,
        OTHER_INVESTMENT_INCOME,
        OTHER_INCOME,
        ALIMONY_AND_CHILD_SUPPORT_RECEIVED,
        PRIVATE_SICKNESS_SCHEME_BENEFITS,
        ACCIDENT_INSURANCE_SCHEME_BENEFITS,
        HOSPITAL_SAVINGS_SCHEME_BENEFITS,
        UNEMPLOYMENT_OR_REDUNDANCY_INSURANCE,
        PERMANENT_HEALTH_INSURANCE,
        ANY_OTHER_SICKNESS_INSURANCE,
        CRITICAL_ILLNESS_COVER,
        TRADE_UNION_SICK_OR_STRIKE_PAY
    ],
    [
        HEALTH_INSURANCE,
        ALIMONY_AND_CHILD_SUPPORT_PAID,
        TRADE_UNIONS_ETC,
        FRIENDLY_SOCIETIES,
        WORK_EXPENSES,
        AVCS,
        OTHER_DEDUCTIONS,
        LOAN_REPAYMENTS,
        PENSION_CONTRIBUTIONS_EMPLOYEE,
        PENSION_CONTRIBUTIONS_EMPLOYER
    ]
)

const LEGACY_HB_INCOME = union( LEGACY_MT_INCOME.included,
    # since these are passported this should only
    # ever matter if we have a 'passporting' switch
    # and it's turned off, but anyway ....
    [
    INCOME_SUPPORT,
    CONTRIB_JOBSEEKERS_ALLOWANCE,
    NON_CONTRIB_JOBSEEKERS_ALLOWANCE,
    NON_CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE,
    CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE,
    CHILD_TAX_CREDIT ] )

const LEGACY_PC_INCOME = setdiff(LEGACY_MT_INCOME.included, [WORKING_TAX_CREDIT] )

const LEGACY_SAVINGS_CREDIT_INCOME = setdiff( LEGACY_MT_INCOME.included,
    [ WORKING_TAX_CREDIT,
    INCAPACITY_BENEFIT,
    CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE,
    # TODO CHECK BOTH?
    CONTRIB_JOBSEEKERS_ALLOWANCE,
    MATERNITY_ALLOWANCE,
    ALIMONY_AND_CHILD_SUPPORT_RECEIVED ])

const DIVIDEND_INCOME = [ STOCKS_SHARES ]

const EXEMPT_INCOME = [
    CARERS_ALLOWANCE,
    FREE_SCHOOL_MEALS,
    DLA_SELF_CARE,
    DLA_MOBILITY,
    CHILD_BENEFIT,
    PENSION_CREDIT,
    BEREAVEMENT_ALLOWANCE,
    ARMED_FORCES_COMPENSATION_SCHEME,
    WAR_WIDOWS_PENSION,
    SEVERE_DISABILITY_ALLOWANCE,
    ATTENDANCE_ALLOWANCE,
    INDUSTRIAL_INJURY_BENEFIT,
    CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE,
    INCAPACITY_BENEFIT,
    INCOME_SUPPORT,
    MATERNITY_ALLOWANCE,
    MATERNITY_GRANT,
    FUNERAL_GRANT,
    GUARDIANS_ALLOWANCE,
    WINTER_FUEL_PAYMENTS,
    WORKING_TAX_CREDIT,
    CHILD_TAX_CREDIT,
    HOUSING_BENEFIT,
    PERSONAL_INDEPENDENCE_PAYMENT_DAILY_LIVING,
    PERSONAL_INDEPENDENCE_PAYMENT_MOBILITY,
    # ...
    SCOTTISH_CARERS_SUPPLEMENT,
    SCOTTISH_DISABILITY_ASSISTANCE_CHILDREN_DAILY_LIVING,
    SCOTTISH_DISABILITY_ASSISTANCE_CHILDREN_MOBILITY,
    SCOTTISH_DISABILITY_ASSISTANCE_OLDER_PEOPLE,
    SCOTTISH_DISABILITY_ASSISTANCE_WORKING_AGE_DAILY_LIVING,
    SCOTTISH_DISABILITY_ASSISTANCE_WORKING_AGE_MOBILITY
]

const ALL_TAXABLE_INCOME = setdiff( ALL_INCOMES, EXEMPT_INCOME )
const NON_SAVINGS_INCOME = setdiff( ALL_TAXABLE_INCOME, DIVIDEND_INCOME, SAVINGS_INCOME )

const DEFAULT_PASSPORTED_BENS = [
        INCOME_SUPPORT,
        NON_CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE,
        NON_CONTRIB_JOBSEEKERS_ALLOWANCE,
        PENSION_CREDIT ]

export inctostr, isettostr, non_zeros

function inctostr( incs :: AbstractVector; round_inc :: Bool = true) :: String
    s =
    """

    |            |              |
    |:-----------|-------------:|
    """
    for i in instances(Incomes)
        if incs[i] != 0
            m = round_inc ? md_format(incs[i]) : "$(incs[i])"
            s *= "|**$(iname(i))**|$m|
            "
        end
    end
    s *= "

    "
    return s
end

function non_zeros( incs :: AbstractVector ) :: Vector{Tuple}
    v = []
    for i in instances(Incomes)
        if incs[i] != 0
            push!(v, (iname(i), incs[i]))
        end
    end
    return v
end

function isettostr( iset )
    s = ""
    for i in iset
        s *= "$(iname(i))\n"
    end
    return s
end

# FIXME move all exports close to the module top
export
    create_incomes_dataframe,
    fill_inc_frame_row!

function create_incomes_dataframe( RT :: DataType, n :: Int ) :: DataFrame
    d = DataFrame(
    pid  = zeros( BigInt, n ),
    hid = zeros( BigInt, n ),
    weight = zeros( RT, n ) )
    # buno = zeros( Int, n ),
    for i in instances(Incomes)
        lab = i2sym(i)
        d[!,lab] = zeros(RT,n)
    end
    return d
end

function fill_inc_frame_row!(
        incd :: DataFrameRow,
        pid :: BigInt,
        hid :: BigInt,
        weight :: Real,
        income :: IncomesArray )
    @assert size( income )[1] == INC_ARRAY_SIZE
    incd.pid = pid
    incd.hid = hid
    incd.weight = weight

    for i in instances(Incomes)
        lab = i2sym(i)
        incd[lab] = income[i]
    end
end # add_to_inc_frame

end # module


text:
# This file is a part of JuliaFEM.
# License is MIT: see https://github.com/JuliaFEM/JuliaFEM.jl/blob/master/LICENSE.md

using JuliaFEM
using JuliaFEM.Testing
importall Base
import JuliaFEM: get_basis, get_dbasis

type TestElement <: AbstractElement
end

function get_basis(element::Element{TestElement}, xi, time)
    1/4*[
        (1-xi[1])*(1-xi[2])
        (1+xi[1])*(1-xi[2])
        (1+xi[1])*(1+xi[2])
        (1-xi[1])*(1+xi[2])]'
end

function get_dbasis(element::Element{TestElement}, xi, time)
    1/4*[
        -(1-xi[2])    (1-xi[2])   (1+xi[2])  -(1+xi[2])
        -(1-xi[1])   -(1+xi[1])   (1+xi[1])   (1-xi[1])]
end

function length(element::Element{TestElement})
    return 4
end

function size(element::Element{TestElement})
    return (2, 4)
end

function get_element()
    element = Element(TestElement, [1, 2, 3, 4])
    X = Dict{Int64, Vector{Float64}}(
        1 => [0.0, 0.0],
        2 => [1.0, 0.0],
        3 => [1.0, 1.0],
        4 => [0.0, 1.0])
    T = Dict{Int64, Float64}(
        1 => 1.0,
        2 => 2.0,
        3 => 3.0,
        4 => 4.0)
    u1 = Dict{Int64, Vector{Float64}}(
        1 => [0.0, 0.0],
        2 => [0.0, 0.0],
        3 => [1/4, 0.0],
        4 => [0.0, 0.0])
    u2 = Dict{Int64, Vector{Float64}}(
        1 => [0.0,  0.0],
        2 => [1.0, -1.0],
        3 => [2.0,  3.0],
        4 => [0.0,  0.0])
    update!(element, "geometry", X)
    update!(element, "temperature", T)
    update!(element, "displacement1", u1)
    update!(element, "displacement2", u2)
    return element
end

@testset "spatial interpolation in basis" begin
    element = get_element()
    @test isapprox(element([0.0, 0.0], 0.0), 1/4*[1 1 1 1])
    @test isapprox(element([0.0, 0.0], 1.0), 1/4*[1 1 1 1])
end

@testset "gradient of shape functions" begin
    element = get_element()
    grad = element([0.0, 0.0], 0.0, Val{:Grad})
    @test isapprox(grad, 1/2*[-1 1 1 -1; -1 -1 1 1])
end

@testset "interpolation of scalar field in spatial domain" begin
    # in unit square: T(X,t) = t*(1 + X[1] + 3*X[2] - 2*X[1]*X[2])
    element = get_element()
    T_known(X) = 1 + X[1] + 3*X[2] - 2*X[1]*X[2]
    T_interpolated = element("temperature", [0.0, 0.0], 0.0)
    @test isapprox(T_interpolated, T_known([0.5, 0.5]))
end

@testset "interpolation of gradient of scalar field in spatial domain" begin
    # in unit square: grad(T)(X) = [1-2X[2], 3-2*X[1]]
    element = get_element()
    gradT = element("temperature", [0.0, 0.0], 0.0, Val{:Grad})
    gradT_expected(X) = [1-2*X[2] 3-2*X[1]]
    @test isapprox(gradT, gradT_expected([0.5, 0.5]))
end

@testset "test interpolation of vector field" begin
    # in unit square, u(X,t) = [1/4*t*X[1]*X[2], 0, 0]
    element = get_element()
    u = element("displacement1", [0.0, 0.0], 0.0)
#   x = X+u
    u_expected(X) = [1/4*X[1]*X[2], 0]
#   @test isapprox(x, [9/16, 1/2])
    @test isapprox(u, u_expected([0.5, 0.5]))
end

@testset "interpolation of gradient of vector_field" begin
    # in unit square, u(X) = t*[X[1]*(X[2]+1), X[1]*(4*X[2]-1)]
    # => u_i,j = t*[X[2]+1 X[1]; 4*X[2]-1 4*X[1]]
    element = get_element()
#    displacement = Field(
#        (0.5, Vector[[0.0, 0.0], [0.5, -0.5], [1.0, 1.5], [0.0, 0.0]]),
#        (1.5, Vector[[0.0, 0.0], [1.5, -1.5], [3.0, 4.5], [0.0, 0.0]]))
    gradu = element("displacement2", [0.0, 0.0], 0.0, Val{:Grad})
    gradu_expected(X) = [X[2]+1 X[1]; 4*X[2]-1 4*X[1]]
    @test isapprox(gradu, gradu_expected([0.5, 0.5]))
end

#= TODO: Fix test
@testset "linear time extrapolation of field" begin
    #T_known(X,t) = t*(1 + X[1] + 3*X[2] - 2*X[1]*X[2])
    T = DVTV()
    update!(T, 0.0 => [0.0, 0.0, 0.0, 0.0])
    update!(T, 1.0 => [1.0, 2.0, 3.0, 4.0])
    @test T(-1.0) == -1.0*[1.0, 2.0, 3.0, 4.0]
    @test T( 3.0) ==  3.0*[1.0, 2.0, 3.0, 4.0]
    # when going to \pm infinity, return the last one.
    @test T(-Inf) ==  0.0*[1.0, 2.0, 3.0, 4.0]
    @test T(+Inf) ==  1.0*[1.0, 2.0, 3.0, 4.0]
end
=#

#= TODO: Fix test
@testset "constant time extrapolation of field" begin
    #T_known(X,t) = t*(1 + X[1] + 3*X[2] - 2*X[1]*X[2])
    T = DVTV()
    update!(T, 0.0 => [0.0, 0.0, 0.0, 0.0])
    update!(T, 1.0 => [1.0, 2.0, 3.0, 4.0])
    @test isapprox(T(-1.0, Val{:constant}), [0.0, 0.0, 0.0, 0.0])
    @test isapprox(T( 3.0, Val{:constant}), [1.0, 2.0, 3.0, 4.0])
end
=#

#= TODO: Fix test
@testset "time extrapolation of field with only one timestep" begin
    T = DVTV()
    update!(T, 0.0 => [1.0, 2.0, 3.0, 4.0])
    @test isapprox(T(1.0), [1.0, 2.0, 3.0, 4.0])
end
=#

#= TODO: Fix test
@testset "interpolation in temporal direction" begin
    field = DCTV()
    update!(field, 0.0 => 0.0)
    update!(field, 2.0 => 1.0)
    update!(field, 4.0 => 2.0)
    @test isapprox(field(-Inf), 0.0)
    @test isapprox(field( 0.0), 0.0)
    @test isapprox(field( 1.0), 0.5)
    @test isapprox(field( 2.0), 1.0)
    @test isapprox(field( 3.0), 1.5)
    @test isapprox(field( 4.0), 2.0)
    @test isapprox(field(+Inf), 2.0)
end
=#

#= TODO: Fix test
@testset "time derivative interpolation in temporal basis in constant velocity" begin
    field = DCTV()
    update!(field, 0.0 => 0.0)
    update!(field, 2.0 => 1.0)
    update!(field, 4.0 => 2.0)
    @test isapprox(field(+Inf, Val{:diff}), 0.5)
    @test isapprox(field(-Inf, Val{:diff}), 0.5)
    @test isapprox(field( 0.0, Val{:diff}), 0.5)
    @test isapprox(field( 0.5, Val{:diff}), 0.5)
    @test isapprox(field( 1.0, Val{:diff}), 0.5)
    @test isapprox(field( 1.5, Val{:diff}), 0.5)
    @test isapprox(field( 2.0, Val{:diff}), 0.5)
end
=#

#= TODO: Fix test
@testset "time derivative interpolation in temporal basis in variable velocity" begin
    pos = DCTV()
    for ti in linspace(0, 2, 5)
        update!(pos, ti => 1/2*ti^2)
    end
    # => ((0.0,0.0),(0.5,0.125),(1.0,0.5),(1.5,1.125),(2.0,2.0))
    velocity = pos(1.0, Val{:diff})
    v1 = (0.500 - 0.125)/0.5
    v2 = (1.125 - 0.500)/0.5
    @test isapprox(velocity, mean([v1, v2])) # = 1.00
    velocity = pos(2.0, Val{:diff})
    @test isapprox(velocity, (2.0-1.125)/0.5) # = 1.75
end
=#

function test_time_derivative_gradient_interpolation_of_field()
    # in unit square, u(X) = t*[X[1]*(X[2]+1), X[1]*(4*X[2]-1)]
    # => u_i,j = t*[X[2]+1 X[1]; 4*X[2]-1 4*X[1]]
    # => d(u_i,j)/dt = [X[2]+1 X[1]; 4*X[2]-1 4*X[1]]
    X = Dict{Int64, Vector{Float64}}(
        1 => [0.0, 0.0],
        2 => [1.0, 0.0],
        3 => [1.0, 1.0],
        4 => [0.0, 1.0])
    u1 = Dict{Int64, Vector{Float64}}(
        1 => [0.0,  0.0],
        2 => [0.5, -0.5],
        3 => [1.0,  1.5],
        4 => [0.0,  0.0])
    u2 = Dict{Int64, Vector{Float64}}(
        1 => [0.0,  0.0],
        2 => [1.5, -1.5],
        3 => [3.0,  4.5],
        4 => [0.0,  0.0])
    element = Element(TestElement, [1, 2, 3, 4])
    update!(element, "geometry", X)
    update!(element, "displacement", 0.5 => u1)
    update!(element, "displacement", 1.5 => u2)

    xi = [0.0, 0.0]
    time = 1.2
    diffgradu = element("displacement", xi, time, Val{:diff}, Val{:Grad})
    diffgradu_expected(X, t) = [X[2]+1 X[1]; 4*X[2]-1 4*X[1]]
    @test diffgradu == diffgradu_expected([0.5, 0.5], 1.2)
end

@testset "some continuum mechanics interpolations" begin
    X = Dict{Int64, Vector{Float64}}(
        1 => [0.0, 0.0],
        2 => [1.0, 0.0],
        3 => [1.0, 1.0],
        4 => [0.0, 1.0])
    u1 = Dict{Int64, Vector{Float64}}(
        1 => [0.0, 0.0],
        2 => [0.0, 0.0],
        3 => [0.0, 0.0],
        4 => [0.0, 0.0])
    u2 = Dict{Int64, Vector{Float64}}(
        1 => [0.0, 0.0],
        2 => [0.0, 0.0],
        3 => [1/4, 0.0],
        4 => [0.0, 0.0])
    element = Element(Quad4, [1, 2, 3, 4])
    update!(element, "geometry", X)
    update!(element, "displacement", 0.0 => u1)
    update!(element, "displacement", 1.0 => u2)

    # from my old home works
    X = element("geometry", [0.0, 0.0], 1.0)
    u = element("displacement", [0.0, 0.0], 1.0)
    x = X + u
    x_expected = [9/16, 1/2]
    gradu = element("displacement", [0.0, 0.0], 1.0, Val{:Grad})
    epsilon = 1/2*(gradu + gradu')
    rotation = 1/2*(gradu - gradu')
    k = 0.25
    epsilon_expected = [
        X[2]*k      1/2*X[1]*k
        1/2*X[1]*k  0]
    rotation_expected = [
        0          k/2*X[1]
        -k/2*X[1]         0]
    F = I + gradu
    F_expected = [
        X[2]*k+1 X[1]*k
        0        1]
    C = F'*F
    C_expected = [
        (X[2]*k+1)^2       (X[2]*k+1)*X[1]*k
        (X[2]*k+1)*X[1]*k  X[1]^2*k^2+1]
    E = 1/2*(F'*F - I)
    E_expected = [
        1/2*(X[2]*k + 1)^2-1/2   1/2*(X[2]*k+1)*X[1]*k
        1/2*(X[2]*k + 1)*X[1]*k  1/2*X[1]^2*k^2]
    U = 1/sqrt(trace(C) + 2*sqrt(det(C)))*(C + sqrt(det(C))*I)
#   U_expected = [1.24235 0.13804; 0.13804 1.02149]

    @test isapprox(x, x_expected)
    @test isapprox(epsilon, epsilon_expected)
    @test isapprox(rotation, rotation_expected)
    @test isapprox(F, F_expected)
    @test isapprox(C, C_expected)
    @test isapprox(E, E_expected)
    # TODO: Fix test
#   @test isapprox(U, U_expected)
end


text:
##########################
# FunctionCell Interface #
##########################
"""
	affinityprop(fs [;kwargs])

Constructs an untrained cell that when piped data inside, clusters the data using the
affinity propagation algorithm; `fs` is a function that is used to calculate the
similarity matrices during training as well as when assigning new data to existing clusters.
By default, `fs(x,y) = 1 - pairwise(CosineDist(),x,y)`

# Keyword arguments (same as in `Clustering`)
  * `maxiter` maximum number of iterations (default `200`)
  * `tol` tolerable change in objective at convergence (default `1.0e-6`)
  * `damp` dampening coefficient; values should be in [0.0, 1.0] (default `0.5`); larger values indicate
slower and probably more stable updates
  * `display` Level of information; can be `:none`, `:final`, `:iter` (default `:none`)

!!! note
	The implementation differes from the one in `Clustering.jl` by allowing the user to
	input directly data and not a similarity matrix to the algorithm.

Read the `Clustering.jl` documentation for more information.
"""
affinityprop(fs::Function=(x,y)->one(eltype(x)) .- Distances.pairwise(Distances.CosineDist(),x,y); kwargs...) =
	FunctionCell(affinityprop, (fs,), ModelProperties(), kwtitle("Affinity propagation", kwargs); kwargs...)

############################
# DataCell/Array Interface #
############################
"""
	affinityprop(x, fs::Function=(x,y)->1-Distances.pairwise(CosineDist(),x,y) [;kwargs])

Performs affinity propagation clustering on the dataset `x` using `fs` to calculate the self-similarity matrix;
When piping data to the resulting clustering (e.g. trained cell), the similarities between each input observation
and all exemplars are returned. One can apply `targets(indmax, ...)` to obtain the corresponding exemplar
index (e.g. cluster) for each input observation.
"""
# Training
affinityprop(x::T where T, fs::Function=(x,y)->1-Distances.pairwise(Distances.CosineDist(),x,y); kwargs...) =
	affinityprop(getx!(x), fs; kwargs...)
affinityprop(x::T where T<:AbstractVector, fs::Function=(x,y)->1-Distances.pairwise(Distances.CosineDist(),x,y); kwargs...) =
	affinityprop(mat(x, LearnBase.ObsDim.Constant{2}()), fs; kwargs...)
affinityprop(x::T where T<:AbstractMatrix, fs::Function=(x,y)->1-Distances.pairwise(Distances.CosineDist(),x,y); kwargs...) =
begin
	# Perform clustering
	affpdata = Clustering.affinityprop(fs(x,x); kwargs...)

	# Build model properties
	modelprops = ModelProperties(nvars(x),length(affpdata.exemplars))

	FunctionCell(affinityprop, Model((fs,x[:,sort(affpdata.exemplars)],affpdata), modelprops), kwtitle("Affinity propagation", kwargs));
end

# Execution
affinityprop(x::T where T<:CellData, model::Model{<:Tuple{<:Function, <:AbstractArray, <:Clustering.AffinityPropResult}}) =
	datacell(affinityprop(getx!(x), model), gety(x))
affinityprop(x::T where T<:AbstractVector, model::Model{<:Tuple{<:Function, <:AbstractArray, <:Clustering.AffinityPropResult}}) =
	affinityprop(mat(x, LearnBase.ObsDim.Constant{2}()), model)
affinityprop(x::T where T<:AbstractMatrix, model::Model{<:Tuple{<:Function, <:AbstractArray, <:Clustering.AffinityPropResult}}) =
	model.data[1](model.data[2], x) #fs(exemplars,x)


text:
module Trapz
    export trapz

    @inline function bringlast(T::Tuple,el)
        ifelse(el==T[1],
            (bringlast(Base.tail(T),el)...,T[1]) ,
            (T[1],bringlast(Base.tail(T),el)...)
        )
    end
    @inline function bringlast(T::Tuple{},el); T; end

    @inline function trapz_colon(k) Colon(); end
    @inline function idxlast(i,::Val{N}) where N; Base.tail((ntuple(trapz_colon,Val(N))...,i)) end
    function killzerodim(A::AbstractArray{fT,0}) where fT
        A[]
    end
    function killzerodim(A::AbstractArray{fT,N}) where {fT,N}
        A
    end
    function killzerodim(A::fT) where {fT<:Number}
        A
    end
    function trapz(x::T1, y::T2) where {N,fT,T1<:AbstractVector{fT},T2<:AbstractArray{fT,N}}
        n = length(x)
        s = size(y)
        @assert s[end]==n
        @inbounds begin
            r = similar(y,Base.reverse(Base.tail(Base.reverse(s))))
            if n <= 1;
                r.=zero(fT)
                @goto retval;
            end
            @fastmath r .= (x[2] - x[1]) .* view(y,idxlast(1,Val(N))...)
            for i in 2:n-1
               @fastmath r .+= (x[i+1] - x[i-1]) .* view(y,idxlast(i,Val(N))...)
            end
            @fastmath r .+= (x[end]-x[end-1]) .* view(y,idxlast(n,Val(N))...)
            @label retval
            return killzerodim(r./2)
        end
    end

    function trapz(x::T1, y::T2, axis::T3) where {N,fT,T1<:AbstractVector{fT},T2<:AbstractArray{fT,N},T3<:Integer}
        @assert 1<=axis<=N
        trapz(x,PermutedDimsArray(y,bringlast(ntuple(identity,Val(N)),axis)))
    end

    function gen_trapz_impl(xs::Type{T}, M) where T <: NTuple{N} where N
        ex = :(trapz(xs[$N], M))
        for i=(N-1):-1:1
            ex = :(trapz(xs[$i],$ex))
        end
        return ex
    end

    @generated function trapz(xs::NTuple{N}, M) where N
        return gen_trapz_impl(xs, M)
    end

    function trapz(xs::NTuple{N}, M::AbstractArray{T,S}, axes::NTuple{N}) where {N, T, S}
        if S > N
            axes = ((i for i=1:S if !in(i,axes))..., axes...)
        end
        trapz(xs, PermutedDimsArray(M, axes))
    end

    """
        trapz(x,y,axis=End)
        Calculates ∫y[..., i (axis) ,...] dx[i]
        for performance reasons x must have the same float type as y.
        1-D Example:
    ```julia
        vx=0:0.01:1
        vy=(x->x^2).(vx)
        trapz(vx,vy)
    ```
            Result ≈ 1/3

        2-D Example:
    ```julia
        vx=0:0.01:1
        vy=0:0.01:2
        z=[x^2+y^2 for x=vx, y=vy]
        trapz((vx,vy),z) # equivalent to trapz(vx, trapz(vy, z))
    ```
            Result ≈ 4/3

        2-D Example in reverse integration order:
    ```julia
        vx=0:0.01:1
        vy=0:0.01:2
        z=[x^2+y^2 for x=vx, y=vy]
        trapz((vy,vx),z,(2,1)) # equivalent to trapz(vy, trapz(vx, z, 1), 1)
    ```
            Result ≈ 4/3
    """
    trapz

end # module

#Some testing code
# using BenchmarkTools,GPUArrays
#
# vx=range(0,1,length=100)
# vy=vx.^2
# My=[x^2+y^2 for x in vx,y in vx]
# Ty=[x^2+y^2+z^2 for x in vx,y in vx,z in vx]
#
#
# Main.Trapz.trapz(vx,JLArray(Ty))
# Main.Trapz.trapz(vx,JLArray(My))
# Main.Trapz.trapz(vx,JLArray(vy))
# Main.Trapz.trapz(vx,My)
# Main.Trapz.trapz(vx,Ty)
# Main.Trapz.trapz(vx,vy)
#


text:
# Belief Propagation                #
# --------------------------------- #
@rule MvNormalMeanCovariance(:out, Marginalisation) (m_μ::PointMass, m_Σ::PointMass) = MvNormalMeanCovariance(mean(m_μ), mean(m_Σ))

@rule MvNormalMeanCovariance(:out, Marginalisation) (m_μ::MultivariateNormalDistributionsFamily, m_Σ::PointMass) = begin
    m_μ_mean, m_μ_cov = mean_cov(m_μ)
    return MvNormalMeanCovariance(m_μ_mean, m_μ_cov + mean(m_Σ))
end

# Variational                       #
# --------------------------------- #
@rule MvNormalMeanCovariance(:out, Marginalisation) (q_μ::Any, q_Σ::Any) = MvNormalMeanCovariance(mean(q_μ), mean(q_Σ))

@rule MvNormalMeanCovariance(:out, Marginalisation) (m_μ::PointMass, q_Σ::Any) = MvNormalMeanCovariance(mean(m_μ), mean(q_Σ))

@rule MvNormalMeanCovariance(:out, Marginalisation) (m_μ::MultivariateNormalDistributionsFamily, q_Σ::Any) = begin
    m_μ_mean, m_μ_cov = mean_cov(m_μ)
    return MvNormalMeanCovariance(m_μ_mean, m_μ_cov + mean(q_Σ))
end