text:
"""
```@setup general
data_dir = joinpath("..", "..", "test", "data") # hide
using CSV, DataFramesMeta, Dates, AbnormalReturns

df_firm = CSV.File(joinpath(data_dir, "daily_ret.csv")) |> DataFrame
df_mkt = CSV.File(joinpath(data_dir, "mkt_ret.csv")) |> DataFrame
df_mkt[!, :mkt] = df_mkt.mktrf .+ df_mkt.rf
df_events = CSV.File(joinpath(data_dir, "firm_earnings_announcements.csv")) |> DataFrame
mkt_data = MarketData(
    df_mkt,
    df_firm
)
```
"""
abstract type CalendarData end

"""
    struct DataVector <: CalendarData
        data::Vector{Float64}
        missing_bdays::SparseVector{Bool, Int}
        dates::ClosedInterval{Date}
        calendar::MarketCalendar
        function DataVector(data, missing_bdays, dates, calendar)
            @assert length(data) == length(missing_bdays) == bdayscount(calendar, dates.left, dates.right) + 1 "Data does not match length of dates or missings"
            new(data, missing_bdays, dates, calendar)
        end
    end

    DataVector(data::AbstractVector, dates::ClosedInterval{Date}, cal::MarketCalendar)

    DataVector(data::AbstractVector, dates::AbstractVector{Date}, cal::MarketCalendar)
"""
struct DataVector <: CalendarData
    data::Vector{Float64}
    missing_bdays::SparseVector{Bool, Int}
    dates::ClosedInterval{Date}
    calendar::MarketCalendar
    function DataVector(data, missing_bdays, dates, calendar)
        @assert length(data) == length(missing_bdays) == bdayscount(calendar, dates.left, dates.right) + 1 "Data does not match length of dates or missings"
        new(data, missing_bdays, dates, calendar)
    end
end

"""
    struct DataMatrix <: CalendarData
        data::Matrix{Float64}
        missing_bdays::SparseVector{Bool, Int}# corresponds to each row with a missing value
        dates::ClosedInterval{Date}
        calendar::MarketCalendar
        function DataMatrix(data, missing_bdays, dates, calendar)
            @assert size(data, 1) == length(missing_bdays) == bdayscount(calendar, dates.left, dates.right) + 1 "Data does not match length of dates or missings"
            new(data, missing_bdays, dates, calendar)
        end
    end

    DataMatrix(data::AbstractMatrix, dates::ClosedInterval{Date}, cal::MarketCalendar)
"""
struct DataMatrix <: CalendarData
    data::Matrix{Float64}
    missing_bdays::SparseVector{Bool, Int}# corresponds to each row with a missing value
    dates::ClosedInterval{Date}
    calendar::MarketCalendar
    function DataMatrix(data, missing_bdays, dates, calendar)
        @assert size(data, 1) == length(missing_bdays) == bdayscount(calendar, dates.left, dates.right) + 1 "Data does not match length of dates or missings"
        new(data, missing_bdays, dates, calendar)
    end
end

struct MarketData{T, MNames, FNames, N1, N2}
    calendar::MarketCalendar
    marketdata::NamedTuple{MNames, NTuple{N1, DataVector}} # column names as symbols
    firmdata::Dict{T, NamedTuple{FNames, NTuple{N2, DataVector}}} # data stored by firm id and then by column name as symbol
end

struct AllowMissing{mssng} end

"""
    mutable struct TimelineTable{Mssng, T, MNames, FNames, N1, N2} <: Tables.AbstractColumns
        parent::MarketData{T, MNames, FNames, N1, N2}
        "The current ID"
        id::T
        "Whether the functions return Vector{Union{Missing, Float64}} or Vector{Float64}"
        allow_missing::Type{AllowMissing{Mssng}}
        "Actual returnable dates that guarentees a square matrix made of the minimum and maximum of the DataVectors for this ID and set of columns"
        dates::ClosedInterval{Date}
        "Index of column names and any lags/leads"
        cols::DictIndex
        "missing days between the dates"
        missing_bdays::SparseVector{Bool, Int}
        "dates requested, which provides what data is automatically returned by functions"
        req_dates::ClosedInterval{Date}
    end

This type provides [Tables.jl](https://github.com/JuliaData/Tables.jl) access to
`DataVector`s.

Functions to update some values are:
- `AbnormalReturns.update_id!`: changes the `id` (which updates `dates` and `missing_bdays`)
- `select!`: changes the `cols` (also updates `dates` and `missing_bdays` if `cols` is different)
- `AbnormalReturns.update_dates!`: changes `req_dates` (does not update anything else)
"""
mutable struct TimelineTable{Mssng, T, MNames, FNames, N1, N2} <: Tables.AbstractColumns
    parent::MarketData{T, MNames, FNames, N1, N2}
    "The current ID"
    id::T
    "Whether the functions return Vector{Union{Missing, Float64}} or Vector{Float64}"
    allow_missing::Type{AllowMissing{Mssng}}
    "Actual returnable dates that guarentees a square matrix made of the minimum and maximum of the DataVectors for this ID and set of columns"
    dates::ClosedInterval{Date}
    "Index of column names and any lags/leads"
    cols::DictIndex
    "missing days between the dates"
    missing_bdays::SparseVector{Bool, Int}
    "dates requested, which provides what data is automatically returned by functions"
    req_dates::ClosedInterval{Date}
end

"""
    function MarketData(
        df_market,
        df_firms;
        date_col_market=:date,
        date_col_firms=:date,
        id_col=:permno,
        valuecols_market=nothing,
        valuecols_firms=nothing
    )

## Arguments
- `df_market`: A Tables.jl compatible source that stores market data, indexed by date.
    The dates must be a unique set. The column name for the date column is specified
    by the keyword argument "date_col_market"
- `df_firms`: A Tables.jl compatible source that stores firm data. Each firm must have a
    unique set of dates. The column name for the date column is specified
    by the keyword argument "date_col_firms" and the firm ID column is specified
    by the keyword argument "id_col"
- `valuecols_market=nothing`: If left as nothing, all other columns in `df_market` are
    used as the value columns. These are the columns that are stored in the resulting
    dataset. Otherwise a vector of Symbol or String specifying column names.
- `valuecols_firms=nothing`: Same as above
- `id_col=:permno`: The column corresponding to the set of firm IDs in `df_firms`

MarketData is the main data storage structure. Data is stored for each firm in
a Dict, where the data itself is a NamedTuple (names corresponding to column names,
such as "ret"), and the keys for the Dict corresponding to firm IDs. The MarketData
struct also stores overall market data and a calendar of dates.

Any firm data must have a corresponding market data date, so there cannot be a
firm return if there is not a market return on that date.

## Example

```@example general
df_firm = CSV.File(joinpath(data_dir, "daily_ret.csv"))
df_mkt = CSV.File(joinpath(data_dir, "mkt_ret.csv"))
df_mkt[!, :mkt] = df_mkt.mktrf .+ df_mkt.rf
mkt_data = MarketData(
    df_mkt,
    df_firm
)
```
"""
function MarketData(
    df_market,
    df_firms;
    date_col_market=:date,
    date_col_firms=:date,
    id_col=:permno,
    valuecols_market=nothing,
    valuecols_firms=nothing
)
    df_market = DataFrame(df_market)
    df_firms = DataFrame(df_firms)
    if valuecols_market === nothing
        valuecols_market = Symbol.([n for n in Symbol.(names(df_market)) if n ∉ [date_col_market]])
    end
    if valuecols_firms === nothing
        valuecols_firms = Symbol.([n for n in Symbol.(names(df_firms)) if n ∉ [date_col_firms, id_col]])
    end

    df_market = select(df_market, vcat([date_col_market], valuecols_market))
    dropmissing!(df_market, date_col_market)
    #dropmissing!(df_market)
    sort!(df_market)

    if any(nonunique(df_market, [date_col_market]))
        @error("There are duplicate date rows in the market data")
    end

    df_firms = select(df_firms, vcat([id_col, date_col_firms], valuecols_firms))
    dropmissing!(df_firms, [id_col, date_col_firms])
    sort!(df_firms, [id_col, date_col_firms])

    if any(nonunique(df_firms, [id_col, date_col_firms]))
        @error("There are duplicate id-date rows in the firm data")
    end

    cal = MarketCalendar(df_market[:, date_col_market])

    market_data = NamedTuple(
        valuecols_market .=>
        DataVector.(
            Tables.columns(df_market[:, valuecols_market]),
            Ref(df_market[:, date_col_market]),
            Ref(cal)
        )
    )

    check_all_businessdays(unique(df_firms[:, date_col_firms]), cal)

    gdf = groupby(df_firms, id_col)
    df_temp = combine(
        gdf,
        date_col_firms => (x -> add_missing_bdays(x, cal)) => date_col_firms
    )
    if nrow(df_temp) > 0
        insertcols!(df_temp, [col => missing for col in valuecols_firms]...)
        df_firms = vcat(
            df_firms,
            df_temp
        )
        sort!(df_firms, [id_col, date_col_firms])
        gdf = groupby(df_firms, id_col)
    end

    col_tab = columntable(gdf)

    firm_data = Dict{typeof(col_tab[id_col][1][1]), NamedTuple{Tuple(valuecols_firms), NTuple{length(valuecols_firms), DataVector}}}()
    sizehint!(firm_data, length(col_tab[id_col]))

    for i in 1:length(col_tab[id_col])
        firm_data[col_tab[id_col][i][1]] = NamedTuple(
            valuecols_firms .=> (
                DataVector(
                    col_tab[col][i],
                    col_tab[date_col_firms][i],
                    cal
                ) for col in valuecols_firms
            )
        )
    end

    MarketData(
        cal,
        market_data,
        firm_data
    )
end

function check_all_businessdays(dates, cal)
    bday_list = isbday(cal, dates)
    if !all(bday_list)
        bday_list_inv = (!).(bday_list)
        if sum(bday_list_inv) <= 3
            @error("Dates $(dates[bday_list_inv]) are not in the MARKET_DATA_CACHE")
        else
            d1_test = findfirst(bday_list_inv)
            d2_test = findlast(bday_list_inv)
            @error("Dates $(dates[d1_test]) ... $(dates[d2_test]) are not in the MARKET_DATA_CACHE")
        end
    end
end

function setup_calendar_data(f, data, dates::ClosedInterval{Date}, cal::MarketCalendar)
    if any(ismissing.(data))
        missing_days = sparsevec(any(ismissing, data, dims=2))
        data = coalesce.(data, zero(nonmissingtype(eltype(data))))
    else
        missing_days = spzeros(Bool, size(data, 1))
    end
    f(
        data,
        missing_days,
        dates,
        cal
    )
end

function DataVector(data::AbstractVector, dates::ClosedInterval{Date}, cal::MarketCalendar)
    setup_calendar_data(
        DataVector,
        data,
        dates,
        cal
    )
end

function DataVector(data::AbstractVector, dates::AbstractVector{Date}, cal::MarketCalendar)
    if any(ismissing.(data))
        if all(ismissing.(data))
            return DataVector(
                zeros(nonmissingtype(eltype(data)), 1),
                sparsevec([1], [true]),
                dates[1] .. dates[1],
                cal
            )
        end
        i = findfirst(!ismissing, data)
        j = findlast(!ismissing, data)
        setup_calendar_data(
            DataVector,
            data[i:j],
            dates[i] .. dates[j],
            cal
        )
    else
        setup_calendar_data(
            DataVector,
            data,
            dates[1] .. dates[end],
            cal
        )
    end
end

function DataMatrix(data::AbstractMatrix, dates::ClosedInterval{Date}, cal::MarketCalendar)
    setup_calendar_data(
        DataMatrix,
        data,
        dates,
        cal
    )
end
function add_missing_bdays(dates, cal)
    out = Date[]
    if length(dates) == bdayscount(cal, dates[1], dates[end]) + 1
        return out
    end
    dates_counter = 1
    for d in listbdays(cal, dates[1], dates[end])
        if dates[dates_counter] > d
            push!(out, d)
        else
            dates_counter += 1
        end
    end
    out
end

function get_missing_bdays(cal::MarketCalendar, missing_bdays::SparseVector{Bool, Int}, dates_missings::ClosedInterval{Date}, new_dates::ClosedInterval{Date})
    missing_bdays[date_range(cal, dates_missings, new_dates)]
end

function get_missing_bdays(data, new_dates::ClosedInterval{Date})
    get_missing_bdays(calendar(data), data_missing_bdays(data), data_dates(data), new_dates)
end

function date_range(cal::MarketCalendar, timeline_dates::ClosedInterval{Date}, new_dates::ClosedInterval{Date})
    dates = dates_min_max(timeline_dates, new_dates)
    s = bdayscount(cal, dt_min(timeline_dates), dt_min(dates)) + 1
    e = s + bdayscount(cal, dt_min(dates), dt_max(dates)) - !isbday(cal, dt_max(dates))
    s:e
end

function date_range(cal::MarketCalendar, data::CalendarData, dt_min::Date, dt_max::Date)
    date_range(cal, data_dates(data), dt_min .. dt_max)
end

function date_range(cal::MarketCalendar, data::CalendarData, dates::ClosedInterval{Date})
    date_range(cal, data_dates(data), dates)
end

# , mssngs::SparseVector{Bool, Int}=spzeros(Bool, size(raw_values(data), 1))
function Base.getindex(data::DataMatrix, dates::ClosedInterval{Date})
    new_dates = dates_min_max(data_dates(data), dates)
    r = date_range(calendar(data), data_dates(data), new_dates)
    data.data[r, :]
end

function Base.getindex(data::DataVector, dates::ClosedInterval{Date})
    new_dates = dates_min_max(data_dates(data), dates)
    r = date_range(calendar(data), data_dates(data), new_dates)
    data.data[r]
end

function combine_missing_bdays(vals::SparseVector{Bool, Int}...)::SparseVector{Bool, Int}
    (|).(vals...)
end

function combine_missing_bdays(vals::Vector{SparseVector{Bool, Int}})::SparseVector{Bool, Int}
    out = sparsevec(zeros(Bool, length(vals[1])))
    for v in vals
        out = out .| v
    end
    out
end

function Base.view(data::DataVector, dates::ClosedInterval{Date}, mssngs::SparseVector{Bool, Int})
    @assert dt_min(data) <= dt_min(dates) && dt_max(dates) <= dt_max(data) "Date range out of bounds"
    r = date_range(calendar(data), data_dates(data), dates)
    @assert length(mssngs) == length(r) "Missing data vector is the wrong length"
    if nnz(mssngs) == 0
        view(data.data, r)
    else
        view(data.data, r[(!).(mssngs)])
    end
end

function Base.view(data::DataMatrix, dates::ClosedInterval{Date}, mssngs::SparseVector{Bool, Int})
    @assert dt_min(data) <= dt_min(dates) && dt_max(dates) <= dt_max(data) "Date range out of bounds"
    r = date_range(calendar(data), data_dates(data), dates)
    @assert length(mssngs) == length(r) "Missing data vector is the wrong length"
    if nnz(mssngs) == 0
        view(data.data, r, :)
    else
        view(data.data, r[(!).(mssngs)], :)
    end
end

function Base.view(data::DataVector, dates::ClosedInterval{Date})
    @assert dt_min(data) <= dt_min(dates) && dt_max(dates) <= dt_max(data) "Date range out of bounds"
    r = date_range(calendar(data), data_dates(data), dates)
    view(data.data, r)
end

function Base.view(data::DataMatrix, dates::ClosedInterval{Date})
    @assert dt_min(data) <= dt_min(dates) && dt_max(dates) <= dt_max(data) "Date range out of bounds"
    r = date_range(calendar(data), data_dates(data), dates)
    view(data.data, r, :)
end

Base.getindex(data::NamedTuple{Names, NTuple{N, DataVector}}, col::TimelineColumn) where {Names, N} = shift(data[Symbol(col)], col.shifts)

function Base.getindex(data::MarketData{T, MNames, FNames}, id::T, col::TimelineColumn) where {T, MNames, FNames}
    shift(pick_data(data, id, Symbol(col)), col.shifts)
end

function Base.getindex(
    data::MarketData{T, MNames, FNames},
    id::T,
    cols::Vector{TimelineColumn}=TimelineColumn.([FNames..., MNames...]),
    allow_mssng::Type{AllowMissing{Mssng}}=AllowMissing{false},
) where {T, MNames, FNames, Mssng}
    real_dates = dates_min_max(get_all_dates(data, id, cols))
    data[id, real_dates, cols, allow_mssng]
end

function Base.getindex(
    data::MarketData{T, MNames, FNames},
    id::T,
    dates::ClosedInterval{Date},
    cols::Vector{TimelineColumn}=TimelineColumn.([FNames..., MNames...]),
    allow_mssng::Type{AllowMissing{Mssng}}=AllowMissing{false}
) where {T, MNames, FNames, Mssng}
    out = TimelineTable(
        data,
        id,
        allow_mssng,
        dates_min_max(get_all_dates(data, id, cols)),
        DictIndex(cols),
        sparsevec(zeros(Bool, bdayscount(calendar(data), dt_min(dates), dt_max(dates)))),
        dates,
    )
    out.missing_bdays = combine_all_missing_bdays(out)
    out
end

function Base.getindex(
    data::MarketData{T},
    id::T,
    dates::ClosedInterval{Date},
    cols::Vector,
    allow_mssng::Type{AllowMissing{Mssng}}=AllowMissing{false}
) where {T, Mssng}
    data[id, dates, TimelineColumn.(cols), allow_mssng]
end

function Base.getindex(data::TimelineTable, col::TimelineColumn)
    parent(data)[data.id, col]
end
Base.getindex(data::TimelineTable, col::Symbol) = data[TimelineColumn(col)]

function Base.getindex(
    data::TimelineTable,
    ::Colon,
    col::TimelineColumn
)
    ret_dates = dates_min_max(data_dates(data), norm_dates(data))
    data[ret_dates, col]
end

function Base.getindex(
    data::TimelineTable{false},
    dates::ClosedInterval{Date},
    col::TimelineColumn
)
    new_mssngs = get_missing_bdays(calendar(data), data_missing_bdays(data), data_dates(data), dates)
    if nnz(new_mssngs) == 0
        data[col][dates]
    else
        data[col][dates][(!).(new_mssngs)]
    end
end

function Base.getindex(
    data::TimelineTable{true},
    dates::ClosedInterval{Date},
    col::TimelineColumn
)
    new_mssngs = get_missing_bdays(calendar(data), data_missing_bdays(data), data_dates(data), dates)
    out = data[col][dates]
    out = allowmissing(out)
    out[new_mssngs] .= missing
    to_add_pre = ifelse(
        dt_min(data) < dt_min(data_dates(data)),
        bdayscount(calendar(data), dt_min(data), dt_min(data_dates(data))),
        0
    )
    to_add_post = ifelse(
        dt_max(data) > dt_max(data_dates(data)),
        bdayscount(calendar(data), dt_max(data_dates(data)), dt_max(data)),
        0
    )
    vcat(
        fill(missing, to_add_pre),
        out,
        fill(missing, to_add_post)
    )
end

function Base.getindex(
    data::TimelineTable,
    ::Colon,
    col::Symbol
)
    data[:, TimelineColumn(col)]
end

function Base.getindex(
    data::TimelineTable,
    dates::ClosedInterval{Date},
    col::Symbol
)
    data[dates, TimelineColumn(col)]
end

function Base.getproperty(
    data::TimelineTable,
    sym::Symbol
)
    if sym == :calendar
        getfield(getfield(data, :parent), :calendar)
    elseif sym == :firmdata
        getfield(getfield(data, :parent), :firmdata)
    elseif sym == :marketdata
        getfield(getfield(data, :parent), :marketdata)
    else
        getfield(data, sym)
    end
end

function get_dates(data::TimelineTable{true})
    listbdays(calendar(data), dt_min(data), dt_max(data))
end

function get_dates(data::TimelineTable{false})
    real_dates = dates_min_max(data_dates(data), norm_dates(data))
    out = listbdays(calendar(data), dt_min(real_dates), dt_max(real_dates))
    new_mssngs = get_missing_bdays(calendar(data), data_missing_bdays(data), data_dates(data), real_dates)
    if nnz(new_mssngs) == 0
        return out
    else
        return out[(!).(new_mssngs)]
    end
end

Base.names(x::TimelineTable) = x.cols.cols

Tables.istable(::Type{<:TimelineTable}) = true
Tables.columnaccess(::Type{<:TimelineTable}) = true
function Tables.schema(obj::TimelineTable{Mssngs}) where {Mssngs}
    col_sym = Tables.columnnames(obj)
    col_types = vcat(
        [Date],
        fill(Mssngs ? Union{Float64, Missing} : Float64, length(col_sym)-1)
    )
    Tables.Schema(col_sym, col_types)
end

Tables.columns(x::TimelineTable) = x
function Tables.getcolumn(x::TimelineTable, i::Int)
    if i == 1
        get_dates(x)
    else
        Tables.getcolumn(x, x.cols.lookup[i-1])# subtract 1 since date is the first column produced
    end
end
function Tables.getcolumn(x::TimelineTable, nm::TimelineColumn)
    x[:, nm]
end
function Tables.getcolumn(x::TimelineTable, nm::Symbol)
    if nm == :date
        return get_dates(x)
    end
    if nm ∈ Tables.columnnames(x)
        pos = findfirst(nm .== Tables.columnnames(x))
        return Tables.getcolumn(x, pos)
    end
end

function Tables.columnnames(x::TimelineTable)
    vcat([:date], Symbol.(String.(names(x))))
end

function Base.length(data::TimelineTable{false})
    real_dates = dates_min_max(data_dates(data), norm_dates(data))
    c = bdayscount(calendar(data), dt_min(real_dates), dt_max(real_dates)) + isbday(calendar(data), dt_max(real_dates))
    new_mssngs = get_missing_bdays(calendar(data), data_missing_bdays(data), data_dates(data), real_dates)
    return c - nnz(new_mssngs)
end

function Base.length(data::TimelineTable{true})
    bdayscount(calendar(data), dt_min(data), dt_max(data)) + isbday(calendar(data), dt_max(data))
end

function Base.length(x::DataVector)
    return length(data_missing_bdays(x)) - nnz(data_missing_bdays(x))
end

DataFrames.dropmissing(data::TimelineTable{false}) = data
DataFrames.allowmissing(data::TimelineTable{true}) = data

function DataFrames.dropmissing(data::TimelineTable{true})
    TimelineTable(
        parent(data),
        data_id(data),
        AllowMissing{false},
        data_dates(data),
        DictIndex(names(data)),
        data_missing_bdays(data),
        norm_dates(data)
    )
end

function DataFrames.allowmissing(data::TimelineTable{false})
    TimelineTable(
        parent(data),
        data_id(data),
        AllowMissing{true},
        data_dates(data),
        DictIndex(names(data)),
        data_missing_bdays(data),
        norm_dates(data)
    )
end

function update_dates!(
    data::TimelineTable,
    dates::ClosedInterval{Date}
)
    data.req_dates = dates
    data
end

function update_id!(
    data::TimelineTable{Mssng, T},
    id::T
) where {Mssng, T}
    if id != data.id
        data.id = id
        data.dates = dates_min_max(get_all_dates(data))
        data.missing_bdays = combine_all_missing_bdays(data)
    end
    data
end

function DataFrames.select!(x::TimelineTable, cols::Vector)
    select!(x, TimelineColumn.(cols))
end
function DataFrames.select!(data::TimelineTable, cols::Vector{TimelineColumn})
    updates = sort(cols) != sort(names(data)) # if the columns are changing, then update dates and missing_bdays
    data.cols = DictIndex(cols)
    if updates
        data.dates = dates_min_max(get_all_dates(data))
        data.missing_bdays = combine_all_missing_bdays(data)
    end
    data
end

function get_all_dates(data::MarketData{T}, id::T, cols::Vector{TimelineColumn}) where {T}
    shift_dates.(
        pick_data.(Ref(data), id, Symbol.(cols)),
        shift_count.(cols)
    )
end

function get_all_dates(data::TimelineTable)
    get_all_dates(parent(data), data.id, names(data))
end

function combine_all_missing_bdays(data::TimelineTable)
    out = sparsevec(
        zeros(
            Bool,
            bdayscount(
                calendar(data),
                dt_min(data_dates(data)),
                dt_max(data_dates(data))
            ) + 1
        )
    )
    for col in names(data)
        mssngs = data_missing_bdays(data[Symbol(col)]) # get based on symbol here since if there are no missings
        # then skip the next step, if there are missings then fetch the new vector and adjust
        if nnz(mssngs) > 0
            out = out .| get_missing_bdays(data[col], data_dates(data))
        end
    end
    out
end

raw_values(x::CalendarData) = x.data
data_dates(x::CalendarData) = x.dates
data_missing_bdays(x::CalendarData) = x.missing_bdays

data_dates(x::TimelineTable) = x.dates
data_missing_bdays(x::TimelineTable) = x.missing_bdays
norm_dates(x::TimelineTable) = x.req_dates
parent(data::TimelineTable) = data.parent
data_id(data::TimelineTable) = data.id

calendar(x::CalendarData) = x.calendar
calendar(x::MarketData) = x.calendar
calendar(x::TimelineTable) = calendar(parent(x))

firmdata(data::MarketData{T}, i::T, sym::Symbol) where {T} = data.firmdata[i][sym]
marketdata(data::MarketData, sym::Symbol) = data.marketdata[sym]
pick_data(data::MarketData{T, MNames, FNames}, id::T, sym::Symbol) where {T, MNames, FNames} = sym ∈ FNames ? firmdata(data, id, sym) : marketdata(data, sym)

pick_data(data::MarketData{T}, id::T, col::TimelineColumn) where {T} = pick_data(data, id, Symbol(col))

data_dates(x::MarketData) = x.dates

data_loc(data::MarketData{T, MNames, FNames}, sym::Symbol) where {T, MNames, FNames} = sym ∈ FNames ? :firmdata : :marketdata
function data_loc(data::MarketData{T, MNames, FNames}, syms::Vector{Symbol}) where {T, MNames, FNames}
    all_locs = data_loc.(Ref(data), syms)
    (
        syms[all_locs .== :firmdata],
        syms[all_locs .== :marketdata]
    )
end

function Base.show(io::IO, data::MarketData{T, MNames, FNames}) where {T, MNames, FNames}
    println(io, "MarketData with ID type $T with $(length(getfield(data, :firmdata))) unique firms")
    println(io, data.calendar)
    println(io, "Market Columns: $(join(MNames, ", "))")
    println(io, "Firm Columns: $(join(FNames, ", "))")
end

function Base.show(io::IO, data::TimelineTable{Mssng, T, MNames, FNames}) where {Mssng, T, MNames, FNames}
    println(io, "TimelineTable for $(data_id(data)) and available columns $(join([MNames..., FNames...], ","))")
    println(io, "Maximum dates given current column selection: $(data_dates(data))")
    println(io, "Currently selected dates: $(norm_dates(data))")
    pretty_table(io, data; header = vcat(["Date"], names(data)), tf=tf_simple)
end

dt_min(x::ClosedInterval{Date}) = x.left
dt_min(x::TimelineTable) = dt_min(norm_dates(x))
dt_min(x::CalendarData) = dt_min(data_dates(x))
dt_max(x::ClosedInterval{Date}) = x.right
dt_max(x::TimelineTable) = dt_max(norm_dates(x))
dt_max(x::CalendarData) = dt_max(data_dates(x))

cal_dt_min(x::CalendarData) = cal_dt_min(calendar(x))
cal_dt_max(x::CalendarData) = cal_dt_max(calendar(x))
cal_dt_min(x::TimelineTable) = cal_dt_min(calendar(x))
cal_dt_max(x::TimelineTable) = cal_dt_max(calendar(x))
cal_dt_min(x::MarketData) = cal_dt_min(calendar(x))
cal_dt_max(x::MarketData) = cal_dt_max(calendar(x))


text:
@testset "Views" begin
  println(testname("View tests"))
  print("  [  ] Create a map view ")
  result = view_index(db, "my_ddoc", "my_view",
  """
  function(doc) {
    if(doc && doc.name) {
      emit(doc.name, 1);
    }
  }""")
  @test result["ok"] == true
  println("\r  [OK] Create a map view")

  print("  [  ] Create a map-reduce view ")
  result = view_index(db, "my_ddoc2", "my_view2",
  """
  function(doc) {
    if(doc && doc.data) {
      emit(doc.data, 1);
    }
  }"""; reduce="_stats")
  @test result["ok"] == true
  println("\r  [OK] Create a map-reduce view")

  print("  [  ] Query view ")
  result = view_query(db, "my_ddoc", "my_view"; include_docs=true, descending=true, conflicts=true, key="adam")
  @test length(result["rows"]) == 1
  println("\r  [OK] Query view")

  print("  [  ] Query view (POST) ")
  result = view_query(db, "my_ddoc", "my_view"; keys=["adam", "billy"])
  @test length(result["rows"]) == 2
  println("\r  [OK] Query view (POST)")

  print("  [  ] Query view (POST + skip) ")
  result = view_query(db, "my_ddoc", "my_view"; skip=1, limit=1)
  @test length(result["rows"]) == 1
  println("\r  [OK] Query view (POST + skip)")

  print("  [  ] Query view (startkey, endkey) ")
  result = view_query(db, "my_ddoc", "my_view"; startkey="adam", endkey="billy", inclusive_end=false)
  @test length(result["rows"]) == 1
  println("\r  [OK] Query view (startkey, endkey)")

  print("  [  ] Query view reduce ")
  result = view_query(db, "my_ddoc2", "my_view2"; key="adam", group=true, group_level=2)
  @test haskey(result, "rows")
  println("\r  [OK] Query view reduce")

  print("  [  ] Query view reduce off ")
  result = view_query(db, "my_ddoc2", "my_view2"; key="adam", reduce=false)
  @test haskey(result, "rows")
  println("\r  [OK] Query view reduce off")
end

ext: jl

max_stars_repo_name: xpqz/couchzilla

max_issues_repo_name: xpqz/couchzilla

max_forks_repo_name: xpqz/couchzilla


text:
#==============================================================================#
# MarketplaceCommerceAnalytics.jl
#
# This file is generated from:
# https://github.com/aws/aws-sdk-js/blob/master/apis/marketplacecommerceanalytics-2015-07-01.normal.json
#==============================================================================#

__precompile__()

module MarketplaceCommerceAnalytics

using AWSCore

"""
    using AWSSDK.MarketplaceCommerceAnalytics.generate_data_set
    generate_data_set([::AWSConfig], arguments::Dict)
    generate_data_set([::AWSConfig]; dataSetType=, dataSetPublicationDate=, roleNameArn=, destinationS3BucketName=, snsTopicArn=, <keyword arguments>)

    using AWSCore.Services.marketplacecommerceanalytics
    marketplacecommerceanalytics([::AWSConfig], "GenerateDataSet", arguments::Dict)
    marketplacecommerceanalytics([::AWSConfig], "GenerateDataSet", dataSetType=, dataSetPublicationDate=, roleNameArn=, destinationS3BucketName=, snsTopicArn=, <keyword arguments>)

# GenerateDataSet Operation

Given a data set type and data set publication date, asynchronously publishes the requested data set to the specified S3 bucket and notifies the specified SNS topic once the data is available. Returns a unique request identifier that can be used to correlate requests with notifications from the SNS topic. Data sets will be published in comma-separated values (CSV) format with the file name {data_set_type}_YYYY-MM-DD.csv. If a file with the same name already exists (e.g. if the same data set is requested twice), the original file will be overwritten by the new file. Requires a Role with an attached permissions policy providing Allow permissions for the following actions: s3:PutObject, s3:GetBucketLocation, sns:GetTopicAttributes, sns:Publish, iam:GetRolePolicy.

# Arguments

## `dataSetType = "customer_subscriber_hourly_monthly_subscriptions", "customer_subscriber_annual_subscriptions", "daily_business_usage_by_instance_type", "daily_business_fees", "daily_business_free_trial_conversions", "daily_business_new_instances", "daily_business_new_product_subscribers", "daily_business_canceled_product_subscribers", "monthly_revenue_billing_and_revenue_data", "monthly_revenue_annual_subscriptions", "disbursed_amount_by_product", "disbursed_amount_by_product_with_uncollected_funds", "disbursed_amount_by_instance_hours", "disbursed_amount_by_customer_geo", "disbursed_amount_by_age_of_uncollected_funds", "disbursed_amount_by_age_of_disbursed_funds", "customer_profile_by_industry", "customer_profile_by_revenue", "customer_profile_by_geography", "sales_compensation_billed_revenue" or "us_sales_and_use_tax_records"` -- *Required*
The desired data set type.

*   **customer_subscriber_hourly_monthly_subscriptions**

    From 2014-07-21 to present: Available daily by 5:00 PM Pacific Time.

*   **customer_subscriber_annual_subscriptions**

    From 2014-07-21 to present: Available daily by 5:00 PM Pacific Time.

*   **daily_business_usage_by_instance_type**

    From 2015-01-26 to present: Available daily by 5:00 PM Pacific Time.

*   **daily_business_fees**

    From 2015-01-26 to present: Available daily by 5:00 PM Pacific Time.

*   **daily_business_free_trial_conversions**

    From 2015-01-26 to present: Available daily by 5:00 PM Pacific Time.

*   **daily_business_new_instances**

    From 2015-01-26 to present: Available daily by 5:00 PM Pacific Time.

*   **daily_business_new_product_subscribers**

    From 2015-01-26 to present: Available daily by 5:00 PM Pacific Time.

*   **daily_business_canceled_product_subscribers**

    From 2015-01-26 to present: Available daily by 5:00 PM Pacific Time.

*   **monthly_revenue_billing_and_revenue_data**

    From 2015-02 to 2017-06: Available monthly on the 4th day of the month by 5:00pm Pacific Time. Data includes metered transactions (e.g. hourly) from two months prior.

    From 2017-07 to present: Available monthly on the 15th day of the month by 5:00pm Pacific Time. Data includes metered transactions (e.g. hourly) from one month prior.

*   **monthly_revenue_annual_subscriptions**

    From 2015-02 to 2017-06: Available monthly on the 4th day of the month by 5:00pm Pacific Time. Data includes up-front software charges (e.g. annual) from one month prior.

    From 2017-07 to present: Available monthly on the 15th day of the month by 5:00pm Pacific Time. Data includes up-front software charges (e.g. annual) from one month prior.

*   **disbursed_amount_by_product**

    From 2015-01-26 to present: Available every 30 days by 5:00 PM Pacific Time.

*   **disbursed_amount_by_product_with_uncollected_funds**

    From 2012-04-19 to 2015-01-25: Available every 30 days by 5:00 PM Pacific Time.

    From 2015-01-26 to present: This data set was split into three data sets: disbursed_amount_by_product, disbursed_amount_by_age_of_uncollected_funds, and disbursed_amount_by_age_of_disbursed_funds.

*   **disbursed_amount_by_instance_hours**

    From 2012-09-04 to present: Available every 30 days by 5:00 PM Pacific Time.

*   **disbursed_amount_by_customer_geo**

    From 2012-04-19 to present: Available every 30 days by 5:00 PM Pacific Time.

*   **disbursed_amount_by_age_of_uncollected_funds**

    From 2015-01-26 to present: Available every 30 days by 5:00 PM Pacific Time.

*   **disbursed_amount_by_age_of_disbursed_funds**

    From 2015-01-26 to present: Available every 30 days by 5:00 PM Pacific Time.

*   **customer_profile_by_industry**

    From 2015-10-01 to 2017-06-29: Available daily by 5:00 PM Pacific Time.

    From 2017-06-30 to present: This data set is no longer available.

*   **customer_profile_by_revenue**

    From 2015-10-01 to 2017-06-29: Available daily by 5:00 PM Pacific Time.

    From 2017-06-30 to present: This data set is no longer available.

*   **customer_profile_by_geography**

    From 2015-10-01 to 2017-06-29: Available daily by 5:00 PM Pacific Time.

    From 2017-06-30 to present: This data set is no longer available.

*   **sales_compensation_billed_revenue**

    From 2016-12 to 2017-06: Available monthly on the 4th day of the month by 5:00pm Pacific Time. Data includes metered transactions (e.g. hourly) from two months prior, and up-front software charges (e.g. annual) from one month prior.

    From 2017-06 to present: Available monthly on the 15th day of the month by 5:00pm Pacific Time. Data includes metered transactions (e.g. hourly) from one month prior, and up-front software charges (e.g. annual) from one month prior.

*   **us_sales_and_use_tax_records**

    From 2017-02-15 to present: Available monthly on the 15th day of the month by 5:00 PM Pacific Time.

## `dataSetPublicationDate = timestamp` -- *Required*
The date a data set was published. For daily data sets, provide a date with day-level granularity for the desired day. For weekly data sets, provide a date with day-level granularity within the desired week (the day value will be ignored). For monthly data sets, provide a date with month-level granularity for the desired month (the day value will be ignored).

## `roleNameArn = ::String` -- *Required*
The Amazon Resource Name (ARN) of the Role with an attached permissions policy to interact with the provided AWS services.

## `destinationS3BucketName = ::String` -- *Required*
The name (friendly name, not ARN) of the destination S3 bucket.

## `destinationS3Prefix = ::String`
(Optional) The desired S3 prefix for the published data set, similar to a directory path in standard file systems. For example, if given the bucket name "mybucket" and the prefix "myprefix/mydatasets", the output file "outputfile" would be published to "s3://mybucket/myprefix/mydatasets/outputfile". If the prefix directory structure does not exist, it will be created. If no prefix is provided, the data set will be published to the S3 bucket root.

## `snsTopicArn = ::String` -- *Required*
Amazon Resource Name (ARN) for the SNS Topic that will be notified when the data set has been published or if an error has occurred.

## `customerDefinedValues = ::Dict{String,String}`
(Optional) Key-value pairs which will be returned, unmodified, in the Amazon SNS notification message and the data set metadata file. These key-value pairs can be used to correlated responses with tracking information from other systems.

# Returns

`GenerateDataSetResult`

# Exceptions

`MarketplaceCommerceAnalyticsException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/marketplacecommerceanalytics-2015-07-01/GenerateDataSet)
"""
@inline generate_data_set(aws::AWSConfig=default_aws_config(); args...) = generate_data_set(aws, args)

@inline generate_data_set(aws::AWSConfig, args) = AWSCore.Services.marketplacecommerceanalytics(aws, "GenerateDataSet", args)

@inline generate_data_set(args) = generate_data_set(default_aws_config(), args)

"""
    using AWSSDK.MarketplaceCommerceAnalytics.start_support_data_export
    start_support_data_export([::AWSConfig], arguments::Dict)
    start_support_data_export([::AWSConfig]; dataSetType=, fromDate=, roleNameArn=, destinationS3BucketName=, snsTopicArn=, <keyword arguments>)

    using AWSCore.Services.marketplacecommerceanalytics
    marketplacecommerceanalytics([::AWSConfig], "StartSupportDataExport", arguments::Dict)
    marketplacecommerceanalytics([::AWSConfig], "StartSupportDataExport", dataSetType=, fromDate=, roleNameArn=, destinationS3BucketName=, snsTopicArn=, <keyword arguments>)

# StartSupportDataExport Operation

Given a data set type and a from date, asynchronously publishes the requested customer support data to the specified S3 bucket and notifies the specified SNS topic once the data is available. Returns a unique request identifier that can be used to correlate requests with notifications from the SNS topic. Data sets will be published in comma-separated values (CSV) format with the file name {data_set_type}_YYYY-MM-DD'T'HH-mm-ss'Z'.csv. If a file with the same name already exists (e.g. if the same data set is requested twice), the original file will be overwritten by the new file. Requires a Role with an attached permissions policy providing Allow permissions for the following actions: s3:PutObject, s3:GetBucketLocation, sns:GetTopicAttributes, sns:Publish, iam:GetRolePolicy.

# Arguments

## `dataSetType = "customer_support_contacts_data" or "test_customer_support_contacts_data"` -- *Required*
Specifies the data set type to be written to the output csv file. The data set types customer_support_contacts_data and test_customer_support_contacts_data both result in a csv file containing the following fields: Product Id, Product Code, Customer Guid, Subscription Guid, Subscription Start Date, Organization, AWS Account Id, Given Name, Surname, Telephone Number, Email, Title, Country Code, ZIP Code, Operation Type, and Operation Time.

*   *customer_support_contacts_data* Customer support contact data. The data set will contain all changes (Creates, Updates, and Deletes) to customer support contact data from the date specified in the from_date parameter.
*   *test_customer_support_contacts_data* An example data set containing static test data in the same format as customer_support_contacts_data

## `fromDate = timestamp` -- *Required*
The start date from which to retrieve the data set in UTC. This parameter only affects the customer_support_contacts_data data set type.

## `roleNameArn = ::String` -- *Required*
The Amazon Resource Name (ARN) of the Role with an attached permissions policy to interact with the provided AWS services.

## `destinationS3BucketName = ::String` -- *Required*
The name (friendly name, not ARN) of the destination S3 bucket.

## `destinationS3Prefix = ::String`
(Optional) The desired S3 prefix for the published data set, similar to a directory path in standard file systems. For example, if given the bucket name "mybucket" and the prefix "myprefix/mydatasets", the output file "outputfile" would be published to "s3://mybucket/myprefix/mydatasets/outputfile". If the prefix directory structure does not exist, it will be created. If no prefix is provided, the data set will be published to the S3 bucket root.

## `snsTopicArn = ::String` -- *Required*
Amazon Resource Name (ARN) for the SNS Topic that will be notified when the data set has been published or if an error has occurred.

## `customerDefinedValues = ::Dict{String,String}`
(Optional) Key-value pairs which will be returned, unmodified, in the Amazon SNS notification message and the data set metadata file.

# Returns

`StartSupportDataExportResult`

# Exceptions

`MarketplaceCommerceAnalyticsException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/marketplacecommerceanalytics-2015-07-01/StartSupportDataExport)
"""
@inline start_support_data_export(aws::AWSConfig=default_aws_config(); args...) = start_support_data_export(aws, args)

@inline start_support_data_export(aws::AWSConfig, args) = AWSCore.Services.marketplacecommerceanalytics(aws, "StartSupportDataExport", args)

@inline start_support_data_export(args) = start_support_data_export(default_aws_config(), args)

end # module MarketplaceCommerceAnalytics

#==============================================================================#
# End of file
#==============================================================================#


text:
module RandomizedPreconditioners

using LinearAlgebra
using Random

include("utils.jl")
include("rangefinder.jl")
include("sketch.jl")
include("preconditioner.jl")

export NystromSketch, EigenSketch, RandomizedSVD, adaptive_sketch
export NystromPreconditioner, NystromPreconditionerInverse

end


text:
module DiffEquations

using BasisFunctions, GridArrays, DomainSets, LinearAlgebra
using FrameFun
import FrameFun: AZ_A, AZ_Zt

import BasisFunctions: operator, coefficienttype, src, dest, apply!, grid

using BasisFunctions: evaluation

"""
A DiffEquation describes a differential equation, with or without boundary conditions.
Parameters:
- Fun is the Expansion that will describe the result

When the equation is solved the equations:
-Diff(Fun) = DRhs on the domain
-diff(Fun) = dRhs on boundary
will hold.
"""
struct BoundaryCondition
    S      :: Dictionary
    diff   :: DictionaryOperator
    DG     :: AbstractGrid
end

export DirichletBC
struct DirichletBC
    dRhs   :: Function
    D      :: Domain
    factor :: Number
    function DirichletBC(dRhs=default_boundary_condition :: Function, D=DomainSets.FullSpace{Float64}(), factor=1.0)
        new(dRhs,D,factor)
    end
end

export NeumannBC
struct NeumannBC
    dRhs   :: Function
    D      :: Domain
    function NeumannBC(dRhs=default_boundary_condition :: Function, D=DomainSets.FullSpace{Float64}())
        new(dRhs,D)
    end
end

function operator(BC :: DirichletBC, S::Dictionary, G::AbstractGrid, D::Domain)
    G = subgrid(G,BC.D)
    BC.factor*evaluation(coefficienttype(S), S, G)
end

function operator(BC :: NeumannBC, S::Dictionary2d, G::AbstractGrid, D::Domain2d)
    G = subgrid(G,BC.D)
    GE = evaluation(coefficienttype(S), S, G)
    dx = Float64[]
    dy = Float64[]
    for i=1:length(G)
        push!(dx, normal(D,G[i])[1])
        push!(dy, normal(D,G[i])[2])
    end
    X = DiagonalOperator(dest(GE), dx)*GE*differentiation(S,(1,0))
    Y = DiagonalOperator(dest(GE), dy)*GE*differentiation(S,(0,1))
    X + Y
end

function operator(BC :: NeumannBC, S::Dictionary1d, G::AbstractGrid1d, D::Domain1d)
    G = subgrid(G,BC.D)
    GE = evaluation(coefficienttype(S), S, G)
    dx = Float64[]
    for i=1:length(G)
        push!(dx, normal(D,G[i])[1])
    end
    DiagonalOperator(dest(GE), dx)*GE*differentiation(S,1)
end

export DiffEquation
struct DiffEquation
    S     :: Dictionary
    D     :: Domain
    Diff  :: DictionaryOperator
    DRhs   :: Function
    BCs    :: Tuple
    SMP     ::  AbstractOperator
    oversamplingfactor
    function DiffEquation(S::Dictionary, D::Domain,Diff::DictionaryOperator, DRhs:: Function, BCs::Tuple, oversamplingfactor=2)
        SMP = sampling_operator(S, D, oversamplingfactor=oversamplingfactor)
        new(S, D, Diff, DRhs, BCs, SMP, oversamplingfactor)
    end
end

# DiffEquation(S::Dictionary, D::Domain, Diff::DictionaryOperator, DRhs::Function, BC::BoundaryCondition, oversamplingfactor=2) = DiffEquation(S,D,Diff,DRhs,(BC,), oversamplingfactor)

boundarygrid(D::DiffEquation) = boundary(supergrid(grid(dest(D.SMP))),D.D)

function operator(D::DiffEquation; incboundary=false, options...)
    B = D.S
    ADiff = pinv(D.Diff)
    ops = incboundary ? Array{DictionaryOperator}(undef, length(D.BCs)+2,1) : Array{DictionaryOperator}(undef, length(D.BCs)+1,1)
    G = grid(dest(D.SMP))

    op = evaluation(coefficienttype(D.S), D.S, G)
    if length(size(dest(op))) > 1
        op = BasisFunctions.LinearizationOperator(dest(op))*op
    end

    cnt=1
    ops[cnt] = op*D.Diff*ADiff
    BG = boundarygrid(D)
    if incboundary
        cnt=cnt+1
        ops[cnt] = evaluation(coefficienttype(D.S), D.S, BG)
    end
    for i = 1:length(D.BCs)
        Ac = operator(D.BCs[i],D.S,BG,D.D)*ADiff
        ops[i+cnt]=Ac
    end
    BlockOperator(ops)
end

function rhs(D::DiffEquation; incboundary = false, options...)
    op = operator(D; incboundary=incboundary, options...)
    rhs = Array{Array{coefficienttype(src(op)),1}}(undef,0)
    G = grid(dest(D.SMP))

    op = evaluation(coefficienttype(D.S), D.S, G)
    S1 = sample(G,D.DRhs, coefficienttype(src(op)))
    push!(rhs,reshape(S1,length(S1)))
    BG = boundarygrid(D)

    if incboundary
        push!(rhs,sample(BG,D.DRhs, coefficienttype(src(op))))
    end
    for i = 1:length(D.BCs)
        push!(rhs,sample(subgrid(BG,D.BCs[i].D),D.BCs[i].dRhs, coefficienttype(src(op))))
    end
    BlockVector(rhs...)
end

struct PDEApproximation <: FrameFun.ApproximationProblem
    D   ::  DiffEquation
end
coefficienttype(ap::PDEApproximation) = coefficienttype(ap.D.S)

AZ_A(ap::PDEApproximation; options...) =
        _AZ_A(ap; options...)

function AZ_Zt(ap::PDEApproximation; options...)
    D = ap.D
    G = grid(dest(D.SMP))
    OP = operator(D; options...)
    Zt = 1/length(supergrid(G))*OP'
    Zt
end

import FrameFun: solve
function solve(D::DiffEquation; solverstyle=AZStyle(), options...)
    Adiff = pinv(D.Diff)
    b = rhs(D; options...)
    OP = operator(D; options...)
    A = solver(solverstyle, PDEApproximation(D), OP; options...)
    coef  = A * b
    Expansion(extensionframe(D.D, dest(A)), Adiff*coef)
end

""" Spectral collocation Operators
A spectral collocation operator is a DictionaryOperator from a Fourier Extension Frame to a GridBasis.
It represents the linear differential operator:
L[u](x) = aX[1](X)*sampler*pD[1](D) + ⋯ + aX[r](X)*sampler*pD[r](D)
where X is a grid multiplication operator for x -> x, and D is the derivative operator
"""
struct FECollocationOperator{T} <: DictionaryOperator{T}
  # L = aX[1]*sampler*pD[1] + ⋯ + aX[r]*sampler*pD[r]
  feframe :: ExtensionFrame # This should be Fourier Extension of odd order to work
  pD :: Vector
  aX :: Vector
  sampler :: DictionaryOperator{T}
  function FECollocationOperator{T}(feframe::ExtensionFrame,pD::Vector,aX::Vector,sampler::DictionaryOperator{T}) where {T}
      r = length(pD)
      @assert r == length(aX)
      for k = 1:r
          @assert src(pD[k]) == feframe
          @assert dest(pD[k]) == feframe # Assumed feframe is of odd order
          @assert src(sampler) == feframe
          @assert src(aX[k]) == dest(sampler)
      end
      new(feframe,pD,aX,sampler)
  end
end

grid_multiplication_operator(a::Function, gb::GridBasis, T=BasisFunctions.operatoreltype(GB)) =
	DiagonalOperator{T}(GB, GB, map(a,grid(GB)))
grid_multiplication_opearator(a::Function, grid::AbstractGrid; options...) =
	grid_multiplication_operator(a,GridBasis(grid); options...)

## Constructors
# All input format checks are done in the first constructor
FECollocationOperator(feframe::ExtensionFrame,pD::Vector,aX::Vector,sampler::DictionaryOperator) = FECollocationOperator{eltype(sampler)}(feframe,pD,aX,sampler)
function FECollocationOperator(feframe::ExtensionFrame,pd::Vector{S1},ax::Vector{S2},samplingfactor::Real) where {S1<:Function,S2<:Function}
    pD = map(p->pseudodifferential_operator(feframe,p),pd)
    gridbasis = GridBasis{coefficienttype(feframe)}(oversampledgrid(feframe,round(Int,samplingfactor*length(feframe))))
    aX = map(a->grid_multiplication_operator(a,gridbasis),ax)
    sampler = evaluation(feframe,grid(gridbasis))
    FECollocationOperator(feframe,pD,aX,sampler)
end
FECollocationOperator(feframe::ExtensionFrame,pd::Vector{S1},ax::Vector{S2}) where {S1<:Function,S2<:Function} = FECollocationOperator(feframe,pd,ax,2)
FECollocationOperator(dom::Domain,ambientdom::Domain,n::Int,pd::Vector{S1},ax::Vector{S2},samplingfactor::Real) where {S1<:Function,S2<:Function} = FECollocationOperator(ExtensionFrame(dom,Fourier(n) → ambientdom),pd,ax,samplingfactor)
FECollocationOperator(dom::Domain,ambientdom::Domain,n,pd::Vector{S1},ax::Vector{S2}) where {S1<:Function,S2<:Function} = FECollocationOperator(dom,ambientdom,n,pd,ax,2)

# Constructors for higher dimensional domains:
FECollocationOperator(dom::Domain,ambientdom::ProductDomain,nn::Tuple,pd::Vector{S1},ax::Vector{S2},samplingfactor::Real) where {S1<:Function,S2<:Function} = FECollocationOperator(ExtensionFrame(dom,tensorproduct(map((x,y)->Fourier(x) → y,nn, components(ambientdom)) )),pd,ax,samplingfactor)
FECollocationOperator(dom::Domain,ambientdom::ProductDomain,nn::Tuple,pd::Vector{S1},ax::Vector{S2}) where {S1<:Function,S2<:Function} = FECollocationOperator(dom,ambientdom,nn,pd,ax,2)

src(L::FECollocationOperator) = L.feframe
dest(L::FECollocationOperator) = dest(sampler(L))

function apply!(L::FECollocationOperator,dest,src,coef_dest,coef_src)
    coef_dest[:] = L.aX[1]*(L.sampler*(L.pD[1]*coef_src))
    for k = 2:length(L.pD)
        coef_dest[:] += L.aX[k]*(L.sampler*(L.pD[k]*coef_src))
    end
    coef_dest
end

sampler(L) = L.sampler
grid(L::FECollocationOperator) = grid(sampler(L))

end # module


text:
"""
    ModelNet10(;kwargs...)

Returns ModelNet10 dataset.

### Optional Key Arguments:
    * `root::String=default_root`   - Root directory of dataset
    * `train::Bool=true`            - Specifies the trainset
    * `transform=nothing`           - Transform to be applied to data point.
    * `categories::Vector{String}`  - Specifies the categories to be used in dataset.

### Examples:

```jldoctest
julia> dset = ModelNet10(train=false)
julia> typeof(dset[1].data) == TriMesh
```
"""
ModelNet10(;
    root::String = default_root,
    train::Bool = true,
    download::Bool = false,
    transform = nothing,
    categories::Vector{String} = MN10_classes,
) = _mn_dataset(root, train, download, transform, categories, 10)

const MN10_classes = [
    "bathtub",
    "bed",
    "chair",
    "desk",
    "dresser",
    "monitor",
    "night_stand",
    "sofa",
    "table",
    "toilet",
]


text:
using Unitful, Gadfly, DataFrames

a = -9.81u"m/s^2"
t = (1:0.5:10)u"s"
v = a .* t
v2 = v .+ 2u"m/s"
h = 0.5 * a .* t.^2
df = DataFrame(time=t, velocity=v, position=h,
               unitlesst=ustrip.(t), unitlessv=ustrip.(v), unitlessh=ustrip.(h),
               position2=reverse(h))

# Test various geometries with Unitful
p1 = plot(df, x=:time, y=:velocity, Geom.bar)
p2 = plot(df, x=:time, y=:position, Geom.point,
          intercept=[-80u"m"], slope=[10u"m/s"], Geom.abline)
p3 = plot(df, x=:time, y=:velocity, Geom.line,
          yintercept=[-20u"m/s", -40u"m/s"], Geom.hline)
# Currently explicitly stated that it loess and lm require arrays of plain numbers
#p4 = plot(df, x=:time, y=:position, Geom.point,
#          Geom.smooth(method=:loess, smoothing=0.2))
p4 = plot(df, x=:position, y=:velocity, Geom.path)
p5 = plot(df, x=:time, y=:position, Geom.step,
          xintercept=[3u"s", 8u"s"], Geom.vline)
p6 = plot(df, layer(x=:time, y=:position, Geom.line),
          layer(x=:time, y=:position2, Geom.bar))
gridstack(reshape([p1,p2,p3,p4,p5,p6], (2,3)))


text:
#=
common code for JuMP examples
=#

const sparse_hypatia = MOI.OptimizerWithAttributes(
    Hypatia.Optimizer,
    MOI.Silent() => true,
    "near_factor" => 1000,
    "tol_feas" => 1e-10,
    "tol_rel_opt" => 1e-9,
    "tol_abs_opt" => 1e-8,
    "tol_illposed" => 1e-9,
    "tol_slow" => 2e-2,
    "tol_inconsistent" => 1e-7,
    "tol_tau_infeas" => 1e-7, # TODO tune
    "syssolver" => Solvers.SymIndefSparseSystemSolver{Float64}(),
    "init_use_indirect" => true,
    "preprocess" => false,
    "reduce" => false,
)

const dense_hypatia = MOI.OptimizerWithAttributes(
    Hypatia.Optimizer,
    MOI.Silent() => true,
    "near_factor" => 1000,
    "tol_feas" => 1e-10,
    "tol_rel_opt" => 1e-9,
    "tol_abs_opt" => 1e-8,
    "tol_illposed" => 1e-9,
    "tol_slow" => 2e-2,
    "tol_inconsistent" => 1e-7,
    "tol_tau_infeas" => 1e-7, # TODO tune
    "syssolver" => Solvers.QRCholDenseSystemSolver{Float64}(),
    "init_use_indirect" => false,
    "preprocess" => true,
    "reduce" => true,
)

# TODO experiment with best options and optimize path in Hypatia
const sep_hypatia = MOI.OptimizerWithAttributes(
    Hypatia.Optimizer,
    MOI.Silent() => true,
    "near_factor" => 1000,
    "tol_feas" => 1e-10,
    "tol_rel_opt" => 1e-9,
    "tol_abs_opt" => 1e-8,
    "tol_illposed" => 1e-9,
    "tol_slow" => 2e-2,
    "tol_inconsistent" => 1e-7,
    "tol_tau_infeas" => 1e-7, # TODO tune
    "syssolver" => Solvers.QRCholDenseSystemSolver{Float64}(),
    "init_use_indirect" => false,
    "preprocess" => true,
    "reduce" => true,
    "stepper" => Solvers.PredOrCentStepper{Float64}(),
)

function run_instance(
    ex_type::Type{<:ExampleInstance},
    inst_data::Tuple,
    inst_options::NamedTuple = NamedTuple();
    default_options::NamedTuple = NamedTuple(),
)
    println("setup example instance")
    Random.seed!(1)
    inst = ex_type(inst_data...)

    println("setup JuMP model")
    setup_time = @elapsed model = build(inst)

    println("set solver options")
    JuMP.set_optimizer(model, MOIPajarito.Optimizer)
    options = (; default_options..., conic_solver = dense_hypatia, inst_options...)
    for (option, value) in pairs(options)
        JuMP.set_optimizer_attribute(model, string(option), value)
    end
    JuMP.set_optimizer_attribute(model, "sep_solver", sep_hypatia)

    println("solve")
    flush(stdout)
    flush(stderr)
    optimize_time = @elapsed JuMP.optimize!(model)
    flush(stdout)
    flush(stderr)
    solve_stats = get_solve_stats(model)

    println("check")
    @testset test_extra(inst, model)

    println("done")
    return (; solve_stats..., setup_time, optimize_time)
end

function get_solve_stats(model::JuMP.Model)
    solve_time = JuMP.solve_time(model)
    primal_obj = JuMP.objective_value(model)
    rel_gap = JuMP.relative_gap(model)
    status = string(JuMP.termination_status(model))

    opt = MOI.get(JuMP.backend(model), MOI.RawSolver())
    paj_ext = opt.use_extended_form
    paj_iter = opt.use_iterative_method

    nodes = -1
    try
        nodes = MOI.get(opt.oa_opt, MOI.NodeCount())
    catch e
        @warn "could not get nodecount"
        println(e, "\n")
    end
    iters = opt.num_iters
    num_subp = length(opt.int_sols_cuts)
    num_cuts = opt.num_cuts

    solve_stats = (;
        paj_ext,
        paj_iter,
        status,
        solve_time,
        nodes,
        iters,
        num_subp,
        num_cuts,
        primal_obj,
        rel_gap,
    )
    return solve_stats
end

# fallback: just check optimal status
function test_extra(inst::ExampleInstance, model::JuMP.Model)
    @test JuMP.termination_status(model) == MOI.OPTIMAL
end


text:
function getTableauSLRK(s, o, tsym, q, p, q̃, p̃, ω, d=Nothing)
    TableauSLRK(tsym, o, s, q, p, q̃, p̃, ω, d)
end

function getTableauSLRKLobIII(s, q, p)
    o = 2s-2
    tsym = Symbol("SLRKLobIII")
    getTableauSLRK(s, o, tsym, q, p, q, p, get_lobatto_ω_matrix(s), get_lobatto_d_vector(s))
end

function getTableauSLRKLobIIIAB(s)
    q = getCoefficientsLobIIIA(s)
    p = getCoefficientsLobIIIB(s)
    getTableauSLRKLobIII(s, q, p)
end

function getTableauSLRKLobIIIC(s)
    q = getCoefficientsLobIII(s)
    p = getCoefficientsLobIIIC(s)
    getTableauSLRKLobIII(s, q, p)
end

function getTableauSLRKLobIIID(s)
    l = getCoefficientsLobIIID(s)
    getTableauSLRKLobIII(s, l, l)
end

function getTableauSLRKLobIIIE(s)
    l = getCoefficientsLobIIIE(s)
    getTableauSLRKLobIII(s, l, l)
end


text:
using ResettableStacks
using Base.Test

S = ResettableStack{}(Tuple{Float64,Float64,Float64})

push!(S,(0.5,0.4,0.3))
push!(S,(0.5,0.4,0.4))
reset!(S)
push!(S,(0.5,0.4,0.3))
@test S.data[1] == (0.5,0.4,0.3)

S = ResettableStack{}(Float64)
for i=1:10
  push!(S,i)
end
@test pop!(S) == 10


text:
using Unitful, LaTeXDatax, JuliaFormatter
using Test
# Supply "overwrite" as a commandline argument to overwrite in formatting step
overwrite = get(ARGS, 1, "") == "overwrite"

cd(@__DIR__)

@testset "LaTeXDatax.jl" begin
    io = IOBuffer()

    # Basic data printing
    LaTeXDatax.printdata(io, "String")
    @test String(take!(io)) == "String"

    LaTeXDatax.printdata(io, 1.25)
    @test String(take!(io)) == "\$1.25\$"

    LaTeXDatax.printdata(io, 3.141592; fmt="%.2f", unitformat=:siunitx)
    @test String(take!(io)) == "\\num{3.14}"

    # keyval printing
    LaTeXDatax.printkeyval(io, :a, 612.2u"nm")
    @test String(take!(io)) == "\\pgfkeyssetvalue{/datax/a}{\$612.2\\;\\mathrm{nm}\$}\n"

    # complete macro
    a = 2
    b = 3.2u"m"
    @datax a b c = 3 * a d = 27 unitformat := :siunitx io := io
    @test String(take!(io)) == """
    \\pgfkeyssetvalue{/datax/a}{\\num{2}}
    \\pgfkeyssetvalue{/datax/b}{\\qty{3.2}{\\meter}}
    \\pgfkeyssetvalue{/datax/c}{\\num{6}}
    \\pgfkeyssetvalue{/datax/d}{\\num{27}}
    """

    # Write to file
    rm.(("data.tex", "test.pdf", "test.log"); force=true)
    @datax a b c = 3 * a d = 27 unitformat := :siunitx filename := "data.tex"
    @test isfile("data.tex")
    @test_nowarn run(`pdflatex --file-line-error --interaction=nonstopmode test.tex`)
    rm("test.aux"; force=true)
end
@testset "Formatting" begin
    is_formatted = JuliaFormatter.format(LaTeXDatax; overwrite)
    @test is_formatted
    if ~is_formatted
        if overwrite
            println("The package has now been formatted. Review the changes and commit.")
        else
            println(
                "The package failed formatting check. Try `JuliaFormatter.format(LaTeXDatax)`",
            )
        end
    end
end


text:
export VMPOPolicy, VMPOTrajectory

using LinearAlgebra: ⋅  # dot product
using Zygote: ignore, dropgrad

## trajectory

const VMPOTrajectory = CircularArraySARTTrajectory

## policy definition

"""
    VMPOPolicy(;kwargs)

V-MPO, an on-policy adaptation of Maximum a Posteriori Policy Optimization (MPO)
that performs policy iteration based on a learned state-value function.

# Keyword arguments

- `approximator`: an [`ActorCritic`](@ref) based on [`NeuralNetworkApproximator`](@ref)
- `update_freq`: update policy every n timesteps
- `γ = 0.99f0`: discount factor
- `ϵ_η = 0.02f0`: temperature η hyperparameter
- `ϵ_α = 0.1f0`: Lagrange multiplier α (discrete) hyperparameter
- `ϵ_αμ = 0.005f0`: Lagrange multiplier α_mu (continuous) hyperparameter
- `ϵ_ασ = 0.00005f0`: Lagrange multiplier α_σ (continuous) hyperparameter
- `n_epochs = 8`: update policy for n epochs
- `dist = Categorical`: `Categorical` - discrete, `Normal` - continuous
- `rng = Random.GLOBAL_RNG`

By default, `dist` is set to `Categorical`, which means it will only work
on environments of discrete actions. To work with environments of continuous
actions `dist` should be set to `Normal` and the `actor` in the `approximator`
should be a `GaussianNetwork`. This algorithm only supports one-dimensional
action space for now.

# Ref paper

[V-MPO: On-Policy Maximum a Posteriori Policy Optimization for Discrete and Continuous Control](https://arxiv.org/abs/1909.12238)
"""
mutable struct VMPOPolicy{A<:ActorCritic,D,R} <: AbstractPolicy
    approximator::A
    γ::Float32
    ϵ_η::Float32
    ϵ_α::Float32
    ϵ_αμ::Float32
    ϵ_ασ::Float32
    η::Vector{Float32}  # 1-element array for temperature η
    α::Vector{Float32}  # 3-element array for Lagrange multipliers: [α, αμ, ασ]
    n_epochs::Int
    update_freq::Int
    update_step::Int
    rng::R
end

function VMPOPolicy(;
    approximator,
    update_freq,
    γ=0.99f0,
    ϵ_η=0.02f0,
    ϵ_α=0.1f0,
    ϵ_αμ=0.005f0,
    ϵ_ασ=0.00005f0,
    n_epochs=8,
    dist=Categorical,
    rng=Random.GLOBAL_RNG
)
    VMPOPolicy{typeof(approximator),dist,typeof(rng)}(
        approximator,
        γ,
        ϵ_η,
        ϵ_α,
        ϵ_αμ,
        ϵ_ασ,
        [1.0f0],                # [η]
        [1.0f0, 1.0f0, 1.0f0],  # [α, αμ, ασ]
        n_epochs,
        update_freq,
        0,
        rng,
    )
end

## policy environment interactions

# discrete action

function RLBase.prob(policy::VMPOPolicy{<:ActorCritic,Categorical}, env::AbstractEnv)
    s = send_to_device(device(policy.approximator), state(env))
    p = policy.approximator.actor(s) |> softmax |> send_to_host
    Categorical(p; check_args=false)
end

function (agent::Agent{<:VMPOPolicy{<:ActorCritic,Categorical}})(env::AbstractEnv)
    dist = prob(agent.policy, env)
    rand(agent.policy.rng, dist)
end

function (policy::VMPOPolicy{<:ActorCritic,Categorical})(
    state::AbstractArray,
    action::AbstractArray,
)
    p = policy.approximator.actor(state) |> softmax
    idx = ignore() do
        CartesianIndex.(action, 1:length(action))
    end
    action_log_prob = log.(p)[idx]
    p, action_log_prob
end

# continuous action

function RLBase.prob(
    policy::VMPOPolicy{<:ActorCritic{<:GaussianNetwork},Normal},
    env::AbstractEnv,
)
    s = send_to_device(device(policy.approximator), state(env))
    μ, logσ = policy.approximator.actor(agent.policy.rng, s)
    Normal(μ, exp(logσ))
end

function (agent::Agent{<:VMPOPolicy{<:ActorCritic{<:GaussianNetwork},Normal}})(
    env::AbstractEnv,
)
    s = send_to_device(device(agent.policy.approximator), state(env))
    # the action is an output of GaussianNetwork which is normalised by tanh(),
    # we increase its stability by limiting it to [-1 + eps, 1 - eps]
    a = agent.policy.approximator.actor(agent.policy.rng, s, is_sampling=true)
    m = one(eltype(a)) - eps(eltype(a))
    clamp.(a, -m, m) |> send_to_host |> first
end

function (policy::VMPOPolicy{<:ActorCritic{<:GaussianNetwork},Normal})(
    state::AbstractArray,
    action::AbstractArray,
)
    μ, logσ = policy.approximator.actor(policy.rng, state)
    action = atanh.(action)
    μ, logσ, normlogpdf(μ, exp.(logσ), reshape(action, size(μ)...))
end

## update policy

function RLBase.update!(p::VMPOPolicy, t::VMPOTrajectory, env::AbstractEnv, ::PreActStage)
    # in the first update, only state & action are inserted into trajectory
    length(t) == 0 && return

    p.update_step += 1
    if p.update_step % p.update_freq == 0
        _update!(p, t)
    end
end

function _update!(p::VMPOPolicy, t::VMPOTrajectory)
    AC = p.approximator
    D = device(AC)
    s = send_to_device(D, t[:state][:, 1:end-1])  # drop the last extra state
    a = send_to_device(D, t[:action][1:end-1])  # drop the last extra action
    is_discrete = isa(p, VMPOPolicy{<:ActorCritic,Categorical})
    π_old = is_discrete ? AC.actor(s) |> softmax : nothing
    μ_old, σ_old = is_discrete ? (nothing, nothing) : begin
        μ, logσ = AC.actor(s)
        μ, exp.(logσ)
    end

    # normalise rewards based on each trajectory
    inds = vcat(0, findall(t[:terminal]), length(t[:terminal]))
    rewards =
        map(
            Flux.normalise,
            view.(Ref(t[:reward]), (:).(inds[1:end-1] .+ 1, inds[2:end])),
        ) |>
        Iterators.flatten |>
        collect

    # calculate discounted rewards
    rewards = discount_rewards(rewards, p.γ, terminal=t[:terminal])
    rewards = send_to_device(D, rewards)

    # calculate advantages and sample top half of it
    advantages = rewards .- AC.critic(s)[1, :]
    top_k_idx = sortperm(advantages |> send_to_host, rev=true)[1:length(advantages)÷2]
    top_k_advs = advantages[top_k_idx]

    for epoch in 1:p.n_epochs
        ps = Flux.params(AC, p.η, p.α)
        gs = gradient(ps) do
            η, α, αμ, ασ = p.η[1], p.α...
            π, logπ = is_discrete ? p(s, a) : (nothing, nothing)
            μ, σ, logπ =
                is_discrete ? (nothing, nothing, logπ) : begin
                    μ, logσ, logπ = p(s, a)
                    μ, exp.(logσ), logπ
                end

            # critic loss
            v = AC.critic(s)[1, :]
            Lᵥ = Flux.Losses.mse(v, rewards) / 2

            # actor loss
            ψ = softmax(top_k_advs / dropgrad(η))
            Lπ = -ψ ⋅ logπ[top_k_idx]

            # loss for temperature η
            Lη = η * p.ϵ_η + η * log(mean(exp.(top_k_advs / η)))

            # loss for Lagrange multiplier α
            Lα = if is_discrete
                KL = Flux.kldivergence(π_old, π, agg=identity)
                mean(α * (p.ϵ_α .- dropgrad(KL)) + dropgrad(α) * KL)
            else
                KLμ = 0.5f0 * (μ .- μ_old) .^ 2 ./ (σ .^ 2)
                Lαμ = mean(αμ * (p.ϵ_αμ .- dropgrad(KLμ)) + dropgrad(αμ) * KLμ)
                KLσ = 0.5f0 * ((σ_old ./ σ) .^ 2 .- 1 .+ 2 * log.(σ ./ σ_old))
                Lασ = mean(ασ * (p.ϵ_ασ .- dropgrad(KLσ)) + dropgrad(ασ) * KLσ)
                Lαμ + Lασ
            end

            loss = Lᵥ + Lπ + Lη + Lα
        end

        # skip if gradient contains NaN or Inf
        if any(x -> !isnothing(x) && any(y -> isnan(y) || isinf(y), x), gs)
            break
        end

        Flux.Optimise.update!(AC.optimizer, ps, gs)

        p.η = clamp.(p.η, 1e-8, p.η)
        p.α = clamp.(p.α, 1e-8, p.α)
    end
end


text:
module SFrameMod

export SFrame, column_names, read_csv, pack_columns, stack, unstack, addrownumber, topk, groupby

using Cxx
import Base: show
import SFrames.FlexibleTypeMod: FlexibleType, FlexType, Cflexible_type
import SFrames.SArrayMod: SArray, SAbstractArray, cval
import SFrames: Util
import SFrames.Util: cstrings, cstring, cstring_map, cstring_vector, jl_vector
import SFrames: head, tail, materialize, ismaterialized, sample, dropna, save, load, unpack

function jlvector(cvector)
    x = FlexibleType[]
    iter = icxx"$cvector.begin();"
    e = icxx"$cvector.end();"
    while icxx"$iter != $e;"
        val = icxx"auto x=*$iter; x;"
        push!(x, FlexibleType(val))
        iter = icxx"$iter + 1;"
    end
    x
end

immutable SFrame
    val
end

function SFrame(;kwargs...)
    s = SFrame(icxx"gl_sframe();")
    for (col, values) in kwargs
        s[col] = values
    end
    s
end

function SFrame(d::Associative)
    s = SFrame()
    for (col, values) in d
        s[col] = values
    end
    s
end

function SFrame(dir::AbstractString)
    load(dir)
end

function Base.copy(x::SFrame)
    SFrame(icxx"gl_sframe($(x.val));")
end

function Base.getindex(s::SFrame, f::SArray)
    icxx("$(s.val)[$(f.val)];")
end

Base.getindex(s::SFrame, f::AbstractVector{Bool}) =
    s[SArray(f)]

function Base.getindex(s::SFrame, idx::Integer)
    icxx"$(s.val)[$(Cint(idx-1))];" |> jlvector
end

Base.eltype(::SFrame) = Vector{FlexibleType}

function Base.start(s::SFrame)
    ra = icxx"$(s.val).range_iterator();"
    (icxx"$ra.begin();", icxx"$ra.end();")
end

function Base.next(s::SFrame, state)
    iter, enditer = state
    value = jlvector(icxx"*$iter;")
    (value, (icxx"$iter+1;", enditer))
end

function Base.done(s::SFrame, state)
    iter, enditer = state
    icxx"$iter == $enditer;"
end

function Base.size(s::SFrame)
    cols = icxx"$(s.val).num_columns();" |> Int
    rows = icxx"$(s.val).size();" |> Int
    (rows, cols)
end

Base.ndims(::SFrame) = 2
Base.length(s::SFrame) = icxx"$(s.val).size();" |> Int

ismaterialized(s::SFrame) = icxx"$(s.val).is_materialized();"

hassize(s::SFrame) = icxx"$(s.val).has_size();"

materialize(s::SFrame) = icxx"$(s.val).materialize();"

function save(s::SFrame, path, format="")
    icxx"$(s.val).save($(cstring(path)), $(cstring(format)));"
end

function load(dir)
    s = SFrame()
    icxx"$(s.val).construct_from_sframe_index($(cstring(dir)));"
    s
end

head(s::SFrame, n) = icxx"$(s.val).head($n);" |> SFrame
tail(s::SFrame, n) = icxx"$(s.val).tail($n);" |> SFrame
sample(s::SFrame, fraction) = icxx"$(s.val).sample($fraction);" |> SFrame
sample(s::SFrame, fraction, seed) =
    icxx"$(s.val).sample($fraction, $seed);" |> SFrame

function topk(s::SFrame, column_name, k=10, reverse=false)
    icxx"$(s.val).topk($(cstring(column_name)), $k, $reverse);" |> SFrame
end

function Base.setindex!(s::SFrame, column::SAbstractArray, name)
    icxx"$(s.val).replace_add_column($(cval(column)), $(cstring(name)));"
    column
end

Base.setindex!(s::SFrame, column, name) = s[name] = SArray(column)

function Base.getindex(s::SFrame, name::AbstractString)
    icxx"$(s.val).select_column($(cstring(name)));" |> SArray
end

function Base.getindex(s::SFrame, names::AbstractVector)
    names_c = cstrings(names)
    icxx"$(s.val).select_columns($names_c);" |> SFrame
end

function Base.delete!(s::SFrame, column)
    icxx"$(s.val).remove_column($(cstring(column)));"
    s
end

function append(s1::SFrame, s2::SFrame)
    icxx"$(s1.val).append($(s2.val));" |> SFrame
end

function Base.join(s1::SFrame, s2::SFrame, joinkeys::AbstractVector, how="inner")
    icxx"$(s1.val).join($(s2.val), $(cstrings(joinkeys)), $(cstring(how)));" |>
    SFrame
end

function Base.join(s1::SFrame, s2::SFrame, joinkeys::Associative, how="inner")
    icxx"$(s1.val).join($(s2.val), $(cstring_map(joinkeys)), $(cstring(how)));" |>
    SFrame
end

function dropna(s::SFrame, columns=[], how="any")
    icxx"$(s.val).dropna($(cstring_vector(columns)), $(cstring(how)));" |> SFrame
end

function fillna(s::SFrame, column, value)
    icxx"$(s.val).fillna($(cstring(column)), $value);" |> SFrame
end

function addrownumber(s::SFrame, column_name="id", start=0)
    #  todo get start=1 working
    icxx"$(s.val).add_row_number($(cstring(column_name)), $(Csize_t(start)));" |> SFrame
end

function Base.unique(s::SFrame)
    icxx"$(s.val).unique();" |> SFrame
end

function Base.sort(s::SFrame, column, rev=false)
    icxx"$(s.val).sort($(cstring(column)), $(!rev));" |> SFrame
end

function Base.sort(s::SFrame, columns::AbstractVector, rev=false)
    icxx"$(s.val).sort($(cstrings(columns)), $(!rev));" |> SFrame
end

function stack(s::SFrame, column, new_column, drop_na=false)
    icxx"$(s.val).stack($(cstring(column)), $(cstring(new_column)), $drop_na);" |>
    SFrame
end

function stack(s::SFrame, column, new_columns::AbstractVector, drop_na=false)
    icxx"$(s.val).stack($(cstring(column)), $(cstring_vector(new_columns)),
        $drop_na);" |> SFrame
end

function unstack(s::SFrame, column, new_column="")
    icxx"$(s.val).unstack($(cstring(column)), $(cstring(new_column)));" |> SFrame
end

function unstack(s::SFrame, columns::AbstractVector, new_column="")
    icxx"$(s.val).unstack($(cstring_vector(columns)), $(cstring(new_column)));" |>
    SFrame
end

function Base.show(io::IO, s::SFrame)
    sstream = icxx"stringstream();"
    icxx"$sstream<<$(s.val)<<endl;"
    c_str = icxx"$sstream.str();"
    ptr = icxx"$c_str.c_str();"
    write(io, bytestring(ptr))
end

function pack_columns(s::SFrame, column_prefix, new_column_name, dtype)
    icxx"$(s.val).pack_columns(
        $(cstring(column_prefix)), $(cstring(new_column_name)),
        flex_type_enum::DICT);" |> SFrame
end

function unpack(s::SFrame, unpack_column, column_name_prefix="X")
    icxx"$(s.val).unpack($(cstring(unpack_column)), $(cstring(column_name_prefix)));" |>
    SFrame
end

function unpack(s::SArray, column_name_prefix="X")
    icxx"$(s.val).unpack($(cstring(column_name_prefix)));" |> SFrame
end

function read_csv(csv_file; use_header=true, row_limit=-1, skip_rows=-1, delimiter=",", na_values=UTF8String[], comment_char="#", escape_char="\\", quote_char="", skip_initial_space=true, column_types=Dict{UTF8String,FlexType}())
    s = SFrame()
    opts = Dict{UTF8String, FlexibleType}()
    opts["use_header"] = FlexibleType(use_header ? 1 : 0)
    if row_limit ≥ 0
        opts["row_limit"] = FlexibleType(row_limit)
    end
    if skip_rows ≥ 0
        opts["skip_rows"] = FlexibleType(skip_rows)
    end
    opts["delimiter"] = FlexibleType(delimiter)
    opts["comment_char"] = FlexibleType(comment_char)
    opts["escape_char"] = FlexibleType(escape_char)
    opts["quote_char"] = FlexibleType(quote_char)
    opts["na_values"] = FlexibleType(na_values)
    opts["skip_initial_space"] = FlexibleType(skip_initial_space ? 1 : 0)
    opts_c = icxx"map<string, flexible_type>();"
    for (k, v) in opts
        icxx"$opts_c[$(cstring(k))] = $(v.val);"
    end
    c_column_types = icxx"map<string, flex_type_enum>();"
    for (k, v) in column_types
        icxx"$c_column_types[$(cstring(k))] = flex_type_enum($v);"
    end
    icxx"$(s.val).construct_from_csvs($(cstring(csv_file)), $opts_c, $c_column_types);";
    s
end

function column_names(s::SFrame)
    cnames = icxx"$(s.val).column_names();"
    jlnames = UTF8String[]
    b = icxx"$cnames.begin();"
    e = icxx"$cnames.end();"
    while icxx"$b!=$e;"
        push!(jlnames, icxx"(*$b).c_str();" |> bytestring)
        b = icxx"$b+1;"
    end
    jlnames
end

function groupby(s::SFrame, groupkeys, operators)
    c_keys = icxx"vector<string>();"
    c_ops = icxx"map<string, aggregate::groupby_descriptor_type>();"
    for key in groupkeys
        icxx"$c_keys.push_back($(cstring(key)));"
    end
    for (col, op) in operators
        icxx"$c_ops[$(cstring(col))] = $op;"
    end
    icxx"$(s.val).groupby($c_keys, $c_ops);" |> SFrame
end

groupby(s::SFrame, groupkey::AbstractString, operators::Associative) =
    groupby(s, [groupkey], operators)

groupby(s::SFrame, groupkeys, operator::Cxx.CppValue) =
    groupby(s, groupkeys, Dict("value"=>operator))

# todo
function generic_apply(f_ptr)
    f = icxx"(vector<flexible_type>*)$f_ptr);"
    f_jl = jl_vector(f, FlexibleType)
    r = (global_f::Function)(f_jl)
    return r.val::Cflexible_type
end

generic_apply_ptr = cfunction(generic_apply, Cflexible_type, (Ptr{Void},))

# Doesn't work yet
function Base.apply(s::SFrame, f)
    global global_f = f
    icxx"return $(s.val).apply(
        (flexible_type (*)(const vector<flexible_type>&))$generic_apply_ptr, flex_type_enum::INTEGER);" |> SArray
end

end


text:
# Base 26 number where alphabets are numbers.

"""
```
    AlphaNumNumeral
```
Numbers represented as alphabets as a base26 number where A, B, C represent
digits.
A = 0, B = 1, ..., Z = 25 etc.
"""
struct AlphaNumNumeral <: Integer
    val::Int
    str::String
end

"""
```
    AlphaNumNumeral(str::String)
    AlphaNumNumeral(n::Int)
```
Constructors for `AlphaNumNumeral`.
"""
AlphaNumNumeral(str::String) = parse(AlphaNumNumeral, str)
AlphaNumNumeral(n::Int) = convert(AlphaNumNumeral, n)

Base.hash(num::AlphaNumNumeral) = xor(hash(num.str), hash(num.val))

"""
```
    @ann_str(str)
```
String decorator for `AlphaNumNumeral` definitions.

#Example

```
julia> ann"BB"
LabelNumerals.AlphaNumNumeral(27, "BB")
```
"""
macro ann_str(str)
    AlphaNumNumeral(str)
end

Base.convert(::Type{T}, num::AlphaNumNumeral) where T <: Integer =
    convert(T, num.val)

function Base.parse(::Type{AlphaNumNumeral}, str::String)
    s = uppercase(str)
    val = 0
    for a in s
        val *= 26
        val += 'A' <= a <= 'Z' ? (a - 'A') :
            throw(DomainError(-3, "Invalid characters"))
    end
    return AlphaNumNumeral(val, str)
end

function Base.convert(::Type{AlphaNumNumeral}, val::Int)
    carr = Vector{Char}()
    tval = val
    while (tval > 0)
        r = rem(tval, 26)
        c = Char('A' + r)
        pushfirst!(carr, c)
        tval = div(tval, 26)
    end
    str = String(carr)
    return AlphaNumNumeral(val, str)
end


text:
module NamedTypeParameters

export @parameterize,
       expand_type_parameters

expand_type_parameters(x::Union{DataType,UnionAll}) = Base.unwrap_unionall(x)

function parse_overrides(typeparameters)
  list_name, list_typeexpr = [], []
  for tp in typeparameters
    name, typeexpr = if tp isa Symbol
      # {..., A, ...} -> {..., <:default_supertype_of_A, ...}
      tp, nothing
    elseif tp.head === :<:
      # {..., A <: SuperType, ...} -> {..., <:SuperType, ...}
      tp.args[1], Expr(:<:, tp.args[2])
    elseif tp.head === :(=)
      # {..., A = Float64, ...} -> {..., Float64, ...}
      tp.args[1], tp.args[2]
    else
      error("Unrecognized type pattern $(tp)")
    end
    push!(list_name, name)
    push!(list_typeexpr, typeexpr)
  end

  list_name, list_typeexpr
end

function parameterize(type, override_parameters=[])

  # !!! This block relies on Base internals
  # - unwrap_unionall (used in expand_type_parameters)
  # - TypeName fields :name, :parameters
  # - TypeVar fields :name, :ub
  expanded_type = expand_type_parameters(type)
  typename, typeparameters = expanded_type.name, expanded_type.parameters
  if length(typeparameters) < length(override_parameters)
    error("Too many parameters for type '$expanded_type'")
  end
  # build parameter list from defaults
  parameter_list = Union{Expr,Symbol}[ Expr(:<:, Symbol(p.ub)) for p in typeparameters ]
  parameter_names = [ p.name for p in typeparameters ]

  override_names, override_typeexprs = parse_overrides(override_parameters)
  if length(unique(override_names)) != length(override_names)
    error("Duplicated parameter names")
  end

  # override
  for (name, typeexpr) in zip(override_names, override_typeexprs)
    # parameter names must be unique, hence, findfirst is enough
    index = findfirst(n -> n == name, parameter_names)
    if isnothing(index)
      error("Unknown parameter name '$name'")
    end
    isnothing(typeexpr) && continue # typename without a supertype falls back to default
    parameter_list[index] = typeexpr
  end

  Expr(:curly, typename.name, parameter_list...)
end

"""
Macro which allows to access type parameters of structs by their name,
ignore certain type parameters in method signatures,
and stop worrying about the order of type parameters.

# Examples

```
julia> struct MyType{A<:Real, B, C, D<:Array, E, F, G} end

julia> @parameterize MyType{C=String, B<:Array, G<:Dict}
MyType{<:Real, <:Array, String, <:Array, <:Any, <:Any, <:Dict}

julia> function foo(m::@parameterize(MyType{G=T, A=T, C=String})) where T
          # ...
       end
```
"""
macro parameterize(signature)
  if signature isa Symbol
    return esc(signature)
  end
  if signature.head !== :curly
    error("Failed to recognize a type in '$signature'")
  end
  type, override_parameters = signature.args[1], signature.args[2:end]
  type = @eval __module__ $type

  esc(parameterize(type, override_parameters))
end

end # module


text:
module HardTestProblems
import BSON
import Statistics: mean

# Multiobjective problems
include("Multiobjective/RW_MOP_2021/RW_MOP_2021.jl")
include("Singleobjective/CEC2020/CEC2020.jl")
include("Bilevel/SMD/SMD.jl")
include("Bilevel/PMM/PMM.jl")

end