text:
abstract AbstractDataFrame <: Associative{String, Any}

type DataFrame <: AbstractDataFrame
    columns::Vector{Any}
    colindex::Index

    function DataFrame(cols::Vector{Any}, colind::Index)
        ncols = length(cols)
        if ncols > 1
            nrows = length(cols[1])
            equallengths = true
            for i in 2:ncols
                equallengths &= length(cols[i]) == nrows
            end
            if !equallengths
                msg = "All columns in a DataFrame must be the same length"
                throw(ArgumentError(msg))
            end
        end
        if length(colind) != ncols
            msg = "Columns and column index must be the same length"
            throw(ArgumentError(msg))
        end
        new(cols, colind)
    end
end

# A DataFrame from keyword arguments
# This also covers the empty DataFrame.
function DataFrame(;kwargs...)
    result = DataFrame({}, Index())
    for (k,v) in kwargs
        result[string(k)] = v
    end
    return result
end

# No-op given a DataFrame
DataFrame(df::DataFrame) = df

# Wrap a scalar in a DataArray, then a DataFrame
function DataFrame(x::Union(Number, String))
    cols = {DataArray([x], falses(1))}
    colind = Index(gennames(1))
    return DataFrame(cols, colind)
end

# Wrap a set of columns in a DataFrame w/ or w/o column names
function DataFrame{T <: String}(cs::Vector{Any},
                                cn::Vector{T} = gennames(length(cs)))
    return DataFrame(cs, Index(cn))
end

# Build a DataFrame from an expression
# TODO: Expand the following to allow unequal lengths that are rep'd
#       to the longest length.
DataFrame(ex::Expr) = based_on(DataFrame(), ex)

# Convert a standard Matrix to a DataFrame w/ pre-specified names
function DataFrame(x::Matrix,
                   cn::Vector = gennames(size(x, 2)))
    n = length(cn)
    cols = Array(Any, n)
    for i in 1:n
        cols[i] = DataArray(x[:, i])
    end
    return DataFrame(cols, Index(cn))
end

function DataFrame{K, V}(d::Associative{K, V})
    # Find the first position with maximum length in the Dict.
    lengths = map(length, values(d))
    max_length = maximum(lengths)
    maxpos = findfirst(lengths .== max_length)
    keymaxlen = keys(d)[maxpos]
    nrows = max_length
    # Start with a blank DataFrame
    df = DataFrame()
    for (k, v) in d
        if length(v) == nrows
            df[k] = v
        elseif rem(nrows, length(v)) == 0    # nrows is a multiple of length(v)
            df[k] = vcat(fill(v, div(nrows, length(v)))...)
        else
            vec = fill(v[1], nrows)
            j = 1
            for i = 1:nrows
                vec[i] = v[j]
                j += 1
                if j > length(v)
                    j = 1
                end
            end
            df[k] = vec
        end
    end
    df
end

# Construct a DataFrame with groupings over the columns
# TODO: Restore grouping
function DataFrame(cs::Vector,
                   cn::Vector,
                   gr::Dict{ByteString, Vector{ByteString}})
    d = DataFrame(cs, cn)
    set_groups(index(d), gr)
    return d
end

# Pandas' Dict of Vectors -> DataFrame constructor w/ explicit column names
function DataFrame(d::Dict)
    cnames = sort(convert(Array{ByteString, 1}, collect(keys(d))))
    p = length(cnames)
    if p == 0
        DataFrame()
    end
    n = length(d[cnames[1]])
    columns = Array(Any, p)
    for j in 1:p
        if length(d[cnames[j]]) != n
            throw(ArgumentError("All columns must have the same length"))
        end
        columns[j] = DataArray(d[cnames[j]])
    end
    return DataFrame(columns, Index(cnames))
end

# Pandas' Dict of Vectors -> DataFrame constructor w/o explicit column names
function DataFrame(d::Dict, cnames::Vector)
    p = length(cnames)
    if p == 0
        DataFrame()
    end
    n = length(d[cnames[1]])
    columns = Array(Any, p)
    for j in 1:p
        if length(d[cnames[j]]) != n
            error("All inputs must have the same length")
        end
        columns[j] = DataArray(d[cnames[j]])
    end
    return DataFrame(columns, Index(cnames))
end

# Initialize empty DataFrame objects of arbitrary size
# t is a Type
function DataFrame(t::Any, nrows::Integer, ncols::Integer)
    columns = Array(Any, ncols)
    for i in 1:ncols
        columns[i] = DataArray(t, nrows)
    end
    cnames = gennames(ncols)
    return DataFrame(columns, Index(cnames))
end

# Initialize empty DataFrame objects of arbitrary size
# Use the default column type
function DataFrame(nrows::Integer, ncols::Integer)
    columns = Array(Any, ncols)
    for i in 1:ncols
        columns[i] = DataArray(DEFAULT_COLUMN_TYPE, nrows)
    end
    cnames = gennames(ncols)
    return DataFrame(columns, Index(cnames))
end

# Initialize an empty DataFrame with specific types and names
function DataFrame(column_types::Vector, cnames::Vector, nrows::Integer)
    p = length(column_types)
    columns = Array(Any, p)
    for j in 1:p
        columns[j] = DataArray(column_types[j], nrows)
        for i in 1:nrows
            # TODO: Find a way to get rid of this line
            # Problem may be in show()
            columns[j][i] = baseval(column_types[j])
            columns[j][i] = NA
        end
    end
    return DataFrame(columns, Index(cnames))
end

# Initialize an empty DataFrame with specific types
function DataFrame(column_types::Vector, nrows::Integer)
    p = length(column_types)
    columns = Array(Any, p)
    cnames = gennames(p)
    for j in 1:p
        columns[j] = DataArray(column_types[j], nrows)
        for i in 1:nrows
            # TODO: Find a way to get rid of this line
            # Problem may be in show()
            columns[j][i] = baseval(column_types[j])
            columns[j][i] = NA
        end
    end
    return DataFrame(columns, Index(cnames))
end

# Initialize from a Vector of Associatives (aka list of dicts)
function DataFrame{D <: Associative}(ds::Vector{D})
    ks = [Set([[k for k in [collect(keys(d)) for d in ds]]...]...)...]
    DataFrame(ds, ks)
end

# Initialize from a Vector of Associatives (aka list of dicts)
function DataFrame{D <: Associative, T <: String}(ds::Vector{D}, ks::Vector{T})
    invoke(DataFrame, (Vector{D}, Vector), ds, ks)
end

function DataFrame{D <: Associative}(ds::Vector{D}, ks::Vector)
    #get column types
    col_types = Any[None for i = 1:length(ks)]
    for d in ds
        for (i,k) in enumerate(ks)
            # TODO: check for user-defined "NA" values, ala pandas
            if haskey(d, k) && !isna(d[k])
                try
                    col_types[i] = promote_type(col_types[i], typeof(d[k]))
                catch
                    col_types[i] = Any
                end
            end
        end
    end
    col_types[col_types .== None] = Any

    # create empty DataFrame, and fill
    df = DataFrame(col_types, ks, length(ds))
    for (i,d) in enumerate(ds)
        for (j,k) in enumerate(ks)
            df[i,j] = get(d, k, NA)
        end
    end

    df
end

# If we have a tuple, convert each value in the tuple to a
# DataVector and then pass the converted columns in, hoping for the best
function DataFrame(vals::Any...)
    p = length(vals)
    columns = Array(Any, p)
    for j in 1:p
        if isa(vals[j], AbstractDataVector)
            columns[j] = vals[j]
        else
            columns[j] = DataArray(vals[j])
        end
    end
    cnames = gennames(p)
    DataFrame(columns, Index(cnames))
end

##############################################################################
##
## Basic properties of a DataFrame
##
##############################################################################

Base.names(df::DataFrame) = names(df.colindex)

names!(df::DataFrame, vals) = names!(df.colindex, vals)

function types(adf::AbstractDataFrame)
    ncols = size(adf, 2)
    res = Array(Type, ncols)
    for j in 1:ncols
        res[j] = eltype(adf[j])
    end
    return res
end

function rename(df::DataFrame, from::Any, to::Any)
    rename(df.colindex, from, to)
end
function rename!(df::DataFrame, from::Any, to::Any)
    rename!(df.colindex, from, to)
end

nrow(df::DataFrame) = ncol(df) > 0 ? length(df.columns[1]) : 0
ncol(df::DataFrame) = length(df.colindex)

Base.size(df::AbstractDataFrame) = (nrow(df), ncol(df))
function Base.size(df::AbstractDataFrame, i::Integer)
    if i == 1
        nrow(df)
    elseif i == 2
        ncol(df)
    else
        throw(ArgumentError("DataFrames have only two dimensions"))
    end
end

Base.length(df::AbstractDataFrame) = ncol(df)
Base.endof(df::AbstractDataFrame) = ncol(df)

Base.ndims(::AbstractDataFrame) = 2

index(df::DataFrame) = df.colindex

##############################################################################
##
## Tools for working with groups of columns
##
##############################################################################

function set_group(df::AbstractDataFrame, newgroup, names)
    set_group(index(df), newgroup, names)
end
function set_groups(df::AbstractDataFrame, gr::Dict{ByteString, Vector{ByteString}})
    set_groups(index(df), gr)
end
function get_groups(df::AbstractDataFrame)
    get_groups(index(df))
end
function rename_group!(df::AbstractDataFrame, a, b)
    rename!(index(df), a, b)
end

function reconcile_groups(olddf::AbstractDataFrame, newdf::AbstractDataFrame)
	# foreach group, restrict range to intersection with newdf colnames
	# add back any groups with non-null range
	old_groups = get_groups(olddf)
	for key in keys(old_groups)
		# this is clunky -- there are better/faster ways of doing this intersection operation
		match_vals = ByteString[]
		for val in old_groups[key]
			if (val in names(newdf))
				push!(match_vals, val)
			end
		end
		if !isempty(match_vals)
			set_group(newdf, key, match_vals)
		end
	end
	newdf
end

# TODO: Restore calls to reconcile_groups()

##############################################################################
##
## getindex() definitions
##
##############################################################################

# Cases:
#
# df[SingleColumnIndex] => AbstractDataVector
# df[MultiColumnIndex] => (Sub)?DataFrame
# df[SingleRowIndex, SingleColumnIndex] => Scalar
# df[SingleRowIndex, MultiColumnIndex] => (Sub)?DataFrame
# df[MultiRowIndex, SingleColumnIndex] => (Sub)?AbstractDataVector
# df[MultiRowIndex, MultiColumnIndex] => (Sub)?DataFrame
#
# General Strategy:
#
# Let getindex(df.colindex, col_inds) from Index() handle the resolution
#  of column indices
# Let getindex(df.columns[j], row_inds) from AbstractDataVector() handle
#  the resolution of row indices

typealias ColumnIndex Union(Real, String, Symbol)

# df[SingleColumnIndex] => AbstractDataVector
function Base.getindex(df::DataFrame, col_ind::ColumnIndex)
    selected_column = df.colindex[col_ind]
    return df.columns[selected_column]
end

# df[MultiColumnIndex] => (Sub)?DataFrame
function Base.getindex{T <: ColumnIndex}(df::DataFrame, col_inds::AbstractVector{T})
    selected_columns = df.colindex[col_inds]
    new_columns = df.columns[selected_columns]
    return DataFrame(new_columns, Index(df.colindex.names[selected_columns]))
end

# df[SingleRowIndex, SingleColumnIndex] => Scalar
function Base.getindex(df::DataFrame, row_ind::Real, col_ind::ColumnIndex)
    selected_column = df.colindex[col_ind]
    return df.columns[selected_column][row_ind]
end

# df[SingleRowIndex, MultiColumnIndex] => (Sub)?DataFrame
function Base.getindex{T <: ColumnIndex}(df::DataFrame, row_ind::Real, col_inds::AbstractVector{T})
    selected_columns = df.colindex[col_inds]
    new_columns = {dv[[row_ind]] for dv in df.columns[selected_columns]}
    return DataFrame(new_columns, Index(df.colindex.names[selected_columns]))
end

# df[MultiRowIndex, SingleColumnIndex] => (Sub)?AbstractDataVector
function Base.getindex{T <: Real}(df::DataFrame, row_inds::AbstractVector{T}, col_ind::ColumnIndex)
    selected_column = df.colindex[col_ind]
    return df.columns[selected_column][row_inds]
end

# df[MultiRowIndex, MultiColumnIndex] => (Sub)?DataFrame
function Base.getindex{R <: Real, T <: ColumnIndex}(df::DataFrame, row_inds::AbstractVector{R}, col_inds::AbstractVector{T})
    selected_columns = df.colindex[col_inds]
    new_columns = {dv[row_inds] for dv in df.columns[selected_columns]}
    return DataFrame(new_columns, Index(df.colindex.names[selected_columns]))
end

# Special cases involving expressions
Base.getindex(df::DataFrame, ex::Expr) = getindex(df, with(df, ex))
Base.getindex(df::DataFrame, ex::Expr, c::ColumnIndex) = getindex(df, with(df, ex), c)
Base.getindex{T <: ColumnIndex}(df::DataFrame, ex::Expr, c::AbstractVector{T}) = getindex(df, with(df, ex), c)
Base.getindex(df::DataFrame, c::Real, ex::Expr) = getindex(df, c, with(df, ex))
Base.getindex{T <: Real}(df::DataFrame, c::AbstractVector{T}, ex::Expr) = getindex(df, c, with(df, ex))
Base.getindex(df::DataFrame, ex1::Expr, ex2::Expr) = getindex(df, with(df, ex1), with(df, ex2))

##############################################################################
##
## setindex!()
##
##############################################################################

function create_new_column_from_scalar(df::DataFrame, val::NAtype)
    n = max(nrow(df), 1)
    return DataArray(Array(DEFAULT_COLUMN_TYPE, n), trues(n))
end

function create_new_column_from_scalar(df::DataFrame, val::Any)
    n = max(nrow(df), 1)
    col_data = Array(typeof(val), n)
    for i in 1:n
        col_data[i] = val
    end
    return DataArray(col_data, falses(n))
end

isnextcol(df::DataFrame, col_ind::String) = true
isnextcol(df::DataFrame, col_ind::Symbol) = true
function isnextcol(df::DataFrame, col_ind::Real)
    return ncol(df) + 1 == int(col_ind)
end

function nextcolname(df::DataFrame)
    return string("x", ncol(df) + 1)
end

# Will automatically add a new column if needed
# TODO: Automatically enlarge column to required size?
function insert_single_column!(df::DataFrame,
                               dv::AbstractVector,
                               col_ind::ColumnIndex)
    dv_n, df_n = length(dv), nrow(df)
    if df_n != 0
        if dv_n != df_n
            #dv = repeat(dv, df_n)
            error("New columns must have the same length as old columns")
        end
    end
    if haskey(df.colindex, col_ind)
        j = df.colindex[col_ind]
        df.columns[j] = dv
    else
        if typeof(col_ind) <: String || typeof(col_ind) <: Symbol
            push!(df.colindex, col_ind)
            push!(df.columns, dv)
        else
            if isnextcol(df, col_ind)
                push!(df.colindex, nextcolname(df))
                push!(df.columns, dv)
            else
                println("Column does not exist: $col_ind")
                error("Cannot assign to non-existent column")
            end
        end
    end
    return dv
end

# Will automatically enlarge a scalar to a DataVector if needed
function insert_single_entry!(df::DataFrame, v::Any, row_ind::Real, col_ind::ColumnIndex)
    if nrow(df) <= 1
        dv = DataArray([v], falses(1))
        insert_single_column!(df, dv, col_ind)
        return dv
    else
        try
            df.columns[df.colindex[col_ind]][row_ind] = v
            return v
        catch
            df.columns[df.colindex[col_ind]][row_ind] = NA
            return NA
        end
    end
end

upgrade_vector(v::Vector) = DataArray(v, falses(length(v)))
upgrade_vector(v::Ranges) = DataArray([v], falses(length(v)))
upgrade_vector(v::BitVector) = DataArray(convert(Array{Bool}, v), falses(length(v)))
upgrade_vector(adv::AbstractDataArray) = adv
function upgrade_scalar(df::DataFrame, v::Any)
    n = max(nrow(df), 1)
    DataArray(fill(v, n), falses(n))
end

# df[SingleColumnIndex] = AbstractVector
function Base.setindex!(df::DataFrame,
                v::AbstractVector,
                col_ind::ColumnIndex)
    insert_single_column!(df, upgrade_vector(v), col_ind)
end

# df[SingleColumnIndex] = Scalar (EXPANDS TO MAX(NROW(DF), 1))
function Base.setindex!(df::DataFrame,
                v::Any,
                col_ind::ColumnIndex)
    insert_single_column!(df, upgrade_scalar(df, v), col_ind)
end

# df[MultiColumnIndex] = DataFrame
function Base.setindex!(df::DataFrame,
                new_df::DataFrame,
                col_inds::AbstractVector{Bool})
    setindex!(df, new_df, find(col_inds))
end
function Base.assign{T <: ColumnIndex}(df::DataFrame,
                                  new_df::DataFrame,
                                  col_inds::AbstractVector{T})
    for i in 1:length(col_inds)
        insert_single_column!(df, new_df[i], col_inds[i])
    end
    return new_df
end

# df[MultiColumnIndex] = AbstractVector (REPEATED FOR EACH COLUMN)
function Base.setindex!(df::DataFrame,
                v::AbstractVector,
                col_inds::AbstractVector{Bool})
    setindex!(df, v, find(col_inds))
end
function Base.assign{T <: ColumnIndex}(df::DataFrame,
                                  v::AbstractVector,
                                  col_inds::AbstractVector{T})
    dv = upgrade_vector(v)
    for col_ind in col_inds
        insert_single_column!(df, dv, col_ind)
    end
    return dv
end

# df[MultiColumnIndex] = Scalar (REPEATED FOR EACH COLUMN; EXPANDS TO MAX(NROW(DF), 1))
function Base.setindex!(df::DataFrame,
                val::Any,
                col_inds::AbstractVector{Bool})
    setindex!(df, val, find(col_inds))
end
function Base.assign{T <: ColumnIndex}(df::DataFrame,
                                  val::Any,
                                  col_inds::AbstractVector{T})
    dv = upgrade_scalar(df, val)
    for col_ind in col_inds
        insert_single_column!(df, dv, col_ind)
    end
    return dv
end

# df[SingleRowIndex, SingleColumnIndex] = Scalar
function Base.setindex!(df::DataFrame,
                v::Any,
                row_ind::Real,
                col_ind::ColumnIndex)
    insert_single_entry!(df, v, row_ind, col_ind)
end

# df[SingleRowIndex, MultiColumnIndex] = Scalar (EXPANDS TO MAX(NROW(DF), 1))
function Base.setindex!(df::DataFrame,
                v::Any,
                row_ind::Real,
                col_inds::AbstractVector{Bool})
    setindex!(df, v, row_ind, find(col_inds))
end
function Base.assign{T <: ColumnIndex}(df::DataFrame,
                                  v::Any,
                                  row_ind::Real,
                                  col_inds::AbstractVector{T})
    for col_ind in col_inds
        insert_single_entry!(df, v, row_ind, col_ind)
    end
    return v
end

# df[SingleRowIndex, MultiColumnIndex] = 1-Row DataFrame
function Base.setindex!(df::DataFrame,
                new_df::DataFrame,
                row_ind::Real,
                col_inds::AbstractVector{Bool})
    setindex!(df, new_df, row_ind, find(col_inds))
end

function Base.assign{T <: ColumnIndex}(df::DataFrame,
                                  new_df::DataFrame,
                                  row_ind::Real,
                                  col_inds::AbstractVector{T})
    for j in 1:length(col_inds)
        col_ind = col_inds[j]
        if haskey(df.colindex, col_ind)
            df.columns[df.colindex[col_ind]][row_ind] = new_df[j][1]
        else
            error("Cannot assign into a non-existent position")
        end
    end
    return new_df
end

# df[MultiRowIndex, SingleColumnIndex] = AbstractVector
function Base.setindex!(df::DataFrame,
                v::AbstractVector,
                row_inds::AbstractVector{Bool},
                col_ind::ColumnIndex)
    setindex!(df, v, find(row_inds), col_ind)
end
function Base.assign{T <: Real}(df::DataFrame,
                           v::AbstractVector,
                           row_inds::AbstractVector{T},
                           col_ind::ColumnIndex)
    dv = upgrade_vector(v)
    if haskey(df.colindex, col_ind)
        df.columns[df.colindex[col_ind]][row_inds] = dv
    else
        error("Cannot assign into a non-existent position")
    end
    return dv
end

# df[MultiRowIndex, SingleColumnIndex] = Single Value
function Base.setindex!(df::DataFrame,
                v::Any,
                row_inds::AbstractVector{Bool},
                col_ind::ColumnIndex)
    setindex!(df, v, find(row_inds), col_ind)
end
function Base.assign{T <: Real}(df::DataFrame,
                           v::Any,
                           row_inds::AbstractVector{T},
                           col_ind::ColumnIndex)
    if haskey(df.colindex, col_ind)
        try
            df.columns[df.colindex[col_ind]][row_inds] = v
            return v
        catch
            df.columns[df.colindex[col_ind]][row_inds] = NA
            return NA
        end
    else
        error("Cannot assign into a non-existent position")
    end
end

# df[MultiRowIndex, MultiColumnIndex] = DataFrame
function Base.setindex!(df::DataFrame,
                new_df::DataFrame,
                row_inds::AbstractVector{Bool},
                col_inds::AbstractVector{Bool})
    setindex!(df, new_df, find(row_inds), find(col_inds))
end
function Base.assign{T <: ColumnIndex}(df::DataFrame,
                                  new_df::DataFrame,
                                  row_inds::AbstractVector{Bool},
                                  col_inds::AbstractVector{T})
    setindex!(df, new_df, find(row_inds), col_inds)
end
function Base.assign{R <: Real}(df::DataFrame,
                           new_df::DataFrame,
                           row_inds::AbstractVector{R},
                           col_inds::AbstractVector{Bool})
    setindex!(df, new_df, row_inds, find(col_inds))
end
function Base.assign{R <: Real, T <: ColumnIndex}(df::DataFrame,
                                             new_df::DataFrame,
                                             row_inds::AbstractVector{R},
                                             col_inds::AbstractVector{T})
    for j in 1:length(col_inds)
        col_ind = col_inds[j]
        if haskey(df.colindex, col_ind)
            df.columns[df.colindex[col_ind]][row_inds] = new_df[:, j]
        else
            error("Cannot assign into a non-existent position")
        end
    end
    return new_df
end

# df[MultiRowIndex, MultiColumnIndex] = AbstractVector
function Base.setindex!(df::DataFrame,
                v::AbstractVector,
                row_inds::AbstractVector{Bool},
                col_inds::AbstractVector{Bool})
    setindex!(df, v, find(row_inds), find(col_inds))
end
function Base.assign{T <: ColumnIndex}(df::DataFrame,
                                  v::AbstractVector,
                                  row_inds::AbstractVector{Bool},
                                  col_inds::AbstractVector{T})
    setindex!(df, v, find(row_inds), col_inds)
end
function Base.assign{R <: Real}(df::DataFrame,
                           v::AbstractVector,
                           row_inds::AbstractVector{R},
                           col_inds::AbstractVector{Bool})
    setindex!(df, v, row_inds, find(col_inds))
end
function Base.assign{R <: Real, T <: ColumnIndex}(df::DataFrame,
                                             v::AbstractVector,
                                             row_inds::AbstractVector{R},
                                             col_inds::AbstractVector{T})
    dv = upgrade_vector(v)
    for j in 1:length(col_inds)
        col_ind = col_inds[j]
        if haskey(df.colindex, col_ind)
            df.columns[df.colindex[col_ind]][row_inds] = dv
        else
            error("Cannot assign into a non-existent position")
        end
    end
    return dv
end

# df[MultiRowIndex, MultiColumnIndex] = Single Item
function Base.setindex!(df::DataFrame,
                v::Any,
                row_inds::AbstractVector{Bool},
                col_inds::AbstractVector{Bool})
    setindex!(df, v, find(row_inds), find(col_inds))
end
function Base.assign{T <: ColumnIndex}(df::DataFrame,
                                  v::Any,
                                  row_inds::AbstractVector{Bool},
                                  col_inds::AbstractVector{T})
    setindex!(df, v, find(row_inds), col_inds)
end
function Base.assign{R <: Real}(df::DataFrame,
                           v::Any,
                           row_inds::AbstractVector{R},
                           col_inds::AbstractVector{Bool})
    setindex!(df, v, row_inds, find(col_inds))
end
function Base.assign{R <: Real, T <: ColumnIndex}(df::DataFrame,
                                             v::Any,
                                             row_inds::AbstractVector{R},
                                             col_inds::AbstractVector{T})
    for j in 1:length(col_inds)
        col_ind = col_inds[j]
        if haskey(df.colindex, col_ind)
            try
                df.columns[df.colindex[col_ind]][row_inds] = v
                return v
            catch
                df.columns[df.colindex[col_ind]][row_inds] = NA
                return NA
            end
        else
            error("Cannot assign into a non-existent position")
        end
    end
end

# Special deletion assignment
Base.setindex!(df::DataFrame, x::Nothing, icol::Int) = delete!(df, icol)

# Special cases involving expressions
function Base.setindex!(df::DataFrame, val::Any, ex::Expr)
    setindex!(df, val, with(df, ex))
end
function Base.setindex!(df::DataFrame, val::Any, ex::Expr, c::ColumnIndex)
    setindex!(df, val, with(df, ex), c)
end
function Base.assign{T <: ColumnIndex}(df::DataFrame, val::Any, ex::Expr, c::AbstractVector{T})
    setindex!(df, val, with(df, ex), c)
end
function Base.setindex!(df::DataFrame, val::Any, c::Real, ex::Expr)
    setindex!(df, val, c, with(df, ex))
end
function Base.assign{T <: Real}(df::DataFrame, val::Any, c::AbstractVector{T}, ex::Expr)
    setindex!(df, val, c, with(df, ex))
end
function Base.setindex!(df::DataFrame, val::Any, ex1::Expr, ex2::Expr)
    setindex!(df, val, with(df, ex1), with(df, ex2))
end

##############################################################################
##
## Associative methods
##
##############################################################################

Base.haskey(df::AbstractDataFrame, key::Any) = haskey(index(df), key)
Base.get(df::AbstractDataFrame, key::Any, default::Any) = haskey(df, key) ? df[key] : default
Base.keys(df::AbstractDataFrame) = keys(index(df))
Base.values(df::DataFrame) = df.columns
Base.empty!(df::DataFrame) = DataFrame() # TODO: Make this work right

Base.isempty(df::AbstractDataFrame) = ncol(df) == 0

function Base.insert!(df::AbstractDataFrame, index::Int, item::Any, name::Any)
    @assert 0 < index <= ncol(df) + 1
    df = copy(df)
    df[name] = item
    # rearrange:
    df[[1:index-1, end, index:end-1]]
end

function Base.insert!(df::AbstractDataFrame, df2::AbstractDataFrame)
    @assert nrow(df) == nrow(df2) || nrow(df) == 0
    df = copy(df)
    for n in names(df2)
        df[n] = df2[n]
    end
    df
end

##############################################################################
##
## Copying
##
##############################################################################

# copy of a data frame does a shallow copy
function Base.copy(df::DataFrame)
	newdf = DataFrame(copy(df.columns), names(df))
	reconcile_groups(df, newdf)
end
function Base.deepcopy(df::DataFrame)
    newdf = DataFrame([copy(x) for x in df.columns], names(df))
    reconcile_groups(df, newdf)
end
#deepcopy_with_groups(df::DataFrame) = DataFrame([copy(x) for x in df.columns], colnames(df), get_groups(df))

##############################################################################
##
## head() and tail()
##
##############################################################################

DataArrays.head(df::AbstractDataFrame, r::Int) = df[1:min(r,nrow(df)), :]
DataArrays.head(df::AbstractDataFrame) = head(df, 6)
DataArrays.tail(df::AbstractDataFrame, r::Int) = df[max(1,nrow(df)-r+1):nrow(df), :]
DataArrays.tail(df::AbstractDataFrame) = tail(df, 6)

# get the structure of a DF
function Base.dump(io::IO, x::AbstractDataFrame, n::Int, indent)
    println(io, typeof(x), "  $(nrow(x)) observations of $(ncol(x)) variables")
    gr = get_groups(x)
    if length(gr) > 0
        pretty_show(io, gr)
        println(io)
    end
    if n > 0
        for col in names(x)[1:end]
            print(io, indent, "  ", col, ": ")
            dump(io, x[col], n - 1, string(indent, "  "))
        end
    end
end

function Base.dump(io::IO, x::AbstractDataVector, n::Int, indent)
    println(io, typeof(x), "(", length(x), ") ", x[1:min(4, end)])
end

# summarize the columns of a DF
# if the column's base type derives from Number,
# compute min, 1st quantile, median, mean, 3rd quantile, and max
# filtering NAs, which are reported separately
# if boolean, report trues, falses, and NAs
# if anything else, punt.
# Note that R creates a summary object, which has a print method. That's
# a reasonable alternative to this.
describe(dv::AbstractDataVector) = describe(STDOUT, dv)
describe(df::DataFrame) = describe(STDOUT, df)
function describe{T<:Number}(io, dv::AbstractDataVector{T})
    if all(isna(dv))
        println(io, " * All NA * ")
        return
    end
    filtered = float(dropna(dv))
    qs = quantile(filtered, [0, .25, .5, .75, 1])
    statNames = ["Min", "1st Qu.", "Median", "Mean", "3rd Qu.", "Max"]
    statVals = [qs[1:3], mean(filtered), qs[4:5]]
    for i = 1:6
        println(io, string(rpad(statNames[i], 8, " "), " ", string(statVals[i])))
    end
    nas = sum(isna(dv))
    println(io, "NAs      $nas")
    println(io, "NA%      $(round(nas*100/length(dv), 2))%")
    return
end
function describe{T}(io, dv::AbstractDataVector{T})
    ispooled = isa(dv, PooledDataVector) ? "Pooled " : ""
    # if nothing else, just give the length and element type and NA count
    println(io, "Length  $(length(dv))")
    println(io, "Type    $(ispooled)$(string(eltype(dv)))")
    println(io, "NAs     $(sum(isna(dv)))")
    println(io, "NA%     $(round(sum(isna(dv))*100/length(dv), 2))%")
    println(io, "Unique  $(length(unique(dv)))")
    return
end

# TODO: clever layout in rows
# TODO: AbstractDataFrame
function describe(io, df::AbstractDataFrame)
    for c in 1:ncol(df)
        col = df[c]
        println(io, names(df)[c])
        describe(io, col)
        println(io, )
    end
end

##############################################################################
##
## We use SubDataFrame's to maintain a reference to a subset of a DataFrame
## without making copies.
##
##############################################################################

# a SubDataFrame is a lightweight wrapper around a DataFrame used most frequently in
# split/apply sorts of operations.
type SubDataFrame <: AbstractDataFrame
    parent::DataFrame
    rows::Vector{Int} # maps from subdf row indexes to parent row indexes

    function SubDataFrame(parent::DataFrame, rows::Vector{Int})
        if any(rows .< 1)
            error("all SubDataFrame indices must be > 0")
        end
        if length(rows) > 0 && maximum(rows) > nrow(parent)
            error("all SubDataFrame indices must be <= the number of rows of the DataFrame")
        end
        new(parent, rows)
    end
end

Base.sub(D::DataFrame, r, c) = sub(D[[c]], r)    # If columns are given, pass in a subsetted parent D.
                                            # Columns are not copies, so it's not expensive.
Base.sub(D::DataFrame, r::Int) = sub(D, [r])
Base.sub(D::DataFrame, rs::Vector{Int}) = SubDataFrame(D, rs)
Base.sub(D::DataFrame, r) = sub(D, getindex(SimpleIndex(nrow(D)), r)) # this is a wacky fall-through that uses light-weight fake indexes!
Base.sub(D::DataFrame, ex::Expr) = sub(D, with(D, ex))

Base.sub(D::SubDataFrame, r, c) = sub(D[[c]], r)
Base.sub(D::SubDataFrame, r::Int) = sub(D, [r])
Base.sub(D::SubDataFrame, rs::Vector{Int}) = SubDataFrame(D.parent, D.rows[rs])
Base.sub(D::SubDataFrame, r) = sub(D, getindex(SimpleIndex(nrow(D)), r)) # another wacky fall-through
Base.sub(D::SubDataFrame, ex::Expr) = sub(D, with(D, ex))
const subset = sub

Base.filter(ex::Expr, df::AbstractDataFrame) = subset(df, ex)
Base.select(ex::Expr, df::AbstractDataFrame) = subset(df, ex)

Base.getindex(df::SubDataFrame, c) = df.parent[df.rows, c]
Base.getindex(df::SubDataFrame, r, c) = df.parent[df.rows[r], c]

Base.setindex!(df::SubDataFrame, v, c) = (df.parent[df.rows, c] = v)
Base.setindex!(df::SubDataFrame, v, r, c) = (df.parent[df.rows[r], c] = v)

nrow(df::SubDataFrame) = length(df.rows)
ncol(df::SubDataFrame) = ncol(df.parent)
Base.names(df::SubDataFrame) = names(df.parent)

# Associative methods:
index(df::SubDataFrame) = index(df.parent)

# delete!() deletes columns; deleterows!() deletes rows
# delete!(df, 1)
# delete!(df, "old")
function Base.delete!(df::DataFrame, inds::Vector{Int})
    for i in 1:length(inds)
        ind = inds[i] - i + 1
        if 1 <= ind <= ncol(df)
            splice!(df.columns, ind)
            delete!(df.colindex, ind)
        else
            throw(ArgumentError("Can't delete a non-existent DataFrame column"))
        end
    end
    return df
end
Base.delete!(df::DataFrame, c::Int) = delete!(df, [c])
Base.delete!(df::DataFrame, c::Any) = delete!(df, df.colindex[c])
Base.delete!(df::SubDataFrame, c::Any) = SubDataFrame(del(df.parent, c), df.rows)

# deleterows!()
function deleterows!(df::DataFrame, keep_inds::Vector{Int})
    for i in 1:ncol(df)
        df.columns[i] = df.columns[i][keep_inds]
    end
end

function without(df::DataFrame, icols::Vector{Int})
    newcols = _setdiff([1:ncol(df)], icols)
    if length(newcols) == 0
        throw(ArgumentError("Empty DataFrame generated by without()"))
    end
    df[newcols]
end
without(df::DataFrame, i::Int) = without(df, [i])
without(df::DataFrame, c::Any) = without(df, df.colindex[c])
without(df::SubDataFrame, c::Any) = SubDataFrame(without(df.parent, c), df.rows)

#### cbind, rbind, hcat, vcat
# hcat() is just cbind()
# rbind(df, ...) only accepts data frames. Finds union of columns, maintaining order
# of first df. Missing data becomes NAs.
# vcat() is just rbind()

# two-argument form, two dfs, references only
function Base.hcat(df1::DataFrame, df2::DataFrame)
    # If df1 had metadata, we should copy that.
    colindex = Index(make_unique([names(df1), names(df2)]))
    columns = [df1.columns, df2.columns]
    d = DataFrame(columns, colindex)
    set_groups(d, get_groups(df1))
    set_groups(d, get_groups(df2))
    return d
end
Base.hcat{T}(df::DataFrame, x::DataVector{T}) = hcat(df, DataFrame({x}))
Base.hcat{T}(df::DataFrame, x::Vector{T}) = hcat(df, DataFrame({DataArray(x)}))
Base.hcat{T}(df::DataFrame, x::T) = hcat(df, DataFrame({DataArray([x])}))

# three-plus-argument form recurses
Base.hcat(a::DataFrame, b, c...) = hcat(hcat(a, b), c...)
cbind(args...) = hcat(args...)

is_group(df::AbstractDataFrame, name::ByteString) = is_group(index(df), name)

Base.similar(df::DataFrame, dims) =
    DataFrame([similar(x, dims) for x in df.columns], names(df))

Base.similar(df::SubDataFrame, dims) =
    DataFrame([similar(df[x], dims) for x in names(df)], names(df))

Base.zeros{T<:String}(::Type{T},args...) = fill("",args...) # needed for string arrays in the `nas` method above

nas{T}(dv::DataArray{T}, dims) =   # TODO move to datavector.jl?
    DataArray(zeros(T, dims), fill(true, dims))

nas{T,R}(dv::PooledDataVector{T,R}, dims) =
    PooledDataArray(DataArrays.RefArray(fill(one(R), dims)), dv.pool)

nas(df::DataFrame, dims) =
    DataFrame([nas(x, dims) for x in df.columns], names(df))

nas(df::SubDataFrame, dims) =
    DataFrame([nas(df[x], dims) for x in names(df)], names(df))

vecbind_type{T}(::Vector{T}) = Vector{T}
vecbind_type{T<:AbstractVector}(x::T) = Vector{eltype(x)}
vecbind_type{T<:AbstractDataVector}(x::T) = DataVector{eltype(x)}
vecbind_type{T}(::PooledDataVector{T}) = DataVector{T}

vecbind_promote_type{T1,T2}(x::Type{Vector{T1}}, y::Type{Vector{T2}}) = Array{promote_type(eltype(x), eltype(y)),1}
vecbind_promote_type{T1,T2}(x::Type{DataVector{T1}}, y::Type{DataVector{T2}}) = DataArray{promote_type(eltype(x), eltype(y)),1}
vecbind_promote_type{T1,T2}(x::Type{Vector{T1}}, y::Type{DataVector{T2}}) = DataArray{promote_type(eltype(x), eltype(y)),1}
vecbind_promote_type{T1,T2}(x::Type{DataVector{T1}}, y::Type{Vector{T2}}) = DataArray{promote_type(eltype(x), eltype(y)),1}
vecbind_promote_type(a, b, c, ds...) = vecbind_promote_type(a, vecbind_promote_type(b, c, ds...))
vecbind_promote_type(a, b, c) = vecbind_promote_type(a, vecbind_promote_type(b, c))

function vecbind_promote_type(a::AbstractVector)
    res = None
    if isdefined(a, 1)
        if length(a) == 1
            return a[1]
        else
            if isdefined(a, 2)
                res = vecbind_promote_type(a[1], a[2])
            else
                res = a[1]
            end
        end
    end
    for i in 3:length(a)
        if isdefined(a, i)
            res = vecbind_promote_type(res, a[i])
        end
    end
    return res
end

constructor{T}(::Type{Vector{T}}, args...) = Array(T, args...)
constructor{T}(::Type{DataVector{T}}, args...) = DataArray(T, args...)

function vecbind(xs::AbstractVector...)
    V = vecbind_promote_type(map(vecbind_type, {xs...}))
    len = sum(length, xs)
    res = constructor(V, len)
    k = 1
    for i in 1:length(xs)
        for j in 1:length(xs[i])
            res[k] = xs[i][j]
            k += 1
        end
    end
    res
end
function vecbind(xs::PooledDataVector...)
    vecbind(map(x -> convert(DataArray, x), xs)...)
end

Base.vcat(df::AbstractDataFrame) = df
Base.vcat{T<:AbstractDataFrame}(dfs::Vector{T}) = vcat(dfs...)
function Base.vcat(dfs::AbstractDataFrame...)
    Nrow = sum(nrow, dfs)
    # build up column names and types
    colnams = names(dfs[1])
    coltyps = types(dfs[1])
    for i in 2:length(dfs)
        cni = names(dfs[i])
        cti = types(dfs[i])
        for j in 1:length(cni)
            cn = cni[j]
            if length(findin([cn], colnams)) == 0  # new column
                push!(colnams, cn)
                push!(coltyps, cti[j])
            end
        end
    end
    Ncol = length(colnams)
    res = DataFrame()
    for i in 1:Ncol
        coldata = {}
        for df in dfs
            push!(coldata,
                  get(df,
                      colnams[i],
                      DataArray(coltyps[i], size(df, 1))))
        end
        res[colnams[i]] = vcat(coldata...)
    end
    res
end

rbind(args...) = vcat(args...)

# DF row operations -- delete and append
# df[1] = nothing
# df[1:3] = nothing
# df3 = rbind(df1, df2...)
# rbind!(df1, df2...)

# split-apply-combine
# co(ap(myfun,
#    sp(df, ["region", "product"])))
# (|>)(x, f::Function) = f(x)
# split(df, ["region", "product"]) |> (apply(nrow)) |> mean
# apply(f::function) = (x -> map(f, x))
# split(df, ["region", "product"]) |> @@@)) |> mean
# how do we add col names to the name space?
# transform(df, :(cat=dog*2, clean=proc(dirty)))
# summarise(df, :(cat=sum(dog), all=string(strs)))

function with(d::Associative, ex::Expr)
    # Note: keys must by symbols
    replace_symbols(x, d::Dict) = x
    replace_symbols(e::Expr, d::Dict) = Expr(e.head, (isempty(e.args) ? e.args : map(x -> replace_symbols(x, d), e.args))...)
    function replace_symbols{K,V}(s::Symbol, d::Dict{K,V})
        if (K == Any || K == Symbol) && haskey(d, s)
            :(_D[$(Meta.quot(s))])
        elseif (K == Any || K <: String) && haskey(d, string(s))
            :(_D[$(string(s))])
        else
            s
        end
    end
    ex = replace_symbols(ex, d)
    global _ex = ex
    f = @eval (_D) -> $ex
    f(d)
end

function within!(d::Associative, ex::Expr)
    # Note: keys must by symbols
    replace_symbols(x, d::Associative) = x
    function replace_symbols{K,V}(e::Expr, d::Associative{K,V})
        if e.head == :(=) # replace left-hand side of assignments:
            if (K == Symbol || (K == Any && isa(collect(keys(d))[1], Symbol)))
                exref = Meta.quot(e.args[1])
                if !haskey(d, e.args[1]) # Dummy assignment to reserve a slot.
                                      # I'm not sure how expensive this is.
                    d[e.args[1]] = collect(values(d))[1]
                end
            else
                exref = string(e.args[1])
                if !haskey(d, exref) # dummy assignment to reserve a slot
                    d[exref] = collect(values(d))[1]
                end
            end
            Expr(e.head,
                 vcat({:(_D[$exref])}, map(x -> replace_symbols(x, d), e.args[2:end]))...)
        else
            Expr(e.head, (isempty(e.args) ? e.args : map(x -> replace_symbols(x, d), e.args))...)
        end
    end
    function replace_symbols{K,V}(s::Symbol, d::Associative{K,V})
        if (K == Any || K == Symbol) && haskey(d, s)
            :(_D[$(Meta.quot(s))])
        elseif (K == Any || K <: String) && haskey(d, string(s))
            :(_D[$(string(s))])
        else
            s
        end
    end
    ex = replace_symbols(ex, d)
    f = @eval (_D) -> begin
        $ex
        _D
    end
    f(d)
end

mygetindex(d, key) = getindex(d, key)
myref{K<:String,V}(d::Associative{K,V}, key) = getindex(d, string(key))
mygetindex(d::AbstractDataFrame, key::Symbol) = getindex(d, string(key))

myhas(d, key) = haskey(d, key)
myhas{K<:String,V}(d::Associative{K,V}, key) = haskey(d, string(key))
myhas(d::AbstractDataFrame, key::Symbol) = haskey(d, string(key))

bestkey(d, key) = key
bestkey{K<:String,V}(d::Associative{K,V}, key) = string(key)
bestkey(d::AbstractDataFrame, key) = string(key)
bestkey(d::NamedArray, key) = string(key)

replace_syms(x) = x
replace_syms(s::Symbol) = :( myhas(d, $(Meta.quot(s))) ? mygetindex(d, $(Meta.quot(s))) : $(esc(s)) )
replace_syms(e::Expr) = Expr(e.head, (isempty(e.args) ? e.args : map(x -> replace_syms(x), e.args))...)

## quot(value) = Base.splicedexpr(:quote, value)  # Toivo special

function transform_helper(d, args...)
    exa = {:(local d = $(esc(d)))}
    for ex in args
        left = ex.args[1]
        right = replace_syms(ex.args[2])
        push!(exa,
            :(d[bestkey(d, $(Meta.quot(left)))] = $(right)))
    end
    push!(exa, :d)
    Expr(:block, exa...)
end

macro transform(df, args...)
    transform_helper(df, args...)
end

macro DataFrame(args...)
    :(DataFrame( @transform(NamedArray(), $(map(esc, args)...)) ))
end

function based_on(d::Associative, ex::Expr)
    # Note: keys must by symbols
    replace_symbols(x, d::Associative) = x
    function replace_symbols{K,V}(e::Expr, d::Associative{K,V})
        if e.head == :(=) # replace left-hand side of assignments:
            if (K == Symbol || (K == Any && isa(collect(keys(d))[1], Symbol)))
                exref = Meta.quot(e.args[1])
                if !haskey(d, e.args[1]) # Dummy assignment to reserve a slot.
                                      # I'm not sure how expensive this is.
                    d[e.args[1]] = collect(values(d))[1]
                end
            else
                exref = string(e.args[1])
                if !haskey(d, exref) # dummy assignment to reserve a slot
                    d[exref] = collect(values(d))[1]
                end
            end
            Expr(e.head,
                 vcat({:(_ND[$exref])}, map(x -> replace_symbols(x, d), e.args[2:end]))...)
        else
            Expr(e.head, (isempty(e.args) ? e.args : map(x -> replace_symbols(x, d), e.args))...)
        end
    end
    function replace_symbols{K,V}(s::Symbol, d::Associative{K,V})
        if (K == Any || K == Symbol) && haskey(d, s)
            :(_D[$(Meta.quot(s))])
        elseif (K == Any || K <: String) && haskey(d, string(s))
            :(_D[$(string(s))])
        else
            s
        end
    end
    ex = replace_symbols(ex, d)
    f = @eval (_D) -> begin
        _ND = similar(_D)
        $ex
        _ND
    end
    f(d)
end

function within!(df::AbstractDataFrame, ex::Expr)
    # By-column operation within a DataFrame that allows replacing or adding columns.
    # Returns the transformed DataFrame.
    #
    # helper function to replace symbols in ex with a reference to the
    # appropriate column in df
    replace_symbols(x, syms::Dict) = x
    function replace_symbols(e::Expr, syms::Dict)
        if e.head == :(=) # replace left-hand side of assignments:
            if !haskey(syms, string(e.args[1]))
                syms[string(e.args[1])] = length(syms) + 1
            end
            Expr(e.head,
                 vcat({:(_DF[$(string(e.args[1]))])}, map(x -> replace_symbols(x, syms), e.args[2:end]))...)
        else
            Expr(e.head, (isempty(e.args) ? e.args : map(x -> replace_symbols(x, syms), e.args))...)
        end
    end
    function replace_symbols(s::Symbol, syms::Dict)
        if (string(s) in keys(syms))
            :(_DF[$(syms[string(s)])])
        else
            s
        end
    end
    # Make a dict of colnames and column positions
    cn_dict = Dict(names(df), 1:ncol(df))
    ex = replace_symbols(ex, cn_dict)
    f = @eval (_DF) -> begin
        $ex
        _DF
    end
    f(df)
end

within(x, args...) = within!(copy(x), args...)

function based_on_f(df::AbstractDataFrame, ex::Expr)
    # Returns a function for use on an AbstractDataFrame

    # helper function to replace symbols in ex with a reference to the
    # appropriate column in a new df
    replace_symbols(x, syms::Dict) = x
    function replace_symbols(e::Expr, syms::Dict)
        if e.head == :(=) # replace left-hand side of assignments:
            if !haskey(syms, string(e.args[1]))
                syms[string(e.args[1])] = length(syms) + 1
            end
            Expr(e.head,
                 vcat({:(_col_dict[$(string(e.args[1]))])}, map(x -> replace_symbols(x, syms), e.args[2:end]))...)
        else
            Expr(e.head, (isempty(e.args) ? e.args : map(x -> replace_symbols(x, syms), e.args))...)
        end
    end
    function replace_symbols(s::Symbol, syms::Dict)
        if (string(s) in keys(syms))
            :(_DF[$(syms[string(s)])])
        else
            s
        end
    end
    # Make a dict of colnames and column positions
    cn_dict = Dict(names(df), [1:ncol(df)])
    ex = replace_symbols(ex, cn_dict)
    @eval (_DF) -> begin
        _col_dict = NamedArray()
        $ex
        DataFrame(_col_dict)
    end
end
function based_on(df::AbstractDataFrame, ex::Expr)
    # By-column operation within a DataFrame.
    # Returns a new DataFrame.
    f = based_on_f(df, ex)
    f(df)
end

function with(df::AbstractDataFrame, ex::Expr)
    # By-column operation with the columns of a DataFrame.
    # Returns the result of evaluating ex.

    # helper function to replace symbols in ex with a reference to the
    # appropriate column in df
    replace_symbols(x, syms::Dict) = x
    replace_symbols(e::Expr, syms::Dict) = Expr(e.head, (isempty(e.args) ? e.args : map(x -> replace_symbols(x, syms), e.args))...)
    function replace_symbols(s::Symbol, syms::Dict)
        if (string(s) in keys(syms))
            :(_DF[$(syms[string(s)])])
        else
            s
        end
    end
    # Make a dict of colnames and column positions
    cn_dict = Dict(names(df), [1:ncol(df)])
    ex = replace_symbols(ex, cn_dict)
    f = @eval (_DF) -> $ex
    f(df)
end

with(df::AbstractDataFrame, s::Symbol) = df[string(s)]

# add function curries to ease pipelining:
with(e::Expr) = x -> with(x, e)
within(e::Expr) = x -> within(x, e)
within!(e::Expr) = x -> within!(x, e)
based_on(e::Expr) = x -> based_on(x, e)

# allow pipelining straight to an expression using within!:
(|>)(x::AbstractDataFrame, e::Expr) = within!(x, e)

##
## Miscellaneous
##

function complete_cases(df::AbstractDataFrame)
    ## Returns a Vector{Bool} of indexes of complete cases (rows with no NA's).
    res = !isna(df[1])
    for i in 2:ncol(df)
        res &= !isna(df[i])
    end
    res
end

complete_cases!(df::AbstractDataFrame) = deleterows!(df, find(complete_cases(df)))

function DataArrays.array(adf::AbstractDataFrame)
    n, p = size(adf)
    T = reduce(typejoin, types(adf))
    res = Array(T, n, p)
    for j in 1:p
        col = adf[j]
        for i in 1:n
            res[i, j] = col[i]
        end
    end
    return res
end

function DataArrays.DataArray(adf::AbstractDataFrame,
                              T::DataType = reduce(typejoin, types(adf)))
    n, p = size(adf)
    dm = DataArray(T, n, p)
    for j in 1:p
        col = adf[j]
        for i in 1:n
            dm[i, j] = col[i]
        end
    end
    return dm
end

function duplicated(df::AbstractDataFrame)
    # Return a Vector{Bool} indicated whether the row is a duplicate
    # of a prior row.
    res = fill(false, nrow(df))
    di = Dict()
    for i in 1:nrow(df)
        if haskey(di, array(df[i, :])) # Used to convert to Any type
            res[i] = true
        else
            di[array(df[i, :])] = 1 # Used to convert to Any type
        end
    end
    res
end

function drop_duplicates!(df::AbstractDataFrame)
    deleterows!(df, find(!duplicated(df)))
end

# Unique rows of an AbstractDataFrame.
Base.unique(df::AbstractDataFrame) = df[!duplicated(df), :]

function duplicatedkey(df::AbstractDataFrame)
    # Here's another (probably a lot faster) way to do `duplicated`
    # by grouping on all columns. It will fail if columns cannot be
    # made into PooledDataVector's.
    gd = groupby(df, names(df))
    idx = [1:length(gd.idx)][gd.idx][gd.starts]
    res = fill(true, nrow(df))
    res[idx] = false
    res
end

function DataArrays.isna(df::DataFrame)
    results = BitArray(size(df))
    for i in 1:nrow(df)
        for j in 1:ncol(df)
            results[i, j] = isna(df[i, j])
        end
    end
    return results
end

function Base.isnan(df::DataFrame)
    p = ncol(df)
    res_columns = Array(Any, p)
    for j in 1:p
        res_columns[j] = isnan(df[j])
    end
    return DataFrame(res_columns, names(df))
end

function Base.isfinite(df::DataFrame)
    p = ncol(df)
    res_columns = Array(Any, p)
    for j in 1:p
        res_columns[j] = isfinite(df[j])
    end
    return DataFrame(res_columns, names(df))
end

# TODO: Use cor and cov for DataMatrix to do this
function Base.cor(df::DataFrame)
    numeric_cols = find(map(t -> isa(t, Number), types(df)))
    cor(array(df[:, numeric_cols]))
end
function Base.cov(df::DataFrame)
    numeric_cols = find(map(t -> isa(t, Number), types(df)))
    cov(array(df[:, numeric_cols]))
end

function cleannames!(df::DataFrame)
    oldnames = map(strip, names(df))
    newnames = map(n -> replace(n, r"\W", "_"), oldnames)
    names!(df, newnames)
    return
end

function Base.flipud(df::DataFrame)
    return df[reverse(1:nrow(df)), :]
end

function flipud!(df::DataFrame)
    df[1:nrow(df), :] = df[reverse(1:nrow(df)), :]
    return
end

##############################################################################
##
## Sorting
##
##############################################################################

#########################################
## Permutation & Ordering types/functions
#########################################
# Sorting in Julia works through Orderings, where each ordering is a type
# which defines a comparison function lt(o::Ord, a, b).

# UserColOrdering: user ordering of a column; this is just a convenience container
#                  which allows a user to specify column specific orderings
#                  with "order(column, rev=true,...)"

type UserColOrdering{T<:ColumnIndex}
    col::T
    kwargs
end

# This is exported, and lets a user define orderings for a particular column
order{T<:ColumnIndex}(col::T; kwargs...) = UserColOrdering{T}(col, kwargs)

# Allow getting the column even if it is not wrapped in a UserColOrdering
_getcol(o::UserColOrdering) = o.col
_getcol(x) = x

###
# Get an Ordering for a single column
###
function ordering(col_ord::UserColOrdering, lt::Function, by::Function, rev::Bool, order::Ordering)
    for (k,v) in col_ord.kwargs
        if     k == :lt;    lt    = v
        elseif k == :by;    by    = v
        elseif k == :rev;   rev   = v
        elseif k == :order; order = v
        else
            error("Unknown keyword argument: ", string(k))
        end
    end

    ord(lt,by,rev,order)
end

ordering(col::ColumnIndex, lt::Function, by::Function, rev::Bool, order::Ordering) =
             ord(lt,by,rev,order)

# DFPerm: defines a permutation on a particular DataFrame, using
#         a single ordering (O<:Ordering) or a list of column orderings
#         (O<:AbstractVector{Ordering}), one per DataFrame column
#
#         If a user only specifies a few columns, the DataFrame
#         contained in the DFPerm only contains those columns, and
#         the permutation induced by this ordering is used to
#         sort the original (presumably larger) DataFrame

type DFPerm{O<:Union(Ordering, AbstractVector), DF<:AbstractDataFrame} <: Ordering
    ord::O
    df::DF
end

function DFPerm{O<:Ordering}(ords::AbstractVector{O}, df::AbstractDataFrame)
    if length(ords) != ncol(df)
        error("DFPerm: number of column orderings does not equal the number of DataFrame columns")
    end
    DFPerm{AbstractVector{O}, typeof(df)}(ords, df)
end

DFPerm{O<:Ordering}(o::O, df::AbstractDataFrame) = DFPerm{O,typeof(df)}(o,df)

# For sorting, a and b are row indices (first two lt definitions)
# For issorted, the default row iterator returns SubDataFrames instead,
# so two more lt function is defined below
function Base.Sort.lt{V<:AbstractVector}(o::DFPerm{V}, a, b)
    for i = 1:ncol(o.df)
        if lt(o.ord[i], o.df[a,i], o.df[b,i])
            return true
        end
        if lt(o.ord[i], o.df[b,i], o.df[a,i])
            return false
        end
    end
    false
end

function Base.Sort.lt{O<:Ordering}(o::DFPerm{O}, a, b)
    for i = 1:ncol(o.df)
        if lt(o.ord, o.df[a,i], o.df[b,i])
            return true
        end
        if lt(o.ord, o.df[b,i], o.df[a,i])
            return false
        end
    end
    false
end

function Base.Sort.lt{V<:AbstractVector}(o::DFPerm{V}, a::AbstractDataFrame, b::AbstractDataFrame)
    for i = 1:ncol(o.df)
        if lt(o.ord[i], a[1,i], b[1,i])
            return true
        end
        if lt(o.ord[i], b[1,i], a[1,i])
            return false
        end
    end
    false
end

function Base.Sort.lt{O<:Ordering}(o::DFPerm{O}, a::AbstractDataFrame, b::AbstractDataFrame)
    for i = 1:ncol(o.df)
        if lt(o.ord, a[1,i], b[1,i])
            return true
        end
        if lt(o.ord, b[1,i], a[1,i])
            return false
        end
    end
    false
end

###
# Get an Ordering for a DataFrame
###

################
## Case 1: no columns requested, so sort by all columns using requested order
################
## Case 1a: single order
######
ordering(df::AbstractDataFrame, lt::Function, by::Function, rev::Bool, order::Ordering) =
    DFPerm(ord(lt, by, rev, order), df)

######
## Case 1b: lt, by, rev, and order are Arrays
######
function ordering(df::AbstractDataFrame, lt::AbstractVector{Function}, by::AbstractVector{Function}, rev::AbstractVector{Bool}, order::AbstractVector)
    if !(length(lt) == length(by) == length(rev) == length(order) == size(df,2))
        throw(ArgumentError("Orderings must be specified for all DataFrame columns"))
    end
    DFPerm([ord(_lt, _by, _rev, _order) for (_lt, _by, _rev, _order) in zip(lt, by, rev, order)], df)
end

################
## Case 2:  Return a regular permutation when there's only one column
################
## Case 2a: The column is given directly
######
ordering(df::AbstractDataFrame, col::ColumnIndex, lt::Function, by::Function, rev::Bool, order::Ordering) =
    Perm(ord(lt, by, rev, order), df[col])

######
## Case 2b: The column is given as a UserColOrdering
######
ordering(df::AbstractDataFrame, col_ord::UserColOrdering, lt::Function, by::Function, rev::Bool, order::Ordering) =
    Perm(ordering(col_ord, lt, by, rev, order), df[col_ord.col])

################
## Case 3:  General case: cols is an iterable of a combination of ColumnIndexes and UserColOrderings
################
## Case 3a: None of lt, by, rev, or order is an Array
######
function ordering(df::AbstractDataFrame, cols::AbstractVector, lt::Function, by::Function, rev::Bool, order::Ordering)

    if length(cols) == 0
        return ordering(df, lt, by, rev, order)
    end

    if length(cols) == 1
        return ordering(df, cols[1], lt, by, rev, order)
    end

    # Collect per-column ordering info

    ords = Ordering[]
    newcols = Int[]

    for col in cols
        push!(ords, ordering(col, lt, by, rev, order))
        push!(newcols, index(df)[(_getcol(col))])
    end

    # Simplify ordering when all orderings are the same
    if all([ords[i] == ords[1] for i = 2:length(ords)])
        return DFPerm(ords[1], df[newcols])
    end

    return DFPerm(ords, df[newcols])
end

######
# Case 3b: cols, lt, by, rev, and order are all arrays
######
function ordering(df::AbstractDataFrame, cols::AbstractVector,
                  lt::AbstractVector{Function}, by::AbstractVector{Function}, rev::AbstractVector{Bool}, order::AbstractVector)

    if !(length(lt) == length(by) == length(rev) == length(order))
        throw(ArgumentError("All ordering arguments must be 1 or the same length."))
    end

    if length(cols) == 0
        return ordering(df, lt, by, rev, order)
    end

    if length(lt) != length(cols)
        throw(ArgumentError("All ordering arguments must be 1 or the same length as the number of columns requested."))
    end

    if length(cols) == 1
        return ordering(df, cols[1], lt[1], by[1], rev[1], order[1])
    end

    # Collect per-column ordering info

    ords = Ordering[]
    newcols = Int[]

    for i in 1:length(cols)
        push!(ords, ordering(cols[i], lt[i], by[i], rev[i], order[i]))
        push!(newcols, index(df)[(_getcol(cols[i]))])
    end

    # Simplify ordering when all orderings are the same
    if all([ords[i] == ords[1] for i = 2:length(ords)])
        return DFPerm(ords[1], df[newcols])
    end

    return DFPerm(ords, df[newcols])
end

######
## At least one of lt, by, rev, or order is an array or tuple, so expand all to arrays
######
function ordering(df::AbstractDataFrame, cols::AbstractVector, lt, by, rev, order)
    to_array(src::AbstractVector, dims) = src
    to_array(src::Tuple, dims) = [src...]
    to_array(src, dims) = fill(src, dims)

    dims = length(cols) > 0 ? length(cols) : size(df,2)
    ordering(df, cols,
             to_array(lt, dims),
             to_array(by, dims),
             to_array(rev, dims),
             to_array(order, dims))
end

#### Convert cols from tuple to Array, if necessary
ordering(df::AbstractDataFrame, cols::Tuple, args...) = ordering(df, [cols...], args...)

###########################
# Default sorting algorithm
###########################

# TimSort is fast for data with structure, but only if the DataFrame is large enough
# TODO: 8192 is informed but somewhat arbitrary

defalg(df::AbstractDataFrame) = size(df, 1) < 8192 ? Sort.MergeSort : SortingAlgorithms.TimSort

# For DataFrames, we can choose the algorithm based on the column type and requested ordering
function defalg{T<:Real}(df::AbstractDataFrame, ::Type{T}, o::Ordering)
    # If we're sorting a single numerical column in forward or reverse,
    # RadixSort will generally be the fastest stable sort
    if isbits(T) && sizeof(T) <= 8 && (o==Order.Forward || o==Order.Reverse)
        SortingAlgorithms.RadixSort
    else
        defalg(df)
    end
end
defalg(df::AbstractDataFrame,        ::Type,          o::Ordering) = defalg(df)
defalg(df::AbstractDataFrame, col    ::ColumnIndex,   o::Ordering) = defalg(df, eltype(df[col]), o)
defalg(df::AbstractDataFrame, col_ord::UserColOrdering, o::Ordering) = defalg(df, col_ord.col, o)
defalg(df::AbstractDataFrame, cols,                   o::Ordering) = defalg(df)

function defalg(df::AbstractDataFrame, o::Ordering; alg=nothing, cols=[])
    alg != nothing && return alg
    defalg(df, cols, o)
end

########################
## Actual sort functions
########################

issorted(df::AbstractDataFrame; cols={}, lt=isless, by=identity, rev=false, order=Forward) =
    issorted(EachRow(df), ordering(df, cols, lt, by, rev, order))

# sort!, sort, and sortperm functions

for s in {:sort!, :sort, :sortperm}
    @eval begin
        function $s(df::AbstractDataFrame; cols={}, alg=nothing,
                    lt=isless, by=identity, rev=false, order=Forward)
            ord = ordering(df, cols, lt, by, rev, order)
            _alg = defalg(df, ord; alg=alg, cols=cols)
            $s(df, _alg, ord)
        end
    end
end

function Base.sort!(df::AbstractDataFrame, a::Base.Sort.Algorithm, o::Base.Sort.Ordering)
    p = sortperm(df, a, o)
    pp = similar(p)
    for col in df.columns
        copy!(pp,p)
        Base.permute!!(col, pp)
    end
    df
end

sort    (df::AbstractDataFrame, a::Algorithm, o::Ordering) = df[sortperm(df, a, o),:]
sortperm(df::AbstractDataFrame, a::Algorithm, o::Union(Perm,DFPerm)) = sort!([1:nrow(df)], a, o)
sortperm(df::AbstractDataFrame, a::Algorithm, o::Ordering) = sortperm(df, a, DFPerm(o,df))

# Extras to speed up sorting
sortperm{V}(d::AbstractDataFrame, a::Algorithm, o::FastPerm{Sort.ForwardOrdering,V}) = sortperm(o.vec)
sortperm{V}(d::AbstractDataFrame, a::Algorithm, o::FastPerm{Sort.ReverseOrdering,V}) = reverse(sortperm(o.vec))

## Deprecated; remove after julia v0.3 is released (or sooner)

typealias ColIndexVec Union(AbstractVector{Integer}, AbstractVector{ASCIIString}, AbstractVector{UTF8String}, AbstractVector{Symbol})

@deprecate sort!(df::AbstractDataFrame, o::Ordering)  sort!(df,order=o)
@deprecate sort(df::AbstractDataFrame, o::Ordering)  sort(df,order=o)
@deprecate sortperm(df::AbstractDataFrame, o::Ordering)  sortperm(df, order=o)

@deprecate sortby!(df::AbstractDataFrame, by::Function)  sort!(df,by=by)

@deprecate sortby!(df::AbstractDataFrame, col ::ColumnIndex, o::Ordering)  sort!(df,cols=col,order=o)
@deprecate sortby!(df::AbstractDataFrame, col ::ColumnIndex)               sort!(df,cols=col)
@deprecate sortby!(df::AbstractDataFrame, cols::ColIndexVec, o::Ordering)  sort!(df,cols=cols,order=o)
@deprecate sortby!(df::AbstractDataFrame, cols::ColIndexVec)               sort!(df,cols=cols)
@deprecate sortby(df::AbstractDataFrame, col ::ColumnIndex, o::Ordering)  sort(df,cols=col,order=o)
@deprecate sortby(df::AbstractDataFrame, col ::ColumnIndex)               sort(df,cols=col)
@deprecate sortby(df::AbstractDataFrame, cols::ColIndexVec, o::Ordering)  sort(df,cols=cols,order=o)
@deprecate sortby(df::AbstractDataFrame, cols::ColIndexVec)               sort(df,cols=cols)

#@deprecate sort!   {O<:Ordering}(df::AbstractDataFrame, ::Type{O})    sort!   (df, order=O())
#@deprecate sort    {O<:Ordering}(df::AbstractDataFrame, ::Type{O})    sort    (df, order=O())
#@deprecate sortperm{O<:Ordering}(df::AbstractDataFrame, ::Type{O})    sortperm(df, order=O())

#@deprecate sortby!{O<:Ordering}(df::AbstractDataFrame, col::ColumnIndex, ::Type{O})                   sort!(df,col=col,order=O())
#@deprecate sortby!{O<:Ordering}(df::AbstractDataFrame, cols::ColIndexVec, ::Type{O})                  sort!(df,cols=cols,order=O())
#@deprecate sortby!{O<:Ordering}(df::AbstractDataFrame, cols::ColIndexVec, o::AbstractArray{O})        sort!(df,DFPerm(o, df[cols]))
#@deprecate sortby {O<:Ordering}(df::AbstractDataFrame, col::ColumnIndex, ::Type{O})                   sort (df,col=col,order=O())
#@deprecate sortby {O<:Ordering}(df::AbstractDataFrame, cols::ColIndexVec, ::Type{O})                  sort (df,cols=cols,order=O())
#@deprecate sortby {O<:Ordering}(df::AbstractDataFrame, cols::ColIndexVec, o::AbstractArray{O})        sort (df,DFPerm(o, df[cols]))

#@deprecate sortby!             (df::AbstractDataFrame, cols::ColIndexVec, o::AbstractArray{DataType}) sort!(df,DFPerm(Ordering[O() for O in o], df[cols]))
#           sortby!             (df::AbstractDataFrame, cols::ColIndexVec, o::AbstractArray)           sortby!(df,cols,DataType[ot for ot in o])
#           sortby!             (df::AbstractDataFrame, col_ord::AbstractArray{Tuple}) = ((cols,o) = zip(col_ord...); sortby!(df, [cols...], [o...]))
#@deprecate sortby              (df::AbstractDataFrame, cols::ColIndexVec, o::AbstractArray{DataType}) sort (df,DFPerm(Ordering[O() for O in o], df[cols]))
#           sortby              (df::AbstractDataFrame, cols::ColIndexVec, o::AbstractArray)           sortby!(df,cols,DataType[ot for ot in o])
#           sortby              (df::AbstractDataFrame, col_ord::AbstractArray{Tuple}) = ((cols,o) = zip(col_ord...); sortby!(df, [cols...], [o...]))

# reorder! for factors by specifying a DataFrame
function DataArrays.reorder(fun::Function, x::PooledDataArray, df::AbstractDataFrame)
    dfc = copy(df)
    dfc["__key__"] = x
    gd = by(dfc, "__key__", df -> colwise(fun, without(df, "__key__")))
    idx = sortperm(gd[[2:ncol(gd)]])
    return PooledDataArray(x, dropna(gd[idx,1]))
end
DataArrays.reorder(x::PooledDataArray, df::AbstractDataFrame) = reorder(:mean, x, df)

DataArrays.reorder(fun::Function, x::PooledDataArray, y::AbstractVector...) =
    reorder(fun, x, DataFrame({y...}))

##############################################################################
##
## Iteration: EachRow, EachCol
##
##############################################################################

# Iteration by rows
type DFRowIterator
    df::AbstractDataFrame
end
EachRow(df::AbstractDataFrame) = DFRowIterator(df)
Base.start(itr::DFRowIterator) = 1
Base.done(itr::DFRowIterator, i::Int) = i > nrow(itr.df)
Base.next(itr::DFRowIterator, i::Int) = (itr.df[i, :], i + 1)
Base.size(itr::DFRowIterator) = (nrow(itr.df), )
Base.length(itr::DFRowIterator) = nrow(itr.df)
Base.getindex(itr::DFRowIterator, i::Any) = itr.df[i, :]
Base.map(f::Function, dfri::DFRowIterator) = [f(row) for row in dfri]

# Iteration by columns
type DFColumnIterator
    df::AbstractDataFrame
end
EachCol(df::AbstractDataFrame) = DFColumnIterator(df)
Base.start(itr::DFColumnIterator) = 1
Base.done(itr::DFColumnIterator, j::Int) = j > ncol(itr.df)
Base.next(itr::DFColumnIterator, j::Int) = (itr.df[:, j], j + 1)
Base.size(itr::DFColumnIterator) = (ncol(itr.df), )
Base.length(itr::DFColumnIterator) = ncol(itr.df)
Base.getindex(itr::DFColumnIterator, j::Any) = itr.df[:, j]
function Base.map(f::Function, dfci::DFColumnIterator)
    # note: `f` must return a consistent length
    res = DataFrame()
    for i = 1:ncol(dfci.df)
        res[i] = f(dfci[i])
    end
    names!(res, names(dfci.df))
    res
end

# Iteration matches that of Associative types (experimental)
Base.start(df::AbstractDataFrame) = 1
Base.done(df::AbstractDataFrame, i) = i > ncol(df)
Base.next(df::AbstractDataFrame, i) = ((names(df)[i], df[i]), i + 1)

##############################################################################
##
## Hashing
##
## Make sure this agrees with is_equals()
##
##############################################################################

function Base.hash(a::AbstractDataFrame)
    h = hash(size(a)) + 1
    for i in 1:ncol(a)
        h = bitmix(h, int(hash(a[i])))
    end
    return uint(h)
end

##############################################################################
##
## Dict conversion
##
## Try to insure this invertible.
## Allow option to flatten a single row.
##
##############################################################################

function dict(adf::AbstractDataFrame, flatten::Bool)
    # TODO: Make flatten an option
    # TODO: Provide a de-data option that makes Vector's, not
    #       DataVector's
    res = Dict{UTF8String, Any}()
    if flatten && nrow(adf) == 1
        for colname in names(adf)
            res[colname] = adf[colname][1]
        end
    else
        for colname in names(adf)
            res[colname] = adf[colname]
        end
    end
    return res
end
dict(adf::AbstractDataFrame) = dict(adf, false)

# TODO: Add proper tests
# adf = DataFrame(quote A = 1:4; B = ["A", "B", "C", "D"] end)
# DataFrames.dict(adf)
# ["B"=>["A", "B", "C", "D"],"A"=>[1, 2, 3, 4]]
# DataFrames.dict(adf[1, :])
# ["B"=>["A"],"A"=>[1]]
# DataFrames.dict(adf[1, :], true)
# ["B"=>"A","A"=>1]

# Pooling

pool(a::AbstractVector) = compact(PooledDataArray(a))

function pool!(df::AbstractDataFrame, cname::Union(Integer, String))
    df[cname] = pool(df[cname])
    return
end

function pool!{T <: Union(Integer, String)}(df::AbstractDataFrame, cnames::Vector{T})
    for cname in cnames
        df[cname] = pool(df[cname])
    end
    return
end

function pool!(df)
    for i in 1:size(df, 2)
        if eltype(df[i]) <: String
            df[i] = pool(df[i])
        end
    end
    return
end


text:
const DCLR_EX_DAYS = 9
const DIST_DAYS = 12
const DIST_RET = :net
const DIST_RET_BOUND = 0.2

const DIST_MIN_ALLOWED = 1#DIST_DAYS*2
const DIST_DAYS_TO_STALE_PRICE = 22

function getidentifierscrspevent(dist::DataFrame; Fdate::Symbol = :exdate)
  local ccm::DataFrame = loadccm()
  ccm = preprocessccm!(ccm)

  dist.eid = collect(1:size(dist,1)) #assign an id to each event

  dist = join(dist, ccm, on=:permno=>:lpermno)
  filter!(r::DataFrameRow -> r[Fdate] ∈ r.linkeffdate:Day(1):r.linkenddate, dist)
  @assert allunique(dist.eid) #no dups

  #rename!(dist, :lpermno=>:permno) #sanitize permno

  return dist
end

function readcrspevent(;fname::String = CRSP_DISTRIBUTIONS_NAME,
  path::String=EVENT_PATH,
  incsvstream::Function = IN_CSV_STREAM,
  crspdateformat::DateFormat = CRSP_DATE_FORMAT,
  Fdate::Symbol=:exdate,
  distdays::Int = DIST_DAYS)

  dist::DataFrame = incsvstream("$path\\$fname.csv.lz4") |> CSV.read |> DataFrame

  names!(dist, (s::Symbol->Symbol(lowercase(string(s)))).(names(dist)))

  rename!(dist, [:dclrdt=>:dclrdate, :exdt=>:exdate, :paydt=>:paydate])

  for f ∈ [:dclrdate, :exdate, :paydate]
    dist[!,f] = (i::MInt->ismissing(i) ? missing : Date("$i", CRSP_DATE_FORMAT)).(dist[!,f])
  end

  essentialfields::Vector{Symbol} = [:permno, :dclrdate, :exdate, :distcd, :divamt]
  dist = dist[completecases(dist[!, essentialfields]), :]

  #time between exdate and dclrdate is important
  dist.dclrexdays = (d::Day->d.value).(dist.exdate .- dist.dclrdate)

  #ensures integrity
  dist = dist[dist.dclrexdays .≥ 0, :]

  #dist = getidentifierscrspevent(dist, Fdate=Fdate) NOTE: We will need to call this method- just not here

  return dist
end

function processcrspevent!(dist::DataFrame,
    dclrexdays::Int = DCLR_EX_DAYS,
    Fret::Symbol = DIST_RET,
    retbound::Float64=DIST_RET_BOUND,
    minallowed = DIST_MIN_ALLOWED,
    daystostaleprice::Int=DIST_DAYS_TO_STALE_PRICE,
    days::Int = DIST_DAYS)

  local sdists::GroupedDataFrame

  #now compute the dividend yield from the declaration date
  dist.yield = Vector{MFloat64}(undef, size(dist,1))
  dist.winsorizedyield = falses(size(dist,1))
  dist.dclrextradingdays = Vector{MInt}(undef, size(dist,1))

  lowerdays::Int = max(daystostaleprice,days)
  sdists = groupby(dist, :eid)
  @mpar for i ∈ 1:length(sdists)
    sdist::SubDataFrame = sdists[i]
    #some basic integrity checks
    @assert maximum(sdist.divamt) == minimum(sdist.divamt)
    @assert size(sdist,1) == days + lowerdays + 1
    @assert maximum(sdist.dclrexdays) == minimum(sdist.dclrexdays)
    @assert sdist.dclrdate[1] == sdist.date[lowerdays + 1] - Day(sdist.dclrexdays[1])

    #find the row for the declaration date
    dclridx::Int = searchsortedfirst(sdist.date, sdist.dclrdate[1])
    (dclridx ≤ 0 || dclridx>size(sdist,1) || sdist.date[dclridx] ≠ sdist.dclrdate[1]) && continue

    #compute the yield and winsorize if necessary
    yield::MFloat64 = sdist.divamt[1]/sdist.price[dclridx]
    (ismissing(yield) || (yield < 0.0)) && continue
    sdist.dclrextradingdays .=  abs(sdist.day[dclridx])
    sdist.winsorizedyield .= yield > retbound
    sdist.yield .= min(yield,retbound)
  end
  #now determine the dividend yields from the exiration

  filter!(r::DataFrameRow->(r.day ≥ -days), dist)

  dist.primary = Vector{Bool}(undef, size(dist,1))
  dist.match = Vector{Bool}(undef, size(dist,1))
  dist.ordinary = Vector{Bool}(undef, size(dist,1))
  dist.dclrexok = Vector{Bool}(undef, size(dist,1))

  sdists = groupby(dist, :eid)
  @mpar for i ∈ 1:length(sdists)
    sdist = sdists[i]
    r::DataFrameRow = sdist[1, :] #examine fields common to each eid

    notmissing::Int = sum((!ismissing).(sdist[!,Fret]))
    sdist.match .= (notmissing ≥ minallowed) && (!ismissing(r.yield))
    sdist.ordinary .= r.distcd == 1232
    sdist.dclrexok .= r.dclrexdays ≥ dclrexdays
  end

  dist.primary .= dist.ordinary .* dist.match .* dist.nooverlap .* dist.dclrexok

  dist[(end-10_000):end,:] |> CSV.write("output\\distest-$(REPLICATION_TYPE[]).csv")
end

ext: jl

max_stars_repo_name: clintonTE/CCA

max_issues_repo_name: clintonTE/CCA

max_forks_repo_name: clintonTE/CCA


text:
# TODO: nonlinear scattering
################################################################################
####### SCATTERING
################################################################################
"""
    ψ_sct, ψ_tot, H = scattering(sim, k, a; H=lu(sparse(I,1,1)), F=[1], file_name="",
        is_linear=true, disp_opt=false, ψ_init=[], ftol=2e-8, iter=150, num_wg_bands=15)

Solves linear inhomogeneous problem.

`k` is the frequency.

`a` is a vector of amplitudes for each channel given in input.

`H` is a factorized wave operator (useful for repeated calculation)

`F` is a multiplicative modifier to the pump profile (can implement saturation this way)

if `file_name` is not empty, output is saved to file

`num_bloch` is the number of bloch frequencies which are used to determine gaps

`num_wg_bands` is the number of bands in the presence of the waveguide to use t
look for guided modes

Note: use build_dispersions! for more control over the dispersion-building process.
"""
function scattering(sim::Simulation, k, a;
        H=lu(sparse(complex(1.,0)*I,1,1)),
        file_name="", is_linear=true, ψ_init=[], ftol=2e-8, iter=150)

    if length(a) < length(sim.sct.channels)
        throw(ArgumentError("number of input amplitudes $(length(a)) less than the number of input channels $(length(sim.sct.channels))"))
    elseif length(a) > length(sim.sct.channels)
        @warn "number of input amplitudes $(length(a)) greater than the number of input channels $(length(sim.sct.channels)), will ignore last $(length(a)-length(sim.sct.channels)) amplitudes"
    end

    build_dispersions!(sim)

    # if is_linear
        ψ, φ₊₋, φ₊, φ₋, H = scattering_l(sim, k, a, H)
    # else
        # ψ, (φ₊₋, φ₊, φ₋, H) = scattering_nl(sim, k, a, H, F, disp_opt=disp_opt, ψ_init=ψ_init, ftol=ftol, iter=iter)
    # end

    if !isempty(file_name)
        fid = open(file_name, "w")
        serialize(fid, (sim, k, ψ) )
        close(fid)
    end

    ψ_sct = ψ
    ψ_tot = ψ + φ₊₋ + φ₊ + φ₋
    ψ_out = ψ + φ₋

    return ψ_sct, ψ_tot, ψ_out, H
end

function scattering_l(sim::Simulation, k, a, H)
    j, φ₊₋, φ₊, φ₋ = ScalarFDFD.synthesize_source(sim, k, a)
    if (H.m == H.n == 1)
        H = ScalarFDFD.factorize_scattering_operator(sim,k)
    end
    ψ = H\j
    return ψ, φ₊₋, φ₊, φ₋, H
end

function factorize_scattering_operator(sim::Simulation, k)
    k²= k^2
    bl_original = ScalarFDFD.set_bl!(sim, :pole)
    try
        ∇² = laplacian(sim, k)
        N = prod(sim.dis.N)
        εt = ε_bl(sim; k=k)

        ɛk² = sparse(1:N, 1:N, ɛt*k², N, N)
        # χk² = sparse(1:N, 1:N, float(sim.tls.D₀)*γ(sim,k)*F.*Ft[:]*k², N, N)

        return lu(∇²+ɛk²)
    finally
        reset_bl!(sim, bl_original)
    end
end

# ################################################################################
# ### NONLINEAR SOLVERS
# ################################################################################
# """
# ψ, ϕ, A, input = scattering_nl(input, k; dispOpt=false, ψ_init=[],
#     F=[1.], A=[], fileName="", ftol=2e-8, iter=750)
#
#     Solves inhomogeneous problem with source defined in incident wave file.
#
#     k is the injection frequency.
#
#     A is a factorized wave operator.
# """
# function scattering_nl(input1::InputStruct, k::Complex128;
#     dispOpt::Bool = false, ψ_init::Array{Complex128,1}=Complex128[], F::Array{Float64,1}=[1.],
#     A::Base.SparseArrays.UMFPACK.UmfpackLU=lufact(speye(1,1)),
#     ftol::Float64=2e-8, iter::Int=150)::Tuple{Array{Complex128,1},Array{Complex128,1},Base.SparseArrays.UMFPACK.UmfpackLU,InputStruct}
#
#         input = open_to_pml(input1)
#
#         j, ∇², φ₊, φ₋ = createJ(input, k, m)
#
#         N = prod(input.N_ext); ε_sm = input.ε_sm; k²= k^2;
#         D₀ = input.D₀; F_sm = input.F_sm
#         ɛk² = sparse(1:N, 1:N, ɛ_sm[:]*k², N, N, +)
#         χk² = sparse(1:N, 1:N, D₀*γ(k,input)*F.*F_sm[:]*k², N, N, +)
#
#         f!(Ψ, fvec) = scattering_residual(Ψ, fvec, j, ∇², εk², k, input)
#         jac!(Ψ, jacarray) = scattering_jacobian(Ψ, jacarray, j, ∇², εk², k, input)
#         df = DifferentiableSparseMultivariateFunction(f!, jac!)
#
#         Ψ_init = Array{Float64}(2*length(j))
#         Ψ_init[1:length(j)]     = real(ψ_init)
#         Ψ_init[length(j)+1:2*length(j)] = imag(ψ_init)
#
#         z = nlsolve(df, Ψ_init, show_trace=dispOpt, ftol=ftol, iterations=iter)
#
#         if converged(z)
#             ψ = z.zero[1:length(j)] + 1im*z.zero[length(j)+1:2*length(j)]
#         else
#             ψ = NaN*ψ
#             println("Warning, solve_scattered did not converge. Returning NaN.")
#         end
#
#         if !isempty(fileName)
#             if truncate
#                 fid = open(fileName,"w")
#                 serialize(fid, ((ψ-φ₊)[input.x̄_inds], φ₊[input.x̄_inds], input, k) )
#                 close(fid)
#             else
#                 fid = open(fileName,"w")
#                 serialize(fid, ((ψ-φ₊), φ₊, input, k) )
#                 close(fid)
#             end
#         end
#
#         if truncate
#             return (ψ-φ₊)[input.x̄_inds], φ₊[input.x̄_inds], A, input
#         else
#             return ψ-φ₊, φ₊, A, input
#         end
# end # end of function computePsi_nonlinear
#
# ################################################################################
# ###  NONLINEAR AUXILLIARIES
# ################################################################################
# """
# scattering_residual(Ψ, fvec, j, ∇², εk², k, input)
# """
# function scattering_residual(Ψ::Array{Float64,1}, fvec::Array{Float64,1}, j,
#                                     ∇², εk², k::Complex128, input::InputStruct)
#
#     ψ = similar(j,Complex128)
#     ind_r = 1:length(j)
#     ind_i = length(j)+1:2*length(j)
#     ψ = Ψ[ind_r] + 1im*Ψ[ind_i]
#     temp = (∇²+ɛk²+χ(ψ,k,input)*k^2)*ψ - j
#     fvec[ind_r] = real(temp)
#     fvec[ind_i] = imag(temp)
# end
#
# """
# scattering_jacobian(Ψ, jacarray, j, ∇², εk², k, input)
# """
# function scattering_jacobian(Ψ::Array{Float64,1}, jacarray, j, ∇², εk², k::Complex128, input::InputStruct)
#     ψ = similar(j,Complex128)
#     ind_r = 1:length(j)
#     ind_i = length(j)+1:2*length(j)
#     ψ = Ψ[ind_r] + 1im*Ψ[ind_i]
#     temp = ∇²+ɛk²+χ(ψ,k,input)*k^2
#     tempr = similar(temp,Float64)
#     tempi = similar(temp,Float64)
#     tr = nonzeros(tempr)
#     ti = nonzeros(tempi)
#     tr[:] = real((nonzeros(temp)))
#     ti[:] = imag((nonzeros(temp)))
#     tempj = [tempr+χʳʳ′(ψ,k,input) -tempi+χʳⁱ′(ψ,k,input); tempi+χⁱʳ′(ψ,k,input) tempr+χⁱⁱ′(ψ,k,input)]
#     jacarray[:,:] = tempj[:,:]
# end
#
# """
# χ(ψ, k, input)
# """
# function χ(ψ::Array{Complex128,1}, k::Complex128, input::InputStruct)::SparseMatrixCSC{Complex{Float64},Int64}
#     N = prod(input.N_ext)
#     h = hb(ψ,k,input)
#     V = input.F_sm[:].*γ(k,input)*input.D₀./(1+h)
#     return sparse(1:N, 1:N, V, N, N, +)
# end
#
# """
# """
# function hb(ψ::Array{Complex128,1},k::Complex128,input::InputStruct)::Array{Float64,1}
#     N = prod(input.N_ext)
#     h = abs2.(γ.(k, input)*ψ)
#     return h
# end
#
# """
# χʳʳ′(Ψ, k, input)
# """
# function χʳʳ′(ψ::Array{Complex128,1}, k::Complex128, input::InputStruct)::SparseMatrixCSC{Float64,Int64}
#     N = prod(input.N_ext); D₀ = input.D₀; F_sm = input.F_sm; γt = γ(k, input)
#     h = hb(ψ,k,input)
#     V = -2D₀.*F_sm[:].*abs2(γt).*real.(γt.*ψ).*real.(ψ)./(1+h).^2
#     return sparse(1:N,1:N,V,N,N,+)
# end
#
# """
# χⁱʳ′(Ψ, k, input)
# """
# function χⁱʳ′(ψ::Array{Complex128,1}, k::Complex128, input::InputStruct)::SparseMatrixCSC{Float64,Int64}
#     N = prod(input.N_ext); D₀ = input.D₀; F_sm = input.F_sm; γt = γ(k, input)
#     h = hb(ψ,k,input)
#     V = -2D₀.*F_sm[:].*abs2(γt).*imag.(γt.*ψ).*real.(ψ)./(1+h).^2
#     return sparse(1:N,1:N,V,N,N,+)
# end
#
# """
# χʳⁱ′(Ψ, k, input)
# """
# function χʳⁱ′(ψ::Array{Complex128,1}, k::Complex128, input::InputStruct)::SparseMatrixCSC{Float64,Int64}
#     N = prod(input.N_ext); D₀ = input.D₀; F_sm = input.F_sm; γt = γ(k, input)
#     h = hb(ψ,k,input)
#     V = -2D₀.*F_sm[:].*abs2(γt).*real.(γt.*ψ).*imag.(ψ)./(1+h).^2
#     return sparse(1:N,1:N,V,N,N,+)
# end
#
# """
# χⁱⁱ′(Ψ, k, input)
# """
# function χⁱⁱ′(ψ::Array{Complex128,1}, k::Complex128, input::InputStruct)::SparseMatrixCSC{Float64,Int64}
#     N = prod(input.N_ext); D₀ = input.D₀; F_sm = input.F_sm; γt = γ(k, input)
#     h = hb(ψ,k,input)
#     V = -2D₀.*F_sm[:].*abs2(γt).*imag.(γt.*ψ).*imag.(ψ)./(1+h).^2
#     return sparse(1:N,1:N,V,N,N,+)
# end


text:
function handle_step(obj::FieldOrekitStepHandler, arg0::FieldOrekitStepInterpolator, arg1::jboolean)
    return jcall(obj, "handleStep", void, (FieldOrekitStepInterpolator, jboolean), arg0, arg1)
end

function init(obj::FieldOrekitStepHandler, arg0::FieldSpacecraftState, arg1::FieldAbsoluteDate)
    return jcall(obj, "init", void, (FieldSpacecraftState, FieldAbsoluteDate), arg0, arg1)
end


text:
@testset "Embed" begin
    include("./embed/embed.jl")
    include("./embed/vocab.jl")
end


text:
using LMCLUS
using Test

@testset "LMCLUS" begin
    include("utils_test.jl")
    include("types_test.jl")
    include("separation_test.jl")
    include("mdl_test.jl")
    include("lmclus_test.jl")
    include("lmclus_p_test.jl")
end


text:
include("../envs/RLEnvironments.jl")
using .RLEnvironments
using Plots
using Distributions: Beta

abstract type Policy end
act(p::Policy) = throw("Method act($(typeof(p))) not implemented")

struct Greedy{T<:AbstractFloat} <: Policy
    avgrewards::Vector{T}
    counts::Vector{T}
    Greedy{T}(n) where T <: AbstractFloat = new(zeros(T, n), zeros(T, n))
end

struct EpsGreedy{T<:AbstractFloat} <: Policy
    avgrewards::Vector{T}
    counts::Vector{T}
    ϵ::Float64
    EpsGreedy{T}(n, ϵ) where T <: AbstractFloat = new(zeros(T, n), zeros(T, n), ϵ)
end

struct OptGreedy{T<:AbstractFloat} <: Policy
    avgrewards::Vector{T}
    counts::Vector{T}
    OptGreedy{T}(n, initvalue) where T <: AbstractFloat = new(fill(T(initvalue), n), zeros(T, n))
end

mutable struct RoundRobin{T<:AbstractFloat} <: Policy
    avgrewards::Vector{T}
    counts::Vector{T}
    n::Int64
    curraction::Int64
    RoundRobin{T}(n) where T <: AbstractFloat = new(zeros(T, n), zeros(T, n), n, n)
end

mutable struct UCB{T<:AbstractFloat} <: Policy
    avgrewards::Vector{T}
    counts::Vector{T}
    n::Int64
    round::Int64
    UCB{T}(n) where T <: AbstractFloat = new(zeros(T, n), zeros(T, n), n, 0)
end

struct ThompsonBeta <: Policy
    successes::Vector{Int64}
    failures::Vector{Int64}
    ThompsonBeta(n) = new(ones(n), ones(n))
end

function greedyaction(p)
    maxaction = argmax(p.avgrewards)
    if p.avgrewards[maxaction] < 1.0
        unused = findall(p.counts .== 0)
        length(unused) > 0 && return rand(unused)
    end
    return maxaction
end

act(p::Union{Greedy, OptGreedy}) = greedyaction(p)

act(p::EpsGreedy) = rand() <= p.ϵ ? rand(1:length(p.avgrewards)) : greedyaction(p)

act(p::RoundRobin) = (p.curraction = (p.curraction % p.n) + 1)

function act(p::UCB)
    p.round += 1
    p.round <= p.n && return p.round
    return argmax(p.avgrewards .+ sqrt.((2.0 * log(p.round)) ./ p.counts))
end

act(p::ThompsonBeta) = argmax(map(x -> rand(Beta(x[1], x[2])), zip(p.successes, p.failures)))

function improve!(p, action, reward)
    p.counts[action] += 1.0
    p.avgrewards[action] += (reward - p.avgrewards[action]) / p.counts[action]
end

function improve!(p::ThompsonBeta, action, reward)
    if reward > 0.0
        p.successes[action] += 1
    else
        p.failures[action] += 1
    end
end

function run!(p, env::Bandit{A, R}; steps=100) where {A, R}
    rewards = zeros(R, steps)
    regrets = zeros(R, steps)
    for i in 1:steps
        action = act(p)
        reward, regret = step!(env, action)
        improve!(p, action, reward)

        rewards[i] = reward #+ (i > 1 ? rewards[i-1] : 0)
        regrets[i] = regret #+ (i > 1 ? regrets[i-1] : 0)
    end
    return rewards, regrets
end

function getpolicies(env::Bandit{A, R}) where {A, R}
    n = length(env.actionspace)
    return [Greedy{R}(n), EpsGreedy{R}(n, 0.03), OptGreedy{R}(n, 1.0),
        RoundRobin{R}(n), UCB{R}(n), ThompsonBeta(n)]
end

title(t) = replace(t, r"\{(.*?)\}" => s"")

env = Bandit()
policies = getpolicies(env)
plt = plot(layout=(length(policies), 1), size=(512, 1024))
for (i, p) in enumerate(policies)
    rs, rgs = run!(p, env)
    plot!([rs rgs], labels=["Reward" "Regret"], title=title("$(typeof(p))"), subplot=i)
end
display(plt)
#savefig(plt, "bandits")


text:
# This file was generated by the Julia Swagger Code Generator
# Do not modify this file directly. Modify the swagger specification instead.

module ServiceCommunityClient

using Random
using Dates
using HTTP
using Swagger
import Swagger: field_name, property_type, hasproperty, validate_property, SwaggerApi, SwaggerModel
import Base: convert, propertynames

include("modelincludes.jl")

include("api_BgpServiceCommunitiesApi.jl")

# export models
export convert, BGPCommunity
export convert, BgpServiceCommunityListResult
export convert, BgpServiceCommunityPropertiesFormat
export convert, Resource
export convert, BgpServiceCommunity

# export operations
export convert, BgpServiceCommunitiesApi

export check_required, field_name, property_type, hasproperty, propertynames, validate_property, convert

end


text:
# ---
# title: 1314. Matrix Block Sum
# id: problem1314
# author: Tian Jun
# date: 2020-10-31
# difficulty: Medium
# categories: Dynamic Programming
# link: <https://leetcode.com/problems/matrix-block-sum/description/>
# hidden: true
# ---
#
# Given a `m * n` matrix `mat` and an integer `K`, return a matrix `answer`
# where each `answer[i][j]` is the sum of all elements `mat[r][c]` for `i - K <=
# r <= i + K, j - K <= c <= j + K`, and `(r, c)` is a valid position in the
# matrix.
#
#
#
# **Example 1:**
#
#
#
#     Input: mat = [[1,2,3],[4,5,6],[7,8,9]], K = 1
#     Output: [[12,21,16],[27,45,33],[24,39,28]]
#
#
# **Example 2:**
#
#
#
#     Input: mat = [[1,2,3],[4,5,6],[7,8,9]], K = 2
#     Output: [[45,45,45],[45,45,45],[45,45,45]]
#
#
#
#
# **Constraints:**
#
#   * `m == mat.length`
#   * `n == mat[i].length`
#   * `1 <= m, n, K <= 100`
#   * `1 <= mat[i][j] <= 100`
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


text:
module CompareShadow

# TODO: OUTDATED
using Printf
import SearchBall, GeometryTypes
const sb = SearchBall
const gt = GeometryTypes

function init_strategies(config; kwargs...)
    ws = config.ws

    radioButtons = []
    for off_idx in eachindex(ws.myPlayers)
        config.printables[string("init_shadow", off_idx)] = re.Signal("")
        config.printables[string("left_shadow", off_idx)] = re.Signal("")
        initShadows[off_idx] = sb.calc_shadow(ws, off_idx, config.field_corners, "init_shadow")
        # initShadows[off_idx] = get_shadow(ws.myPlayers[off_idx], ws, field_corners)
        config.manipulators[string("DoffShadow",off_idx)] = (sb.check_button(string("draw shaodws for off",off_idx)),
        () -> begin
            if !sb.get_value(config.manipulators[string("DoffShadow",off_idx)][1], Bool)
                compare_shadow(ws, off_idx, initShadows[off_idx], config.field_corners)
            else
                [delete!(config.drawables, string("shadow", off_idx, idx)) for idx in 1:(length(ws.myPlayers)+length(ws.opponents))]
                push!(config.printables[string("shadow", off_idx)], "")
                push!(config.printables[string("left_shadow", off_idx)], "")
            end
        end
        )
        config.printables[string("shadow", off_idx)] = re.Signal("")
        rb = sb.radio_button(string(off_idx))
        config.manipulators["DNrb$off_idx"] = (rb,
        () -> begin
            if sb.get_value(config.manipulators["Ddraw function"][1], Bool)
                vals, ins = sb.sample(config, off_idx, x -> sum([sum(sb.polygon_area.(sb.polygon_intersection(@view(i[2:end]), @view(x[2:end])))) for i in initShadows[off_idx]]))
                sb.send_background(vals, ins, config)
            end
        end)
        push!(radioButtons, rb)
    end
    config.manipulators["show function for"] = sb.radio_button_group(radioButtons)

    config.manipulators["Ddraw function"] = (sb.check_button("draw background"),
        () -> begin
            if !sb.get_value(config.manipulators["Ddraw function"][1], Bool)
                off_idx = parse(Int, sb.get_value(config.manipulators["show function for"], String))
                vals, ins = sb.sample(config, off_idx, x -> sum([sum(sb.polygon_area.(sb.polygon_intersection(@view(i[2:end]), @view(x[2:end])))) for i in initShadows[off_idx]]))
                sb.send_background(vals, ins, config)
            else
                delete!(config.drawables, "_fieldBackground")
                delete!(config.drawables, "_color_bar")
                delete!(config.drawables, "color_bar_edge")
                delete!(config.drawables, "text_min")
                delete!(config.drawables, "text_33p")
                delete!(config.drawables, "text_66p")
                delete!(config.drawables, "text_max")
            end
        end
        )
    return
end

function compare_shadow(ws::sb.WorldState, off_idx::Int, initShadows::Vector{Vector{gt.Point2{Float64}}}, field_corners::AbstractVector{<:gt.Point2{<:Number}})
    currentShadows = sb.calc_shadow(ws, off_idx, field_corners)
    overlapping_shadow = sum([sum(sb.polygon_area.(sb.polygon_intersection(@view(i[2:end]), @view(c[2:end])))) for i in initShadows for c in currentShadows])
    # overlapping_shadow = 0.0
    push!(config.printables[string("left_shadow", off_idx)], @sprintf("%.2fm²", overlapping_shadow))
end

const initShadows = Dict{Int, Vector{Vector{gt.Point2{Float64}}}}()
function calc_myPlayers(config::sb.Config, dt::Number, set_type::Symbol, moved_object::Union{sb.Object{<:Number}, Nothing}=nothing)
    ws = config.ws

    if moved_object[1]==:opponent
        for off_idx in eachindex(ws.myPlayers)
            if !sb.is_inside(ws.myPlayers[off_idx], config.field_corners)
                continue
            end
            initShadows[off_idx] = sb.calc_shadow(ws, off_idx, config.field_corners, "init_shadow")
        end
    end
    for off_idx in eachindex(ws.myPlayers)
        if !sb.is_inside(ws.myPlayers[off_idx], config.field_corners)
            for k in keys(config.drawables)
                if startswith(k, string("shadow", off_idx))
                    delete!(config.drawables, k)
                end
            end
            continue
        end
        if !sb.get_value(config.manipulators[string("DoffShadow",off_idx)][1], Bool)
            continue
        end
        compare_shadow(ws, off_idx, initShadows[off_idx], config.field_corners)
    end
    offender_for_function = parse(Int, sb.get_value(config.manipulators["show function for"], String))
    if set_type ∈ [:reset, :release] && sb.get_value(config.manipulators["Ddraw function"][1], Bool) && (moved_object[1]!=:myPlayer || moved_object[2]!= offender_for_function)
        vals, ins = sb.sample(config, offender_for_function, x -> sum([sum(sb.polygon_area.(sb.polygon_intersection(@view(i[2:end]), @view(x[2:end])))) for i in initShadows[offender_for_function]]))
        sb.send_background(vals, ins, config)
    end
    return
end

function calc_opponend(config::sb.Config, dt::Number, set_type::Symbol, moved_object::Union{sb.Object{<:Number}, Nothing}=nothing)
    ws = config.ws

    return
end

function run_strategies(config::sb.Config, dt::Number=0, set_type::Symbol=:default, moved_object::Union{sb.Object{<:Number}, Nothing}=nothing)
    calc_myPlayers(config, dt, set_type, moved_object)
    calc_opponend(config, dt, set_type, moved_object)
    return
end

end