text:
using Pkg
Pkg.add("Agents")
using Agents

using Pkg
Pkg.add("CSV")
using CSV

global dims_glob = (100,150);
global maxSize = 23; # after this size we no longer grow our crab
global minTemp = 9; # dormancy temp
global lowTemp = 10.8; # from Brylawski & Miller 2007 - the minimum physiological temp
global lowSalt = 10;
global step = 1;
global nurseryx = 40 # the horizontal limiter on the space for spawn re-entry
global nurseryy = 40 # the vertical limiter on the space for spawn re-entry
println("Initialized.")

global temps_all = CSV.read("model_temps.csv") # the observed temp at each grid cell
global salt_all = CSV.read("model_salt.csv") # the observed salt at each grid cell

println("Read data.")

function getSalt(tempTup,simTime)
    saltstring = salt[convert(Int,tempTup[1]),convert(Int, convert(Int,tempTup[2])+150*simTime)]
    if saltstring == "NA"
        return "NA"
    else
        return parse(Float64, saltstring)
    end
end

function getTemp(tempTup,simTime)
    tempstring = temps[convert(Int,tempTup[1]),convert(Int, convert(Int,tempTup[2])+150*simTime)]
    if tempstring == "NA"
        return "NA"
    else
        return parse(Float64, tempstring)
    end
end

# each agent is a crab that contains its own position, its own ID, its own other physiological params
mutable struct ChesapeakeBayCrab <: AbstractAgent
    id::Int
    pos::NTuple{2, Int}
    temp::Float64 # the temperature where the crab is placed
    salt::Float64 # the salinity where thecrab is placed
    simTime::Int # time since the simulation began
    age::Int
    sex::Int
    size::Float64
    mated::Int
    numSperm::Int
    eggs::Int
    degreeDays::Int
    tOpt::Int
    lifespan::Int
end

# really we have 120 million crabs
function crab_model_2D(;dims = dims_glob, M = 120)
  global step = 1;
  space = Space(dims, periodic = true)
  age = rand(0:10)
  model = ABM(ChesapeakeBayCrab, space; properties = step, scheduler = random_activation)
  for i in 1:M # add agents in random nodes
      if rand(0:10) > 7
          age = rand(200:1100)
      else
          age = rand(0:200)
      end
      sex = rand(0:1)
      mated = 0
      if rand(1:10) == 1 # fewer need to start out mated. ~10% mated at any given time.
          mated = 1
      end
      tOpt = rand(20:26)
      degreeDays = rand(0:24)
      simTime = 0
      pos = (rand(1:100), rand(1:150))
      temp = getTemp(pos,simTime)
      while temp == "NA"
          pos = (rand(1:100), rand(1:150))
          temp = getTemp(pos,simTime)
      end
      salt = getSalt(pos,simTime)
      size = rand(0:23)
      numSperm = size * 2
      eggs = round(rand(0:46), digits=0)
      lifespan = rand(200:1100)
      node_to_add = coord2vertex(pos, model)
      add_agent!(node_to_add, model, temp, salt, simTime, age, sex, size, mated, numSperm, eggs, degreeDays, tOpt, lifespan)
  end
  numSpawns = round(0,digits=0)#M/75, digits = 0)
  simTime = 0
  for i in 1:numSpawns
    age = 0
    sex = rand(0:1)
    numSperm = 0
    size = rand(0:5)
    mated = 0
    tOpt = rand(20:26)
    degreeDays = 0
    lifespan = rand(200:1100)
    pos = (rand(nurseryx:100), rand(1:nurseryy)) # spawns can't start just anywhere
    temp = getTemp(pos,simTime)
    numSperm = 0
    eggs = 0
    while temp == "NA"
        pos = (rand(nurseryx:100), rand(1:nurseryy))
        temp = getTemp(pos,simTime)
    end
    salt = getSalt(pos,simTime)
    node_to_add = coord2vertex(pos, model)
    add_agent!(node_to_add, model, temp, salt, simTime, age, sex, size, mated, numSperm, eggs, degreeDays, tOpt, lifespan)
  end
  return model
end

function bay_step!(bay)
    if nagents(bay) > 1
        for node in nodes(bay, by = :random)
            if nagents(bay) == 0
                return
            end
            nc = get_node_contents(node, bay)
            for curr_agent in nc
                crab = id2agent(curr_agent, bay)
                randnumber = rand();
                if crab.age >= crab.lifespan
                    delete!(bay.agents, crab.id)

                    twoDpos = node
                    splice!(bay.space.agent_positions[twoDpos],
                      findfirst(a->a==crab.id, bay.space.agent_positions[twoDpos]))
                elseif (crab.salt < lowSalt) & (crab.temp < lowTemp) & (randnumber <= (0.25 / 365))
                    delete!(bay.agents, crab.id)
                    twoDpos = node
                    splice!(bay.space.agent_positions[twoDpos],
                      findfirst(a->a==crab.id, bay.space.agent_positions[twoDpos]))
                elseif (crab.temp < lowTemp) & (randnumber <= (0.20 / 365))
                    delete!(bay.agents, crab.id)
                    twoDpos = node
                    splice!(bay.space.agent_positions[twoDpos],
                      findfirst(a->a==crab.id, bay.space.agent_positions[twoDpos]))
                elseif (randnumber <= (0.17 / 365))
                    delete!(bay.agents, crab.id)
                    twoDpos = node
                    splice!(bay.space.agent_positions[twoDpos],
                      findfirst(a->a==crab.id, bay.space.agent_positions[twoDpos]))
                end
            end
        end
    end
end

function agent_step_2d!(agent, model)
    temp = getTemp(agent.pos,agent.simTime)
    salt = getSalt(agent.pos,agent.simTime)
    agent.age += 1
    if (agent.age > 800)
        agent.mated = 2 # don't care about mating anymore
    end
    agent.temp = temp
    agent.salt = salt
    agent.simTime = agent.simTime + 1
    if temp > minTemp
        spawnx = 1 # the horizontal limiter on the space for spawning - any part of lower bay
        spawny = 23 # the vertical limiter on the space for spawning - lower part of bay
        agent_node = coord2vertex(agent.pos, model)
        agent.degreeDays += round(agent.temp - minTemp,digits=0)
        probMolt = 0.669 * log(agent.degreeDays) - 3.71 # from Brylawski & Miller 2007
        growthAmount = real(rand(1044:1598)) / 1000
        if (probMolt > rand()) & ((agent.size + growthAmount) < maxSize)
            agent.size += growthAmount
        end
        if agent.pos[2] > 80
            if agent.sex == 0 # if a female
                agent_node = coord2vertex(agent.pos, model)
                neighboring_nodes = node_neighbors(agent_node, model)
                push!(neighboring_nodes, agent_node) # also consider current node
                available_agents = []
                for curr in neighboring_nodes
                    append!(available_agents, get_node_contents(curr, model))
                end
                done_ = 0
                while length(available_agents) > 0 & done_ == 0
                    tester = rand(available_agents)
                    random_neighbor_agent = id2agent(tester, model)
                    if (random_neighbor_agent.numSperm > 0) & (random_neighbor_agent.sex == 1)
                        agent.mated = 1
                        agent.eggs = random_neighbor_agent.numSperm
                        if (random_neighbor_agent.mated == 1)
                            random_neighbor_agent.mated = 2 # now this sperm donor doesn't care about mating
                            # this will cause there to be a better global distribution of crabs.
                            # and possibly fewer overall crabs.
                        else
                            random_neighbor_agent.mated = 1
                        end
                        done_ = 1
                    end

                    deleteat!(available_agents, findfirst(x->x == tester,available_agents))
                end

                if (rand(1:1000) == 1) & (agent.mated == 0)
                    agent.eggs = rand(0:23) * 2
                    agent.mated = 1
                end
            else
                agent.numSperm = convert(Int,round(agent.size,digits=0))
            end
        end

        # Randomly decide whether the agent will move position. Each coord can change by up to 1
        newx = agent.pos[1] + rand(-5:5);
        newy = agent.pos[2] + rand(-5:5);

        # Take into account that females that have mated are trying to go southeast (higher x & lower y)
        if (agent.mated == 1) & (agent.sex == 0 | agent.pos[2] > 125)
           newx = agent.pos[1] + rand(-5:5); # + 1 + rand(-5:5);
           newy = agent.pos[2] - 8 + rand(-2:2);
        elseif (agent.pos[2] > 125)
           newy = agent.pos[2] - 4 + rand(-2:2);
        end

        # Take into account that individuals that have not mated are trying to go north & stay west
        if (agent.mated == 0) & (agent.pos[2] < 80) & (agent.pos[1] > 40) & (agent.pos[1] < 65)
            newx = agent.pos[1] + rand(-5:5);
            newy = agent.pos[2] + 8 + rand(-2:2);
        end

        # Take into account that individuals that are in branches need to get to the main shaft eventually
        if (agent.mated < 2) & (agent.pos[2] < 80)
            if agent.pos[1] < 40
                newx = agent.pos[1] + rand(-2:4);
            elseif agent.pos[1] > 55
                newx = agent.pos[1] + rand(-4:2);
            end
        end

        if (agent.pos[1] > 55) & (agent.pos[2] < 40)
            newx = agent.pos[1] - 1 + rand(-4:2);
        end

        # distribute crabs that don't care about mating to low-density areas.
        if (agent.mated == 2)
            agent_node = coord2vertex(agent.pos, model)
            currmin = 5
            for cell in node_neighbors(agent_node, model)
                neighboringcrabs = get_node_contents(cell, model)
                if (length(neighboringcrabs) < currmin)
                    (newx,newy) = vertex2coord(cell, model)
                    currmin = length(neighboringcrabs)
                end
            end
        end

        # Make sure we don't go off the grid
        if newx > dims_glob[1]
            newx = agent.pos[1]
        elseif newx < 1
            newx = 1
        end

        if newy > dims_glob[2]
            newy = agent.pos[2]
        elseif newy < 1
            newy = 1
        end

        temp = getTemp((newx,newy),agent.simTime)
        numNAs = 0
        while temp == "NA"
            changex = rand(-2:2);
            changey = rand(-2:2);
            if agent.mated == 0
                changey = rand(-2:5);
            else
                changey = rand(-8:0);
            end

            if (agent.mated < 2)
                if agent.pos[1] < 40
                    changex = rand(-1:2);
                elseif agent.pos[1] > 55
                    changex = rand(-2:1); #+ rand(-5:5);
                end
            end

            if changex == 0
                changex = rand(-1:1)
            end
            if changey == 0
                changey = rand(-1:1)
            end
            newx = agent.pos[1] + changex
            newy = agent.pos[2] + changey

            # Make sure we don't go off the grid
            if newx > dims_glob[1]
                newx = dims_glob[1]
            elseif newx < 1
                newx = 1
            end

            if newy > dims_glob[2]
                newy = dims_glob[2]
            elseif newy < 1
                newy = 1
            end

            temp = getTemp((newx,newy),agent.simTime)
            numNAs += 1

            if (temp == "NA") & (numNAs >= 3)
                newx = agent.pos[1]
                newy = agent.pos[2]
                temp = agent.temp
            end

        end

        agent.pos = (newx,newy)
        agent.temp = temp
        agent.salt = getSalt(agent.pos,agent.simTime)
        if (agent.pos[1] > spawnx) & (agent.pos[2] <= spawny) & (agent.mated == 1) & (rand(1:100) == 1)
            numSpawns = round(agent.eggs * 0.3, digits=0) #0.35#rand(20:60); #220);
            simTime = agent.simTime
            for i in 1:numSpawns
                age = 0
                sex = rand(0:1)
                numSperm = 0
                size = rand(0:5)
                mated = 0
                tOpt = rand(20:26)
                degreeDays = 0
                lifespan = rand(200:1100)
                pos = (rand(nurseryx:100), rand(1:nurseryy)) # spawns can't start just anywhere
                temp = getTemp(pos,simTime)
                numSperm = 0
                eggs = 0
                while temp == "NA"
                    pos = (rand(nurseryx:100), rand(1:nurseryy))
                    temp = getTemp(pos,simTime)
                end
                salt = getSalt(pos,simTime)
                node_to_add = coord2vertex(pos, model)
                add_agent!(node_to_add, model, temp, salt, simTime, age, sex, size, mated, numSperm, eggs, degreeDays, tOpt, lifespan)
            end
            agent.mated = 0
        end
    end
end

using Random
global temps_trunc = CSV.read("model_temps_truncated.csv")[:,1:(150*365)]
global salts_trunc = CSV.read("model_salts_truncated.csv")[:,1:(150*365)]
global temps = hcat(temps_trunc, CSV.read("model_temps_climatechange.csv")[:,1:(150*1460)],makeunique=true)
# without salt being changed ->
global salt = salt_all[:,1:length(temps[1,:])]
model = crab_model_2D(;M = 2000)
agent_properties = [:id, :pos, :temp, :simTime, :age, :sex, :mated, :eggs, :size, :numSperm, :degreeDays, :tOpt, :lifespan]
numdays = Int(round(length(temps[1,:]) / 150, digits=0)-1)
data = step!(model, agent_step_2d!, bay_step!, numdays, agent_properties, when =[50,100,150,160,170,180,190,200,300,400,500,600,700,800,900,1000,1100,1400,1700,1800,numdays],replicates=3,parallel=true,step0=true)

CSV.write(joinpath("output","climatechangetemps_29August2020.csv",data)

global temps = hcat(temps_trunc, CSV.read("model_temps_truncated.csv")[:,1:(150*1460)], makeunique=true)
model2 = crab_model_2D(;M = 2000)
data2 = step!(model2, agent_step_2d!, bay_step!, numdays, agent_properties, when =[50,100,150,160,170,180,190,200,300,400,500,600,700,800,900,1000,1100,1400,1700,1800,numdays],replicates=3,parallel=true,step0=true)

Pkg.add("DataFrames")
using DataFrames
CSV.write(joinpath("output","normaltruncatedtemps_29August2020.csv"),data2)

max_stars_repo_name: akrinos/bluecrab-agent

max_issues_repo_name: akrinos/bluecrab-agent

max_forks_repo_name: akrinos/bluecrab-agent


text:
"""
    H1, A1, B1 = pslq_step(H, A, B)

Perform one step of the PSLQ agorithm. Before the first step H should be initialized
by calling `make_H(x)` from and array `x` of reals and `A`, `B` should be identity matriices.
The step returns modified `H` and `B` accordingly. The follow identities are maintained:
`A1 * B1 == A * B` and `B1 * H1 == B * H * G` where `G` is orthonormal.
"""
function pslq_step(H::Matrix{T}, A::Matrix{Ti}, B::Matrix{Ti}) where {T<:Real,Ti<:Integer}
    pslq_step!(copy(H), copy(A), copy(B), UnitLowerTriangular(similar(B)))
end
function pslq_step!(H::Matrix{T}, A::Matrix{Ti}, B::Matrix{Ti},
                    D::UnitLowerTriangular{Ti,Matrix{Ti}}) where {T<:Real,Ti<:Integer}
    n = check_H(H)
    check_square(D, n)
    check_square(A, n)
    check_square(B, n)
    make_DH!(D, H, nint)
    γ = 2.0
    j = findmaxj(H, γ)
    exrows!(H, j, j+1, 1:j)
    givenscol!(H, j)
    rdiv!(B, D)
    excols!(B, j, j+1, 1:n)
    lmul!(D, A)
    exrows!(A, j, j+1, 1:n)
    H, A, B
end

function findmaxj(H::Matrix, γ::Real)
    # δ = sqrt(3/4 - 1/γ^2) must be > 0
    # j = findmax(abs.(diag(H) .* γ .^ (1:n-1)))[2]
    n = min(size(H)...)
    j = 1
    msf = abs(H[1,1]) / γ
    for k = 2:n
        hkk = abs(H[k,k])
        if hkk >= msf
            j, msf = k, hkk
        end
        msf /= γ
    end
    j
end
function check_H(H::AbstractMatrix)
    n = size(H, 1)
    if size(H, 2) != n - 1 || n < 2
        throw(DimensionMismatch("Matrix needs to be nx(n-1) with n >= 2 " *
                                "but has size $(size(H))"))
    end
    n
end
function check_square(B::AbstractMatrix, n::Integer)
    n = size(B, 1)
    if size(B, 2) != n
        throw(DimensionMismatch("Matrix needs to be square with size $n but is $(size(B))"))
    end
    n
end
nint(x) = Integer(round(x))
integertype(::Type{T}) where T = typeof(Integer(T(0)))
integertype(::Type{BigFloat}) = BigInt

# calculate D and D*H in one loop (modified Hermite reduction)
function make_DH(H::Matrix{T}, nint::Function=nint) where {T<:Real}
    n = size(H, 1)
    D = Matrix{integertype(T)}(undef, n, n)
    make_DH!(UnitLowerTriangular(D), copy(H), nint)
end
function make_DH!(DU::UnitLowerTriangular, H::Matrix, nint::Function)
    n = check_H(H)
    D = DU.data
    for i = 2:n
        for j = 1:i-1
            D[i,j] = 0
        end
        for j = i-1:-1:1
            q = nint(H[i,j] / H[j,j])
            D[i,j] -= q
            H[i,j] -= q * H[j,j]
            for k = 1:j-1
                D[i,k] -= q * D[j,k]
                H[i,k] -= q * H[j,k]
            end
        end
    end
    DU, H
end

function make_H(x::Vector{T}) where T<:Real
    n = length(x)
    H = zeros(T, n, n-1)
    b = abs(x[n])
    a = b^2
    for j = n-1:-1:1
        a += x[j]^2
        c = b
        b = sqrt(a)
        H[j,j] = c / b
        d = -x[j] / (c * b)
        for i = j+1:n
            H[i,j] = x[i] * d
        end
    end
    H
end

function givenscol!(H::Matrix{T}, j::Integer) where T<:Real
    if j == size(H, 2)
        return refresh!(H, j)
    end
    n = size(H, 1)
    a, b, c = H[j,j], H[j+1,j+1], H[j+1,j]
    d = hypot(a, b)
    a /= d
    b /= d
    H[j,j] = d
    H[j+1,j], H[j+1,j+1] = c * a, -c * b
    for k = j+2:n
        u, v = H[k,j], H[k,j+1]
        H[k,j], H[k,j+1] = u*a + v*b, -u*b + v*a
    end
    refresh!(H, j+1)
end

function exrows!(H::Matrix, i1::Integer, i2::Integer, r::UnitRange)
    for k in r
        H[i1,k], H[i2,k] = H[i2,k], H[i1,k]
    end
    H
end

function excols!(H::Matrix, i1::Integer, i2::Integer, r::UnitRange)
    for k in r
        H[k,i1], H[k,i2] = H[k,i2], H[k,i1]
    end
    H
end

"""
    refresh!(H, j)
divide column `j` by `H[j,j]/abs(H[j,j])` in order to make `H[j,j]` real and positive.
`H` is assumed to be lower trapezoidal.
"""
function refresh!(H::Matrix, col::Integer)
    m, n = size(H)
    h = H[col, col]
    if 0 < col <= m && !(isreal(h) && real(h) >= 0)
        h /= abs(h)
        @inbounds for k = col:m
            H[k,col] /= h
        end
    end
    H
end

function refresh!(H::Matrix)
    n = size(H, 2)
    @inbounds for i = 1:n
        refresh!(H, i)
    end
    H
end

"""
    lqrefresh(A, H)

calculate LQ-factorization of `A * H`and return refreshed `L`.
"""
function lqrefresh(A::Matrix, H::Matrix)
    L = lq!(A * H).L # lq! not availbale for BigFloat
    refresh!(L)
end
function lqrefresh(A::Matrix, H::Matrix{BigFloat})
    L = Matrix(qr!(H'*A').R')
    refresh!(L)
end

ext: jl


text:
using CRC32c
using Base.Threads
using CodecZlib
using BufferedStreams
using MacroTools: @forward
using ProtoBuf: ProtoType
using TranscodingStreams: NoopStream

# Ref: https://github.com/tensorflow/tensorflow/blob/295ad2781683835be974faba0a191528d8079768/tensorflow/core/lib/hash/crc32c.h#L50-L59

const MASK_DELTA = 0xa282ead8

mask(crc::UInt32) = ((crc >> 15) | (crc << 17)) + MASK_DELTA

function unmask(masked_crc::UInt32)
    rot = masked_crc - MASK_DELTA
    ((rot >> 17) | (rot << 15))
end

"""

Ref: https://github.com/tensorflow/tensorflow/blob/295ad2781683835be974faba0a191528d8079768/tensorflow/core/lib/io/record_reader.cc#L164-L199

Each record is stored in the following format:

```
uint64 n
uint32 masked_crc32_of_n
byte   data[n]
uint32 masked_crc32_of_data
```
"""
function read_record(io::IO)
    n = Base.read(io, sizeof(UInt64))
    masked_crc32_n = Base.read(io, UInt32)
    crc32c(n) == unmask(masked_crc32_n) || error("record corrupted, did you set the correct compression?")

    data = Base.read(io, Int(reinterpret(UInt64, n)[]))  # !!! watch https://github.com/JuliaIO/TranscodingStreams.jl/pull/104
    masked_crc32_data = Base.read(io, UInt32)
    crc32c(data) == unmask(masked_crc32_data) || error("record corrupted, did you set the correct compression?")
    data
end

"""
    read([f=identity], s::Union{String,Vector{String}};kwargs...)

Read tensorflow records from file(s).

# Keyword Arguments

- `compression=nothing`. No compression by default. Optional values are `:zlib` and `:gzip`.
- `bufsize=10*1024*1024`. Set the buffer size of internal `BufferedOutputStream`. The default value is `10M`. Suggested value is between `1M`~`100M`.
- `channel_size=1000`. The number of pre-fetched elements.
- `eltype=Example`. Change it to the type of result `f(::Example)` if `f` is provided.

!!! note

    To enable reading records from multiple files concurrently, remember to set the number of threads correctly (See [JULIA_NUM_THREADS](https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_NUM_THREADS)).
"""
read(s; kw...) = read(identity, s; kw...)

function read(
    f,
    files;
    compression = nothing,
    bufsize = 10 * 1024 * 1024,
    channel_size = 1_000,
    record_type = Example,
)

    file_itr(file::AbstractString) = [file]
    file_itr(files) = files

    Channel{record_type}(channel_size) do ch
        @threads for file_name in file_itr(files)
            open(decompressor_stream(compression), file_name, "r") do io
                buffered_io = BufferedInputStream(io, bufsize)
                while !eof(buffered_io)
                    instance = readproto(IOBuffer(read_record(buffered_io)), record_type())
                    put!(ch, f(instance))
                end
            end
        end
    end
end

#####
# TFRecordWriter
#####

struct TFRecordWriter{X<:IO}
    io::X
end

"""
    write(file_name, xs;compression=nothing, bufsize=1024*1024)

`xs` is assumed to be an iterator. Its element must support converting to an `Example`.

Supported `compression` methods are: `:gzip` or `:zlib`.
Default value is `nothing`, which means do not do compression.
`bufsize` is used to set the size of buffer used by an internal
`BufferedOutputStream`, the default value is `1M` (1024*1024).
You may want to change it to a larger value when writing large datasets,
for example `100M`.
"""
function write(s::AbstractString, x; compression=nothing, bufsize=1024*1024)
    open(compressor_stream(compression), s, "w") do io
        buffered_io = BufferedOutputStream(open(s, "w"), bufsize)
        write(buffered_io, x)
    end
end

function write(io::IO, xs)
    for x in xs
        write(io, convert(Example, x))
    end
end

function write(io::IO, x::ProtoType)
    buff = IOBuffer()
    writeproto(buff, x)

    data_crc = mask(crc32c(seekstart(buff)))
    data = take!(seekstart(buff))
    n = length(data)

    buff = IOBuffer()
    Base.write(buff, n)
    n_crc = mask(crc32c(seekstart(buff)))

    Base.write(io, n)
    Base.write(io, n_crc)
    Base.write(io, data)
    Base.write(io, data_crc)
end

#####
# convert
#####

Base.convert(::Type{Feature}, x::Int) = Feature(;int64_list=Int64List(value=[x]))
Base.convert(::Type{Feature}, x::Bool) = Feature(;int64_list=Int64List(value=[Int(x)]))
Base.convert(::Type{Feature}, x::Float32) = Feature(;float_list=FloatList(value=[x]))
Base.convert(::Type{Feature}, x::AbstractString) = Feature(;bytes_list=BytesList(value=[unsafe_wrap(Vector{UInt8}, x)]))

Base.convert(::Type{Feature}, x::Vector{Int}) = Feature(;int64_list=Int64List(value=x))
Base.convert(::Type{Feature}, x::Vector{Bool}) = Feature(;int64_list=Int64List(value=convert(Vector{Int}, x)))
Base.convert(::Type{Feature}, x::Vector{Float32}) = Feature(;float_list=FloatList(value=x))
Base.convert(::Type{Feature}, x::Vector{<:AbstractString}) = Feature(;bytes_list=BytesList(value=[unsafe_wrap(Vector{UInt8}, s) for s in x]))
Base.convert(::Type{Feature}, x::Vector{Array{UInt8,1}}) = Feature(;bytes_list=BytesList(value=x))

Base.convert(::Type{Features}, x::Dict) = Features(;feature=Dict(k=>convert(Feature, v) for (k, v) in x))

function Base.convert(::Type{Example}, x::Dict)
    d = Example()
    d.features = convert(Features, x)
    d
end

# (De)compression

function compressor_stream(compression)
    if isnothing(compression)
        NoopStream
    elseif compression === :gzip
        GzipCompressorStream
    elseif compression === :zlib
        ZlibCompressorStream
    else
        throw(ArgumentError("Unsupported compression method: $compression"))
    end
end

function decompressor_stream(compression)
    if isnothing(compression)
        NoopStream
    elseif compression === :gzip
        GzipDecompressorStream
    elseif compression === :zlib
        ZlibDecompressorStream
    else
        throw(ArgumentError("Unsupported decompression method: $compression"))
    end
end


text:
# This file is a part of BAT.jl, licensed under the MIT License (MIT).

using BAT
using Test

using Random, Statistics, LinearAlgebra
using Distributions, PDMats
using StableRNGs

@testset "standard_normal" begin
    stblrng() = StableRNG(789990641)

    @testset "StandardUvNormal" begin
        @test @inferred(BAT.StandardUvNormal()) isa BAT.StandardUvNormal{Float64}
        @test @inferred(BAT.StandardUvNormal{Float32}()) isa BAT.StandardUvNormal{Float32}
        @test @inferred(BAT.StandardUvNormal{Float64}()) isa BAT.StandardUvNormal{Float64}

        @test @inferred(Normal(BAT.StandardUvNormal{Float32}())) isa Normal{Float32}
        @test @inferred(Normal(BAT.StandardUvNormal{Float64}())) isa Normal{Float64}
        @test @inferred(Normal(BAT.StandardUvNormal())) == Normal()
        @test @inferred(convert(Normal, BAT.StandardUvNormal())) == Normal()

        d = BAT.StandardUvNormal()
        dref = Normal()

        @test @inferred(minimum(d)) == minimum(dref)
        @test @inferred(maximum(d)) == maximum(dref)

        @test @inferred(params(d)) == params(dref)
        @test @inferred(partype(d)) == partype(dref)

        @test @inferred(location(d)) == location(dref)
        @test @inferred(scale(d)) == scale(dref)

        @test @inferred(eltype(typeof(d))) == eltype(typeof(dref))
        @test @inferred(eltype(d)) == eltype(dref)

        @test @inferred(length(d)) == length(dref)
        @test @inferred(size(d)) == size(dref)

        @test @inferred(mean(d)) == mean(dref)
        @test @inferred(median(d)) == median(dref)
        @test @inferred(mode(d)) == mode(dref)
        @test @inferred(modes(d)) ≈ modes(dref)

        @test @inferred(var(d)) == var(dref)
        @test @inferred(std(d)) == std(dref)
        @test @inferred(skewness(d)) == skewness(dref)
        @test @inferred(kurtosis(d)) == kurtosis(dref)

        @test @inferred(entropy(d)) == entropy(dref)

        for x in [-Inf, -1.3, 0.0, 1.3, +Inf]
            @test @inferred(gradlogpdf(d, x)) == gradlogpdf(dref, x)

            @test @inferred(logpdf(d, x)) == logpdf(dref, x)
            @test @inferred(pdf(d, x)) == pdf(dref, x)
            @test @inferred(logcdf(d, x)) == logcdf(dref, x)
            @test @inferred(cdf(d, x)) == cdf(dref, x)
            @test @inferred(logccdf(d, x)) == logccdf(dref, x)
            @test @inferred(ccdf(d, x)) == ccdf(dref, x)
        end

        for p in [0.0, 0.25, 0.75, 1.0]
            @test @inferred(quantile(d, p)) == quantile(dref, p)
            @test @inferred(cquantile(d, p)) == cquantile(dref, p)
        end

        for t in [-3, 0, 3]
            @test @inferred(mgf(d, t)) == mgf(dref, t)
            @test @inferred(cf(d, t)) == cf(dref, t)
        end

        @test @inferred(rand(stblrng(), d)) == rand(stblrng(), d)
        @test @inferred(rand(stblrng(), d, 5)) == rand(stblrng(), d, 5)

        @test @inferred(truncated(BAT.StandardUvNormal{Float32}(), -2.2f0, 3.1f0)) isa Truncated{Normal{Float32}}
        @test truncated(BAT.StandardUvNormal{Float32}(), -2.2f0, 3.1f0) == truncated(Normal(0.0f0, 1.0f0), -2.2f0, 3.1f0)

        @test @inferred(product_distribution(fill(BAT.StandardUvNormal{Float32}(), 3))) isa BAT.StandardMvNormal{Float32}
        @test product_distribution(fill(BAT.StandardUvNormal{Float32}(), 3)) == BAT.StandardMvNormal{Float32}(3)
    end

    @testset "StandardMvNormal" begin
        @test @inferred(BAT.StandardMvNormal(3)) isa BAT.StandardMvNormal{Float64}
        @test @inferred(BAT.StandardMvNormal{Float32}(3)) isa BAT.StandardMvNormal{Float32}
        @test @inferred(BAT.StandardMvNormal{Float64}(3)) isa BAT.StandardMvNormal{Float64}

        @test @inferred(Distributions.Product(BAT.StandardMvNormal{Float32}(3))) isa Distributions.Product{Continuous,BAT.StandardUvNormal{Float32}}
        @test @inferred(Distributions.Product(BAT.StandardMvNormal{Float64}(3))) isa Distributions.Product{Continuous,BAT.StandardUvNormal{Float64}}
        @test @inferred(Distributions.Product(BAT.StandardMvNormal{Float64}(3))) == Distributions.Product(Fill(BAT.StandardUvNormal(), 3))
        @test @inferred(convert(Product, BAT.StandardMvNormal(3))) == Distributions.Product(Fill(BAT.StandardUvNormal(), 3))

        @test @inferred(MvNormal(BAT.StandardMvNormal{Float32}(3))) isa MvNormal{Float32}
        @test @inferred(MvNormal(BAT.StandardMvNormal{Float64}(3))) isa MvNormal{Float64}
        @test @inferred(MvNormal(BAT.StandardMvNormal{Float64}(3))) == MvNormal(ScalMat(3, 1.0))
        @test @inferred(convert(MvNormal, BAT.StandardMvNormal(3))) == MvNormal(ScalMat(3, 1.0))

        d = BAT.StandardMvNormal(3)
        dref = MvNormal(ScalMat(3, 1.0))

        @test @inferred(eltype(typeof(d))) == eltype(typeof(dref))
        @test @inferred(eltype(d)) == eltype(dref)

        @test @inferred(length(d)) == length(dref)
        @test @inferred(size(d)) == size(dref)

        @test @inferred(view(BAT.StandardMvNormal{Float32}(7), 3)) isa BAT.StandardUvNormal{Float32}
        @test_throws BoundsError view(BAT.StandardMvNormal{Float32}(7), 9)
        @test @inferred(view(BAT.StandardMvNormal{Float32}(7), 2:4)) isa BAT.StandardMvNormal{Float32}
        @test view(BAT.StandardMvNormal{Float32}(7), 2:4) == BAT.StandardMvNormal{Float32}(3)
        @test_throws BoundsError view(BAT.StandardMvNormal{Float32}(7), 2:8)

        @test @inferred(params(d)) == params(dref)
        @test @inferred(partype(d)) == partype(dref)

        @test @inferred(mean(d)) == mean(dref)
        @test @inferred(var(d)) == var(dref)
        @test @inferred(cov(d)) == cov(dref)

        @test @inferred(mode(d)) == mode(dref)
        @test @inferred(modes(d)) == modes(dref)

        @test @inferred(invcov(d)) == invcov(dref)
        @test @inferred(logdetcov(d)) == logdetcov(dref)

        @test @inferred(entropy(d)) == entropy(dref)

        for x in fill.([-Inf, -1.3, 0.0, 1.3, +Inf], 3)
            @test @inferred(insupport(d, x)) == insupport(dref, x)
            @test @inferred(logpdf(d, x)) == logpdf(dref, x)
            @test @inferred(pdf(d, x)) == pdf(dref, x)
            @test @inferred(sqmahal(d, x)) == sqmahal(dref, x)
            @test @inferred(gradlogpdf(d, x)) == gradlogpdf(dref, x)
        end

        @test @inferred(rand(stblrng(), d)) == rand(stblrng(), d)
        @test @inferred(rand!(stblrng(), d, zeros(3))) == rand!(stblrng(), d, zeros(3))
        @test @inferred(rand!(stblrng(), d, zeros(3, 10))) == rand!(stblrng(), d, zeros(3, 10))
    end
end


text:
abstract type BDDNode end

const BinBoolType = Union{Integer,Bool}

struct BDDTerminal <: BDDNode
  value::Bool
  f_low::WeakKeyDict{BDDNode,Bool}
  f_high::WeakKeyDict{BDDNode,Bool}

  function BDDTerminal(value::BinBoolType)
    if value==0 || value==false
      value=0
    elseif value==1 || value==true
      value=1
    else
      throw(ArgumentError("expected a binary value, got $(value)"))
    end

    if !haskey(Tnodes,value)
      Tnodes[value]=new(value,WeakKeyDict{BDDNode,Bool}(),
                              WeakKeyDict{BDDNode,Bool}())
    end

    return Tnodes[value]
  end
end

const Tnodes = Dict{Bool,BDDTerminal}()

function invert!(a::BDDTerminal,result_cache=Dict())
  if a in keys(result_cache)
    return result_cache[a]
  end

  result_cache[a]=BDD(!a.value)

  return result_cache[a]
end

function ~(a::BDDNode)
  return invert!(a,Dict())
end

function string(a::BDDTerminal)
  if a.value
    return "1"
  else
    return "0"
  end
end

mutable struct BDDNonTerminal <: BDDNode
  var::String
  low::BDDNode
  high::BDDNode
  f_low::WeakKeyDict{BDDNode,Bool}
  f_high::WeakKeyDict{BDDNode,Bool}

  function BDDNonTerminal(var::String,low::BDDNode,high::BDDNode)
    if low===high
      return low
    end

    if length(high.f_high)>length(low.f_low)
      nodeDict=low.f_low
      test=(node->node.high===high)
    else
      nodeDict=high.f_high
      test=(node->node.low===low)
    end

    for node in keys(nodeDict)
      if node.var==var && test(node)
        return node
      end
    end
    node=new(var,low,high,WeakKeyDict{BDDNode,Bool}(),
                          WeakKeyDict{BDDNode,Bool}())

    high.f_high[node]=true
    low.f_low[node]=true

    return node
  end
end

function SAT_assignments!(A::Array{Dict{String, BinBoolType}},
                          tmp_a::Dict{String, BinBoolType},
                          node::BDDNode)
  if typeof(node)==BDDTerminal
    if node.value==true || node.value==1
      push!(A, copy(tmp_a))
    end
  else
    if (typeof(node.high)!=BDDTerminal ||
        node.high.value==false || node.high.value==0)
      tmp_a[node.var] = 0
    end
    SAT_assignments!(A, tmp_a, node.low)
    delete!(tmp_a, node.var)

    if (typeof(node.low)!=BDDTerminal ||
        node.low.value==false || node.low.value==0)
      tmp_a[node.var] = 1
    end
    SAT_assignments!(A, tmp_a, node.high)
    delete!(tmp_a, node.var)
  end
end

function SAT_assignments(node::BDDNode)::Array{Dict{String, BinBoolType}}
  tmp_a = Dict{String, BinBoolType}()
  A = Dict{String, BinBoolType}[]

  SAT_assignments!(A, tmp_a, node)

  return A
end

function invert!(a::BDDNonTerminal,result_cache=Dict())
  if haskey(result_cache,a)
    return result_cache[a]
  end

  result_cache[a]=BDDNonTerminal(a.var,
                                 invert!(a.low,result_cache),
                                 invert!(a.high,result_cache))

  return result_cache[a]
end

function string(a::BDDNonTerminal)
  repr=String[]

  if typeof(a.low)==BDDTerminal
    if a.low.value
      push!(repr,"~$(a.var)")
    end
  else
    push!(repr,"(~$(a.var) & $(a.low))")
  end

  if typeof(a.high)==BDDTerminal
    if a.high.value
      push!(repr,"$(a.var)")
    end
  else
    push!(repr,"($(a.var) & $(a.high))")
  end

  if length(repr)==2
    return "($(repr[1]) | $(repr[2]))"
  end

  return repr[1]
end

print(io::IO, x::BDDNode) = print(io, string(x))
println(io::IO, x::BDDNode) = println(io, string(x))

show(io::IO, x::BDDNode) = show(io, "$(string(x))")

==(a::BDDNode,b::BDDNode) = (a === b)

==(a::BDDNode,b::BinBoolType) = (a === BDD(b))

==(a::BinBoolType,b::BDDNode) = (BDD(a) === b)

function BDD(value::BinBoolType)
  return BDDTerminal(value)
end

function BDD(var::String,low::BDDNode,high::BDDNode)
  return BDDNonTerminal(var,low,high)
end

function respect!(bdd::BDDNode,O::Ordering,checked::Set)
  global dynamicOBDDordering

  if in(bdd,checked)
    return true
  end

  if typeof(bdd)==BDDTerminal
    push!(checked,bdd)

    return true
  end

  if typeof(bdd)==BDDNonTerminal
    if !in(bdd.var,O)
      if dynamicOBDDordering

      else
        throw(ArgumentError("$(bdd.var) is not in $(O)"))
      end
    end

    for son in BDDNode[bdd.low,bdd.high]
      if typeof(son) == BDDNonTerminal && !inorder(O,bdd.var,son.var)
        return false
      end
    end

    if (respect!(bdd.low,O,checked) && respect!(bdd.high,O,checked))
      push!(checked,bdd)

      return true
    else
      return false
    end
  end
end

function respect(bdd::BDDNode,O::Ordering)
  return respect!(bdd,O,Set{BDDNode}())
end

function ancestors!(A::BDDNode,checked::Set{BDDNode})
  anc=Set{BDDNode}()
  stack=BDDNode[A]

  while !isempty(stack)
    node=pop!(stack)

    if !in(node,anc)&&!in(node,checked)
      push!(anc,node)

      for father in union(keys(node.f_low),keys(node.f_high))
        push!(stack,father)
      end
    end
  end

  return anc
end

function ancestors(A::BDDNode)
  return ancestors!(A,Set{BDDNode}())
end

function descendents!(A::BDDNode,checked)
  desc=Set{BDDNode}()
  to_visit=BDDNode[A]

  while !isempty(to_visit)
    node=pop!(to_visit)

    if !in(node,desc)
      push!(desc,node)

      if typeof(node)==BDDNonTerminal
        push!(to_visit,node.low)
        push!(to_visit,node.high)
      end
    end
  end

  return desc
end

function descendents(A::BDDNode)
  return descendents!(A,Set{BDDNode}())
end

function variables(A::BDDNode)
  vars=Set{String}()
  for node in descendents(A)
    if typeof(node)==BDDNonTerminal
      push!(vars,node.var)
    end
  end

  return vars
end

function applyoperator!(operator::Function,A::BDDNode,B::BDDNode,
                        O::Ordering,result_cache::Dict)
  if !haskey(result_cache,A)
    result_cache[A]=Dict()
  end

  if haskey(result_cache[A],B)
    return result_cache[A][B]
  end

  result_cache[A][B]=compute!(operator,A,B,O,result_cache)

  return result_cache[A][B]
end

function applyoperator(operator::Function,A::BDDNode,B::BDDNode,O::Ordering)
  return applyoperator!(operator,A,B,O,Dict())
end

function compute!(operator::Function,A::BDDNode,B::BDDNode,
                  O::Ordering,result_cache::Dict)
  if typeof(A)==BDDTerminal
    if typeof(B)==BDDTerminal
      return BDD(operator(A.value,B.value))
    end
    low=applyoperator!(operator,A,B.low,O,result_cache)
    high=applyoperator!(operator,A,B.high,O,result_cache)
    return BDD(B.var,low,high)
  end

  if typeof(B)==BDDTerminal || inorder(O,A.var,B.var)
    low=applyoperator!(operator,A.low,B,O,result_cache)
    high=applyoperator!(operator,A.high,B,O,result_cache)
    return BDD(A.var,low,high)
  end

  if A.var==B.var
    low=applyoperator!(operator,A.low,B.low,O,result_cache)
    high=applyoperator!(operator,A.high,B.high,O,result_cache)

    return BDD(A.var,low,high)
  end

  if inorder(O,B.var,A.var)
    low=applyoperator!(operator,A,B.low,O,result_cache)
    high=applyoperator!(operator,A,B.high,O,result_cache)
    return BDD(B.var,low,high)
  end

  throw(ArgumentError("Unsupported configuration $(A) $(B)"))
end

function compute(operator::Function,A::BDDNode,B::BDDNode, O::Ordering)
  return compute!(operator,A,B,O,Dict())
end

function restrict(A::BDDNode,var::String,value::Integer)
  if value==0
    return restrict(A,var,false)
  end

  if value==1
    return restrict(A,var,true)
  end

  throw(ArgumentError("expected a binary value, got $(value)"))
end

function restrict!(A::BDDNode,var::String,value::Bool,result_cache::Dict)
  if haskey(result_cache,A)
    return result_cache[A]
  end

  result_cache[A]=computerestriction!(A,var,value,result_cache)

  return result_cache[A]
end

function restrict(A::BDDNode, assignment::Dict{String})
   new_node = A

   for (var, value) in assignment
      new_node = restrict(new_node, var, value)
   end

   return new_node
end

function restrict(A::BDDNode,var::String,value::Bool)
  return restrict!(A,var,value,Dict())
end

function computerestriction!(A::BDDNode,var::String,value::Bool,
                             result_cache::Dict)
  if typeof(A)==BDDTerminal
    return A
  end

  if A.var==var
    if value
      return restrict!(A.high,var,value,result_cache)
    else
      return restrict!(A.low,var,value,result_cache)
    end
  else
    low=restrict!(A.low,var,value,result_cache)
    high=restrict!(A.high,var,value,result_cache)

    return BDD(A.var,low,high)
  end
end


text:
### A Pluto.jl notebook ###
# v0.11.3

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing
        el
    end
end

# ╔═╡ fd0763a0-b163-11ea-23b4-a7bae7052e19
md"# File picker"

# ╔═╡ 592d3ac0-b15e-11ea-313c-db05bc7303cc
md"# Notebook interaction"

# ╔═╡ 6dde0352-b15e-11ea-2fa8-7327cc366c1a
md"## Running multiple cells"

# ╔═╡ f0b821b0-b15f-11ea-1f64-dd33aa85b54e
md"## Moving cells"

# ╔═╡ 4980fc10-b163-11ea-081b-c1335699a8f6
md"## Stopping cells"

# ╔═╡ d2c1d090-b162-11ea-0c17-2b234c098cf9
md"# CodeMirror"

# ╔═╡ d890a190-b162-11ea-31dd-8d603787e5c5
md"## Autocomplete"

# ╔═╡ e141f910-b162-11ea-039b-3ba1414cbd07

# ╔═╡ c7a1f21e-c137-11ea-1d11-8fb5508ac80c
md"## Selections"

# ╔═╡ 39724bae-b160-11ea-0af5-737755a57e48
md"# Network"

# ╔═╡ f33dc5b0-b160-11ea-15e3-1b54697b5872

# ╔═╡ 95492630-b15f-11ea-3f16-410fe87cd32c
md"# Errors"

# ╔═╡ c1c894f0-b166-11ea-31e0-dd1cddb7b7ca
md"## Multiple definitions"

# ╔═╡ d05d7df0-b166-11ea-2d8f-39b31a42890d
1
2

# ╔═╡ bdeba8e0-b166-11ea-3913-3138ab428083
md"## Stack traces"

# ╔═╡ 84888e20-b160-11ea-1d61-c5934251d6dd
html"

"

# ╔═╡ ea3f77f0-b166-11ea-046e-ef39bfc57d0f
md"## Bad errors"

# ╔═╡ ee92a570-b166-11ea-38fa-0f3563b3f790
TODO: add that error showing error to the Julia tests

# ╔═╡ 4f32c16e-b15e-11ea-314e-01700baeb92d
md"# Bonds"

# ╔═╡ 3a14b3f0-b165-11ea-153d-796416ee5ccc
md"## Lossy"

# ╔═╡ 59966a90-b163-11ea-1786-e56e45f06dd0
md"## Recursive"

# ╔═╡ 55bade10-b163-11ea-327f-132c93471713
md"## Scrolling"

# ╔═╡ 431d17c0-cfff-11ea-39b5-394b34438544
md"### `text/html`"

# ╔═╡ 32b5edc0-b15d-11ea-09d6-3b889f6d397a
md"# Rich display

## `image/svg+xml` and `image/jpeg`"

# ╔═╡ 3be84600-b166-11ea-1d24-59212363543f
md"## `text/plain`"

# ╔═╡ 95898660-b166-11ea-1db1-df7f3c4f1353
"I am not bold"

# ╔═╡ 2859a890-b161-11ea-14e9-b7ddaf08195a
md"## Tree view"

# ╔═╡ 88bd7aae-b15f-11ea-270e-ab00e6a01203
["asdf", "I am not bold"]

# ╔═╡ 479df620-b161-11ea-0d2b-293fac4e46bb
md"## Markdown"

# ╔═╡ 57a82490-b162-11ea-0fdb-73c57b08830f
md"Writing a notebook is not just about writing the final document — Pluto empowers the experiments and discoveries that are essential to getting there.

Writing a notebook is not just about writing the final document — Pluto empowers the experiments and discoveries that are essential to getting there."

# ╔═╡ 5dbbc6c0-b162-11ea-0890-7f0e994b2691
md"Writing a notebook is not just about writing the final document — Pluto empowers the experiments and discoveries that are essential to getting there."

# ╔═╡ e9a68626-d04c-11ea-35ae-cb157d97fdf7
md"## CSS defaults"

# ╔═╡ 2afb1380-d04d-11ea-27e6-6733f186032a
md"that are essential to **bold getting** there.

that are essential to _italics getting_ there.

that are essential to _**bold italics getting**_ there.

that are essential to [linky](asfdfds) getting there.

that are essential to `Base.get` getting there.

that are essential to **`Base.bold`** getting there.

that are essential to _`Base.italics`_ getting there.

that are essential to _**`Base.bitald`**_ getting there.

"

# ╔═╡ 539c6442-d04d-11ea-2b5d-8faeb152a6fb
md"""
### Heading 3 - hello **bold** but also _italics_ and _**both**_
Writing a notebook is not just about writing the final document — Pluto empowers the experiments and discoveries that are essential to getting there.
#### Heading 4 - hello **bold** but also _italics_ and _**both**_
Writing a notebook is not just about writing the final document — Pluto empowers the experiments and discoveries that are essential to getting there.
##### Heading 5 - hello **bold** but also _italics_ and _**both**_
Writing a notebook is not just about writing the final document — Pluto empowers the experiments and discoveries that are essential to getting there.
###### Heading 6 - hello **bold** but also _italics_ and _**both**_
Writing a notebook is not just about writing the final document — Pluto empowers the experiments and discoveries that are essential to getting there.
"""

# ╔═╡ fcf2e354-d04d-11ea-1340-1d6211796d5e
md"""
### Heading 3 - `hey` hello **`bold`** but also _`italics`_ and _**`both`**_
Writing a notebook is not just about writing the final document — Pluto empowers the experiments and discoveries that are essential to getting there.
#### Heading 4 - `hey` hello **`bold`** but also _`italics`_ and _**`both`**_
Writing a notebook is not just about writing the final document — Pluto empowers the experiments and discoveries that are essential to getting there.
##### Heading 5 - `hey` hello **`bold`** but also _`italics`_ and _**`both`**_
Writing a notebook is not just about writing the final document — Pluto empowers the experiments and discoveries that are essential to getting there.
###### Heading 6 - `hey` hello **`bold`** but also _`italics`_ and _**`both`**_
Writing a notebook is not just about writing the final document — Pluto empowers the experiments and discoveries that are essential to getting there.
"""

# ╔═╡ e8c7abe6-d050-11ea-3999-e3f1d4994c0c
md"Hey I am referencing [^1], [^two], [^Raa97] and [^another_one_here].

[^another_one_here]: Heyy"

# ╔═╡ 95de7fac-d04d-11ea-0d83-f1b05c9bd972
md"""
> **Blockquote:** Writing a notebook is not just about writing the final document — Pluto empowers the experiments and discoveries that are essential to getting there.
"""

# ╔═╡ 33b2934e-d04e-11ea-0f18-0dfec350bb5e
md"""Here is some generic code:
```
Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

with one very long line that should wrap on spaces with one very long line that should wrap on spaces with one very long line that should wrap on spaces with one very long line that should wrap

andoneverylonglinethathasnospacesandoneverylonglinethathasnospacesandoneverylonglinethathasnospacesandoneverylonglinethathasnospacesandoneverylong
```
And this is Julia code:
```julia
if (isfile(path))
    # 4 spaces
    return launch_notebook_response(path, title="Failed to load notebook", advice="The file <code>$(htmlesc(path))</code> could not be loaded. Please <a href='https://github.com/fonsp/Pluto.jl/issues'>report this error</a>!")
else
	# 1 tab
	return error_response(404, "Can't find a file here", "Please check whether <code>$(htmlesc(path))</code> exists.")
end
```

"""

# ╔═╡ a70fc678-d04d-11ea-0390-9705003cd285
md"---"

# ╔═╡ 29565386-d04e-11ea-3a40-47639d1fc22f
md"---
---"

# ╔═╡ 3804ce76-d68e-11ea-1641-b9d5e4d9bace
md"Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic

---

typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum."

# ╔═╡ 2ee00948-d04e-11ea-3413-e5fb7dd2a25e
md"""
| Syntax      | Description | Test Text     |
| :---        |    :----:   |          ---: |
| Header      | Title       | Here's this   |
| Paragraph   | Text        | And more      |
"""

# ╔═╡ f0e92382-d682-11ea-3ede-65cd0f124ea4
md"Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum."

# ╔═╡ ac82922e-d04e-11ea-0bac-8530cacc5c72
md"""
Stage | Direct Products | ATP Yields
----: | --------------: | ---------:
Glycolysis | 2 ATP ||
^^ | 2 NADH | 3--5 ATP |
Pyruvaye oxidation | 2 NADH | 5 ATP |
Citric acid cycle | 2 ATP ||
^^ | 6 NADH | 15 ATP |
^^ | 2 FADH2 | 3 ATP |
**30--32** ATP |||
[Net ATP yields per hexose]
"""

# ╔═╡ ee43c1b6-d04f-11ea-3a3d-1384ff255535
md"""
!!! tip
    asdf

!!! note "Note - Wowie!"

    asdf

!!! info
	I don't work?

!!! warning

    asdf

    line 2

!!! danger

    asdf

    line 2

"""

# ╔═╡ 17f53606-d051-11ea-3183-b3a4cb7180db
md"### Footnote defs

asdfdasd

[^1]: adsf
[^two]:
    asdfasfd

    line 2
[^thrice]:
    Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.

    line 2"

# ╔═╡ 46fc284a-d682-11ea-34b6-69874efcaf65
md"### Text wrapping"

# ╔═╡ 1bb05fc0-b15d-11ea-3dae-7734f66a0c56
md"# Testing machinery"

# ╔═╡ 9ac925d0-b15d-11ea-2abd-7db360900be0
html"""

"""

# ╔═╡ 878a4750-b15e-11ea-2584-8feba490699f
using Test

# ╔═╡ 7370dcc0-b15e-11ea-234b-23584c864b61
ma = 1

# ╔═╡ 75b21a30-b15e-11ea-3046-2170ec097e63
mb = 2

# ╔═╡ 7b74dd40-b15e-11ea-291a-d7e10a185718
@test ma + mb == 3

# ╔═╡ 9dc4a0a0-b15f-11ea-361c-87742cf3f2a2
function ef(x)

	sqrt(-x)
end

# ╔═╡ aab109c0-b15f-11ea-275d-31e21fcda8c4
ef(1)

# ╔═╡ 976bc2a0-b160-11ea-3e7a-9f033b0f2daf
function eg(x)

	sqrt(-x)
end

# ╔═╡ 9c74f9b2-b160-11ea-35fb-917cb1120f5b
eg(1)

# ╔═╡ 41a75500-b165-11ea-2519-bbd0feaef6cf
@bind bl1 html"<input type='range' max='100000'>"

# ╔═╡ 4ccbf670-b165-11ea-1951-c17ffb8a58cf
sleep(.5); bl1

# ╔═╡ 8bb26902-b165-11ea-048c-d7f7a72006ee
@assert bl1 isa Int64

# ╔═╡ e559eaf0-b165-11ea-0d81-ffc480afe8f3
@bind bl2 html"<input type='range' max='100000'>"

# ╔═╡ e63be680-b165-11ea-0fd3-bd4e0bf92eb8
bl2

# ╔═╡ f2c0bb90-b162-11ea-24a1-3f864a09e5ee
@bind bw1 html"<input type='range' value='0'>"

# ╔═╡ a4d4ac28-cfff-11ea-3f14-15d2928d2c88
zeros((bw1, bw1))

# ╔═╡ 56e6f440-b15e-11ea-1327-09932af5b5bd
HTML("

")

# ╔═╡ 2296ac80-b163-11ea-3d00-ed366fa9ce3e
@bind bw2 html"<input type='range' value='0'>"

# ╔═╡ 20d72230-b163-11ea-39c2-69bf2c422d50
HTML("

")

# ╔═╡ 55d116d6-cfff-11ea-25fc-056ce62c8bcd
zeros((bw2, bw2))

# ╔═╡ 76c98394-cfff-11ea-0b6c-25260a8a3bb9
zeros((10,10));

# ╔═╡ 52cb1264-d824-11ea-332a-55964f3d8b90
begin
	struct A end
	struct B end

	function Base.show(io::IO, ::MIME"image/svg+xml", x::A)
		write(io, read(download("https://raw.githubusercontent.com/fonsp/Pluto.jl/master/frontend/img/logo.svg")))
	end
	function Base.show(io::IO, ::MIME"image/jpg", x::B)
		write(io, read(download("https://fonsp.com/img/doggoSmall.jpg?raw=true")))
	end
	nothing
end

# ╔═╡ 5d59acfe-d824-11ea-1d7b-07551a2b11d4
A()

# ╔═╡ 64d929aa-d824-11ea-2cc1-835fbe38be11
B()

# ╔═╡ 661c112e-d824-11ea-3612-4104449c409e
[A(), B()]

# ╔═╡ 42f0a872-b166-11ea-0c71-355d62f67fca
ra = 1:100

# ╔═╡ 794bc212-b166-11ea-0840-fddb29190841
1:13

# ╔═╡ 23f41dd2-b15c-11ea-17d2-45b3e83093ba
Ref(Dict(:a => [1,md"![](https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/PDS_70.jpg/567px-PDS_70.jpg)", md"# Hello"], [3,4] => :b))

# ╔═╡ 4d452956-d682-11ea-3aeb-cd7d1b2f67dc
s="12345678012345678012345678012345678012345678012345678012345678012345678012345678012345678012345678012345678056780123456780123456780123456780123456780123456780123456780123456780123456780120123456780\n\n\"\"\n\n5678012345678012

7801234567801234567801234567 7801234567801234567801234567 7801234567801234567801234567 7801234567801234567801234567 7801234567801234567801234567

👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👦👩👩👧👩👩👧👩👩👧👩👩👧👩👩👧👩👩👧👩👩👧👩👩👧👩👩👧👩👩👧👩👩👧❤❤❤✔

Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum."

# ╔═╡ 4e320206-d682-11ea-3dfe-b77f6e96f33a
Text(s)

# ╔═╡ 7e2cc6c0-b15d-11ea-32b0-15394cdebd35
function ask(kind, str::Markdown.MD)
	HTML("
" * sprint(show, MIME"text/html"(), str) * "
")
end

# ╔═╡ ce7f9d90-b163-11ea-0ff7-cf7378644741
ask("visual", md"During launch: 👆👆 Header should be BLUE and say LOADING")

# ╔═╡ 11f3ae90-b164-11ea-1027-0d2e6e7048dd
ask("visual", md"Hover shows path outlined and RENAME button")

# ╔═╡ 26d2b310-b164-11ea-0029-f7b04ee5ba73
ask("manual", md"Autocomplete shows on every keystroke")

# ╔═╡ 1c6229f0-b165-11ea-0f1d-674022c43971
ask("manual", md"Autocompleting a **directory** using ENTER does what you expect")

# ╔═╡ 427e4980-b164-11ea-3604-69e5af4ad390
ask("manual", md"Delete `jl` from the end, it gives **one** suggestion: the original path, with a sheet-of-paper icon")

# ╔═╡ 6e04dd80-b164-11ea-0b33-7dc165574a35
ask("manual", md"Delete `?.jl` (4 charachters) from the end, it gives **two** suggestions:
- the original path, with a sheet-of-paper icon
- a new suggested path, which ends with `.jl (new)` and has a different icon

1. Click RENAME
1. _There is a confirmation pop-up_
1. Click CANCEL
1. _The path changes back to the original_
1. Change the name again
1. Press `ENTER`
1. _There is a confirmation pop-up_
1. Click OK
1. _Path remains the same_
1. _The notebook is LOADING for a second_
")

# ╔═╡ 5fd66630-b15e-11ea-29b3-5fdf839d105b
ask("visual", md"Fold symbol should follow page scroll for long cells")

# ╔═╡ 9b26e0c0-b15e-11ea-1fbb-a34d46c520fa
ask("manual", md"Swap names `ma` and `mb`, then use `Ctrl+S` to run both cells. Then again, now use 'Submit all changes' button at top of notebook")

# ╔═╡ f4015940-b15f-11ea-0f3a-1714c79023c3
ask("manual", md"Move this cell:
- before itself (should not change pos)
- after itself (should not change pos)
- somewhere else

Drag to end of the screen, **should autoscroll**")

# ╔═╡ 1b569b72-b167-11ea-1462-63674f7e13db
ask("manual", md"Test autocomplete")

# ╔═╡ cd3f70fc-c137-11ea-2871-936287684d2c
ask("manual", md"Select a chunk of code, then put your cursor in another cell

Selection of the original cell should collapse")

# ╔═╡ f7297d72-c137-11ea-3a9b-a10be62c3976
ask("manual", md"Put your cursor in this cell, scroll down 2 screens, and click somewhere else

Page should not jump back to this cell")

# ╔═╡ 4902abb2-b160-11ea-1120-f501bf151fc2
ask("manual", md"Refresh quickly & slowly a couple of times")

# ╔═╡ 58721630-b160-11ea-30b4-e9667b4df6da
ask("manual", md"Change UUID in URL to something invalid - should redirect to `/`")

# ╔═╡ c8595660-b166-11ea-0b6f-31d05ec9f51f
ask("manual", md"Click on the error")

# ╔═╡ e4a7c0b0-b15f-11ea-2eff-ad120d8cb859
ask("manual", md"Click both yellow items in the stack trace, the relevant line should be selected and scrolled into view")

# ╔═╡ ba734f30-b15f-11ea-0490-a55c3739e4bf
ask("visual", md"Stack trace should have **5** frames and they should look nice

The last **2** should have yellow 'filenames'")

# ╔═╡ a4f07420-b160-11ea-2c62-9115046e4e24
ask("manual broken", md"The `eg` cell should get unfolded when the stack frame is clicked")

# ╔═╡ 5f37b100-b165-11ea-2f65-158f1392aea9
ask("manual", md"Move the slider, the second cell should **not** execute on intermediate values. It should **not** be executing 500ms after releasing the slider.")

# ╔═╡ b3047b10-b165-11ea-014e-25d14d54558c
ask("manual", md"1. Move the slider
1. Uncomment the second cell and run
1. _The second cell has the slider value_
1. Move the slider
1. _Second cell updates_")

# ╔═╡ 0d1ca080-b163-11ea-2927-7b46d602f837
ask("manual", md"Moving slider should **not scroll** window")

# ╔═╡ 0c499910-b163-11ea-3fee-c5d006a97ff7
ask("manual", md"Moving slider should **scroll** window to **keep this message at a fixed position**")

# ╔═╡ b5cf05fa-cfff-11ea-2c43-6748c5d90a1e
ask("manual", md"**Delete** semicolon below, then **add** semicolon back. CodeMirror should stay fixed on screen.")

# ╔═╡ 6f4ba610-d824-11ea-2e8f-a1ec77bcaad3
ask("visual", md"**Pluto logo** and **dogs** visible")

# ╔═╡ 7fc898f4-d824-11ea-0edd-ff2084f0652f
ask("visual", md"**Pluto logo** and **dogs** visible")

# ╔═╡ 486c7770-b166-11ea-22df-b38b69fb51ad
ask("visual", md"Shows assignee `ra`")

# ╔═╡ 5f28c770-b166-11ea-3099-afb5ec07119b
ask("visual", md"Output can be scrolled horizontally")

# ╔═╡ 80e7b1f0-b166-11ea-1434-ff74883f762d
ask("visual", md"No scrollbar")

# ╔═╡ 893d7bf2-b166-11ea-1580-63694795488b
ask("visual", md"HTML tags are visible")

# ╔═╡ de56f5d2-b15c-11ea-320b-31b4da177c3a
ask("manual", md"I must be an expandable tree")

# ╔═╡ f3e429be-b15e-11ea-2a93-4516283aa4c0
ask("visual", md"Element alignment should be on baseline")

# ╔═╡ 4096de22-b15f-11ea-0178-17a2ac624e96
ask("visual broken", md"HTML should be escaped")

# ╔═╡ 9df64d10-b161-11ea-2c8c-ff6f0b978cc2
ask("visual", md"Tests visibile inside the following Markdown")

# ╔═╡ 78ef6e12-b162-11ea-1474-b3e749a7b705
ask("visual", md"Assignment to `低调又牛逼的类型系统` must be visible")

# ╔═╡ e16977de-b15a-11ea-3cbd-35af9445d31d
低调又牛逼的类型系统 = md"""
 $(ask("visual", md"Heading in simplified Chinese:"))

#### 低调又牛逼的类型系统

 $(ask("visual", md"Code and LaTeX math inline"))

 比如，`sin.(A)` 会计算 A 中每个元素的 $\sin(A)$ 值

 $(ask("visual", md"Bigger and centered LaTeX math"))

$$\frac{\pi^2}{2}$$

 $(ask("visual broken", md"Heading & text in Arabic displayed right-to-left"))

## الكَوكَب

يظن بأن كوكب الأرض هو مركز الكون

 $(ask("visual", md"Greek"))

Πλθτο ισ εεν πλανεετ! Some maths:

 $(ask("visual broken", md"Identical symbols:"))

$$\pi = π$$

 $(ask("visual", md"Image **should not** fill page when widescreen, image **should** fill page when narrow screen"))

![](https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/PDS_70.jpg/567px-PDS_70.jpg)"""

# ╔═╡ 21bd9950-b15d-11ea-2632-41b1c66563bd
ask("visual", md"These three paragraphs must have equal spacing between them")

# ╔═╡ Cell order:
# ╟─ce7f9d90-b163-11ea-0ff7-cf7378644741
# ╠═878a4750-b15e-11ea-2584-8feba490699f
# ╟─fd0763a0-b163-11ea-23b4-a7bae7052e19
# ╟─11f3ae90-b164-11ea-1027-0d2e6e7048dd
# ╟─26d2b310-b164-11ea-0029-f7b04ee5ba73
# ╟─1c6229f0-b165-11ea-0f1d-674022c43971
# ╟─427e4980-b164-11ea-3604-69e5af4ad390
# ╟─6e04dd80-b164-11ea-0b33-7dc165574a35
# ╟─592d3ac0-b15e-11ea-313c-db05bc7303cc
# ╟─5fd66630-b15e-11ea-29b3-5fdf839d105b
# ╟─6dde0352-b15e-11ea-2fa8-7327cc366c1a
# ╟─9b26e0c0-b15e-11ea-1fbb-a34d46c520fa
# ╠═7370dcc0-b15e-11ea-234b-23584c864b61
# ╠═75b21a30-b15e-11ea-3046-2170ec097e63
# ╠═7b74dd40-b15e-11ea-291a-d7e10a185718
# ╠═f0b821b0-b15f-11ea-1f64-dd33aa85b54e
# ╟─f4015940-b15f-11ea-0f3a-1714c79023c3
# ╟─4980fc10-b163-11ea-081b-c1335699a8f6
# ╟─d2c1d090-b162-11ea-0c17-2b234c098cf9
# ╟─d890a190-b162-11ea-31dd-8d603787e5c5
# ╟─1b569b72-b167-11ea-1462-63674f7e13db
# ╠═e141f910-b162-11ea-039b-3ba1414cbd07
# ╟─c7a1f21e-c137-11ea-1d11-8fb5508ac80c
# ╠═cd3f70fc-c137-11ea-2871-936287684d2c
# ╠═f7297d72-c137-11ea-3a9b-a10be62c3976
# ╟─39724bae-b160-11ea-0af5-737755a57e48
# ╟─4902abb2-b160-11ea-1120-f501bf151fc2
# ╟─58721630-b160-11ea-30b4-e9667b4df6da
# ╠═f33dc5b0-b160-11ea-15e3-1b54697b5872
# ╟─95492630-b15f-11ea-3f16-410fe87cd32c
# ╟─c1c894f0-b166-11ea-31e0-dd1cddb7b7ca
# ╟─c8595660-b166-11ea-0b6f-31d05ec9f51f
# ╠═d05d7df0-b166-11ea-2d8f-39b31a42890d
# ╟─bdeba8e0-b166-11ea-3913-3138ab428083
# ╟─e4a7c0b0-b15f-11ea-2eff-ad120d8cb859
# ╟─ba734f30-b15f-11ea-0490-a55c3739e4bf
# ╠═aab109c0-b15f-11ea-275d-31e21fcda8c4
# ╟─84888e20-b160-11ea-1d61-c5934251d6dd
# ╠═9dc4a0a0-b15f-11ea-361c-87742cf3f2a2
# ╟─a4f07420-b160-11ea-2c62-9115046e4e24
# ╟─976bc2a0-b160-11ea-3e7a-9f033b0f2daf
# ╠═9c74f9b2-b160-11ea-35fb-917cb1120f5b
# ╟─ea3f77f0-b166-11ea-046e-ef39bfc57d0f
# ╠═ee92a570-b166-11ea-38fa-0f3563b3f790
# ╟─4f32c16e-b15e-11ea-314e-01700baeb92d
# ╟─3a14b3f0-b165-11ea-153d-796416ee5ccc
# ╟─5f37b100-b165-11ea-2f65-158f1392aea9
# ╠═41a75500-b165-11ea-2519-bbd0feaef6cf
# ╠═4ccbf670-b165-11ea-1951-c17ffb8a58cf
# ╠═8bb26902-b165-11ea-048c-d7f7a72006ee
# ╟─b3047b10-b165-11ea-014e-25d14d54558c
# ╠═e559eaf0-b165-11ea-0d81-ffc480afe8f3
# ╠═e63be680-b165-11ea-0fd3-bd4e0bf92eb8
# ╟─59966a90-b163-11ea-1786-e56e45f06dd0
# ╟─55bade10-b163-11ea-327f-132c93471713
# ╟─431d17c0-cfff-11ea-39b5-394b34438544
# ╟─0d1ca080-b163-11ea-2927-7b46d602f837
# ╠═f2c0bb90-b162-11ea-24a1-3f864a09e5ee
# ╠═a4d4ac28-cfff-11ea-3f14-15d2928d2c88
# ╠═56e6f440-b15e-11ea-1327-09932af5b5bd
# ╠═20d72230-b163-11ea-39c2-69bf2c422d50
# ╠═55d116d6-cfff-11ea-25fc-056ce62c8bcd
# ╟─0c499910-b163-11ea-3fee-c5d006a97ff7
# ╠═2296ac80-b163-11ea-3d00-ed366fa9ce3e
# ╟─b5cf05fa-cfff-11ea-2c43-6748c5d90a1e
# ╠═76c98394-cfff-11ea-0b6c-25260a8a3bb9
# ╟─32b5edc0-b15d-11ea-09d6-3b889f6d397a
# ╟─52cb1264-d824-11ea-332a-55964f3d8b90
# ╟─6f4ba610-d824-11ea-2e8f-a1ec77bcaad3
# ╠═5d59acfe-d824-11ea-1d7b-07551a2b11d4
# ╠═64d929aa-d824-11ea-2cc1-835fbe38be11
# ╟─7fc898f4-d824-11ea-0edd-ff2084f0652f
# ╠═661c112e-d824-11ea-3612-4104449c409e
# ╟─3be84600-b166-11ea-1d24-59212363543f
# ╟─486c7770-b166-11ea-22df-b38b69fb51ad
# ╟─5f28c770-b166-11ea-3099-afb5ec07119b
# ╠═42f0a872-b166-11ea-0c71-355d62f67fca
# ╟─80e7b1f0-b166-11ea-1434-ff74883f762d
# ╠═794bc212-b166-11ea-0840-fddb29190841
# ╟─893d7bf2-b166-11ea-1580-63694795488b
# ╟─95898660-b166-11ea-1db1-df7f3c4f1353
# ╟─2859a890-b161-11ea-14e9-b7ddaf08195a
# ╟─de56f5d2-b15c-11ea-320b-31b4da177c3a
# ╟─f3e429be-b15e-11ea-2a93-4516283aa4c0
# ╟─23f41dd2-b15c-11ea-17d2-45b3e83093ba
# ╟─4096de22-b15f-11ea-0178-17a2ac624e96
# ╠═88bd7aae-b15f-11ea-270e-ab00e6a01203
# ╟─479df620-b161-11ea-0d2b-293fac4e46bb
# ╟─9df64d10-b161-11ea-2c8c-ff6f0b978cc2
# ╟─78ef6e12-b162-11ea-1474-b3e749a7b705
# ╟─e16977de-b15a-11ea-3cbd-35af9445d31d
# ╟─21bd9950-b15d-11ea-2632-41b1c66563bd
# ╟─57a82490-b162-11ea-0fdb-73c57b08830f
# ╟─5dbbc6c0-b162-11ea-0890-7f0e994b2691
# ╟─e9a68626-d04c-11ea-35ae-cb157d97fdf7
# ╟─2afb1380-d04d-11ea-27e6-6733f186032a
# ╟─539c6442-d04d-11ea-2b5d-8faeb152a6fb
# ╠═fcf2e354-d04d-11ea-1340-1d6211796d5e
# ╟─e8c7abe6-d050-11ea-3999-e3f1d4994c0c
# ╟─95de7fac-d04d-11ea-0d83-f1b05c9bd972
# ╟─33b2934e-d04e-11ea-0f18-0dfec350bb5e
# ╠═a70fc678-d04d-11ea-0390-9705003cd285
# ╠═29565386-d04e-11ea-3a40-47639d1fc22f
# ╠═3804ce76-d68e-11ea-1641-b9d5e4d9bace
# ╟─2ee00948-d04e-11ea-3413-e5fb7dd2a25e
# ╟─f0e92382-d682-11ea-3ede-65cd0f124ea4
# ╟─ac82922e-d04e-11ea-0bac-8530cacc5c72
# ╠═ee43c1b6-d04f-11ea-3a3d-1384ff255535
# ╟─17f53606-d051-11ea-3183-b3a4cb7180db
# ╟─46fc284a-d682-11ea-34b6-69874efcaf65
# ╟─4d452956-d682-11ea-3aeb-cd7d1b2f67dc
# ╠═4e320206-d682-11ea-3dfe-b77f6e96f33a
# ╟─1bb05fc0-b15d-11ea-3dae-7734f66a0c56
# ╠═9ac925d0-b15d-11ea-2abd-7db360900be0
# ╠═7e2cc6c0-b15d-11ea-32b0-15394cdebd35


text:
# ------------------------------------------------------------
# Motivation

# Project requirements
# - 22G raw data + runtime overhead
# - as many cpu's as possible
#
# Consider:
# - AWS c5.4xlarge with 16 vCPU, 32 GB RAM

# Valuation matrix:  dims = (100000, 10000, 3)
#     for 100K securities
#          10K monte carlo states
#            3 return attributes
#
# Total memory footprint (GiB)
#     julia> 100_000 * 10_000 * 3 * 8 / 1024 / 1024 / 1024
#     22.351741790771484

# ----------------------------------------------------------------
# Generate test data
# ----------------------------------------------------------------
function locate_file(index)
    id = index - 1
    dir = string(id % 100)
    joinpath(dir, "sec$(id).dat")
end

function make_data_directories(folder)
    for i in 0:99
        mkpath(joinpath(folder, string(i)))
    end
end

function generate_test_data(nfiles)
    for i in 1:nfiles
        A = rand(10000, 3)
        file = locate_file(i)
        open(file, "w") do io
            write(io, A)
        end
    end
end

# put test data in a separate directory
folder = joinpath(ENV["HOME"], "data", "julia_book_ch06_data")
make_data_directories(folder)
cd(folder)
generate_test_data(100_000)

# data directory
#=
$ ls
0   13  18  22  27  31  36  40  45  5   54  59  63  68  72  77  81  86  90  95
1   14  19  23  28  32  37  41  46  50  55  6   64  69  73  78  82  87  91  96
10  15  2   24  29  33  38  42  47  51  56  60  65  7   74  79  83  88  92  97
11  16  20  25  3   34  39  43  48  52  57  61  66  70  75  8   84  89  93  98
12  17  21  26  30  35  4   44  49  53  58  62  67  71  76  80  85  9   94  99

$ ls -l 0 | tail -6
-rw-rw-r-- 1 ec2-user ec2-user 240000 Jul  9 07:05 sec99400.dat
-rw-rw-r-- 1 ec2-user ec2-user 240000 Jul  9 07:05 sec99500.dat
-rw-rw-r-- 1 ec2-user ec2-user 240000 Jul  9 07:05 sec99600.dat
-rw-rw-r-- 1 ec2-user ec2-user 240000 Jul  9 07:05 sec99700.dat
-rw-rw-r-- 1 ec2-user ec2-user 240000 Jul  9 07:05 sec99800.dat
-rw-rw-r-- 1 ec2-user ec2-user 240000 Jul  9 07:05 sec99900.dat
=#

# ----------------------------------------------------------------
# Fast read into memory
# ----------------------------------------------------------------

# From unix command line, use the `-p` option to start child processes.
#     $ julia -p 16

using Distributed
using SharedArrays

# Read a single data file into a segment of the shared array `dest`
# The segment size is specified as in `dims`.
@everywhere function read_val_file!(index, dest)
    filename = locate_file(index)
    (nstates, nattrs) = size(dest)[1:2]
    open(filename) do io
        nbytes = nstates * nattrs * 8
        buffer = read(io, nbytes)
        A = reinterpret(Float64, buffer)
        dest[:, :, index] = A
    end
end

# obtain path to data file
@everywhere function locate_file(index)
    id = index - 1
    dir = string(id % 100)
    joinpath(dir, "sec$(id).dat")
end

# main program
function load_data!(nfiles, dest)
    @sync @distributed for i in 1:nfiles
        read_val_file!(i, dest)
    end
end

# main program
@everywhere cd(joinpath(ENV["HOME"], "data", "julia_book_ch06_data"))
nfiles  = 50_000;
nstates = 10_000;
nattr   = 3;
valuation = SharedArray{Float64}(nstates, nattr, nfiles);

@time load_data!(nfiles, valuation);

# It took about 3 minutes to load data 100,000 files
#=
julia> @time load_data!(nfiles, valuation);
180.975685 seconds (1.49 M allocations: 76.647 MiB, 0.01% gc time)
=#