text:
module ProfileView

using Profile, UUIDs
using Colors

import Base: isequal, show

include("tree.jl")
include("pvtree.jl")

using .Tree
using .PVTree

include("svgwriter.jl")

struct TagData
    ip::UInt
    status::Int
end
const TAGNONE = TagData(UInt(0), -1)

mutable struct ProfileData
    img
    lidict
    imgtags
    fontsize
end

const bkg = colorant"white"
const fontcolor = colorant"black"
const gccolor = colorant"red"
const colors = distinguishable_colors(13, [bkg,fontcolor,gccolor],
                                      lchoices=Float64[65, 70, 75, 80],
                                      cchoices=Float64[0, 50, 60, 70],
                                      hchoices=range(0, stop=330, length=24))[4:end]

function have_display()
    !Sys.isunix() && return true
    Sys.isapple() && return true
    return haskey(ENV, "DISPLAY")
end

function __init__()
    if (isdefined(Main, :IJulia) && !isdefined(Main, :PROFILEVIEW_USEGTK)) || !have_display()
        # @eval import ProfileViewSVG
        include(joinpath(@__DIR__, "ProfileViewSVG.jl"))
        @eval import .ProfileViewSVG
        @eval begin
            view(data = Profile.fetch(); C = false, lidict = nothing, colorgc = true, fontsize = 12, combine = true, pruned = []) = ProfileViewSVG.view(data; C=C, lidict=lidict, colorgc=colorgc, fontsize=fontsize, combine=combine, pruned=pruned)
        end
    else
        # @eval import ProfileViewGtk
        include(joinpath(@__DIR__, "ProfileViewGtk.jl"))
        @eval import .ProfileViewGtk
        @eval begin
            view(data = Profile.fetch(); C = false, lidict = nothing, colorgc = true, fontsize = 12, combine = true, pruned = []) = ProfileViewGtk.view(data; C=C, lidict=lidict, colorgc=colorgc, fontsize=fontsize, combine=combine, pruned=pruned)

            @doc """
    closeall()

Closes all windows opened by ProfileView.
"""
            closeall() = ProfileViewGtk.closeall()
        end
    end
end

function prepare(data; C = false, lidict = nothing, colorgc = true, combine = true, pruned = [])
    bt, uip, counts, lidict, lkup = prepare_data(data, lidict)
    prepare_image(bt, uip, counts, lidict, lkup, C, colorgc, combine, pruned)
end

function prepare_data(data, lidict)
    bt, counts = tree_aggregate(data)
    if isempty(counts)
        Profile.warning_empty()
        error("Nothing to view")
    end
    len = Int[length(x) for x in bt]
    keep = len .> 0
    if length(data) == Profile.maxlen_data()
        keep[end] = false
    end
    bt = bt[keep]
    counts = counts[keep]
    # Display has trouble with very large images. If needed, pretend
    # we took fewer samples.
    ncounts = sum(counts)
    if ncounts > 10^4
        counts = [floor(Int, c/(ncounts/10^4)) for c in counts]  # uniformly reduce the number of backtraces
        keep = counts .> 0
        counts = counts[keep]
        bt = bt[keep]
        if isempty(counts)
            error("No backtraces survived pruning.")
        end
    end
    # Do code address lookups on all unique instruction pointers
    uip = unique(vcat(bt...))
    if lidict == nothing
        lkup = Vector{StackTraces.StackFrame}[Profile.lookup(ip) for ip in uip]
        lidict = Dict(zip(uip, lkup))
    else
        lkup = [lidict[ip] for ip in uip]
    end
    bt, uip, counts, lidict, lkup
end

prepare_data(::Nothing, ::Nothing) = nothing, nothing, nothing, nothing, nothing

function prepare_image(bt, uip, counts, lidict, lkup, C, colorgc, combine,
                       pruned)
    nuip = length(uip)
    isjl = Dict(zip(uip, [all(x->!x.from_c, l) for l in lkup]))
    isgc = Dict(zip(uip, [any(is_noninferrable_call, l) for l in lkup]))
    isjl[UInt(0)] = false  # needed for root below
    isgc[UInt(0)] = false
    p = Profile.liperm(map(first, lkup))
    rank = similar(p)
    rank[p] = 1:length(p)
    ip2so = Dict(zip(uip, rank))
    so2ip = Dict(zip(rank, uip))
    # Build the graph
    level = 0
    w = sum(counts)
    root = Tree.Node(PVData(1:w))
    PVTree.buildgraph!(root, bt, counts, 0, ip2so, so2ip, lidict)
    PVTree.setstatus!(root, isgc)
#     Tree.showedges(STDOUT, root, x -> string(get(lidict, x.ip, "root"), ", hspan = ", x.hspan, ", status = ", x.status))
#     Tree.showedges(STDOUT, root, x -> string(get(lidict, x.ip, "root"), ", status = ", x.status))
#     Tree.showedges(STDOUT, root, x -> x.status == 0 ? nothing : string(get(lidict, x.ip, "root"), ", status = ", x.status))
#     checkidentity(ip2so, so2ip)
#     checkcontains(root, ip2so, so2ip, lidict)
#     checkstatus(root, isgc, isjl, C, lidict)
    counts = zeros(Int, length(uip))
    if !C
        pruned_ips = Set()
        pushpruned!(pruned_ips, pruned, lidict)
        PVTree.prunegraph!(root, isjl, lidict, ip2so, counts, pruned_ips)
    end
#     for ip in uip
#         println(counts[ip2so[ip]], ": ", lidict[ip])
#     end
#     if !C
#         havegc = any([isgc[ip] for ip in uip])
#         if havegc
#             @assert checkprunedgc(root, false)
#         end
#     end
#     println("\nPruned:")
#     Tree.showedges(STDOUT, root, x -> string(get(lidict, x.ip, "root"), ", status = ", x.status))
    # Generate a "tagged" image
    rowtags = Any[fill(TAGNONE, w)]
    buildtags!(rowtags, root, 1)
    imgtags = hcat(rowtags...)
    img = buildimg(imgtags, colors, bkg, gccolor, colorgc, combine, lidict)
    img, lidict, imgtags
end

function svgwrite(io::IO, data, lidict; C = false, colorgc = true, fontsize = 12, combine = true, pruned = [])
    img, lidict, imgtags = prepare(data, C=C, lidict=lidict, colorgc=colorgc, combine=combine, pruned=pruned)
    pd = ProfileData(img, lidict, imgtags, fontsize)
    show(io, "image/svg+xml", pd)
end
function svgwrite(filename::AbstractString, data, lidict; kwargs...)
    open(filename, "w") do file
        svgwrite(file, data, lidict; kwargs...)
    end
    nothing
end
function svgwrite(io::IO; kwargs...)
    data, lidict = Profile.retrieve()
    svgwrite(io, data, lidict; kwargs...)
end
function svgwrite(filename::AbstractString; kwargs...)
    data, lidict = Profile.retrieve()
    svgwrite(filename, data, lidict; kwargs...)
end

Base.showable(::MIME"image/svg+xml", pd::ProfileData) = true

function show(f::IO, ::MIME"image/svg+xml", pd::ProfileData)
    img = pd.img
    lidict = pd.lidict
    imgtags = pd.imgtags
    fontsize = pd.fontsize

    ncols, nrows = size(img)
    leftmargin = rightmargin = 10
    width = 1000
    topmargin = 30
    botmargin = 40
    rowheight = 15
    height = ceil(rowheight*nrows + botmargin + topmargin)
    xstep = (width - (leftmargin + rightmargin)) / ncols
    ystep = (height - (topmargin + botmargin)) / nrows
    avgcharwidth = 6  # for Verdana 12 pt font
    function eschtml(str)
        s = replace(str, '<' => "<")
        s = replace(s, '>' => ">")
        s = replace(s, '&' => "&")
        s
    end
    function printrec(f, samples, xstart, xend, y, tag, rgb)
        width = xend - xstart
        li = lidict[tag.ip]
        info = join(["$(l.func) in $(l.file):$(l.line)" for l in li], "; ")
        shortinfo = join(["$(l.func) in $(basename(string(l.file))):$(l.line)" for l in li], "; ")
        info = eschtml(info)
        shortinfo = eschtml(shortinfo)
        #if avgcharwidth*3 > width
        #    shortinfo = ""
        #elseif length(shortinfo) * avgcharwidth > width
        #    nchars = int(width/avgcharwidth)-2
        #    shortinfo = eschtml(info[1:nchars] * "..")
        #end
        red = round(Integer,255*rgb.r)
        green = round(Integer,255*rgb.g)
        blue = round(Integer,255*rgb.b)
        print(f, """<rect vector-effect="non-scaling-stroke" x="$xstart" y="$y" width="$width" height="$ystep" fill="rgb($red,$green,$blue)" rx="2" ry="2" data-shortinfo="$shortinfo" data-info="$info"/>\n""")
        #if shortinfo != ""
        println(f, """\n<text text-anchor="" x="$(xstart+4)" y="$(y+11.5)" font-size="12" font-family="Verdana" fill="rgb(0,0,0)" ></text>""")
        # end
    end

    fig_id = string("fig-", replace(string(uuid4()), "-" => ""))
    svgheader(f, fig_id, width=width, height=height)
    # rectangles are on a grid and split across multiple columns (must span similar adjacent ones together)
    for r in 1:nrows
        # top of rectangle:
        y = height - r*ystep - botmargin
        # local vars:
        prevtag = TAGNONE
        xstart = xend = 0.0
        for c in 1:ncols
            tag = imgtags[c,r]
            if prevtag == TAGNONE && prevtag != tag
                # Very first in span
                xstart = (c-1) * xstep + leftmargin
            elseif tag != prevtag && tag != TAGNONE && prevtag != TAGNONE
                # End of old span and start of new one
                xend = (c-1) * xstep + leftmargin
                samples = round(Int, (xend - xstart)/xstep)
                printrec(f, samples, xstart, xend, y, prevtag, img[c-1,r])
                xstart = xend
            elseif tag == TAGNONE && tag != prevtag
                # at end of span and start of nothing
                xend = (c-1) * xstep + leftmargin
                samples = round(Int, (xend - xstart)/xstep)
                printrec(f, samples, xstart, xend, y, prevtag, img[c-1,r])
                xstart = 0.0
            elseif c == ncols && tag != TAGNONE
                # end of span at last element of row
                xend = (c-1) * xstep + leftmargin
                samples = round(Int,(xend - xstart)/xstep)
                printrec(f, samples, xstart, xend, y, tag, img[c,r])
                xstart = 0.0
            else
                # in middle of span
            end
            prevtag = tag
        end
    end
    svgfinish(f, fig_id)
end

function buildtags!(rowtags, parent, level)
    if isleaf(parent)
        return
    end
    w = length(rowtags[1])
    if length(rowtags) < level
        push!(rowtags, fill(TAGNONE, w))
    end
    t = rowtags[level]
    for c in parent
        t[c.data.hspan] .= Ref(TagData(c.data.ip, c.data.status))
        buildtags!(rowtags, c, level+1)
    end
end

function buildimg(imgtags, colors, bkg, gccolor, colorgc::Bool, combine::Bool, lidict)
    w = size(imgtags,1)
    h = size(imgtags,2)
    img = fill(bkg, w, h)
    colorlen = round(Int, length(colors)/2)
    for j = 1:h
        coloroffset = colorlen*iseven(j)
        colorindex = 1
        lasttag = TAGNONE
        status = 0
        first = 0
        nextcolor = colors[coloroffset + colorindex]
        for i = 1:w
            t = imgtags[i,j]
            if t != TAGNONE
                if t != lasttag && (lasttag == TAGNONE || !(combine && lidict[lasttag.ip] == lidict[t.ip]))
                    if first != 0
                        colorindex = fillrow!(img, j, first:i-1, colorindex, colorlen, nextcolor, gccolor, status & colorgc)
                        nextcolor = colors[coloroffset + colorindex]
                        status = t.status
                    end
                    first = i
                    lasttag = t
                else
                    status |= t.status
                end
            else
                if first != 0
                    # We transitioned from tag->none, render the previous range
                    colorindex = fillrow!(img, j, first:i-1, colorindex, colorlen, nextcolor, gccolor, status & colorgc)
                    nextcolor = colors[coloroffset + colorindex]
                    first = 0
                    lasttag = TAGNONE
                end
            end
        end
        if first != 0
            # We got to the end of a row, render the previous range
            fillrow!(img, j, first:w, colorindex, colorlen, nextcolor, gccolor, status & colorgc)
        end
    end
    img
end

function fillrow!(img, j, rng::UnitRange{Int}, colorindex, colorlen, regcolor, gccolor, status)
    if status > 0
        img[rng,j] .= gccolor
        return colorindex
    else
        img[rng,j] .= regcolor
        return mod1(colorindex+1, colorlen)
    end
end

is_noninferrable_call(f) = f.func == :jl_invoke || f.func == :jl_apply_generic

#### Debugging code

function checkidentity(ip2so, so2ip)
    for (k,v) in ip2so
        @assert so2ip[v] == k
    end
end

function checkcontains(root, ip2so, so2ip, lidict)
    flag = contains(root, ip2so)
    if !all(flag)
        missing = findall(!flag)
        println("missing ips:")
        for i in missing
            @show i
            @show so2ip[i]
            println(lidict[so2ip[i]])
        end
        error("Internal error: the tree does not contain all ips")
    end
end

# This skips the parent, gets everything else
# (to avoid a problem with root with ip=0)
function contains(parent::Node, ip2so::Dict)
    ret = Array(Bool, 0)
    contains!(ret, parent, ip2so)
    @show length(ip2so)
    @show length(ret)
    return ret
end

function contains!(ret, parent::Node, ip2so::Dict)
    for c in parent
        indx = ip2so[c.data.ip]
        setindexsafe!(ret, indx, true)
        contains!(ret, c, ip2so)
    end
end

function setindexsafe!(a, i::Integer, val)
    if i > length(a)
        insert!(a, i, val)
    else
        a[i] = val
    end
end

function checkstatus(parent::Node, isgc::Dict, isjl::Dict, C, lidict)
    if isgc[parent.data.ip] && parent.data.status == 0
        @show lidict[parent.data.ip]
        error("gc should be set, and it isn't")
    end
    for c in parent
        checkstatus(c, isgc, isjl, C, lidict)
    end
end

function checkprunedgc(parent::Node, tf::Bool)
    tf |= parent.data.status > 0
    if !tf
        for c in parent
            tf = checkprunedgc(c, tf)
        end
    end
    tf
end

function pushpruned!(pruned_ips, pruned, lidict)
    for (ip, liv) in lidict
        for li in liv
            if (li.func, basename(string(li.file))) in pruned
                push!(pruned_ips, ip)
                break
            end
        end
    end
end

## A tree representation
# Identify and counts repetitions of all unique backtraces
function tree_aggregate(data::Vector{UInt64})
    iz = findall(iszero, data)  # find the breaks between backtraces
    treecount = Dict{Vector{UInt64},Int}()
    istart = 1
    for iend in iz
        tmp = data[iend - 1 : -1 : istart]
        treecount[tmp] = get(treecount, tmp, 0) + 1
        istart = iend + 1
    end
    bt = Vector{Vector{UInt64}}(undef, 0)
    counts = Vector{Int}(undef, 0)
    for (k, v) in treecount
        if !isempty(k)
            push!(bt, k)
            push!(counts, v)
        end
    end
    return (bt, counts)
end

end


text:
using DynamicGrids, Test, Adapt
using DynamicGrids: SimData

output = ArrayOutput(BitArray(rand(Bool, 10, 10));
    tspan=1:10,
    mask=BitArray(rand(Bool, 10, 10)),
    aux=(aux1=BitArray(rand(Bool, 10, 10)),),
)

rs1 = Ruleset(Life(); opt=NoOpt())
rs2 = Ruleset(Life(); opt=SparseOpt())
sd1 = SimData(output, rs1)
sd2 = SimData(output, rs2)
Adapt.adapt(Array, rs1)
Adapt.adapt(Array, rs2)
b_sd1 = Adapt.adapt(Array, sd1)
b_sd2 = Adapt.adapt(Array, sd2)
@test b_sd2.extent.init._default_ isa Array
@test b_sd2.extent.mask isa Array
@test b_sd2.extent.aux.aux1 isa Array
@test parent(b_sd2.grids[:_default_].source) isa Array
@test parent(b_sd2.grids[:_default_].dest) isa Array
@test parent(b_sd2.grids[:_default_].optdata.sourcestatus) isa Array
@test Adapt.adapt(Array, output)[1] isa Array

ext: jl


text:
using Test
using EnergySystemModeling

@test true


text:
"""
A basis of the classicale Hermite polynomials. These polynomials are orthogonal
on the real line `(-∞,∞)` with respect to the weight function
`w(x)=exp(-x^2)`.
"""
struct Hermite{T} <: OPS{T}
    n           ::  Int
end

Hermite(n::Int) = Hermite{Float64}(n)

similar(b::Hermite, ::Type{T}, n::Int) where {T} = Hermite{T}(n)

support(b::Hermite{T}) where {T} = DomainSets.FullSpace(T)

first_moment(b::Hermite{T}) where {T} = sqrt(T(pi))

measure(b::Hermite{T}) where {T} = HermiteWeight{T}()
interpolation_grid(b::Hermite{T}) where T = HermiteNodes{T}(length(b))
iscompatible(b::Hermite,grid::HermiteNodes) = length(b) == length(grid)
isorthogonal(dict::Hermite, measure::GaussHermite) = opsorthogonal(dict, measure)
isorthogonal(::Hermite, ::HermiteWeight) = true
issymmetric(::Hermite) = true

gauss_rule(dict::Hermite{T}) where T = GaussHermite{T}(length(dict))

# See DLMF, Table 18.9.1
# http://dlmf.nist.gov/18.9#i
rec_An(b::Hermite, n::Int) = 2
rec_Bn(b::Hermite, n::Int) = 0
rec_Cn(b::Hermite, n::Int) = 2*n

function innerproduct_native(d1::Hermite, i::PolynomialDegree, d2::Hermite, j::PolynomialDegree, measure::HermiteWeight; options...)
	T = coefficienttype(d1)
	if i == j
		sqrt(convert(T, pi)) * (1<<value(i)) * factorial(value(i))
	else
		zero(T)
	end
end

show(io::IO, b::Hermite{Float64}) = print(io, "Hermite($(length(b)))")
show(io::IO, b::Hermite{T}) where T = print(io, "Hermite{$(T)}($(length(b)))")


text:
#Written By Johnathan Bizzano

include("../NeuralNetTest.jl")

#Hyperparameters
const train_sample_size = 10
const test_sample_size = 10
const iterations = 5
const scalefactor = .8
const crossoverrate = .7
const population_size = 10
const IsVerbose = true

const InputType = Float32
const StorageType = Float32

function MNistDataSet(isTrainingData::Bool; T::Type = InputType)
    isTrainingData && return LazyDataSet(Data.Reader{T, 28 ^ 2, 1}(row ->
        DataEntry{T, 28^2, 1}(MNIST.traintensor(T, row), Array([T(MNIST.trainlabels(row))]))), 60000)
    return LazyDataSet(Data.Reader{T, 28 ^ 2, 1}(row ->
        DataEntry{T, 28^2, 1}(MNIST.testtensor(T, row), Array([T(MNIST.testlabels(row))]))), 10000)
end

function test()

    argMaxAndSubtractOne = Processor.Chain(Processor.SingleFunc(Single.Add(-1)), Processor.ArgMaxIdx())

    designer = NeuralNet.ModelDesigner(input_shape=(28, 28), inputtype=InputType, outputtype=InputType,
                                    storagetype=StorageType, output_shape=(1), error = Error.Meanabserr(),
                                    postprocessor = argMaxAndSubtractOne)

    rng_init = Initializer.RNG(-1.0:.001:1.0)

    NeuralNet.push!(designer,
            Layers.ConvolutionalLayer(28, rng_init; inputdims=(28, 28), outputdims=(28, 28), kerneldims = (3,3)),
            Layers.MaxPoolingLayer(; inputdims=(28, 28), outputdims=(28, 28), kerneldims = (2,2)))

    push!(designer, Layers.UniformNodeSet(rng_init, Nodes.Weight(; functional=Functional.∑(; dropoutprob = .2)), input_size=28^2, output_size=128))
    push!(designer, Layers.SoftmaxLayer(; size = 10))

    neuralnet = designer()
    nettrainer = trainer(neuralnet, optimizer=Optimizer.de_rand_1_bin(; population_size = population_size,
                        iterations = iterations, scalefactor = scalefactor, crossoverrate = crossoverrate, IsVerbose = IsVerbose))

    trainingData = RandomBatch(MNistDataSet(true), train_sample_size)
    testData = RandomBatch(MNistDataSet(false), test_sample_size)

    for i in 1:10
        entry = trainingData[i]
        output = neuralnet(entry.inputs)[1]
        real = entry.outputs[1]
        println("Test #$i. Actual:$real. Predicted:$output")
    end

    train!(nettrainer, trainingData; epochs=1, IsVerbose = IsVerbose, testset = testData)

    for i in 1:10
        entry = trainingData[i]
        output = neuralnet(entry.inputs)[1]
        real = entry.outputs[1]
        println("Test #$i. Actual:$real. Predicted:$output")
    end
end

test()


text:
using Gadfly
using Colors
using MLLibrary

path=joinpath(Pkg.dir("MLLibrary"),"resources","datasets")

threshold =.1
delimiter = ","
#resources = ["3percent-miscategorization","10percent-miscatergorization","abalone.data","Seperable.csv"]
resources=[
["res"=>"abalone.data" ,"proxy"=>[Dict("column"=>1,"proxy"=>["M" "F" "I"])], "indices"=>[1 2 3 4 5 6 7]]
["res"=>"Seperable.csv", "proxy"=>[],"indices"=>[1 2 3 4 5 6 7 8 9]]
["res"=>"3percent-miscategorization.csv", "proxy"=>[],"indices"=>[1 2 3 4 5 6 7 8 9]]
["res"=>"10percent-miscatergorization.csv", "proxy"=>[],"indices"=>[1 2 3 4 5 6 7 8 9]]
]
errors = Dict()

outfile = open(joinpath(Pkg.dir("MLLibrary"),"resources","paResources","Week2","ConfusionMatrix.txt"),"w")
for r  in resources

  println("Running Tests for ",r["res"])
  errors[r["res"]]=Dict()
  res = joinpath(path,r["res"])
  proxyColumns = r["proxy"]

  srand(1234)
  trainingSet = MLLibrary.DataSampling.ThresholdSampling.getTrainingSet(res,delimiter,threshold)

  srand(1234)

  testSet = MLLibrary.DataSampling.ThresholdSampling.getTestSet(res,delimiter,threshold)

  training_y = trainingSet[:,end]
  test_y = testSet[:,end]
  trainingSet=trainingSet[:,1:end-1]
  testSet=testSet[:,1:end-1]
  trainingSet=MLLibrary.DataManipulation.DataTransform.addProxyColumns(trainingSet,proxyColumns)
  testSet=MLLibrary.DataManipulation.DataTransform.addProxyColumns(testSet,proxyColumns)

  trainingSet= MLLibrary.DataManipulation.DataTransform.convertToFloat(trainingSet)
  testSet= MLLibrary.DataManipulation.DataTransform.convertToFloat(testSet)
  training_y= MLLibrary.DataManipulation.DataTransform.convertToFloat(training_y)
  test_y= MLLibrary.DataManipulation.DataTransform.convertToFloat(test_y)

  #println(r["indices"])
  for i in r["indices"]
    #println(i)
    #println(size(trainingSet))
    column= trainingSet[:,i]
    average = mean(column)

    standarddeviation=std(column)
    trainingSet = MLLibrary.DataManipulation.DataTransform.zScaleColumn(trainingSet,i,average,standarddeviation)
    testSet = MLLibrary.DataManipulation.DataTransform.zScaleColumn(testSet,i,average,standarddeviation)
  end

#println(testSet)
  println("We got our Datasets!")
  #println(testSet)
  for K in [1 3 5 7 9]
    predicted_y=MLLibrary.LearningAlgorithms.KNeighbors.KNearestNeighbors(trainingSet,testSet,training_y,K)

    confusionMatrix=MLLibrary.Utils.ErrorAnalysis.generateConfusionMatrix(test_y,predicted_y)
    errors[r["res"]][K] = 1 - MLLibrary.Utils.ErrorAnalysis.calculateWeightedSuccessRate(confusionMatrix)
    println(errors[r["res"]][K])

    #MLLibrary.Utils.ErrorAnalysis.calculateErrorRate(test_y,predicted_y)

  end

  theset = errors[r["res"]]
  Kmin = collect(keys(theset))[indmin(collect(values(theset)))]
  println(Kmin)
  optimal_y = MLLibrary.LearningAlgorithms.KNeighbors.KNearestNeighbors(trainingSet,testSet,training_y,Kmin)

  confusionmatrix =MLLibrary.Utils.ErrorAnalysis.generateConfusionMatrix(test_y,optimal_y)

  #println(size(confusionmatrix))
  write(outfile, r["res"],"\n","Minimum K: ","$Kmin", "\n","ConfusionMatrix: \n")
  largesize =size(confusionmatrix)[1]>3

  for i=1:size(confusionmatrix)[1]
    for j=1:size(confusionmatrix)[2]
      if(largesize)
        @printf(outfile,"%3d",confusionmatrix[i,j])
      else
        @printf(outfile,"%7d",confusionmatrix[i,j])
      end

    end
    @printf(outfile,"%s","\n")
  end

  write(outfile,"\n\n")

end

close(outfile)

p = plot(

  Guide.ylabel("Error Rate"),
  Guide.xlabel("K-Value"),
  Guide.title("Error Rate per K value"),
  #color="DataSet"
)
index = 1
color_keys = []
colors = distinguishable_colors(length(errors))

for (k,v) in errors
  a = layer(
    x=collect(keys(v)),
    y=collect(values(v)),
    Geom.line,
    Theme(default_color=(colors[index]))
    #Guide.colorkey(k)
  )
  push!(color_keys,k)

  index+=1
  push!(p,a)
end

push!(p,  Guide.manual_color_key("Legend", color_keys, colors))

draw(PNG(joinpath(Pkg.dir("MLLibrary"),"resources","paResources","Week2","ErrorRatesVsK.png"), 800px, 600px), p)

max_stars_repo_name: ajermaky/CustomMLLibrary

max_issues_repo_name: ajermaky/CustomMLLibrary

max_forks_repo_name: ajermaky/CustomMLLibrary


text:
using Catalan
using Base.Test

# catalan
@test catalan(5) == 42
@test catalan(30) == BigInt("3814986502092304")

# derangement
@test derangement(4) == 9
@test derangement(24) == BigInt("228250211305338670494289")

# doublefactorial
@test doublefactorial(70) == BigInt("355044260642859198243475901411974413130137600000000")

# fibonacci
@test fibonacci(5) == 5
@test fibonacci(101) == BigInt("573147844013817084101")

# hyperfactorial
@test hyperfactorial(8) == BigInt("55696437941726556979200000")

# lassalle
@test lassalle(14) == BigInt("270316008395632253340")

# legendresymbol
@test legendresymbol(1001,9907) == jacobisymbol(1001,9907)

# lucas
@test lucas(10) == 123
@test lucas(100) == BigInt("792070839848372253127")

# multifactorial
@test multifactorial(40,2) == doublefactorial(40)

# multinomial
@test multinomial(1,4,4,2) == 34650

# primorial
@test primorial(17) == 510510

# stirlings1
@test sum([abs(stirlings1(8, i)) for i = 0:8]) == factorial(8)

# bell
@test bell(7) == 877
@test bell(42) == BigInt("35742549198872617291353508656626642567")
@test_throws bell(-1)

# integer_partitions
@test integer_partitions(5) == {[1, 1, 1, 1, 1], [2, 1, 1, 1], [2, 2, 1], [3, 1, 1], [3, 2], [4, 1], [5]}
@test_throws integer_partitions(-1)

include("youngdiagrams.jl")


text:
Base.write(io::IO, ::MIME"text/plain", ::Nothing) = nothing
function Base.read(io::IO, ::MIME"text/plain", ::Nothing)
    readline(io)
    return nothing
end

Base.write(io::IO, ::MIME"text/plain", i::Integer) = print(io, i)
Base.read(io::IO, ::MIME"text/plain", ::Type{I}) where {I<:Integer} = parse(I, readline(io))

Base.write(io::IO, ::MIME"text/plain", r::Float64) = print(io, r)
Base.read(io::IO, ::MIME"text/plain", ::Type{F}) where {F<:AbstractFloat} = parse(F, readline(io))

max_stars_repo_name: tawheeler/Records

max_issues_repo_name: tawheeler/Records

max_forks_repo_name: tawheeler/Records


text:
using Lynx
using Luxor
using Colors

init("Random particles", 800, 600)

framerate!(60)

N = 100
xoff = 0.
yoff = 1000.

function update(dt)
    global xoff, yoff;
    background("#111")

    for n in 1:N
        x = noise(xoff + n) * @width
        y = noise(yoff + n) * @height
        hue = mapr(n, 1:N, 0:360)
        color = HSL(hue, .85, .65)
        sethue(color)
        circle(x, y, 5, :fill)
    end

    sethue("white")
    fontsize(14)
    fps = floor(Int, @framerate)
    text("Framerate: $(fps)fps", @width()/2, 20, halign = :center)

    xoff += 0.5 * dt
    yoff += 0.5 * dt
end

run!(update)


text:
export meshcat,
       meshcat_material,
       meshcat_glass_material,
       meshcat_metal_material
import Base.convert

#=
ATTENTION!!!

Use Chrome debugger to visualize threeJS examples, break the execution and
evaluate in the console:

JSON.stringify(scene.toJSON(), undefined, 2)

=#

## Primitives

send_meshcat(vis::Visualizer, obj) =
  write(vis.core, pack(obj))

send_setobject(vis, path, obj) =
  let msg = (type="set_object", path=path, object=obj),
      data = pack(msg)
    #vis.core.tree[path].object = data
    write(vis.core, data)
    path
  end

send_settransform(vis, path, transform) =
  let msg = (type="set_transform", path=path, matrix=convert(Vector{Float32}, transform)),
      data = pack(msg)
    #vis.core.tree[path].transform = data
    write(vis.core, data)
    nothing
  end

send_setproperty(vis, path, property, value) =
  let msg = (type="set_property", path=path, property=property, value=value),
      data = pack(msg)
    #vis.core.tree[path].properties[property] = data
    write(vis.core, data)
    nothing
  end

send_setvisible(vis, path, value) =
  send_meshcat(vis, (type="set_property", path=path, property="visible", value=value))

#send_setproperty(connection(meshcat), "/Background/hide_background", "value", true)
#send_meshcat(connection(meshcat), (type="hide_background",))

send_delobject(vis, path) =
  send_meshcat(vis, (type="delete", path=path))

meshcat_point(p::Loc) =
  let p = in_world(p)
    [cx(p),cz(p),cy(p)]
  end

meshcat_transform(p::Loc) =
  # MeshCat swaps Y with Z
  let m = translated_cs(p.cs, p.x, p.y, p.z).transform*[1 0 0 0; 0 0 1 0; 0 1 0 0; 0 0 0 1]
    m[:]
  end

meshcat_metadata(type="Object") =
  (type=type, version=4.5)

meshcat_color(color) =
  "0x$(hex(color))"

meshcat_material(color) =
  (uuid=string(uuid1()),
   type="MeshLambertMaterial",
    #"vertexColors" => 0,
    #"transparent" => false,
    #"opacity" => 1.0,
    #"depthTest" => true,
    #"linewidth" => 1.0,
    #"depthFunc" => 3,
    side=2,
    color=meshcat_color(color),
    #color="0xAAAAAA",
    #"reflectivity" => 0.5,
    #depthWrite=true
    )
meshcat_glass_material(opacity=0.5, color=RGB(0.9,0.9,1.0)) =
  (uuid=string(uuid1()),
   type="MeshPhysicalMaterial",
    #"vertexColors" => 0,
   transparent=true,
   opacity=opacity,
    #"depthTest" => true,
    #"linewidth" => 1.0,
    #"depthFunc" => 3,
   side=2,
   color=meshcat_color(color),
   reflectivity=0.1,
    #depthWrite=true
  )
meshcat_metal_material(roughness=0.5, color=RGB(1.0,1.0,1.0)) =
  (uuid=string(uuid1()),
   type="MeshStandardMaterial",
   metalness=1,
   roughness=roughness,
   side=2,
   color=meshcat_color(color),
  )

meshcat_line_material(color) =
  (uuid=string(uuid1()),
   type="LineBasicMaterial",
   color=meshcat_color(color),
   #linewidth=2, Due to limitations of the OpenGL Core Profile with the WebGL renderer on most platforms linewidth will always be 1 regardless of the set value.
   #depthFunc=3,
   #depthTest=true,
   depthWrite=true,
   #stencilWrite=false,
   #stencilWriteMask=255,
   #stencilFunc=519,
   #stencilRef=0,
   #stencilFuncMask=255,
   #stencilFail=7680,
   #stencilZFail=7680,
   #stencilZPass=7680
   )

meshcat_object(type, geom, material, p=u0(world_cs)) =
  (metadata=meshcat_metadata(),
   geometries=[geom],
   materials=[material],
   object=(uuid=string(uuid1()),
           type=type,
           geometry=geom.uuid,
           material=material.uuid,
           matrix=meshcat_transform(p)))

meshcat_object_2D(type, geom, shapes, material) =
 (metadata=meshcat_metadata(),
  shapes=shapes,
  geometries=[geom],
  materials=[material],
  object=(uuid=string(uuid1()),
          type=type,
          geometry=geom.uuid,
          material=material.uuid,
          matrix=(1, 0, 0, 0,
                  0, 1, 0, 0,
                  0, 0,-1, 0,
                  0, 0, 0, 1)))

meshcat_object_shapes(type, geom, shapes, material, p=u0(world_cs)) =
 (metadata=meshcat_metadata(),
  shapes=shapes,
  geometries=[geom],
  materials=[material],
  object=(uuid=string(uuid1()),
          type=type,
          geometry=geom.uuid,
          material=material.uuid,
          matrix=(translated_cs(p.cs, p.x, p.y, p.z).transform*[1 0 0 0; 0 1 0 0; 0 0 -1 0; 0 0 0 1])[:])) # meshcat_transform(p))) #(1, 0, 0, 0, 0, 1, 0, 0, 0, 0,-1, 0, 0, 0, 0, 1)))

meshcat_buffer_geometry_attributes_position(vertices) =
  (itemSize=3,
   type="Float32Array",
   array=convert(Vector{Float32}, reduce(vcat, meshcat_point.(vertices))))

meshcat_line(vertices, material) =
  let geom = (uuid=string(uuid1()),
              type="BufferGeometry",
              data=(
                attributes=(
                  position=meshcat_buffer_geometry_attributes_position(vertices),),))
    meshcat_object("Line", geom, material)
  end

#=
Three.js uses 2D locations and 3D locations
=#

abstract type Meshcat2D end

convert(::Type{Meshcat2D}, p::Loc) = (cx(p),cy(p))
meshcat_2d(p::Loc) = let z =  @assert(abs(cz(p)) < 1e-10); (cx(p),cy(p)) end
meshcat_3d(p::Loc) = (cx(p),cy(p),cz(p))
meshcat_line_curve_2d(v1::Loc, v2::Loc) =
  (type="LineCurve", v1=meshcat_2d(v1), v2=meshcat_2d(v2))
meshcat_line_curve_3d(v1::Loc, v2::Loc) =
  (type="LineCurve3", v1=meshcat_3d(v1), v2=meshcat_3d(v2))

#=
Three.js provides a hierarchy of curves.
Curve - Abstract
2D curves:
  ArcCurve
  CubicBezierCurve
  EllipseCurve
  LineCurve
  QuadraticBezierCurve
  SplineCurve
3D curves:
  CatmullRomCurve3
  CubicBezierCurve3
  LineCurve3
  QuadraticBezierCurve3
Sequences:
  CurvePath - Abstract
    Path
      Shape
=#
abstract type MeshcatCurve end
abstract type MeshcatCurve2D <: MeshcatCurve end
abstract type MeshcatCurve3D <: MeshcatCurve end
abstract type MeshcatCurvePath <: MeshcatCurve end
abstract type MeshcatPath <: MeshcatCurvePath end
abstract type MeshcatShape <: MeshcatPath end

abstract type MeshcatCurves end

meshcat_curve(path) = convert(MeshcatCurve, path)
convert(::Type{MeshcatCurve}, p::CircularPath) =
  (type="EllipseCurve",
   aX=cx(p.center), aY=cy(p.center),
   xRadius=p.radius, yRadius=p.radius,
   aStartAngle=0, aEndAngle=2π,
   aClockwise=false,
   aRotation=0)
convert(::Type{MeshcatCurve}, p::OpenPolygonalPath) =
  let ps = path_vertices(p)
    length(ps) == 2 ?
      meshcat_line_curve_2d(ps[1], ps[2]) :
      convert(MeshcatPath, p)
  end

meshcat_path(path) = convert(MeshcatPath, path)
convert(::Type{MeshcatPath}, path) =
  (type="Path",
   curves=meshcat_curves(path),
   autoclose=false,
   currentPoint=(0,0))

meshcat_curves(path) = convert(MeshcatCurves, path)
convert(::Type{MeshcatCurves}, path) =
  [meshcat_curve(path)]
convert(::Type{MeshcatCurves}, vs::Locs) =
  [meshcat_line_curve_2d(v1, v2)
   for (v1,v2) in zip(vs, circshift(vs, -1))]
convert(::Type{MeshcatCurves}, p::Union{RectangularPath, ClosedPolygonalPath}) =
  convert(MeshcatCurves, path_vertices(p))

meshcat_shape(path) = convert(MeshcatShape, path)
convert(::Type{MeshcatShape}, p::Region) =
  (uuid=string(uuid1()),
   type="Shape",
   curves=meshcat_curves(p.paths[1]),
   autoclose=false,
   currentPoint=(0,0),
   holes=meshcat_path.(p.paths[2:end]))
convert(::Type{MeshcatShape}, p::Path) =
  (uuid=string(uuid1()),
   type="Shape",
   curves=meshcat_curves(p),
   autoclose=false,
   currentPoint=(0,0),
   holes=[])

meshcat_surface_2d(path, material, p=u0(world_cs)) =
  let shape = meshcat_shape(path),
      geom = (uuid=string(uuid1()),
              type="ShapeBufferGeometry",
              shapes=[shape.uuid],
              curveSegments=64)
    meshcat_object_shapes("Mesh", geom, [shape], material, p)
  end

#=
backend(meshcat)
delete_all_shapes()
add_object(
  meshcat,
  meshcat_surface_polygon_2d(
    path_vertices(closed_polygonal_path(regular_polygon_vertices(5, u0(), 3))),
    [path_vertices(closed_polygonal_path(regular_polygon_vertices(4, u0(), 1)))],
  material(meshcat)))

dump(meshcat_surface_polygon_2d(
  path_vertices(closed_polygonal_path(regular_polygon_vertices(5, u0(), 3))),
  [path_vertices(closed_polygonal_path(regular_polygon_vertices(4, u0(), 1)))],
  material(meshcat)))
dump(meshcat_surface_2d(
    path_set(closed_polygonal_path(regular_polygon_vertices(5, u0(), 3)),
             closed_polygonal_path(regular_polygon_vertices(4, u0(), 1))),
    material(meshcat)))

add_object(meshcat, meshcat_surface_polygon(regular_polygon_vertices(5), material(meshcat)))
add_object(
  meshcat,
  meshcat_surface_2d(
    path_set(
      circular_path(u0(), 2),
      circular_path(u0(), 1)),
    material(meshcat)))

add_object(
 meshcat,
  meshcat_surface_2d(circular_path(u0(), 3), material(meshcat)))

add_object(
  meshcat,
  meshcat_surface_2d(
    closed_polygonal_path(regular_polygon_vertices(5, u0(), 5)),
    material(meshcat)))
add_object(
  meshcat,
  meshcat_surface_2d(
    closed_polygonal_path(regular_polygon_vertices(4, u0(), 1)),
    material(meshcat)))

add_object(
  meshcat,
  meshcat_surface_2d(
    path_set(
      closed_polygonal_path(regular_polygon_vertices(5, u0(), 3)),
      closed_polygonal_path(regular_polygon_vertices(4, u0(), 1))),
    material(meshcat)))

dump(meshcat_surface_polygon(regular_polygon_vertices(5), material(meshcat)))

dump(meshcat_surface_2d(
  path_set(
    circular_path(u0(), 5),
    circular_path(u0(), 1)),
  material(meshcat)))
=#

meshcat_surface_polygon(vertices, material) =
  let n = length(vertices)
    n <= 0 ?
      meshcat_mesh(vertices, n < 4 ? [(0,1,2)] : [(0,1,2),(2,3,0)], material) :
      let ps = in_world.(vertices),
          n = vertices_normal(ps),
          cs = cs_from_o_vz(ps[1], n),
          vs = [in_cs(v, cs) for v in vertices]
        meshcat_surface_2d(closed_polygonal_path(vs), material, u0(cs))
      end
  end

#=
reset_backend()
delete_all_shapes()
add_object(meshcat, meshcat_surface_polygon([x(1), x(3), xz(3,4), xz(1,4)], material(meshcat)))
=#

meshcat_circle_mesh(center, radius, start_angle, amplitude, material) =
  let geom = (uuid=string(uuid1()),
              type="CircleBufferGeometry",
              radius=radius,
              segments=64,
              thetaStart=start_angle,
              thetaLength=amplitude),
      cs = cs_from_o_vz(center, vx(-1, center.cs))
    meshcat_object("Mesh", geom, material, u0(cs))
  end

meshcat_centered_box(p, dx, dy, dz, material) =
  let geom = (uuid=string(uuid1()),
              type="BoxBufferGeometry",
              width=dx,
              depth=dy,
              height=dz)
    meshcat_object("Mesh", geom, material, p)
  end

meshcat_box(p, dx, dy, dz, material) =
  meshcat_centered_box(p+vxyz(dx/2, dy/2, dz/2, p.cs), dx, dy, dz, material)

#=
send_delobject(v, "/meshcat")
send_setobject(v, "/meshcat/box1", meshcat_box(xyz(1,1,1), 1, 2, 3))
send_setobject(v, "/meshcat/box2", meshcat_box(loc_from_o_vx_vy(u0(), vxy(1,1), vxy(-1,1)), 1, 2, 3))
send_delobject(v, "/meshcat/box2")
=#

meshcat_sphere(p, r, material) =
  let geom = (uuid=string(uuid1()),
              type="SphereBufferGeometry",
              radius=r,
              widthSegments=64,
              heightSegments=64)
    meshcat_object("Mesh", geom, material, p)
  end

meshcat_torus(p, re, ri, material) =
  let geom = (uuid=string(uuid1()),
              type="TorusBufferGeometry",
              radius=re,
              tube=ri,
              radialSegments=64,
              tubularSegments=32)
    meshcat_object_shapes("Mesh", geom, [], material, p)
  end

meshcat_centered_cone(p, r, h, material) =
  let geom = (uuid=string(uuid1()),
              type="ConeBufferGeometry",
              radius=r,
              height=h,
              radialSegments=64)
    meshcat_object("Mesh", geom, material, p)
  end

meshcat_cone(p, r, h, material) =
  meshcat_centered_cone(p+vz(h/2, p.cs), r, h, material)

meshcat_centered_cone_frustum(p, rb, rt, h, material) =
  let geom = (uuid=string(uuid1()),
              type="CylinderBufferGeometry",
              radiusTop=rt,
              radiusBottom=rb,
              height=h,
              radialSegments=64)
    meshcat_object("Mesh", geom, material, p)
  end

meshcat_cone_frustum(p, rb, rt, h, material) =
  meshcat_centered_cone_frustum(p+vz(h/2, p.cs), rb, rt, h, material)

meshcat_cylinder(p, r, h, material) =
  meshcat_centered_cone_frustum(p+vz(h/2, p.cs), r, r, h, material)

meshcat_extrusion_z(profile, h, material, p=u0(world_cs)) =
  let shape = meshcat_shape(profile),
      geom = (uuid=string(uuid1()),
              type="ExtrudeBufferGeometry",
              shapes=[shape.uuid],
              options=(
                #steps=2,
                depth=-h,
                bevelEnabled=false,
                #bevelThickness=1,
                #bevelSize=1,
                #bevelOffset=0,
                #bevelSegments=1,
                #extrudePath=meshcat_path(path),
                curveSegments=64
                ))
    meshcat_object_shapes("Mesh", geom, [shape], material, p)
  end

meshcat_faces(si, sj, closed_u, closed_v) =
  let idx(i,j) = (i-1)*sj+(j-1),
      idxs = [],
      quad(a,b,c,d) = (push!(idxs, (a, b, d)); push!(idxs, (d, b, c)))
    for i in 1:si-1
      for j in 1:sj-1
        quad(idx(i,j), idx(i+1,j), idx(i+1,j+1), idx(i,j+1))
      end
      if closed_v
        quad(idx(i,sj), idx(i+1,sj), idx(i+1,1), idx(i,1))
      end
    end
    if closed_u
      for j in 1:sj-1
        quad(idx(si,j), idx(1,j), idx(1,j+1), idx(si,j+1))
      end
      if closed_v
        quad(idx(si,sj), idx(1,sj), idx(1,1), idx(si,1))
      end
    end
    idxs
  end
#=
delete_all_shapes()
add_object(
  meshcat,
  meshcat_extrusion_z(
    path_set(
      circular_path(u0(), 2),
      circular_path(u0(), 1)),
    10,
    material(meshcat),
    y(10)))

delete_all_shapes()
add_object(
  meshcat,
  meshcat_extrusion(open_polygonal_path([x(0), z(10)]),
                    closed_polygonal_path(regular_polygon_vertices(5, u0(), 3)),
                    material(meshcat)))
dump(meshcat_extrusion(open_polygonal_path([x(0), z(10)]),
                  closed_polygonal_path(regular_polygon_vertices(5, u0(), 3)),
                  material(meshcat)))
=#

#=
send_setobject(v, "/meshcat/sphere1", meshcat_sphere(xyz(-2,-3,0), 1))
send_setobject(v, "/meshcat/cylinder1", meshcat_cylinder(loc_from_o_vx_vy(x(-3), vxy(1,1), vxz(-1,1)), 1, 5))
=#
meshcat_mesh(vertices, faces, material) =
  let geom = (uuid=string(uuid1()),
              type="BufferGeometry",
              data=(
                attributes=(
                  position=meshcat_buffer_geometry_attributes_position(vertices),
                  #uv=?
                  ),
                index=(
                  itemSize=3,
                  type="Uint32Array",
                  array=convert(Vector{UInt32}, collect(Iterators.flatten(faces))))))
    meshcat_object("Mesh", geom, material, u0(world_cs))
  end
#=
meshcat_extruded_surface_polygon(outer_vertices, holes_vertices, material) =
  let ps = in_world.(outer_vertices),
      n = vertices_normal(ps),
      cs = cs_from_o_vz(ps[1], n),
      vs = [in_cs(v, cs) for v in outer_vertices],
      hsvs = [[in_cs(v, cs) for v in hvs] for hvs in holes_vertices],
      curves = [(#metadata=meshcat_metadata("Curve"),
                 type="LineCurve",
                 v1=(cx(v1),cy(v1)), v2=(cx(v2),cy(v2)))
                for (v1,v2) in zip(vs, circshift(vs, -1))],
      shape = (uuid=string(uuid1()),
               type="Shape",
               curves=curves,
               autoclose=false,
               currentPoint=(cx(vs[1]), cy(vs[1])),
               holes=[],
               ),
      geom = (uuid=string(uuid1()),
              type="ShapeBufferGeometry",
              shapes=[shape.uuid],
              #curveSegments=12,
              )
    #meshcat_object_2D("Mesh", geom, [shape], material, )
    meshcat_object_shapes("Mesh", geom, [shape], material, u0(cs))
  end
=#

send_setview(v, camera::Loc, target::Loc, lens::Real, aperture::Real) =
  let (x1,y1,z1) = raw_point(camera),
      (x2,y2,z2) = raw_point(target)
    send_settransform(v, "/Cameras/default", [
      1, 0, 0, 0,
      0, 1, 0, 0,
      0, 0, 1, 0,
      x2, y2, z2, 1])
    send_setproperty(v, "/Cameras/default/rotated/<object>", "zoom", 1)
    send_setproperty(v, "/Cameras/default/rotated/<object>", "fov", view_angles(lens)[2])
    send_setproperty(v, "/Cameras/default/rotated/<object>", "position", [x1-x2,z1-z2,y2-y1])
  end

#send_setproperty(connection(meshcat), "/Orbit/<object>", "target", [2,0,0])
####################################################
abstract type MCATKey end
const MCATId = Union{String, NamedTuple} #Materials are tuples
const MCATRef = GenericRef{MCATKey, MCATId}
const MCATNativeRef = NativeRef{MCATKey, MCATId}
const MCATUnionRef = UnionRef{MCATKey, MCATId}
const MCATSubtractionRef = SubtractionRef{MCATKey, MCATId}

struct MCATLayer
  name
  material
  line_material
end

mcat_layer(name, color) =
  MCATLayer(name, meshcat_material(color), meshcat_line_material(color))

mutable struct MCATBackend{K,T} <: RemoteBackend{K,T}
  connection::Union{Missing,Visualizer}
  count::Int64
  layer::MCATLayer
  view::View
  sun_altitude::Real
  sun_azimuth::Real
end

const MCAT = MCATBackend{MCATKey, MCATId}

material(b::MCAT) = b.layer.material
line_material(b::MCAT) = b.layer.line_material

KhepriBase.after_connecting(b::MCAT) =
  begin
    set_material(b, material_point, meshcat_material(RGB(0.1,0.1,0.9)))
    set_material(b, material_curve, meshcat_material(RGB(0.9,0.1,0.1)))
    set_material(b, material_surface, meshcat_material(RGB(0.1,0.9,0.1)))
    set_material(b, material_basic, meshcat_material(RGB(0.8,0.8,0.8)))
    set_material(b, material_glass, meshcat_glass_material())
	  set_material(b, material_metal, meshcat_metal_material())
	  set_material(b, material_wood, meshcat_material(RGB(0.5,0.5,0.5)))
	  set_material(b, material_concrete, meshcat_material(RGB(0.2,0.2,0.2)))
	  set_material(b, material_plaster, meshcat_material(RGB(0.7,0.7,0.7)))
	  set_material(b, material_grass, meshcat_material(RGB(0.1,0.7,0.1)))
  end

KhepriBase.start_connection(b::MCAT) =
  let (width, height) = render_size(),
      vis = Visualizer()
    display(MCATViewer(vis))
    send_setproperty(vis, "/Cameras/default/rotated/<object>", "far", 50000)
    vis
  end

meshcat = MCAT(missing,
               0,
               mcat_layer("default", RGB(1,1,1)),
               default_view(),
               90,
               0)

KhepriBase.backend_name(b::MCAT) = "MeshCat"
#=
To visualize, we piggyback on Julia's display mechanisms
=#

display_meshcat(io, vis, (w, h)=render_size()) =
  let frame = vis.core
    print(io, """

    """)
    MeshCat.wait_for_server(frame)
  end

struct MCATViewer
  visualizer
end

Base.show(
  io::IO,
  ::Union{
    MIME"text/html",
    MIME"juliavscode/html",
    MIME"application/prs.juno.plotpane+html"},
  v::MCATViewer) = display_meshcat(io, v.visualizer)

export display_view
display_view(b=current_backend()) = error("Needs specialization")
display_view(b::MCAT) = MCATViewer(connection(b))

const meshcat_root_path = "/Khepri"

next_id(b::MCATBackend{K,T}) where {K,T} =
  begin
      b.count += 1
      string(meshcat_root_path, "/", b.layer.name, "/", b.count - 1)
  end

add_object(b::MCAT, obj) =
  send_setobject(connection(b), next_id(b), obj)

has_boolean_ops(::Type{MCAT}) = HasBooleanOps{false}()
void_ref(b::MCAT) = MCATNativeRef("")

reset_backend(b::MCAT) =
  begin
    display(render(connection(b)))
    set_view(get_view(b)...)
  end

new_backend(b::MCAT) =
  begin
    reset(b.connection)
    display_view(b)
  end
#=
#@bdef(b_point(p, mat))
=#
KhepriBase.b_line(b::MCAT, ps, mat) =
  add_object(b, meshcat_line(ps, mat))
#=
KhepriBase.b_nurbs_curve(b::MCAT, order, ps, knots, weights, closed, mat) =
  b_line(b, ps, closed, mat)

KhepriBase.b_spline(b::MCAT, ps, v1, v2, interpolator, mat) =
  let ci = curve_interpolator(ps, false),
      cpts = curve_control_points(ci),
      n = length(cpts),
      knots = curve_knots(ci)
    b_nurbs_curve(b, 5, cpts, knots, fill(1.0, n), false, mat)
  end

KhepriBase.b_closed_spline(b::MCAT, ps, mat) =
  let ci = curve_interpolator(ps, true),
      cpts = curve_control_points(ci),
      n = length(cpts),
      knots = curve_knots(ci)
    b_nurbs_curve(b, 5, cpts, knots, fill(1.0, n), true, mat)
  end

KhepriBase.b_circle(b::MCAT, c, r, mat) =
  b_closed_spline(b, regular_polygon_vertices(32, c, r, 0, true), mat)

KhepriBase.b_arc(b::MCAT, c, r, α, Δα, mat) =
  b_spline(b,
    [center + vpol(r, a, center.cs)
     for a in division(α, α + Δα, Δα*32/2/π, false)],
    nothing, nothing, # THIS NEEDS TO BE FIXED
    mat)

KhepriBase.b_rectangle(b::MCAT, c, dx, dy, mat) =
  b_polygon(b, [c, add_x(c, dx), add_xy(c, dx, dy), add_y(c, dy)], mat)

# First tier: everything is a triangle or a set of triangles
=#
KhepriBase.b_trig(b::MCAT, p1, p2, p3, mat) =
  let ps = [p1, p2, p3]
    add_object(b, meshcat_mesh(ps, [(0,1,2)], mat))
  end
KhepriBase.b_quad(b::MCAT, p1, p2, p3, p4, mat) =
  let ps = [p1, p2, p3, p4]
    add_object(b, meshcat_mesh(ps, [(0,1,2),(2,3,0)], mat))
  end

#=

KhepriBase.b_ngon(b::MCAT, ps, pivot, smooth, mat) =
  [(b_trig(b, pivot, ps[i], ps[i+1], mat)
    for i in 1:size(ps,1)-1)...,
	 b_trig(b, pivot, ps[end], ps[1], mat)]

KhepriBase.b_quad_strip(b::MCAT, ps, qs, smooth, mat) =
  [b_quad(b, ps[i], ps[i+1], qs[i+1], qs[i], mat)
   for i in 1:size(ps,1)-1]

KhepriBase.b_quad_strip_closed(b::MCAT, ps, qs, smooth, mat) =
  b_quad_strip(b, [ps..., ps[1]], [qs..., qs[1]], smooth, mat)

=#
KhepriBase.b_surface_polygon(b::MCAT, ps, mat) =
  add_object(b, meshcat_surface_polygon(ps, mat))
KhepriBase.b_surface_circle(b::MCAT, c, r, mat) =
  add_object(b, meshcat_circle_mesh(c, r, 0, 2pi, mat))
KhepriBase.b_surface_arc(b::MCAT, c, r, α, Δα, mat) =
  add_object(b, meshcat_circle_mesh(c, r, α, Δα, mat))
KhepriBase.b_surface_grid(b::MCAT, ptss, closed_u, closed_v, smooth_u, smooth_v, mat) =
  let si = size(ptss, 1),
      sj = size(ptss, 2),
      idxs = meshcat_faces(si, sj, closed_u, closed_v)
    add_object(b, meshcat_mesh(reshape(permutedims(ptss),:), idxs, mat))
  end
KhepriBase.b_surface_mesh(b::MCAT, vertices, faces, mat) =
  add_object(b, meshcat_mesh(vertices, faces, mat))
#=

# Parametric surface
#=
parametric {
    function { sin(u)*cos(v) }
    function { sin(u)*sin(v) }
    function { cos(u) }

    <0,0>, <2*pi,pi>
    contained_by { sphere{0, 1.1} }
    max_gradient ??
    accuracy 0.0001
    precompute 10 x,y,z
    pigment {rgb 1}
  }
=#

=#
KhepriBase.b_cylinder(b::MCAT, cb, r, h, mat) =
	add_object(b, meshcat_cylinder(cb, r, h, mat))
KhepriBase.b_box(b::MCAT, c, dx, dy, dz, mat) =
  add_object(b, meshcat_box(c, dx, dy, dz, mat))
KhepriBase.b_sphere(b::MCAT, c, r, mat) =
  add_object(b, meshcat_sphere(c, r, mat))
KhepriBase.b_cone(b::MCAT, cb, r, h, mat) =
	add_object(b, meshcat_cone(cb, r, h, mat))
KhepriBase.b_cone_frustum(b::MCAT, cb, rb, h, rt, mat) =
	add_object(b, meshcat_cone_frustum(cb, rb, rt, h, mat))
KhepriBase.b_torus(b::MCAT, c, ra, rb, mat) =
  add_object(b, meshcat_torus(c, ra, rb, mat))

KhepriBase.b_extrusion(b::MCAT, profile::Region, v, cb, bmat, tmat, smat) =
  let cs = cs_from_o_vz(cb, v)
    add_object(b, meshcat_extrusion_z(profile, norm(v), tmat, u0(cs)))
  end
#=
setprop!(
  connection(meshcat)["/Cameras/default/rotated/<object>"], #"Cameras","default","rotated","<object>"],
  "zoom",
  1.0)
=#

#=
MeshCat.setcontrol!(
  connection(meshcat)["Background"], #"Cameras","default","rotated","<object>"],
  "hide_background")

{
    type: "set_property",
    path: "/Cameras/default/rotated/<object>",
    property: "zoom",
    value: 2.0
}
=#

KhepriBase.b_set_view(b::MCAT, camera, target, lens, aperture) =
  let v = connection(b)
    send_setview(v, camera, target, lens, aperture)
    b.camera = camera
    b.target = target
    b.lens = lens
  end

KhepriBase.b_delete_ref(b::MCAT, r::MCATId) =
  send_delobject(connection(b), r)

KhepriBase.b_delete_all_refs(b::MCAT) =
  send_delobject(connection(b), meshcat_root_path)
#=

backend_stroke(b::MCAT, path::OpenSplinePath) =
  if (path.v0 == false) && (path.v1 == false)
    add_object(b, meshcat_line(path_frames(path), line_material(b)))
  elseif (path.v0 != false) && (path.v1 != false)
    @remote(b, InterpSpline(path.vertices, path.v0, path.v1))
  else
    @remote(b, InterpSpline(
                     path.vertices,
                     path.v0 == false ? path.vertices[2]-path.vertices[1] : path.v0,
                     path.v1 == false ? path.vertices[end-1]-path.vertices[end] : path.v1))
  end
backend_stroke(b::MCAT, path::ClosedSplinePath) =
    add_object(b, meshcat_line(path_frames(path), line_material(b)))
backend_fill(b::MCAT, path::ClosedSplinePath) =
    add_object(b, meshcat_surface_polygon(path_frames(path), material(b)))

#=
smooth_pts(pts) = in_world.(path_frames(open_spline_path(pts)))

=#

# Layers
current_layer(b::MCAT) =
  b.layer

current_layer(layer, b::MCAT) =
  b.layer = layer

backend_create_layer(b::MCAT, name::String, active::Bool, color::RGB) =
  begin
    @assert active
    mcat_layer(name, color)
  end

#=
create_ground_plane(shapes, material=default_MCAT_ground_material()) =
  if shapes == []
    error("No shapes selected for analysis. Use add-MCAT-shape!.")
  else
    let (p0, p1) = bounding_box(union(shapes)),
        (center, ratio) = (quad_center(p0, p1, p2, p3),
                  distance(p0, p4)/distance(p0, p2));
     ratio == 0 ?
      error("Couldn"t compute height. Use add-MCAT-shape!.") :
      let pts = map(p -> intermediate_loc(center, p, ratio*10), [p0, p1, p2, p3]);
         create_surface_layer(pts, 0, ground_layer(), material)
        end
       end
  end
        w = max(floor_extra_factor()*distance(p0, p1), floor_extra_width())
        with(current_layer,floor_layer()) do
          box(xyz(min(p0.x, p1.x)-w, min(p0.y, p1.y)-w, p0.z-1-floor_distance()),
              xyz(max(p0.x, p1.x)+w, max(p0.y, p1.y)+w, p0.z-0-floor_distance()))
        end
      end
    end

=#

####################################################
=#


text:
include("test_params.jl")

using Distributed
procs = addprocs(4)
using Test, Pkg
project_path = splitdir(Pkg.project().path)[1]
@everywhere procs begin
    Base.MainInclude.eval(
        quote
            using Pkg
            Pkg.activate($$project_path)
        end,
    )
end
@everywhere using SymbolicRegression
@everywhere _inv(x::Float32)::Float32 = 1.0f0 / x
X = rand(Float32, 5, 100) .+ 1
y = 1.2f0 .+ 2 ./ X[3, :]

options = SymbolicRegression.Options(;
    default_params..., binary_operators=(+, *), unary_operators=(_inv,), npopulations=8
)
hallOfFame = EquationSearch(X, y; niterations=8, options=options, procs=procs)
rmprocs(procs)

dominating = calculate_pareto_frontier(X, y, hallOfFame, options)
best = dominating[end]
# Test the score
@test best.loss < maximum_residual / 10


text:
Base.:(==)(x::InfExtendedReal, y::InfExtendedReal) = isinf(x)==isinf(y) && x.val==y.val

Base.hash(x::InfExtendedReal, h::UInt) = hash(x.val, h)

Base.:<(x::InfExtendedReal, y::InfExtendedReal) =
  if isinf(x)
    signbit(x) && !isneginf(y)
  elseif isinf(y)
    !signbit(y)
  else
    x.val < y.val
  end

Base.signbit(x::InfExtendedReal) = signbit(x.val)

Base.sign(x::InfExtendedReal{T}) where {T<:Real} = convert(T, sign(x.val))

Base.isapprox(x::InfExtendedReal, y::InfExtendedReal; opts...) = isinf(x)==isinf(y) && isapprox(x.val, y.val; opts...)


text:
module TMazeCumulantConstants
    const MAX = 50
    const MIN = -50
end
module TMazeCumulantSchedules

using Distributions
import ..CumulantSchedule
import ..Curiosity: get_cumulant, update!, get_cumulant_eval_values
import ..TMazeCumulantConstants: MAX, MIN

mutable struct DrifterDistractor <: CumulantSchedule
    constant1::Float64
    constant2::Float64
    drifter_mean::Float64
    drifter_std::Float64
    distractor_mean::Float64
    distractor_std::Float64

    # function DrifterDistractor(constants, drifter_mean, drifter_std, distractor_mean, distractor_std)
    #     new(constants, constants, drifter_mean, drifter_std, distractor_mean, distractor_std)
    # end

end

function get_cumulant(cs::DrifterDistractor, goal::String)
    if goal == "G1"
        clamp(MIN, rand(Normal(cs.distractor_mean, cs.distractor_std)), MAX)
    elseif goal == "G2"
        cs.constant1
    elseif goal == "G3"
        cs.drifter_mean
    elseif goal == "G4"
        cs.constant2
    end
end

function get_cumulant_eval_values(self::DrifterDistractor)
    # Used for scaling the eval set based on the end values
    num_cumulants = 4
    cumulants = zeros(num_cumulants)
    cumulants[1] = self.distractor_mean
    cumulants[2] = self.constant1
    cumulants[3] = self.drifter_mean
    cumulants[4] = self.constant2
    return cumulants
end

# function update!(env::TabularTMaze, self::DrifterDistractor, pos)
function update!(self::DrifterDistractor, pos)
    self.drifter_mean += rand(Normal(0, self.drifter_std))
    self.drifter_mean = clamp(MIN,self.drifter_mean,MAX)
end

mutable struct Constant <: CumulantSchedule
    c1::Float64
    c2::Float64
    c3::Float64
    c4::Float64
end

Constant(c = 1.0) = Constant(c,c,c,c)

function get_cumulant(cs::Constant, goal::String)
    if goal == "G1"
        cs.c1
    elseif goal == "G2"
        cs.c2
    elseif goal == "G3"
        cs.c3
    elseif goal == "G4"
        cs.c4
    end
end

get_cumulant_eval_values(cs::Constant) = [cs.c1, cs.c2, cs.c3, cs.c4]

function update!(::Constant, pos)
end

mutable struct DrifterDrifterDistractor <: CumulantSchedule
    constant1::Float64

    drifter_1_mean::Float64
    drifter_1_std::Float64
    drifter_2_mean::Float64
    drifter_2_std::Float64
    distractor_mean::Float64
    distractor_std::Float64

end

function get_cumulant(cs::DrifterDrifterDistractor, goal::String)
    if goal == "G1"
        clamp(MIN, rand(Normal(cs.distractor_mean, cs.distractor_std)), MAX)
    elseif goal == "G2"
        cs.constant1
    elseif goal == "G3"
        cs.drifter_1_mean
    elseif goal == "G4"
        cs.drifter_2_mean
    end
end

function get_cumulant_eval_values(self::DrifterDrifterDistractor)
    # Used for scaling the eval set based on the end values
    num_cumulants = 4
    cumulants = zeros(num_cumulants)
    cumulants[1] = self.distractor_mean
    cumulants[2] = self.constant1
    cumulants[3] = self.drifter_1_mean
    cumulants[4] = self.drifter_2_mean
    return cumulants
end

# function update!(env::TabularTMaze, self::DrifterDistractor, pos)
function update!(self::DrifterDrifterDistractor, pos)
    self.drifter_1_mean += rand(Normal(0, self.drifter_1_std))
    self.drifter_1_mean = clamp(MIN,self.drifter_1_mean,MAX)

    self.drifter_2_mean += rand(Normal(0, self.drifter_2_std))
    self.drifter_2_mean = clamp(MIN,self.drifter_2_mean,MAX)
end

end

max_stars_repo_name: MatthewMcLeod/curiosity

max_issues_repo_name: MatthewMcLeod/curiosity

max_forks_repo_name: MatthewMcLeod/curiosity


text:
function fetch_structure_types(xroot)
    stype_vals = findall("//member[@values]", xroot)
    res = Dict()
    for stype ∈ stype_vals
        type = stype.parentnode["name"]
        stype_value = stype["values"]
        res[Symbol(type)] = Symbol(stype_value)
    end
    res
end

const structure_types = fetch_structure_types(xroot)


text:
using Documenter, DiscreteEvents

x = 2 # set global (Main) variable for mocking fclosure.jl doctest line 90

makedocs(
    modules = [DiscreteEvents],
    format = Documenter.HTML(prettyurls = get(ENV, "CI", nothing) == "true"),
    sitename = "DiscreteEvents.jl",
    authors  = "Paul Bayer",
    pages = [
        "Home" => "index.md",
        "News" => "news.md",
        "Introduction" => "intro.md",
        "Manual" => [
            "setup.md",
            "clocks.md",
            "events.md",
            "processes.md",
            "actors.md",
            "resources.md",
            "parallel.md",
            "usage.md"],
        "Internals" => [
            "internals.md",
            "troubleshooting.md",
            "history.md"]
    ]
)

deploydocs(
    repo   = "github.com/pbayer/DiscreteEvents.jl.git",
    target = "build",
    deps   = nothing,
    make   = nothing,
    devbranch = "master",
    devurl = "dev",
    versions = ["stable" => "v^", "v#.#", "dev" => "dev"]
)


text:
# This file is a part of Julia. License is MIT: https://julialang.org/license

import .Libc: RawFD, dup
if Sys.iswindows()
    import .Libc: WindowsRawSocket
    const OS_HANDLE = WindowsRawSocket
    const INVALID_OS_HANDLE = WindowsRawSocket(Ptr{Cvoid}(-1))
else
    const OS_HANDLE = RawFD
    const INVALID_OS_HANDLE = RawFD(-1)
end

## types ##
abstract type IOServer end
abstract type LibuvServer <: IOServer end
abstract type LibuvStream <: IO end

# IO
# +- GenericIOBuffer{T<:AbstractArray{UInt8,1}} (not exported)
# +- AbstractPipe (not exported)
# .  +- Pipe
# .  +- Process (not exported)
# .  +- ProcessChain (not exported)
# +- BufferStream
# +- DevNull (not exported)
# +- Filesystem.File
# +- LibuvStream (not exported)
# .  +- PipeEndpoint (not exported)
# .  +- TCPSocket
# .  +- TTY (not exported)
# .  +- UDPSocket
# +- IOBuffer = Base.GenericIOBuffer{Array{UInt8,1}}
# +- IOStream

# IOServer
# +- LibuvServer
# .  +- PipeServer
# .  +- TCPServer

# Redirectable = Union{IO, FileRedirect, Libc.RawFD} (not exported)

function stream_wait(x, c...) # for x::LibuvObject
    preserve_handle(x)
    try
        return wait(c...)
    finally
        unpreserve_handle(x)
    end
end

bytesavailable(s::LibuvStream) = bytesavailable(s.buffer)

function eof(s::LibuvStream)
    if isopen(s) # fast path
        bytesavailable(s) > 0 && return false
    else
        return bytesavailable(s) <= 0
    end
    wait_readnb(s,1)
    return !isopen(s) && bytesavailable(s) <= 0
end

# Limit our default maximum read and buffer size,
# to avoid DoS-ing ourself into an OOM situation
const DEFAULT_READ_BUFFER_SZ = 10485760 # 10 MB

# manually limit our write size, if the OS doesn't support full-size writes
if Sys.iswindows()
    const MAX_OS_WRITE = UInt(0x1FF0_0000) # 511 MB (determined semi-empirically, limited to 31 MB on XP)
else
    const MAX_OS_WRITE = UInt(typemax(Csize_t))
end

const StatusUninit      = 0 # handle is allocated, but not initialized
const StatusInit        = 1 # handle is valid, but not connected/active
const StatusConnecting  = 2 # handle is in process of connecting
const StatusOpen        = 3 # handle is usable
const StatusActive      = 4 # handle is listening for read/write/connect events
const StatusClosing     = 5 # handle is closing / being closed
const StatusClosed      = 6 # handle is closed
const StatusEOF         = 7 # handle is a TTY that has seen an EOF event (pretends to be closed until reseteof is called)
const StatusPaused      = 8 # handle is Active, but not consuming events, and will transition to Open if it receives an event
function uv_status_string(x)
    s = x.status
    if x.handle == C_NULL
        if s == StatusClosed
            return "closed"
        elseif s == StatusUninit
            return "null"
        end
        return "invalid status"
    elseif s == StatusUninit
        return "uninit"
    elseif s == StatusInit
        return "init"
    elseif s == StatusConnecting
        return "connecting"
    elseif s == StatusOpen
        return "open"
    elseif s == StatusActive
        return "active"
    elseif s == StatusPaused
        return "paused"
    elseif s == StatusClosing
        return "closing"
    elseif s == StatusClosed
        return "closed"
    elseif s == StatusEOF
        return "eof"
    end
    return "invalid status"
end

mutable struct PipeEndpoint <: LibuvStream
    handle::Ptr{Cvoid}
    status::Int
    buffer::IOBuffer
    cond::ThreadSynchronizer
    closenotify::ThreadSynchronizer
    sendbuf::Union{IOBuffer, Nothing}
    lock::ReentrantLock
    throttle::Int
    function PipeEndpoint(handle::Ptr{Cvoid}, status)
        lock = Threads.SpinLock()
        p = new(handle,
                status,
                PipeBuffer(),
                ThreadSynchronizer(lock),
                ThreadSynchronizer(lock),
                nothing,
                ReentrantLock(),
                DEFAULT_READ_BUFFER_SZ)
        associate_julia_struct(handle, p)
        finalizer(uvfinalize, p)
        return p
    end
end

function PipeEndpoint()
    pipe = PipeEndpoint(Libc.malloc(_sizeof_uv_named_pipe), StatusUninit)
    err = ccall(:uv_pipe_init, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Cint), eventloop(), pipe.handle, 0)
    uv_error("failed to create pipe endpoint", err)
    pipe.status = StatusInit
    return pipe
end

function PipeEndpoint(fd::OS_HANDLE)
    pipe = PipeEndpoint()
    err = ccall(:uv_pipe_open, Int32, (Ptr{Cvoid}, OS_HANDLE), pipe.handle, fd)
    uv_error("pipe_open", err)
    pipe.status = StatusOpen
    return pipe
end
if OS_HANDLE != RawFD
    PipeEndpoint(fd::RawFD) = PipeEndpoint(Libc._get_osfhandle(fd))
end

mutable struct TTY <: LibuvStream
    handle::Ptr{Cvoid}
    status::Int
    buffer::IOBuffer
    cond::ThreadSynchronizer
    closenotify::ThreadSynchronizer
    sendbuf::Union{IOBuffer, Nothing}
    lock::ReentrantLock
    throttle::Int
    @static if Sys.iswindows(); ispty::Bool; end
    function TTY(handle::Ptr{Cvoid}, status)
        lock = Threads.SpinLock()
        tty = new(
            handle,
            status,
            PipeBuffer(),
            ThreadSynchronizer(lock),
            ThreadSynchronizer(lock),
            nothing,
            ReentrantLock(),
            DEFAULT_READ_BUFFER_SZ)
        associate_julia_struct(handle, tty)
        finalizer(uvfinalize, tty)
        @static if Sys.iswindows()
            tty.ispty = ccall(:jl_ispty, Cint, (Ptr{Cvoid},), handle) != 0
        end
        return tty
    end
end

function TTY(fd::OS_HANDLE)
    tty = TTY(Libc.malloc(_sizeof_uv_tty), StatusUninit)
    err = ccall(:uv_tty_init, Int32, (Ptr{Cvoid}, Ptr{Cvoid}, OS_HANDLE, Int32),
        eventloop(), tty.handle, fd, 0)
    uv_error("TTY", err)
    tty.status = StatusOpen
    return tty
end
if OS_HANDLE != RawFD
    TTY(fd::RawFD) = TTY(Libc._get_osfhandle(fd))
end

show(io::IO, stream::LibuvServer) = print(io, typeof(stream), "(",
    _fd(stream), " ",
    uv_status_string(stream), ")")
show(io::IO, stream::LibuvStream) = print(io, typeof(stream), "(",
    _fd(stream), " ",
    uv_status_string(stream), ", ",
    bytesavailable(stream.buffer), " bytes waiting)")

# Shared LibuvStream object interface

function isreadable(io::LibuvStream)
    bytesavailable(io) > 0 && return true
    isopen(io) || return false
    return ccall(:uv_is_readable, Cint, (Ptr{Cvoid},), io.handle) != 0
end

function iswritable(io::LibuvStream)
    isopen(io) || return false
    io.status == StatusClosing && return false
    return ccall(:uv_is_writable, Cint, (Ptr{Cvoid},), io.handle) != 0
end

lock(s::LibuvStream) = lock(s.lock)
unlock(s::LibuvStream) = unlock(s.lock)

rawhandle(stream::LibuvStream) = stream.handle
unsafe_convert(::Type{Ptr{Cvoid}}, s::Union{LibuvStream, LibuvServer}) = s.handle

function init_stdio(handle::Ptr{Cvoid})
    t = ccall(:jl_uv_handle_type, Int32, (Ptr{Cvoid},), handle)
    if t == UV_FILE
        fd = ccall(:jl_uv_file_handle, OS_HANDLE, (Ptr{Cvoid},), handle)
        # TODO: Replace ios.c file with libuv fs?
        # return File(fd)
        @static if Sys.iswindows()
            # TODO: Get ios.c to understand native handles
            fd = ccall(:_open_osfhandle, RawFD, (WindowsRawSocket, Int32), fd, 0)
        end
        # TODO: Get fdio to work natively with file descriptors instead of integers
        return fdio(cconvert(Cint, fd))
    elseif t == UV_TTY
        return TTY(handle, StatusOpen)
    elseif t == UV_TCP
        Sockets = require(PkgId(UUID((0x6462fe0b_24de_5631, 0x8697_dd941f90decc)), "Sockets"))
        return Sockets.TCPSocket(handle, StatusOpen)
    elseif t == UV_NAMED_PIPE
        return PipeEndpoint(handle, StatusOpen)
    else
        throw(ArgumentError("invalid stdio type: $t"))
    end
end

"""
    open(fd::OS_HANDLE) -> IO

Take a raw file descriptor wrap it in a Julia-aware IO type,
and take ownership of the fd handle.
Call `open(Libc.dup(fd))` to avoid the ownership capture
of the original handle.

!!! warn
    Do not call this on a handle that's already owned by some
    other part of the system.
"""
function open(h::OS_HANDLE)
    t = ccall(:uv_guess_handle, Cint, (OS_HANDLE,), h)
    if t == UV_FILE
        @static if Sys.iswindows()
            # TODO: Get ios.c to understand native handles
            h = ccall(:_open_osfhandle, RawFD, (WindowsRawSocket, Int32), h, 0)
        end
        # TODO: Get fdio to work natively with file descriptors instead of integers
        return fdio(cconvert(Cint, h))
    elseif t == UV_TTY
        return TTY(h)
    elseif t == UV_TCP
        Sockets = require(PkgId(UUID((0x6462fe0b_24de_5631, 0x8697_dd941f90decc)), "Sockets"))
        return Sockets.TCPSocket(h)
    elseif t == UV_NAMED_PIPE
        pipe = PipeEndpoint(h)
        @static if Sys.iswindows()
            if ccall(:jl_ispty, Cint, (Ptr{Cvoid},), pipe.handle) != 0
                # replace the Julia `PipeEndpoint` type with a `TTY` type,
                # if we detect that this is a cygwin pty object
                pipe_handle, pipe_status = pipe.handle, pipe.status
                pipe.status = StatusClosed
                pipe.handle = C_NULL
                return TTY(pipe_handle, pipe_status)
            end
        end
        return pipe
    else
        throw(ArgumentError("invalid stdio type: $t"))
    end
end

if OS_HANDLE != RawFD
    function open(fd::RawFD)
        h = Libc.dup(Libc._get_osfhandle(fd)) # make a dup to steal ownership away from msvcrt
        try
            io = open(h)
            ccall(:_close, Cint, (RawFD,), fd) # on success, destroy the old libc handle
            return io
        catch ex
            ccall(:CloseHandle, stdcall, Cint, (OS_HANDLE,), h) # on failure, destroy the new nt handle
            rethrow(ex)
        end
    end
end

function isopen(x::Union{LibuvStream, LibuvServer})
    if x.status == StatusUninit || x.status == StatusInit
        throw(ArgumentError("$x is not initialized"))
    end
    return x.status != StatusClosed && x.status != StatusEOF
end

function check_open(x::Union{LibuvStream, LibuvServer})
    if !isopen(x) || x.status == StatusClosing
        throw(IOError("stream is closed or unusable", 0))
    end
end

function wait_connected(x::Union{LibuvStream, LibuvServer})
    check_open(x)
    lock(x.cond)
    try
        while x.status == StatusConnecting
            stream_wait(x, x.cond)
            check_open(x)
        end
    finally
        unlock(x.cond)
    end
end

function wait_readbyte(x::LibuvStream, c::UInt8)
    if isopen(x) # fast path
        occursin(c, x.buffer) && return
    else
        return
    end
    preserve_handle(x)
    lock(x.cond)
    try
        while isopen(x) && !occursin(c, x.buffer)
            start_reading(x) # ensure we are reading
            wait(x.cond)
        end
    finally
        if isempty(x.cond)
            stop_reading(x) # stop reading iff there are currently no other read clients of the stream
        end
        unpreserve_handle(x)
        unlock(x.cond)
    end
    nothing
end

function wait_readnb(x::LibuvStream, nb::Int)
    if isopen(x) # fast path
        bytesavailable(x.buffer) >= nb && return
    else
        return
    end
    oldthrottle = x.throttle
    preserve_handle(x)
    lock(x.cond)
    try
        while isopen(x) && bytesavailable(x.buffer) < nb
            x.throttle = max(nb, x.throttle)
            start_reading(x) # ensure we are reading
            wait(x.cond)
        end
    finally
        if isempty(x.cond)
            stop_reading(x) # stop reading iff there are currently no other read clients of the stream
        end
        if oldthrottle <= x.throttle <= nb
            x.throttle = oldthrottle
        end
        unpreserve_handle(x)
        unlock(x.cond)
    end
    nothing
end

function wait_close(x::Union{LibuvStream, LibuvServer})
    lock(x.closenotify)
    try
        if isopen(x)
            stream_wait(x, x.closenotify)
        end
    finally
        unlock(x.closenotify)
    end
    nothing
end

function close(stream::Union{LibuvStream, LibuvServer})
    if stream.status == StatusInit
        ccall(:jl_forceclose_uv, Cvoid, (Ptr{Cvoid},), stream.handle)
        return nothing
    end
    lock(stream.closenotify)
    try
        if isopen(stream) || stream.status == StatusEOF
            should_wait = uv_handle_data(stream) != C_NULL
            if stream.status != StatusClosing
                ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), stream.handle)
                stream.status = StatusClosing
            end
            if should_wait
                stream_wait(stream, stream.closenotify)
            end
        end
    finally
        unlock(stream.closenotify)
    end
    return nothing
end

function uvfinalize(uv::Union{LibuvStream, LibuvServer})
    if uv.handle != C_NULL
        disassociate_julia_struct(uv.handle) # not going to call the usual close hooks
        if uv.status != StatusUninit
            close(uv)
        else
            Libc.free(uv.handle)
        end
        uv.status = StatusClosed
        uv.handle = C_NULL
    end
    nothing
end

if Sys.iswindows()
    ispty(s::TTY) = s.ispty
    ispty(s::IO) = false
end

"""
    displaysize([io::IO]) -> (lines, columns)

Return the nominal size of the screen that may be used for rendering output to
this `IO` object.
If no input is provided, the environment variables `LINES` and `COLUMNS` are read.
If those are not set, a default size of `(24, 80)` is returned.

# Examples
```jldoctest
julia> withenv("LINES" => 30, "COLUMNS" => 100) do
           displaysize()
       end
(30, 100)
```

To get your TTY size,

```julia
julia> displaysize(stdout)
(34, 147)
```
"""
displaysize(io::IO) = displaysize()
displaysize() = (parse(Int, get(ENV, "LINES",   "24")),
                 parse(Int, get(ENV, "COLUMNS", "80")))::Tuple{Int, Int}

function displaysize(io::TTY)
    local h::Int, w::Int
    default_size = displaysize()

    @static if Sys.iswindows()
        if ispty(io)
            # io is actually a libuv pipe but a cygwin/msys2 pty
            try
                h, w = parse.(Int, split(read(open(Base.Cmd(String["stty", "size"]), "r", io).out, String)))
                h > 0 || (h = default_size[1])
                w > 0 || (w = default_size[2])
                return h, w
            catch
                return default_size
            end
        end
    end

    s1 = Ref{Int32}(0)
    s2 = Ref{Int32}(0)
    Base.uv_error("size (TTY)", ccall(:uv_tty_get_winsize,
                                      Int32, (Ptr{Cvoid}, Ptr{Int32}, Ptr{Int32}),
                                      io, s1, s2) != 0)
    w, h = s1[], s2[]
    h > 0 || (h = default_size[1])
    w > 0 || (w = default_size[2])
    return h, w
end

in(key_value::Pair{Symbol,Bool}, ::TTY) = key_value.first === :color && key_value.second === have_color
haskey(::TTY, key::Symbol) = key === :color
getindex(::TTY, key::Symbol) = key === :color ? have_color : throw(KeyError(key))
get(::TTY, key::Symbol, default) = key === :color ? have_color : default

### Libuv callbacks ###

## BUFFER ##
## Allocate space in buffer (for immediate use)
function alloc_request(buffer::IOBuffer, recommended_size::UInt)
    ensureroom(buffer, Int(recommended_size))
    ptr = buffer.append ? buffer.size + 1 : buffer.ptr
    nb = length(buffer.data) - ptr + 1
    return (pointer(buffer.data, ptr), nb)
end

notify_filled(buffer::IOBuffer, nread::Int, base::Ptr{Cvoid}, len::UInt) = notify_filled(buffer, nread)

function notify_filled(buffer::IOBuffer, nread::Int)
    if buffer.append
        buffer.size += nread
    else
        buffer.ptr += nread
    end
end

function alloc_buf_hook(stream::LibuvStream, size::UInt)
    throttle = UInt(stream.throttle)
    return alloc_request(stream.buffer, (size > throttle) ? throttle : size)
end

function uv_alloc_buf(handle::Ptr{Cvoid}, size::Csize_t, buf::Ptr{Cvoid})
    hd = uv_handle_data(handle)
    if hd == C_NULL
        ccall(:jl_uv_buf_set_len, Cvoid, (Ptr{Cvoid}, Csize_t), buf, 0)
        return nothing
    end
    stream = unsafe_pointer_to_objref(hd)::LibuvStream

    local data::Ptr{Cvoid}, newsize::Csize_t
    if stream.status != StatusActive
        data = C_NULL
        newsize = 0
    else
        (data, newsize) = alloc_buf_hook(stream, UInt(size))
        if data == C_NULL
            newsize = 0
        end
        # avoid aliasing of `nread` with `errno` in uv_readcb
        # or exceeding the Win32 maximum uv_buf_t len
        maxsize = @static Sys.iswindows() ? typemax(Cint) : typemax(Cssize_t)
        newsize > maxsize && (newsize = maxsize)
    end

    ccall(:jl_uv_buf_set_base, Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}), buf, data)
    ccall(:jl_uv_buf_set_len, Cvoid, (Ptr{Cvoid}, Csize_t), buf, newsize)
    nothing
end

function uv_readcb(handle::Ptr{Cvoid}, nread::Cssize_t, buf::Ptr{Cvoid})
    stream_unknown_type = @handle_as handle LibuvStream
    nrequested = ccall(:jl_uv_buf_len, Csize_t, (Ptr{Cvoid},), buf)
    function readcb_specialized(stream::LibuvStream, nread::Int, nrequested::UInt)
        lock(stream.cond)
        try
            if nread < 0
                if nread == UV_ENOBUFS && nrequested == 0
                    # remind the client that stream.buffer is full
                    notify(stream.cond)
                elseif nread == UV_EOF
                    if isa(stream, TTY)
                        stream.status = StatusEOF # libuv called uv_stop_reading already
                        notify(stream.cond)
                        notify(stream.closenotify)
                    elseif stream.status != StatusClosing
                        # begin shutdown of the stream
                        ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), stream.handle)
                        stream.status = StatusClosing
                    end
                else
                    # This is a fatal connection error. Shutdown requests as per the usual
                    # close function won't work and libuv will fail with an assertion failure
                    ccall(:jl_forceclose_uv, Cvoid, (Ptr{Cvoid},), stream)
                    notify_error(stream.cond, _UVError("read", nread))
                end
            else
                notify_filled(stream.buffer, nread)
                notify(stream.cond)
            end
        finally
            unlock(stream.cond)
        end

        # Stop background reading when
        # 1) there's nobody paying attention to the data we are reading
        # 2) we have accumulated a lot of unread data OR
        # 3) we have an alternate buffer that has reached its limit.
        if stream.status == StatusPaused ||
           (stream.status == StatusActive &&
            ((bytesavailable(stream.buffer) >= stream.throttle) ||
             (bytesavailable(stream.buffer) >= stream.buffer.maxsize)))
            # save cycles by stopping kernel notifications from arriving
            ccall(:jl_uv_read_stop, Cint, (Ptr{Cvoid},), stream)
            stream.status = StatusOpen
        end
        nothing
    end
    readcb_specialized(stream_unknown_type, Int(nread), UInt(nrequested))
end

function reseteof(x::TTY)
    if x.status == StatusEOF
        x.status = StatusOpen
    end
    nothing
end

function _uv_hook_close(uv::Union{LibuvStream, LibuvServer})
    lock(uv.closenotify)
    try
        uv.handle = C_NULL
        uv.status = StatusClosed
        # notify any listeners that exist on this libuv stream type
        notify(uv.closenotify)
        notify(uv.cond)
    finally
        unlock(uv.closenotify)
    end
    nothing
end

##########################################
# Pipe Abstraction
#  (composed of two half-pipes: .in and .out)
##########################################

mutable struct Pipe <: AbstractPipe
    in::PipeEndpoint # writable
    out::PipeEndpoint # readable
end

"""
Construct an uninitialized Pipe object.

The appropriate end of the pipe will be automatically initialized if
the object is used in process spawning. This can be useful to easily
obtain references in process pipelines, e.g.:

```
julia> err = Pipe()

# After this `err` will be initialized and you may read `foo`'s
# stderr from the `err` pipe.
julia> run(pipeline(pipeline(`foo`, stderr=err), `cat`), wait=false)
```
"""
Pipe() = Pipe(PipeEndpoint(), PipeEndpoint())
pipe_reader(p::Pipe) = p.out
pipe_writer(p::Pipe) = p.in

function link_pipe!(pipe::Pipe;
                    reader_supports_async = false,
                    writer_supports_async = false)
     link_pipe!(pipe.out, reader_supports_async, pipe.in, writer_supports_async)
     return pipe
end

show(io::IO, stream::Pipe) = print(io,
    "Pipe(",
    _fd(stream.in), " ",
    uv_status_string(stream.in), " => ",
    _fd(stream.out), " ",
    uv_status_string(stream.out), ", ",
    bytesavailable(stream), " bytes waiting)")

## Functions for PipeEndpoint and PipeServer ##

function open_pipe!(p::PipeEndpoint, handle::OS_HANDLE)
    if p.status != StatusInit
        error("pipe is already in use or has been closed")
    end
    err = ccall(:uv_pipe_open, Int32, (Ptr{Cvoid}, OS_HANDLE), p.handle, handle)
    uv_error("pipe_open", err)
    p.status = StatusOpen
    return p
end

function link_pipe!(read_end::PipeEndpoint, reader_supports_async::Bool,
                    write_end::PipeEndpoint, writer_supports_async::Bool)
    rd, wr = link_pipe(reader_supports_async, writer_supports_async)
    try
        try
            open_pipe!(read_end, rd)
        catch
            close_pipe_sync(rd)
            rethrow()
        end
        read_end.status = StatusOpen
        open_pipe!(write_end, wr)
    catch
        close_pipe_sync(wr)
        rethrow()
    end
    write_end.status = StatusOpen
    nothing
end

function link_pipe(reader_supports_async::Bool, writer_supports_async::Bool)
    UV_NONBLOCK_PIPE = 0x40
    fildes = Ref{Pair{OS_HANDLE, OS_HANDLE}}(INVALID_OS_HANDLE => INVALID_OS_HANDLE) # read (in) => write (out)
    err = ccall(:uv_pipe, Int32, (Ptr{Pair{OS_HANDLE, OS_HANDLE}}, Cint, Cint),
                fildes,
                reader_supports_async * UV_NONBLOCK_PIPE,
                writer_supports_async * UV_NONBLOCK_PIPE)
    uv_error("pipe", err)
    return fildes[]
end

if Sys.iswindows()
    function close_pipe_sync(handle::WindowsRawSocket)
        ccall(:CloseHandle, stdcall, Cint, (WindowsRawSocket,), handle)
        nothing
    end
else
    function close_pipe_sync(handle::RawFD)
        ccall(:close, Cint, (RawFD,), handle)
        nothing
    end
end

## Functions for any LibuvStream ##

# flow control

function start_reading(stream::LibuvStream)
    if stream.status == StatusOpen
        if !isreadable(stream)
            error("tried to read a stream that is not readable")
        end
        # libuv may call the alloc callback immediately
        # for a TTY on Windows, so ensure the status is set first
        stream.status = StatusActive
        ret = ccall(:jl_uv_read_start, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}),
                    stream, uv_jl_alloc_buf::Ptr{Cvoid}, uv_jl_readcb::Ptr{Cvoid})
        return ret
    elseif stream.status == StatusPaused
        stream.status = StatusActive
        return Int32(0)
    elseif stream.status == StatusActive
        return Int32(0)
    else
        return Int32(-1)
    end
end

if Sys.iswindows()
    # the low performance version of stop_reading is required
    # on Windows due to a NT kernel bug that we can't use a blocking
    # stream for non-blocking (overlapped) calls,
    # and a ReadFile call blocking on one thread
    # causes all other operations on that stream to lockup
    function stop_reading(stream::LibuvStream)
        if stream.status == StatusActive
            stream.status = StatusOpen
            ccall(:jl_uv_read_stop, Cint, (Ptr{Cvoid},), stream)
        end
        nothing
    end
else
    function stop_reading(stream::LibuvStream)
        if stream.status == StatusActive
            stream.status = StatusPaused
        end
        nothing
    end
end

# bulk read / write

readbytes!(s::LibuvStream, a::Vector{UInt8}, nb = length(a)) = readbytes!(s, a, Int(nb))
function readbytes!(s::LibuvStream, a::Vector{UInt8}, nb::Int)
    sbuf = s.buffer
    @assert sbuf.seekable == false
    @assert sbuf.maxsize >= nb

    if bytesavailable(sbuf) >= nb
        return readbytes!(sbuf, a, nb)
    end

    if nb <= SZ_UNBUFFERED_IO # Under this limit we are OK with copying the array from the stream's buffer
        wait_readnb(s, nb)
        return readbytes!(sbuf, a, nb)
    else
        try
            stop_reading(s) # Just playing it safe, since we are going to switch buffers.
            newbuf = PipeBuffer(a, maxsize = nb)
            newbuf.size = 0 # reset the write pointer to the beginning
            s.buffer = newbuf
            write(newbuf, sbuf)
            wait_readnb(s, Int(nb))
            compact(newbuf)
            return bytesavailable(newbuf)
        finally
            s.buffer = sbuf
            if !isempty(s.cond)
                start_reading(s) # resume reading iff there are currently other read clients of the stream
            end
        end
    end
    @assert false # unreachable
end

function read(stream::LibuvStream)
    wait_readnb(stream, typemax(Int))
    return take!(stream.buffer)
end

function unsafe_read(s::LibuvStream, p::Ptr{UInt8}, nb::UInt)
    sbuf = s.buffer
    @assert sbuf.seekable == false
    @assert sbuf.maxsize >= nb

    if bytesavailable(sbuf) >= nb
        return unsafe_read(sbuf, p, nb)
    end

    if nb <= SZ_UNBUFFERED_IO # Under this limit we are OK with copying the array from the stream's buffer
        wait_readnb(s, Int(nb))
        unsafe_read(sbuf, p, nb)
    else
        try
            stop_reading(s) # Just playing it safe, since we are going to switch buffers.
            newbuf = PipeBuffer(unsafe_wrap(Array, p, nb), maxsize = Int(nb))
            newbuf.size = 0 # reset the write pointer to the beginning
            s.buffer = newbuf
            write(newbuf, sbuf)
            wait_readnb(s, Int(nb))
            nb == bytesavailable(newbuf) || throw(EOFError())
        finally
            s.buffer = sbuf
            if !isempty(s.cond)
                start_reading(s) # resume reading iff there are currently other read clients of the stream
            end
        end
    end
    nothing
end

function read(this::LibuvStream, ::Type{UInt8})
    wait_readnb(this, 1)
    buf = this.buffer
    @assert buf.seekable == false
    return read(buf, UInt8)
end

function readavailable(this::LibuvStream)
    wait_readnb(this, 1)
    buf = this.buffer
    @assert buf.seekable == false
    return take!(buf)
end

function readuntil(this::LibuvStream, c::UInt8; keep::Bool=false)
    wait_readbyte(this, c)
    buf = this.buffer
    @assert buf.seekable == false
    return readuntil(buf, c, keep=keep)
end

uv_write(s::LibuvStream, p::Vector{UInt8}) = uv_write(s, pointer(p), UInt(sizeof(p)))

function uv_write(s::LibuvStream, p::Ptr{UInt8}, n::UInt)
    ct = current_task()
    uvw = uv_write_async(s, p, n, ct)
    preserve_handle(ct)
    try
        # wait for the last chunk to complete (or error)
        # assume that any errors would be sticky,
        # (so we don't need to monitor the error status of the intermediate writes)
        wait()
    finally
        if uv_req_data(uvw) != C_NULL
            # uvw is still alive,
            # so make sure we won't get spurious notifications later
            uv_req_set_data(uvw, C_NULL)
        else
            # done with uvw
            Libc.free(uvw)
        end
        unpreserve_handle(ct)
    end
    return Int(n)
end

# helper function for uv_write that returns the uv_write_t struct for the write
# rather than waiting on it
function uv_write_async(s::LibuvStream, p::Ptr{UInt8}, n::UInt, reqdata)
    check_open(s)
    while true
        uvw = Libc.malloc(_sizeof_uv_write)
        uv_req_set_data(uvw, reqdata)
        nwrite = min(n, MAX_OS_WRITE) # split up the write into chunks the OS can handle.
        # TODO: use writev, when that is added to uv-win
        err = ccall(:jl_uv_write,
                    Int32,
                    (Ptr{Cvoid}, Ptr{Cvoid}, UInt, Ptr{Cvoid}, Ptr{Cvoid}),
                    s, p, nwrite, uvw,
                    uv_jl_writecb_task::Ptr{Cvoid})
        if err < 0
            Libc.free(uvw)
            uv_error("write", err)
        end
        n -= nwrite
        p += nwrite
        if n == 0
            return uvw
        end
    end
end

# Optimized send
# - smaller writes are buffered, final uv write on flush or when buffer full
# - large isbits arrays are unbuffered and written directly

function unsafe_write(s::LibuvStream, p::Ptr{UInt8}, n::UInt)
    if s.sendbuf === nothing
        return uv_write(s, p, UInt(n))
    end

    buf = s.sendbuf
    totb = bytesavailable(buf) + n
    if totb < buf.maxsize
        nb = unsafe_write(buf, p, n)
    else
        flush(s)
        if n > buf.maxsize
            nb = uv_write(s, p, n)
        else
            nb = unsafe_write(buf, p, n)
        end
    end
    return nb
end

function flush(s::LibuvStream)
    buf = s.sendbuf
    if buf !== nothing
        if bytesavailable(buf) > 0
            arr = take!(buf)        # Array of UInt8s
            uv_write(s, arr)
            return
        end
    end
    uv_write(s, Ptr{UInt8}(Base.eventloop()), UInt(0)) # zero write from a random pointer to flush current queue
    return
end

buffer_writes(s::LibuvStream, bufsize) = (s.sendbuf=PipeBuffer(bufsize); s)

## low-level calls to libuv ##

function write(s::LibuvStream, b::UInt8)
    buf = s.sendbuf
    if buf !== nothing
        if bytesavailable(buf) + 1 < buf.maxsize
            return write(buf, b)
        end
    end
    return write(s, Ref{UInt8}(b))
end

function uv_writecb_task(req::Ptr{Cvoid}, status::Cint)
    d = uv_req_data(req)
    if d != C_NULL
        uv_req_set_data(req, C_NULL) # let the Task know we got the writecb
        t = unsafe_pointer_to_objref(d)::Task
        if status < 0
            err = _UVError("write", status)
            schedule(t, err, error=true)
        else
            schedule(t)
        end
    else
        # no owner for this req, safe to just free it
        Libc.free(req)
    end
    nothing
end

_fd(x::IOStream) = RawFD(fd(x))
_fd(x::Union{OS_HANDLE, RawFD}) = x

function _fd(x::Union{LibuvStream, LibuvServer})
    fd = Ref{OS_HANDLE}(INVALID_OS_HANDLE)
    if x.status != StatusUninit && x.status != StatusClosed
        err = ccall(:uv_fileno, Int32, (Ptr{Cvoid}, Ptr{OS_HANDLE}), x.handle, fd)
        # handle errors by returning INVALID_OS_HANDLE
    end
    return fd[]
end

for (x, writable, unix_fd, c_symbol) in
        ((:stdin, false, 0, :jl_uv_stdin),
         (:stdout, true, 1, :jl_uv_stdout),
         (:stderr, true, 2, :jl_uv_stderr))
    f = Symbol("redirect_", lowercase(string(x)))
    _f = Symbol("_", f)
    @eval begin
        function ($_f)(stream)
            global $x
            posix_fd = _fd(stream)
            @static if Sys.iswindows()
                ccall(:SetStdHandle, stdcall, Int32, (Int32, OS_HANDLE),
                    $(-10 - unix_fd), Libc._get_osfhandle(posix_fd))
            end
            dup(posix_fd, RawFD($unix_fd))
            $x = stream
            nothing
        end
        function ($f)(handle::Union{LibuvStream, IOStream})
            $(_f)(handle)
            unsafe_store!(cglobal($(Expr(:quote, c_symbol)), Ptr{Cvoid}),
                handle.handle)
            return handle
        end
        function ($f)()
            p = link_pipe!(Pipe())
            read, write = p.out, p.in
            ($f)($(writable ? :write : :read))
            return (read, write)
        end
    end
end

"""
    redirect_stdout([stream]) -> (rd, wr)

Create a pipe to which all C and Julia level [`stdout`](@ref) output
will be redirected.
Returns a tuple `(rd, wr)` representing the pipe ends.
Data written to [`stdout`](@ref) may now be read from the `rd` end of
the pipe. The `wr` end is given for convenience in case the old
[`stdout`](@ref) object was cached by the user and needs to be replaced
elsewhere.

If called with the optional `stream` argument, then returns `stream` itself.

!!! note
    `stream` must be a `TTY`, a `Pipe`, or a socket.
"""
redirect_stdout

"""
    redirect_stderr([stream]) -> (rd, wr)

Like [`redirect_stdout`](@ref), but for [`stderr`](@ref).

!!! note
    `stream` must be a `TTY`, a `Pipe`, or a socket.
"""
redirect_stderr

"""
    redirect_stdin([stream]) -> (rd, wr)

Like [`redirect_stdout`](@ref), but for [`stdin`](@ref).
Note that the order of the return tuple is still `(rd, wr)`,
i.e. data to be read from [`stdin`](@ref) may be written to `wr`.

!!! note
    `stream` must be a `TTY`, a `Pipe`, or a socket.
"""
redirect_stdin

for (F,S) in ((:redirect_stdin, :stdin), (:redirect_stdout, :stdout), (:redirect_stderr, :stderr))
    @eval function $F(f::Function, stream)
        STDOLD = $S
        local ret
        $F(stream)
        try
            ret = f()
        finally
            $F(STDOLD)
        end
        ret
    end
end

"""
    redirect_stdout(f::Function, stream)

Run the function `f` while redirecting [`stdout`](@ref) to `stream`.
Upon completion, [`stdout`](@ref) is restored to its prior setting.

!!! note
    `stream` must be a `TTY`, a `Pipe`, or a socket.
"""
redirect_stdout(f::Function, stream)

"""
    redirect_stderr(f::Function, stream)

Run the function `f` while redirecting [`stderr`](@ref) to `stream`.
Upon completion, [`stderr`](@ref) is restored to its prior setting.

!!! note
    `stream` must be a `TTY`, a `Pipe`, or a socket.
"""
redirect_stderr(f::Function, stream)

"""
    redirect_stdin(f::Function, stream)

Run the function `f` while redirecting [`stdin`](@ref) to `stream`.
Upon completion, [`stdin`](@ref) is restored to its prior setting.

!!! note
    `stream` must be a `TTY`, a `Pipe`, or a socket.
"""
redirect_stdin(f::Function, stream)

mark(x::LibuvStream)     = mark(x.buffer)
unmark(x::LibuvStream)   = unmark(x.buffer)
reset(x::LibuvStream)    = reset(x.buffer)
ismarked(x::LibuvStream) = ismarked(x.buffer)

function peek(s::LibuvStream)
    mark(s)
    try read(s, UInt8)
    finally
        reset(s)
    end
end

# BufferStream's are non-OS streams, backed by a regular IOBuffer
mutable struct BufferStream <: LibuvStream
    buffer::IOBuffer
    r_c::Condition
    close_c::Condition
    is_open::Bool
    buffer_writes::Bool
    lock::ReentrantLock

    BufferStream() = new(PipeBuffer(), Condition(), Condition(), true, false, ReentrantLock())
end

isopen(s::BufferStream) = s.is_open
function close(s::BufferStream)
    s.is_open = false
    notify(s.r_c)
    notify(s.close_c)
    nothing
end
uvfinalize(s::BufferStream) = nothing

read(s::BufferStream, ::Type{UInt8}) = (wait_readnb(s, 1); read(s.buffer, UInt8))
unsafe_read(s::BufferStream, a::Ptr{UInt8}, nb::UInt) = (wait_readnb(s, Int(nb)); unsafe_read(s.buffer, a, nb))
bytesavailable(s::BufferStream) = bytesavailable(s.buffer)

isreadable(s::BufferStream) = s.buffer.readable
iswritable(s::BufferStream) = s.buffer.writable

function wait_readnb(s::BufferStream, nb::Int)
    while isopen(s) && bytesavailable(s.buffer) < nb
        wait(s.r_c)
    end
end

show(io::IO, s::BufferStream) = print(io,"BufferStream() bytes waiting:",bytesavailable(s.buffer),", isopen:", s.is_open)

function wait_readbyte(s::BufferStream, c::UInt8)
    while isopen(s) && !occursin(c, s.buffer)
        wait(s.r_c)
    end
end

wait_close(s::BufferStream) = if isopen(s); wait(s.close_c); end
start_reading(s::BufferStream) = Int32(0)
stop_reading(s::BufferStream) = nothing

write(s::BufferStream, b::UInt8) = write(s, Ref{UInt8}(b))
function unsafe_write(s::BufferStream, p::Ptr{UInt8}, nb::UInt)
    rv = unsafe_write(s.buffer, p, nb)
    !(s.buffer_writes) && notify(s.r_c)
    return rv
end

function eof(s::BufferStream)
    wait_readnb(s, 1)
    return !isopen(s) && bytesavailable(s) <= 0
end

# If buffer_writes is called, it will delay notifying waiters till a flush is called.
buffer_writes(s::BufferStream, bufsize=0) = (s.buffer_writes=true; s)
flush(s::BufferStream) = (notify(s.r_c); nothing)

max_stars_repo_name: hamzaelsaawy/julia

max_issues_repo_name: hamzaelsaawy/julia

max_forks_repo_name: hamzaelsaawy/julia


text:
using Pkg
Pkg.add("WebIO")
Pkg.build("WebIO")
using WebIO, IJulia
WebIO.install_jupyter_labextension()

# Run 'include("ijulia-startup.jl")' in julia, to install all the things that are needed.