text:
using AWS
using Base: CmdRedirect
using Base64
using Dates: datetime2unix, now

const invalid_access_key = "ThisIsMyInvalidAccessKey"
const invalid_secret_key = "ThisIsMyInvalidSecretKey"

function get_auth(config::AWSConfig; params::AbstractDict=Dict())
    id = lpad(!haskey(params, "registryIds") ? "" : first(params["registryIds"]), 12, '0')
    return Dict(
        "authorizationData" => [
            Dict(
                "authorizationToken" => base64encode("AWS:password"),
                "proxyEndpoint" => "https://$(id).dkr.ecr.us-east-1.amazonaws.com",
            ),
        ],
    )
end

get_caller_identity_patch = @patch function AWSTools.get_caller_identity()
    account_id = join(rand(0:9, 12), "")
    return Dict(
        "GetCallerIdentityResult" => Dict(
            Dict(
                "Account" => account_id,
                "Arn" => "arn:aws:iam::$account_id:user/UserName",
                "UserId" => join(rand('A':'Z', 21), ""),
            ),
        ),
    )
end
sts_assume_role = @patch function AWSTools.STS.assume_role(
    role_arn, role_session_name; aws_config
)
    return Dict(
        "AssumeRoleResult" => Dict(
            "Credentials" => Dict(
                "AccessKeyId" => "TESTACCESSKEYID",
                "SecretAccessKey" => "TESTSECRETACEESSKEY",
                "SessionToken" => "TestSessionToken",
                "Expiration" => datetime2unix(now()),
            ),
        ),
    )
end

function instance_metadata_patch(result)
    @patch HTTP.get(args...; kwargs...) = result
end

get_authorization_token_patch = @patch function AWSTools.ECR.get_authorization_token(
    config::AWSConfig, params::AbstractDict
)
    return get_auth(config; params=params)
end

get_authorization_token_no_param_patch = @patch function AWSTools.ECR.get_authorization_token(
    config::AWSConfig
)
    return get_auth(config)
end
describe_stacks_patch = @patch function AWSTools.CloudFormation.describe_stacks(
    config, params
)
    responses = Dict(
        Dict("StackName" => "stackname", "return_raw" => true) => """
                                                                  <DescribeStacksResponse xmlns="http://cloudformation.amazonaws.com/doc/2010-05-15/">
                                                                    <DescribeStacksResult>
                                                                        <Stacks>
                                                                          <member>
                                                                            <StackId>Stack Id</StackId>
                                                                            <StackName>Stack Name</StackName>
                                                                            <Description>Stack Description</Description>
                                                                          </member>
                                                                        </Stacks>
                                                                    </DescribeStacksResult>
                                                                  </DescribeStacksResponse>
                                                                  """,
        Dict("StackName" => "1-stack-output-stackname", "return_raw" => true) => """
                                                                                <DescribeStacksResponse xmlns="http://cloudformation.amazonaws.com/doc/2010-05-15/">
                                                                                <DescribeStacksResult>
                                                                                <Stacks>
                                                                                <member>
                                                                                <Outputs>
                                                                                <member>
                                                                                <OutputKey>TestBucketArn1</OutputKey>
                                                                                <OutputValue>arn:aws:s3:::test-bucket-1</OutputValue>
                                                                                </member>
                                                                                </Outputs>
                                                                                <StackId>Stack Id</StackId>
                                                                                <StackName>Stack Name</StackName>
                                                                                <Description>Stack Description</Description>
                                                                                </member>
                                                                                </Stacks>
                                                                                </DescribeStacksResult>
                                                                                </DescribeStacksResponse>
                                                                                """,
        Dict("StackName" => "multiple-stack-outputs-stackname", "return_raw" => true) => """
                                                                                <DescribeStacksResponse xmlns="http://cloudformation.amazonaws.com/doc/2010-05-15/">
                                                                                <DescribeStacksResult>
                                                                                <Stacks>
                                                                                <member>
                                                                                <Outputs>
                                                                                <member>
                                                                                <OutputKey>TestBucketArn1</OutputKey>
                                                                                <OutputValue>arn:aws:s3:::test-bucket-1</OutputValue>
                                                                                </member>
                                                                                <member>
                                                                                <OutputKey>TestBucketArn2</OutputKey>
                                                                                <OutputValue>arn:aws:s3:::test-bucket-2</OutputValue>
                                                                                </member>
                                                                                </Outputs>
                                                                                <StackId>Stack Id</StackId>
                                                                                <StackName>Stack Name</StackName>
                                                                                <Description>Stack Description</Description>
                                                                                </member>
                                                                                </Stacks>
                                                                                </DescribeStacksResult>
                                                                                </DescribeStacksResponse>
                                                                                """,
        Dict("StackName" => "empty-value", "return_raw" => true) => """
                                                                    <DescribeStacksResponse xmlns="http://cloudformation.amazonaws.com/doc/2010-05-15/">
                                                                      <DescribeStacksResult>
                                                                        <Stacks>
                                                                          <member>
                                                                            <Outputs>
                                                                              <member>
                                                                                <OutputKey>ParquetConversionTriggerName</OutputKey>
                                                                                <OutputValue></OutputValue>
                                                                              </member>
                                                                            </Outputs>
                                                                            <StackId>Stack Id</StackId>
                                                                            <StackName>Stack Name</StackName>
                                                                            <Description>Stack Description</Description>
                                                                          </member>
                                                                        </Stacks>
                                                                      </DescribeStacksResult>
                                                                    </DescribeStacksResponse>
                                                                    """,
        Dict("StackName" => "export", "return_raw" => true) => """
                                                               <DescribeStacksResponse xmlns="http://cloudformation.amazonaws.com/doc/2010-05-15/">
                                                                 <DescribeStacksResult>
                                                                   <Stacks>
                                                                     <member>
                                                                       <Outputs>
                                                                         <member>
                                                                           <Description>Exported output for use in other stacks</Description>
                                                                           <ExportName>ExportedKey</ExportName>
                                                                           <OutputKey>Key</OutputKey>
                                                                           <OutputValue>Value</OutputValue>
                                                                         </member>
                                                                       </Outputs>
                                                                       <StackId>Stack Id</StackId>
                                                                       <StackName>Stack Name</StackName>
                                                                       <Description>Stack Description</Description>
                                                                     </member>
                                                                   </Stacks>
                                                                 </DescribeStacksResult>
                                                               </DescribeStacksResponse>
                                                               """,
    )

    # So we can test that we get an error using the invalid access and secret keys
    access_key = config.credentials.access_key_id
    secret_key = config.credentials.secret_key
    if access_key == invalid_access_key && secret_key == invalid_secret_key
        throw(
            AWSException(
                HTTP.StatusError(
                    403,
                    "",
                    "",
                    HTTP.Messages.Response(
                        403,
                        """
<ErrorResponse xmlns="http://cloudformation.amazonaws.com/doc/2010-05-15/">
    <Error>
    <Type>Sender</Type>
    InvalidClientTokenId
    <Message>The security token included in the request is invalid.</Message>
    </Error>
    <RequestId>cff5beb8-4b7b-11e9-9c2b-43c18f6078dc</RequestId>
</ErrorResponse>
""",
                    ),
                ),
            ),
        )
    else
        return responses[params]
    end
end

function throttle_patch(allow)
    describe_stacks_throttle_count = 0
    @patch function AWSTools.CloudFormation.describe_stacks(config, params)
        describe_stacks_throttle_count += 1
        if !(describe_stacks_throttle_count in allow)
            error_message = """
                <ErrorResponse xmlns="http://cloudformation.amazonaws.com/doc/2010-05-15/">
                    <Error>
                        <Type>Sender</Type>
                        Throttling
                        <Message>Rate exceeded</Message>
                    </Error>
                    <RequestId>d0c477ac-f267-11e8-9d2b-93e3aa6368c5</RequestId>
                </ErrorResponse>
                """
            response = HTTP.Messages.Response(400, error_message)
            http_error = HTTP.ExceptionRequest.StatusError(400, "", "", response)
            throw(AWSException(http_error))
        end
        responses = Dict(
            Dict("StackName" => "stackname", "return_raw" => true) => """
                                                                      <DescribeStacksResponse xmlns="http://cloudformation.amazonaws.com/doc/2010-05-15/">
                                                                        <DescribeStacksResult>
                                                                            <Stacks>
                                                                              <member>
                                                                                <StackId>Stack Id</StackId>
                                                                                <StackName>Stack Name</StackName>
                                                                                <Description>Stack Description</Description>
                                                                                <ThrottleCount>$describe_stacks_throttle_count</ThrottleCount>
                                                                              </member>
                                                                            </Stacks>
                                                                        </DescribeStacksResult>
                                                                      </DescribeStacksResponse>
                                                                      """,
        )

        return responses[params]
    end
end


text:
function makeplots(picdir; Plotter=PyPlot, extension="svg")

    p=plotting_multiscene(Plotter=Plotter)

    fname=joinpath(picdir,"plotting_multiscene."*extension)
    save(fname,p,Plotter=Plotter)
    @test isfile(fname)
    println("multiscene")

    p=plotting_func3d(Plotter=Plotter)
    fname=joinpath(picdir,"plotting_func3d."*extension)
    save(fname,p,Plotter=Plotter)
    @test isfile(fname)
    println("func3d")

    p=plotting_func2d(Plotter=Plotter)
    fname=joinpath(picdir,"plotting_func2d."*extension)
    save(fname,p,Plotter=Plotter)
    @test isfile(fname)
    println("func2d")

    p=plotting_func1d(Plotter=Plotter)
    fname=joinpath(picdir,"plotting_func1d."*extension)
    save(fname,p,Plotter=Plotter)
    @test isfile(fname)
    println("func1d")

    p=plotting_vec2d(Plotter=Plotter)
    fname=joinpath(picdir,"plotting_vec2d."*extension)
    save(fname,p,Plotter=Plotter)
    @test isfile(fname)
    println("vec2d")

    p=plotting_stream2d(Plotter=Plotter)
    fname=joinpath(picdir,"plotting_stream2d."*extension)
    save(fname,p,Plotter=Plotter)
    @test isfile(fname) || GridVisualize.plottername!="PyPlot"
    println("stream2d")

    p=plotting_grid3d(Plotter=Plotter)
    fname=joinpath(picdir,"plotting_grid3d."*extension)
    save(fname,p,Plotter=Plotter)
    @test isfile(fname)
    println("grid3d")

    p=plotting_grid2d(Plotter=Plotter)
    fname=joinpath(picdir,"plotting_grid2d."*extension)
    save(fname,p,Plotter=Plotter)
    @test isfile(fname)
    println("grid2d")

    p=plotting_grid1d(Plotter=Plotter)
    fname=joinpath(picdir,"plotting_grid1d."*extension)
    save(fname,p,Plotter=Plotter)
    @test isfile(fname)
    println("grid1d")

    true
end

ext: jl


text:
"""
    partition!(predicate, v::AbstractVector, r::AbstractRange) → Union{Nothing,Int}

Calling `k = partition!(f, v)` reorders `v` such that `all(f, v[k:end])` and
`all(x -> !f(x), v[1:k-1]` are true. If there is no element in `v` for which the
predicate is false, then `k` is `nothing`.
"""
function partition!(predicate, v::AbstractVector, range::AbstractRange = 1 : length(v))
    @inbounds begin
        isempty(range) && return nothing

        idx = range[1]
        to = range[end]

        while idx ≤ to && predicate(v[idx])
            idx += 1
        end

        idx > to && return nothing

        for j = idx+1:to
            if predicate(v[j])
                v[idx], v[j] = v[j], v[idx]
                idx += 1
            end
        end
        return idx
    end
end


text:
using StanSample, Test

if haskey(ENV, "JULIA_CMDSTAN_HOME")

  TestDir = @__DIR__
  tmpdir = mktempdir()
  println()

  test_bernoulli = [
    "test_keyedarray/test_bernoulli_keyedarray_01.jl",
    "test_keyedarray/test_keyedarray.jl",
  ]

  @testset "Bernoulli array tests" begin
      include(joinpath(TestDir, "test_bernoulli/test_bernoulli_keyedarray_01.jl"))
      println("\n")

      if success(rc)

        sdf = read_summary(sm)
        @test sdf[sdf.parameters .== :theta, :mean][1] ≈ 0.33 rtol=0.05

        (samples, parameters) = read_samples(sm, :array;
          return_parameters=true)
        @test size(samples) == (1000, 1, 6)
        @test length(parameters) == 1

        (samples, parameters) = read_samples(sm, :array;
          return_parameters=true, include_internals=true)
        @test size(samples) == (1000, 8, 6)
        @test length(parameters) == 8

        samples = read_samples(sm, :array;
          include_internals=true)
        @test size(samples) == (1000, 8, 6)

        samples = read_samples(sm, :array)
        @test size(samples) == (1000, 1, 6)
      end

      include(joinpath(TestDir, "test_bernoulli/test_bernoulli_array_02.jl"))
      if success(rc)
        sdf = read_summary(sm)
        @test sdf[sdf.parameters .== :theta, :mean][1] ≈ 0.33 rtol=0.05

        (samples, parameters) = read_samples(sm, :array;
          return_parameters=true)
        @test size(samples) == (500, 1, 4)
        @test length(parameters) == 1

        samples = read_samples(sm, :array;
          include_internals=true)
        @test size(samples) == (500, 8, 4)
      end
  end

  basic_run_tests = [
    "test_bernoulli/test_bernoulli_array_01.jl",
    "test_basic_runs/test_bernoulli_dict.jl",
    "test_basic_runs/test_bernoulli_array_dict_1.jl",
    "test_basic_runs/test_bernoulli_array_dict_2.jl",
    "test_basic_runs/test_parse_interpolate.jl"
  ]

  @testset "Bernoulli basic run tests" begin
    for test in basic_run_tests
      println("\nTesting: $test.")
      include(joinpath(TestDir, test))
      @test sdf[sdf.parameters .== :theta, :mean][1] ≈ 0.33 rtol=0.05
    end
    println()
  end

  sample_settings_tests = [
    "test_sample_settings/test_bernoulli.jl"
  ]

  @testset "Bernoulli Sample() settings tests" begin
    for test in sample_settings_tests
      println("\nTesting: $test.")
      include(joinpath(TestDir, test))
      @test stanmodel.method.adapt.delta ≈ 0.85 atol=0.01
    end
    println()
  end

  generate_quantities_tests = [
    "test_generate_quantities/test_generate_quantities.jl"
  ]

  @testset "Generate_quantities tests" begin
    for test in generate_quantities_tests
      println("\nTesting: $test.")
      include(joinpath(TestDir, test))
    end
    println()
  end

  test_tables_interface = [
    "test-tables-interface/ex-00.jl",
    "test-tables-interface/ex-01.jl",
    "test-tables-interface/ex-02.jl",
    "test-tables-interface/ex-03.jl",
    "test-tables-interface/ex-04.jl",
    "test-tables-interface/ex-05.jl"
  ]

  @testset "Tables.jl interface" begin
    for test in test_tables_interface
      println("\nTesting: $test.")
      include(joinpath(TestDir, test))
    end
    println()
  end

  test_dimensionaldata = [
    "test_dimensionaldata/test_dimensionaldata.jl",
  ]

  @testset "DimensionalData interface" begin
    for test in test_dimensionaldata
      println("\nTesting: $test.")
      include(joinpath(TestDir, test))
    end
    println()
  end

  test_keywords = [
    "test_keywords/test_bernoulli_keyedarray_01.jl",
    "test_keywords/test_bernoulli_keyedarray_02.jl",
    "test_keywords/test_bernoulli_keyedarray_03.jl",
  ]

  @testset "Seed and n_chains keywords" begin
    for test in test_keywords
      println("\nTesting: $test.")
      include(joinpath(TestDir, test))
    end
    println()
  end

  include(joinpath(TestDir, ))
  println()

else
  println("\nJULIA_CMDSTAN_HOME not set. Skipping tests")
end


text:
import accrue.cryptoerase.runtime.Condition;

class C {
    int[] foo = new int[] { m() };
    static final Condition c = new Condition();

    int m() {
	int y = ({L /c T}) 42;
	return y;
    }

    public static void main(String[] args) {
	new C().m();
    }
}

max_stars_repo_name: HarvardPL/cryptoerase

max_issues_repo_name: HarvardPL/cryptoerase

max_forks_repo_name: HarvardPL/cryptoerase


text:
using
  Documenter,
  OceanTurb

makedocs(
     modules = [OceanTurb],
       clean = true,
     doctest = false,
   checkdocs = :all,
      format = Documenter.HTML(prettyurls = get(ENV, "CI", nothing) == "true"),
     authors = "Gregory L. Wagner",
    sitename = "OceanTurb.jl",

       pages = Any[
                "Home" => "index.md",
                "Turbulence, fluxes, and physics" =>  "basics.md",
                "Numerical methods" => "numerics.md",
                "Turbulence models" => Any[
                  "models/kpp.md",
                  "models/modular_kpp.md",
                  "models/tke_mass_flux.md",
                  "models/pacanowskiphilander.md"],
                "DocStrings" => Any[
                      "man/types.md",
                      "man/functions.md"]
                      ]
)

deploydocs(
  repo = "github.com/glwagner/OceanTurb.jl.git",
)


text:
# Construct the various pieces of the DLRM as functions.
struct DLRM
    bottom_mlp
    embedding_tables
    interaction
    top_mlp
end

function create_dlrm(;
        m_sparse,       # Output feature size of the embedding tables.
        ln_emb,         # collection of "n" sizes for the embedding tables.
        ln_bot,         # dimensions for the bottom mlp
        ln_top,         # dimensions for the top mlp
        arch_interaction_op,        # interaction between sparse and dense dimensions
        sigmoid_bot = -1,
        sigmoid_top = -1,
        loss_threshold = 0.0,
    )

    # Create embedding layers
    embedding_tables = create_embedding(m_sparse, ln_emb)
    bottom_mlp = create_mlp(ln_bot, sigmoid_bot)
    interaction = arch_interaction_op
    top_mlp = create_mlp(ln_top, sigmoid_top)

    return DLRM(
        bottom_mlp,
        embedding_tables,
        interaction,
        top_mlp
    )
end

function runlayers(layers, x)
    for l in layers
        x = l(x)
    end
    return x
end

function (D::DLRM)(dense_input, sparse_inputs...)
    # Run the bottom MLP
    dense_features = runlayers(D.bottom_mlp, dense_input)

    # Run the embedding tables.
    sparse_features = []
    for (i, indices) in enumerate(sparse_inputs)
        push!(sparse_features, D.embedding_tables[i](indices))
    end

    # Run the interaction.
    Z = D.interaction(dense_features, sparse_features)

    # Run the top MLP
    return runlayers(D.top_mlp, Z)
end

function create_embedding(sparse_dim::Integer, sizes)
    tables = []
    for sz in sizes
        # Create the table.
        table = Flux.param(randn(Float32, sparse_dim, sz))

        # The Facebook implementations use the embedding bag approach, but looking at the
        # output is is just performing a standard embedding - so that's what we do here.
        f = function(x)
            n = nGraph.embedding(x, nGraph.Node(table))
            nGraph.__inplace(table)
            return n
        end
        push!(tables, f)
    end
    return tables
end

function create_mlp(sizes, sigmoid_layer)
    layers = []
    for i in 1:length(sizes) - 1
        m = sizes[i]
        n = sizes[i+1]

        if i == sigmoid_layer
            σ = Flux.sigmoid
        else
            σ = Flux.relu
        end
        L = Dense(m, n, σ)
        push!(layers, L)
    end
    return layers
end

function kaggle_dlrm(;m_sparse = 16, batchsize = 128, dense_input_size = 13)
    # Extracted from the Facebook implementation.
    embedding_sizes = 5 .* [
        1461     ,
        586      ,
        10131227 ,
        2202608  ,
        306      ,
        24       ,
        12518    ,
        634      ,
        4        ,
        93146    ,
        5684     ,
        8351593  ,
        3195     ,
        28       ,
        14993    ,
        5461306  ,
        11       ,
        5653     ,
        2173     ,
        4        ,
        7046547  ,
        18       ,
        16       ,
        286181   ,
        105      ,
        142572   ,
    ]

    bottom_mlp_sizes = [13, 512, 256, 64, 16]
    bottom_mlp_sigmoid = -1

    # Replace the first entry with 745 becuase we aren't able to do the triangular selection
    # thing easily in nGraph.
    top_mlp_sizes = [745, 512, 256, 1]
    top_mlp_sigmoid = 3

    interaction = dot_interaction

    dlrm = create_dlrm(;
        m_sparse = m_sparse,
        ln_emb = embedding_sizes,
        ln_bot = bottom_mlp_sizes,
        ln_top = top_mlp_sizes,
        arch_interaction_op = interaction,
        sigmoid_bot = bottom_mlp_sigmoid,
        sigmoid_top = top_mlp_sigmoid,
    )

    backend = nGraph.Backend("CPU")

    # Create appropriate inputs.
    dense_input = randn(Float32, dense_input_size, batchsize)
    sparse_inputs = [rand(Int32(1):Int32(e), batchsize) for e in embedding_sizes]

    loss(y, x...) = Flux.crossentropy(dlrm(x...), y)

    expected_values = rand((Float32(0),Float32(1)), 1, batchsize)
    kw = (optimizer = nGraph.SGD(Float32(0.1)),)

    return loss, (expected_values, dense_input, sparse_inputs...), kw
end

#####
##### dot interactions
#####

_extrude_reshape(x) = reshape(x, size(x, 1), 1, size(x, 2))

function dot_interaction(dense_features, sparse_features)
    # Concatenate features along the second dimension
    # Reshape everything to a 3d vector
    T′ = dense_features
    for S in sparse_features
        T′ = cat(T′, S; dims = 1)
    end
    T = reshape(
        T′,
        size(dense_features, 1),
        :,
        size(dense_features, 2),
    )

    @show size(T)
    # Perform a dot product
    Z = nGraph.bmm(T, T; transpose_b = true)
    Z = reshape(Z, :, size(Z, 3))

    @show size(dense_features)
    @show size(Z)

    # Change everything to a row vector - concatenate dense features onto the
    return cat(dense_features, Z; dims = 1)
end

max_stars_repo_name: darchr/AutoTM

max_issues_repo_name: darchr/AutoTM

max_forks_repo_name: darchr/AutoTM


text:
module TestExtras

export @constinferred, @constinferred_broken
export @timedtestset
export ConstInferred

include("constinferred.jl")
include("timedtest.jl")

using .ConstInferred: @constinferred, @constinferred_broken
using .TimedTests: @timedtestset

end


text:
using Test
using TropicalNumbers

@testset "tropical" begin
    @test Tropical(3) * Tropical(4) == Tropical(7)
    @test Tropical(3) + Tropical(4) == Tropical(4)
    @test Tropical(4) + Tropical(-1) == Tropical(4)
    @test zero(Tropical(2)).n .< -99999
    @test zero(Tropical(2.0)) == Tropical(-Inf)
    @test one(Tropical(2)) == Tropical(0)
    @test Tropical(2.0) ≈ Tropical(2.0 + 1e-10)
    @test Tropical(2) ≈ Tropical(2.0)
    @test TropicalF32(2.0).n isa Float32
    @test TropicalF16(2.0).n isa Float16
    @test TropicalF64(2.0).n isa Float64

    @test Tropical(0//3) * Tropical(1//3) == Tropical(1//3)
    @test Tropical(1//3) * Tropical(0//3) == Tropical(1//3)
    @test Tropical(1//3) * Tropical(1//3) == Tropical(2//3)

    @test Tropical(1//3) * Tropical(1//0) == Tropical(1//0)
    @test Tropical(1//3) * Tropical(-1//0) == Tropical(-1//0)
    @test Tropical(1//0) * Tropical(1//1) == Tropical(1//0)
    @test Tropical(-1//0) * Tropical(-1//1) == Tropical(-1//0)
    @test content(Tropical(3.0)) == 3.0
    @test Tropical{Float32}(Tropical(0.0)) isa Tropical{Float32}
    println(TropicalF64(3))

    # promote and convert
    t1 = Tropical(2)
    t2 = Tropical(2.0)
    @test TropicalF64(t1) === Tropical(2.0)
    @test promote(t1, t2) === (Tropical(2.0), t2)

    @test content(TropicalF64) == Float64

    @test promote(Tropical{Float64}(1), Tropical{Int64}(2)) == (Tropical{Float64}(1), Tropical{Float64}(2))
    @test promote(Tropical{Float64}(1)) == (Tropical{Float64}(1),)
    @test promote_type(Tropical{Float64}, TropicalF32) == TropicalF64
    @test promote_type(Tropical{Float64}, TropicalF32, Tropical{Int32}) == TropicalF64

    @test Tropical(3) / Tropical(4) == Tropical(-1)
    @test Tropical(3) ÷ Tropical(4) == Tropical(-1)
    @test inv(Tropical(3)) == Tropical(-3)

    x = Tropical(2.0)
    @test x * true == x * one(x)
    @test x / true == x / one(x)
    @test x ÷ true == x ÷ one(x)
    @test x * false == x * zero(x)
    @test x / false == x / zero(x)
    @test x ÷ false == x ÷ zero(x)
    @test true * x == one(x) * x
    @test true / x == one(x) / x
    @test true ÷ x == one(x) ÷ x
    @test false * x == zero(x) * x
    @test false / x == zero(x) / x
    @test false ÷ x == zero(x) ÷ x
    @test isnan(Tropical(NaN))
    @test !isnan(Tropical(-Inf))
end


text:
################################################################################
#                           DERIVATIVE METHOD DATATYPES
################################################################################
"""
    OrthogonalCollocation{Q <: MeasureToolbox.AbstractUnivariateMethod
                          } <: GenerativeDerivativeMethod

A `DataType` for storing information about orthogonal collocation over finite
elements to approximate derivatives. The constructor is of the form:
```
    OrthogonalCollocation(num_nodes::Int,
                          [quad::AbstractUnivariateMethod = GaussLobatto])
```

**Fields**
- `num_nodes::Int`: The number of collocation points (nodes) per finite element.
- `quadrature_method::Q`: The quadrature method uses to choose the collocation points.
"""
struct OrthogonalCollocation{Q <: MeasureToolbox.AbstractUnivariateMethod
                             } <: GenerativeDerivativeMethod
    num_nodes::Int
    quadrature_method::Q
    function OrthogonalCollocation{Q}(
        num_nodes::Int,
        quad::Q
        ) where {Q <: MeasureToolbox.AbstractUnivariateMethod}
        return new{Q}(num_nodes, quad)
    end
end

# Define Lobatto constructor
function OrthogonalCollocation(num_nodes::Int, quad::Q
    )::OrthogonalCollocation{Q} where {Q <: MeasureToolbox.GaussLobatto}
    num_nodes >= 2 || error("Must specify at least 2 collocation points (i.e., " *
                            "the bounds of each support interval with no internal " *
                            "support nodes).")
    return OrthogonalCollocation{Q}(num_nodes, quad)
end

# Define default constructor
function OrthogonalCollocation(num_nodes::Int
    )::OrthogonalCollocation{MeasureToolbox.GaussLobatto}
    return OrthogonalCollocation(num_nodes, MeasureToolbox.GaussLobatto())
end

################################################################################
#                                HELPER METHODS
################################################################################
"""
    generative_support_info(method::AbstractDerivativeMethod)::AbstractGenerativeInfo

Return the [`AbstractGenerativeInfo`](@ref) associated with `method`. This is
intended as an internal method and should be extended for user-defined derivative
methods are [`GenerativeDerivativeMethod`](@ref)s.
"""
function generative_support_info(method::AbstractDerivativeMethod)
    error("`generative_support_info` is undefined for derivative method type " *
          "`$(typeof(method))`.")
end

# NonGenerativeDerivativeMethods
function generative_support_info(
    method::NonGenerativeDerivativeMethod
    )::NoGenerativeSupports
    return NoGenerativeSupports()
end

# OrthogonalCollocation with GaussLabatto
function generative_support_info(
    method::OrthogonalCollocation{MeasureToolbox.GaussLobatto}
    )::AbstractGenerativeInfo
    num_nodes = method.num_nodes
    if num_nodes == 2
        return NoGenerativeSupports()
    else
        (nodes, _) = FastGaussQuadrature.gausslobatto(method.num_nodes)
        deleteat!(nodes, length(nodes))
        deleteat!(nodes, 1)
        return UniformGenerativeInfo(nodes, MeasureToolbox.InternalGaussLobatto, -1, 1)
    end
end

# OrthogonalCollocation fallback
function generative_support_info(method::OrthogonalCollocation{Q}) where {Q}
    error("Invalid `OrthogonalCollocation` with unknown quadrature method `$Q`.")
end

"""
    support_label(method::GenerativeDerivativeMethod)

Return the support label associated with `method` if there is one, errors otherwise.
This depends on [`generative_support_info`](@ref generative_support_info(::AbstractDerivativeMethod))
being defined for the type of `method`.
"""
function support_label(method::AbstractDerivativeMethod)::DataType
    error("`support_label` not defined for derivative methods of type " *
          "`$(typeof(method))`.")
end

# Extend support_label for GenerativeDerivativeMethods
function support_label(method::GenerativeDerivativeMethod)::DataType
    return support_label(generative_support_info(method))
end

"""
    make_reduced_expr(vref::GeneralVariableRef, pref::GeneralVariableRef,
                      support::Float64, write_model::Union{InfiniteModel, JuMP.Model})

Given the argument variable `vref` and the operator parameter `pref` from a
derivative, build and return the reduced expression in accordance to the support
`support` with respect to `pref`. New point/semi-infinite variables will be written to
`write_model`. This is solely intended as a helper function for derivative
evaluation.
"""
function make_reduced_expr(vref::GeneralVariableRef, pref::GeneralVariableRef,
                           support::Float64, write_model::JuMP.AbstractModel)
    return make_reduced_expr(vref, _index_type(vref), pref, support, write_model)
end

# MeasureIndex
function make_reduced_expr(mref, ::Type{MeasureIndex}, pref, support, write_model)
    data = DiscreteMeasureData(pref, [1], [support], InternalLabel, # NOTE the label will not be used
                               default_weight, NaN, NaN, false)
    return expand_measure(mref, data, write_model)
end

# TODO prevent the redundant generation of point and semi-infinite variables for overlapping expressions
# InfiniteVariableIndex/DerivativeIndex
function make_reduced_expr(vref,
    ::Union{Type{InfiniteVariableIndex}, Type{DerivativeIndex}, Type{ParameterFunctionIndex}},
    pref,
    support,
    write_model
    )
    prefs = parameter_list(vref)
    # only one parameter so we have to make a point variable (we know that this must be pref)
    if length(prefs) == 1
        return make_point_variable_ref(write_model, vref, [support])
    # there are other parameters so make semi-infinite variable
    else
        pindex = findfirst(isequal(pref), prefs)
        return make_semi_infinite_variable_ref(write_model, vref, [pindex], [support])
    end
end

# SemiInfiniteVariableIndex
function make_reduced_expr(vref, ::Type{SemiInfiniteVariableIndex}, pref, support,
                           write_model)::GeneralVariableRef
    # get the preliminary info
    dvref = dispatch_variable_ref(vref)
    ivref = infinite_variable_ref(vref)
    var_prefs = parameter_list(dvref)
    orig_prefs = parameter_list(ivref)
    eval_supps = eval_supports(dvref)
    pindex = findfirst(isequal(pref), orig_prefs)
    # we only have 1 parameter so we need to make a point variable
    if length(var_prefs) == 1
        processed_support = _make_point_support(orig_prefs, eval_supps, pindex, support)
        return make_point_variable_ref(write_model, ivref, processed_support)
    # otherwise we need to make another semi-infinite variable
    else
        indices = collect(keys(eval_supps))
        vals = map(k -> eval_supps[k], indices)
        push!(indices, pindex)
        push!(vals, support)
        return make_semi_infinite_variable_ref(write_model, ivref, indices, vals)
    end
end

################################################################################
#                        EVALUATE_DERIVIATIVE DEFINITIONS
################################################################################
"""
    evaluate_derivative(dref::GeneralVariableRef,
                        method::AbstractDerivativeMethod,
                        write_model::JuMP.AbstractModel)::Vector{JuMP.AbstractJuMPScalar}

Build expressions for derivative `dref` evaluated in accordance with `method`.
The expressions are of the form `lhs - rhs`, where `lhs` is a function of derivatives evaluated at some supports
for certain infinite parameter, and `rhs` is a function of the derivative arguments
evaluated at some supports for certain infinite parameter. For example, for finite difference
methods at point `t = 1`, `lhs` is `Δt * ∂/∂t[T(1)]`, and `rhs` could be `T(1+Δt) - T(1)` in
case of forward difference mode. This is intended as a helper function for `evaluate`, which
will take the the expressions generated by this method and generate constraints that approximate
the derivative values by setting the expressions as 0. However, one can extend this function
to encode custom methods for approximating derivatives. This should invoke
`add_derivative_supports` if the method is generative and users will likely find
it convenient to use `make_reduced_expr`.
"""
function evaluate_derivative(
    dref::GeneralVariableRef,
    method::AbstractDerivativeMethod,
    write_model::JuMP.AbstractModel)
    error("`evaluate_derivative` not defined for derivative method of type " *
          "$(typeof(method)).")
end

## Define helper methods for finite difference
# Forward
function _make_difference_expr(dref::GeneralVariableRef,
                               vref::GeneralVariableRef,
                               pref::GeneralVariableRef,
                               index::Int,
                               ordered_supps::Vector{Float64},
                               write_model::JuMP.AbstractModel,
                               type::Forward)::JuMP.AbstractJuMPScalar
    curr_value = ordered_supps[index]
    next_value = ordered_supps[index+1]
    return JuMP.@expression(_Model, (next_value - curr_value) *
                                    make_reduced_expr(dref, pref, curr_value, write_model) -
                                    make_reduced_expr(vref, pref, next_value, write_model) +
                                    make_reduced_expr(vref, pref, curr_value, write_model))
end

# Central
function _make_difference_expr(dref::GeneralVariableRef,
                               vref::GeneralVariableRef,
                               pref::GeneralVariableRef,
                               index::Int,
                               ordered_supps::Vector{Float64},
                               write_model::JuMP.AbstractModel,
                               type::Central)::JuMP.AbstractJuMPScalar
    prev_value = ordered_supps[index-1]
    next_value = ordered_supps[index+1]
    curr_value = ordered_supps[index]
    return JuMP.@expression(_Model, (next_value - prev_value) *
                                    make_reduced_expr(dref, pref, curr_value, write_model) -
                                    make_reduced_expr(vref, pref, next_value, write_model) +
                                    make_reduced_expr(vref, pref, prev_value, write_model))
end

# Backward
function _make_difference_expr(dref::GeneralVariableRef,
                               vref::GeneralVariableRef,
                               pref::GeneralVariableRef,
                               index::Int,
                               ordered_supps::Vector{Float64},
                               write_model::JuMP.AbstractModel,
                               type::Backward)::JuMP.AbstractJuMPScalar
    prev_value = ordered_supps[index-1]
    curr_value = ordered_supps[index]
    return JuMP.@expression(_Model, (curr_value - prev_value) *
                                    make_reduced_expr(dref, pref, curr_value, write_model) -
                                    make_reduced_expr(vref, pref, curr_value, write_model) +
                                    make_reduced_expr(vref, pref, prev_value, write_model))
end

# Fallback
function _make_difference_expr(dref, vref, pref, index, supps, model, type)
    error("Undefined `FiniteDifference` technique `$(typeof(type))`.")
end

# evaluate_derivative for FiniteDifference
function evaluate_derivative(
    dref::GeneralVariableRef,
    method::FiniteDifference,
    write_model::JuMP.AbstractModel
    )::Vector{JuMP.AbstractJuMPScalar}
    # gather the arugment and parameter
    vref = derivative_argument(dref)
    pref = operator_parameter(dref)
    # get the supports and check validity
    ordered_supps = sort!(supports(pref, label = All))
    n_supps = length(ordered_supps)
    n_supps <= 1 && error("$(pref) does not have enough supports for derivative evaluation.")
    # make the expressions
    exprs = Vector{JuMP.AbstractJuMPScalar}(undef, n_supps - 2)
    for i in eachindex(exprs)
        exprs[i] = _make_difference_expr(dref, vref, pref, i + 1, ordered_supps,
                                         write_model, method.technique)
    end
    if method.add_boundary_constraint && method.technique isa Forward
        push!(exprs, _make_difference_expr(dref, vref, pref, 1, ordered_supps,
                                           write_model, Forward()))
    elseif method.add_boundary_constraint && method.technique isa Backward
        push!(exprs, _make_difference_expr(dref, vref, pref, n_supps, ordered_supps,
                                           write_model, Backward()))
    end
    return exprs
end

# Evaluate_derivative for OrthogonalCollocation with Labatto
# This is based on the collocation scheme presented in "Nonlinear Modeling,
# Estimation and Predictive Control in APMonitor"
function evaluate_derivative(
    dref::GeneralVariableRef,
    method::OrthogonalCollocation{MeasureToolbox.GaussLobatto},
    write_model::JuMP.AbstractModel
    )::Vector{JuMP.AbstractJuMPScalar}
    # gather the arugment and parameter
    vref = derivative_argument(dref)
    pref = operator_parameter(dref)
    # ensure that the internal supports are added in case they haven't already
    add_generative_supports(pref) # this checks for enough supports
    # gather the supports and indices of the internal supports
    all_supps = supports(pref, label = All) # sorted by virtue of SortedDict
    bool_inds = map(s -> !(support_label(method) in s),
                    values(_parameter_supports(dispatch_variable_ref(pref))))
    interval_inds = findall(bool_inds)
    n_nodes = method.num_nodes - 2 # we want the # of internal nodes
    # compute M matrix based on the node values and generate expressions using
    # the M matrix and `make_reduced_expr`
    exprs = Vector{JuMP.AbstractJuMPScalar}(undef, length(all_supps) - 1)
    counter = 1
    for i in Iterators.take(eachindex(interval_inds), length(interval_inds) - 1)
        # collect the supports
        lb = all_supps[interval_inds[i]]
        i_nodes = all_supps[interval_inds[i]+1:interval_inds[i+1]] .- lb
        # build the matrices in memory order (column-wise using the transpose form)
        M1t = Matrix{Float64}(undef, n_nodes+1, n_nodes+1)
        M2t = Matrix{Float64}(undef, n_nodes+1, n_nodes+1)
        for j in eachindex(i_nodes) # support index
            for k in eachindex(i_nodes) # polynomial index
                M1t[k, j] = k * i_nodes[j]^(k-1)
                M2t[k, j] = i_nodes[j]^k
            end
        end
        Minvt = M1t \ M2t
        for j in eachindex(i_nodes)
            exprs[counter] = JuMP.@expression(_Model, sum(Minvt[k, j] *  make_reduced_expr(dref, pref, i_nodes[k] + lb, write_model)
                                                          for k in eachindex(i_nodes)) -
                                                      make_reduced_expr(vref, pref, i_nodes[j] + lb, write_model) +
                                                      make_reduced_expr(vref, pref, lb, write_model))
            counter += 1
        end
    end
    return exprs
end

################################################################################
#                              EVALUATION METHODS
################################################################################
"""
    evaluate(dref::DerivativeRef)::Nothing

Numerically evaluate `dref` by computing its auxiliary derivative constraints
(e.g., collocation equations) and add them to the model. For normal usage, it is
recommended that this method not be called directly and instead have TranscriptionOpt
handle these equations. Errors if `evaluate_derivative` is not
defined for the derivative method employed.

The resulting constraints can be accessed via `derivative_constraints`.

**Example**
```julia-repl
julia> m = InfiniteModel(); @infinite_parameter(m, t in [0,2]); @variable(m, T, Infinite(t));

julia> dref = @deriv(T,t)
∂/∂t[T(t)]

julia> add_supports(t, [0, 0.5, 1, 1.5, 2])

julia> evaluate(dref)

julia> derivative_constraints(dref)
Feasibility
4-element Array{InfOptConstraintRef,1}:
 0.5 ∂/∂t[T(t)](0.5) - T(0.5) + T(0) = 0.0
 0.5 ∂/∂t[T(t)](1) - T(1) + T(0.5) = 0.0
 0.5 ∂/∂t[T(t)](1.5) - T(1.5) + T(1) = 0.0
 0.5 ∂/∂t[T(t)](2) - T(2) + T(1.5) = 0.0
```
"""
function evaluate(dref::DerivativeRef)::Nothing
    if !has_derivative_constraints(dref)
        # collect the basic info
        method = derivative_method(dref)
        model = JuMP.owner_model(dref)
        constr_list = _derivative_constraint_dependencies(dref)
        # get the expressions
        gvref = GeneralVariableRef(model, JuMP.index(dref).value, DerivativeIndex)
        exprs = evaluate_derivative(gvref, method, model)
        # add the constraints
        for expr in exprs
            cref = JuMP.@constraint(model, expr == 0)
            push!(constr_list, JuMP.index(cref))
        end
        # update the parameter status
        pref = dispatch_variable_ref(operator_parameter(dref))
        _set_has_derivative_constraints(pref, true)
    end
    return
end

"""
    evaluate_all_derivatives!(model::InfiniteModel)::Nothing

Evaluate all the derivatives in `model` by adding the corresponding auxiliary
equations to `model`. See [`evaluate`](@ref) for more information.

**Example**
```julia-repl
julia> m = InfiniteModel();

julia> @infinite_parameter(m, t in [0,2], supports = [0, 1, 2]);

julia> @infinite_parameter(m, x in [0,1], supports = [0, 0.5, 1]);

julia> @variable(m, T, Infinite(x, t));

julia> dref1 = @deriv(T, t); dref2 = @deriv(T, x^2);

julia> evaluate_all_derivatives!(m)

julia> print(m)
Feasibility
Subject to
 ∂/∂t[T(x, t)](x, 1) - T(x, 1) + T(x, 0) = 0.0, ∀ x ∈ [0, 1]
 ∂/∂t[T(x, t)](x, 2) - T(x, 2) + T(x, 1) = 0.0, ∀ x ∈ [0, 1]
 0.5 ∂/∂x[T(x, t)](0.5, t) - T(0.5, t) + T(0, t) = 0.0, ∀ t ∈ [0, 2]
 0.5 ∂/∂x[T(x, t)](1, t) - T(1, t) + T(0.5, t) = 0.0, ∀ t ∈ [0, 2]
 0.5 ∂/∂x[∂/∂x[T(x, t)]](0.5, t) - ∂/∂x[T(x, t)](0.5, t) + ∂/∂x[T(x, t)](0, t) = 0.0, ∀ t ∈ [0, 2]
 0.5 ∂/∂x[∂/∂x[T(x, t)]](1, t) - ∂/∂x[T(x, t)](1, t) + ∂/∂x[T(x, t)](0.5, t) = 0.0, ∀ t ∈ [0, 2]
```
"""
function evaluate_all_derivatives!(model::InfiniteModel)::Nothing
    for dref in all_derivatives(model)
        evaluate(dref)
    end
    return
end


text:
using Random
using naumer_Dimensionality_2022
using CSV

rng = MersenneTwister(4321)

const max_samples = 10000

dataset1 = samplePSD(rng,max_samples,1,1)
dataset2 = samplePSD(rng,max_samples,2,1)
dataset3 = samplePSD(rng,max_samples,3,1)

function minimumDistance(sample, dataset)
    return minimum(sqrt.(sum(abs2, dataset .- sample,dims=(1,2))))
end

function EstimateMinDist(rng, dataset, N, iterations)
    total = 0
    for i in 1:iterations
        mini_dataset = dataset[:,:,randperm(max_samples)[1:(N+1)]]
        chosen_index = rand(1:(N+1))
        chosen_sample = mini_dataset[:,:,chosen_index]
        mini_dataset = cat(mini_dataset[:,:,1:chosen_index-1],mini_dataset[:,:,chosen_index+1:end]; dims=3)
        dist = minimumDistance(chosen_sample, mini_dataset)
        total += dist
    end
    return total/iterations
end

dist1 = zeros(500)
dist2 = zeros(500)
dist3 = zeros(500)

Threads.@threads for i in 1:500
    dist1[i] = EstimateMinDist(rng, dataset1, i, 5000)
    dist2[i] = EstimateMinDist(rng, dataset2, i, 5000)
    dist3[i] = EstimateMinDist(rng, dataset3, i, 5000)
end

CSV.write("SamplingData.csv",(dist1 = dist1, dist2 = dist2, dist3 = dist3))


text:
module PiCraft

using Sockets
using GZip

include("api.jl")
include("blocks.jl")
include("turtle.jl")
include("drawing.jl")
include("NBT.jl")
include("buildTools.jl")

end


text:
module Olson

using Base.Dates

import ..TimeZones: TZDATA_DIR, COMPILED_DIR, ZERO, MIN_GMT_OFFSET, MAX_GMT_OFFSET,
    MIN_SAVE, MAX_SAVE, ABS_DIFF_OFFSET, Time
import ..TimeZones: TimeZone, FixedTimeZone, VariableTimeZone, Transition, Time

# Zone type maps to an Olson Timezone database entity
type Zone
    gmtoffset::Time
    save::Time
    rules::AbstractString
    format::AbstractString
    until::Nullable{DateTime}
    until_flag::Char
end

function Base.isless(x::Zone,y::Zone)
    x_dt = get(x.until, typemax(DateTime))
    y_dt = get(y.until, typemax(DateTime))

    # Easy to compare until's if they are using the same flag. Alternatively, it should
    # be safe to compare different until flags if the DateTimes are far enough apart.
    if x.until_flag == y.until_flag || abs(x_dt - y_dt) > ABS_DIFF_OFFSET
        return isless(x_dt, y_dt)
    else
        error("Unable to compare zones when until datetimes are too close and flags are mixed")
    end
end

# Rules govern how Daylight Savings transitions happen for a given time zone
type Rule
    from::Nullable{Int}  # First year rule applies
    to::Nullable{Int}    # Rule applies up until, but not including this year
    month::Int           # Month in which DST transition happens
    on::Function         # Anonymous boolean function to determine day
    at::Time             # Hour and minute at which the transition happens
    at_flag::Char        # Local wall time (w), UTC time (u), Local Standard time (s)
    save::Time           # How much time is "saved" in daylight savings transition
    letter::AbstractString  # Timezone abbr letter(s). ie. CKT ("") => CKHST ("HS")
end

typealias ZoneDict Dict{AbstractString,Array{Zone}}
typealias RuleDict Dict{AbstractString,Array{Rule}}
typealias OrderedRuleDict Dict{AbstractString,Tuple{Array{Date},Array{Rule}}}

# Min and max years that we create DST transition DateTimes for (inclusive)
const MIN_YEAR = year(typemin(DateTime))  # Essentially the begining of time
const MAX_YEAR = 2037                     # year(unix2datetime(typemax(Int32))) - 1

# Helper functions/data
const MONTHS = Dict("Jan"=>1,"Feb"=>2,"Mar"=>3,"Apr"=>4,"May"=>5,"Jun"=>6,
                "Jul"=>7,"Aug"=>8,"Sep"=>9,"Oct"=>10,"Nov"=>11,"Dec"=>12)

const DAYS = Dict("Mon"=>1,"Tue"=>2,"Wed"=>3,"Thu"=>4,"Fri"=>5,"Sat"=>6,"Sun"=>7)

# Create adjuster functions such as "lastSun".
for (abbr, dayofweek) in DAYS
    sym = Symbol("last" * abbr)
    @eval (
        function $sym(dt)
            return dayofweek(dt) == $dayofweek &&
            dayofweekofmonth(dt) == daysofweekinmonth(dt)
        end
    )
end

function parseflag(s::AbstractString)
    if s == "" || s == "w"
        return 'w'
    elseif s == "u"
        return 'u'
    elseif s == "s"
        return 's'
    else
        throw(ArgumentError("Unhandled flag $s"))
    end
end

# Olson time zone dates can be a single year (1900), yyyy-mm-dd (1900-Jan-01),
# or minute-precision (1900-Jan-01 2:00).
# They can also be given in Local Wall Time, UTC time (u), or Local Standard time (s)
function parsedate(s::AbstractString)
    s = replace(s, r"\s+", " ")
    num_periods = length(split(s, " "))
    s, flag = num_periods > 3 && isalpha(s[end]) ? (s[1:end-1], s[end:end]) : (s, "")
    if contains(s,"lastSun")
        dt = DateTime(replace(s, "lastSun", "1", 1), "yyyy uuu d H:MM:SS")
        dt = tonext(lastSun, dt; same=true)
    elseif contains(s,"lastSat")
        dt = DateTime(replace(s, "lastSat", "1", 1), "yyyy uuu d H:MM:SS")
        dt = tonext(lastSat, dt; same=true)
    elseif contains(s,"Sun>=1")
        dt = DateTime(replace(s,"Sun>=", "", 1),"yyyy uuu d H:MM:SS")
        dt = tonext(d -> dayofweek(d) == Sun, dt; same=true)
    else
        format = join(split("yyyy uuu dd HH:MM:SS", " ")[1:num_periods], ' ')
        periods = Dates.parse(s, DateFormat(format))

        # Deal with zone "Pacific/Apia" which has a 24:00 datetime.
        if length(periods) > 3 && periods[4] == Hour(24)
            periods[4] = Hour(0)
            periods[3] += Day(1)
        end
        dt = DateTime(periods...)
    end

    # TODO: I feel like there are issues here.
    # If the time is UTC, we add back the offset and any saved amount
    # If it's local standard time, we just need to add any saved amount
    # return letter == 's' ? (dt - save) : (dt - offset - save)

    return dt, parseflag(flag)
end

function asutc(dt::DateTime, flag::Char, offset::Time, save::Time)
    if flag == 'u'
        # In UTC
        return dt
    elseif flag == 'w'
        # In local wall time, add back offset and saved amount
        return dt - offset - save
    elseif flag == 's'
        # In local standard time, add back any saved amount
        return dt - offset
    else
        error("Unknown flag: $flag")
    end
end

function abbr_string(format::AbstractString, save::Time, letter::AbstractString="")
    # Note: using @sprintf would make sense but unfortunately it doesn't accept a
    # format as a variable.
    abbr = replace(format,"%s",letter,1)

    # If the abbreviation contains a slash then the first string is the abbreviation
    # for standard time and the second is the abbreviation for daylight saving time.
    abbrs = split(abbr, '/')
    if length(abbrs) > 1
        abbr = save == ZERO ? first(abbrs) : last(abbrs)
    end

    # Some time zones (e.g. "Europe/Ulyanovsk") do not have abbreviations for the various
    # rules and instead hardcode the offset as the name.
    if ismatch(r"[+-]\d{2}", abbr)
        abbr = ""
    end

    return abbr
end

function ruleparse(from, to, rule_type, month, on, at, save, letter)
    from_int = Nullable{Int}(from == "min" ? nothing : parse(Int, from))
    to_int = Nullable{Int}(to == "only" ? from_int : to == "max" ? nothing : parse(Int, to))
    month_int = MONTHS[month]

    # Now we need to get the right anonymous function
    # for determining the right day for transitioning
    if ismatch(r"last\w\w\w", on)
        # We pre-built these functions above
        # They follow the format: "lastSun", "lastMon".
        on_func = eval(Symbol(on))
    elseif ismatch(r"\w\w\w[<>]=\d\d?", on)
        # The first day of the week that occurs before or after a given day of month.
        # i.e. Sun>=8 refers to the Sunday after the 8th of the month
        # or in other words, the 2nd Sunday.
        dow::Int = DAYS[match(r"\w\w\w", on).match]
        dom::Int = parse(Int, match(r"\d\d?", on).match)
        if ismatch(r"<=", on)
            on_func = (dt -> day(dt) <= dom && dayofweek(dt) == dow)
        else
            on_func = (dt -> day(dt) >= dom && dayofweek(dt) == dow)
        end
    elseif ismatch(r"\d\d?", on)
        # Matches just a plain old day of the month
        dom = parse(Int, on)
        on_func = dt -> day(dt) == dom
    else
        error("Can't parse day of month for DST change")
    end
    # Now we get the time of the transition
    c = at[end:end]
    at_hm = Time(isalpha(c) ? at[1:end-1] : at)
    at_flag = parseflag(isalpha(c) ? c : "")
    save_hm = Time(save)
    letter = letter == "-" ? "" : letter

    # Report unexpected save values that could cause issues during resolve.
    save_hm < MIN_SAVE && warn("Discovered save $save_hm less than the expected min $MIN_SAVE")
    save_hm > MAX_SAVE && warn("Discovered save $save_hm larger than the expected max $MAX_SAVE")

    # Now we've finally parsed everything we need
    return Rule(
        from_int,
        to_int,
        month_int,
        on_func,
        at_hm,
        at_flag,
        save_hm,
        letter,
    )
end

function zoneparse(gmtoff, rules, format, until="")
    # Get our offset and abbreviation string for this period
    offset = Time(gmtoff)

    # Report unexpected offsets that could cause issues during resolve.
    offset < MIN_GMT_OFFSET && warn("Discovered offset $offset less than the expected min $MIN_GMT_OFFSET")
    offset > MAX_GMT_OFFSET && warn("Discovered offset $offset larger than the expected max $MAX_GMT_OFFSET")

    # "zzz" represents a NULL entry
    format = format == "zzz" ? "" : format

    # Parse the date the line rule applies up to
    until_tuple = until == "" ? (nothing, 'w') : parsedate(until)
    until_dt, until_flag = Nullable{DateTime}(until_tuple[1]), until_tuple[2]

    if rules == "-" || ismatch(r"\d",rules)
        save = Time(rules)
        rules = ""
    else
        save = ZERO
    end

    return Zone(
        offset,
        save,
        rules,
        format,
        until_dt,
        until_flag,
    )
end

"""
Rules are typically ordered by the "from" than "in" fields. Since rules also
contain a "to" field the written ordering can be problematic for resolving
time zone transitions.

Example:

    # Rule  NAME    FROM    TO  TYPE    IN  ON  AT      SAVE    LETTER/S
    Rule    Poland  1918    1919    -   Sep 16  2:00s   0       -
    Rule    Poland  1919    only    -   Apr 15  2:00s   1:00    S
    Rule    Poland  1944    only    -   Apr  3  2:00s   1:00    S

    A simplistic way of iterating through the rules by years could yield the rules
    in the wrong order:

    # ON         AT      SAVE    LETTER/S
    1918-09-16   2:00s   0       -
    1919-09-16   2:00s   0       -
    1919-04-15   2:00s   1:00    S
    1944-04-03   2:00s   1:00    S

    The order_rules function will expand the rules such that they can be ordered by the
    "on" date which ensures we process the rules in the correct order:

    1918-09-16   2:00s   0       -
    1919-04-15   2:00s   1:00    S
    1919-09-16   2:00s   0       -
    1944-04-03   2:00s   1:00    S
"""
function order_rules(rules::Array{Rule}; max_year::Integer=MAX_YEAR)
    dates = Date[]
    ordered = Rule[]

    # Note: Typically rules are orderd by "from" and "in". Unfortunately
    for rule in rules
        start_year = max(get(rule.from, MIN_YEAR), MIN_YEAR)
        end_year = min(get(rule.to, max_year), max_year)

        # For each year the rule applies compute the transition date
        for rule_year in start_year:end_year
            # Determine the rule transition day by starting at the
            # beginning of the month and applying our "on" function
            # until we reach the correct day.
            date = Date(rule_year, rule.month)
            try
                # The "on" function should evaluate to a day within the current month.
                date = tonext(rule.on, date; same=true, limit=daysinmonth(date))
            catch e
                if isa(e, ArgumentError)
                    error("Unable to find matching day in month $(year(date))/$(month(date))")
                else
                    rethrow(e)
                end
            end
            push!(dates, date)
            push!(ordered, rule)
        end
    end

    perm = sortperm(dates)
    dates = dates[perm]
    ordered = ordered[perm]

    # Since we are not yet taking offsets or flags into account yet
    # there is a small chance that the results are not ordered correctly.
    last_date = typemin(Date)
    for (i, date) in enumerate(dates)
        if i > 1 && date - last_date <= ABS_DIFF_OFFSET
            error("Dates are probably not in order")
        end
        last_date = date
    end

    return dates, ordered
end

"""
Resolves a named zone into TimeZone. Updates ordered with any new rules that
were required to be ordered.
"""
function resolve!(zone_name::AbstractString, zoneset::ZoneDict, ruleset::RuleDict,
    ordered::OrderedRuleDict; max_year::Integer=MAX_YEAR, debug=false)

    transitions = Transition[]
    cutoff = Nullable{DateTime}()

    # Set some default values and starting DateTime increment and away we go...
    start_utc = DateTime(MIN_YEAR)
    max_until = DateTime(max_year, 12, 31, 23, 59, 59)
    save = ZERO
    letter = ""
    start_rule = Nullable{Rule}()

    # zones = Set{FixedTimeZone}()

    # Zone needs to be in ascending order to ensure that start_utc is being applied
    # to the correct transition.
    for zone in sort(zoneset[zone_name])

        # Break at the beginning of the loop instead of the end so that we know an
        # future zone exists beyond max_year and we can set cutoff.
        if year(start_utc) > max_year
            cutoff = Nullable(start_utc)
            break
        end

        offset = zone.gmtoffset
        format = zone.format
        # save = zone.save
        rule_name = zone.rules
        until = get(zone.until, max_until)
        cutoff = Nullable{DateTime}()  # Reset cutoff

        if rule_name == ""
            save = zone.save
            abbr = abbr_string(format, save)

            if debug
                rule_name = "\"\""  # Just for display purposes
                println("Zone Start $rule_name, $(zone.gmtoffset), $save, $(start_utc)u, $(until)$(zone.until_flag), $abbr")
            end

            tz = FixedTimeZone(abbr, Second(offset), Second(save))
            if isempty(transitions) || last(transitions).zone != tz
                push!(transitions, Transition(start_utc, tz))
            end
        else
            # Only order the rule if it hasn't already been processed. We'll go one year
            # further than the max_year to ensure we get an accurate cutoff DateTime.
            if !haskey(ordered, rule_name)
                ordered[rule_name] = order_rules(ruleset[rule_name]; max_year=max_year + 1)
            end

            dates, rules = ordered[rule_name]

            # TODO: We could avoid this search if the rule_name haven't changed since the
            # last iteration.
            index = searchsortedlast(dates, start_utc)

            if !isnull(start_rule)
                rule = get(start_rule)
                save = rule.save
                letter = rule.letter

                start_rule = Nullable{Rule}()
            elseif index == 0
                save = ZERO

                # Find the first occurrence of of standard-time
                for rule in rules
                    if rule.save == save
                        letter = rule.letter
                        break
                    end
                end
            else
                rule = rules[index]
                save = rule.save
                letter = rule.letter
            end
            abbr = abbr_string(format, save, letter)

            debug && println("Zone Start $rule_name, $(zone.gmtoffset), $save, $(start_utc)u, $(until)$(zone.until_flag), $abbr")

            tz = FixedTimeZone(abbr, Second(offset), Second(save))
            if isempty(transitions) || last(transitions).zone != tz
                push!(transitions, Transition(start_utc, tz))
            end

            index = max(index, 1)
            for (date, rule) in zip(dates[index:end], rules[index:end])
                # Add "at" since it could be larger than 23:59:59.
                dt = DateTime(date) + rule.at

                # Convert rule and until datetimes into UTC using the latest
                # offset and save values that occurred prior to this rule.
                dt_utc = asutc(dt, rule.at_flag, offset, save)
                until_utc = asutc(until, zone.until_flag, offset, save)

                if dt_utc == until_utc
                    start_rule = Nullable{Rule}(rule)
                elseif dt_utc > until_utc
                    cutoff = Nullable{DateTime}(dt_utc)
                end

                dt_utc >= until_utc && break

                # Need to be careful when we update save/letter.
                save = rule.save
                letter = rule.letter
                abbr = abbr_string(format, save, letter)

                if debug
                    status = dt_utc >= start_utc ? "Rule" : "Skip"
                    println("$status $(year(date)), $(dt)$(rule.at_flag), $(dt_utc)u, $save, $abbr")
                end

                # TODO: Is start_utc inclusive or exclusive?
                dt_utc >= start_utc || continue

                tz = FixedTimeZone(abbr, Second(offset), Second(save))

                # TODO: We can maybe reduce memory usage by reusing the same
                # FixedTimeZone object.
                # Note: By default pushing onto a set will replace objects.
                # if !(tz in zones)
                #     push!(zones, tz)
                # else
                #     tz = first(intersect(zones, Set([tz])))
                # end

                if isempty(transitions) || last(transitions).zone != tz
                    push!(transitions, Transition(dt_utc, tz))
                end
            end
        end

        start_utc = asutc(until, zone.until_flag, offset, save)
        debug && println("Zone End   $rule_name, $offset, $save, $(start_utc)u")
    end

    debug && println("Cutoff     $(isnull(cutoff) ? "nothing" : get(cutoff))")

    # Note: Transitions array is expected to be ordered and should be if both
    # zones and rules were ordered.
    if length(transitions) > 1 || !isnull(cutoff)
        return VariableTimeZone(zone_name, transitions, cutoff)
    else
        # Although unlikely the time zone name in the transition and the zone_name
        # could be different. We'll ignore this issue at the moment.
        return transitions[1].zone
    end
end

function resolve(zoneset::ZoneDict, ruleset::RuleDict; max_year::Integer=MAX_YEAR, debug=false)
    ordered = OrderedRuleDict()
    timezones = Dict{AbstractString,TimeZone}()

    for zone_name in keys(zoneset)
        tz = resolve!(zone_name, zoneset, ruleset, ordered; max_year=max_year, debug=debug)
        timezones[zone_name] = tz
    end

    return timezones
end

function resolve(zone_name::AbstractString, zoneset::ZoneDict, ruleset::RuleDict; max_year::Integer=MAX_YEAR, debug=false)
    ordered = OrderedRuleDict()
    return resolve!(zone_name, zoneset, ruleset, ordered; max_year=max_year, debug=debug)
end

function tzparse(tzfile::AbstractString)
    zones = ZoneDict()
    rules = RuleDict()
    links = Dict{AbstractString,AbstractString}()

    # For the intial pass we'll collect the zone and rule lines.
    open(tzfile) do fp
        kind = name = ""
        for line in eachline(fp)
            # Lines that start with whitespace can be considered a "continuation line"
            # which means the last found kind/name should persist.
            persist = ismatch(r"^\s", line)

            line = strip(replace(chomp(line), r"#.*$", ""))
            length(line) > 0 || continue
            line = replace(line, r"\s+", " ")

            if !persist
                kind, name, line = split(line, ' '; limit=3)
            end

            if kind == "Rule"
                rule = ruleparse(split(line, ' ')...)
                rules[name] = push!(get(rules, name, Rule[]), rule)
            elseif kind == "Zone"
                zone = zoneparse(split(line, ' '; limit=4)...)
                zones[name] = push!(get(zones, name, Zone[]), zone)
            elseif kind == "Link"
                dest = line
                links[dest] = name
            else
                warn("Unhandled line found with type: $kind")
            end
        end
    end

    # Turn links into zones.
    # Note: it would be more computationally efficient to pass the links around
    # and only resolve a zone once.
    for (dest, source) in links
        zones[dest] = zones[source]
    end

    return zones, rules
end

function load(tzdata_dir::AbstractString=TZDATA_DIR; max_year::Integer=MAX_YEAR)
    timezones = Dict{AbstractString,TimeZone}()
    for filename in readdir(tzdata_dir)
        zones, rules = tzparse(joinpath(tzdata_dir, filename))
        merge!(timezones, resolve(zones, rules; max_year=max_year))
    end
    return timezones
end

function compile(tzdata_dir::AbstractString=TZDATA_DIR, dest_dir::AbstractString=COMPILED_DIR; max_year::Integer=MAX_YEAR)
    timezones = load(tzdata_dir; max_year=max_year)

    isdir(dest_dir) || error("Destination directory doesn't exist")

    for (name, timezone) in timezones
        parts = split(name, "/")
        tz_dir, tz_file = joinpath(dest_dir, parts[1:end-1]...), parts[end]

        isdir(tz_dir) || mkpath(tz_dir)

        open(joinpath(tz_dir, tz_file), "w") do fp
            serialize(fp, timezone)
        end
    end
end

end # module


text:
import Pkg;
Pkg.add("HTTP")
Pkg.add("Dates")

using HTTP, Sockets, Dates

println("Starting my server ...")

# Write to a file, this file is stored in the dir where
# the julia program runs

# in case program runs in container data is stored inside container
# at /app dir as specified WORKDIR in Dockerfile

# To persist the data file over container sessions:
# create a 'volume' : docker volume create myvolumename
#
# Then start the container by binding the /app dir in container to
# myvolumename: docker run -v myvolumename:/app -p 80:80 juliacon
open("serverlocalstorage.txt", "a") do io
       write(io, "Start the server\n")
   end;

HTTP.listen("0.0.0.0", 80) do http
    println("Caught the next request ..." )

    d_now = Dates.format(now(), "HH:MM:SS")

    #if desired can see into the container via
    # docker exec -t -i mycontainer_id /bin/bash
    open("serverlocalstorage.txt", "a") do io
           write(io, "Serving a request\n")
           write(io, d_now)
           write(io, "----\n")
    end;

    HTTP.setheader(http, "Content-Type" => "text/html")
    write(http, "MKEJ Hello World of VCP<BR>")
    write(http, string(d_now))
    return
end

max_stars_repo_name: mjtkeijsers/juliawebcon

max_issues_repo_name: mjtkeijsers/juliawebcon

max_forks_repo_name: mjtkeijsers/juliawebcon


text:
################################################################################
#
#  Map/NumField.jl
#
################################################################################

# This implements maps between arbitrary number fields.
#
# Basic idea
# ----------
#
# If f : K -> * is a map from a number field K, then f is completely determined by
#   - f(gen(K)), if K is an AnticNumberField
#   - f(gen(K)), f|_base_field(K), if K is an NfRel
#   - f(gens(K)), if K is an NfAbsNS
#   - f(gens(K)), f|_base_field(K), if K is an NfRelNS
#
# Thus our map type NumFieldMor has fields
#   - header (as usual)
#   - image_data, which define the morphism (as described above)
#   - inverse_data, which define the inverse morphism (if it exists)
#
# To untangle the data defining the morphism and the morphism itself, we
# introduce the types
#
#   MapDataFromAnticNumberField, MapDataFromNfRel, MapDataFromNfAbsNS,
#   MapDataFromNfRelNS
#
# They store the data defining the morphism (these constructions are recursive
# in the relative setting).
#
# Because we know what the type of the inverse_data will be, we can also fully
# initialize the type, even if we don't have access to the preimage data (yet).
#
# Applying a morphism
# -------------------
#
# The application of a morphism is delegated the MapData* types. They implement
# an image function, e.g., with signature
#
#     image(MapDataFromAnticNumberField{NfRel{nf_elem}}, L, x::nf_elem),
#
# which gets called when f : K -> L is a map from AnticNumberField to
# NfRel{nf_elem}. (More precisely, f(a) = image(f.image_data, codomain(f), a))
#
# Note that we do not store L inside explicitely inside MapData* and this
# becomes a problem if f.isid is true. Thus we need to pass L to the function.
#
# Difference to the old system
# ----------------------------
#
# To allow creation of morphism between arbitrary types, we have to be a bit
# careful when it comes to morphism of relative fields.
#
# Assume L/K is an NfRel{nf_elem} and f : L -> L is a morphism, mapping K to K.
# Before, we were basically storing implicitely f(gen(L)) as well as g : K ->
# K. Instead we are storing f(gen(L)) as well as f(L(gen(K)). This new system
# is much more flexible and does not require awkward special cases, which do
# not scale. The (minor) price we pay is that some morphism are a bit slower to
# evaluate.
#
# We are mitigating this regression by storing (in this case) whether f is the
# identity on base_field(K).
#
# Creating morphisms
# ------------------
# This follows now a very easy rule. Let L be any number field. Constructing
# a homorphism K -> L can be constructed as follows:
#
# If K is an AnticNumberField:
#   hom(K, L, ::elem_type(L))
# If K is an NfAbsNS:
#   hom(K, L, ::Vector{elem_type(L)}
# If K is an NfRel:
#   hom(K, L, g, ::elem_type(L)), where g can be
#     - a homomorphism base_field(K) -> L
#     - data defining a homomorphism base_field(K) -> L
#       (e.g., to support
#        hom(::NfRel{nf_elem}, AnticNumberField, nf_elem, nf_elem))
#     - a homorphism base_field(K) -> base_field(L)
#   hom(K, L, ::elem_type(L))
#     - this assumes that the base_field of K embeds naturally into L
# If K is an NfRelNS:
#   hom(K, L, g, ::Vector{elem_type(L)}), where g can be
#     - a homomorphism base_field(K) -> L
#     - data defining a homomorphism base_field(K) -> L
#       (e.g., to support
#        hom(::NfRelNS{nf_elem}, AnticNumberField, nf_elem, Vector{nf_elem}))
#     - a homorphism base_field(K) -> base_field(L)
#   hom(K, L, ::Vector{elem_type(L)})
#     - this assumes that the base_field of K embeds naturally into L
#
# We also get a nice syntax to create inverses for free:
#
#     hom(K, L, ..., inverse = (x))
#
#  where x is such that hom(L, K, x) works.
#

export restrict

################################################################################
#
#   The NumFieldMor type
#
################################################################################

mutable struct NumFieldMor{S, T, U, V, W} <: Map{S, T, HeckeMap, NumFieldMor}
  header::MapHeader{S, T}
  image_data::U
  inverse_data::V
  absolute_basis::Vector{W}
  absolute_basis_matrix_image::fmpq_mat
  rref::Tuple{fmpq_mat, fmpq_mat}
  pivots_of_rref::Vector{Int}

  function NumFieldMor{S, T, U, V}() where {S, T, U, V}
    z = new{S, T, U, V, elem_type(S)}()
    return z
  end

  function NumFieldMor(K::NumField, L::NumField)
    z = new{typeof(K), typeof(L), map_data_type(K, L), map_data_type(L, K), elem_type(K)}()
    z.header = MapHeader(K, L)
    return z
  end

  function NumFieldMor{S, T, U, V}(h::MapHeader{S, T}, i::U, p::V) where {S, T, U, V}
    z = new{S, T, U, V, elem_type(S)}(h, i, p)
    return z
  end
end

################################################################################
#
#  The hom function
#
################################################################################

function hom(K::S, L::T, x...; inverse = nothing,
                               check::Bool = true,
                               compute_inverse = false) where {S <: NumField,
                                                               T <: NumField}
  header = MapHeader(K, L)

  # Check if data defines a morphism
  image_data = map_data(K, L, x..., check = check)

  if inverse !== nothing
    # Check if data defines a morphism
    # This goes through _validata_data, since we don't want to split the
    # argument if for example the argument is a Vector
    inverse_data = _map_data(L, K, inverse, check = check)

    z = NumFieldMor{S, T, typeof(image_data),
                       typeof(inverse_data)}(header, image_data, inverse_data)

  else
    z = NumFieldMor{S, T, typeof(image_data), map_data_type(L, K)}()
  end

  z.header = header
  z.image_data = image_data

  if compute_inverse
    _assure_has_inverse_data(z)
  end

  return z
end

################################################################################
#
#  Some type acrobatic. This should be moved to where types will be ending up.
#
################################################################################

base_field_type(::AnticNumberField) = FlintRationalField

base_field_type(::NfAbsNS) = FlintRationalField

base_field_type(::NfRel{T}) where {T} = parent_type(T)

base_field_type(::NfRelNS{T}) where {T} = parent_type(T)

base_field_type(::Type{NfRelNS{T}}) where {T} = parent_type(T)

base_field_type(::Type{NfRel{T}}) where {T} = parent_type(T)

elem_type(::Type{NfRelNS{T}}) where {T} = NfRelNSElem{T}

elem_type(::NfRelNS{T}) where {T} = NfRelNSElem{T}

parent_type(::Type{NfRelNSElem{T}}) where {T} = NfRelNS{T}

elem_type(::Type{NfAbsNS}) = NfAbsNSElem

elem_type(::NfAbsNS) = NfAbsNSElem

parent_type(::Type{NfAbsNSElem}) = NfAbsNS

################################################################################
#
#  Image and preimage function
#
################################################################################

function image(f::NumFieldMor, x)
  @assert domain(f) === parent(x)
  return image(f.image_data, codomain(f), x)
end

function preimage(f::NumFieldMor, x)
  return image(f.inverse_data, domain(f), x)
end

################################################################################
#
#  Now the types to store the data for the maps
#
################################################################################

# From AnticNumberField into something
mutable struct MapDataFromAnticNumberField{T}
  prim_image::T
  isid::Bool

  function MapDataFromAnticNumberField{T}(x::Bool) where T
    @assert x
    z = new{T}()
    z.isid = true
    return z
  end

  function MapDataFromAnticNumberField{T}(x::T) where T
    z = new{T}(x, false)
    return z
  end
end

# Helper functions to create the type
map_data_type(K::AnticNumberField, L::NumField) = map_data_type(AnticNumberField, typeof(L))

map_data_type(::Type{AnticNumberField}, T::Type{<:NumField}) = MapDataFromAnticNumberField{elem_type(T)}

# Test if data u, v specfiying a map K -> L define the same morphism
function _isequal(K, L, u::MapDataFromAnticNumberField{T},
                        v::MapDataFromAnticNumberField{S}) where {T, S}
  if u.isid && v.isid
    return true
  end

  z = gen(K)

  return image(u, L, z) == image(v, L, z)
end

# Image function
function image(f::MapDataFromAnticNumberField, L, y::nf_elem)
  f.isid && return L(y)
  z = parent(defining_polynomial(parent(y)))(y)
  return evaluate(z, f.prim_image)
end

# Functions to create and validate the data
#
map_data(K::AnticNumberField, L, ::Bool) = MapDataFromAnticNumberField{elem_type(L)}(true)

function map_data(K::AnticNumberField, L, x::NumFieldElem; check = true)
  if parent(x) === L
    xx = x
  else
    xx = L(x)::elem_type(L)
  end

  if check
    if !iszero(evaluate(defining_polynomial(K), xx))
      error("Data does not define a morphism")
    end
  end
  return MapDataFromAnticNumberField{elem_type(L)}(xx)
end

function map_data(K::AnticNumberField, L, g::NumFieldMor; check = true)
  if check
    K !== domain(g) && error("Data does not define a morphism")
  end

  z = image_primitive_element(g)

  return map_data(K, L, z; check = false)
end

# From NfRel into something
mutable struct MapDataFromNfRel{T, S}
  prim_image::T
  base_field_map_data::S
  isid::Bool

  function MapDataFromNfRel{T, S}(x::T, y::S) where {T, S}
    z = new{T, S}(x, y, false)
    return z
  end

  function MapDataFromNfRel{T, S}(x::Bool) where {T, S}
    @assert x
    z = new{T, S}()
    z.isid = true
    return z
  end
end

# Helper functions to create the type

function map_data_type(T::Type{<: NfRel}, L::Type{<:Any})
  MapDataFromNfRel{elem_type(L), map_data_type(base_field_type(T), L)}
end

map_data_type(K::NfRel, L::NumField) = map_data_type(typeof(K), typeof(L))

# Test if data u, v specfiying a map K -> L define the same morphism
function _isequal(K, L, u::MapDataFromNfRel{T, S}, v::MapDataFromNfRel{T, S}) where {T, S}
  if u.isid && v.isid
    return true
  end

  return image(u, L, gen(K)) == image(v, L, gen(K)) &&
         _isequal(base_field(K), L, u.base_field_map_data, v.base_field_map_data)
end

# Image function
function image(f::MapDataFromNfRel, L, y)
  f.isid && return L(y)
  # TODO: Cache the polynomial ring
  Ly, = PolynomialRing(L, "y", cached = false)
  z = map_coefficients(t -> image(f.base_field_map_data, L, t), y.data, parent = Ly)
  return evaluate(z, f.prim_image)
end

# Functions to validate and create the data.

function map_data(K::NfRel, L, ::Bool)
  z = MapDataFromNfRel{elem_type(L), map_data_type(base_field(K), L)}(true)
  z.base_field_map_data = map_data(base_field(K), L, true)
  return z
end

function map_data(K::NfRel, L, x...; check = true)
  z = map_data(base_field(K), L, Base.front(x)...; check = check)

  local yy::elem_type(L)

  if Base.last(x) isa NumFieldMor
    domain(Base.last(x)) !== K && error("Data does not define a morphism")
    _y = image_primitive_element(Base.last(x))
    if parent(_y) === L
      yy = _y
    else
      yy = L(_y)::elem_type(L)
    end
  else
    y = Base.last(x)
    if parent(y) === L
      yy = y
    else
      yy = L(y)::elem_type(L)
    end
  end

  if check
    y = evaluate(map_coefficients(w -> image(z, L, w), defining_polynomial(K), cached = false), yy)
    !iszero(y) && error("Data does not define a morphism")
  end

  @assert typeof(yy) == elem_type(L)
  @assert typeof(z) == map_data_type(base_field(K), L)

  return MapDataFromNfRel{typeof(yy), typeof(z)}(yy, z)
end

# From NfAbsNS into something
mutable struct MapDataFromNfAbsNS{T}
  images::T
  isid::Bool

  function MapDataFromNfAbsNS{T}(x::T) where {T}
    z = new{T}(x, false)
    return z
  end

  function MapDataFromNfAbsNS{T}(x::Bool) where {T}
    @assert x
    z = new{T}()
    z.isid = true
    return z
  end
end

function _isequal(K, L, u::MapDataFromNfAbsNS{T}, v::MapDataFromNfAbsNS{T}) where {T}
  # If one is the identity, this means that K === L
  if (u.isid && !v.isid)
    return v.images == gens(K)
  elseif (v.isid && !u.isid)
    return u.images == gens(K)
  elseif u.isid && v.isid
    return true
  end

  return v.images == u.images
end

function image(f::MapDataFromNfAbsNS, L, y)
  f.isid && return L(y)
  return msubst(y.data, f.images)
end

map_data_type(K::NfAbsNS, L) = MapDataFromNfAbsNS{Vector{elem_type(L)}}

map_data_type(T::Type{NfAbsNS}, L::Type{<:Any}) = MapDataFromNfAbsNS{Vector{elem_type(L)}}

map_data(K::NfAbsNS, L, ::Bool) = MapDataFromNfAbsNS{Vector{elem_type(L)}}(true)

function map_data(K::NfAbsNS, L, x::Vector; check = true)
  if length(x) != ngens(K)
    error("Data does not define a morphism")
  end

  local xx::Vector{elem_type(L)}

  if x isa Vector{elem_type(L)}
    if parent(x[1]) !== L
      error("Data does not define a morphism")
    end
    xx = x
  else
    xx = map(L, x)::Vector{elem_type(L)}
  end

  if check
    for i in 1:ngens(K)
      if !iszero(evaluate(K.pol[i], xx))
        error("Data does not define a morphism")
      end
    end
  end

  @assert typeof(xx) == Vector{elem_type(L)}

  return MapDataFromNfAbsNS{typeof(xx)}(xx)
end

# From NfRelNS into something
mutable struct MapDataFromNfRelNS{T, S}
  images::T
  base_field_map_data::S
  isid::Bool

  function MapDataFromNfRelNS{T, S}(x::T, y::S) where {T, S}
    z = new{T, S}(x, y, false)
    return z
  end

  function MapDataFromNfRelNS{T, S}(x::Bool) where {T, S}
    @assert x
    z = new{T, S}()
    z.isid = true
    return z
  end
end

function _isequal(K, L, u::MapDataFromNfRelNS{T, S}, v::MapDataFromNfRelNS{T, S}) where {T, S}
   if u.isid && v.isid
    return true
  end

  return all(g -> image(u, L, g) == image(v, L, g), gens(K)) && _isequal(base_field(K), base_field(L), u.base_field_map_data, v.base_field_map_data)
end

function image(f::MapDataFromNfRelNS, L, y)
  f.isid && return L(y)
  z = map_coefficients(w -> image(f.base_field_map_data, L, w), y.data, cached = false)
  return evaluate(z, f.images)
end

function map_data_type(T::Type{<: NfRelNS}, L::Type{<:Any})
  MapDataFromNfRelNS{Vector{elem_type(L)}, map_data_type(base_field_type(T), L)}
end

map_data_type(K::NfRelNS, L) = MapDataFromNfRelNS{Vector{elem_type(L)}, map_data_type(base_field(K), L)}

function map_data(K::NfRelNS, L, ::Bool)
  z = MapDataFromNfRelNS{Vector{elem_type(L)}, map_data_type(base_field(K), L)}(true)
  z.base_field_map_data = map_data(base_field(K), L, true)
  return z
end

function map_data(K::NfRelNS, L, x...; check = true)
  z = map_data(base_field(K), L, Base.front(x)...; check = check)

  local yy::Vector{elem_type(L)}

  if Base.last(x) isa NumFieldMor
    domain(Base.last(x)) !== K && error("")
    _y = image_generators(Base.last(x))
    if parent(_y[1]) === L
      yy = _y
    else
      yy = map(L, _y)::Vector{elem_type(L)}
    end
  else
    y = Base.last(x)
    if !(y isa Vector)
      error("")
    end
    if parent(y[1]) === L
      yy = y
    else
      yy = map(L, y)::Vector{elem_type(L)}
    end
  end

  if check
    for i in 1:ngens(K)
      w = evaluate(map_coefficients(w -> image(z, L, w), K.pol[i], cached = false), yy)
      !iszero(w) && error("Data does not define a morphism")
    end
  end

  @assert typeof(yy) == Vector{elem_type(L)}
  @assert typeof(z) == map_data_type(base_field(K), L)

  return MapDataFromNfRelNS{typeof(yy), typeof(z)}(yy, z)
end

################################################################################
#
#  Equality
#
################################################################################

function Base.:(==)(u::NumFieldMor, v::NumFieldMor)
  if (domain(u) != domain(v)) || (codomain(u) != codomain(v))
    return false
  end
  return _isequal(domain(u), codomain(u), u.image_data, v.image_data)
end

################################################################################
#
#  Identity morphism
#
################################################################################

function id_hom(K::NumField)
  z = NumFieldMor{typeof(K), typeof(K), map_data_type(K, K), map_data_type(K, K)}(MapHeader(K, K), map_data(K, K, true), map_data(K, K, true))
end

################################################################################
#
#  Helper functions to compare morphisms
#
################################################################################

_convert_map_data(g::NumFieldMor, L) = __convert_map_data(g.image_data, L)

__convert_map_data(d::MapDataFromAnticNumberField, L) = MapDataFromAnticNumberField{elem_type(L)}(d.isid ? true : L(d.prim_image))

__convert_map_data(d::MapDataFromNfRel, L) = MapDataFromNfReld{elem_type(L)}(L(d.prim_image), d.isid ? true : __convert_map_data(d.base_field_map_data), d.isid)

################################################################################
#
#  Helper functions to pass through inverse data
#
################################################################################

@inline _validate_data(L, K, inverse) = validate_data(L, K, inverse)

@inline _validate_data(L, K, inverse::Tuple) = validate_data(L, K, inverse...)

@inline _map_data(L, K, inverse; check::Bool) = map_data(L, K, inverse, check = check)

@inline _map_data(L, K, inverse::Tuple; check::Bool) = map_data(L, K, inverse..., check = check)

################################################################################
#
#  Morphism type
#
################################################################################

morphism_type(K::NumField) = morphism_type(typeof(K), typeof(K))

morphism_type(K::Type{T}) where T <: NumField = morphism_type(T, T)

morphism_type(K::S, L::T) where {S <: NumField, T <: NumField} = morphism_type(S, T)

morphism_type(::Type{S}, ::Type{T}) where {S <: NumField, T <: NumField} = NumFieldMor{S, T, map_data_type(S, T), map_data_type(T, S), elem_type(S)}

################################################################################
#
#  Type aliases
#
################################################################################

const NfToNfMor = morphism_type(AnticNumberField, AnticNumberField)

const NfAbsNSToNfAbsNS = morphism_type(NfAbsNS, NfAbsNS)

const NfAbsToNfAbsNS = morphism_type(AnticNumberField, NfAbsNS)

const NfToNfRel =  morphism_type(AnticNumberField, NfRel{nf_elem})

const NfRelToNfRelMor_nf_elem_nf_elem =  morphism_type(NfRel{nf_elem}, NfRel{nf_elem})

const NfRelToNf = morphism_type(NfRel{nf_elem}, AnticNumberField)

const NfRelNSToNfRelNSMor_nf_elem = morphism_type(NfRelNS{nf_elem}, NfRelNS{nf_elem})

const NfRelToNfRelNSMor_nf_elem = morphism_type(NfRel{nf_elem}, NfRelNS{nf_elem})

################################################################################
#
#  Images of primitive elements/generators
#
################################################################################

function image_primitive_element(f::NumFieldMor{AnticNumberField})
  if f.image_data.isid
    return codomain(f)(gen(domain(f)))
  end
  return f.image_data.prim_image
end

function preimage_primitive_element(f::NfToNfMor)
  if f.inverse_data.isid
    return codomain(f)(gen(domain(f)))
  else
    return f.inverse_data.prim_image
  end
end

function image_generators(f::NumFieldMor{<:NfRelNS})
  return f.image_data.images
end

function image_generators(f::NumFieldMor{<:NfAbsNS})
  return f.image_data.images
end

function image_primitive_element(f::NumFieldMor{<:NfRel})
  if f.image_data.isid
    return gen(domain(f))
  end
  return f.image_data.prim_image
end

################################################################################
#
#  Preimage computation
#
# ##############################################################################

function _assert_has_preimage_data(f::NumFieldMor)
  if isdefined(f, :absolute_basis_matrix_image)
    return nothing
  end
  K = domain(f)
  L = codomain(f)
  b = absolute_basis(K)
  d = absolute_degree(K)
  n = absolute_degree(L)
  M = zero_matrix(FlintQQ, n, d)
  for i in 1:d
    c = f(b[i])
    cc = absolute_coordinates(c)
    for j in 1:length(cc)
      M[j, i] = cc[j]
    end
  end

  r, R, U =  _rref_with_trans(M)
  pivots = _get_pivots_ut(R)

  f.absolute_basis_matrix_image = M
  f.absolute_basis = b
  f.pivots_of_rref = pivots
  f.rref = R, U

  return nothing
end

function haspreimage(f::NumFieldMor, g::NumFieldElem)
  if isdefined(f, :inverse_data)
    return true, image(f.inverse_data, domain(f), g)
  end
  @assert parent(g) === codomain(f)
  cc = absolute_coordinates(g)
  K = domain(f)
  _assert_has_preimage_data(f)
  fl, s = can_solve_given_rref(f.rref[1], f.rref[2], f.pivots_of_rref, cc)
  if !fl
    return false, zero(K)
  else
    b = f.absolute_basis
    # This is suboptimal
    prim_preimg = reduce(+, (s[i, 1] * b[i] for i in 1:length(b)), init = zero(K))::elem_type(K)
    return true, prim_preimg
  end
end

function preimage(f::NumFieldMor, g::NumFieldElem)
  fl, y = haspreimage(f, g)
  @assert fl
  return y
end

################################################################################
#
#  Computation of the inverse (data)
#
################################################################################

function _assure_has_inverse_data(f::NumFieldMor)
  if isdefined(f, :inverse_data)
    return nothing
  else
    pr = _compute_inverse_data(f, domain(f), codomain(f))
    f.inverse_data = pr
    return nothing
  end
end

function inv(f::NumFieldMor{S, T}) where {S, T}
  _assure_has_inverse_data(f)
  pr = f.inverse_data
  hd = MapHeader(codomain(f), domain(f))
  g = NumFieldMor{T, S, map_data_type(T, S), map_data_type(S, T)}(hd, pr, f.image_data)

  return g
end

# into AnticNumberField
function _compute_inverse_data(f#= image data =#, K, L::AnticNumberField)
  return _compute_inverse_data(f#= image data =#, K, L, L)
end

function _compute_inverse_data(f#= image data =#, K, LL, L::AnticNumberField)
  g = LL(gen(L))
  fl, prim_preimg = haspreimage(f, LL(g))
  @assert fl
  return MapDataFromAnticNumberField{typeof(prim_preimg)}(prim_preimg)
end

# into NfAbsNS
function _compute_inverse_data(f#= image data =#, K, L::NfAbsNS)
  return _compute_inverse_data(f, K, L, L)
end

function _compute_inverse_data(f#= image data =#, K, LL, L::NfAbsNS)
  preimg_gens = elem_type(K)[]
  for g in gens(L)
    fl, preimg = haspreimage(f, LL(g))
    @assert fl
    push!(preimg_gens, preimg)
  end
  return MapDataFromNfAbsNS{typeof(preimg_gens)}(preimg_gens)
end

# into NfRel
function _compute_inverse_data(f#= image data =#, K, L::NfRel)
  return _compute_inverse_data(f#= image data =#, K, L, L)
end

function _compute_inverse_data(f#= image data =#, K, LL, L::NfRel)
  g = gen(L)
  fl, preimg = haspreimage(f, LL(g))
  inverse_data_base_field = _compute_inverse_data(f, K, LL, base_field(L))
  return MapDataFromNfRel{typeof(preimg), typeof(inverse_data_base_field)}(preimg, inverse_data_base_field)
end

# into NfRelNS

function _compute_inverse_data(f#= image data =#, K, L::NfRelNS)
  return _compute_inverse_data(f, K, L, L)
end

function _compute_inverse_data(f, K, LL, L::NfRelNS)
  preimg_gens = elem_type(K)[]
  for g in gens(L)
    fl, preimg = haspreimage(f, LL(g))
    push!(preimg_gens, preimg)
  end
  inverse_data_base_field = _compute_inverse_data(f, K, LL, base_field(L))
  return MapDataFromNfRelNS{typeof(preimg_gens), typeof(inverse_data_base_field)}(preimg_gens, inverse_data_base_field)
end

################################################################################
#
#  Auxillary map_data function
#
################################################################################

function map_data(K::NumField, L::NumField; check = true)
  return map_data(K, L, true)
end

################################################################################
#
#  Composition
#
################################################################################

# f : K -> L, g : L -> M
function _compose(f::MapDataFromAnticNumberField, g#= map data =#, K, L, M)
  return map_data_type(K, M)(image(g, M, image(f, L, gen(K))))
end

function _compose(f::MapDataFromNfRel, g#= map data =#, K, L, M)
  return map_data_type(K, M)(image(g, M, image(f, L, gen(K))),
                             _compose(f.base_field_map_data, g, base_field(K), L, M))
end

function _compose(f::MapDataFromNfAbsNS, g#= map data =#, K, L, M)
  return map_data_type(K, M)(elem_type(M)[image(g, M, image(f, L, u)) for u in gens(K)])
end

function _compose(f::MapDataFromNfRelNS, g#= map data =#, K, L, M)
  return map_data_type(K, M)(elem_type(M)[image(g, M, image(f, L, u)) for u in gens(K)],
                             _compose(f.base_field_map_data, g, base_field(K), L, M))
end

function Base.:(*)(f::NumFieldMor, g::NumFieldMor)
  @req codomain(f) === domain(g) "Composition: Maps are not compatible"
  z = NumFieldMor(domain(f), codomain(g))
  z.image_data = _compose(f.image_data, g.image_data, domain(f), codomain(f), codomain(g))
  if isdefined(f, :inverse_data) && isdefined(g, :inverse_data)
    z.inverse_data = _compose(g.inverse_data, f.inverse_data, codomain(g), domain(g), domain(f))
  end
  return z
end

################################################################################
#
#  Powering
#
################################################################################

function ^(f::NumFieldMor, b::Int)
  K = domain(f)
  @assert K == codomain(f)
  d = absolute_degree(K)
  b = mod(b, d)
  if b == 0
    return id_hom(K)
  elseif b == 1
    return f
  else
    bit = ~((~UInt(0)) >> 1)
    while (UInt(bit) & b) == 0
      bit >>= 1
    end
    z = f
    bit >>= 1
    while bit != 0
      z = z * z
      if (UInt(bit) & b) != 0
        z = z * f
      end
      bit >>= 1
    end
    return z
  end
end

^(a::NumFieldMor, n::IntegerUnion)  = _generic_power(a, n)

################################################################################
#
#  Hashing
#
################################################################################

function Base.hash(f::MapDataFromAnticNumberField, K, L, h::UInt)
  if f.isid
    return xor(hash(L, h), hash(K, h))
  else
    return hash(f.prim_image, h)
  end
end

function Base.hash(f::MapDataFromNfRel, K, L, h::UInt)
  if f.isid
    h = xor(hash(L, h), hash(K, h))
  else
    h = hash(f.prim_image, h)
  end
  h = hash(f.base_field_map_data, base_field(K), L, h)
  return h
end

function Base.hash(f::MapDataFromNfAbsNS, K, L, h::UInt)
  if f.isid
    return xor(hash(L, h), hash(K, h))
  else
    G = f.images
    for g in G
      h = hash(g, h)
    end
    return h
  end
end

function Base.hash(f::MapDataFromNfRelNS, K, L, h::UInt)
  if f.isid
    h = xor(hash(L, h), hash(K, h))
  else
    G = f.images
    for g in G
      h = hash(g, h)
    end
  end
  h = hash(f.base_field_map_data, base_field(K), L, h)
end

Base.hash(f::NumFieldMor, h::UInt) = hash(f.image_data, domain(f), codomain(f), h)

################################################################################
#
#  Restriction
#
################################################################################

function restrict(f::NumFieldMor, K::NonSimpleNumField)
  k = domain(f)
  return hom(K, k, [k(x) for x in gens(K)])*f
end

function restrict(f::NumFieldMor, K::SimpleNumField)
  k = domain(f)
  return hom(K, k, k(gen(K)))*f
end


text:
using Documenter, Analyzer

makedocs()


text:
using  JLD,Dates
## Load Data
startDate = Date("2012-03-12")
endDate = Date("2015-02-27")
save_fold =   "/home/Domenico/Dropbox/Dynamic_Networks/data/emid_data/juliaFiles/"
file_nameStart = "Weekly_eMid_Estimates"
file_nameEnd = "_from_$(Dates.format(startDate,"yyyy_mm_dd"))"*
        "_to_" *"$(Dates.format(endDate,"yyyy_mm_dd"))"*".jld"

save_path = save_fold*file_nameStart* "_W1_" *file_nameEnd#
@load(save_path,estSSW1,estGasW1,estGasTargW1,estSSW1_1GW,estGasW1_1GW,AeMidWeekly_T, YeMidWeekly_T ,weekInd,degsIO_T)

# test netmodel direceted binary with covariates on eMid Data
using  StaticNets ,Utilities

AeMidWeekly_T = BitArray(YeMidWeekly_T )
AmatsEMid = AeMidWeekly_T[:,:,2:end]
#use the presence or absence of a link as regressor
regs0eMid = AeMidWeekly_T[:,:,1:end-1]
#regs0eMid = YeMidWeekly_T[:,:,1:end-1]#use the weight of previous time link

##


text:
function print_sparse_statistics(S)
    println("size: ", size(S), "; rank: ", rank(S))
    println("sparsity: ", nnz(S)/length(S))
    println("memory: ", Base.summarysize(S))
end