<commit_before>using HDF5

"""
    save(path::String, name::String, obj, [mode="w"])

Save an object in Merlin HDF5 format.
* mode: "w" (overrite) or "r+" (append)
"""
function save(path::String, data::Pair{String}...; mode="w")
    info("Saving objects...")
    mkpath(dirname(path))
    h5open(path, mode) do h
        for (k,v) in data
            h5save(h, k, v)
        end
    end
    nothing
end

"""
    load(path::String, name::String)

Load an object from Merlin HDF5 format.
"""
function load(path::String, name::String)
    h5open(path, "r") do h
        h5load(h[name])
    end
end

readas(::Type{Void}, x) = nothing
writeas(x::Void) = string(x)

readas(::Type{Char}, x) = x[1]
writeas(x::Char) = string(x)

readas(::Type{Symbol}, x) = parse(x)
writeas(x::Symbol) = string(x)

readas(::Type{DataType}, x) = eval(parse(x))
writeas(x::DataType) = string(x)

readas(::Type{Function}, x) = eval(parse(x))
writeas(x::Function) = string(x)

readas{T<:HDF5.BitsKindOrString}(::Type{T}, x) = x
writeas(x::HDF5.BitsKindOrString) = x

function readas{T,N}(::Type{Array{T,N}}, x)
    if T <: HDF5.BitsKindOrString
        x
    elseif N == 1
        data = Array(T, length(x))
        for (k,v) in x
            data[parse(Int,k)] = v
        end
        data
    else
        throw("Invalid data: $x")
    end
end
function writeas{T,N}(x::Array{T,N})
    if T <: HDF5.BitsKindOrString
        x
    elseif N == 1
        Dict(i=>x[i] for i=1:length(x))
    else
        throw("x::$(typeof(x)) is not supported.")
    end
end

function readas{K<:HDF5.BitsKindOrString,V<:HDF5.BitsKindOrString}(::Type{Dict{K,V}}, x)
    dict = Dict{K,V}()
    for line in split(x, "\n")
        items = split(chomp(line), "\t")
        k = K <: String ? String(items[1]) : parse(K,items[1])
        v = V <: String ? String(items[2]) : parse(V,items[2])
        dict[k] = v
    end
    dict
end
function writeas{K<:HDF5.BitsKindOrString,V<:HDF5.BitsKindOrString}(x::Dict{K,V})
    lines = String[]
    for (k,v) in x
        push!(lines, "$k\t$v")
    end
    join(lines, "\n")
end

function readas{T<:Tuple}(::Type{T}, x)
    data = Array(Any, length(x))
    for (k,v) in x
        data[parse(Int,k)] = v
    end
    tuple(data...)
end
writeas(x::Tuple) = Dict(i=>x[i] for i=1:length(x))

function readas(T::Type, x)
    values = map(name -> x[string(name)], fieldnames(T))
    T(values...)<commit_msg>Fix hdf5<commit_after>using HDF5
import HDF5.BitsKindOrString

"""
    save(path::String, name::String, obj, [mode="w"])

Save an object in Merlin HDF5 format.
* mode: "w" (overrite) or "r+" (append)
"""
function save(path::String, data::Pair{String}...; mode="w")
    info("Saving objects...")
    mkpath(dirname(path))
    h5open(path, mode) do h
        for (k,v) in data
            h5save(h, k, v)
        end
    end
    nothing
end

"""
    load(path::String, name::String)

Load an object from Merlin HDF5 format.
"""
function load(path::String, name::String)
    h5open(path, "r") do h
        h5load(h[name])
    end
end

readas(::Type{Void}, x) = nothing
writeas(x::Void) = string(x)

readas(::Type{Char}, x) = x[1]
writeas(x::Char) = string(x)

readas(::Type{Symbol}, x) = parse(x)
writeas(x::Symbol) = string(x)

readas(::Type{DataType}, x) = eval(parse(x))
writeas(x::DataType) = string(x)

readas(::Type{Function}, x) = eval(parse(x))
writeas(x::Function) = string(x)

readas{T<:BitsKindOrString}(::Type{T}, x) = x
writeas(x::BitsKindOrString) = x

function readas{T,N}(::Type{Array{T,N}}, x)
    if T <: BitsKindOrString
        x
    elseif N == 1
        data = Array(T, length(x))
        for (k,v) in x
            data[parse(Int,k)] = v
        end
        data
    else
        throw("Invalid data: $x")
    end
end
function writeas{T,N}(x::Array{T,N})
    if T <: BitsKindOrString
        x
    elseif N == 1
        Dict(i=>x[i] for i=1:length(x))
    else
        throw("x::$(typeof(x)) is not supported.")
    end
end

function readas{K<:BitsKindOrString,V<:BitsKindOrString}(::Type{Dict{K,V}}, x)
    dict = Dict{K,V}()
    for line in split(x, "\n")
        items = split(chomp(line), "\t")
        k = K <: String ? String(items[1]) : parse(K,items[1])
        v = V <: String ? String(items[2]) : parse(V,items[2])
        dict[k] = v
    end
    dict
end
function writeas{K<:BitsKindOrString,V<:BitsKindOrString}(x::Dict{K,V})
    lines = String[]
    for (k,v) in x
        push!(lines, "$k\t$v")
    end
    join(lines, "\n")
end

function readas{T<:Tuple}(::Type{T}, x)
    data = Array(Any, length(x))
    for (k,v) in x
        data[parse(Int,k)] = v
    end
    tuple(data...)
end
writeas(x::Tuple) = Dict(i=>x[i] for i=1:length(x))

function readas(T::Type, x)
    values = map(name -> x[string(name)], fieldnames(T))
    T(values...)
{'fraction_non_alphanumeric': 0.19180672268907564, 'fraction_numerical': 0.008823529411764706, 'mean_word_length': 3.129228100607112, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2425098', 'n_tokens_mistral': 1888, 'n_tokens_neox': 1891, 'n_words': 412}