<commit_before>module NRRD

using Images, SIUnits, SIUnits.ShortUnits
import Images: imread, imwrite
import Zlib

typedict = [
    "signed char" => Int8,
    "int8" => Int8,
    "int8_t" => Int8,
    "uchar" => Uint8,
    "unsigned char" => Uint8,
    "uint8" => Uint8,
    "uint8_t" => Uint8,
    "short" => Int16,
    "short int" => Int16,
    "signed short" => Int16,
    "signed short int" => Int16,
    "int16" => Int16,
    "int16_t" => Int16,
    "ushort" => Uint16,
    "unsigned short" => Uint16,
    "unsigned short int" => Uint16,
    "uint16" => Uint16,
    "uint16_t" => Uint16,
    "int" => Int32,
    "signed int" => Int32,
    "int32" => Int32,
    "int32_t" => Int32,
    "uint" => Uint32,
    "unsigned int" => Uint32,
    "uint32" => Uint32,
    "uint32_t" => Uint32,
    "longlong" => Int64,
    "long long" => Int64,
    "long long int" => Int64,
    "signed long long" => Int64,
    "signed long long int" => Int64,
    "int64" => Int64,
    "int64_t" => Int64,
    "ulonglong" => Uint64,
    "unsigned long long" => Uint64,
    "unsigned long long int" => Uint64,
    "uint64" => Uint64,
    "uint64_t" => Uint64,
    "float" => Float32,
    "double" => Float64]

function myendian()
    if ENDIAN_BOM == 0x04030201
        return "little"
    elseif ENDIAN_BOM == 0x01020304
        return "big"
    end
end

function imread{S<:IO}(stream::S, ::Type{Images.NRRDFile})
    version = ascii(read(stream, Uint8, 4))
    skipchars(stream,isspace)
    header = Dict{ASCIIString, UTF8String}()
    comments = Array(ASCIIString, 0)
    # Read until we encounter a blank line, which is the separator between the header and data
    line = strip(readline(stream))
    while !isempty(line)
        if line[1] != '#'
            key, value = split(line, ":")
            header[key] = strip(value)
        else
            cmt = strip(lstrip(line, collect("#")))
            if !isempty(cmt)
                push!(comments, cmt)
            end
        end
        line = strip(readline(stream))
    end
    # Check to see whether the data are in an external file
    sdata = stream
    if haskey(header, "data file")
        sdata = open(header["data file"])
    elseif haskey(header, "datafile")
        sdata = open(header["datafile"])
    end
    if header["encoding"] == "gzip"
        sdata = Zlib.Reader(sdata)
    end
    # Parse properties and read the data
    sz = parse_vector_int(header["sizes"])
    T = typedict[header["type"]]
    props = Dict{ASCIIString, Any}()
    local A
    if header["encoding"] == "raw" && prod(sz) > 10^8
        # Use memory-mapping for large files
        fn = stream2name(sdata)
        datalen = div(filesize(fn) - position(sdata), sizeof(T))
        strds = [1,cumprod(sz)]
        k = length(sz)
        sz[k] = div(datalen, strds[k])
        while sz[k] == 0 && k > 1
            pop!(sz)
            k -= 1
            sz[k] = div(datalen, strds[k])
        end
        A = mmap_array(T, tuple(sz...), sdata, position(sdata))
        if haskey(header, "endian")
            if header["endian"] != myendian()
                props["bswap"] = true
            end
        end
    elseif header["encoding"] == "raw" || header["encoding"] == "gzip"
        A = read(sdata, T, sz...)
        if haskey(header, "endian")
            if header["endian"] != myendian() && eltype(A) != Uint8
                A = reshape([bswap(a) for a in A], size(A))
            end
        end
    else
      error("\"", header["encoding"], "\" encoding not supported.")
    end
    # Optional fields
    if haskey(header, "kinds")
        kinds = split(header["kinds"], " ")
        for i = 1:length(kinds)
            k = kinds[i]
            if k == "time"
                props["timedim"] = i
            elseif in(k, ("3-color","4-color","list","point","vector","covariant-vector","normal","2-vector","3-vector","4-vector","3-gradient","3-normal","scalar","complex","quaternion"))
                props["colordim"] = i
                props["colorspace"] = k
            elseif k == "RGB-color"
                props["colordim"] = i
                props["colorspace"] = "RGB"
            elseif k == "HSV-color"
                props["colordim"] = i
                props["colorspace"] = "HSV"
            elseif k == "RGBA-color"
                props["colordim"] = i
                props["colorspace"] = "RGBA"
            elseif contains(k, "matrix")
                error("matrix types are not yet supported")
            end
        end
    end
    if !haskey(props, "colordim")
        props["colorspace"] = "Gray"
    end
    if haskey(header, "min") || haskey(header, "max")
        mn = typemin(T)
        mx = typemax(T)
        if T <: Integer
            if haskey(header, "min")
                mn = convert(T, parseint(header["min"]))
            end
            if haskey(header, "max")
                mx = convert(T, parseint(header["max"]))
            end
        else
            if haskey(header, "min")
                mn = convert(T, parsefloat(header["min"]))
            end
            if haskey(header, "max")
                mx = convert(T, parsefloat(header["max"]))
            end
        end
        props["limits"] = (mn, mx)
    end
    if haskey(header, "spacings")
        ps = parse_vector_float(header["spacings"])
        keep = trues(length(ps))
        cd = get(props, "colordim", 0)
        if 1 <= cd <= length(keep)
            keep[cd] = false
        end
        td = get(props, "timedim", 0)
        if 1 <= td <= length(keep)
            keep[td] = false
        end
        if haskey(header, "space units")
            ustrs = split(header["space units"], " ")
            if length(ustrs) != length(ps)
                warn("space units parsing did not agree with spacings")
                @show u
                @show ps
                props["pixelspacing"] = ps[keep]
            else
                u = Array(SIUnits.SIQuantity, 0)
                for i = 1:length(ustrs)
                    if keep[i]
                        utmp = ps[i]*eval(symbol(ustrs[i][2:end-1]))
                        push!(u, utmp)
                    end
                end
                props["pixelspacing"] = u
            end
        else
            props["pixelspacing"] = ps[keep]
        end
    end
    if !isempty(comments)
        props["comments"] = comments
    end
    img = Image(A, props)
    spatialorder = ["x", "y", "z"]
    img.properties["spatialorder"] = spatialorder[1:sdims(img)]
    img
end

function imwrite(img, sheader::IO, ::Type{Images.NRRDFile}, props::Dict = Dict{ASCIIString,Any}())
    println(sheader, "NRRD0001")
    # Write the datatype
    T = get(props, "type", eltype(data(img)))
    if T<:FloatingPoint
        println(sheader, "type: ", (T == Float32) ? "float" : "double")
    else
        println(sheader, "type: ", lowercase(string(T)))
    end
    # Extract size and kinds
    sz = get(props, "sizes", size(img))
    kinds = ["space" for i = 1:length(sz)]
    td = timedim(img)
    if td != 0<commit_msg>NRRD: make extra props a keyword argument, so it can be passed to imwrite<commit_after>module NRRD

using Images, SIUnits, SIUnits.ShortUnits
import Images: imread, imwrite
import Zlib

typedict = [
    "signed char" => Int8,
    "int8" => Int8,
    "int8_t" => Int8,
    "uchar" => Uint8,
    "unsigned char" => Uint8,
    "uint8" => Uint8,
    "uint8_t" => Uint8,
    "short" => Int16,
    "short int" => Int16,
    "signed short" => Int16,
    "signed short int" => Int16,
    "int16" => Int16,
    "int16_t" => Int16,
    "ushort" => Uint16,
    "unsigned short" => Uint16,
    "unsigned short int" => Uint16,
    "uint16" => Uint16,
    "uint16_t" => Uint16,
    "int" => Int32,
    "signed int" => Int32,
    "int32" => Int32,
    "int32_t" => Int32,
    "uint" => Uint32,
    "unsigned int" => Uint32,
    "uint32" => Uint32,
    "uint32_t" => Uint32,
    "longlong" => Int64,
    "long long" => Int64,
    "long long int" => Int64,
    "signed long long" => Int64,
    "signed long long int" => Int64,
    "int64" => Int64,
    "int64_t" => Int64,
    "ulonglong" => Uint64,
    "unsigned long long" => Uint64,
    "unsigned long long int" => Uint64,
    "uint64" => Uint64,
    "uint64_t" => Uint64,
    "float" => Float32,
    "double" => Float64]

function myendian()
    if ENDIAN_BOM == 0x04030201
        return "little"
    elseif ENDIAN_BOM == 0x01020304
        return "big"
    end
end

function imread{S<:IO}(stream::S, ::Type{Images.NRRDFile})
    version = ascii(read(stream, Uint8, 4))
    skipchars(stream,isspace)
    header = Dict{ASCIIString, UTF8String}()
    comments = Array(ASCIIString, 0)
    # Read until we encounter a blank line, which is the separator between the header and data
    line = strip(readline(stream))
    while !isempty(line)
        if line[1] != '#'
            key, value = split(line, ":")
            header[key] = strip(value)
        else
            cmt = strip(lstrip(line, collect("#")))
            if !isempty(cmt)
                push!(comments, cmt)
            end
        end
        line = strip(readline(stream))
    end
    # Check to see whether the data are in an external file
    sdata = stream
    if haskey(header, "data file")
        sdata = open(header["data file"])
    elseif haskey(header, "datafile")
        sdata = open(header["datafile"])
    end
    if header["encoding"] == "gzip"
        sdata = Zlib.Reader(sdata)
    end
    # Parse properties and read the data
    sz = parse_vector_int(header["sizes"])
    T = typedict[header["type"]]
    props = Dict{ASCIIString, Any}()
    local A
    if header["encoding"] == "raw" && prod(sz) > 10^8
        # Use memory-mapping for large files
        fn = stream2name(sdata)
        datalen = div(filesize(fn) - position(sdata), sizeof(T))
        strds = [1,cumprod(sz)]
        k = length(sz)
        sz[k] = div(datalen, strds[k])
        while sz[k] == 0 && k > 1
            pop!(sz)
            k -= 1
            sz[k] = div(datalen, strds[k])
        end
        A = mmap_array(T, tuple(sz...), sdata, position(sdata))
        if haskey(header, "endian")
            if header["endian"] != myendian()
                props["bswap"] = true
            end
        end
    elseif header["encoding"] == "raw" || header["encoding"] == "gzip"
        A = read(sdata, T, sz...)
        if haskey(header, "endian")
            if header["endian"] != myendian() && eltype(A) != Uint8
                A = reshape([bswap(a) for a in A], size(A))
            end
        end
    else
      error("\"", header["encoding"], "\" encoding not supported.")
    end
    # Optional fields
    if haskey(header, "kinds")
        kinds = split(header["kinds"], " ")
        for i = 1:length(kinds)
            k = kinds[i]
            if k == "time"
                props["timedim"] = i
            elseif in(k, ("3-color","4-color","list","point","vector","covariant-vector","normal","2-vector","3-vector","4-vector","3-gradient","3-normal","scalar","complex","quaternion"))
                props["colordim"] = i
                props["colorspace"] = k
            elseif k == "RGB-color"
                props["colordim"] = i
                props["colorspace"] = "RGB"
            elseif k == "HSV-color"
                props["colordim"] = i
                props["colorspace"] = "HSV"
            elseif k == "RGBA-color"
                props["colordim"] = i
                props["colorspace"] = "RGBA"
            elseif contains(k, "matrix")
                error("matrix types are not yet supported")
            end
        end
    end
    if !haskey(props, "colordim")
        props["colorspace"] = "Gray"
    end
    if haskey(header, "min") || haskey(header, "max")
        mn = typemin(T)
        mx = typemax(T)
        if T <: Integer
            if haskey(header, "min")
                mn = convert(T, parseint(header["min"]))
            end
            if haskey(header, "max")
                mx = convert(T, parseint(header["max"]))
            end
        else
            if haskey(header, "min")
                mn = convert(T, parsefloat(header["min"]))
            end
            if haskey(header, "max")
                mx = convert(T, parsefloat(header["max"]))
            end
        end
        props["limits"] = (mn, mx)
    end
    if haskey(header, "spacings")
        ps = parse_vector_float(header["spacings"])
        keep = trues(length(ps))
        cd = get(props, "colordim", 0)
        if 1 <= cd <= length(keep)
            keep[cd] = false
        end
        td = get(props, "timedim", 0)
        if 1 <= td <= length(keep)
            keep[td] = false
        end
        if haskey(header, "space units")
            ustrs = split(header["space units"], " ")
            if length(ustrs) != length(ps)
                warn("space units parsing did not agree with spacings")
                @show u
                @show ps
                props["pixelspacing"] = ps[keep]
            else
                u = Array(SIUnits.SIQuantity, 0)
                for i = 1:length(ustrs)
                    if keep[i]
                        utmp = ps[i]*eval(symbol(ustrs[i][2:end-1]))
                        push!(u, utmp)
                    end
                end
                props["pixelspacing"] = u
            end
        else
            props["pixelspacing"] = ps[keep]
        end
    end
    if !isempty(comments)
        props["comments"] = comments
    end
    img = Image(A, props)
    spatialorder = ["x", "y", "z"]
    img.properties["spatialorder"] = spatialorder[1:sdims(img)]
    img
end

function imwrite(img, sheader::IO, ::Type{Images.NRRDFile}; props::Dict = Dict{ASCIIString,Any}())
    println(sheader, "NRRD0001")
    # Write the datatype
    T = get(props, "type", eltype(data(img)))
    if T<:FloatingPoint
        println(sheader, "type: ", (T == Float32) ? "float" : "double")
    else
        println(sheader, "type: ", lowercase(string(T)))
    end
    # Extract size and kinds
    sz = get(props, "sizes", size(img))
    kinds = ["space" for i = 1:length(sz)]
    td = timedim(img)
    if td != 0
{'fraction_non_alphanumeric': 0.1480537862703468, 'fraction_numerical': 0.022363765038924274, 'mean_word_length': 1.9136082474226803, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4785112', 'n_tokens_mistral': 4825, 'n_tokens_neox': 4589, 'n_words': 1136}