<commit_before>"""
    Datatype

A `Datatype` represents the layout of the data in memory.

# Usage

    Datatype(T)

Either return the predefined `Datatype` corresponding to `T`, or create a new `Datatype`
for the Julia type `T`, calling [`Types.commit!`](@ref) so that it can be used for
communication operations.

Note that this can only be called on types for which `isbitstype(T)` is `true`.
"""
@mpi_handle Datatype MPI_Datatype

const DATATYPE_NULL = _Datatype(MPI_DATATYPE_NULL)

const MPI_Datatype_default = MPI_Datatype == Cint ? MPI_DATATYPE_NULL : C_NULL
Datatype() = Datatype(MPI_Datatype_default)


function free(dt::Datatype)
    if dt.val != DATATYPE_NULL.val && !Finalized()
        @mpichk ccall((:MPI_Type_free, libmpi), Cint, (Ptr{MPI_Datatype},), dt)
    end
    return nothing
end

# attributes
function _type_null_copy_fn(oldtype::MPI_Datatype, type_keyval::Cint,
                            extra_state::Ptr{Cvoid},
                            attribute_val_in::Ptr{Cvoid}, attribute_val_out::Ptr{Cvoid},
                            flag::Ptr{Cint})
    unsafe_store!(flag, Cint(0))
    return MPI_SUCCESS
end
function _type_null_delete_fn(oldtype::MPI_Datatype, type_keyval::Cint,
                           attribute_val_in::Ptr{Cvoid}, extra_state::Ptr{Cvoid})
    return MPI_SUCCESS
end

function create_keyval(::Type{Datatype})
    ref = Ref(Cint(0))
    @mpichk ccall((:MPI_Type_create_keyval, libmpi), Cint,
                  (Ptr{Cvoid},
                   Ptr{Cvoid},
                   Ptr{Cint},
                   Ptr{Cvoid}),
                  @cfunction(_type_null_copy_fn,Cint,(MPI_Datatype, Cint, Ptr{Cvoid},
                                                      Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cint})),
                  @cfunction(_type_null_delete_fn,Cint,(MPI_Datatype, Cint, Ptr{Cvoid},
                                                      Ptr{Cvoid})),
                  ref, C_NULL)
    return ref[]
end
function set_attr!(datatype::Datatype, keyval::Cint, attrval::Ptr{Cvoid})
    @mpichk ccall((:MPI_Type_set_attr, libmpi), Cint,
                  (MPI_Datatype, Cint, Ptr{Cvoid}), datatype, keyval, attrval)
    return nothing
end
function get_attr(datatype::Datatype, keyval::Cint)
    flagref = Ref(Cint(0))
    attrref = Ref{Ptr{Cvoid}}(C_NULL)
    @mpichk ccall((:MPI_Type_get_attr, libmpi), Cint,
                  (MPI_Datatype, Cint, Ptr{Ptr{Cvoid}}, Ptr{Cint}), datatype, keyval, attrref, flagref)
    if flagref[] == 0
        return nothing
    end
    return attrref[]
end
function del_attr!(datatype::Datatype, keyval::Cint)
    flagref = Ref(Cint(0))
    attrref = Ref{Ptr{Cvoid}}(C_NULL)
    @mpichk ccall((:MPI_Type_delete_attr, libmpi), Cint,
                  (MPI_Datatype, Cint), datatype, keyval)
    return nothing
end

# names
function get_name(datatype::Datatype)
    MPI.Initialized() || return ""
    buffer = Array{UInt8}(undef, MPI_MAX_OBJECT_NAME)
    lenref = Ref{Cint}()
    @mpichk ccall((:MPI_Type_get_name, libmpi), Cint,
                  (MPI_Datatype, Ptr{UInt8}, Ptr{Cint}),
                  datatype, buffer, lenref)
    return String(resize!(buffer, lenref[]))
end

# datatype attribute to store Julia type
const JULIA_TYPE_PTR_ATTR = Ref(Cint(0))
push!(mpi_init_hooks, () -> JULIA_TYPE_PTR_ATTR[] = create_keyval(Datatype))

"""
    to_type(datatype::Datatype)

Return the Julia type corresponding to the MPI [`Datatype`](@ref) `datatype`, or `nothing`
if it doesn't correspond directly.
"""
function to_type(datatype::Datatype)
    if MPI.Initialized()
        ptr = get_attr(datatype, JULIA_TYPE_PTR_ATTR[])
        if !isnothing(ptr)
            return unsafe_pointer_to_objref(ptr)
        end
    end
    return nothing
end


# predefined
for (mpiname, T) in [
    :INT8_T            => Int8
    :UINT8_T           => UInt8
    :INT16_T           => Int16
    :UINT16_T          => UInt16
    :INT32_T           => Int32
    :UINT32_T          => UInt32
    :INT64_T           => Int64
    :UINT64_T          => UInt64
    :BYTE              => UInt8
    :SHORT             => Cshort
    :UNSIGNED_SHORT    => Cushort
    :INT               => Cint
    :UNSIGNED          => Cuint
    :LONG              => Clong
    :UNSIGNED_LONG     => Culong
    :CHAR              => Cchar
    :SIGNED_CHAR       => Cchar
    :UNSIGNED_CHAR     => Cuchar
    :WCHAR             => Cwchar_t
    :FLOAT             => Float32
    :DOUBLE            => Float64
    :C_FLOAT_COMPLEX   => ComplexF32
    :C_DOUBLE_COMPLEX  => ComplexF64]

    @eval if @isdefined($(Symbol(:MPI_,mpiname)))
        const $mpiname = _Datatype($(Symbol(:MPI_,mpiname)))
        if !hasmethod(Datatype, Tuple{Type{$T}})
            Datatype(::Type{$T}) = $mpiname
            push!(mpi_init_hooks, () -> set_attr!($mpiname, JULIA_TYPE_PTR_ATTR[], pointer_from_objref($T)))
        end
    end
end


@generated function Datatype(::Type{T}) where {T}
    newtype = Datatype()
    quote
        datatype = $newtype
        # lazily initialize so that it can be safely precompiled
        if datatype.val === MPI.MPI_Datatype_default
            Types.create!(datatype, T)
            Types.commit!(datatype)
            set_attr!(datatype, JULIA_TYPE_PTR_ATTR[], pointer_from_objref(T))
        end
        return datatype
    end
end


function Base.show(io::IO, datatype::Datatype)
    show(io, Datatype)
    print(io, '(')
    juliatype = to_type(datatype)
    if isnothing(juliatype)
        show(io, datatype.val)
    else
        show(io, juliatype)
    end
    print(io, ')')
    name = get_name(datatype)
    if name != ""
        print(io, ": ")
        print(io, name)
    end
end



module Types

import MPI
import MPI: @mpichk, libmpi, _doc_external,
    Datatype, MPI_Datatype, MPI_Aint,
    free

"""
    lb, extent = MPI.Types.extent(dt::MPI.Datatype)

Gets the lowerbound `lb` and the extent `extent` in bytes.<commit_msg>Add bool datatype handling (#522)<commit_after>"""
    Datatype

A `Datatype` represents the layout of the data in memory.

# Usage

    Datatype(T)

Either return the predefined `Datatype` corresponding to `T`, or create a new `Datatype`
for the Julia type `T`, calling [`Types.commit!`](@ref) so that it can be used for
communication operations.

Note that this can only be called on types for which `isbitstype(T)` is `true`.
"""
@mpi_handle Datatype MPI_Datatype

const DATATYPE_NULL = _Datatype(MPI_DATATYPE_NULL)

const MPI_Datatype_default = MPI_Datatype == Cint ? MPI_DATATYPE_NULL : C_NULL
Datatype() = Datatype(MPI_Datatype_default)


function free(dt::Datatype)
    if dt.val != DATATYPE_NULL.val && !Finalized()
        @mpichk ccall((:MPI_Type_free, libmpi), Cint, (Ptr{MPI_Datatype},), dt)
    end
    return nothing
end

# attributes
function _type_null_copy_fn(oldtype::MPI_Datatype, type_keyval::Cint,
                            extra_state::Ptr{Cvoid},
                            attribute_val_in::Ptr{Cvoid}, attribute_val_out::Ptr{Cvoid},
                            flag::Ptr{Cint})
    unsafe_store!(flag, Cint(0))
    return MPI_SUCCESS
end
function _type_null_delete_fn(oldtype::MPI_Datatype, type_keyval::Cint,
                           attribute_val_in::Ptr{Cvoid}, extra_state::Ptr{Cvoid})
    return MPI_SUCCESS
end

function create_keyval(::Type{Datatype})
    ref = Ref(Cint(0))
    @mpichk ccall((:MPI_Type_create_keyval, libmpi), Cint,
                  (Ptr{Cvoid},
                   Ptr{Cvoid},
                   Ptr{Cint},
                   Ptr{Cvoid}),
                  @cfunction(_type_null_copy_fn,Cint,(MPI_Datatype, Cint, Ptr{Cvoid},
                                                      Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cint})),
                  @cfunction(_type_null_delete_fn,Cint,(MPI_Datatype, Cint, Ptr{Cvoid},
                                                      Ptr{Cvoid})),
                  ref, C_NULL)
    return ref[]
end
function set_attr!(datatype::Datatype, keyval::Cint, attrval::Ptr{Cvoid})
    @mpichk ccall((:MPI_Type_set_attr, libmpi), Cint,
                  (MPI_Datatype, Cint, Ptr{Cvoid}), datatype, keyval, attrval)
    return nothing
end
function get_attr(datatype::Datatype, keyval::Cint)
    flagref = Ref(Cint(0))
    attrref = Ref{Ptr{Cvoid}}(C_NULL)
    @mpichk ccall((:MPI_Type_get_attr, libmpi), Cint,
                  (MPI_Datatype, Cint, Ptr{Ptr{Cvoid}}, Ptr{Cint}), datatype, keyval, attrref, flagref)
    if flagref[] == 0
        return nothing
    end
    return attrref[]
end
function del_attr!(datatype::Datatype, keyval::Cint)
    flagref = Ref(Cint(0))
    attrref = Ref{Ptr{Cvoid}}(C_NULL)
    @mpichk ccall((:MPI_Type_delete_attr, libmpi), Cint,
                  (MPI_Datatype, Cint), datatype, keyval)
    return nothing
end

# names
function get_name(datatype::Datatype)
    MPI.Initialized() || return ""
    buffer = Array{UInt8}(undef, MPI_MAX_OBJECT_NAME)
    lenref = Ref{Cint}()
    @mpichk ccall((:MPI_Type_get_name, libmpi), Cint,
                  (MPI_Datatype, Ptr{UInt8}, Ptr{Cint}),
                  datatype, buffer, lenref)
    return String(resize!(buffer, lenref[]))
end

# datatype attribute to store Julia type
const JULIA_TYPE_PTR_ATTR = Ref(Cint(0))
push!(mpi_init_hooks, () -> JULIA_TYPE_PTR_ATTR[] = create_keyval(Datatype))

"""
    to_type(datatype::Datatype)

Return the Julia type corresponding to the MPI [`Datatype`](@ref) `datatype`, or `nothing`
if it doesn't correspond directly.
"""
function to_type(datatype::Datatype)
    if MPI.Initialized()
        ptr = get_attr(datatype, JULIA_TYPE_PTR_ATTR[])
        if !isnothing(ptr)
            return unsafe_pointer_to_objref(ptr)
        end
    end
    return nothing
end


# predefined
for (mpiname, T) in [
    :INT8_T            => Int8
    :UINT8_T           => UInt8
    :INT16_T           => Int16
    :UINT16_T          => UInt16
    :INT32_T           => Int32
    :UINT32_T          => UInt32
    :INT64_T           => Int64
    :UINT64_T          => UInt64
    :BYTE              => UInt8
    :SHORT             => Cshort
    :UNSIGNED_SHORT    => Cushort
    :INT               => Cint
    :UNSIGNED          => Cuint
    :LONG              => Clong
    :UNSIGNED_LONG     => Culong
    :CHAR              => Cchar
    :SIGNED_CHAR       => Cchar
    :UNSIGNED_CHAR     => Cuchar
    :WCHAR             => Cwchar_t
    :FLOAT             => Float32
    :DOUBLE            => Float64
    :C_FLOAT_COMPLEX   => ComplexF32
    :C_DOUBLE_COMPLEX  => ComplexF64
    :C_BOOL            => Bool
]

    @eval if @isdefined($(Symbol(:MPI_,mpiname)))
        const $mpiname = _Datatype($(Symbol(:MPI_,mpiname)))
        if !hasmethod(Datatype, Tuple{Type{$T}})
            Datatype(::Type{$T}) = $mpiname
            push!(mpi_init_hooks, () -> begin
                  @assert Types.size($mpiname) == sizeof($T)
                  set_attr!($mpiname, JULIA_TYPE_PTR_ATTR[], pointer_from_objref($T))
                  end)
        end
    end
end


@generated function Datatype(::Type{T}) where {T}
    newtype = Datatype()
    quote
        datatype = $newtype
        # lazily initialize so that it can be safely precompiled
        if datatype.val === MPI.MPI_Datatype_default
            Types.create!(datatype, T)
            Types.commit!(datatype)
            set_attr!(datatype, JULIA_TYPE_PTR_ATTR[], pointer_from_objref(T))
        end
        return datatype
    end
end


function Base.show(io::IO, datatype::Datatype)
    show(io, Datatype)
    print(io, '(')
    juliatype = to_type(datatype)
    if isnothing(juliatype)
        show(io, datatype.val)
    else
        show(io, juliatype)
    end
    print(io, ')')
    name = get_name(datatype)
    if name != ""
        print(io, ": ")
        print(io, name)
    end
end



module Types

import MPI
import MPI: @mpichk, libmpi, _doc_external,
    Datatype, MPI_Datatype, MPI_Aint,
    free

# ??
function size(dt::Datatype)
    dtsize = Ref{Cint}()
    @mpichk ccall((:MPI_Type_size, libmpi), Cint,
                  (MPI_Datatype, Ptr{Cint}),
                  dt, dtsize)
    return dtsize[]
end


"""
    lb, extent = MPI.Types.extent(dt::MPI.Datatype)

Gets the lowerbound `lb` and the extent `extent` in bytes.
{'fraction_non_alphanumeric': 0.11606848864010537, 'fraction_numerical': 0.007655581165623971, 'mean_word_length': 2.4426183054689714, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4354948', 'n_tokens_mistral': 4244, 'n_tokens_neox': 4089, 'n_words': 869}