<commit_before>using Compat

const MPIDatatype = Union{Char,
                            Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64,
                            UInt64,
                            Float32, Float64, ComplexF32, ComplexF64}
MPIBuffertype{T} = Union{Ptr{T}, Array{T}, SubArray{T}, Ref{T}}

# Type used to hold constant void pointers such as MPI_IN_PLACE and MPI_BOTTOM
primitive type ConstantPtr sizeof(Ptr{Cvoid})*8 end
Base.cconvert(::Type{Ptr{T}}, x::ConstantPtr) where {T} = reinterpret(Ptr{T}, x)

MPIBuffertypeOrConst{T} = Union{MPIBuffertype{T}, ConstantPtr}

fieldoffsets(::Type{T}) where {T} = Int[fieldoffset(T, i) for i in 1:length(fieldnames(T))]

# Define a function mpitype(T) that returns the MPI datatype code for
# a given type T. In the case the the type does not exist, it is created and
# then returned. The dictonary is defined in __init__ so the module can be
# precompiled

# accessor and creation function for getting MPI datatypes
function mpitype(::Type{T}) where T
    if haskey(mpitype_dict, T)  # if the datatype already exists
      return mpitype_dict[T]
    end

    if !isbitstype(T)
        throw(ArgumentError("Type must be isbitstype()"))
    end

    # get the data from the type
    fieldtypes = T.types
    offsets = fieldoffsets(T)
    nfields = Cint(length(fieldtypes))

    if nfields == 0  # primitive type
      if sizeof(T) == 0
        error("Can't convert 0-size type to MPI")
      end
      nfields, blocklengths, displacements, types = factorPrimitiveType(T)
    else  # struct
      # put data in MPI format
      blocklengths = ones(Cint, nfields)
      displacements = zeros(Cptrdiff_t, nfields)  # size_t == MPI_Aint ?
      types = zeros(Cint, nfields)
      for i=1:nfields
          displacements[i] = offsets[i]
          # create an MPI_Datatype for the current field if it does not exist yet
          types[i] = mpitype(fieldtypes[i])
      end

    end

    newtype = Type_Create_Struct(nfields, blocklengths, displacements, types)

    # commit the datatatype
    Type_Commit(newtype)

    # add it to the dictonary of known types
    recordDataType(T, newtype)

    return mpitype_dict[T]
end


function factorPrimitiveType(::Type{T}) where {T}

  tsize = sizeof(T)  # size in bytes
  displacements = zeros(Cptrdiff_t, 0)  # size_t == MPI_Aint ?
  types = zeros(Cint, 0)

  # put largest sizes first
  mpi_types = [MPI_INTEGER8, MPI_INTEGER4, MPI_INTEGER2, MPI_INTEGER1]
  mpi_sizes = [8, 4, 2, 1]
  curr_disp = 0

  while curr_disp != tsize
    remsize = tsize - curr_disp
    for i=1:length(mpi_types)
      size_i = mpi_sizes[i]
      # because each size is a multiple of the smaller sizes, taking the largest
      # size always results in the smallest number of types and doesn't result
      # in avoidable small remainders
      if remsize >= size_i
        push!(types, mpi_types[i])
        push!(displacements, curr_disp)
        curr_disp += size_i
        break
      end
    end
  end
  
  nfields = length(types)
  blocklengths = ones(Cint, nfields)

  return nfields, blocklengths, displacements, types
end


mutable struct Comm
    val::Cint
    Comm(val::Integer) = new(val)
end
const COMM_NULL  = Comm(MPI_COMM_NULL)
const COMM_SELF  = Comm(MPI_COMM_SELF)
const COMM_WORLD = Comm(MPI_COMM_WORLD)

mutable struct Op
    val::Cint
end
const OP_NULL = Op(MPI_OP_NULL)
const BAND    = Op(MPI_BAND)
const BOR     = Op(MPI_BOR)
const BXOR    = Op(MPI_BXOR)
const LAND    = Op(MPI_LAND)
const LOR     = Op(MPI_LOR)
const LXOR    = Op(MPI_LXOR)
const MAX     = Op(MPI_MAX)
const MIN     = Op(MPI_MIN)
const NO_OP   = Op(MPI_NO_OP)
const PROD    = Op(MPI_PROD)
const REPLACE = Op(MPI_REPLACE)
const SUM     = Op(MPI_SUM)

mutable struct Request
    val::Cint
    buffer
end
const REQUEST_NULL = Request(MPI_REQUEST_NULL, nothing)

mutable struct Info
    val::Cint
    function Info()
        newinfo = Ref{Cint}()
        ccall(MPI_INFO_CREATE, Nothing, (Ptr{Cint}, Ref{Cint}), newinfo, 0)
        info = new(newinfo[])        
        refcount_inc()        
        finalizer(info) do x
            ccall(MPI_INFO_FREE, Nothing, (Ref{Cint}, Ref{Cint}), x.val, 0)
            x.val = MPI_INFO_NULL
            refcount_dec()
        end
        info
    end

    function Info(val::Cint)
        if val != MPI_INFO_NULL
            error("Info can only be created using Info()")
        end
        return new(MPI_INFO_NULL)
    end
end
const INFO_NULL = Info(MPI_INFO_NULL)

# the info functions assume that Fortran hidden arguments are placed at the end of the argument list
function Info_set(info::Info,key::AbstractString,value::AbstractString)
    @assert isascii(key) && isascii(value)
    ccall(MPI_INFO_SET, Nothing,
          (Ref{Cint}, Ptr{UInt8}, Ptr{UInt8}, Ref{Cint}, Csize_t, Csize_t),
           info.val, key, value, 0, sizeof(key), sizeof(value))
end

function Info_get(info::Info,key::AbstractString)
    @assert isascii(key)
    keyexists=Ref{Bool}()
    len=Ref{Cint}()
    ccall(MPI_INFO_GET_VALUELEN, Nothing,
          (Ref{Cint}, Ptr{UInt8}, Ptr{Cint}, Ptr{Bool}, Ref{Cint}, Csize_t),
           info.val, key, len, keyexists, 0, sizeof(key))
    if keyexists[]
        value=" "^(len[])
        ccall(MPI_INFO_GET, Nothing,
              (Ref{Cint}, Ptr{UInt8}, Ptr{Cint}, Ptr{UInt8}, Ptr{Bool}, Ref{Cint}, Csize_t, Csize_t),
               info.val, key, len, value, keyexists, 0, sizeof(key), sizeof(value))
    else
        value=""
    end
    value
end

function Info_delete(info::Info,key::AbstractString)
    @assert isascii(key)
    ccall(MPI_INFO_DELETE, Nothing,
          (Ref{Cint}, Ptr{UInt8}, Ref{Cint}, Csize_t), info.val, key, 0, sizeof(key))
end
Info_free(info::Info) = finalize(info)

mutable struct Status
    val::Array{Cint,1}
    Status() = new(Array{Cint}(undef, MPI_STATUS_SIZE))
end
Get_error(stat::Status) = Int(stat.val[MPI_ERROR])
Get_source(stat::Status) = Int(stat.val[MPI_SOURCE])
Get_tag(stat::Status) = Int(stat.val[MPI_TAG])

mutable struct Win
    val::Cint
    Win() = new(0)
end
<commit_msg>allocated buffer correctly for strings<commit_after>using Compat

const MPIDatatype = Union{Char,
                            Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64,
                            UInt64,
                            Float32, Float64, ComplexF32, ComplexF64}
MPIBuffertype{T} = Union{Ptr{T}, Array{T}, SubArray{T}, Ref{T}}

# Type used to hold constant void pointers such as MPI_IN_PLACE and MPI_BOTTOM
primitive type ConstantPtr sizeof(Ptr{Cvoid})*8 end
Base.cconvert(::Type{Ptr{T}}, x::ConstantPtr) where {T} = reinterpret(Ptr{T}, x)

MPIBuffertypeOrConst{T} = Union{MPIBuffertype{T}, ConstantPtr}

fieldoffsets(::Type{T}) where {T} = Int[fieldoffset(T, i) for i in 1:length(fieldnames(T))]

# Define a function mpitype(T) that returns the MPI datatype code for
# a given type T. In the case the the type does not exist, it is created and
# then returned. The dictonary is defined in __init__ so the module can be
# precompiled

# accessor and creation function for getting MPI datatypes
function mpitype(::Type{T}) where T
    if haskey(mpitype_dict, T)  # if the datatype already exists
      return mpitype_dict[T]
    end

    if !isbitstype(T)
        throw(ArgumentError("Type must be isbitstype()"))
    end

    # get the data from the type
    fieldtypes = T.types
    offsets = fieldoffsets(T)
    nfields = Cint(length(fieldtypes))

    if nfields == 0  # primitive type
      if sizeof(T) == 0
        error("Can't convert 0-size type to MPI")
      end
      nfields, blocklengths, displacements, types = factorPrimitiveType(T)
    else  # struct
      # put data in MPI format
      blocklengths = ones(Cint, nfields)
      displacements = zeros(Cptrdiff_t, nfields)  # size_t == MPI_Aint ?
      types = zeros(Cint, nfields)
      for i=1:nfields
          displacements[i] = offsets[i]
          # create an MPI_Datatype for the current field if it does not exist yet
          types[i] = mpitype(fieldtypes[i])
      end

    end

    newtype = Type_Create_Struct(nfields, blocklengths, displacements, types)

    # commit the datatatype
    Type_Commit(newtype)

    # add it to the dictonary of known types
    recordDataType(T, newtype)

    return mpitype_dict[T]
end


function factorPrimitiveType(::Type{T}) where {T}

  tsize = sizeof(T)  # size in bytes
  displacements = zeros(Cptrdiff_t, 0)  # size_t == MPI_Aint ?
  types = zeros(Cint, 0)

  # put largest sizes first
  mpi_types = [MPI_INTEGER8, MPI_INTEGER4, MPI_INTEGER2, MPI_INTEGER1]
  mpi_sizes = [8, 4, 2, 1]
  curr_disp = 0

  while curr_disp != tsize
    remsize = tsize - curr_disp
    for i=1:length(mpi_types)
      size_i = mpi_sizes[i]
      # because each size is a multiple of the smaller sizes, taking the largest
      # size always results in the smallest number of types and doesn't result
      # in avoidable small remainders
      if remsize >= size_i
        push!(types, mpi_types[i])
        push!(displacements, curr_disp)
        curr_disp += size_i
        break
      end
    end
  end
  
  nfields = length(types)
  blocklengths = ones(Cint, nfields)

  return nfields, blocklengths, displacements, types
end


mutable struct Comm
    val::Cint
    Comm(val::Integer) = new(val)
end
const COMM_NULL  = Comm(MPI_COMM_NULL)
const COMM_SELF  = Comm(MPI_COMM_SELF)
const COMM_WORLD = Comm(MPI_COMM_WORLD)

mutable struct Op
    val::Cint
end
const OP_NULL = Op(MPI_OP_NULL)
const BAND    = Op(MPI_BAND)
const BOR     = Op(MPI_BOR)
const BXOR    = Op(MPI_BXOR)
const LAND    = Op(MPI_LAND)
const LOR     = Op(MPI_LOR)
const LXOR    = Op(MPI_LXOR)
const MAX     = Op(MPI_MAX)
const MIN     = Op(MPI_MIN)
const NO_OP   = Op(MPI_NO_OP)
const PROD    = Op(MPI_PROD)
const REPLACE = Op(MPI_REPLACE)
const SUM     = Op(MPI_SUM)

mutable struct Request
    val::Cint
    buffer
end
const REQUEST_NULL = Request(MPI_REQUEST_NULL, nothing)

mutable struct Info
    val::Cint
    function Info()
        newinfo = Ref{Cint}()
        ccall(MPI_INFO_CREATE, Nothing, (Ptr{Cint}, Ref{Cint}), newinfo, 0)
        info = new(newinfo[])        
        refcount_inc()        
        finalizer(info) do x
            ccall(MPI_INFO_FREE, Nothing, (Ref{Cint}, Ref{Cint}), x.val, 0)
            x.val = MPI_INFO_NULL
            refcount_dec()
        end
        info
    end

    function Info(val::Cint)
        if val != MPI_INFO_NULL
            error("Info can only be created using Info()")
        end
        return new(MPI_INFO_NULL)
    end
end
const INFO_NULL = Info(MPI_INFO_NULL)

# the info functions assume that Fortran hidden arguments are placed at the end of the argument list
function Info_set(info::Info,key::AbstractString,value::AbstractString)
    @assert isascii(key) && isascii(value)
    ccall(MPI_INFO_SET, Nothing,
          (Ref{Cint}, Ptr{UInt8}, Ptr{UInt8}, Ref{Cint}, Csize_t, Csize_t),
           info.val, key, value, 0, sizeof(key), sizeof(value))
end

function Info_get(info::Info,key::AbstractString)
    @assert isascii(key)
    keyexists=Ref{Cint}()
    len=Ref{Cint}()
    ccall(MPI_INFO_GET_VALUELEN, Nothing,
          (Ref{Cint}, Ptr{UInt8}, Ptr{Cint}, Ptr{Cint}, Ref{Cint}, Csize_t),
           info.val, key, len, keyexists, 0, sizeof(key))
    if keyexists[] != 0
        buffer = Vector{UInt8}(undef, len[])
        ccall(MPI_INFO_GET, Nothing,
              (Ref{Cint}, Ptr{UInt8}, Ptr{Cint}, Ptr{UInt8}, Ptr{Cint}, Ref{Cint}, Csize_t, Csize_t),
              info.val, key, len, buffer, keyexists, 0, sizeof(key), sizeof(buffer))
        return String(buffer)
    else
        return ""
    end
end

function Info_delete(info::Info,key::AbstractString)
    @assert isascii(key)
    ccall(MPI_INFO_DELETE, Nothing,
          (Ref{Cint}, Ptr{UInt8}, Ref{Cint}, Csize_t), info.val, key, 0, sizeof(key))
end
Info_free(info::Info) = finalize(info)

mutable struct Status
    val::Array{Cint,1}
    Status() = new(Array{Cint}(undef, MPI_STATUS_SIZE))
end
Get_error(stat::Status) = Int(stat.val[MPI_ERROR])
Get_source(stat::Status) = Int(stat.val[MPI_SOURCE])
Get_tag(stat::Status) = Int(stat.val[MPI_TAG])

mutable struct Win
    val::Cint
    Win() = new(0)
end

{'fraction_non_alphanumeric': 0.10659981918303608, 'fraction_numerical': 0.009040848195939838, 'mean_word_length': 3.152901023890785, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1823954', 'n_tokens_mistral': 4412, 'n_tokens_neox': 4259, 'n_words': 1131}