<commit_before>e##############################################################################
# Julia wrapper generator using libclang from the LLVM project                                #
###############################################################################

module wrap_c
    version = v"0.0.0"

using Clang.cindex
import ..cindex.TypKind, ..cindex.CurKind

### Wrappable type hierarchy

abstract CArg

type IntrinsicArg <: CArg
    cursor::cindex.CXCursor
end

type TypedefArg <: CArg
    cursor::cindex.CXCursor
end

type PtrArg <: CArg
    cursor::cindex.CXCursor
end

type StructArg <: CArg
    cursor::cindex.CXCursor
    typedef::Any
end

type FunctionArg <: CArg
    cursor::cindex.CXCursor
end

type EnumArg <: CArg
    cursor::cindex.CXCursor
    typedef::Any
end

reserved_words = ["type", "end"]

### Execution context for wrap_c
typealias StringsArray Array{ASCIIString,1}

# InternalOptions
type InternalOptions
    wrap_structs::Bool
end
InternalOptions() = InternalOptions(false)

# WrapContext object stores shared information about the wrapping session
type WrapContext
    out_path::ASCIIString
    common_file::ASCIIString
    clang_includes::StringsArray             # clang include paths
    clang_extra_args::StringsArray         # additional {"-Arg", "value"} pairs for clang
    header_wrapped::Function                     # called to determine cursor inclusion status
    header_library::Function                     # called to determine shared library for given header
    header_outfile::Function                     # called to determine output file group for given header
    index::cindex.CXIndex
    common_stream
    cache_wrapped::Set{ASCIIString}
    output_streams::Dict{ASCIIString, IOStream}
    options::InternalOptions
end
WrapContext(op,cmn,incl,extra,hwrap,hlib,hout) = WrapContext(op,cmn,incl,extra,hwrap,hlib,hout,
                                                                                                    cindex.idx_create(1,1),None,Set{ASCIIString}(),
                                                                                                    Dict{ASCIIString,IOStream}(), InternalOptions())



### These helpers will be written to the generated file
helper_macros = "macro c(ret_type, func, arg_types, lib)
    local args_in = Any[ symbol(string('a',x)) for x in 1:length(arg_types.args) ]
    quote
        \$(esc(func))(\$(args_in...)) = ccall( (\$(string(func)), \$(Expr(:quote, lib)) ), \$ret_type, \$arg_types, \$(args_in...) )
    end
end

macro ctypedef(fake_t,real_t)
    quote
        typealias \$fake_t \$real_t
    end
end"

### libclang types to Julia types

c_jl = {
    TypKind.VOID                => Void,
    TypKind.BOOL                => Bool,
    TypKind.CHAR_U            => Uint8,
    TypKind.UCHAR             => Cuchar,
    TypKind.CHAR16            => Uint16,
    TypKind.CHAR32            => Uint32,
    TypKind.USHORT            => Uint16,
    TypKind.UINT                => Uint32,
    TypKind.ULONG             => Culong,
    TypKind.ULONGLONG     => Culonglong,
    TypKind.CHAR_S            => Uint8,         # TODO check
    TypKind.SCHAR             => Uint8,         # TODO check
    TypKind.WCHAR             => Char,
    TypKind.SHORT             => Int16,
    TypKind.INT                 => Cint,
    TypKind.LONG                => Clong,
    TypKind.LONGLONG        => Clonglong,
    TypKind.FLOAT             => Cfloat,
    TypKind.DOUBLE            => Cdouble,
    TypKind.LONGDOUBLE    => Float64,     # TODO detect?
    TypKind.ENUM                => Cint,            # TODO arch check?
    TypKind.NULLPTR         => C_NULL,
                                                                        # TypKind.UINT128 => TODO
    "size_t"                        => :Csize_t,
    "ptrdiff_t"                 => :Cptrdiff_t
    
    }

# Convert clang type to julia type
function ctype_to_julia(cutype::CXType)
    typkind = ty_kind(cutype)
    # Special cases: TYPEDEF, POINTER
    if (typkind == TypKind.POINTER)
        ptr_ctype = cindex.getPointeeType(cutype)
        ptr_jltype = ctype_to_julia(ptr_ctype)
        return Ptr{ptr_jltype}
    elseif (typkind == TypKind.TYPEDEF || typkind == TypKind.RECORD)
        basename = string( spelling( cindex.getTypeDeclaration(cutype) ) )
        return symbol( get(c_jl, basename, basename))
    else
        # TODO: missing mappings should generate a warning
        return symbol( string( get(c_jl, typkind, :Void) ))
    end
end

### eliminate symbol from the final type representation
rep_type(t) = begin
    replace(string(t), ":", "")
end

### Tuple representation without quotes
rep_args(v) = begin        
    o = IOBuffer()
    print(o, "(")
    s = first = start(v)
    r = (!done(v,s))
    while r
        x,s = next(v,s)
        print(o, x)
        done(v,s) && break
        print(o, ", ")
    end
    r && (s == next(v,first)[2]) && print(o, ",")
    print(o, ")")
    seek(o, 0)
    readall(o)
end

### Build array of include definitions: ["-I", "incpath",...] plus extra args
function build_clang_args(includes, extras)
    reduce(vcat, [], vcat([["-I",x] for x in includes], extras))
end

### Retrieve function arguments for a given cursor<commit_msg>Add WrapContext.fixup_names for name remaps<commit_after>e##############################################################################
# Julia wrapper generator using libclang from the LLVM project                                #
###############################################################################

module wrap_c
    version = v"0.0.0"

using Clang.cindex
import ..cindex.TypKind, ..cindex.CurKind

### Wrappable type hierarchy

abstract CArg

type IntrinsicArg <: CArg
    cursor::cindex.CXCursor
end

type TypedefArg <: CArg
    cursor::cindex.CXCursor
end

type PtrArg <: CArg
    cursor::cindex.CXCursor
end

type StructArg <: CArg
    cursor::cindex.CXCursor
    typedef::Any
end

type FunctionArg <: CArg
    cursor::cindex.CXCursor
end

type EnumArg <: CArg
    cursor::cindex.CXCursor
    typedef::Any
end

reserved_words = ["type", "end"]

### Execution context for wrap_c
typealias StringsArray Array{ASCIIString,1}

# InternalOptions
type InternalOptions
    wrap_structs::Bool
end
InternalOptions() = InternalOptions(false)

# WrapContext object stores shared information about the wrapping session
type WrapContext
    out_path::ASCIIString
    common_file::ASCIIString
    clang_includes::StringsArray             # clang include paths
    clang_extra_args::StringsArray         # additional {"-Arg", "value"} pairs for clang
    header_wrapped::Function                     # called to determine cursor inclusion status
    header_library::Function                     # called to determine shared library for given header
    header_outfile::Function                     # called to determine output file group for given header
    index::cindex.CXIndex
    fixup_names::Dict{ASCIIString, ASCIIString}
    common_stream
    cache_wrapped::Set{ASCIIString}
    output_streams::Dict{ASCIIString, IOStream}
    options::InternalOptions
end
WrapContext(op,cmn,incl,extra,hwrap,hlib,hout) = WrapContext(op,cmn,incl,extra,hwrap,hlib,hout,
                                                                                                    cindex.idx_create(1,1),None,Set{ASCIIString}(),
                                                                                                    Dict{ASCIIString,IOStream}(), InternalOptions())



### These helpers will be written to the generated file
helper_macros = "macro c(ret_type, func, arg_types, lib)
    local args_in = Any[ symbol(string('a',x)) for x in 1:length(arg_types.args) ]
    quote
        \$(esc(func))(\$(args_in...)) = ccall( (\$(string(func)), \$(Expr(:quote, lib)) ), \$ret_type, \$arg_types, \$(args_in...) )
    end
end

macro ctypedef(fake_t,real_t)
    quote
        typealias \$fake_t \$real_t
    end
end"

### libclang types to Julia types

c_jl = {
    TypKind.VOID                => Void,
    TypKind.BOOL                => Bool,
    TypKind.CHAR_U            => Uint8,
    TypKind.UCHAR             => Cuchar,
    TypKind.CHAR16            => Uint16,
    TypKind.CHAR32            => Uint32,
    TypKind.USHORT            => Uint16,
    TypKind.UINT                => Uint32,
    TypKind.ULONG             => Culong,
    TypKind.ULONGLONG     => Culonglong,
    TypKind.CHAR_S            => Uint8,         # TODO check
    TypKind.SCHAR             => Uint8,         # TODO check
    TypKind.WCHAR             => Char,
    TypKind.SHORT             => Int16,
    TypKind.INT                 => Cint,
    TypKind.LONG                => Clong,
    TypKind.LONGLONG        => Clonglong,
    TypKind.FLOAT             => Cfloat,
    TypKind.DOUBLE            => Cdouble,
    TypKind.LONGDOUBLE    => Float64,     # TODO detect?
    TypKind.ENUM                => Cint,            # TODO arch check?
    TypKind.NULLPTR         => C_NULL,
                                                                        # TypKind.UINT128 => TODO
    "size_t"                        => :Csize_t,
    "ptrdiff_t"                 => :Cptrdiff_t
    
    }

# Convert clang type to julia type
function ctype_to_julia(cutype::CXType)
    typkind = ty_kind(cutype)
    # Special cases: TYPEDEF, POINTER
    if (typkind == TypKind.POINTER)
        ptr_ctype = cindex.getPointeeType(cutype)
        ptr_jltype = ctype_to_julia(ptr_ctype)
        return Ptr{ptr_jltype}
    elseif (typkind == TypKind.TYPEDEF || typkind == TypKind.RECORD)
        basename = string( spelling( cindex.getTypeDeclaration(cutype) ) )
        return symbol( get(c_jl, basename, basename))
    else
        # TODO: missing mappings should generate a warning
        return symbol( string( get(c_jl, typkind, :Void) ))
    end
end

### eliminate symbol from the final type representation
function rep_type(t)
    replace(string(t), ":", "")
end

### Tuple representation without quotes
function rep_args(v)
    o = IOBuffer()
    print(o, "(")
    s = first = start(v)
    r = (!done(v,s))
    while r
        x,s = next(v,s)
        print(o, x)
        done(v,s) && break
        print(o, ", ")
    end
    r && (s == next(v,first)[2]) && print(o, ",")
    print(o, ")")
    seek(o, 0)
    readall(o)
end

### Build array of include definitions: ["-I", "incpath",...] plus extra args
function build_clang_args(includes, extras)
    reduce(vcat, [], vcat([["-I",x] for x in includes], extras))
end

### Retrieve function arguments for a given cursor
{'fraction_non_alphanumeric': 0.1332068311195446, 'fraction_numerical': 0.0058823529411764705, 'mean_word_length': 2.175, 'pattern_counts': {'":': 2, '<': 15, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5235430', 'n_tokens_mistral': 3200, 'n_tokens_neox': 3093, 'n_words': 739}