text:
####################################################################################################
###################################### Memory Addressing ###########################################
####################################################################################################

# Operation names and dispatch pipeline:
# `vload` and `vstore!` are the user API functions. They load from an `AbstractStridedPointer` and
# are indexed via a `::Tuple` or `Unroll{AU,F,N,AV,W,M,X,<:Tuple}`.
# These calls are forwarded to `_vload` and `_vstore!`, appending information like  `register_size()`,
# whether the operations can be assumed to be aligned, and for `vstore!` whether to add alias scope
# metadata and a nontemporal hint (non-temporal requires alignment).
#
# The tuple (Cartesian) indices are then linearized and put in terms of the number of bytes, and
# forwarded to `__vload` and `__vstore!`.
#
# The name mangling was introduced to help with discoverability, and to mayke the dispatch chain clearer.
# `methods(vload)` and `methods(vstore!)` now return much fewer methods, so users have an easier time
# assessing the API.

"""
  Unroll{AU,F,N,AV,W,M,X}(i::I)

 - AU: Unrolled axis
 - F: Factor, step size per unroll. If AU == AV, `F == W` means successive loads. `1` would mean offset by `1`, e.g. `x{1:8]`, `x[2:9]`, and `x[3:10]`.
 - N: How many times is it unrolled
 - AV: Vectorized axis # 0 means not vectorized, some sort of reduction
 - W: vector width
 - M: bitmask indicating whether each factor is masked
 - X: stride between loads of vectors along axis `AV`.
 - i::I - index
"""
struct Unroll{AU,F,N,AV,W,M,X,I}
  i::I
end
@inline Unroll{AU,F,N,AV,W}(i::I) where {AU,F,N,AV,W,I} =
  Unroll{AU,F,N,AV,W,zero(UInt),1,I}(i)
@inline Unroll{AU,F,N,AV,W,M}(i::I) where {AU,F,N,AV,W,M,I} = Unroll{AU,F,N,AV,W,M,1,I}(i)
@inline Unroll{AU,F,N,AV,W,M,X}(i::I) where {AU,F,N,AV,W,M,X,I} =
  Unroll{AU,F,N,AV,W,M,X,I}(i)
@inline data(u::Unroll) = getfield(u, :i)
const TupleIndex = Union{Tuple,Unroll{<:Any,<:Any,<:Any,<:Any,<:Any,<:Any,<:Any,<:Tuple}}
@inline function linear_index(
  ptr::AbstractStridedPointer,
  u::Unroll{AU,F,N,AV,W,M,X,I},
) where {AU,F,N,AV,W,M,X,I<:TupleIndex}
  p, i = linear_index(ptr, data(u))
  # Unroll{AU,F,N,AV,W,M,typeof(i)}(i)
  p, Unroll{AU,F,N,AV,W,M,X}(i)
end
unroll_params(::Type{Unroll{AU,F,N,AV,W,M,X,I}}) where {AU,F,N,AV,W,M,X,I} =
  (AU, F, N, AV, W, M, X, I)
const NestedUnroll{W,AV,X,I,AUO,FO,NO,MO,AUI,FI,NI,MI} =
  Unroll{AUO,FO,NO,AV,W,MO,X,Unroll{AUI,FI,NI,AV,W,MI,X,I}}

const VectorIndexCore{W} = Union{Vec{W},MM{W},Unroll{<:Any,<:Any,<:Any,<:Any,W}}
const VectorIndex{W} =
  Union{VectorIndexCore{W},LazyMulAdd{<:Any,<:Any,<:VectorIndexCore{W}}}
const IntegerIndex = Union{IntegerTypes,LazyMulAdd{<:Any,<:Any,<:IntegerTypes}}
const Index = Union{IntegerIndex,VectorIndex}

const VectorIndexNoUnrollCore{W} = Union{Vec{W},MM{W}}
const VectorIndexNoUnroll{W} =
  Union{VectorIndexNoUnrollCore{W},LazyMulAdd{<:Any,<:Any,<:VectorIndexNoUnrollCore{W}}}
const IndexNoUnroll = Union{IntegerIndex,VectorIndexNoUnroll}

# const BoolVec = Union{AbstractMask,VecUnroll{<:Any, <:Any, Bool, <: AbstractMask}}

const SCOPE_METADATA = """
!1 = !{!\"noaliasdomain\"}
!2 = !{!\"noaliasscope\", !1}
!3 = !{!2}
"""
const LOAD_SCOPE_FLAGS = ", !alias.scope !3";
const STORE_SCOPE_FLAGS = ", !noalias !3";

# use TBAA?
# const TBAA_STR = """
# !4 = !{!5, !5, i64 0}
# !5 = !{!"jtbaa_mutab", !6, i64 0}
# !6 = !{!"jtbaa_value", !7, i64 0}
# !7 = !{!"jtbaa_data", !8, i64 0}
# !8 = !{!"jtbaa", !9, i64 0}
# !9 = !{!"jtbaa"}
# """;
# const TBAA_FLAGS = ", !tbaa !4";
const TBAA_STR = TBAA_FLAGS = ""
# const TBAA_STR = """
# !4 = !{!"jtbaa", !5, i64 0}
# !5 = !{!"jtbaa"}
# !6 = !{!"jtbaa_data", !4, i64 0}
# !7 = !{!8, !8, i64 0}
# !8 = !{!"jtbaa_arraybuf", !6, i64 0}
# """;
# const TBAA_FLAGS = ", !tbaa !7";
const LOAD_SCOPE_TBAA = SCOPE_METADATA * TBAA_STR
const LOAD_SCOPE_TBAA_FLAGS = LOAD_SCOPE_FLAGS * TBAA_FLAGS

"""
An omnibus offset constructor.

The general motivation for generating the memory addresses as LLVM IR rather than combining multiple lllvmcall Julia functions is
that we want to minimize the `inttoptr` and `ptrtoint` calculations as we go back and fourth. These can get in the way of some
optimizations, such as memory address calculations.
It is particulary import for `gather` and `scatter`s, as these functions take a `Vec{W,Ptr{T}}` argument to load/store a
`Vec{W,T}` to/from. If `sizeof(T) < sizeof(Int)`, converting the `<W x \$(typ)*` vectors of pointers in LLVM to integer
vectors as they're represented in Julia will likely make them too large to fit in a single register, splitting the operation
into multiple operations, forcing a corresponding split of the `Vec{W,T}` vector as well.
This would all be avoided by not promoting/widenting the `<W x \$(typ)>` into a vector of `Int`s.

For this last issue, an alternate workaround would be to wrap a `Vec` of 32-bit integers with a type that defines it as a pointer for use with
internal llvmcall functions, but I haven't really explored this optimization.
"""
function offset_ptr(
  ::Type{T},
  ind_type::Symbol,
  indargname::Char,
  ibits::Int,
  W::Int,
  X::Int,
  M::Int,
  O::Int,
  forgep::Bool,
  rs::Int,
) where {T}
  T_sym = JULIA_TYPES[T]
  offset_ptr(T_sym, sizeof_T, ind_type, indargname, ibits, W, X, M, O, forgep, rs)
end
function offset_ptr(
  T_sym::Symbol,
  ind_type::Symbol,
  indargname::Char,
  ibits::Int,
  W::Int,
  X::Int,
  M::Int,
  O::Int,
  forgep::Bool,
  rs::Int,
)
  sizeof_T = JULIA_TYPE_SIZE[T_sym]
  i = 0
  Morig = M
  isbit = T_sym === :Bit
  if isbit
    typ = "i1"
    vtyp = isone(W) ? typ : "<$W x i1>"
    M = max(1, M >> 3)
    O >>= 3
    if !((isone(X) | iszero(X)) && (ind_type !== :Vec))
      throw(ArgumentError("indexing BitArrays with a vector not currently supported."))
    end
  else
    typ = LLVM_TYPES_SYM[T_sym]
    vtyp = vtype(W, typ) # vtyp is dest type
  end
  instrs = String[]
  if M == 0
    ind_type = :StaticInt
  elseif ind_type === :StaticInt
    M = 0
  end
  if isone(W)
    X = 1
  end
  if iszero(M)
    tz = intlog2(sizeof_T)
    tzf = sizeof_T
    index_gep_typ = typ
  else
    tz = min(trailing_zeros(M), 3)
    tzf = 1 << tz
    index_gep_typ = ((tzf == sizeof_T) | iszero(M)) ? typ : "i$(tzf << 3)"
    M >>= tz
  end
  # after this block, we will have a index_gep_typ pointer
  if iszero(O)
    push!(instrs, "%ptr.$(i) = inttoptr $(JULIAPOINTERTYPE) %0 to $(index_gep_typ)*")
    i += 1
  else # !iszero(O)
    if !iszero(O & (tzf - 1)) # then index_gep_typ works for the constant offset
      offset_gep_typ = "i8"
      offset = O
    else # then we need another intermediary
      offset_gep_typ = index_gep_typ
      offset = O >> tz
    end
    push!(instrs, "%ptr.$(i) = inttoptr $(JULIAPOINTERTYPE) %0 to $(offset_gep_typ)*")
    i += 1
    push!(
      instrs,
      "%ptr.$(i) = getelementptr inbounds $(offset_gep_typ), $(offset_gep_typ)* %ptr.$(i-1), i32 $(offset)",
    )
    i += 1
    if forgep && iszero(M) && (iszero(X) || isone(X))
      push!(
        instrs,
        "%ptr.$(i) = ptrtoint $(offset_gep_typ)* %ptr.$(i-1) to $(JULIAPOINTERTYPE)",
      )
      i += 1
      return instrs, i
    elseif offset_gep_typ != index_gep_typ
      push!(
        instrs,
        "%ptr.$(i) = bitcast $(offset_gep_typ)* %ptr.$(i-1) to $(index_gep_typ)*",
      )
      i += 1
    end
  end
  # will do final type conversion
  if ind_type === :Vec
    if isone(M)
      indname = indargname
    else
      indname = "indname"
      constmul = llvmconst(W, "i$(ibits) $M")
      push!(instrs, "%$(indname) = mul nsw <$W x i$(ibits)> %$(indargname), $(constmul)")
    end
    push!(
      instrs,
      "%ptr.$(i) = getelementptr inbounds $(index_gep_typ), $(index_gep_typ)* %ptr.$(i-1), <$W x i$(ibits)> %$(indname)",
    )
    i += 1
    if forgep
      push!(
        instrs,
        "%ptr.$(i) = ptrtoint <$W x $index_gep_typ*> %ptr.$(i-1) to <$W x $JULIAPOINTERTYPE>",
      )
      i += 1
    elseif index_gep_typ != vtyp
      push!(
        instrs,
        "%ptr.$(i) = bitcast <$W x $index_gep_typ*> %ptr.$(i-1) to <$W x $typ*>",
      )
      i += 1
    end
    return instrs, i
  end
  if ind_type === :Integer
    if isbit
      if (Morig & 7) == 0 || ispow2(Morig)
        if abs(Morig) ≥ 8
          M = Morig >> 3
          if M != 1
            indname = "indname"
            push!(instrs, "%$(indname) = mul nsw i$(ibits) %$(indargname), $M")
          else
            indname = indargname
          end
        else
          shifter = 3 - intlog2(Morig)
          push!(instrs, "%shiftedind = ashr i$(ibits) %$(indargname), $shifter")
          if Morig > 0
            indname = "shiftedind"
          else
            indname = "indname"
            push!(instrs, "%$(indname) = mul i$(ibits) %shiftedind, -1")
          end
        end
      else
        throw(
          ArgumentError(
            "Scale factors on bit accesses must be an integer multiple of 8 or an integer power of 2.",
          ),
        )
        indname = "0"
      end
    else
      if isone(M)
        indname = indargname
      elseif iszero(M)
        indname = "0"
      else
        indname = "indname"
        push!(instrs, "%$(indname) = mul nsw i$(ibits) %$(indargname), $M")
      end
      # TODO: if X != 1 and X != 0, check if it is better to gep -> gep, or broadcast -> add -> gep
    end
    push!(
      instrs,
      "%ptr.$(i) = getelementptr inbounds $(index_gep_typ), $(index_gep_typ)* %ptr.$(i-1), i$(ibits) %$(indname)",
    )
    i += 1
  end
  # ind_type === :Integer || ind_type === :StaticInt
  if !(isone(X) | iszero(X)) # vec
    # LLVM assumes integers are signed for indexing
    # therefore, we need to set the integer size to be at least `nextpow2(intlog2(X*W-1)+2)` bits
    # to avoid overflow
    vibytes = max(min(4, rs ÷ W), nextpow2(intlog2(X * W - 1) + 2) >> 3)
    vityp = "i$(8vibytes)"
    vi = join((X * w for w ∈ 0:W-1), ", $vityp ")
    if typ !== index_gep_typ
      push!(instrs, "%ptr.$(i) = bitcast $(index_gep_typ)* %ptr.$(i-1) to $(typ)*")
      i += 1
    end
    push!(
      instrs,
      "%ptr.$(i) = getelementptr inbounds $(typ), $(typ)* %ptr.$(i-1), <$W x $(vityp)> <$vityp $vi>",
    )
    i += 1
    if forgep
      push!(
        instrs,
        "%ptr.$(i) = ptrtoint <$W x $typ*> %ptr.$(i-1) to <$W x $JULIAPOINTERTYPE>",
      )
      i += 1
    end
    return instrs, i
  end
  if forgep # if forgep, just return now
    push!(instrs, "%ptr.$(i) = ptrtoint $(index_gep_typ)* %ptr.$(i-1) to $JULIAPOINTERTYPE")
    i += 1
  elseif index_gep_typ != vtyp
    push!(instrs, "%ptr.$(i) = bitcast $(index_gep_typ)* %ptr.$(i-1) to $(vtyp)*")
    i += 1
  end
  instrs, i
end

gep_returns_vector(W::Int, X::Int, M::Int, ind_type::Symbol) =
  (!isone(W) && ((ind_type === :Vec) || !(isone(X) | iszero(X))))
#::Type{T}, ::Type{I}, W::Int = 1, ivec::Bool = false, constmul::Int = 1) where {T <: NativeTypes, I <: Integer}
function gep_quote(
  ::Type{T},
  ind_type::Symbol,
  ::Type{I},
  W::Int,
  X::Int,
  M::Int,
  O::Int,
  forgep::Bool,
  rs::Int,
) where {T,I}
  T_sym = JULIA_TYPES[T]
  I_sym = JULIA_TYPES[I]
  gep_quote(T_sym, ind_type, I_sym, W, X, M, O, forgep, rs)
end
function gep_quote(
  T_sym::Symbol,
  ind_type::Symbol,
  I_sym::Symbol,
  W::Int,
  X::Int,
  M::Int,
  O::Int,
  forgep::Bool,
  rs,
)
  sizeof_T = JULIA_TYPE_SIZE[T_sym]
  sizeof_I = JULIA_TYPE_SIZE[I_sym]
  if W > 1 && ind_type !== :Vec
    X, Xr = divrem(X, sizeof_T)
    iszero(Xr) || throw(
      ArgumentError(
        "sizeof($T_sym) == $sizeof_T, but stride between vector loads is given as $X, which is not a positive integer multiple.",
      ),
    )
  end
  if iszero(O) && (iszero(X) | isone(X)) && (iszero(M) || ind_type === :StaticInt)
    return Expr(:block, Expr(:meta, :inline), :ptr)
  end
  ibits = 8sizeof_I
  # ::Type{T}, ind_type::Symbol, indargname = '1', ibytes::Int, W::Int = 1, X::Int = 1, M::Int = 1, O::Int = 0, forgep::Bool = false
  instrs, i = offset_ptr(T_sym, ind_type, '1', ibits, W, X, M, O, true, rs)
  ret = Expr(:curly, :Ptr, T_sym)
  lret = JULIAPOINTERTYPE
  if gep_returns_vector(W, X, M, ind_type)
    ret = Expr(:curly, :_Vec, W, ret)
    lret = "<$W x $lret>"
  end

  args = Expr(:curly, :Tuple, Expr(:curly, :Ptr, T_sym))
  largs = String[JULIAPOINTERTYPE]
  arg_syms = Union{Symbol,Expr}[:ptr]

  if !(iszero(M) || ind_type === :StaticInt)
    push!(arg_syms, Expr(:call, :data, :i))
    if ind_type === :Integer
      push!(args.args, I_sym)
      push!(largs, "i$(ibits)")
    else
      push!(args.args, Expr(:curly, :_Vec, W, I_sym))
      push!(largs, "<$W x i$(ibits)>")
    end
  end
  push!(instrs, "ret $lret %ptr.$(i-1)")
  llvmcall_expr("", join(instrs, "\n"), ret, args, lret, largs, arg_syms)
end

@generated function _gep(
  ptr::Ptr{T},
  i::I,
  ::StaticInt{RS},
) where {I<:IntegerTypes,T<:NativeTypes,RS}
  gep_quote(T, :Integer, I, 1, 1, 1, 0, true, RS)
end
@generated function _gep(
  ptr::Ptr{T},
  ::StaticInt{N},
  ::StaticInt{RS},
) where {N,T<:NativeTypes,RS}
  gep_quote(T, :StaticInt, Int, 1, 1, 0, N, true, RS)
end
@generated function _gep(
  ptr::Ptr{T},
  i::LazyMulAdd{M,O,I},
  ::StaticInt{RS},
) where {T<:NativeTypes,I<:IntegerTypes,O,M,RS}
  gep_quote(T, :Integer, I, 1, 1, M, O, true, RS)
end
@inline _gep(ptr, i::IntegerIndex, ::StaticInt) = ptr + _materialize(i)
@generated function _gep(
  ptr::Ptr{T},
  i::Vec{W,I},
  ::StaticInt{RS},
) where {W,T<:NativeTypes,I<:IntegerTypes,RS}
  gep_quote(T, :Vec, I, W, 1, 1, 0, true, RS)
end
@generated function _gep(
  ptr::Ptr{T},
  i::LazyMulAdd{M,O,Vec{W,I}},
  ::StaticInt{RS},
) where {W,T<:NativeTypes,I<:IntegerTypes,M,O,RS}
  gep_quote(T, :Vec, I, W, 1, M, O, true, RS)
end
@inline gep(ptr::Ptr, i) = _gep(ptr, i, register_size())

@inline increment_ptr(ptr::AbstractStridedPointer) = pointer(ptr)
@inline function increment_ptr(ptr::AbstractStridedPointer, i::Tuple)
  ioffset = _offset_index(i, offsets(ptr))
  p, li = tdot(ptr, ioffset, strides(ptr))
  _gep(p, li, Zero())
end
# @inline function increment_ptr(ptr::AbstractStridedPointer{T,N,C,B,R,X,NTuple{N,Zero}}, i::Tuple) where {T,N,C,B,R,X}
#   p, li = tdot(ptr, i, strides(ptr))
#   _gep(p, li, Zero())
# end
@inline increment_ptr(p::StridedBitPointer) = offsets(p)
@inline bmap(::F, ::Tuple{}, y::Tuple{}) where {F} = ()
@inline bmap(::F, ::Tuple{}, y::Tuple) where {F} = ()
@inline bmap(::F, x::Tuple{X,Vararg{Any,K}}, ::Tuple{}) where {F,X,K} = x
#  (first(x), bmap(f, Base.tail(x), ())...)
@inline bmap(
  f::F,
  x::Tuple{X,Vararg{Any,KX}},
  y::Tuple{Y,Vararg{Any,KY}},
) where {F,X,Y,KX,KY} = (f(first(x), first(y)), bmap(f, Base.tail(x), Base.tail(y))...)
@inline increment_ptr(p::StridedBitPointer, i::Tuple) = bmap(vsub_nsw, offsets(p), i)
@inline increment_ptr(p::AbstractStridedPointer, o, i::Tuple) =
  increment_ptr(reconstruct_ptr(p, o), i)
@inline function reconstruct_ptr(sp::AbstractStridedPointer, p::Ptr)
  similar_with_offset(sp, p, offsets(sp))
end
# @inline function reconstruct_ptr(sp::AbstractStridedPointer, p::Ptr)
#   similar_no_offset(sp, p)
# end
@inline function reconstruct_ptr(
  ptr::StridedBitPointer{N,C,B,R},
  offs::NTuple{N,Int},
) where {N,C,B,R}
  stridedpointer(
    pointer(ptr),
    ArrayInterface.StrideIndex{N,R,C}(strides(ptr), offs),
    StaticInt{B}(),
  )
end

@inline function gesp(ptr::AbstractStridedPointer, i::Tuple{Vararg{IntegerIndex}})
  similar_no_offset(ptr, increment_ptr(ptr, i))
end
@inline function gesp(
  ptr::StridedBitPointer{N,C,B,R},
  i::Tuple{Vararg{IntegerIndex,K}},
) where {N,C,B,R,K}
  stridedpointer(
    pointer(ptr),
    ArrayInterface.StrideIndex{N,R,C}(strides(ptr), increment_ptr(ptr, i)),
    StaticInt{B}(),
  )
end
@inline vsub_nsw(::NullStep, _) = Zero()
@inline vsub_nsw(::NullStep, ::LazyMulAdd) = Zero() # avoid ambiguity
@inline vsub_nsw(::NullStep, ::NullStep) = Zero()
@inline vsub_nsw(::NullStep, ::StaticInt) = Zero()

@inline select_null_offset(i::Tuple{}, off::Tuple{}) = ()
@inline select_null_offset(i::Tuple{I1,Vararg}, off::Tuple{O1}) where {I1,O1} = (Zero(),)
@inline select_null_offset(i::Tuple{NullStep,Vararg}, off::Tuple{O1}) where {O1} =
  (first(off),)
@inline select_null_offset(
  i::Tuple{I1,I2,Vararg},
  off::Tuple{O1,O2,Vararg},
) where {I1,I2,O1,O2} = (Zero(), select_null_offset(Base.tail(i), Base.tail(off))...)
@inline select_null_offset(
  i::Tuple{NullStep,I2,Vararg},
  off::Tuple{O1,O2,Vararg},
) where {I2,O1,O2} = (first(off), select_null_offset(Base.tail(i), Base.tail(off))...)

@inline function gesp(
  ptr::AbstractStridedPointer,
  i::Tuple{Vararg{Union{NullStep,IntegerIndex}}},
)
  ioffset = _offset_index(i, offsets(ptr))
  offs = select_null_offset(i, offsets(ptr))
  similar_with_offset(ptr, gep(zero_offsets(ptr), ioffset), offs)
end
@inline gesp(ptr::AbstractStridedPointer, i::Tuple{NullStep,Vararg{NullStep,N}}) where {N} =
  ptr
@inline gesp(ptr::AbstractStridedPointer, i::Tuple{Vararg{Any,N}}) where {N} =
  gesp(ptr, Tuple(CartesianVIndex(i)))#flatten

function vload_quote(
  ::Type{T},
  ::Type{I},
  ind_type::Symbol,
  W::Int,
  X::Int,
  M::Int,
  O::Int,
  mask::Bool,
  align::Bool,
  rs::Int,
) where {T<:NativeTypes,I<:Integer}
  T_sym = JULIA_TYPES[T]
  I_sym = JULIA_TYPES[I]
  vload_quote(T_sym, I_sym, ind_type, W, X, M, O, mask, align, rs)
end
function vload_quote(
  T_sym::Symbol,
  I_sym::Symbol,
  ind_type::Symbol,
  W::Int,
  X::Int,
  M::Int,
  O::Int,
  mask::Bool,
  align::Bool,
  rs::Int,
)
  isbit = T_sym === :Bit
  if !isbit && W > 1
    sizeof_T = JULIA_TYPE_SIZE[T_sym]
    if W * sizeof_T > rs
      if ((T_sym === :Int64) | (T_sym === :UInt64)) && ((W * sizeof_T) == 2rs)
        return vload_trunc_quote(
          T_sym,
          I_sym,
          ind_type,
          W,
          X,
          M,
          O,
          mask,
          align,
          rs,
          :(_Vec{$W,$T_sym}),
        )
        # else
        #   return vload_split_quote(W, sizeof_T, mask, align, rs, T_sym)
      end
    else
      return vload_quote(
        T_sym,
        I_sym,
        ind_type,
        W,
        X,
        M,
        O,
        mask,
        align,
        rs,
        :(_Vec{$W,$T_sym}),
      )
    end
  end
  jtyp = isbit ? (isone(W) ? :Bool : mask_type_symbol(W)) : T_sym
  vload_quote(T_sym, I_sym, ind_type, W, X, M, O, mask, align, rs, jtyp)
  # jtyp_expr = Expr(:(.), :Base, QuoteNode(jtyp)) # reduce latency, hopefully
  # vload_quote(T_sym, I_sym, ind_type, W, X, M, O, mask, align, rs, jtyp_expr)
end
function vload_trunc_quote(
  T_sym::Symbol,
  I_sym::Symbol,
  ind_type::Symbol,
  W::Int,
  X::Int,
  M::Int,
  O::Int,
  mask::Bool,
  align::Bool,
  rs::Int,
  ret::Union{Symbol,Expr},
)
  call = vload_quote_llvmcall(T_sym, I_sym, ind_type, W, X, M, O, mask, align, rs, ret)
  call = Expr(:call, :%, call, Core.ifelse(T_sym === :Int64, :Int32, :UInt32))
  Expr(:block, Expr(:meta, :inline), call)
end
# function vload_split_quote(W::Int, sizeof_T::Int, mask::Bool, align::Bool, rs::Int, T_sym::Symbol)
#     D, r1 = divrem(W * sizeof_T, rs)
#     Wnew, r2 = divrem(W, D)
#     (iszero(r1) & iszero(r2)) || throw(ArgumentError("If loading more than a vector, Must load a multiple of the vector width."))
#     q = Expr(:block,Expr(:meta,:inline))
#     # ind_type = :StaticInt, :Integer, :Vec
#     push!(q.args, :(isplit = splitvectortotuple(StaticInt{$D}(), StaticInt{$Wnew}(), i)))
#     mask && push!(q.args, :(msplit = splitvectortotuple(StaticInt{$D}(), StaticInt{$Wnew}(), m)))
#     t = Expr(:tuple)
#     alignval = Expr(:call, align ? :True : :False)
#     for d ∈ 1:D
#         call = Expr(:call, :__vload, :ptr)
#         push!(call.args, Expr(:ref, :isplit, d))
#         mask && push!(call.args, Expr(:ref, :msplit, d))
#         push!(call.args, alignval, Expr(:call, Expr(:curly, :StaticInt, rs)))
#         v_d = Symbol(:v_, d)
#         push!(q.args, Expr(:(=), v_d, call))
#         push!(t.args, v_d)
#     end
#     push!(q.args, :(VecUnroll($t)::VecUnroll{$(D-1),$Wnew,$T_sym,Vec{$Wnew,$T_sym}}))
#     q
# end

@inline function _mask_scalar_load(
  ptr::Ptr{T},
  i::IntegerIndex,
  m::AbstractMask{1},
  ::A,
  ::StaticInt{RS},
) where {T,A,RS}
  Bool(m) ? __vload(ptr, i, A(), StaticInt{RS}()) : zero(T)
end
@inline function _mask_scalar_load(
  ptr::Ptr{T},
  m::AbstractMask{1},
  ::A,
  ::StaticInt{RS},
) where {T,A,RS}
  Bool(m) ? __vload(ptr, i, A(), StaticInt{RS}()) : zero(T)
end
@inline function _mask_scalar_load(
  ptr::Ptr{T},
  i::IntegerIndex,
  m::AbstractMask{W},
  ::A,
  ::StaticInt{RS},
) where {T,A,RS,W}
  s = __vload(ptr, i, A(), StaticInt{RS}())
  ifelse(
    m,
    _vbroadcast(StaticInt{W}(), s, StaticInt{RS}()),
    _vzero(StaticInt{W}(), T, StaticInt{RS}()),
  )
end
@inline function _mask_scalar_load(
  ptr::Ptr{T},
  m::AbstractMask{W},
  ::A,
  ::StaticInt{RS},
) where {T,A,RS,W}
  s = __vload(ptr, A(), StaticInt{RS}())
  ifelse(
    m,
    _vbroadcast(StaticInt{W}(), s, StaticInt{RS}()),
    _vzero(StaticInt{W}(), T, StaticInt{RS}()),
  )
end

function vload_quote_llvmcall(
  T_sym::Symbol,
  I_sym::Symbol,
  ind_type::Symbol,
  W::Int,
  X::Int,
  M::Int,
  O::Int,
  mask::Bool,
  align::Bool,
  rs::Int,
  ret::Union{Symbol,Expr},
)
  if mask && W == 1
    if M == O == 0
      return quote
        $(Expr(:meta, :inline))
        _mask_scalar_load(ptr, m, $(align ? :True : :False)(), StaticInt{$rs}())
      end
    else
      return quote
        $(Expr(:meta, :inline))
        _mask_scalar_load(ptr, i, m, $(align ? :True : :False)(), StaticInt{$rs}())
      end
    end
  end

  decl, instrs, args, lret, largs, arg_syms =
    vload_quote_llvmcall_core(T_sym, I_sym, ind_type, W, X, M, O, mask, align, rs)

  return llvmcall_expr(decl, instrs, ret, args, lret, largs, arg_syms, true)
end
function vload_quote_llvmcall_core(
  T_sym::Symbol,
  I_sym::Symbol,
  ind_type::Symbol,
  W::Int,
  X::Int,
  M::Int,
  O::Int,
  mask::Bool,
  align::Bool,
  rs::Int,
)
  sizeof_T = JULIA_TYPE_SIZE[T_sym]

  reverse_load = ((W > 1) & (X == -sizeof_T)) & (ind_type !== :Vec)
  if reverse_load
    X = sizeof_T
    O -= (W - 1) * sizeof_T
  end
  # if (X == -sizeof_T) & (!mask)
  #     return quote
  #         $(Expr(:meta,:inline))
  #         vload(ptr, i
  #     end
  # end
  sizeof_I = JULIA_TYPE_SIZE[I_sym]
  ibits = 8sizeof_I
  if W > 1 && ind_type !== :Vec
    X, Xr = divrem(X, sizeof_T)
    iszero(Xr) || throw(
      ArgumentError(
        "sizeof($T_sym) == $sizeof_T, but stride between vector loads is given as $X, which is not a positive integer multiple.",
      ),
    )
  end
  instrs, i = offset_ptr(T_sym, ind_type, '1', ibits, W, X, M, O, false, rs)
  grv = gep_returns_vector(W, X, M, ind_type)
  # considers booleans to only occupy 1 bit in memory, so they must be handled specially
  isbit = T_sym === :Bit
  if isbit
    # @assert !grv "gather's not are supported with `BitArray`s."
    mask = false # TODO: not this?
    # typ = "i$W"
    alignment = (align & (!grv)) ? cld(W, 8) : 1
    typ = "i1"
  else
    alignment = (align & (!grv)) ? _get_alignment(W, T_sym) : _get_alignment(0, T_sym)
    typ = LLVM_TYPES_SYM[T_sym]
  end

  decl = LOAD_SCOPE_TBAA
  dynamic_index = !(iszero(M) || ind_type === :StaticInt)

  vtyp = vtype(W, typ)
  if mask
    if reverse_load
      decl *= truncate_mask!(instrs, '1' + dynamic_index, W, 0, true) * "\n"
    else
      truncate_mask!(instrs, '1' + dynamic_index, W, 0, false)
    end
  end
  if grv
    loadinstr = "$vtyp @llvm.masked.gather." * suffix(W, T_sym) * '.' * ptr_suffix(W, T_sym)
    decl *= "declare $loadinstr(<$W x $typ*>, i32, <$W x i1>, $vtyp)"
    m = mask ? m = "%mask.0" : llvmconst(W, "i1 1")
    passthrough = mask ? "zeroinitializer" : "undef"
    push!(
      instrs,
      "%res = call $loadinstr(<$W x $typ*> %ptr.$(i-1), i32 $alignment, <$W x i1> $m, $vtyp $passthrough)" *
      LOAD_SCOPE_TBAA_FLAGS,
    )
  elseif mask
    suff = suffix(W, T_sym)
    loadinstr = "$vtyp @llvm.masked.load." * suff * ".p0" * suff
    decl *= "declare $loadinstr($vtyp*, i32, <$W x i1>, $vtyp)"
    push!(
      instrs,
      "%res = call $loadinstr($vtyp* %ptr.$(i-1), i32 $alignment, <$W x i1> %mask.0, $vtyp zeroinitializer)" *
      LOAD_SCOPE_TBAA_FLAGS,
    )
  else
    push!(
      instrs,
      "%res = load $vtyp, $vtyp* %ptr.$(i-1), align $alignment" * LOAD_SCOPE_TBAA_FLAGS,
    )
  end
  if isbit
    lret = string('i', max(8, nextpow2(W)))
    if W > 1
      if reverse_load
        # isbit means mask is set to false, so we definitely need to declare `bitreverse`
        bitreverse = "i$(W) @llvm.bitreverse.i$(W)(i$(W))"
        decl *= "declare $bitreverse"
        resbit = "resbitreverse"
        push!(instrs, "%$(resbit) = call $bitreverse(i$(W) %res")
      else
        resbit = "res"
      end
      if W < 8
        push!(instrs, "%resint = bitcast <$W x i1> %$(resbit) to i$(W)")
        push!(instrs, "%resfinal = zext i$(W) %resint to i8")
      elseif ispow2(W)
        push!(instrs, "%resfinal = bitcast <$W x i1> %$(resbit) to i$(W)")
      else
        Wpow2 = nextpow2(W)
        push!(instrs, "%resint = bitcast <$W x i1> %$(resbit) to i$(W)")
        push!(instrs, "%resfinal = zext i$(W) %resint to i$(Wpow2)")
      end
    else
      push!(instrs, "%resfinal = zext i1 %res to i8")
    end
    push!(instrs, "ret $lret %resfinal")
  else
    lret = vtyp
    if reverse_load
      reversemask = '<' * join(map(x -> string("i32 ", W - x), 1:W), ", ") * '>'
      push!(
        instrs,
        "%resreversed = shufflevector $vtyp %res, $vtyp undef, <$W x i32> $reversemask",
      )
      push!(instrs, "ret $vtyp %resreversed")
    else
      push!(instrs, "ret $vtyp %res")
    end
  end
  args = Expr(:curly, :Tuple, Expr(:curly, :Ptr, T_sym))
  largs = String[JULIAPOINTERTYPE]
  arg_syms = Union{Symbol,Expr}[:ptr]
  if dynamic_index
    push!(arg_syms, :(data(i)))
    if ind_type === :Integer
      push!(args.args, I_sym)
      push!(largs, "i$(ibits)")
    else
      push!(args.args, :(_Vec{$W,$I_sym}))
      push!(largs, "<$W x i$(ibits)>")
    end
  end
  if mask
    push!(arg_syms, :(data(m)))
    push!(args.args, mask_type(nextpow2(W)))
    push!(largs, "i$(max(8,nextpow2(W)))")
  end
  return decl, join(instrs, "\n"), args, lret, largs, arg_syms
end
function vload_quote(
  T_sym::Symbol,
  I_sym::Symbol,
  ind_type::Symbol,
  W::Int,
  X::Int,
  M::Int,
  O::Int,
  mask::Bool,
  align::Bool,
  rs::Int,
  ret::Union{Symbol,Expr},
)
  call = vload_quote_llvmcall(T_sym, I_sym, ind_type, W, X, M, O, mask, align, rs, ret)
  if (W > 1) & (T_sym === :Bit)
    call = Expr(:call, Expr(:curly, :Mask, W), call)
  end
  Expr(:block, Expr(:meta, :inline), call)
end

# vload_quote(T, ::Type{I}, ind_type::Symbol, W::Int, X, M, O, mask, align = false)

# ::Type{T}, ::Type{I}, ind_type::Symbol, W::Int, X::Int, M::Int, O::Int, mask::Bool, align::Bool, rs::Int

function index_summary(::Type{StaticInt{N}}) where {N}
  #I,    ind_type, W, X, M, O
  Int, :StaticInt, 1, 1, 0, N
end
function index_summary(::Type{I}) where {I<:IntegerTypesHW}
  #I, ind_type, W, X, M, O
  I, :Integer, 1, 1, 1, 0
end
function index_summary(::Type{Vec{W,I}}) where {W,I<:IntegerTypes}
  #I, ind_type, W, X, M, O
  I, :Vec, W, 1, 1, 0
end
function index_summary(::Type{MM{W,X,I}}) where {W,X,I<:IntegerTypes}
  #I, ind_type, W,  X, M, O
  IT, ind_type, _, __, M, O = index_summary(I)
  # inherit from parent, replace `W` and `X`
  IT, ind_type, W, X, M, O
end
function index_summary(::Type{LazyMulAdd{LMAM,LMAO,LMAI}}) where {LMAM,LMAO,LMAI}
  I, ind_type, W, X, M, O = index_summary(LMAI)
  I, ind_type, W, X * LMAM, M * LMAM, LMAO + O * LMAM
end

# no index, no mask
@generated function __vload(
  ptr::Ptr{T},
  ::A,
  ::StaticInt{RS},
) where {T<:NativeTypes,A<:StaticBool,RS}
  vload_quote(T, Int, :StaticInt, 1, 1, 0, 0, false, A === True, RS)
end
# no index, mask
@generated function __vload(
  ptr::Ptr{T},
  ::A,
  m::AbstractMask,
  ::StaticInt{RS},
) where {T<:NativeTypesExceptFloat16,A<:StaticBool,RS}
  vload_quote(T, Int, :StaticInt, 1, 1, 0, 0, true, A === True, RS)
end
# index, no mask
@generated function __vload(
  ptr::Ptr{T},
  i::I,
  ::A,
  ::StaticInt{RS},
) where {A<:StaticBool,T<:NativeTypes,I<:Index,RS}
  IT, ind_type, W, X, M, O = index_summary(I)
  vload_quote(T, IT, ind_type, W, X, M, O, false, A === True, RS)
end
# index, mask
@generated function __vload(
  ptr::Ptr{T},
  i::I,
  m::AbstractMask,
  ::A,
  ::StaticInt{RS},
) where {A<:StaticBool,T<:NativeTypesExceptFloat16,I<:Index,RS}
  IT, ind_type, W, X, M, O = index_summary(I)
  vload_quote(T, IT, ind_type, W, X, M, O, true, A === True, RS)
end

# Float16 with mask
@inline function __vload(
  ptr::Ptr{Float16},
  ::A,
  m::AbstractMask,
  ::StaticInt{RS},
) where {A<:StaticBool,RS}
  reinterpret(Float16, __vload(reinterpret(Ptr{Int16}, ptr), A(), m, StaticInt{RS}()))
end
@inline function __vload(
  ptr::Ptr{Float16},
  i::I,
  m::AbstractMask,
  ::A,
  ::StaticInt{RS},
) where {A<:StaticBool,I<:Index,RS}
  reinterpret(Float16, __vload(reinterpret(Ptr{Int16}, ptr), i, m, A(), StaticInt{RS}()))
end

@inline function _vload_scalar(
  ptr::Ptr{Bit},
  i::Union{Integer,StaticInt},
  ::A,
  ::StaticInt{RS},
) where {RS,A<:StaticBool}
  d = i >> 3
  r = i & 7
  u = __vload(Base.unsafe_convert(Ptr{UInt8}, ptr), d, A(), StaticInt{RS}())
  (u >> r) % Bool
end
@inline function __vload(
  ptr::Ptr{Bit},
  i::IntegerTypesHW,
  ::A,
  ::StaticInt{RS},
) where {A<:StaticBool,RS}
  _vload_scalar(ptr, i, A(), StaticInt{RS}())
end
# avoid ambiguities
@inline __vload(
  ptr::Ptr{Bit},
  ::StaticInt{N},
  ::A,
  ::StaticInt{RS},
) where {N,A<:StaticBool,RS} = _vload_scalar(ptr, StaticInt{N}(), A(), StaticInt{RS}())

# Entry points, `vload` and `vloada`
# No index, so forward straight to `__vload`
@inline vload(ptr::AbstractStridedPointer) = __vload(ptr, False(), register_size())
@inline vloada(ptr::AbstractStridedPointer) = __vload(ptr, True(), register_size())
# Index, so forward to `_vload` to linearize.
@inline vload(ptr::AbstractStridedPointer, i::Union{Tuple,Unroll}) =
  _vload(ptr, i, False(), register_size())
@inline vloada(ptr::AbstractStridedPointer, i::Union{Tuple,Unroll}) =
  _vload(ptr, i, True(), register_size())
@inline vload(
  ptr::AbstractStridedPointer,
  i::Union{Tuple,Unroll},
  m::Union{AbstractMask,Bool,VecUnroll},
) = _vload(ptr, i, m, False(), register_size())
@inline vloada(
  ptr::AbstractStridedPointer,
  i::Union{Tuple,Unroll},
  m::Union{AbstractMask,Bool,VecUnroll},
) = _vload(ptr, i, m, True(), register_size())

@inline function __vload(
  ptr::Ptr{T},
  i::Number,
  b::Bool,
  ::A,
  ::StaticInt{RS},
) where {T,A<:StaticBool,RS}
  b ? __vload(ptr, i, A(), StaticInt{RS}()) : zero(T)
end
@inline vwidth_from_ind(i::Tuple) = vwidth_from_ind(i, StaticInt(1), StaticInt(0))
@inline vwidth_from_ind(i::Tuple{}, ::StaticInt{W}, ::StaticInt{U}) where {W,U} =
  (StaticInt{W}(), StaticInt{U}())
@inline function vwidth_from_ind(
  i::Tuple{<:AbstractSIMDVector{W},Vararg},
  ::Union{StaticInt{1},StaticInt{W}},
  ::StaticInt{U},
) where {W,U}
  vwidth_from_ind(Base.tail(i), StaticInt{W}(), StaticInt{W}(U))
end
@inline function vwidth_from_ind(
  i::Tuple{<:VecUnroll{U,W},Vararg},
  ::Union{StaticInt{1},StaticInt{W}},
  ::Union{StaticInt{0},StaticInt{U}},
) where {U,W}
  vwidth_from_ind(Base.tail(i), StaticInt{W}(), StaticInt{W}(U))
end

@inline function _vload(
  ptr::Ptr{T},
  i::Tuple,
  b::Bool,
  ::A,
  ::StaticInt{RS},
) where {T,A<:StaticBool,RS}
  if b
    _vload(ptr, i, A(), StaticInt{RS}())
  else
    zero_init(T, vwidth_from_ind(i), StaticInt{RS}())
  end
end
@inline function _vload(
  ptr::Ptr{T},
  i::Unroll{AU,F,N,AV,W,M,X,I},
  b::Bool,
  ::A,
  ::StaticInt{RS},
) where {T,AU,F,N,AV,W,M,X,I,A<:StaticBool,RS}
  m = max_mask(Val{W}()) & b
  _vload(ptr, i, m, A(), StaticInt{RS}())
end
@inline function __vload(
  ptr::Ptr{T},
  i::I,
  m::Bool,
  ::A,
  ::StaticInt{RS},
) where {T,A<:StaticBool,RS,I<:IntegerIndex}
  m ? __vload(ptr, i, A(), StaticInt{RS}()) : zero(T)
end
@inline function __vload(
  ptr::Ptr{T},
  i::I,
  m::Bool,
  ::A,
  ::StaticInt{RS},
) where {T,A<:StaticBool,RS,W,I<:VectorIndex{W}}
  _m = max_mask(Val{W}()) & m
  __vload(ptr, i, _m, A(), StaticInt{RS}())
end

function vstore_quote(
  ::Type{T},
  ::Type{I},
  ind_type::Symbol,
  W::Int,
  X::Int,
  M::Int,
  O::Int,
  mask::Bool,
  align::Bool,
  noalias::Bool,
  nontemporal::Bool,
  rs::Int,
) where {T<:NativeTypes,I<:Integer}
  T_sym = JULIA_TYPES[T]
  I_sym = JULIA_TYPES[I]
  vstore_quote(T_sym, I_sym, ind_type, W, X, M, O, mask, align, noalias, nontemporal, rs)
end
function vstore_quote(
  T_sym::Symbol,
  I_sym::Symbol,
  ind_type::Symbol,
  W::Int,
  X::Int,
  M::Int,
  O::Int,
  mask::Bool,
  align::Bool,
  noalias::Bool,
  nontemporal::Bool,
  rs::Int,
)
  sizeof_T = JULIA_TYPE_SIZE[T_sym]
  sizeof_I = JULIA_TYPE_SIZE[I_sym]
  ibits = 8sizeof_I

  reverse_store = (((W > 1) & (X == -sizeof_T)) & (ind_type !== :Vec)) & (T_sym ≢ :Bit) # TODO: check if `T_sym` can actually be `Bit`; don't we cast?
  if reverse_store
    X = sizeof_T
    O -= (W - 1) * sizeof_T
  end

  if W > 1 && ind_type !== :Vec
    X, Xr = divrem(X, sizeof_T)
    iszero(Xr) || throw(
      ArgumentError(
        "sizeof($T_sym) == $sizeof_T, but stride between vector loads is given as $X, which is not a positive integer multiple.",
      ),
    )
  end
  instrs, i = offset_ptr(T_sym, ind_type, '2', ibits, W, X, M, O, false, rs)

  grv = gep_returns_vector(W, X, M, ind_type)
  align != nontemporal # should I do this?
  alignment = (align & (!grv)) ? _get_alignment(W, T_sym) : _get_alignment(0, T_sym)

  decl = noalias ? SCOPE_METADATA * TBAA_STR : TBAA_STR
  metadata = noalias ? STORE_SCOPE_FLAGS * TBAA_FLAGS : TBAA_FLAGS
  dynamic_index = !(iszero(M) || ind_type === :StaticInt)

  typ = LLVM_TYPES_SYM[T_sym]
  lret = vtyp = vtype(W, typ)
  if mask
    if reverse_store
      decl *= truncate_mask!(instrs, '2' + dynamic_index, W, 0, true) * "\n"
    else
      truncate_mask!(instrs, '2' + dynamic_index, W, 0, false)
    end
  end
  if reverse_store
    reversemask = '<' * join(map(x -> string("i32 ", W - x), 1:W), ", ") * '>'
    push!(
      instrs,
      "%argreversed = shufflevector $vtyp %1, $vtyp undef, <$W x i32> $reversemask",
    )
    argtostore = "%argreversed"
  else
    argtostore = "%1"
  end
  if grv
    storeinstr =
      "void @llvm.masked.scatter." * suffix(W, T_sym) * '.' * ptr_suffix(W, T_sym)
    decl *= "declare $storeinstr($vtyp, <$W x $typ*>, i32, <$W x i1>)"
    m = mask ? m = "%mask.0" : llvmconst(W, "i1 1")
    push!(
      instrs,
      "call $storeinstr($vtyp $(argtostore), <$W x $typ*> %ptr.$(i-1), i32 $alignment, <$W x i1> $m)" *
      metadata,
    )
    # push!(instrs, "call $storeinstr($vtyp $(argtostore), <$W x $typ*> %ptr.$(i-1), i32 $alignment, <$W x i1> $m)")
  elseif mask
    suff = suffix(W, T_sym)
    storeinstr = "void @llvm.masked.store." * suff * ".p0" * suff
    decl *= "declare $storeinstr($vtyp, $vtyp*, i32, <$W x i1>)"
    push!(
      instrs,
      "call $storeinstr($vtyp $(argtostore), $vtyp* %ptr.$(i-1), i32 $alignment, <$W x i1> %mask.0)" *
      metadata,
    )
  elseif nontemporal
    push!(
      instrs,
      "store $vtyp $(argtostore), $vtyp* %ptr.$(i-1), align $alignment, !nontemporal !{i32 1}" *
      metadata,
    )
  else
    push!(
      instrs,
      "store $vtyp $(argtostore), $vtyp* %ptr.$(i-1), align $alignment" * metadata,
    )
  end
  push!(instrs, "ret void")
  ret = :Cvoid
  lret = "void"
  ptrtyp = Expr(:curly, :Ptr, T_sym)
  args = if W > 1
    Expr(:curly, :Tuple, ptrtyp, Expr(:curly, :NTuple, W, Expr(:curly, :VecElement, T_sym)))
  else
    Expr(:curly, :Tuple, ptrtyp, T_sym)
  end
  largs = String[JULIAPOINTERTYPE, vtyp]
  arg_syms = Union{Symbol,Expr}[:ptr, Expr(:call, :data, :v)]
  if dynamic_index
    push!(arg_syms, :(data(i)))
    if ind_type === :Integer
      push!(args.args, I_sym)
      push!(largs, "i$(ibits)")
    else
      push!(args.args, :(_Vec{$W,$I_sym}))
      push!(largs, "<$W x i$(ibits)>")
    end
  end
  if mask
    push!(arg_syms, :(data(m)))
    push!(args.args, mask_type(W))
    push!(largs, "i$(max(8,nextpow2(W)))")
  end
  llvmcall_expr(decl, join(instrs, "\n"), ret, args, lret, largs, arg_syms)
end

# no index, no mask, scalar store
@generated function __vstore!(
  ptr::Ptr{T},
  v::VT,
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {
  T<:NativeTypesExceptBit,
  VT<:NativeTypes,
  A<:StaticBool,
  S<:StaticBool,
  NT<:StaticBool,
  RS,
}
  if VT !== T
    return Expr(
      :block,
      Expr(:meta, :inline),
      :(__vstore!(ptr, convert($T, v), $A(), $S(), $NT(), StaticInt{$RS}())),
    )
  end
  vstore_quote(
    T,
    Int,
    :StaticInt,
    1,
    1,
    0,
    0,
    false,
    A === True,
    S === True,
    NT === True,
    RS,
  )
end
# no index, no mask, vector store
@generated function __vstore!(
  ptr::Ptr{T},
  v::V,
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {
  T<:NativeTypesExceptBit,
  W,
  VT<:NativeTypes,
  V<:AbstractSIMDVector{W,VT},
  A<:StaticBool,
  S<:StaticBool,
  NT<:StaticBool,
  RS,
}
  if V !== Vec{W,T}
    return Expr(
      :block,
      Expr(:meta, :inline),
      :(__vstore!(ptr, convert(Vec{$W,$T}, v), $A(), $S(), $NT(), StaticInt{$RS}())),
    )
  end
  vstore_quote(
    T,
    Int,
    :StaticInt,
    W,
    sizeof(T),
    0,
    0,
    false,
    A === True,
    S === True,
    NT === True,
    RS,
  )
end
# index, no mask, scalar store
@generated function __vstore!(
  ptr::Ptr{T},
  v::VT,
  i::I,
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {
  T<:NativeTypesExceptBit,
  VT<:NativeTypes,
  I<:Index,
  A<:StaticBool,
  S<:StaticBool,
  NT<:StaticBool,
  RS,
}
  IT, ind_type, W, X, M, O = index_summary(I)
  if VT !== T || W > 1
    if W > 1
      return Expr(
        :block,
        Expr(:meta, :inline),
        :(__vstore!(ptr, convert(Vec{$W,$T}, v), i, $A(), $S(), $NT(), StaticInt{$RS}())),
      )
    else
      return Expr(
        :block,
        Expr(:meta, :inline),
        :(__vstore!(ptr, convert($T, v), i, $A(), $S(), $NT(), StaticInt{$RS}())),
      )
    end
  end
  vstore_quote(T, IT, ind_type, W, X, M, O, false, A === True, S === True, NT === True, RS)
end
# index, no mask, vector store
@generated function __vstore!(
  ptr::Ptr{T},
  v::V,
  i::I,
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {
  T<:NativeTypesExceptBit,
  W,
  VT<:NativeTypes,
  V<:AbstractSIMDVector{W,VT},
  I<:Index,
  A<:StaticBool,
  S<:StaticBool,
  NT<:StaticBool,
  RS,
}
  if V !== Vec{W,T}
    return Expr(
      :block,
      Expr(:meta, :inline),
      :(__vstore!(ptr, convert(Vec{$W,$T}, v), i, $A(), $S(), $NT(), StaticInt{$RS}())),
    )
  end
  IT, ind_type, _W, X, M, O = index_summary(I)
  # don't want to require vector indices...
  (W == _W || _W == 1) || throw(
    ArgumentError(
      "Vector width: $W, index width: $(_W). They must either be equal, or index width == 1.",
    ),
  )
  if (W != _W) & (_W == 1)
    X *= sizeof(T)
  end
  vstore_quote(T, IT, ind_type, W, X, M, O, false, A === True, S === True, NT === True, RS)
end
# index, mask, scalar store
@generated function __vstore!(
  ptr::Ptr{T},
  v::VT,
  i::I,
  m::AbstractMask{W},
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {
  W,
  T<:NativeTypesExceptBit,
  VT<:NativeTypes,
  I<:Index,
  A<:StaticBool,
  S<:StaticBool,
  NT<:StaticBool,
  RS,
}
  IT, ind_type, _W, X, M, O = index_summary(I)
  (W == _W || _W == 1) || throw(
    ArgumentError(
      "Vector width: $W, index width: $(_W). They must either be equal, or index width == 1.",
    ),
  )
  if W == 1
    return Expr(
      :block,
      Expr(:meta, :inline),
      :(
        Bool(m) &&
        __vstore!(ptr, convert($T, v), data(i), $A(), $S(), $NT(), StaticInt{$RS}())
      ),
    )
  else
    return Expr(
      :block,
      Expr(:meta, :inline),
      :(__vstore!(ptr, convert(Vec{$W,$T}, v), i, m, $A(), $S(), $NT(), StaticInt{$RS}())),
    )
  end
  # vstore_quote(T, IT, ind_type, W, X, M, O, true, A===True, S===True, NT===True, RS)
end
# no index, mask, vector store
@generated function __vstore!(
  ptr::Ptr{T},
  v::V,
  m::AbstractMask{W},
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {
  T<:NativeTypesExceptBit,
  W,
  VT<:NativeTypes,
  V<:AbstractSIMDVector{W,VT},
  A<:StaticBool,
  S<:StaticBool,
  NT<:StaticBool,
  RS,
}
  if W == 1
    return Expr(
      :block,
      Expr(:meta, :inline),
      :(
        Bool(m) &&
        __vstore!(ptr, convert($T, v), data(i), $A(), $S(), $NT(), StaticInt{$RS}())
      ),
    )
  elseif V !== Vec{W,T}
    return Expr(
      :block,
      Expr(:meta, :inline),
      :(__vstore!(ptr, convert(Vec{$W,$T}, v), m, $A(), $S(), $NT(), StaticInt{$RS}())),
    )
  end
  vstore_quote(
    T,
    Int,
    :StaticInt,
    W,
    sizeof(T),
    0,
    0,
    true,
    A === True,
    S === True,
    NT === True,
    RS,
  )
end
# index, mask, vector store
@generated function __vstore!(
  ptr::Ptr{T},
  v::V,
  i::I,
  m::AbstractMask{W},
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {
  T<:NativeTypesExceptBit,
  W,
  VT<:NativeTypes,
  V<:AbstractSIMDVector{W,VT},
  I<:Index,
  A<:StaticBool,
  S<:StaticBool,
  NT<:StaticBool,
  RS,
}
  if W == 1
    return Expr(
      :block,
      Expr(:meta, :inline),
      :(
        Bool(m) &&
        __vstore!(ptr, convert($T, v), data(i), $A(), $S(), $NT(), StaticInt{$RS}())
      ),
    )
  elseif V !== Vec{W,T}
    return Expr(
      :block,
      Expr(:meta, :inline),
      :(__vstore!(ptr, convert(Vec{$W,$T}, v), i, m, $A(), $S(), $NT(), StaticInt{$RS}())),
    )
  end
  IT, ind_type, _W, X, M, O = index_summary(I)
  (W == _W || _W == 1) || throw(
    ArgumentError(
      "Vector width: $W, index width: $(_W). They must either be equal, or index width == 1.",
    ),
  )
  if (W != _W) & (_W == 1)
    X *= sizeof(T)
  end
  vstore_quote(T, IT, ind_type, W, X, M, O, true, A === True, S === True, NT === True, RS)
end

# no index, mask, vector store
@generated function __vstore!(
  ptr::Ptr{Float16},
  v::V,
  m::AbstractMask{W},
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {W,V<:AbstractSIMDVector{W,Float16},A<:StaticBool,S<:StaticBool,NT<:StaticBool,RS}
  __vstore!(
    reinterpret(Ptr{Int16}, ptr),
    reinterpret(Int16, v),
    m,
    A(),
    S(),
    NT(),
    StaticInt{RS}(),
  )
end
# index, mask, vector store
@inline function __vstore!(
  ptr::Ptr{Float16},
  v::V,
  i::I,
  m::AbstractMask{W},
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {
  W,
  V<:AbstractSIMDVector{W,Float16},
  I<:Index,
  A<:StaticBool,
  S<:StaticBool,
  NT<:StaticBool,
  RS,
}
  __vstore!(
    reinterpret(Ptr{Int16}, ptr),
    reinterpret(Int16, v),
    i,
    m,
    A(),
    S(),
    NT(),
    StaticInt{RS}(),
  )
end

# BitArray stores
@inline function __vstore!(
  ptr::Ptr{Bit},
  v::AbstractSIMDVector{W,B},
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {B<:Union{Bit,Bool},W,A<:StaticBool,S<:StaticBool,NT<:StaticBool,RS}
  __vstore!(
    Base.unsafe_convert(Ptr{mask_type(StaticInt{W}())}, ptr),
    tounsigned(v),
    A(),
    S(),
    NT(),
    StaticInt{RS}(),
  )
end
@inline bytetobit(x::Union{Vec,VecUnroll}) = x >> 3
@inline bytetobit(x::Union{MM,LazyMulAdd}) = data(x) >> 3
@inline bytetobit(x::Union{MM,LazyMulAdd,Unroll}) = data(x) >> 3

@inline function __vstore!(
  ptr::Ptr{Bit},
  v::AbstractSIMDVector{W,B},
  i::VectorIndex{W},
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {B<:Union{Bit,Bool},W,A<:StaticBool,S<:StaticBool,NT<:StaticBool,RS}
  __vstore!(
    Base.unsafe_convert(Ptr{mask_type(StaticInt{W}())}, ptr),
    tounsigned(v),
    bytetobit(i),
    A(),
    S(),
    NT(),
    StaticInt{RS}(),
  )
end
@inline function __vstore!(
  ptr::Ptr{Bit},
  v::AbstractSIMDVector{W,B},
  i::VectorIndex{W},
  m::AbstractMask,
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {B<:Union{Bit,Bool},W,A<:StaticBool,S<:StaticBool,NT<:StaticBool,RS}
  ishift = data(i) >> 3
  p = Base.unsafe_convert(Ptr{mask_type(StaticInt{W}())}, ptr)
  u = bitselect(data(m), __vload(p, ishift, A(), StaticInt{RS}()), tounsigned(v))
  __vstore!(p, u, ishift, A(), S(), NT(), StaticInt{RS}())
end
@inline function __vstore!(
  f::F,
  ptr::Ptr{Bit},
  v::AbstractSIMDVector{W,B},
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {B<:Union{Bit,Bool},F<:Function,W,A<:StaticBool,S<:StaticBool,NT<:StaticBool,RS}
  __vstore!(
    f,
    Base.unsafe_convert(Ptr{mask_type(StaticInt{W}())}, ptr),
    tounsigned(v),
    A(),
    S(),
    NT(),
    StaticInt{RS}(),
  )
end
@inline function __vstore!(
  f::F,
  ptr::Ptr{Bit},
  v::AbstractSIMDVector{W,B},
  i::VectorIndex{W},
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {W,B<:Union{Bit,Bool},F<:Function,A<:StaticBool,S<:StaticBool,NT<:StaticBool,RS}
  __vstore!(
    f,
    Base.unsafe_convert(Ptr{mask_type(StaticInt{W}())}, ptr),
    tounsigned(v),
    data(i) >> 3,
    A(),
    S(),
    NT(),
    StaticInt{RS}(),
  )
end
@inline function __vstore!(
  f::F,
  ptr::Ptr{Bit},
  v::AbstractSIMDVector{W,B},
  i::VectorIndex{W},
  m::AbstractMask,
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {W,B<:Union{Bit,Bool},F<:Function,A<:StaticBool,S<:StaticBool,NT<:StaticBool,RS}
  ishift = data(i) >> 3
  p = Base.unsafe_convert(Ptr{mask_type(StaticInt{W}())}, ptr)
  u = bitselect(data(m), __vload(p, ishift, A(), StaticInt{RS}()), tounsigned(v))
  __vstore!(f, p, u, ishift, A(), S(), NT(), StaticInt{RS}())
end

# Can discard `f` if we have a vector index
@inline function __vstore!(
  f::F,
  ptr::Ptr{T},
  v::AbstractSIMDVector{W},
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {
  T<:NativeTypesExceptBit,
  F<:Function,
  A<:StaticBool,
  S<:StaticBool,
  NT<:StaticBool,
  RS,
  W,
}
  __vstore!(ptr, f(v), A(), S(), NT(), StaticInt{RS}())
end
@inline function __vstore!(
  f::F,
  ptr::Ptr{T},
  v::AbstractSIMDVector{W},
  i::IntegerIndex,
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {
  T<:NativeTypesExceptBit,
  F<:Function,
  A<:StaticBool,
  S<:StaticBool,
  NT<:StaticBool,
  RS,
  W,
}
  __vstore!(ptr, f(v), i, A(), S(), NT(), StaticInt{RS}())
end
@inline function __vstore!(
  f::F,
  ptr::Ptr{T},
  v::AbstractSIMDVector{W},
  i::VectorIndex{W},
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {
  W,
  T<:NativeTypesExceptBit,
  F<:Function,
  A<:StaticBool,
  S<:StaticBool,
  NT<:StaticBool,
  RS,
}
  # __vstore!(ptr, convert(Vec{W,T}, v), i, A(), S(), NT(), StaticInt{RS}()) # discard `f`
  __vstore!(ptr, v, i, A(), S(), NT(), StaticInt{RS}()) # discard `f`
end
@inline function __vstore!(
  f::F,
  ptr::Ptr{T},
  v::AbstractSIMDVector{W},
  i::VectorIndex{W},
  m::AbstractMask{W},
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {
  W,
  T<:NativeTypesExceptBit,
  F<:Function,
  A<:StaticBool,
  S<:StaticBool,
  NT<:StaticBool,
  RS,
}
  # __vstore!(ptr, convert(Vec{W,T}, v), i, m, A(), S(), NT(), StaticInt{RS}())
  __vstore!(ptr, f(v), i, m, A(), S(), NT(), StaticInt{RS}())
end
@inline function __vstore!(
  f::F,
  ptr::Ptr,
  v::AbstractSIMDVector,
  i::Index,
  m::Bool,
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {F<:Function,A<:StaticBool,S<:StaticBool,NT<:StaticBool,RS}
  # __vstore!(ptr, convert(Vec{W,T}, v), i, m, A(), S(), NT(), StaticInt{RS}())
  m && __vstore!(f, ptr, v, i, A(), S(), NT(), StaticInt{RS}())
end

@inline function __vstore!(
  f::F,
  ptr::Ptr{T},
  v::NativeTypes,
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {T<:NativeTypesExceptBit,F<:Function,A<:StaticBool,S<:StaticBool,NT<:StaticBool,RS}
  __vstore!(ptr, v, A(), S(), NT(), StaticInt{RS}())
end
@inline function __vstore!(
  f::F,
  ptr::Ptr{T},
  v::NativeTypes,
  i::IntegerIndex,
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {T<:NativeTypesExceptBit,F<:Function,A<:StaticBool,S<:StaticBool,NT<:StaticBool,RS}
  __vstore!(ptr, v, i, A(), S(), NT(), StaticInt{RS}())
end

for (store, align, alias, nontemporal) ∈ [
  (:vstore!, False(), False(), False()),
  (:vstorea!, True(), False(), False()),
  (:vstorent!, True(), False(), True()),
  (:vnoaliasstore!, False(), True(), False()),
  (:vnoaliasstorea!, True(), True(), False()),
  (:vnoaliasstorent!, True(), True(), True()),
]
  @eval begin
    @inline function $store(ptr::AbstractStridedPointer, v)
      __vstore!(ptr, v, $align, $alias, $nontemporal, register_size())
    end
    @inline function $store(ptr::AbstractStridedPointer, v, i::Union{Tuple,Unroll})
      _vstore!(ptr, v, i, $align, $alias, $nontemporal, register_size())
    end
    @inline function $store(
      ptr::AbstractStridedPointer,
      v,
      i::Union{Tuple,Unroll},
      m::Union{AbstractMask,VecUnroll},
    )
      _vstore!(ptr, v, i, m, $align, $alias, $nontemporal, register_size())
    end
    @inline function $store(ptr::AbstractStridedPointer, v, i::Union{Tuple,Unroll}, b::Bool)
      b && _vstore!(ptr, v, i, $align, $alias, $nontemporal, register_size())
    end

    @inline function $store(f::F, ptr::AbstractStridedPointer, v) where {F<:Function}
      __vstore!(f, ptr, v, $align, $alias, $nontemporal, register_size())
    end
    @inline function $store(
      f::F,
      ptr::AbstractStridedPointer,
      v,
      i::Union{Tuple,Unroll},
    ) where {F<:Function}
      _vstore!(f, ptr, v, i, $align, $alias, $nontemporal, register_size())
    end
    @inline function $store(
      f::F,
      ptr::AbstractStridedPointer,
      v,
      i::Union{Tuple,Unroll},
      m::Union{AbstractMask,VecUnroll},
    ) where {F<:Function}
      _vstore!(f, ptr, v, i, m, $align, $alias, $nontemporal, register_size())
    end
    @inline function $store(
      f::F,
      ptr::AbstractStridedPointer,
      v,
      i::Union{Tuple,Unroll},
      b::Bool,
    ) where {F<:Function}
      b && _vstore!(f, ptr, v, i, $align, $alias, $nontemporal, register_size())
    end
  end
end
@inline function __vstore!(
  ptr::Ptr,
  v,
  i,
  b::Bool,
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {A<:StaticBool,S<:StaticBool,NT<:StaticBool,RS}
  b && __vstore!(ptr, v, i, A(), S(), NT(), StaticInt{RS}())
  nothing
end

@generated function prefetch(ptr::Ptr{Cvoid}, ::Val{L}, ::Val{R}) where {L,R}
  L ∈ (0, 1, 2, 3) || throw(
    ArgumentError(
      "Prefetch intrinsic requires a locality argument of 0, 1, 2, or 3, but received $L.",
    ),
  )
  R ∈ (0, 1) || throw(
    ArgumentError(
      "Prefetch intrinsic requires a read/write argument of 0, 1, but received $R.",
    ),
  )
  decl = "declare void @llvm.prefetch(i8*, i32, i32, i32)"
  instrs = """
      %addr = inttoptr $JULIAPOINTERTYPE %0 to i8*
      call void @llvm.prefetch(i8* %addr, i32 $R, i32 $L, i32 1)
      ret void
  """
  llvmcall_expr(
    decl,
    instrs,
    :Cvoid,
    :(Tuple{Ptr{Cvoid}}),
    "void",
    [JULIAPOINTERTYPE],
    [:ptr],
  )
end
@inline prefetch(ptr::Ptr{T}, ::Val{L}, ::Val{R}) where {T,L,R} =
  prefetch(Base.unsafe_convert(Ptr{Cvoid}, ptr), Val{L}(), Val{R}())

@inline function prefetch(
  ptr::Union{AbstractStridedPointer,Ptr},
  i,
  ::Val{Locality},
  ::Val{ReadOrWrite},
) where {Locality,ReadOrWrite}
  prefetch(gep(ptr, i), Val{Locality}(), Val{ReadOrWrite}())
end
@inline prefetch(ptr) = nothing
@inline prefetch(ptr::Ptr) = prefetch(reinterpret(Ptr{Cvoid}, ptr), Val{3}(), Val{0}())
@inline prefetch(ptr::Ptr, ::Val{L}) where {L} = prefetch(ptr, Val{L}(), Val{0}())
@inline prefetch(ptr::Ptr, i) = prefetch(ptr, i, Val{3}(), Val{0}())
@inline prefetch(ptr::Ptr, i, ::Val{L}) where {L} = prefetch(ptr, i, Val{L}(), Val{0}())

@inline prefetch0(x, i) = prefetch(gep(stridedpointer(x), (data(i),)), Val{3}(), Val{0}())
@inline prefetch0(x, I::Tuple) =
  prefetch(gep(stridedpointer(x), data.(I)), Val{3}(), Val{0}())
@inline prefetch0(x, i, j) =
  prefetch(gep(stridedpointer(x), (data(i), data(j))), Val{3}(), Val{0}())
# @inline prefetch0(x, i, j, oi, oj) = prefetch(gep(stridedpointer(x), (data(i) + data(oi) - 1, data(j) + data(oj) - 1)), Val{3}(), Val{0}())
@inline prefetch1(x, i) = prefetch(gep(stridedpointer(x), (data(i),)), Val{2}(), Val{0}())
@inline prefetch1(x, i, j) =
  prefetch(gep(stridedpointer(x), (data(i), data(j))), Val{2}(), Val{0}())
# @inline prefetch1(x, i, j, oi, oj) = prefetch(gep(stridedpointer(x), (data(i) + data(oi) - 1, data(j) + data(oj) - 1)), Val{2}(), Val{0}())
@inline prefetch2(x, i) = prefetch(gep(stridedpointer(x), (data(i),)), Val{1}(), Val{0}())
@inline prefetch2(x, i, j) =
  prefetch(gep(stridedpointer(x), (data(i), data(j))), Val{1}(), Val{0}())
# @inline prefetch2(x, i, j, oi, oj) = prefetch(gep(stridedpointer(x), (data(i) + data(oi) - 1, data(j) + data(oj) - 1)), Val{1}(), Val{0}())

@generated function lifetime_start!(ptr::Ptr{T}, ::Val{L}) where {L,T}
  decl = "declare void @llvm.lifetime.start(i64, i8* nocapture)"
  instrs = "%ptr = inttoptr $JULIAPOINTERTYPE %0 to i8*\ncall void @llvm.lifetime.start(i64 $(L*sizeof(T)), i8* %ptr)\nret void"
  llvmcall_expr(decl, instrs, :Cvoid, :(Tuple{Ptr{$T}}), "void", [JULIAPOINTERTYPE], [:ptr])
end
@generated function lifetime_end!(ptr::Ptr{T}, ::Val{L}) where {L,T}
  decl = "declare void @llvm.lifetime.end(i64, i8* nocapture)"
  instrs = "%ptr = inttoptr $JULIAPOINTERTYPE %0 to i8*\ncall void @llvm.lifetime.end(i64 $(L*sizeof(T)), i8* %ptr)\nret void"
  llvmcall_expr(decl, instrs, :Cvoid, :(Tuple{Ptr{$T}}), "void", [JULIAPOINTERTYPE], [:ptr])
end

@inline lifetime_start!(ptr::Ptr) = lifetime_start!(ptr, Val{-1}())
@inline lifetime_end!(ptr::Ptr) = lifetime_end!(ptr, Val{-1}())
# Fallback is to do nothing. Intention is (e.g.) for PaddedMatrices/StackPointers.
@inline lifetime_start!(::Any) = nothing
@inline lifetime_end!(::Any) = nothing

@generated function compressstore!(
  ptr::Ptr{T},
  v::Vec{W,T},
  mask::AbstractMask{W,U},
) where {W,T<:NativeTypes,U<:Unsigned}
  typ = LLVM_TYPES[T]
  vtyp = "<$W x $typ>"
  mtyp_input = LLVM_TYPES[U]
  mtyp_trunc = "i$W"
  instrs = String["%ptr = inttoptr $JULIAPOINTERTYPE %1 to $typ*"]
  truncate_mask!(instrs, '2', W, 0)
  decl = "declare void @llvm.masked.compressstore.$(suffix(W,T))($vtyp, $typ*, <$W x i1>)"
  push!(
    instrs,
    "call void @llvm.masked.compressstore.$(suffix(W,T))($vtyp %0, $typ* %ptr, <$W x i1> %mask.0)\nret void",
  )
  llvmcall_expr(
    decl,
    join(instrs, "\n"),
    :Cvoid,
    :(Tuple{_Vec{$W,$T},Ptr{$T},$U}),
    "void",
    [vtyp, JULIAPOINTERTYPE, "i$(8sizeof(U))"],
    [:(data(v)), :ptr, :(data(mask))],
  )
end

@generated function expandload(
  ptr::Ptr{T},
  mask::AbstractMask{W,U},
) where {W,T<:NativeTypes,U<:Unsigned}
  typ = LLVM_TYPES[T]
  vtyp = "<$W x $typ>"
  vptrtyp = "<$W x $typ*>"
  mtyp_input = LLVM_TYPES[U]
  mtyp_trunc = "i$W"
  instrs = String[]
  push!(instrs, "%ptr = inttoptr $JULIAPOINTERTYPE %0 to $typ*")
  if mtyp_input == mtyp_trunc
    push!(instrs, "%mask = bitcast $mtyp_input %1 to <$W x i1>")
  else
    push!(instrs, "%masktrunc = trunc $mtyp_input %1 to $mtyp_trunc")
    push!(instrs, "%mask = bitcast $mtyp_trunc %masktrunc to <$W x i1>")
  end
  decl = "declare $vtyp @llvm.masked.expandload.$(suffix(W,T))($typ*, <$W x i1>, $vtyp)"
  push!(
    instrs,
    "%res = call $vtyp @llvm.masked.expandload.$(suffix(W,T))($typ* %ptr, <$W x i1> %mask, $vtyp zeroinitializer)\nret $vtyp %res",
  )
  llvmcall_expr(
    decl,
    join(instrs, "\n"),
    :(_Vec{$W,$T}),
    :(Tuple{Ptr{$T},$U}),
    vtyp,
    [JULIAPOINTERTYPE, "i$(8sizeof(U))"],
    [:ptr, :(data(mask))],
  )
end

# fallback definitions
@generated function __vload(p::Ptr{T}, i::Index, ::A, ::StaticInt{RS}) where {T,A,RS}
  if Base.allocatedinline(T)
    Expr(:block, Expr(:meta, :inline), :(unsafe_load(p + convert(Int, i))))
  else
    Expr(
      :block,
      Expr(:meta, :inline),
      :(ccall(
        :jl_value_ptr,
        Ref{$T},
        (Ptr{Cvoid},),
        unsafe_load(Base.unsafe_convert(Ptr{Ptr{Cvoid}}, p) + convert(Int, i)),
      )),
    )
  end
end
@generated function __vstore!(
  p::Ptr{T},
  v::T,
  i::Index,
  ::A,
  ::S,
  ::NT,
  ::StaticInt{RS},
) where {T,A,S,NT,RS}
  if Base.allocatedinline(T)
    Expr(
      :block,
      Expr(:meta, :inline),
      :(unsafe_store!(p + convert(Int, i), v); return nothing),
    )
  else
    Expr(
      :block,
      Expr(:meta, :inline),
      :(
        unsafe_store!(
          Base.unsafe_convert(Ptr{Ptr{Cvoid}}, p) + convert(Int, i),
          Base.pointer_from_objref(v),
        );
        return nothing
      ),
    )
  end
end

# @inline function Base.getindex(A::AbstractArray{<:Number}, i::Unroll)
#   sp, pres = stridedpointer_preserve(A)
#   GC.@preserve pres begin
#     x = vload(sp, i)
#   end
#   return x
# end
# @inline function Base.getindex(A::ArrayInterface.AbstractArray2{<:Number}, i::Unroll)
#   sp, pres = stridedpointer_preserve(A)
#   GC.@preserve pres begin
#     x = vload(sp, i)
#   end
#   return x
# end
# @inline function Base.setindex!(A::AbstractArray{<:Number}, v, i::Unroll)
#   sp, pres = stridedpointer_preserve(A)
#   GC.@preserve pres begin
#     vstore!(sp, v, i)
#   end
#   return v
# end
# @inline function Base.setindex!(A::ArrayInterface.AbstractArray2{<:Number}, v, i::Unroll)
#   sp, pres = stridedpointer_preserve(A)
#   GC.@preserve pres begin
#     vstore!(sp, v, i)
#   end
#   return v
# end

# @inline function Base.getindex(A::AbstractArray{<:Number}, i::AbstractSIMD)
#   sp, pres = stridedpointer_preserve(A)
#   GC.@preserve pres begin
#     x = vload(zero_offsets(sp), (vsub_nw(i,ArrayInterface.offset1(A)),))
#   end
#   return x
# end
# @inline function Base.getindex(A::ArrayInterface.AbstractArray2{<:Number}, i::AbstractSIMD)
#   sp, pres = stridedpointer_preserve(A)
#   GC.@preserve pres begin
#     x = vload(zero_offsets(sp), (vsub_nw(i,ArrayInterface.offset1(A)),))
#   end
#   return x
# end

# @inline function Base.getindex(A::AbstractArray{<:Number}, i::Vararg{Union{Integer,AbstractSIMD},K}) where {K}
#   sp, pres = stridedpointer_preserve(A)
#   GC.@preserve pres begin
#     x = vload(sp, i)
#   end
#   return x
# end
# @inline function Base.getindex(A::ArrayInterface.AbstractArray2{<:Number}, i::Vararg{Union{Integer,AbstractSIMD},K}) where {K}
#   sp, pres = stridedpointer_preserve(A)
#   GC.@preserve pres begin
#     x = vload(sp, i)
#   end
#   return x
# end


text:
using FactCheck, Base.Test, Images, Colors, FixedPointNumbers
using Compat.view

srand(1234)

facts("Algorithms") do
    # Comparison of each element in arrays with a scalar
    approx_equal(ar, v) = @compat all(abs.(ar.-v) .< sqrt(eps(v)))
    approx_equal(ar::Images.AbstractImage, v) = approx_equal(Images.data(ar), v)

	context("Flip dimensions") do
		A = UInt8[200 150; 50 1]
		img_x = grayim(A)
		img_y = permutedims(img_x, [2, 1])

		@fact raw(flipdim(img_x, "x")) --> raw(flipdim(img_x, 1))
		@fact raw(flipdim(img_x, "x")) --> flipdim(A, 1)
		@fact raw(flipdim(img_y, "x")) --> raw(flipdim(img_y, 2))
		@fact raw(flipdim(img_y, "x")) --> flipdim(A', 2)

		@fact raw(flipdim(img_x, "y")) --> raw(flipdim(img_x, 2))
		@fact raw(flipdim(img_x, "y")) --> flipdim(A, 2)
		@fact raw(flipdim(img_y, "y")) --> raw(flipdim(img_y, 1))
		@fact raw(flipdim(img_y, "y")) --> flipdim(A', 1)

		@fact raw(flipx(img_x)) --> raw(flipdim(img_x, "x"))
		@fact raw(flipx(img_y)) --> raw(flipdim(img_y, "x"))

		@fact raw(flipy(img_x)) --> raw(flipdim(img_x, "y"))
		@fact raw(flipy(img_y)) --> raw(flipdim(img_y, "y"))
	end

    context("Arithmetic") do
        img = convert(Images.Image, zeros(3,3))
        img2 = (img .+ 3)/2
        @fact all(img2 .== 1.5) --> true
        img3 = 2img2
        @fact all(img3 .== 3) --> true
        img3 = copy(img2)
        img3[img2 .< 4] = -1
        @fact all(img3 .== -1) --> true
        img = convert(Images.Image, rand(3,4))
        A = rand(3,4)
        img2 = img .* A
        @fact all(Images.data(img2) == Images.data(img).*A) --> true
        img2 = convert(Images.Image, A)
        img2 = img2 .- 0.5
        img3 = 2img .* img2
        img2 = img ./ A
        img2 = (2img).^2
        # Same operations with Color images
        img = Images.colorim(zeros(Float32,3,4,5))
        img2 = (img .+ RGB{Float32}(1,1,1))/2
        @fact all(img2 .== RGB{Float32}(1,1,1)/2) --> true
        img3 = 2img2
        @fact all(img3 .== RGB{Float32}(1,1,1)) --> true
        A = fill(2, 4, 5)
        @fact all(A.*img2 .== fill(RGB{Float32}(1,1,1), 4, 5)) --> true
        img2 = img2 .- RGB{Float32}(1,1,1)/2
        A = rand(UInt8,3,4)
        img = reinterpret(Gray{UFixed8}, Images.grayim(A))
        imgm = mean(img)
        imgn = img/imgm
        @fact reinterpret(Float64, Images.data(imgn)) --> roughly(convert(Array{Float64}, A/mean(A)))
        @fact imcomplement([Gray(0.2)]) --> [Gray(0.8)]
        @fact imcomplement([Gray{U8}(0.2)]) --> [Gray{U8}(0.8)]
        @fact imcomplement([RGB(0,0.3,1)]) --> [RGB(1,0.7,0)]
        @fact imcomplement([RGBA(0,0.3,1,0.7)]) --> [RGBA(1.0,0.7,0.0,0.7)]
        @fact imcomplement([RGBA{U8}(0,0.6,1,0.7)]) --> [RGBA{U8}(1.0,0.4,0.0,0.7)]

        img = rand(1:10,10,10)
        img2 = rand(1:2,10,10)
        img3 = reinterpret(Gray{U8}, grayim(rand(UInt8,10,10)))
        @fact all([entropy(img, kind=kind) for kind in [:shannon,:nat,:hartley]] .≥ 0) --> true
        @fact all([entropy(img2, kind=kind) for kind in [:shannon,:nat,:hartley]] .≥ 0) --> true
        @fact all([entropy(img3, kind=kind) for kind in [:shannon,:nat,:hartley]] .≥ 0) --> true
    end

    context("Reductions") do
        A = rand(5,5,3)
        img = Images.colorim(A, "RGB")
        s12 = sum(img, (1,2))
        @fact Images.colorspace(s12) --> "RGB"
        s3 = sum(img, (3,))
        @fact Images.colorspace(s3) --> "Unknown"
        A = [NaN, 1, 2, 3]
        @fact Images.meanfinite(A, 1) --> roughly([2])
        A = [NaN 1 2 3;
             NaN 6 5 4]
        @test_approx_eq Images.meanfinite(A, 1) [NaN 3.5 3.5 3.5]
        @test_approx_eq Images.meanfinite(A, 2) [2, 5]'
        @test_approx_eq Images.meanfinite(A, (1,2)) [3.5]
        @fact Images.minfinite(A) --> 1
        @fact Images.maxfinite(A) --> 6
        @fact Images.maxabsfinite(A) --> 6
        A = rand(10:20, 5, 5)
        @fact minfinite(A) --> minimum(A)
        @fact maxfinite(A) --> maximum(A)
        A = reinterpret(UFixed8, rand(0x00:0xff, 5, 5))
        @fact minfinite(A) --> minimum(A)
        @fact maxfinite(A) --> maximum(A)
        A = rand(Float32,3,5,5)
        img = Images.colorim(A, "RGB")
        dc = Images.data(Images.meanfinite(img, 1))-reinterpret(RGB{Float32}, mean(A, 2), (1,5))
        @fact maximum(map(abs, dc)) --> less_than(1e-6)
        dc = Images.minfinite(img)-RGB{Float32}(minimum(A, (2,3))...)
        @fact abs(dc) --> less_than(1e-6)
        dc = Images.maxfinite(img)-RGB{Float32}(maximum(A, (2,3))...)
        @fact abs(dc) --> less_than(1e-6)

        a = convert(Array{UInt8}, [1, 1, 1])
        b = convert(Array{UInt8}, [134, 252, 4])
        @fact Images.sad(a, b) --> 387
        @fact Images.ssd(a, b) --> 80699
        af = reinterpret(UFixed8, a)
        bf = reinterpret(UFixed8, b)
        @fact Images.sad(af, bf) --> roughly(387f0/255)
        @fact Images.ssd(af, bf) --> roughly(80699f0/255^2)
        ac = reinterpret(RGB{UFixed8}, a)
        bc = reinterpret(RGB{UFixed8}, b)
        @fact Images.sad(ac, bc) --> roughly(387f0/255)
        @fact Images.ssd(ac, bc) --> roughly(80699f0/255^2)
        ag = reinterpret(RGB{UFixed8}, a)
        bg = reinterpret(RGB{UFixed8}, b)
        @fact Images.sad(ag, bg) --> roughly(387f0/255)
        @fact Images.ssd(ag, bg) --> roughly(80699f0/255^2)

        a = rand(15,15)
        @fact_throws ErrorException (Images.@test_approx_eq_sigma_eps a rand(13,15) [1,1] 0.01)
        @fact_throws ErrorException (Images.@test_approx_eq_sigma_eps a rand(15,15) [1,1] 0.01)
        @fact (Images.@test_approx_eq_sigma_eps a a [1,1] 0.01) --> nothing
        @fact (Images.@test_approx_eq_sigma_eps a a+0.01*rand(size(a)) [1,1] 0.01) --> nothing
        @fact_throws ErrorException (Images.@test_approx_eq_sigma_eps a a+0.5*rand(size(a)) [1,1] 0.01)
        a = colorim(rand(3,15,15))
        @fact (Images.@test_approx_eq_sigma_eps a a [1,1] 0.01) --> nothing
        @fact_throws ErrorException (Images.@test_approx_eq_sigma_eps a colorim(rand(3,15,15)) [1,1] 0.01)

        a = rand(15,15)
        @fact_throws ErrorException Images.test_approx_eq_sigma_eps(a, rand(13,15), [1,1], 0.01)
        @fact_throws ErrorException Images.test_approx_eq_sigma_eps(a, rand(15,15), [1,1], 0.01)
        @fact Images.test_approx_eq_sigma_eps(a, a, [1,1], 0.01) --> 0.0
        @fact Images.test_approx_eq_sigma_eps(a, a+0.01*rand(size(a)), [1,1], 0.01) --> greater_than(0.0)
        @fact_throws ErrorException Images.test_approx_eq_sigma_eps(a, a+0.5*rand(size(a)), [1,1], 0.01)
        a = colorim(rand(3,15,15))
        @fact Images.test_approx_eq_sigma_eps(a, a, [1,1], 0.01) --> 0.0
        @fact_throws ErrorException Images.test_approx_eq_sigma_eps(a, colorim(rand(3,15,15)), [1,1], 0.01)

        @fact Images.test_approx_eq_sigma_eps(a[:,1:end-1], a[1:end-1,:], [3,3], 0.1) --> less_than(0.1)
        @fact_throws Images.test_approx_eq_sigma_eps(a[:,1:end-1], a[1:end-1,:], [3,3], 0.01)

        a = zeros(10, 10)
        int_img = integral_image(a)
        @fact all(int_img == a) --> true

        a = ones(10,10)
        int_img = integral_image(a)
        chk = Array(1:10)
        @fact all([vec(int_img[i, :]) == chk * i for i in 1:10]) --> true

        int_sum = boxdiff(int_img, 1, 1, 5, 2)
        @fact int_sum --> 10.0
        int_sum = boxdiff(int_img, 1:5, 1:2)
        @fact int_sum --> 10.0
        int_sum = boxdiff(int_img, CartesianIndex((1, 1)), CartesianIndex((5, 2)))
        @fact int_sum --> 10.0
        int_sum = boxdiff(int_img, 1, 1, 2, 5)
        @fact int_sum --> 10.0
        int_sum = boxdiff(int_img, 1:2, 1:5)
        @fact int_sum --> 10.0
        int_sum = boxdiff(int_img, CartesianIndex((1, 1)), CartesianIndex((2, 5)))
        @fact int_sum --> 10.0
        int_sum = boxdiff(int_img, 4, 4, 8, 8)
        @fact int_sum --> 25.0
        int_sum = boxdiff(int_img, 4:8, 4:8)
        @fact int_sum --> 25.0
        int_sum = boxdiff(int_img, CartesianIndex((4, 4)), CartesianIndex((8, 8)))
        @fact int_sum --> 25.0

        a = reshape(1:100, 10, 10)
        int_img = integral_image(a)
        @fact int_img[diagind(int_img)] == Array([1, 26,  108,  280,  575, 1026, 1666, 2528, 3645, 5050]) --> true

        int_sum = boxdiff(int_img, 1, 1, 3, 3)
        @fact int_sum --> 108
        int_sum = boxdiff(int_img, 1:3, 1:3)
        @fact int_sum --> 108
        int_sum = boxdiff(int_img, CartesianIndex((1, 1)), CartesianIndex((3, 3)))
        @fact int_sum --> 108
        int_sum = boxdiff(int_img, 1, 1, 5, 2)
        @fact int_sum --> 80
        int_sum = boxdiff(int_img, 1:5, 1:2)
        @fact int_sum --> 80
        int_sum = boxdiff(int_img, CartesianIndex((1, 1)), CartesianIndex((5, 2)))
        @fact int_sum --> 80
        int_sum = boxdiff(int_img, 4, 4, 8, 8)
        @fact int_sum --> 1400
        int_sum = boxdiff(int_img, 4:8, 4:8)
        @fact int_sum --> 1400
        int_sum = boxdiff(int_img, CartesianIndex((4, 4)), CartesianIndex((8, 8)))
        @fact int_sum --> 1400

        img = zeros(40, 40)
        img[10:30, 10:30] = 1
        pyramid = gaussian_pyramid(img, 3, 2, 1.0)
        @fact size(pyramid[1]) == (40, 40) --> true
        @fact size(pyramid[2]) == (20, 20) --> true
        @fact size(pyramid[3]) == (10, 10) --> true
        @fact size(pyramid[4]) == (5, 5) --> true
        @fact isapprox(pyramid[1][20, 20], 1.0, atol = 0.01) --> true
        @fact isapprox(pyramid[2][10, 10], 1.0, atol = 0.01) --> true
        @fact isapprox(pyramid[3][5, 5], 1.0, atol = 0.05) --> true
        @fact isapprox(pyramid[4][3, 3], 0.9, atol = 0.025) --> true

        for p in pyramid
            h, w = size(p)
            @fact all(Bool[isapprox(v, 0, atol = 0.06) for v in p[1, :]]) --> true
            @fact all(Bool[isapprox(v, 0, atol = 0.06) for v in p[:, 1]]) --> true
            @fact all(Bool[isapprox(v, 0, atol = 0.06) for v in p[h, :]]) --> true
            @fact all(Bool[isapprox(v, 0, atol = 0.06) for v in p[:, w]]) --> true
        end
    end

    context("fft and ifft") do
        A = rand(Float32, 3, 5, 6)
        img = Images.colorim(A)
        imgfft = fft(img)
        @fact Images.data(imgfft) --> roughly(fft(A, 2:3))
        @fact Images.colordim(imgfft) --> 1
        img2 = ifft(imgfft)
        @fact img2 --> roughly(reinterpret(Float32, img))
    end

    context("Array padding") do
        A = [1 2; 3 4]
        @fact Images.padindexes(A, 1, 0, 0, "replicate") --> [1,2]
        @fact Images.padindexes(A, 1, 1, 0, "replicate") --> [1,1,2]
        @fact Images.padindexes(A, 1, 0, 1, "replicate") --> [1,2,2]
        @fact Images.padarray(A, (0,0), (0,0), "replicate") --> A
        @fact Images.padarray(A, (1,2), (2,0), "replicate") --> [1 1 1 2; 1 1 1 2; 3 3 3 4; 3 3 3 4; 3 3 3 4]
        @fact Images.padindexes(A, 1, 1, 0, "circular") --> [2,1,2]
        @fact Images.padindexes(A, 1, 0, 1, "circular") --> [1,2,1]
        @fact Images.padarray(A, [2,1], [0,2], "circular") --> [2 1 2 1 2; 4 3 4 3 4; 2 1 2 1 2; 4 3 4 3 4]
        @fact Images.padindexes(A, 1, 1, 0, "symmetric") --> [1,1,2]
        @fact Images.padindexes(A, 1, 0, 1, "symmetric") --> [1,2,2]
        @fact Images.padarray(A, (1,2), (2,0), "symmetric") --> [2 1 1 2; 2 1 1 2; 4 3 3 4; 4 3 3 4; 2 1 1 2]
        @fact Images.padarray(A, (1,2), (2,0), "value", -1) --> [-1 -1 -1 -1; -1 -1 1 2; -1 -1 3 4; -1 -1 -1 -1; -1 -1 -1 -1]
        A = [1 2 3; 4 5 6]
        @fact Images.padindexes(A, 2, 1, 0, "reflect") --> [2,1,2,3]
        @fact Images.padindexes(A, 2, 0, 1, "reflect") --> [1,2,3,2]
        @fact Images.padarray(A, (1,2), (2,0), "reflect") --> [6 5 4 5 6; 3 2 1 2 3; 6 5 4 5 6; 3 2 1 2 3; 6 5 4 5 6]
        A = [1 2; 3 4]
        @fact Images.padarray(A, (1,1)) --> [1 1 2 2; 1 1 2 2; 3 3 4 4; 3 3 4 4]
        @fact Images.padarray(A, (1,1), "replicate", "both") --> [1 1 2 2; 1 1 2 2; 3 3 4 4; 3 3 4 4]
        @fact Images.padarray(A, (1,1), "circular", "pre") --> [4 3 4; 2 1 2; 4 3 4]
        @fact Images.padarray(A, (1,1), "symmetric", "post") --> [1 2 2; 3 4 4; 3 4 4]
        A = ["a" "b"; "c" "d"]
        @fact Images.padarray(A, (1,1)) --> ["a" "a" "b" "b"; "a" "a" "b" "b"; "c" "c" "d" "d"; "c" "c" "d" "d"]
        @fact_throws ErrorException Images.padindexes(A, 1, 1, 1, "unknown")
        @fact_throws ErrorException Images.padarray(A, (1,1), "unknown")
        # issue #292
        A = trues(3,3)
        @fact typeof(Images.padarray(A, (1,2), (2,1), "replicate")) --> BitArray{2}
        @fact typeof(Images.padarray(Images.grayim(A), (1,2), (2,1), "replicate")) --> BitArray{2}
        # issue #525
        A = falses(10,10,10)
        B = view(A,1:8,1:8,1:8)
        @fact isa(padarray(A, ones(Int,3), ones(Int,3), "replicate"), BitArray{3}) --> true
        @fact isa(padarray(B, ones(Int,3), ones(Int,3), "replicate"), BitArray{3}) --> true
    end

    @compat context("Filtering") do
        EPS = 1e-14
        imgcol = Images.colorim(rand(3,5,6))
        imgcolf = convert(Images.Image{RGB{UFixed8}}, imgcol)
        for T in (Float64, Int)
            A = zeros(T,3,3); A[2,2] = 1
            kern = rand(3,3)
            @fact maximum(abs.(Images.imfilter(A, kern) - rot180(kern))) --> less_than(EPS)
            kern = rand(2,3)
            @fact maximum(abs.(Images.imfilter(A, kern)[1:2,:] - rot180(kern))) --> less_than(EPS)
            kern = rand(3,2)
            @fact maximum(abs.(Images.imfilter(A, kern)[:,1:2] - rot180(kern))) --> less_than(EPS)
        end
        kern = zeros(3,3); kern[2,2] = 1
        @fact maximum(map(abs, imgcol - Images.imfilter(imgcol, kern))) --> less_than(EPS)
        @fact maximum(map(abs, imgcolf - Images.imfilter(imgcolf, kern))) --> less_than(EPS)
        for T in (Float64, Int)
            # Separable kernels
            A = zeros(T,3,3); A[2,2] = 1
            kern = rand(3).*rand(3)'
            @fact maximum(abs.(Images.imfilter(A, kern) - rot180(kern))) --> less_than(EPS)
            kern = rand(2).*rand(3)'
            @fact maximum(abs.(Images.imfilter(A, kern)[1:2,:] - rot180(kern))) --> less_than(EPS)
            kern = rand(3).*rand(2)'
            @fact maximum(abs.(Images.imfilter(A, kern)[:,1:2] - rot180(kern))) --> less_than(EPS)
        end
        A = zeros(3,3); A[2,2] = 1
        kern = rand(3,3)
        @fact maximum(abs.(Images.imfilter_fft(A, kern) - rot180(kern))) --> less_than(EPS)
        kern = rand(2,3)
        @fact maximum(abs.(Images.imfilter_fft(A, kern)[1:2,:] - rot180(kern))) --> less_than(EPS)
        kern = rand(3,2)
        @fact maximum(abs.(Images.imfilter_fft(A, kern)[:,1:2] - rot180(kern))) --> less_than(EPS)
        kern = zeros(3,3); kern[2,2] = 1
        @fact maximum(map(abs, imgcol - Images.imfilter_fft(imgcol, kern))) --> less_than(EPS)
        @fact maximum(map(abs, imgcolf - Images.imfilter_fft(imgcolf, kern))) --> less_than(EPS)

        @fact approx_equal(Images.imfilter(ones(4,4), ones(3,3)), 9.0) --> true
        @fact approx_equal(Images.imfilter(ones(3,3), ones(3,3)), 9.0) --> true
        @fact approx_equal(Images.imfilter(ones(3,3), [1 1 1;1 0.0 1;1 1 1]), 8.0) --> true
        img = convert(Images.Image, ones(4,4))
        @fact approx_equal(Images.imfilter(img, ones(3,3)), 9.0) --> true
        A = zeros(5,5,3); A[3,3,[1,3]] = 1
        @fact Images.colordim(A) --> 3
        kern = rand(3,3)
        kernpad = zeros(5,5); kernpad[2:4,2:4] = kern
        Af = Images.imfilter(A, kern)

        @fact cat(3, rot180(kernpad), zeros(5,5), rot180(kernpad)) --> roughly(Af)
        Aimg = permutedims(convert(Images.Image, A), [3,1,2])
        @fact Images.imfilter(Aimg, kern) --> roughly(permutedims(Af, [3,1,2]))
        @fact approx_equal(Images.imfilter(ones(4,4),ones(1,3),"replicate"), 3.0) --> true

        A = zeros(5,5); A[3,3] = 1
        kern = rand(3,3)
        Af = Images.imfilter(A, kern, "inner")
        @fact Af --> rot180(kern)
        Afft = Images.imfilter_fft(A, kern, "inner")
        @fact Af --> roughly(Afft)
        h = [0.24,0.87]
        hfft = Images.imfilter_fft(eye(3), h, "inner")
        hfft[abs.(hfft) .< 3eps()] = 0
        @fact Images.imfilter(eye(3), h, "inner") --> roughly(hfft)  # issue #204

        # circular
        A = zeros(3, 3)
        A[3,2] = 1
        kern = rand(3,3)
        @fact Images.imfilter_fft(A, kern, "circular") --> roughly(kern[[1,3,2],[3,2,1]])

        A = zeros(5, 5)
        A[5,3] = 1
        kern = rand(3,3)
        @fact Images.imfilter_fft(A, kern, "circular")[[1,4,5],2:4] --> roughly(kern[[1,3,2],[3,2,1]])

        A = zeros(5, 5)
        A[5,3] = 1
        kern = rand(3,3)
        @fact Images.imfilter(A, kern, "circular")[[1,4,5],2:4] --> roughly(kern[[1,3,2],[3,2,1]])

        @fact approx_equal(Images.imfilter_gaussian(ones(4,4), [5,5]), 1.0) --> true
        A = fill(convert(Float32, NaN), 3, 3)
        A[1,1] = 1
        A[2,1] = 2
        A[3,1] = 3
        @fact Images.imfilter_gaussian(A, [0,0]) --> exactly(A)
        @test_approx_eq Images.imfilter_gaussian(A, [0,3]) A
        B = copy(A)
        B[isfinite.(B)] = 2
        @test_approx_eq Images.imfilter_gaussian(A, [10^3,0]) B
        @fact maximum(map(abs, Images.imfilter_gaussian(imgcol, [10^3,10^3]) - mean(imgcol))) --> less_than(1e-4)
        @fact maximum(map(abs, Images.imfilter_gaussian(imgcolf, [10^3,10^3]) - mean(imgcolf))) --> less_than(1e-4)
        A = rand(4,5)
        img = reinterpret(Images.Gray{Float64}, Images.grayim(A))
        imgf = Images.imfilter_gaussian(img, [2,2])
        @fact reinterpret(Float64, Images.data(imgf)) --> roughly(Images.imfilter_gaussian(A, [2,2]))
        A = rand(3,4,5)
        img = Images.colorim(A)
        imgf = Images.imfilter_gaussian(img, [2,2])
        @fact reinterpret(Float64, Images.data(imgf)) --> roughly(Images.imfilter_gaussian(A, [0,2,2]))

        A = zeros(Int, 9, 9); A[5, 5] = 1
        @fact maximum(abs.(Images.imfilter_LoG(A, [1,1]) - Images.imlog(1.0))) --> less_than(EPS)
        @fact maximum(Images.imfilter_LoG([0 0 0 0 1 0 0 0 0], [1,1]) - sum(Images.imlog(1.0),1)) --> less_than(EPS)
        @fact maximum(Images.imfilter_LoG([0 0 0 0 1 0 0 0 0]', [1,1]) - sum(Images.imlog(1.0),2)) --> less_than(EPS)

        @fact Images.imaverage() --> fill(1/9, 3, 3)
        @fact Images.imaverage([3,3]) --> fill(1/9, 3, 3)
        @fact_throws ErrorException Images.imaverage([5])
    end

    context("Features") do
        A = zeros(Int, 9, 9); A[5, 5] = 1
        @fact all(x->x<eps(),[blob_LoG(A, 2.0.^[0.5,0,1])[1]...] - [0.3183098861837907,sqrt(2),5,5]) --> true
        @fact all(x->x<eps(),[blob_LoG(A, [1])[1]...] - [0.3183098861837907,sqrt(2),5,5]) --> true
        A = zeros(Int, 9, 9); A[[1:2;5],5]=1
        @fact findlocalmaxima(A) --> [(5,5)]
        @fact findlocalmaxima(A,2) --> [(1,5),(2,5),(5,5)]
        @fact findlocalmaxima(A,2,false) --> [(2,5),(5,5)]
        A = zeros(Int, 9, 9, 9); A[[1:2;5],5,5]=1
        @fact findlocalmaxima(A) --> [(5,5,5)]
        @fact findlocalmaxima(A,2) --> [(1,5,5),(2,5,5),(5,5,5)]
        @fact findlocalmaxima(A,2,false) --> [(2,5,5),(5,5,5)]
    end

    context("Restriction") do
        imgcol = Images.colorim(rand(3,5,6))
        A = reshape([convert(UInt16, i) for i = 1:60], 4, 5, 3)
        B = Images.restrict(A, (1,2))
        Btarget = cat(3, [ 0.96875  4.625   5.96875;
                           2.875   10.5    12.875;
                           1.90625  5.875   6.90625],
                      [ 8.46875  14.625 13.46875;
                        17.875    30.5   27.875;
                        9.40625  15.875 14.40625],
                      [15.96875  24.625 20.96875;
                       32.875    50.5   42.875;
                       16.90625  25.875 21.90625])
        @fact B --> roughly(Btarget)
        Argb = reinterpret(RGB, reinterpret(UFixed16, permutedims(A, (3,1,2))))
        B = Images.restrict(Argb)
        Bf = permutedims(reinterpret(Float64, B), (2,3,1))
        @fact Bf --> roughly(Btarget/reinterpret(one(UFixed16)), 1e-12)
        Argba = reinterpret(RGBA{UFixed16}, reinterpret(UFixed16, A))
        B = Images.restrict(Argba)
        @fact reinterpret(Float64, B) --> roughly(Images.restrict(A, (2,3))/reinterpret(one(UFixed16)), 1e-12)
        A = reshape(1:60, 5, 4, 3)
        B = Images.restrict(A, (1,2,3))
        @fact cat(3, [ 2.6015625  8.71875 6.1171875;
                       4.09375   12.875   8.78125;
                       3.5390625 10.59375 7.0546875],
                     [10.1015625 23.71875 13.6171875;
                      14.09375   32.875   18.78125;
                      11.0390625 25.59375 14.5546875]) --> roughly(B)
        imgcol["pixelspacing"] = [1,1]
        imgr = Images.restrict(imgcol, (1,2))
        @fact pixelspacing(imgr) --> [2,2]
        @fact pixelspacing(imgcol) --> [1,1]  # issue #347
        # Issue #395
        img1 = colorim(fill(0.9, 3, 5, 5))
        img2 = colorim(fill(U8(0.9), 3, 5, 5))
        @fact separate(restrict(img1)) --> roughly(separate(restrict(img2)), 0.01)
    end

    context("Erode/ dilate") do
        A = zeros(4,4,3)
        A[2,2,1] = 0.8
        A[4,4,2] = 0.6
        Ae = Images.erode(A)
        @fact Ae --> zeros(size(A))
        Ad = Images.dilate(A)
        Ar = [0.8 0.8 0.8 0;
              0.8 0.8 0.8 0;
              0.8 0.8 0.8 0;
              0 0 0 0]
        Ag = [0 0 0 0;
              0 0 0 0;
              0 0 0.6 0.6;
              0 0 0.6 0.6]
        @fact Ad --> cat(3, Ar, Ag, zeros(4,4))
        Ae = Images.erode(Ad)
        Ar = [0.8 0.8 0 0;
              0.8 0.8 0 0;
              0 0 0 0;
              0 0 0 0]
        Ag = [0 0 0 0;
              0 0 0 0;
              0 0 0 0;
              0 0 0 0.6]
        @fact Ae --> cat(3, Ar, Ag, zeros(4,4))
        # issue #311
        @fact Images.dilate(trues(3)) --> trues(3)
    end

    context("Extrema_filter") do
        # 2d case
        A = zeros(5,5)
        A[2,2] = 0.8
        A[4,4] = 0.6
        minval, maxval = extrema_filter(A, 2)
        matching = [vec(A[2:end]) .!= vec(maxval); false]
        @fact A[reshape(matching, size(A))] --> [0.8, 0.6]
        # 3d case
        A = zeros(5,5,5)
        A[2,2,2] = 0.7
        A[4,4,3] = 0.5
        minval, maxval = extrema_filter(A, 2)
        matching = [vec(A[2:end]) .!= vec(maxval); false]
        @fact A[reshape(matching, size(A))] --> [0.7, 0.5]
        # 4d case
        A = zeros(5,5,5,5)
        A[2,2,2,2] = 0.7
        A[4,4,3,1] = 0.4
        A[3,4,3,2] = 0.5
        minval, maxval = extrema_filter(A, 2)
        matching = [vec(A[2:end]) .!= vec(maxval); false]
        @fact A[reshape(matching, size(A))] --> [0.4,0.7,0.5]
        x, y, z, t = ind2sub(size(A), find(A .== 0.4))
        @fact x[1] --> 4
        @fact y[1] --> 4
        @fact z[1] --> 3
        @fact t[1] --> 1
        # 2d case
        A = rand(5,5)/10
        A[2,2] = 0.8
        A[4,4] = 0.6
        minval, maxval = extrema_filter(A, [2, 2])
        matching = falses(A)
        matching[2:end, 2:end] = maxval .== A[2:end, 2:end]
        @fact sort(A[matching])[end-1:end] --> [0.6, 0.8]
        # 3d case
        A = rand(5,5,5)/10
        A[2,2,2] = 0.7
        A[4,4,2] = 0.4
        A[2,2,4] = 0.5
        minval, maxval = extrema_filter(A, [2, 2, 2])
        matching = falses(A)
        matching[2:end, 2:end, 2:end] = maxval .== A[2:end, 2:end, 2:end]
        @fact sort(A[matching])[end-2:end] --> [0.4, 0.5, 0.7]
        # 4d case
        A = rand(5,5,5,5)/10
        A[2,2,2,2] = 0.7
        A[4,4,2,3] = 0.4
        A[2,2,4,3] = 0.5
        minval, maxval = extrema_filter(A, [2, 2, 2, 2])
        matching = falses(A)
        matching[2:end, 2:end, 2:end, 2:end] = maxval .== A[2:end, 2:end, 2:end, 2:end]
        @fact sort(A[matching])[end-2:end] --> [0.4, 0.5, 0.7]
    end

    context("Opening / closing") do
        A = zeros(4,4,3)
        A[2,2,1] = 0.8
        A[4,4,2] = 0.6
        Ao = Images.opening(A)
        @fact Ao --> zeros(size(A))
        A = zeros(10,10)
        A[4:7,4:7] = 1
        B = copy(A)
        A[5,5] = 0
        Ac = Images.closing(A)
        @fact Ac --> B
    end

    context("Morphological Top-hat") do
        A = zeros(13, 13)
        A[2:3, 2:3] = 1
        Ae = copy(A)
        A[5:9, 5:9] = 1
        Ao = Images.tophat(A)
        @fact Ao --> Ae
        Aoo = Images.tophat(Ae)
        @fact Aoo --> Ae
    end

    context("Morphological Bottom-hat") do
        A = ones(13, 13)
        A[2:3, 2:3] = 0
        Ae = 1 - copy(A)
        A[5:9, 5:9] = 0
        Ao = Images.bothat(A)
        @fact Ao --> Ae
    end

    context("Morphological Gradient") do
        A = zeros(13, 13)
        A[5:9, 5:9] = 1
        Ao = Images.morphogradient(A)
        Ae = zeros(13, 13)
        Ae[4:10, 4:10] = 1
        Ae[6:8, 6:8] = 0
        @fact Ao --> Ae
        Aee = Images.dilate(A) - Images.erode(A)
        @fact Aee --> Ae
    end

    context("Morphological Laplacian") do
        A = zeros(13, 13)
        A[5:9, 5:9] = 1
        Ao = Images.morpholaplace(A)
        Ae = zeros(13, 13)
        Ae[4:10, 4:10] = 1
        Ae[5:9, 5:9] = -1
        Ae[6:8, 6:8] = 0
        @fact Ao --> Ae
        Aee = Images.dilate(A) + Images.erode(A) - 2A
        @fact Aee --> Ae
    end

    context("Label components") do
        A = [true  true  false true;
             true  false true  true]
        lbltarget = [1 1 0 2;
                     1 0 2 2]
        lbltarget1 = [1 2 0 4;
                      1 0 3 4]
        @fact Images.label_components(A) --> lbltarget
        @fact Images.label_components(A, [1]) --> lbltarget1
        connectivity = [false true  false;
                        true  false true;
                        false true  false]
        @fact Images.label_components(A, connectivity) --> lbltarget
        connectivity = trues(3,3)
        lbltarget2 = [1 1 0 1;
                      1 0 1 1]
        @fact Images.label_components(A, connectivity) --> lbltarget2
        @fact component_boxes(lbltarget) --> Vector{Tuple}[[(1,2),(2,3)],[(1,1),(2,2)],[(1,3),(2,4)]]
        @fact component_lengths(lbltarget) --> [2,3,3]
        @fact component_indices(lbltarget) --> Array{Int64}[[4,5],[1,2,3],[6,7,8]]
        @fact component_subscripts(lbltarget) --> Array{Tuple}[[(2,2),(1,3)],[(1,1),(2,1),(1,2)],[(2,3),(1,4),(2,4)]]
        @fact component_centroids(lbltarget) --> Tuple[(1.5,2.5),(4/3,4/3),(5/3,11/3)]
    end

    context("Phantoms") do
        P = [ 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0;
              0.0  0.0  1.0  0.2  0.2  1.0  0.0  0.0;
              0.0  0.0  0.2  0.3  0.3  0.2  0.0  0.0;
              0.0  0.0  0.2  0.0  0.2  0.2  0.0  0.0;
              0.0  0.0  0.2  0.0  0.0  0.2  0.0  0.0;
              0.0  0.0  0.2  0.2  0.2  0.2  0.0  0.0;
              0.0  0.0  1.0  0.2  0.2  1.0  0.0  0.0;
              0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0 ]
        Q = Images.shepp_logan(8)
        @fact norm((P-Q)[:]) --> less_than(1e-10)
        P = [ 0.0  0.0  0.0   0.0   0.0   0.0   0.0  0.0;
              0.0  0.0  2.0   1.02  1.02  2.0   0.0  0.0;
              0.0  0.0  1.02  1.03  1.03  1.02  0.0  0.0;
              0.0  0.0  1.02  1.0   1.02  1.02  0.0  0.0;
              0.0  0.0  1.02  1.0   1.0   1.02  0.0  0.0;
              0.0  0.0  1.02  1.02  1.02  1.02  0.0  0.0;
              0.0  0.0  2.0   1.02  1.02  2.0   0.0  0.0;
              0.0  0.0  0.0   0.0   0.0   0.0   0.0  0.0 ]
        Q = Images.shepp_logan(8, highContrast=false)
        @fact norm((P-Q)[:]) --> less_than(1e-10)
    end

    context("Image resize") do
        img = convert(Images.Image, zeros(10,10))
        img2 = Images.imresize(img, (5,5))
        @fact length(img2) --> 25
    end

    context("Interpolations") do

        img = zeros(Float64, 5, 5)
        @fact bilinear_interpolation(img, 4.5, 5.5) --> 0.0
        @fact bilinear_interpolation(img, 4.5, 3.5) --> 0.0

        for i in [1.0, 2.0, 5.0, 7.0, 9.0]
            img = ones(Float64, 5, 5) * i
            @fact (bilinear_interpolation(img, 3.5, 4.5) == i) --> true
            @fact (bilinear_interpolation(img, 3.2, 4) == i) --> true # X_MAX == X_MIN
            @fact (bilinear_interpolation(img, 3.2, 4) == i) --> true # Y_MAX == Y_MIN
            @fact (bilinear_interpolation(img, 3.2, 4) == i) --> true # BOTH EQUAL
            @fact (bilinear_interpolation(img, 2.8, 1.9) == i) --> true
            # One dim out of bounds
            @fact isapprox(bilinear_interpolation(img, 0.5, 1.5), 0.5 * i) --> true
            @fact isapprox(bilinear_interpolation(img, 0.5, 1.6), 0.5 * i) --> true
            @fact isapprox(bilinear_interpolation(img, 0.5, 1.8), 0.5 * i) --> true
            # Both out of bounds (corner)
            @fact isapprox(bilinear_interpolation(img, 0.5, 0.5), 0.25 * i) --> true
        end

        img = reshape(1.0:1.0:25.0, 5, 5)
        @fact bilinear_interpolation(img, 1.5, 2) --> 6.5
        @fact bilinear_interpolation(img, 2, 1.5) --> 4.5
        @fact bilinear_interpolation(img, 2, 1) --> 2.0
        @fact bilinear_interpolation(img, 1.5, 2.5) --> 9.0
        @fact bilinear_interpolation(img, 1.5, 3.5) --> 14.0
        @fact bilinear_interpolation(img, 1.5, 4.5) --> 19.0
        @fact bilinear_interpolation(img, 1.5, 5.5) --> 10.75

    end

end

ext: jl


text:
using ADCME
using LinearAlgebra
using AdFem
using DelimitedFiles

p = readdlm("fenics/x.txt")[:]
f = readdlm("fenics/f.txt")[:]

mesh = Mesh(8, 8, 1. /8)
f0 = compute_interaction_term(p, mesh)
sess = Session(); init(sess)
f1 = run(sess, f0)

@show norm(f - f1)

p = readdlm("fenics/x2.txt")[:]
f = readdlm("fenics/f2.txt")[:]

mesh = Mesh(8, 8, 1. /8, degree=2)
f0 = compute_interaction_term(p, mesh)
sess = Session(); init(sess)
f1 = run(sess, f0)

E = Int64.(readdlm("fenics/edges.txt"))
Edof = get_edge_dof(E, mesh)
DOF = [1:mesh.nnode; Edof .+ mesh.nnode]
f1 = f1[[DOF; DOF .+ mesh.ndof]]
@show norm(f - f1)


text:
type Grouped{T, I}
    source::T
    group_indices::I
    group_levels
    groupbys
    predicate_aliases
end

type GroupLevels
    levels::Vector{Vector}
end

Base.indices(group_levels::GroupLevels, j) = group_levels.levels[j]

function Base.isequal(grouped1::Grouped, grouped2::Grouped)
    isequal(grouped1.source, grouped2.source) || return false
    isequal(grouped1.group_indices, grouped2.group_indices) || return false
    isequal(grouped1.groupbys, grouped2.groupbys) || return false
    return true
end

function Base.hash(grouped::Grouped)
    h = hash(grouped.source) + 1
    h = hash(grouped.indices, h)
    h = hash(grouped.groupbys, h)
    return @compat UInt(h)
end


text:
@testset "Exchange responses for backtesting"
  @testset "Software-implemented limit order"
    plim = 150.0
    buyord = TradingLogic.Order("buysome", 10, plim,
                                :buy, :limit, :pending, "aux")
    sellord = TradingLogic.Order("sellsome", 10, plim,
                                 :sell, :limit, :pending, "aux")
    @test TradingLogic.plimitcheck(buyord, 175.0) == false
    @test TradingLogic.query_orderstatus(buyord, 175.0) == :pending
    @test TradingLogic.plimitcheck(buyord, 125.0) == true
    @test TradingLogic.query_orderstatus(buyord, 125.0) == :complete
    @test TradingLogic.plimitcheck(sellord, 175.0) == true
    @test TradingLogic.query_orderstatus(sellord, 175.0) == :complete
    @test TradingLogic.plimitcheck(sellord, 125.0) == false
    @test TradingLogic.query_orderstatus(sellord, 125.0) == :pending
  end
  @testset "Market order and order submission"
    ordid = "markord"
    mord = TradingLogic.Order(ordid, 10, NaN,
                              :buy, :market, :pending, "aux")
    # buy no matter how high
    @test TradingLogic.query_orderstatus(mord, 1e7) == :complete
    # simulated order submission
    @test TradingLogic.submit_ordernew(mord, false) == "FAIL"
    @test TradingLogic.submit_ordernew(mord, true) == ordid
  end
  @testset "Cancel pending limit-order"
    limord = TradingLogic.Order("buysome", 10, 100.0,
                                :buy, :limit, :pending, "aux")
    @test TradingLogic.submit_ordercancel(limord) == true
    # can not cancel non-pending order
    @test TradingLogic.submit_ordercancel(
      TradingLogic.emptyorder()) == false
  end
end


text:
using Keras
using Flux
using Base.Test
using DataFlow:Call, constant, vertex, syntax

Keras.new_type(a, b) = Keras.new_type(a, nothing, b)

vcall(a...) = vertex(Call(), constant.(a)...)

a = rand(10,10,1,1)
@testset "Pool tests" begin
    # MaxPool test
    temp = Keras.new_type(:MaxPool, Dict{Any,Any}(Pair{Any,Any}("name", "max_pooling2d_1"),
            Pair{Any,Any}("strides", Any[1, 1]),Pair{Any,Any}("padding", "valid"),
                Pair{Any,Any}("pool_size", Any[2, 2])))
    @test maxpool(a, (2,2), pad=(0,0), stride=(1,1)) ==
                (Keras.ops[:MaxPool](temp))(permutedims(a, (3,2,1,4)))

    # MeanPool test
    temp = Keras.new_type(:MaxPool, Dict{Any,Any}(Pair{Any,Any}("name", "mean_pooling2d_1"),
            Pair{Any,Any}("strides", Any[1, 1]),Pair{Any,Any}("padding", "valid"),
                Pair{Any,Any}("pool_size", Any[2, 2])))
    @test meanpool(a, (2,2), pad=(0,0), stride=(1,1)) ==
                (Keras.ops[:AveragePooling2D](temp))(a)

    # Concatenate tests
    t1, t2 = rand(4,4,4), rand(4,4,4)
    @test vcall(Keras.ops[:Concatenate](temp), 3, t1, t2) |> syntax |> eval ==
                cat(3, t1, t2)

    # GlobalAveragePooling2D
    @test (Keras.ops[:GlobalAveragePooling2D](a))(a) == mean(a, (1,2))

    # Conv test
    #w = rand(3,3,1,32)
    #b = rand(32)
    #temp = Keras.new_type(:Conv, Dict{Any,Any}(Pair{Any,Any}("name", "conv2d_1"),
    #                Pair{Any,Any}("strides", Any[3, 3]),Pair{Any,Any}("kernel_size",
    #                     Any[3, 3]),Pair{Any,Any}("activation", "relu")))
    #global weight = Dict{Any, Any}()
    #weight["conv2d_1"] = Dict{Any, Any}()
    #weight["conv2d_1"]["conv2d_1"] = Dict{Any, Any}()
    #weight["conv2d_1"]["conv2d_1"]["kernel:0"] = w
    #weight["conv2d_1"]["conv2d_1"]["bias:0"] = b
    #@test Conv(relu, w, b, (3,3), (0,0), (1,1))(a) ==
    #            vcall(Keras.ops[:Conv](temp), a) |> syntax |> eval

    ## Workaround for handling global variables.

    # Flatten test
    ip = rand(5,5)
    a = rand(10,10)
    println(vcall(Keras.ops[:Flatten](ip), a) |> syntax |> eval |> size)
    @test vcall(Keras.ops[:Flatten](ip), a) |> syntax |> eval ==
                reshape(permutedims(a, reverse(range(1, ndims(a)))), (length(a),1))

    #Reshape test
    d = Dict{Any, Any}()
    d["target_shape"] = [20,5]
    a = Keras.new_type(:Reshape, d)
    t = rand(10,10)
    @test (Keras.ops[:Reshape](a))(t) == reshape(t, (20,5))
end