<commit_before>{. compile: "pymodpkg/private/pyarrayobject_c.c" .}

# Copyright (c) 2015 SnapDisco Pty Ltd, Australia.
# All rights reserved.
# 
# This source code is licensed under the terms of the MIT license
# found in the "LICENSE" file in the root directory of this source tree.

## Nim wrappers for functions in the Numpy C-API Array API.
##
## Numpy C-API Array API documentation here:
##  http://docs.scipy.org/doc/numpy/reference/c-api.html
##  http://docs.scipy.org/doc/numpy/reference/c-api.array.html
##  http://docs.scipy.org/doc/numpy/reference/c-api.config.html
##  http://docs.scipy.org/doc/numpy/reference/c-api.dtype.html
##  http://docs.scipy.org/doc/numpy/reference/c-api.types-and-structures.html
##
## Most of these functions are preceded by the prefix `PyArray_` in the C-API.
##
## Instead, we prefix the PyArrayObject attribute accessor functions with `get`
## (in place of the `PyArray_` prefix).  For example:
##  - PyArray_NDIM -> getNDIM
##  - PyArray_SHAPE -> getSHAPE
##  - etc.
##
## We prefix PyArrayObject creation functions with `create` (in place of the
## `PyArray_` prefix).  These functions allocate new PyArrayObject instances.
## For example:
##  - PyArray_SimpleNew -> createSimpleNew
##  - PyArray_NewLikeArray -> createNewLikeArray
##
## Additionally, if a PyArrayObject creation function allocates new memory for
## a copy of the underlying data, we suffix the function name with `NewData`.
## For example:
##  - PyArray_NewCopy -> createNewCopyNewData
##
## If a PyArrayObject creation function _might_ allocate new memory for a copy
## of the data, we suffix the function name with `MaybeNewData`.  Rather than
## just returning a `ptr PyArrayObject`, a function like this will return a
## tuple[newArray: ptr PyArrayObject, hasNewData: bool].  For example:
##  - PyArray_NewShape -> createNewShapeMaybeNewData
##
## For functions like `createNewShapeMaybeNewData` that might allocate a copy
## of the underlying data (or might not), we also provide an alternate version
## `createNewShapeNewData` that _always_ allocates a new copy of the underlying
## data.  This alternate version simply returns a `ptr PyArrayObject`.
##
## Finally, if a function doesn't return anything, we prefix its name with `do`.
## For example:
##  - PyArray_CopyInto -> doCopyInto
##  - PyArray_FILLWBYTE -> doFILLWBYTE
##
## The Numpy C functions to create new arrays require both a shape parameter &
## a dtype parameter.
##
## In the Numpy C-API, an array's shape is described using the attribute pair
## `(nd: cint, dims: ptr npy_intp)`.  This pair of attributes is also supplied
## to Numpy C-API array creation functions to specify the shape of a new array.
## Pymod provides the wrapper procs `arr.getNDIM()` & `arr.getDIMS()` to access
## these attributes.
##
## However, it is not necessary to use these two wrapper procs; Pymod makes it
## easier to specify a shape parameter.  In Pymod, there are 2 ways to specify
## a shape parameter:
##  1. `CArrayProxy[npy_intp]`, a simple Nim wrapper type around
##     `(nd: cint, dims: ptr npy_intp)`.  A `CArrayProxy[npy_intp]` instance
##     is returned by the PyArrayObject accessors `.dimensions`, `strides` &
##     `.shape`.
##  2. `openArray[int]`, so you can supply Nim array literals; eg, `[5, 4, 3]`.
##
## There are 4 ways to specify a dtype parameter:
##  1, `dtype: ptr PyArrayDescr`, as per the Numpy C-API.
##  2. `typenum: cint`, as per the Numpy C-API.
##  3. `typenum: CNpyTypes`, a Nim enum that matches the C-API enum `NPY_TYPES`
##     in `<numpy/ndarraytypes.h>`; eg: `CNpyTypes.NPY_UBYTE`.
##  4. `nptype: NpType`, a Nim enum that contains *only* the dtypes that Pymod
##     supports; eg: `np_uint8`.  This is the recommended method in Pymod.
##
## Any proc that either allocates a new PyObject that must be memory-managed
## (such as a PyArrayObject), or could raise an exception, will be provided as
## a template wrapper around an implementation proc.  This enables us to use
## the template's instantiation info in debugging info and exception traces.
##
## Here's the full list of procs & templates:
##
##  - getDescrFromType(typenum: cint): ptr PyArrayDescr
##  - getDescrFromType(typenum: CNpyTypes): ptr PyArrayDescr
##  - getDescrFromType(nptype: NpType): ptr PyArrayDescr
##
##  - canCastArrayTo(arr: ptr PyArrayObject, toType: ptr PyArray_Descr,
##        casting: NpyCastingAlternatives): bool
##
##  - createNewLikeArray(prototype: ptr PyArrayObject, order: NpyOrderAlternatives,
##        dtype: ptr PyArrayDescr, subOk: bool): ptr PyArrayObject
##
##  - createSimpleNew(dims: CArrayProxy[npy_intp], nptype: NpType): ptr PyArrayObject
##  - createSimpleNew(dims: openarray[int], nptype: NpType): ptr PyArrayObject
##
##  - createNewCopyNewData(old: ptr PyArrayObject, order: NpyOrderAlternatives): ptr PyArrayObject
##  - copy(old: ptr PyArrayObject): ptr PyArrayObject  # an alias for `createNewCopyNewData`
##  - createAsTypeNewData(old: ptr PyArrayObject, newtype: NpType): ptr PyArrayObject
##
##  - doCopyInto(dest: ptr PyArrayObject, src: ptr PyArrayObject)
##  - doFILLWBYTE(arr: ptr PyArrayObject, val: cint)
##
##  - doResizeDataInplace(old: ptr PyArrayObject, newShape: openarray[int], doRefCheck: bool=true)
##  - doResizeDataInplaceNumRows(old: ptr PyArrayObject, newNumRows: int, doRefCheck: bool=true)


## New procs to add:
##
##  - createNewShapeMaybeNewData(old: ptr PyArrayObject, newShape: openarray[int],
##        order: NpyOrderAlternatives): tuple[newArray: ptr PyArrayObject, hasNewData: bool]
##  - createNewShapeNewData(old: ptr PyArrayObject, newShape: openarray[int],
##        order: NpyOrderAlternatives): ptr PyArrayObject

##  - createSwapAxes(old: ptr PyArrayObject, axis1: cint, axis2: cint): ptr PyArrayObject
##  - createTranspose(old: ptr PyArrayObject): ptr PyArrayObject
##  - createTranspose(old: ptr PyArrayObject, permute: openarray[int]): ptr PyArrayObject
##  - createFlattenNewData(arr: ptr PyArrayObject,
##        order: NpyOrderAlternatives): ptr PyArrayObject
##  - createRavelMaybeNewData(arr: ptr PyArrayObject,
##        order: NpyOrderAlternatives): tuple[newArray: ptr PyArrayObject, hasNewData: bool]

##  - createEmpty(nd: cint, dims: ptr npy_intp, dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(nd: cint, dims: ptr npy_intp, typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(nd: cint, dims: ptr npy_intp, typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(nd: cint, dims: ptr npy_intp, nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject
##
##  - createEmpty(dims: CArrayProxy[npy_intp], dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(dims: CArrayProxy[npy_intp], typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(dims: CArrayProxy[npy_intp], typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(dims: CArrayProxy[npy_intp], nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject
##
##  - createEmpty(dims: openarray[int], dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(dims: openarray[int], typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(dims: openarray[int], typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(dims: openarray[int], nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject
##
##  - createZeros(nd: cint, dims: ptr npy_intp, dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(nd: cint, dims: ptr npy_intp, typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(nd: cint, dims: ptr npy_intp, typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(nd: cint, dims: ptr npy_intp, nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject
##
##  - createZeros(dims: CArrayProxy[npy_intp], dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(dims: CArrayProxy[npy_intp], typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(dims: CArrayProxy[npy_intp], typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(dims: CArrayProxy[npy_intp], nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject
##
##  - createZeros(dims: openarray[int], dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(dims: openarray[int], typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(dims: openarray[int], typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(dims: openarray[int], nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject
##
##  - createFull(nd: cint, dims: ptr npy_intp, fillValue: double, dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(nd: cint, dims: ptr npy_intp, fillValue: double, typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(nd: cint, dims: ptr npy_intp, fillValue: double, typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(nd: cint, dims: ptr npy_intp, fillValue: double, nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject
##
##  - createFull(nd: cint, dims: CArrayProxy[npy_intp], fillValue: double, dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(nd: cint, dims: CArrayProxy[npy_intp], fillValue: double, typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(nd: cint, dims: CArrayProxy[npy_intp], fillValue: double, typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(nd: cint, dims: CArrayProxy[npy_intp], fillValue: double, nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject
##
##  - createFull(dims: openarray[int], fillValue: double, dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(dims: openarray[int], fillValue: double, typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(dims: openarray[int], fillValue: double, typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(dims: openarray[int], fillValue: double, nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject

##  - createEmptyLike(prototype: ptr PyArrayObject,
##        dtype: ptr PyArrayDescr=nil,
##        order: NpyOrderAlternatives=NpyOrderAlternatives.keeporder,
##        subOk: bool=true):
##            ptr PyArrayObject
##
##  - createZerosLike(prototype: ptr PyArrayObject,
##        dtype: ptr PyArrayDescr=nil,
##        order: NpyOrderAlternatives=NpyOrderAlternatives.keeporder,
##        subOk: bool=true):
##            ptr PyArrayObject
##
##  - createFullLike(prototype: ptr PyArrayObject,
##        fillValue: double,
##        dtype: ptr PyArrayDescr=nil,
##        order: NpyOrderAlternatives=NpyOrderAlternatives.keeporder,
##        subOk: bool=true):
##            ptr PyArrayObject


import strutils
import typetraits  # name(t: typedesc)

import pymodpkg/miscutils
import pymodpkg/pyobject

import pymodpkg/private/membrain

import pymodpkg/private/nptypes
export nptypes.NumpyCompatibleNimType
export nptypes.NpType
export nptypes.toNpType

import pymodpkg/private/cnpytypes
export cnpytypes.CNpyTypes
export cnpytypes.toNpType
export cnpytypes.toCNpyTypes

import pymodpkg/private/pyarraydescrtype
export pyarraydescrtype.PyArrayDescr
export pyarraydescrtype.dkind
export pyarraydescrtype.dtype
export pyarraydescrtype.byteorder
export pyarraydescrtype.dflags
export pyarraydescrtype.type_num
export pyarraydescrtype.elsize
export pyarraydescrtype.alignment

import pymodpkg/private/pyarrayenums
export pyarrayenums.NpyOrderAlternatives
export pyarrayenums.NpyCastingAlternatives

import pymodpkg/private/pyarrayflags
export pyarrayflags.NPY_ARRAY_BEHAVED
export pyarrayflags.NPY_ARRAY_CARRAY
export pyarrayflags.NPY_ARRAY_CARRAY_RO
export pyarrayflags.PyArrayFlags
export pyarrayflags.flags

import pymodpkg/private/pyarrayobjecttype
export pyarrayobjecttype.PyArrayObject
export pyarrayobjecttype.npy_intp
export pyarrayobjecttype.CArrayProxy
export pyarrayobjecttype.getLen
export pyarrayobjecttype.getPtr
export pyarrayobjecttype.items
export pyarrayobjecttype.`[]`
export pyarrayobjecttype.`$`

export pyarrayobjecttype.getNDIM
export pyarrayobjecttype.getDIMS
export pyarrayobjecttype.getSHAPE
export pyarrayobjecttype.getDATA
export pyarrayobjecttype.getSTRIDES
export pyarrayobjecttype.getDIM
export pyarrayobjecttype.getSTRIDE
export pyarrayobjecttype.getDESCR
export pyarrayobjecttype.getDTYPE
export pyarrayobjecttype.getFLAGS
export pyarrayobjecttype.getITEMSIZE
export pyarrayobjecttype.getTYPE
export pyarrayobjecttype.getPtr

export pyarrayobjecttype.data
export pyarrayobjecttype.nd
export pyarrayobjecttype.ndim
export pyarrayobjecttype.enumerateDimensions
export pyarrayobjecttype.dimensions
export pyarrayobjecttype.elcount
export pyarrayobjecttype.shape
export pyarrayobjecttype.enumerateStrides
export pyarrayobjecttype.strides
export pyarrayobjecttype.descr
export pyarrayobjecttype.dtype

import pymodpkg/private/pyarrayiters
export pyarrayiters.PyArrayForwardIter
export pyarrayiters.`[]`
export pyarrayiters.`[]=`
export pyarrayiters.inc
export pyarrayiters.derefInc
export pyarrayiters.incFast
export pyarrayiters.PyArrayRandAccIter
export pyarrayiters.PyArrayIterBounds
export pyarrayiters.contains
export pyarrayiters.dec
export pyarrayiters.`+`
export pyarrayiters.`-`
export pyarrayiters.`==`
export pyarrayiters.`!=`
export pyarrayiters.`<=`
export pyarrayiters.`<`


## A convenient and plausible maximum number of dimensions to support.
## (This is Numpy's internal limit.)
const NPY_MAXDIMS = 32


## Allow implicit conversion from `ptr PyArrayObject` to `ptr PyObject`
## (since we have that pesky "strong static typing" thing in Nim).
##  http://nim-lang.org/manual.html#converters
##  http://nim-lang.org/manual.html#convertible-relation
converter toPyObject*(obj: ptr PyArrayObject): ptr PyObject = cast[ptr PyObject](obj)
converter toPyObject*(obj: ptr PyArrayDescr): ptr PyObject = cast[ptr PyObject](obj)


# http://nim-lang.org/system.html#instantiationInfo,
type InstantiationInfoTuple = tuple[filename: string, line: int]


proc assertArrayType*(obj: ptr PyArrayObject, NT: typedesc[NumpyCompatibleNimType],
    ii: InstantiationInfoTuple, procname: string) =
  let obj_dtype = obj.dtype  # This requires a lookup in a case statement.
  if toNpType(NT) != obj_dtype:
    let msg = "$1: PyArrayObject supplied dtype `$2` does not match specified Nim type `$3` [File \"$4\", line $5]" %
        [procname, $obj_dtype, NT.name, ii.filename, $ii.line]
    raise newException(ObjectConversionError, msg)


proc assertArrayNdim*(obj: ptr PyArrayObject, expected_nd: Positive,
    ii: InstantiationInfoTuple, procname: string) =
  let obj_nd = obj.nd
  if expected_nd != obj_nd:
    let msg = "$1: PyArrayObject supplied ndim (== $2) does not match specified ndim (== $3) [File \"$4\", line $5]" %
        [procname, $obj_nd, $expected_nd, ii.filename, $ii.line]
    raise newException(ObjectConversionError, msg)


proc assertArrayCContigForIterator*(obj: ptr PyArrayObject,
    ii: InstantiationInfoTuple, procname: string) =
  let is_c_contig: bool = flagBitIsOn(getFLAGS(obj), c_contiguous)
  if not is_c_contig:
    let msg = "$1: PyArrayObject iterator can only be used with C-contiguous data [File \"$2\", line $3]" %
        [procname, ii.filename, $ii.line]
    raise newException(AssertionError, msg)


proc iterateFlatImpl(arr: ptr PyArrayObject, NimT: typedesc[NumpyCompatibleNimType],
    ii: InstantiationInfoTuple, procname: string{lit}):
    PyArrayForwardIter[NimT] =
  assertArrayType(arr, NimT, ii, procname)
  assertArrayCContigForIterator(arr, ii, procname)
  result = initPyArrayForwardIter[NimT](arr)


template iterateFlat*(arr: ptr PyArrayObject, NimT: typedesc[NumpyCompatibleNimType]):
    PyArrayForwardIter[NimT] =
  ## Return a PyArrayForwardIter over type `NimT`.
  ##
  ## A PyArrayForwardIter is an iterator that can only step forward in
  ## single increments.  That is, it can only be incremented and dereferenced.
  ##
  ## Due to the limited manner in which the PyArrayForwardIter's position
  ## can be changed, a PyArrayForwardIter is slightly more predictable than
  ## a PyArrayRandAccIter.  So prefer to use PyArrayForwardIter
  ## if you can.
  ##
  ## NOTE:  This proc requires that the PyArrayObject data is C-contiguous;
  ## else, an AssertionError will be raised.
  ##
  ## Here's an example of how you use this type of iterator:
  ##
  ##   let dt = arr.dtype
  ##   if dt == np_int32:
  ##       let bounds = arr.getBounds(int32)
  ##       var iter = arr.iterateFlat(int32)
  ##       while iter in bounds:
  ##           doSomethingWith(iter[])
  ##           inc(iter)
  ##

  # http://nim-lang.org/system.html#instantiationInfo,
  let ii = instantiationInfo()
  iterateFlatImpl(arr, NimT, ii, "iterateFlat")


iterator items*[T](iter: PyArrayForwardIter[T]):
    PyArrayForwardIter[T] {.inline.} =
  let bounds = iter.getBounds()
  var iter = iter
  while iter in bounds:
    yield iter
    inc(iter)


iterator values*(arr: ptr PyArrayObject, NimT: typedesc[NumpyCompatibleNimType]):
    NimT {.inline.} =
  let bounds = arr.getBounds(NimT)
  var iter = arr.iterateFlat(NimT)
  while iter in bounds:
    yield iter[]
    inc(iter)


proc accessFlatImpl(arr: ptr PyArrayObject; NimT: typedesc[NumpyCompatibleNimType];
    ii: InstantiationInfoTuple; procname: string{lit}; initOffset, incDelta: int):
    PyArrayRandAccIter[NimT] =
  assertArrayType(arr, NimT, ii, procname)
  assertArrayCContigForIterator(arr, ii, procname)
  result = initPyArrayRandAccIter[NimT](arr, initOffset, incDelta)<commit_msg>Added two inline iterators, both called `iterateFlatFast`.<commit_after>{. compile: "pymodpkg/private/pyarrayobject_c.c" .}

# Copyright (c) 2015 SnapDisco Pty Ltd, Australia.
# All rights reserved.
# 
# This source code is licensed under the terms of the MIT license
# found in the "LICENSE" file in the root directory of this source tree.

## Nim wrappers for functions in the Numpy C-API Array API.
##
## Numpy C-API Array API documentation here:
##  http://docs.scipy.org/doc/numpy/reference/c-api.html
##  http://docs.scipy.org/doc/numpy/reference/c-api.array.html
##  http://docs.scipy.org/doc/numpy/reference/c-api.config.html
##  http://docs.scipy.org/doc/numpy/reference/c-api.dtype.html
##  http://docs.scipy.org/doc/numpy/reference/c-api.types-and-structures.html
##
## Most of these functions are preceded by the prefix `PyArray_` in the C-API.
##
## Instead, we prefix the PyArrayObject attribute accessor functions with `get`
## (in place of the `PyArray_` prefix).  For example:
##  - PyArray_NDIM -> getNDIM
##  - PyArray_SHAPE -> getSHAPE
##  - etc.
##
## We prefix PyArrayObject creation functions with `create` (in place of the
## `PyArray_` prefix).  These functions allocate new PyArrayObject instances.
## For example:
##  - PyArray_SimpleNew -> createSimpleNew
##  - PyArray_NewLikeArray -> createNewLikeArray
##
## Additionally, if a PyArrayObject creation function allocates new memory for
## a copy of the underlying data, we suffix the function name with `NewData`.
## For example:
##  - PyArray_NewCopy -> createNewCopyNewData
##
## If a PyArrayObject creation function _might_ allocate new memory for a copy
## of the data, we suffix the function name with `MaybeNewData`.  Rather than
## just returning a `ptr PyArrayObject`, a function like this will return a
## tuple[newArray: ptr PyArrayObject, hasNewData: bool].  For example:
##  - PyArray_NewShape -> createNewShapeMaybeNewData
##
## For functions like `createNewShapeMaybeNewData` that might allocate a copy
## of the underlying data (or might not), we also provide an alternate version
## `createNewShapeNewData` that _always_ allocates a new copy of the underlying
## data.  This alternate version simply returns a `ptr PyArrayObject`.
##
## Finally, if a function doesn't return anything, we prefix its name with `do`.
## For example:
##  - PyArray_CopyInto -> doCopyInto
##  - PyArray_FILLWBYTE -> doFILLWBYTE
##
## The Numpy C functions to create new arrays require both a shape parameter &
## a dtype parameter.
##
## In the Numpy C-API, an array's shape is described using the attribute pair
## `(nd: cint, dims: ptr npy_intp)`.  This pair of attributes is also supplied
## to Numpy C-API array creation functions to specify the shape of a new array.
## Pymod provides the wrapper procs `arr.getNDIM()` & `arr.getDIMS()` to access
## these attributes.
##
## However, it is not necessary to use these two wrapper procs; Pymod makes it
## easier to specify a shape parameter.  In Pymod, there are 2 ways to specify
## a shape parameter:
##  1. `CArrayProxy[npy_intp]`, a simple Nim wrapper type around
##     `(nd: cint, dims: ptr npy_intp)`.  A `CArrayProxy[npy_intp]` instance
##     is returned by the PyArrayObject accessors `.dimensions`, `strides` &
##     `.shape`.
##  2. `openArray[int]`, so you can supply Nim array literals; eg, `[5, 4, 3]`.
##
## There are 4 ways to specify a dtype parameter:
##  1, `dtype: ptr PyArrayDescr`, as per the Numpy C-API.
##  2. `typenum: cint`, as per the Numpy C-API.
##  3. `typenum: CNpyTypes`, a Nim enum that matches the C-API enum `NPY_TYPES`
##     in `<numpy/ndarraytypes.h>`; eg: `CNpyTypes.NPY_UBYTE`.
##  4. `nptype: NpType`, a Nim enum that contains *only* the dtypes that Pymod
##     supports; eg: `np_uint8`.  This is the recommended method in Pymod.
##
## Any proc that either allocates a new PyObject that must be memory-managed
## (such as a PyArrayObject), or could raise an exception, will be provided as
## a template wrapper around an implementation proc.  This enables us to use
## the template's instantiation info in debugging info and exception traces.
##
## Here's the full list of procs & templates:
##
##  - getDescrFromType(typenum: cint): ptr PyArrayDescr
##  - getDescrFromType(typenum: CNpyTypes): ptr PyArrayDescr
##  - getDescrFromType(nptype: NpType): ptr PyArrayDescr
##
##  - canCastArrayTo(arr: ptr PyArrayObject, toType: ptr PyArray_Descr,
##        casting: NpyCastingAlternatives): bool
##
##  - createNewLikeArray(prototype: ptr PyArrayObject, order: NpyOrderAlternatives,
##        dtype: ptr PyArrayDescr, subOk: bool): ptr PyArrayObject
##
##  - createSimpleNew(dims: CArrayProxy[npy_intp], nptype: NpType): ptr PyArrayObject
##  - createSimpleNew(dims: openarray[int], nptype: NpType): ptr PyArrayObject
##
##  - createNewCopyNewData(old: ptr PyArrayObject, order: NpyOrderAlternatives): ptr PyArrayObject
##  - copy(old: ptr PyArrayObject): ptr PyArrayObject  # an alias for `createNewCopyNewData`
##  - createAsTypeNewData(old: ptr PyArrayObject, newtype: NpType): ptr PyArrayObject
##
##  - doCopyInto(dest: ptr PyArrayObject, src: ptr PyArrayObject)
##  - doFILLWBYTE(arr: ptr PyArrayObject, val: cint)
##
##  - doResizeDataInplace(old: ptr PyArrayObject, newShape: openarray[int], doRefCheck: bool=true)
##  - doResizeDataInplaceNumRows(old: ptr PyArrayObject, newNumRows: int, doRefCheck: bool=true)


## New procs to add:
##
##  - createNewShapeMaybeNewData(old: ptr PyArrayObject, newShape: openarray[int],
##        order: NpyOrderAlternatives): tuple[newArray: ptr PyArrayObject, hasNewData: bool]
##  - createNewShapeNewData(old: ptr PyArrayObject, newShape: openarray[int],
##        order: NpyOrderAlternatives): ptr PyArrayObject

##  - createSwapAxes(old: ptr PyArrayObject, axis1: cint, axis2: cint): ptr PyArrayObject
##  - createTranspose(old: ptr PyArrayObject): ptr PyArrayObject
##  - createTranspose(old: ptr PyArrayObject, permute: openarray[int]): ptr PyArrayObject
##  - createFlattenNewData(arr: ptr PyArrayObject,
##        order: NpyOrderAlternatives): ptr PyArrayObject
##  - createRavelMaybeNewData(arr: ptr PyArrayObject,
##        order: NpyOrderAlternatives): tuple[newArray: ptr PyArrayObject, hasNewData: bool]

##  - createEmpty(nd: cint, dims: ptr npy_intp, dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(nd: cint, dims: ptr npy_intp, typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(nd: cint, dims: ptr npy_intp, typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(nd: cint, dims: ptr npy_intp, nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject
##
##  - createEmpty(dims: CArrayProxy[npy_intp], dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(dims: CArrayProxy[npy_intp], typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(dims: CArrayProxy[npy_intp], typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(dims: CArrayProxy[npy_intp], nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject
##
##  - createEmpty(dims: openarray[int], dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(dims: openarray[int], typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(dims: openarray[int], typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createEmpty(dims: openarray[int], nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject
##
##  - createZeros(nd: cint, dims: ptr npy_intp, dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(nd: cint, dims: ptr npy_intp, typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(nd: cint, dims: ptr npy_intp, typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(nd: cint, dims: ptr npy_intp, nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject
##
##  - createZeros(dims: CArrayProxy[npy_intp], dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(dims: CArrayProxy[npy_intp], typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(dims: CArrayProxy[npy_intp], typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(dims: CArrayProxy[npy_intp], nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject
##
##  - createZeros(dims: openarray[int], dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(dims: openarray[int], typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(dims: openarray[int], typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createZeros(dims: openarray[int], nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject
##
##  - createFull(nd: cint, dims: ptr npy_intp, fillValue: double, dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(nd: cint, dims: ptr npy_intp, fillValue: double, typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(nd: cint, dims: ptr npy_intp, fillValue: double, typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(nd: cint, dims: ptr npy_intp, fillValue: double, nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject
##
##  - createFull(nd: cint, dims: CArrayProxy[npy_intp], fillValue: double, dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(nd: cint, dims: CArrayProxy[npy_intp], fillValue: double, typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(nd: cint, dims: CArrayProxy[npy_intp], fillValue: double, typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(nd: cint, dims: CArrayProxy[npy_intp], fillValue: double, nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject
##
##  - createFull(dims: openarray[int], fillValue: double, dtype: ptr PyArrayDescr,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(dims: openarray[int], fillValue: double, typenum: cint,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(dims: openarray[int], fillValue: double, typenum: CNpyTypes,
##        isFortranOrder: bool=false): ptr PyArrayObject
##  - createFull(dims: openarray[int], fillValue: double, nptype: NpType,
##        isFortranOrder: bool=false): ptr PyArrayObject

##  - createEmptyLike(prototype: ptr PyArrayObject,
##        dtype: ptr PyArrayDescr=nil,
##        order: NpyOrderAlternatives=NpyOrderAlternatives.keeporder,
##        subOk: bool=true):
##            ptr PyArrayObject
##
##  - createZerosLike(prototype: ptr PyArrayObject,
##        dtype: ptr PyArrayDescr=nil,
##        order: NpyOrderAlternatives=NpyOrderAlternatives.keeporder,
##        subOk: bool=true):
##            ptr PyArrayObject
##
##  - createFullLike(prototype: ptr PyArrayObject,
##        fillValue: double,
##        dtype: ptr PyArrayDescr=nil,
##        order: NpyOrderAlternatives=NpyOrderAlternatives.keeporder,
##        subOk: bool=true):
##            ptr PyArrayObject


import strutils
import typetraits  # name(t: typedesc)

import pymodpkg/miscutils
import pymodpkg/pyobject

import pymodpkg/private/membrain

import pymodpkg/private/nptypes
export nptypes.NumpyCompatibleNimType
export nptypes.NpType
export nptypes.toNpType

import pymodpkg/private/cnpytypes
export cnpytypes.CNpyTypes
export cnpytypes.toNpType
export cnpytypes.toCNpyTypes

import pymodpkg/private/pyarraydescrtype
export pyarraydescrtype.PyArrayDescr
export pyarraydescrtype.dkind
export pyarraydescrtype.dtype
export pyarraydescrtype.byteorder
export pyarraydescrtype.dflags
export pyarraydescrtype.type_num
export pyarraydescrtype.elsize
export pyarraydescrtype.alignment

import pymodpkg/private/pyarrayenums
export pyarrayenums.NpyOrderAlternatives
export pyarrayenums.NpyCastingAlternatives

import pymodpkg/private/pyarrayflags
export pyarrayflags.NPY_ARRAY_BEHAVED
export pyarrayflags.NPY_ARRAY_CARRAY
export pyarrayflags.NPY_ARRAY_CARRAY_RO
export pyarrayflags.PyArrayFlags
export pyarrayflags.flags

import pymodpkg/private/pyarrayobjecttype
export pyarrayobjecttype.PyArrayObject
export pyarrayobjecttype.npy_intp
export pyarrayobjecttype.CArrayProxy
export pyarrayobjecttype.getLen
export pyarrayobjecttype.getPtr
export pyarrayobjecttype.items
export pyarrayobjecttype.`[]`
export pyarrayobjecttype.`$`

export pyarrayobjecttype.getNDIM
export pyarrayobjecttype.getDIMS
export pyarrayobjecttype.getSHAPE
export pyarrayobjecttype.getDATA
export pyarrayobjecttype.getSTRIDES
export pyarrayobjecttype.getDIM
export pyarrayobjecttype.getSTRIDE
export pyarrayobjecttype.getDESCR
export pyarrayobjecttype.getDTYPE
export pyarrayobjecttype.getFLAGS
export pyarrayobjecttype.getITEMSIZE
export pyarrayobjecttype.getTYPE
export pyarrayobjecttype.getPtr

export pyarrayobjecttype.data
export pyarrayobjecttype.nd
export pyarrayobjecttype.ndim
export pyarrayobjecttype.enumerateDimensions
export pyarrayobjecttype.dimensions
export pyarrayobjecttype.elcount
export pyarrayobjecttype.shape
export pyarrayobjecttype.enumerateStrides
export pyarrayobjecttype.strides
export pyarrayobjecttype.descr
export pyarrayobjecttype.dtype

import pymodpkg/private/pyarrayiters
export pyarrayiters.PyArrayForwardIter
export pyarrayiters.`[]`
export pyarrayiters.`[]=`
export pyarrayiters.inc
export pyarrayiters.derefInc
export pyarrayiters.incFast
export pyarrayiters.PyArrayRandAccIter
export pyarrayiters.PyArrayIterBounds
export pyarrayiters.contains
export pyarrayiters.dec
export pyarrayiters.`+`
export pyarrayiters.`-`
export pyarrayiters.`==`
export pyarrayiters.`!=`
export pyarrayiters.`<=`
export pyarrayiters.`<`


## A convenient and plausible maximum number of dimensions to support.
## (This is Numpy's internal limit.)
const NPY_MAXDIMS = 32


## Allow implicit conversion from `ptr PyArrayObject` to `ptr PyObject`
## (since we have that pesky "strong static typing" thing in Nim).
##  http://nim-lang.org/manual.html#converters
##  http://nim-lang.org/manual.html#convertible-relation
converter toPyObject*(obj: ptr PyArrayObject): ptr PyObject = cast[ptr PyObject](obj)
converter toPyObject*(obj: ptr PyArrayDescr): ptr PyObject = cast[ptr PyObject](obj)


# http://nim-lang.org/system.html#instantiationInfo,
type InstantiationInfoTuple = tuple[filename: string, line: int]


proc assertArrayType*(obj: ptr PyArrayObject, NT: typedesc[NumpyCompatibleNimType],
    ii: InstantiationInfoTuple, procname: string) =
  let obj_dtype = obj.dtype  # This requires a lookup in a case statement.
  if toNpType(NT) != obj_dtype:
    let msg = "$1: PyArrayObject supplied dtype `$2` does not match specified Nim type `$3` [File \"$4\", line $5]" %
        [procname, $obj_dtype, NT.name, ii.filename, $ii.line]
    raise newException(ObjectConversionError, msg)


proc assertArrayNdim*(obj: ptr PyArrayObject, expected_nd: Positive,
    ii: InstantiationInfoTuple, procname: string) =
  let obj_nd = obj.nd
  if expected_nd != obj_nd:
    let msg = "$1: PyArrayObject supplied ndim (== $2) does not match specified ndim (== $3) [File \"$4\", line $5]" %
        [procname, $obj_nd, $expected_nd, ii.filename, $ii.line]
    raise newException(ObjectConversionError, msg)


proc assertArrayCContigForIterator*(obj: ptr PyArrayObject,
    ii: InstantiationInfoTuple, procname: string) =
  let is_c_contig: bool = flagBitIsOn(getFLAGS(obj), c_contiguous)
  if not is_c_contig:
    let msg = "$1: PyArrayObject iterator can only be used with C-contiguous data [File \"$2\", line $3]" %
        [procname, ii.filename, $ii.line]
    raise newException(AssertionError, msg)


proc iterateFlatImpl(arr: ptr PyArrayObject, NimT: typedesc[NumpyCompatibleNimType],
    ii: InstantiationInfoTuple, procname: string{lit}):
    PyArrayForwardIter[NimT] =
  assertArrayType(arr, NimT, ii, procname)
  assertArrayCContigForIterator(arr, ii, procname)
  result = initPyArrayForwardIter[NimT](arr)


template iterateFlat*(arr: ptr PyArrayObject, NimT: typedesc[NumpyCompatibleNimType]):
    PyArrayForwardIter[NimT] =
  ## Return a PyArrayForwardIter over type `NimT`.
  ##
  ## A PyArrayForwardIter is an iterator that can only step forward in
  ## single increments.  That is, it can only be incremented and dereferenced.
  ##
  ## Due to the limited manner in which the PyArrayForwardIter's position
  ## can be changed, a PyArrayForwardIter is slightly more predictable than
  ## a PyArrayRandAccIter.  So prefer to use PyArrayForwardIter
  ## if you can.
  ##
  ## NOTE:  This proc requires that the PyArrayObject data is C-contiguous;
  ## else, an AssertionError will be raised.
  ##
  ## Here's an example of how you use this type of iterator:
  ##
  ##   let dt = arr.dtype
  ##   if dt == np_int32:
  ##       let bounds = arr.getBounds(int32)
  ##       var iter = arr.iterateFlat(int32)
  ##       while iter in bounds:
  ##           doSomethingWith(iter[])
  ##           inc(iter)
  ##

  # http://nim-lang.org/system.html#instantiationInfo,
  let ii = instantiationInfo()
  iterateFlatImpl(arr, NimT, ii, "iterateFlat")


iterator items*[T](iter: PyArrayForwardIter[T]):
    PyArrayForwardIter[T] {.inline.} =
  let bounds = iter.getBounds()
  var iter = iter
  while iter in bounds:
    yield iter
    inc(iter)

iterator iterateFlatFast*[T](arr: ptr PyArrayObject; NimT: typedesc[NumpyCompatibleNimType];
    positiveDelta: Positive): PyArrayForwardIter[T] {.inline.} =
  ## "Fast forward"
  # http://nim-lang.org/system.html#instantiationInfo,
  let ii = instantiationInfo()
  let bounds = arr.getBounds()
  var iter = arr.iterateFlatImpl(NimT, ii, "iterateFlatFast")
  while iter in bounds:
    yield iter
    incFast(iter, positiveDelta)

iterator iterateFlatFast*[T](arr: ptr PyArrayObject; NimT: typedesc[NumpyCompatibleNimType];
    positiveOffset, positiveDelta: Positive): PyArrayForwardIter[T] {.inline.} =
  ## "Fast forward"
  # http://nim-lang.org/system.html#instantiationInfo,
  let ii = instantiationInfo()
  let bounds = arr.getBounds()
  var iter = arr.iterateFlatImpl(NimT, ii, "iterateFlatFast")
  incFast(iter, positiveOffset)
  while iter in bounds:
    yield iter
    incFast(iter, positiveDelta)

iterator values*(arr: ptr PyArrayObject, NimT: typedesc[NumpyCompatibleNimType]):
    NimT {.inline.} =
  let bounds = arr.getBounds(NimT)
  var iter = arr.iterateFlat(NimT)
  while iter in bounds:
    yield iter[]
    inc(iter)


proc accessFlatImpl(arr: ptr PyArrayObject; NimT: typedesc[NumpyCompatibleNimType];
    ii: InstantiationInfoTuple; procname: string{lit}; initOffset, incDelta: int):
    PyArrayRandAccIter[NimT] =
  assertArrayType(arr, NimT, ii, procname)
  assertArrayCContigForIterator(arr, ii, procname)
  result = initPyArrayRandAccIter[NimT](arr, initOffset, incDelta)
{'fraction_non_alphanumeric': 0.11025422827358791, 'fraction_numerical': 0.0020742474204871824, 'mean_word_length': 4.87027786450203, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2457713', 'n_tokens_mistral': 12511, 'n_tokens_neox': 12289, 'n_words': 3318}