<commit_before>from libcpp.vector cimport vector
from libcpp cimport bool

from primitiv.device cimport wrapDevice, _Device
from primitiv.shape cimport _Shape, normShape
from primitiv.graph cimport _Graph, wrapNode, Node, _Node
from primitiv.parameter cimport _Parameter

cimport numpy as np
import numpy as np

cdef inline vector[float] ndarrays_to_vector(list arrays):
    cdef vector[float] result
    cdef np.float32_t *np_data
    cdef unsigned datasize
    cdef np.ndarray data_tmp
    if len(arrays) == 0:
        raise TypeError("arrays contains no item")
    datasize = arrays[0].size
    shape = arrays[0].shape
    result.resize(len(arrays) * datasize)
    for j, data in enumerate(arrays):
        if shape != data.shape:
            raise TypeError("arrays contains different shaped ndarrays")
        data_tmp = np.array(data, dtype=np.float32, order="F")
        np_data = <np.float32_t *> data_tmp.data
        for i in range(datasize):
            result[j * datasize + i] = np_data[i]
    return result

cdef inline vector[unsigned] ndarray_to_vector_unsigned(np.ndarray array):
    cdef vector[unsigned] result
    cdef unsigned *np_data
    cdef unsigned datasize
    cdef np.ndarray array_tmp

    if array.dtype != np.uint32:
        raise TypeError("numpy.ndarray must be constructed from uint32 data")

    datasize = array.size<commit_msg>Specify type of iterator variables<commit_after>from libcpp.vector cimport vector
from libcpp cimport bool

from primitiv.device cimport wrapDevice, _Device
from primitiv.shape cimport _Shape, normShape
from primitiv.graph cimport _Graph, wrapNode, Node, _Node
from primitiv.parameter cimport _Parameter

cimport numpy as np
import numpy as np

cdef inline vector[float] ndarrays_to_vector(list arrays):
    cdef vector[float] result
    cdef np.float32_t *np_data
    cdef unsigned datasize
    cdef np.ndarray data_tmp
    cdef unsigned j, i
    if len(arrays) == 0:
        raise TypeError("arrays contains no item")
    datasize = arrays[0].size
    shape = arrays[0].shape
    result.resize(len(arrays) * datasize)
    for j, data in enumerate(arrays):
        if shape != data.shape:
            raise TypeError("arrays contains different shaped ndarrays")
        data_tmp = np.array(data, dtype=np.float32, order="F")
        np_data = <np.float32_t *> data_tmp.data
        for i in range(datasize):
            result[j * datasize + i] = np_data[i]
    return result

cdef inline vector[unsigned] ndarray_to_vector_unsigned(np.ndarray array):
    cdef vector[unsigned] result
    cdef unsigned *np_data
    cdef unsigned datasize
    cdef np.ndarray array_tmp

    if array.dtype != np.uint32:
        raise TypeError("numpy.ndarray must be constructed from uint32 data")

    datasize = array.size
{'fraction_non_alphanumeric': 0.07756132756132757, 'fraction_numerical': 0.00937950937950938, 'mean_word_length': 3.465378421900161, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4872698', 'n_tokens_mistral': 890, 'n_tokens_neox': 845, 'n_words': 265}