<commit_before># Copyright 2016 <NAME>

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


cimport cython
import types
from cpython cimport array
from cpython.list cimport PyList_GET_SIZE, PyList_GET_ITEM

cpdef rebuild(data, index, size):
    cdef SparseArray r = SparseArray()
    r._len = size
    r.data = data
    r.index = index
    r.non_zero = len(index)
    return r


@cython.freelist(512)
cdef class SparseArray:
    def __cinit__(self):
        self.non_zero = 0
        self._len = 0
        self.index = array.array('I')
        self.data = array.array('d')

    @classmethod
    def fromlist(cls, lst):
        cdef SparseArray res
        cdef unsigned int k
        cdef double v
        cdef Py_ssize_t i = 0
        try:            
            res = cls.empty(len(lst), len(lst) - lst.count(0))
        except AttributeError:
            k = 0
            for v in lst:
                if v == 0:
                    k += 1
            res = cls.empty(len(lst), len(lst) - k)
        for k, v in enumerate(lst):
            if v == 0:
                continue
            res.index.data.as_uints[i] = k
            res.data.data.as_doubles[i] = v
            i += 1
        return res

    @classmethod
    def index_data(cls, index_data, size):
        cdef SparseArray r = cls()
        r._len = size
        r.index = array.array('I', [x[0] for x in index_data])
        r.data = array.array('d', [x[1] for x in index_data])
        r.non_zero = len(index_data)
        return r

    @classmethod
    def constant(cls, value, index, size):
        cdef SparseArray r = cls()
        cdef double *data_value 
        cdef Py_ssize_t i, index_size = len(index)
        r._len = size
        r.non_zero = index_size
        r.index = array.array('I', index)
        r.data = array.clone(r.data, index_size, zero=False)
        data_value = r.data.data.as_doubles
        for i in range(index_size):
            data_value[i] = value
        return r
        
        
    @property
    def density(self):
        return self.non_zero / float(self._len)

    @property
    def used_memory(self):
        return self.data.itemsize * self.non_zero + self.index.itemsize * self.non_zero

    @property
    def maximum_memory(self):
        return self.data.itemsize * self._len + self.index.itemsize * self._len

    cdef double getitem(self, unsigned int k):
        cdef unsigned int start=0, end=self.non_zero, half
        cdef unsigned int *a = self.index.data.as_uints
        cdef double *a_value = self.data.data.as_doubles
        if k >= self._len:
            return 0
        while end - start > 1:
            half = start + (end - start) // 2
            if a[half] > k:
                end = half
            elif a[half] < k:
                start = half
            else:
                return a_value[half]
        if a[start] == k:
            return a_value[start]
        elif a[end] == k:
            return a_value[end]
        return 0

    def __getitem__(self, i):
        return self.getitem(i)
    
    def __len__(self):
        return self._len

    def size(self):
        return self._len
<commit_msg>getitem (2)<commit_after># Copyright 2016 <NAME>

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


cimport cython
import types
from cpython cimport array
from cpython.list cimport PyList_GET_SIZE, PyList_GET_ITEM

cpdef rebuild(data, index, size):
    cdef SparseArray r = SparseArray()
    r._len = size
    r.data = data
    r.index = index
    r.non_zero = len(index)
    return r


@cython.freelist(512)
cdef class SparseArray:
    def __cinit__(self):
        self.non_zero = 0
        self._len = 0
        self.index = array.array('I')
        self.data = array.array('d')

    @classmethod
    def fromlist(cls, lst):
        cdef SparseArray res
        cdef unsigned int k
        cdef double v
        cdef Py_ssize_t i = 0
        try:            
            res = cls.empty(len(lst), len(lst) - lst.count(0))
        except AttributeError:
            k = 0
            for v in lst:
                if v == 0:
                    k += 1
            res = cls.empty(len(lst), len(lst) - k)
        for k, v in enumerate(lst):
            if v == 0:
                continue
            res.index.data.as_uints[i] = k
            res.data.data.as_doubles[i] = v
            i += 1
        return res

    @classmethod
    def index_data(cls, index_data, size):
        cdef SparseArray r = cls()
        r._len = size
        r.index = array.array('I', [x[0] for x in index_data])
        r.data = array.array('d', [x[1] for x in index_data])
        r.non_zero = len(index_data)
        return r

    @classmethod
    def constant(cls, value, index, size):
        cdef SparseArray r = cls()
        cdef double *data_value 
        cdef Py_ssize_t i, index_size = len(index)
        r._len = size
        r.non_zero = index_size
        r.index = array.array('I', index)
        r.data = array.clone(r.data, index_size, zero=False)
        data_value = r.data.data.as_doubles
        for i in range(index_size):
            data_value[i] = value
        return r
        
        
    @property
    def density(self):
        return self.non_zero / float(self._len)

    @property
    def used_memory(self):
        return self.data.itemsize * self.non_zero + self.index.itemsize * self.non_zero

    @property
    def maximum_memory(self):
        return self.data.itemsize * self._len + self.index.itemsize * self._len

    cdef double getitem(self, unsigned int k):
        cdef unsigned int start=0, end=self.non_zero, half
        cdef unsigned int *a = self.index.data.as_uints
        cdef double *a_value = self.data.data.as_doubles
        if k >= self._len:
            return 0
        while end - start > 1:
            half = start + (end - start) // 2
            if a[half] > k:
                end = half
            elif a[half] < k:
                start = half
            elif a[half] == k:
                return a_value[half]
            else:
                break
        if a[start] == k:
            return a_value[start]
        elif a[end] == k:
            return a_value[end]
        return 0

    def __getitem__(self, i):
        return self.getitem(i)
    
    def __len__(self):
        return self._len

    def size(self):
        return self._len

{'fraction_non_alphanumeric': 0.09336240970423879, 'fraction_numerical': 0.0074962518740629685, 'mean_word_length': 1.8519238243295764, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3280959', 'n_tokens_mistral': 2323, 'n_tokens_neox': 2244, 'n_words': 699}