<commit_before>"""Manage bit sets as ordered list of integers very efficiently, relying on RoaringBitmaps.
"""

import array

from pyroaring import BitMap
import numpy as np
from  collections.abc import Iterable
# pragma no cover
# pylint: disable=invalid-name
_integer_types = (int, np.integer)


class bitmap(BitMap,object):
    # pylint: disable=invalid-name
    """
    Derive from an efficient and light-weight ordered set of 32 bits integers.
    """
    def __new__(cls, values=None, copy_on_write=False, optimize=True, no_init=False):
        if isinstance(values, slice):
            values = range(values.start, values.stop,
                           (values.step if values.step else 1))
        return super(bitmap, cls).__new__(cls, values,
                                          copy_on_write,
                                          optimize,
                                          no_init)
        #BitMap.__init__(self, values, copy_on_write)

    def clear(self):
        "Clear the bitmap in-place"
        self &= NIL_BITMAP

    def __contains__(self, other):
        if isinstance(other, _integer_types):
            return BitMap.__contains__(self, other)
        other = self.asbitmap(other)
        return other <= self

    def __repr__(self):
        l = len(self)
        if l > 10:
            values = ', '.join([str(n) for n in self[0:6]])
            values += '...(%d)...%d)' % (l, self[l-1])
        else:
            values = ', '.join([str(n) for n in self])
        return 'bitmap([%s])' % values

    # def __binary_op__(self, other, function):
    #     if other is None:
    #         other = NIL_BITMAP
    #     try:
    #         return bitmap(obj=function(self.__obj__, other.__obj__))
    #     except AttributeError:
    #         raise TypeError('Not a bitmap.')

    # def __binary_op_inplace__(self, other, function):
    #     if other is None:
    #         other = NIL_BITMAP
    #     return BitMap.__binary_op_inplace__(self, other, function)

    def __getitem__(self, values):
        bm = BitMap.__getitem__(self, values)
        if isinstance(bm, BitMap):
            return bitmap(bm)
        if (isinstance(bm, TypeError) and isinstance(values, Iterable)):
            bmval = bitmap(values)
            if bmval in self:
                return bmval
        if isinstance(bm, Exception):
            raise bm
        raise KeyError(f"wrong key{values}")

                <commit_msg>backtrack on the previous fix<commit_after>"""Manage bit sets as ordered list of integers very efficiently, relying on RoaringBitmaps.
"""

import array

from pyroaring import BitMap
import numpy as np
from  collections.abc import Iterable
# pragma no cover
# pylint: disable=invalid-name
_integer_types = (int, np.integer)


class bitmap(BitMap,object):
    # pylint: disable=invalid-name
    """
    Derive from an efficient and light-weight ordered set of 32 bits integers.
    """
    def __new__(cls, values=None, copy_on_write=False, optimize=True, no_init=False):
        if isinstance(values, slice):
            values = range(values.start, values.stop,
                           (values.step if values.step else 1))
        return super(bitmap, cls).__new__(cls, values,
                                          copy_on_write,
                                          optimize,
                                          no_init)
        #BitMap.__init__(self, values, copy_on_write)

    def clear(self):
        "Clear the bitmap in-place"
        self &= NIL_BITMAP

    def __contains__(self, other):
        if isinstance(other, _integer_types):
            return BitMap.__contains__(self, other)
        other = self.asbitmap(other)
        return other <= self

    def __repr__(self):
        l = len(self)
        if l > 10:
            values = ', '.join([str(n) for n in self[0:6]])
            values += '...(%d)...%d)' % (l, self[l-1])
        else:
            values = ', '.join([str(n) for n in self])
        return 'bitmap([%s])' % values

    # def __binary_op__(self, other, function):
    #     if other is None:
    #         other = NIL_BITMAP
    #     try:
    #         return bitmap(obj=function(self.__obj__, other.__obj__))
    #     except AttributeError:
    #         raise TypeError('Not a bitmap.')

    # def __binary_op_inplace__(self, other, function):
    #     if other is None:
    #         other = NIL_BITMAP
    #     return BitMap.__binary_op_inplace__(self, other, function)

    def __getitem__(self, values):
        bm = BitMap.__getitem__(self, values)
        if isinstance(bm, BitMap):
            return bitmap(bm)
        if isinstance(bm, Exception):
            raise bm
        return bm

                
{'fraction_non_alphanumeric': 0.10304697418535759, 'fraction_numerical': 0.0033855268726195515, 'mean_word_length': 1.945171339563863, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '213244', 'n_tokens_mistral': 1362, 'n_tokens_neox': 1324, 'n_words': 377}