<commit_before># -*- coding: utf-8 -*-
##
## $Id$
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""Unit tests for the intbitset data structure."""

__revision__ = "$Id$"

import unittest

from invenio.intbitset import intbitset

try:
    set()
except NameError:
    from sets import Set as set

class IntBitSetTest(unittest.TestCase):
    """Test functions related to intbitset data structure."""
    def setUp(self):
        self.sets = [
            [10, 20],
            [10, 40],
            [60, 70],
            [60, 80],
            [10, 20, 60, 70],
            [10, 40, 60, 80],
        ]
        self.fncs_list = [
            (intbitset.__and__, set.__and__, int.__and__, False),
            (intbitset.__or__, set.__or__, int.__or__, False),
            (intbitset.__xor__, set.__xor__, int.__xor__, False),
            (intbitset.__sub__, set.__sub__, int.__sub__, False),
            (intbitset.__iand__, set.__iand__, int.__and__, True),
            (intbitset.__ior__, set.__ior__, int.__or__, True),
            (intbitset.__ixor__, set.__ixor__, int.__xor__, True),
            (intbitset.__isub__, set.__isub__, int.__sub__, True),
        ]

        self.cmp_list = [
            (intbitset.__eq__, set.__eq__, lambda x, y: cmp(x, y) == 0),
            (intbitset.__ge__, set.__ge__, lambda x, y: cmp(x, y) >= 0),
            (intbitset.__gt__, set.__gt__, lambda x, y: cmp(x, y) > 0),
            (intbitset.__le__, set.__le__, lambda x, y: cmp(x, y) <= 0),
            (intbitset.__lt__, set.__lt__, lambda x, y: cmp(x, y) < 0),
            (intbitset.__ne__, set.__ne__, lambda x, y: cmp(x, y) != 0),
        ]

    def _helper_sanity_test(self, intbitset1):
        wordbitsize = intbitset1.get_wordbitsize()
        size1 = intbitset1.get_size()
        allocated1 = intbitset1.get_allocated()
        creator_list = intbitset1.extract_finite_list()
        up_to1 = creator_list and max(creator_list) or -1
        self.failUnless(up_to1 <= size1 * wordbitsize < allocated1 * wordbitsize)
        tmp = intbitset(intbitset1.fastdump())
        size1 = intbitset1.get_size()
        allocated1 = intbitset1.get_allocated()
        creator_list = intbitset1.extract_finite_list()
        up_to1 = creator_list and max(creator_list) or -1
        self.failUnless(up_to1 <= size1 * wordbitsize < allocated1 * wordbitsize)


    def _helper_test_via_fncs_list(self, fncs, intbitset1, intbitset2):
        orig1 = intbitset(intbitset1)
        orig2 = intbitset(intbitset2)

        trailing1 = intbitset1.is_infinite()
        trailing2 = intbitset2.is_infinite()

        if fncs[3]:
            fncs[0](intbitset1, intbitset2)
            trailing1 = fncs[2](trailing1, trailing2) > 0
            up_to = intbitset1.extract_finite_list() and max(intbitset1.extract_finite_list()) or -1
        else:
            intbitset3 = fncs[0](intbitset1, intbitset2)
            trailing3 = fncs[2](trailing1, trailing2) > 0
            up_to = intbitset3.extract_finite_list() and max(intbitset3.extract_finite_list()) or -1

        set1 = set(orig1.extract_finite_list(up_to))
        set2 = set(orig2.extract_finite_list(up_to))

        if fncs[3]:
            fncs[1](set1, set2)
        else:
            set3 = fncs[1](set1, set2)

        self._helper_sanity_test(intbitset1)
        self._helper_sanity_test(intbitset2)

        if fncs[3]:
            self.assertEqual(set1 & set(intbitset1.extract_finite_list(up_to)), set(intbitset1.extract_finite_list(up_to)), "%s not equal to %s after executing %s(%s, %s)" % (set1, set(intbitset1.extract_finite_list(up_to)), fncs[0].__name__, repr(orig1), repr(orig2)))
            self.assertEqual(set1 | set(intbitset1.extract_finite_list(up_to)), set1, "%s not equal to %s after executing %s(%s, %s)" % (set1, set(intbitset1.extract_finite_list(up_to)), fncs[0].__name__, repr(orig1), repr(orig2)))
            self.assertEqual(trailing1, intbitset1.is_infinite(), "%s is not %s as it is supposed to be after executing %s(%s, %s)" % (intbitset1, trailing1 and 'infinite' or 'finite', fncs[0].__name__, repr(orig1), repr(orig2)))
        else:
            self._helper_sanity_test(intbitset3)
            self.assertEqual(set3 & set(intbitset3.extract_finite_list(up_to)), set(intbitset3.extract_finite_list(up_to)), "%s not equal to %s after executing %s(%s, %s)" % (set3, set(intbitset3.extract_finite_list(up_to)), fncs[0].__name__, repr(orig1), repr(orig2)))
            self.assertEqual(set3 | set(intbitset3.extract_finite_list(up_to)), set3, "%s not equal to %s after executing %s(%s, %s)" % (set3, set(intbitset3.extract_finite_list(up_to)), fncs[0].__name__, repr(orig1), repr(orig2)))
            self.assertEqual(trailing3, intbitset3.is_infinite(), "%s is not %s as it is supposed to be after executing %s(%s, %s)" % (intbitset3, trailing3 and 'infinite' or 'finite', fncs[0].__name__, repr(orig1), repr(orig2)))


    def _helper_test_normal_set(self, fncs):
        for set1 in self.sets:
            for set2 in self.sets:
                self._helper_test_via_fncs_list(fncs, intbitset(set1), intbitset(set2))<commit_msg>Perfectioned the tests, adding some meaningful failure messages, and some big lists. It appears that ixor doesn't work well between two infinite intbitsets. It will be fixed some day in the future since it's not used anywhere in CDS Invenio code.<commit_after># -*- coding: utf-8 -*-
##
## $Id$
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""Unit tests for the intbitset data structure."""

__revision__ = "$Id$"

import unittest

from invenio.intbitset import intbitset

try:
    set()
except NameError:
    from sets import Set as set

class IntBitSetTest(unittest.TestCase):
    """Test functions related to intbitset data structure."""
    def setUp(self):
        self.sets = [
            [10, 20],
            [10, 40],
            [60, 70],
            [60, 80],
            [10, 20, 60, 70],
            [10, 40, 60, 80],
            [1000],
            [10000],
            [23, 45, 67, 89, 110, 130, 174, 1002, 2132, 23434],
            [700, 2000]
        ]
        self.fncs_list = [
            (intbitset.__and__, set.__and__, int.__and__, False),
            (intbitset.__or__, set.__or__, int.__or__, False),
            (intbitset.__xor__, set.__xor__, int.__xor__, False),
            (intbitset.__sub__, set.__sub__, int.__sub__, False),
            (intbitset.__iand__, set.__iand__, int.__and__, True),
            (intbitset.__ior__, set.__ior__, int.__or__, True),
            (intbitset.__ixor__, set.__ixor__, int.__xor__, True),
            (intbitset.__isub__, set.__isub__, int.__sub__, True),
        ]

        self.cmp_list = [
            (intbitset.__eq__, set.__eq__, lambda x, y: cmp(x, y) == 0),
            (intbitset.__ge__, set.__ge__, lambda x, y: cmp(x, y) >= 0),
            (intbitset.__gt__, set.__gt__, lambda x, y: cmp(x, y) > 0),
            (intbitset.__le__, set.__le__, lambda x, y: cmp(x, y) <= 0),
            (intbitset.__lt__, set.__lt__, lambda x, y: cmp(x, y) < 0),
            (intbitset.__ne__, set.__ne__, lambda x, y: cmp(x, y) != 0),
        ]

    def _helper_sanity_test(self, intbitset1, msg=''):
        wordbitsize = intbitset1.get_wordbitsize()
        size1 = intbitset1.get_size()
        allocated1 = intbitset1.get_allocated()
        creator_list = intbitset1.extract_finite_list()
        up_to1 = creator_list and max(creator_list) or -1
        self.failUnless(up_to1 <= size1 * wordbitsize < allocated1 * wordbitsize, "up_to1=%s, size1=%s, allocated1=%s while testing %s during %s" % (up_to1, size1 * wordbitsize, allocated1 * wordbitsize, intbitset1, msg))
        tmp = intbitset(intbitset1.fastdump())
        size1 = intbitset1.get_size()
        allocated1 = intbitset1.get_allocated()
        creator_list = intbitset1.extract_finite_list()
        up_to1 = creator_list and max(creator_list) or -1
        self.failUnless(up_to1 <= size1 * wordbitsize < allocated1 * wordbitsize, "After serialization up_to1=%s, size1=%s, allocated1=%s while testing %s during %s" % (up_to1, size1 * wordbitsize, allocated1 * wordbitsize, intbitset1, msg))


    def _helper_test_via_fncs_list(self, fncs, intbitset1, intbitset2):
        orig1 = intbitset(intbitset1)
        orig2 = intbitset(intbitset2)

        msg = "Testing %s(%s, %s)" % (fncs[0].__name__, repr(intbitset1), repr(intbitset2))

        trailing1 = intbitset1.is_infinite()
        trailing2 = intbitset2.is_infinite()

        if fncs[3]:
            fncs[0](intbitset1, intbitset2)
            trailing1 = fncs[2](trailing1, trailing2) > 0
            up_to = intbitset1.extract_finite_list() and max(intbitset1.extract_finite_list()) or -1
        else:
            intbitset3 = fncs[0](intbitset1, intbitset2)
            trailing3 = fncs[2](trailing1, trailing2) > 0
            up_to = intbitset3.extract_finite_list() and max(intbitset3.extract_finite_list()) or -1

        set1 = set(orig1.extract_finite_list(up_to))
        set2 = set(orig2.extract_finite_list(up_to))

        if fncs[3]:
            fncs[1](set1, set2)
        else:
            set3 = fncs[1](set1, set2)

        self._helper_sanity_test(intbitset1, msg)
        self._helper_sanity_test(intbitset2, msg)

        if fncs[3]:
            self.assertEqual(set1 & set(intbitset1.extract_finite_list(up_to)), set(intbitset1.extract_finite_list(up_to)), "%s not equal to %s after executing %s(%s, %s)" % (set1, set(intbitset1.extract_finite_list(up_to)), fncs[0].__name__, repr(orig1), repr(orig2)))
            self.assertEqual(set1 | set(intbitset1.extract_finite_list(up_to)), set1, "%s not equal to %s after executing %s(%s, %s)" % (set1, set(intbitset1.extract_finite_list(up_to)), fncs[0].__name__, repr(orig1), repr(orig2)))
            self.assertEqual(trailing1, intbitset1.is_infinite(), "%s is not %s as it is supposed to be after executing %s(%s, %s)" % (intbitset1, trailing1 and 'infinite' or 'finite', fncs[0].__name__, repr(orig1), repr(orig2)))
        else:
            self._helper_sanity_test(intbitset3, msg)
            self.assertEqual(set3 & set(intbitset3.extract_finite_list(up_to)), set(intbitset3.extract_finite_list(up_to)), "%s not equal to %s after executing %s(%s, %s)" % (set3, set(intbitset3.extract_finite_list(up_to)), fncs[0].__name__, repr(orig1), repr(orig2)))
            self.assertEqual(set3 | set(intbitset3.extract_finite_list(up_to)), set3, "%s not equal to %s after executing %s(%s, %s)" % (set3, set(intbitset3.extract_finite_list(up_to)), fncs[0].__name__, repr(orig1), repr(orig2)))
            self.assertEqual(trailing3, intbitset3.is_infinite(), "%s is not %s as it is supposed to be after executing %s(%s, %s)" % (intbitset3, trailing3 and 'infinite' or 'finite', fncs[0].__name__, repr(orig1), repr(orig2)))


    def _helper_test_normal_set(self, fncs):
        for set1 in self.sets:
            for set2 in self.sets:
                self._helper_test_via_fncs_list(fncs, intbitset(set1), intbitset(set2))
{'fraction_non_alphanumeric': 0.1259188239054011, 'fraction_numerical': 0.037152444870565675, 'mean_word_length': 3.0918600849950963, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3385596', 'n_tokens_mistral': 4439, 'n_tokens_neox': 4195, 'n_words': 1096}