<commit_before># matrices.py - boolean matrices as row bitsets and column bitsets

"""Boolean matrices as collections of row and column vectors."""

import bitsets

__all__ = ['Relation']


Vector = bitsets.bases.MemberBits
"""Single row or column of a boolean matrix as bit vector."""


class Vectors(bitsets.series.Tuple):
    """Paired collection of rows or columns of a boolean matrix relation.

    Trailing zeros see https://stackoverflow.com/q/63917579/3456664
    """

    def _pair_with(self, relation, index, other):
        if hasattr(self, 'prime'):
            raise RuntimeError(f'{self!r} attempt _pair_with {other!r}')

        self.relation = relation
        self.relation_index = index

        Prime = other.BitSet.supremum  # noqa: N806
        Double = self.BitSet.supremum  # noqa: N806

        _prime = other.BitSet.fromint
        _double = self.BitSet.fromint

        def prime(bitset):
            """FCA derivation operator (extent->intent, intent->extent)."""
            prime = Prime

            i = 0
            while i < len(other):
                if bitset:
                    trailing_zeros = (bitset & -bitset).bit_length() - 1
                    if trailing_zeros:
                        bitset >>= trailing_zeros
                        i += trailing_zeros
                    else:
                        prime &= other[i]
                        bitset >>= 1
                        i += 1
                else:
                    break

            return _prime(prime)
<commit_msg>simplify leftover using shift<commit_after># matrices.py - boolean matrices as row bitsets and column bitsets

"""Boolean matrices as collections of row and column vectors."""

import bitsets

__all__ = ['Relation']


Vector = bitsets.bases.MemberBits
"""Single row or column of a boolean matrix as bit vector."""


class Vectors(bitsets.series.Tuple):
    """Paired collection of rows or columns of a boolean matrix relation.

    Trailing zeros see https://stackoverflow.com/q/63917579/3456664
    """

    def _pair_with(self, relation, index, other):
        if hasattr(self, 'prime'):
            raise RuntimeError(f'{self!r} attempt _pair_with {other!r}')

        self.relation = relation
        self.relation_index = index

        Prime = other.BitSet.supremum  # noqa: N806
        Double = self.BitSet.supremum  # noqa: N806

        _prime = other.BitSet.fromint
        _double = self.BitSet.fromint

        def prime(bitset):
            """FCA derivation operator (extent->intent, intent->extent)."""
            prime = Prime

            i = 0
            while i < len(other):
                if bitset:
                    trailing_zeros = (bitset & -bitset).bit_length() - 1
                    if trailing_zeros:
                        shift = trailing_zeros
                    else:
                        prime &= other[i]
                        shift = 1
                    bitset >>= shift
                    i += shift
                else:
                    break

            return _prime(prime)

{'fraction_non_alphanumeric': 0.09090909090909091, 'fraction_numerical': 0.01596611274030629, 'mean_word_length': 1.6835664335664335, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '469998', 'n_tokens_mistral': 883, 'n_tokens_neox': 802, 'n_words': 241}