<commit_before>#    Copyright 2016 <NAME> aka penguinolog
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Bitfield module

Implements BitField in Python
"""

import copy
import functools
import math


def _mangle(cls_name, attr_name):
    """Mangle attribute

    :param cls_name: class name
    :type cls_name: str
    :param attr_name: attribute name
    :type attr_name: str
    :return: mangled attribute
    :rtype: str
    """
    return '_{cls_name!s}__{attr_name!s}'.format(
        cls_name=cls_name,
        attr_name=attr_name
    )


def _is_descriptor(obj):
    """Returns True if obj is a descriptor, False otherwise."""
    return (
        hasattr(obj, '__get__') or
        hasattr(obj, '__set__') or
        hasattr(obj, '__delete__')
    )


def _is_dunder(name):
    """Returns True if a __dunder__ name, False otherwise."""
    return (name[:2] == name[-2:] == '__' and
            name[2:3] != '_' and
            name[-3:-2] != '_' and
            len(name) > 4)


def _is_sunder(name):
    """Returns True if a _sunder_ name, False otherwise."""
    return (name[0] == name[-1] == '_' and
            name[1:2] != '_' and
            name[-2:-1] != '_' and
            len(name) > 2)


def _mapping_filter(item):
    """Filter for namig records from namespace

    :param item: namespace item
    :type item: tuple
    :rtype: bool
    """
    name, obj = item

    if name in {'_index_'}:
        return True

    # Descriptors, special methods, protected
    if _is_descriptor(obj) or _is_dunder(name) or name.startswith('_'):
        return False

    # Index / slice / slice from iterable
    if isinstance(obj, int) or (
        isinstance(obj, slice) and obj.step is None
    ) or (
        isinstance(obj, (tuple, list)) and len(obj) == 2 and
        isinstance(obj[0], int) and isinstance(obj[1], int) and
        obj[0] < obj[1]
    ):
        return True

    # Not nested
    if not isinstance(obj, dict):
        return False

    # Process nested
    return all((_mapping_filter(value) for value in obj.items()))


def _get_idx(val):
    """Internal method for usage in repr. Moved from class implementation."""
    if isinstance(val, int):
        return {val}
    if isinstance(val, (tuple, list)):
        return set(range(*val))
    if isinstance(val, slice):
        if val.start is not None:
            return set(range(val.start, val.stop))
        return set(range(val.stop))
    if isinstance(val, dict):
        return set(range(*val['_index_']))


def _process_indexes(mapping):
    global_index = set()
    for key, val in mapping.items():
        index = _get_idx(val)
        if global_index - index != global_index:
            raise IndexError(
                'Mapping key {key} has intersection with other keys '
                'on indexes {indexes}'.format(
                    key=key,
                    indexes=list(sorted(index - global_index))
                ))
        global_index |= index
    return functools.reduce(
        lambda res, idx: res | 1 << idx,
        global_index,
        0
    )


class BitField(object):
    """Fake class for BitFieldMeta compilation"""
    pass


class BitFieldMeta(type):
    """Metaclass for BitField class and subclasses construction"""
    def __new__(mcs, name, bases, classdict):
        """BitField metaclass

        :type name: str
        :type bases: tuple
        :type classdict: dict
        :returns: new class
        """

        for base in bases:
            if base is not BitField and issubclass(base, BitField):
                raise TypeError("Cannot extend BitField")

        mapping = {}
        if '_index_' in classdict:
            raise ValueError(
                '_index_ is reserved index for slicing nested BitFields'
            )
        for m_key, m_val in filter(
            _mapping_filter,
            classdict.copy().items()
        ):
            mapping[m_key] = m_val
            del classdict[m_key]  # drop

        if '_length_' not in classdict:
            classdict['_length_'] = None

        garbage = {
            name: obj for name, obj in classdict.items()
            if not (_is_dunder(name) or _is_sunder(name) or _is_descriptor(obj))
        }

        if garbage:
            raise TypeError(
                'Several data is not recognized in class structure: '
                '{!r}'.format(garbage)
            )

        if mapping:
            mask = _process_indexes(mapping)
            classdict['_mapping_'] = property(
                fget=lambda _: copy.deepcopy(mapping),
                doc="""Read-only mapping structure"""
            )
            classdict['_mask_'] = property(
                fget=lambda _: mask,
                doc="""Read-only data binary mask"""
            )
        else:
            # None for structure and mask
            classdict['_mapping_'] = property(
                fget=lambda _: None,
                doc="""Read-only mapping structure"""
            )
            classdict['_mask_'] = property(<commit_msg>Refactor & continue development<commit_after>#    Copyright 2016 <NAME> aka penguinolog
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Bitfield module

Implements BitField in Python
"""

import copy
import functools
import math


def _mangle(cls_name, attr_name):
    """Mangle attribute

    :param cls_name: class name
    :type cls_name: str
    :param attr_name: attribute name
    :type attr_name: str
    :return: mangled attribute
    :rtype: str
    """
    return '_{cls_name!s}__{attr_name!s}'.format(
        cls_name=cls_name,
        attr_name=attr_name
    )


def _is_descriptor(obj):
    """Returns True if obj is a descriptor, False otherwise."""
    return (
        hasattr(obj, '__get__') or
        hasattr(obj, '__set__') or
        hasattr(obj, '__delete__')
    )


def _is_dunder(name):
    """Returns True if a __dunder__ name, False otherwise."""
    return (name[:2] == name[-2:] == '__' and
            name[2:3] != '_' and
            name[-3:-2] != '_' and
            len(name) > 4)


def _is_sunder(name):
    """Returns True if a _sunder_ name, False otherwise."""
    return (name[0] == name[-1] == '_' and
            name[1:2] != '_' and
            name[-2:-1] != '_' and
            len(name) > 2)


def _mapping_filter(item):
    """Filter for namig records from namespace

    :param item: namespace item
    :type item: tuple
    :rtype: bool
    """
    name, obj = item

    if name in {'_index_'}:
        return True

    # Descriptors, special methods, protected
    if _is_descriptor(obj) or _is_dunder(name) or name.startswith('_'):
        return False

    # Index / slice / slice from iterable
    if isinstance(obj, int) or (
        isinstance(obj, slice) and obj.step is None
    ) or (
        isinstance(obj, (tuple, list)) and len(obj) == 2 and
        isinstance(obj[0], int) and isinstance(obj[1], int) and
        obj[0] < obj[1]
    ):
        return True

    # Not nested
    if not isinstance(obj, dict):
        return False

    # Process nested
    return all((_mapping_filter(value) for value in obj.items()))


def _get_idx(val):
    """Internal method for usage in repr. Moved from class implementation."""
    if isinstance(val, int):
        return {val}
    if isinstance(val, (tuple, list)):
        return set(range(*val))
    if isinstance(val, slice):
        if val.start is not None:
            return set(range(val.start, val.stop))
        return set(range(val.stop))
    if isinstance(val, dict):
        return set(range(*val['_index_']))


def _process_indexes(mapping):
    global_index = set()
    for key, val in mapping.items():
        index = _get_idx(val)
        if global_index - index != global_index:
            raise IndexError(
                'Mapping key {key} has intersection with other keys '
                'on indexes {indexes}'.format(
                    key=key,
                    indexes=list(sorted(index - global_index))
                ))
        global_index |= index
    return functools.reduce(
        lambda res, idx: res | 1 << idx,
        global_index,
        0
    )


class BitField(object):
    """Fake class for BitFieldMeta compilation"""
    pass


class BitFieldMeta(type):
    """Metaclass for BitField class and subclasses construction"""
    def __new__(mcs, name, bases, classdict):
        """BitField metaclass

        :type name: str
        :type bases: tuple
        :type classdict: dict
        :returns: new class
        """

        for base in bases:
            if base is not BitField and issubclass(base, BitField):
                raise TypeError("Cannot extend BitField")

        mapping = {}
        if '_index_' in classdict:
            raise ValueError(
                '_index_ is reserved index for slicing nested BitFields'
            )
        for m_key, m_val in filter(
            _mapping_filter,
            classdict.copy().items()
        ):
            mapping[m_key] = m_val
            del classdict[m_key]  # drop

        if '_length_' not in classdict:
            classdict['_length_'] = None

        garbage = {
            name: obj for name, obj in classdict.items()
            if not (
                _is_dunder(name) or _is_sunder(name) or _is_descriptor(obj)
            )
        }

        if garbage:
            raise TypeError(
                'Several data is not recognized in class structure: '
                '{!r}'.format(garbage)
            )

        if mapping:
            mask = _process_indexes(mapping)
            classdict['_mapping_'] = property(
                fget=lambda _: copy.deepcopy(mapping),
                doc="""Read-only mapping structure"""
            )
            classdict['_mask_'] = property(
                fget=lambda _: mask,
                doc="""Read-only data binary mask"""
            )
        else:
            # None for structure and mask
            classdict['_mapping_'] = property(
                fget=lambda _: None,
                doc="""Read-only mapping structure"""
            )
            classdict['_mask_'] = property(
{'fraction_non_alphanumeric': 0.10415547196847573, 'fraction_numerical': 0.005194339960594662, 'mean_word_length': 2.0510928961748633, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7618210', 'n_tokens_mistral': 3244, 'n_tokens_neox': 3157, 'n_words': 1007}