<commit_before>'''Implementation of the actual Markov Chain dictionary.

'''

import collections

# from markoverror import CorpusError, MarkovChainError

class MarkovChain(collections.defaultdict):
    '''This class is essentially a fancy defaultdict that always
    uses a list for it's default factory method.

    Key methods such as __contains__ have been modified to reflect the
    nature of the keys (tuples)  for MarkovChains.

    It also provides modified comparison methods, key checking
    and operator shortcuts (chain & other_chain, chain + other_chain. etc).

    '''
    
    # Pretty standard magic methods.
    
    def __init__(self, **kwargs):
        self.order = kwargs.get('order', 2)
        super(MarkovChain, self).__init__(list)

    def __repr__(self):
        pattern = "<{} Order:{} Length:{}>"
        return pattern.format(type(self).__name__, self.order, len(self))

    # Modified equality testers.

    def __eq__(self, other):
        '''Equality for Markov chains is resolved based on order,
        rather than content. Instead of running self.order == other.order
        every time (despite that being more explicit), this allows a
        shortcut to that.

        '''
        if not isinstance(other, MarkovChain):
            return NotImplemented
        return self.order == other.order

    def __ne__(self, other):
        '''See self.__eq__ for implementation details.

        '''
        return not self.__eq__(other)

    # Inequality comparions.
    # These are actually set comparisons.

    def __lt__(self, other):
        '''Tests if self is a proper subset of other.
        That is set(self.keys()) <= set(self.keys()) and 
                set(self.keys()) != set(other.keys())

        '''
        if not isinstance(other, MarkovChain):
            return NotImplemented
        return set(self.keys()) < set(other.keys())

    def __le__(self, other):
        '''Tests if every element in self is in other.

        '''
        if not isinstance(other, MarkovChain):
            return NotImplemented
        return set(self.keys()) <= set(other.keys())

    def __gt__(self, other):
        '''Tests if self is a proper superset of other.
        That is set(self.keys()) > set(other.keys()) and
                set(self.keys()) != set(other.keys())

        '''
        if not isinstance(other, MarkovChain):
            return NotImplemented
        return set(self.keys()) > set(other.keys())

    def __ge__(self, other):
        '''Tests if every element in other is in self.

        '''
        if not isinstance(other, MarkovChain):
            return NotImplemented
        return set(self.keys()) >= set(other.keys())

    # Bitwise and arithmetic operators.
    # I'll leave this unimplmented for now..
    # Implementation has given me a headache before.

    def __and__(self, other):
        raise NotImplementedError

    def __iand__(self, other):
        raise NotImplementedError

    def __or__(self, other):
        raise NotImplementedError

    def __ior__(self, other):
        raise NotImplementedError

    def __add__(self, other):
        raise NotImplementedError

    def __iadd__(self, other):
        raise NotImplementedError

    def __sub__(self, other):
        raise NotImplementedError

    def __isub__(self, other):
        raise NotImplementedError

    # container methods

    def __contains__(self, search):
        if isinstance(search, tuple):
            return bool(self._search_by_sequence(search))
        return any(search in key for key in self.keys())

    def __setitem__(self, key, value):
        #DANGER WILL ROBINSON!
        '''Over riden method to ensure data intergity.
        Ensures that key is a tuple with the proper length
        and that value is a list.

        Type checking...I know.

        '''

        if not isinstance(key, tuple):
            raise TypeError('MarkovChain expects tuple as key, {} provided.'
                                 ''.format(type(key).__name__))
        elif len(key) != self.order:
            raise AttributeError('MarkovChain expects key with length {}, '
                                 '{} length given'.format(self.order, len(key)))
        elif not isinstance(value, list):
            raise AttributeError('MarkovChain expects a list as a value, '
                                 '{} provided.'.format(type(value).__name__))
        else:
            super(MarkovChain, self).__setitem__(key, value)
<commit_msg>Created BitwiseOpFeature for operators on the chains.<commit_after>'''Implementation of the actual Markov Chain dictionary.

'''

import collections

# from markoverror import CorpusError, MarkovChainError

class MarkovChain(collections.defaultdict):
    '''This class is essentially a fancy defaultdict that always
    uses a list for it's default factory method.

    Key methods such as __contains__ have been modified to reflect the
    nature of the keys (tuples)  for MarkovChains.

    It also provides modified comparison methods, key checking
    and operator shortcuts (chain & other_chain, chain + other_chain. etc).

    '''
    
    # Pretty standard magic methods.
    
    def __init__(self, **kwargs):
        self.order = kwargs.get('order', 2)
        super(MarkovChain, self).__init__(list)

    def __repr__(self):
        pattern = "<{} Order:{} Length:{}>"
        return pattern.format(type(self).__name__, self.order, len(self))

    # Modified equality testers.

    def __eq__(self, other):
        '''Equality for Markov chains is resolved based on order,
        rather than content. Instead of running self.order == other.order
        every time (despite that being more explicit), this allows a
        shortcut to that.

        '''
        if not isinstance(other, MarkovChain):
            return NotImplemented
        return self.order == other.order

    def __ne__(self, other):
        '''See self.__eq__ for implementation details.

        '''
        return not self.__eq__(other)

    # Inequality comparions.
    # These are actually set comparisons.

    def __lt__(self, other):
        '''Tests if self is a proper subset of other.
        That is set(self.keys()) <= set(self.keys()) and 
                set(self.keys()) != set(other.keys())

        '''
        if not isinstance(other, MarkovChain):
            return NotImplemented
        return set(self.keys()) < set(other.keys())

    def __le__(self, other):
        '''Tests if every element in self is in other.

        '''
        if not isinstance(other, MarkovChain):
            return NotImplemented
        return set(self.keys()) <= set(other.keys())

    def __gt__(self, other):
        '''Tests if self is a proper superset of other.
        That is set(self.keys()) > set(other.keys()) and
                set(self.keys()) != set(other.keys())

        '''
        if not isinstance(other, MarkovChain):
            return NotImplemented
        return set(self.keys()) > set(other.keys())

    def __ge__(self, other):
        '''Tests if every element in other is in self.

        '''
        if not isinstance(other, MarkovChain):
            return NotImplemented
        return set(self.keys()) >= set(other.keys())

    # container methods

    def __contains__(self, search):
        if isinstance(search, tuple):
            return bool(self._search_by_sequence(search))
        return any(search in key for key in self.keys())

    def __setitem__(self, key, value):
        #DANGER WILL ROBINSON!
        '''Over riden method to ensure data intergity.
        Ensures that key is a tuple with the proper length
        and that value is a list.

        Type checking...I know.

        '''

        if not isinstance(key, tuple):
            raise TypeError('MarkovChain expects tuple as key, {} provided.'
                                 ''.format(type(key).__name__))
        elif len(key) != self.order:
            raise AttributeError('MarkovChain expects key with length {}, '
                                 '{} length given'.format(self.order, len(key)))
        elif not isinstance(value, list):
            raise AttributeError('MarkovChain expects a list as a value, '
                                 '{} provided.'.format(type(value).__name__))
        else:
            super(MarkovChain, self).__setitem__(key, value)

{'fraction_non_alphanumeric': 0.09965306854886948, 'fraction_numerical': 0.00023926306974518484, 'mean_word_length': 2.300434267666798, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1047353', 'n_tokens_mistral': 2239, 'n_tokens_neox': 2135, 'n_words': 745}