<commit_before>#!/usr/bin/env python
# pylint: disable=W0142
from copy import copy

class Intervals:
    '''Very simple interval implementation for integers (might also work on floats)'''

    def __init__(self, *args, **kwargs):
        '''if keywords *sane* is True (default: False), checks will not be done on given data.'''
        self._data = list(args)
        if not kwargs.get('sane', False):
            self._data = [d for d in self._data if d[1] > d[0]]
            self._enforce_order()
            self._enforce_no_overlap()

    def _enforce_order(self):
        self._data.sort(key=lambda d: d[0])

    def _enforce_no_overlap(self, start_at=0):
        i = start_at
        while i+1 < len(self._data):
            if self._data[i][1] >= self._data[i+1][0]:
                # beginning of i+1-th range is contained in i-th range
                if self._data[i][1] < self._data[i+1][1]:
                    # i+1-th range is longer, thus enlarge i-th range
                    self._data[i][1] = self._data[i+1][1]

                # removed contained range
                del self._data[i+1]
            i += 1

    def __and__(self, other):
        '''combines two intervals, under the assumption that they are sane'''
        return Intervals(*(self._data+other._data))

    def __len__(self):
        '''returns sum of range lengths'''
        return sum(upper-lower for (lower, upper) in self._data)

    def __contains__(self, needle):
        return any(lower <= needle < upper for (lower, upper) in self._data)

    def __repr__(self):
        return str(self.__class__) + '(' + ', '.join(map(list.__repr__, self._data)) + ')'

    def __cmp__(self, other):
        if self._data == other._data:
            return 0
        else:
            return 1

if __name__ == '__main__':
    assert Intervals([0, 10])._data == [[0, 10]]
    assert Intervals([0, 10], [1, 9])._data == [[0, 10]]
    assert Intervals([0, 10], [5, 15])._data == [[0, 15]]
    assert Intervals([-5, 5], [0, 10])._data == [[-5, 10]]
    assert Intervals([0, 9], [10, 11])._data == [[0, 9], [10, 11]]
    assert Intervals([0, 10], [10, 11])._data == [[0, 11]]
    assert Intervals([0, 5]) & Intervals([1, 9]) == Intervals([0, 9])
    assert Intervals([0, 5]) & Intervals([5, 9]) == Intervals([0, 9])
    assert Intervals([2, 4]) & Intervals([0, 9]) == Intervals([0, 9])
    assert len(Intervals([1, 2])) == 1
    assert 10 not in Intervals([0, 10]) and 0 in Intervals([0, 10])<commit_msg>cleaed up intervals module<commit_after>#!/usr/bin/env python
'''
A simple interval implementation
'''

class Intervals(object):
    '''Very simple interval implementation for integers (might also work on floats)'''

    def __init__(self, *args, **kwargs):
        '''If keywords *sane* is True (default: False), checks will not be done on given data.'''
        self.data = list(args)
        if not kwargs.get('sane', False):
            self.data = [d for d in self.data if d[1] > d[0]]
            self._enforce_order()
            self._enforce_no_overlap()

    def _enforce_order(self):
        '''Enforces the order of all entries in internal storage'''
        self.data.sort(key=lambda d: d[0])

    def _enforce_no_overlap(self, start_at=0):
        '''Enforces that no ranges overlap in internal storage'''
        i = start_at
        while i+1 < len(self.data):
            if self.data[i][1] >= self.data[i+1][0]:
                # beginning of i+1-th range is contained in i-th range
                if self.data[i][1] < self.data[i+1][1]:
                    # i+1-th range is longer, thus enlarge i-th range
                    self.data[i][1] = self.data[i+1][1]

                # removed contained range
                del self.data[i+1]
            i += 1

    def __and__(self, other):
        '''Combines two intervals, under the assumption that they are sane'''
        return Intervals(*(self.data+other.data))

    def __len__(self):
        '''Returns sum of range lengths'''
        return sum(upper-lower for (lower, upper) in self.data)

    def __contains__(self, needle):
        return any(lower <= needle < upper for (lower, upper) in self.data)

    def __repr__(self):
        return str(self.__class__) + '(' + ', '.join([list.__repr__(d) for d in self.data]) + ')'

    def __cmp__(self, other):
        if self.data == other.data:
            return 0
        else:
            return 1

if __name__ == '__main__':
    assert Intervals([0, 10]).data == [[0, 10]]
    assert Intervals([0, 10], [1, 9]).data == [[0, 10]]
    assert Intervals([0, 10], [5, 15]).data == [[0, 15]]
    assert Intervals([-5, 5], [0, 10]).data == [[-5, 10]]
    assert Intervals([0, 9], [10, 11]).data == [[0, 9], [10, 11]]
    assert Intervals([0, 10], [10, 11]).data == [[0, 11]]
    assert Intervals([0, 5]) & Intervals([1, 9]) == Intervals([0, 9])
    assert Intervals([0, 5]) & Intervals([5, 9]) == Intervals([0, 9])
    assert Intervals([2, 4]) & Intervals([0, 9]) == Intervals([0, 9])
    assert len(Intervals([1, 2])) == 1
    assert 10 not in Intervals([0, 10]) and 0 in Intervals([0, 10])
{'fraction_non_alphanumeric': 0.16513041772896647, 'fraction_numerical': 0.04118454598940969, 'mean_word_length': 2.495544893762851, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2061167', 'n_tokens_mistral': 1787, 'n_tokens_neox': 1629, 'n_words': 502}