<commit_before>INFINITY = float('inf')
NEGATIVE_INFINITY = -INFINITY


class IntervalSet:
    __slots__ = ('intervals', 'size')

    def __init__(self, intervals, disjoint=False):
        self.intervals = intervals

        if not disjoint:
            self.intervals = union_overlapping(self.intervals)

        self.size = sum(i.size for i in self.intervals)

    def __repr__(self):
        return repr(self.intervals)

    def __iter__(self):
        return iter(self.intervals)

    def __len__(self):
        return len(self.intervals)

    def __getitem__(self, i):
        return self.intervals[i]

    def __nonzero__(self):
        return self.size != 0

    def __sub__(self, other):
        return self.intersect( other.complement() )

    def complement(self):
        complementary = []
        cursor = NEGATIVE_INFINITY

        for interval in self.intervals:
            if cursor < interval.start:
                complementary.append( Interval(cursor, interval.start) )
                cursor = interval.end

        if cursor < INFINITY:
            complementary.append( Interval(cursor, INFINITY) )

        return IntervalSet(complementary, disjoint=True)

    def intersect(self, other): #XXX The last major bottleneck. Factorial-time hell.
        # Then again, this function is entirely unused...
        if (not self) or (not other):
            return IntervalSet([])

        #earliest = max(self.intervals[0].start, other.intervals[0].start)
        #latest = min(self.intervals[-1].end, other.intervals[-1].end)

        #mine = [i for i in self.intervals if i.start >= earliest and i.end <= latest]
        #theirs = [i for i in other.intervals if i.start >= earliest and i.end <= latest]

        intersections = [x for x in (i.intersect(j)
                                     for i in self.intervals
                                     for j in other.intervals)
                         if x]

        return IntervalSet(intersections, disjoint=True)

    def intersect_interval(self, interval):
        intersections = [x for x in (i.intersect(interval)
                                     for i in self.intervals)
                         if x]
        return IntervalSet(intersections, disjoint=True)

    def union(self, other):
        return IntervalSet( sorted(self.intervals + other.intervals) )


class Interval:
    __slots__ = ('start', 'end', 'tuple', 'size')

    def __init__(self, start, end):
        if end - start < 0:
            raise ValueError("Invalid interval start=%s end=%s" % (start, end))

        self.start = start
        self.end = end
        self.tuple = (start, end)
        self.size = self.end - self.start<commit_msg>Resolves #2692 can't unpickle Interval/IntervalSet<commit_after>INFINITY = float('inf')
NEGATIVE_INFINITY = -INFINITY


class IntervalSet:

    def __init__(self, intervals, disjoint=False):
        self.intervals = intervals

        if not disjoint:
            self.intervals = union_overlapping(self.intervals)

        self.size = sum(i.size for i in self.intervals)

    def __repr__(self):
        return repr(self.intervals)

    def __iter__(self):
        return iter(self.intervals)

    def __len__(self):
        return len(self.intervals)

    def __getitem__(self, i):
        return self.intervals[i]

    def __nonzero__(self):
        return self.size != 0

    def __sub__(self, other):
        return self.intersect( other.complement() )

    def complement(self):
        complementary = []
        cursor = NEGATIVE_INFINITY

        for interval in self.intervals:
            if cursor < interval.start:
                complementary.append( Interval(cursor, interval.start) )
                cursor = interval.end

        if cursor < INFINITY:
            complementary.append( Interval(cursor, INFINITY) )

        return IntervalSet(complementary, disjoint=True)

    def intersect(self, other): #XXX The last major bottleneck. Factorial-time hell.
        # Then again, this function is entirely unused...
        if (not self) or (not other):
            return IntervalSet([])

        #earliest = max(self.intervals[0].start, other.intervals[0].start)
        #latest = min(self.intervals[-1].end, other.intervals[-1].end)

        #mine = [i for i in self.intervals if i.start >= earliest and i.end <= latest]
        #theirs = [i for i in other.intervals if i.start >= earliest and i.end <= latest]

        intersections = [x for x in (i.intersect(j)
                                     for i in self.intervals
                                     for j in other.intervals)
                         if x]

        return IntervalSet(intersections, disjoint=True)

    def intersect_interval(self, interval):
        intersections = [x for x in (i.intersect(interval)
                                     for i in self.intervals)
                         if x]
        return IntervalSet(intersections, disjoint=True)

    def union(self, other):
        return IntervalSet( sorted(self.intervals + other.intervals) )


class Interval:

    def __init__(self, start, end):
        if end - start < 0:
            raise ValueError("Invalid interval start=%s end=%s" % (start, end))

        self.start = start
        self.end = end
        self.tuple = (start, end)
        self.size = self.end - self.start
{'fraction_non_alphanumeric': 0.09990627928772258, 'fraction_numerical': 0.002999062792877226, 'mean_word_length': 2.0526315789473686, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4095071', 'n_tokens_mistral': 1579, 'n_tokens_neox': 1491, 'n_words': 417}