<commit_before>from collections import defaultdict
from regulus.topo import Partition


class Merge(object):
    def __init__(self, level, is_max, src, dest):
        self.level = level
        self.is_max = is_max
        self.src = src
        self.dest = dest


class PartitionNode(object):
    _id_generator = -1

    @staticmethod
    def gen_id():
        PartitionNode._id_generator += 1
        return PartitionNode._id_generator

    @staticmethod
    def reset():
        PartitionNode._id_generator = -1

    def __init__(self, persistence, base_pts=None, min_idx=None, max_idx=None, from_partition=None, is_max=None):
        self.id = PartitionNode.gen_id()
        self.persistence = persistence
        self.span = []
        self.parent = None
        self.children = []

        self.extrema = []
        self.base_pts = base_pts if base_pts is not None else []
        self.min_idx = min_idx
        self.max_idx = max_idx
        self.max_merge = is_max

        if from_partition is not None:
            self.min_idx = from_partition.min_idx
            self.max_idx = from_partition.max_idx
            self.children.append(from_partition)
            from_partition.parent = self

    def add_child(self, child):
        child.parent = self
        self.children.append(child)
        # if child.min_idx != self.min_idx and child.max_idx != self.max_idx:
        #     print("ERROR: child {} [{} {}] merged into parent {} [{} {}] without a matching extrema".format(child.id,
        #                                             child.min_idx, child.max_idx, self.id, self.min_idx, self.max_idx))


class Builder(object):
    def __init__(self, debug=False):
        self.base = None
        self.merges = []
        self.min_map = defaultdict(set)
        self.max_map = defaultdict(set)
        self.active = set()
        self.root = None
        self.pts = []
        self.original_pts = set()
        self.debug = debug
        self.mapping = dict()
        self.unique = set()
        self.all = dict()
        self.data_pts = []
        self.single = 0

    def data(self, pts):
        self.data_pts = pts
        return self

    def msc(self, base, hierarchy):
        _, maxima = zip(*base.keys())
        maxima = set(maxima)
        self.base = base
        for dying, (persistence, surviving, saddle) in hierarchy.items():
            self.merges.append(Merge(persistence, dying in maxima, surviving, saddle))
        return self

    def build(self):
        self.prepare()
        self.merge()

        # get root
        if len(self.active) != 1:
            raise RuntimeError('Error: found {} roots'.format(len(self.active)))
        self.root = self.active.pop()

        self.single = 0
        idx = self.build_idx(self.root, 0)
        print('found {} singles'.format(self.single))
        print('len(idx)=', idx)

        self.test_uniques()

        self.pts.extend([self.root.min_idx, self.root.max_idx])
        self.rename(self.root, 0)
        return self

    # internal

    def merge(self):
        for record in self.merges:
            # print(record.level, record.is_max, record.src, record.dest)
            if record.src == record.dest:
                continue

            # merge.dest may have been merged already (same persistence level: degenerate case)
            dest = self.current(record.dest)
            src = self.current(record.src)

            if src == dest:
                continue

            record.dest = dest
            record.src = src
            self.mapping[record.src] = record.dest

            if record.is_max:
                self.collapse(record, self.max_map, lambda item: item.min_idx)
            else:
                self.collapse(record, self.min_map, lambda item: item.max_idx)

    def prepare(self):
        PartitionNode.reset()
        for key, value in self.base.items():
            m, x = [int(s) for s in key.split(',')]
            p = PartitionNode(0, list(value), m, x)
            self.add(p)

        # self.find_unique()
        self.remove_non_unique()

        self.merges.sort(key=lambda m: (m.level, m.src))
        high = self.merges[-1].level
        for merge in self.merges:
            merge.level /= high

        if self.debug:
            for partition in self.active:
                self.check_partition(partition)
<commit_msg>Parsing the base_partitions using a tuple as a key rather than a string that needs parsed.<commit_after>from collections import defaultdict
from regulus.topo import Partition


class Merge(object):
    def __init__(self, level, is_max, src, dest):
        self.level = level
        self.is_max = is_max
        self.src = src
        self.dest = dest


class PartitionNode(object):
    _id_generator = -1

    @staticmethod
    def gen_id():
        PartitionNode._id_generator += 1
        return PartitionNode._id_generator

    @staticmethod
    def reset():
        PartitionNode._id_generator = -1

    def __init__(self, persistence, base_pts=None, min_idx=None, max_idx=None, from_partition=None, is_max=None):
        self.id = PartitionNode.gen_id()
        self.persistence = persistence
        self.span = []
        self.parent = None
        self.children = []

        self.extrema = []
        self.base_pts = base_pts if base_pts is not None else []
        self.min_idx = min_idx
        self.max_idx = max_idx
        self.max_merge = is_max

        if from_partition is not None:
            self.min_idx = from_partition.min_idx
            self.max_idx = from_partition.max_idx
            self.children.append(from_partition)
            from_partition.parent = self

    def add_child(self, child):
        child.parent = self
        self.children.append(child)
        # if child.min_idx != self.min_idx and child.max_idx != self.max_idx:
        #     print("ERROR: child {} [{} {}] merged into parent {} [{} {}] without a matching extrema".format(child.id,
        #                                             child.min_idx, child.max_idx, self.id, self.min_idx, self.max_idx))


class Builder(object):
    def __init__(self, debug=False):
        self.base = None
        self.merges = []
        self.min_map = defaultdict(set)
        self.max_map = defaultdict(set)
        self.active = set()
        self.root = None
        self.pts = []
        self.original_pts = set()
        self.debug = debug
        self.mapping = dict()
        self.unique = set()
        self.all = dict()
        self.data_pts = []
        self.single = 0

    def data(self, pts):
        self.data_pts = pts
        return self

    def msc(self, base, hierarchy):
        _, maxima = zip(*base.keys())
        maxima = set(maxima)
        self.base = base
        for dying, (persistence, surviving, saddle) in hierarchy.items():
            self.merges.append(Merge(persistence, dying in maxima, surviving, saddle))
        return self

    def build(self):
        self.prepare()
        self.merge()

        # get root
        if len(self.active) != 1:
            raise RuntimeError('Error: found {} roots'.format(len(self.active)))
        self.root = self.active.pop()

        self.single = 0
        idx = self.build_idx(self.root, 0)
        print('found {} singles'.format(self.single))
        print('len(idx)=', idx)

        self.test_uniques()

        self.pts.extend([self.root.min_idx, self.root.max_idx])
        self.rename(self.root, 0)
        return self

    # internal

    def merge(self):
        for record in self.merges:
            # print(record.level, record.is_max, record.src, record.dest)
            if record.src == record.dest:
                continue

            # merge.dest may have been merged already (same persistence level: degenerate case)
            dest = self.current(record.dest)
            src = self.current(record.src)

            if src == dest:
                continue

            record.dest = dest
            record.src = src
            self.mapping[record.src] = record.dest

            if record.is_max:
                self.collapse(record, self.max_map, lambda item: item.min_idx)
            else:
                self.collapse(record, self.min_map, lambda item: item.max_idx)

    def prepare(self):
        PartitionNode.reset()
        for (m, x), value in self.base.items():
            p = PartitionNode(0, list(value), m, x)
            self.add(p)

        # self.find_unique()
        self.remove_non_unique()

        self.merges.sort(key=lambda m: (m.level, m.src))
        high = self.merges[-1].level
        for merge in self.merges:
            merge.level /= high

        if self.debug:
            for partition in self.active:
                self.check_partition(partition)

{'fraction_non_alphanumeric': 0.10752688172043011, 'fraction_numerical': 0.0022878059940517046, 'mean_word_length': 1.8902479338842975, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3675611', 'n_tokens_mistral': 2614, 'n_tokens_neox': 2533, 'n_words': 603}