<commit_before>import numpy as np
import logging
import time
import concurrent.futures

from hytra.core.probabilitygenerator import IlpProbabilityGenerator, computeDivisionFeaturesOnCloud, computeRegionFeaturesOnCloud, DummyExecutor
from hytra.util.progressbar import ProgressBar

def getLogger():
    return logging.getLogger(__name__)

class ConflictingSegmentsProbabilityGenerator(IlpProbabilityGenerator):
    """
    Specialization of the probability generator that computes all the features on its own,
    to have more than one segmentation hypotheses per timeframe.

    First step: make sure that objects from different hypotheses have different IDs

    * do that by adding the maxId of the "previous" segmentation hypothesis for that frame
    * store reference which hypothesis this segment comes from in Traxel, so that we can reconstruct a result from the graph and images

    """

    def __init__(self, 
                 ilpOptions,
                 additionalLabelImageFilenames,
                 additionalLabelImagePaths,
                 turnOffFeatures=[], 
                 useMultiprocessing=True, 
                 pluginPaths=['hytra/plugins'],
                 verbose=False):
        """
        """
        super(ConflictingSegmentsProbabilityGenerator, self).__init__(ilpOptions,
                                                                      turnOffFeatures,
                                                                      useMultiprocessing,
                                                                      pluginPaths,
                                                                      verbose)
                                                                      
        # store the additional segmentation hypotheses and check that they are of the same size
        self._labelImageFilenames = additionalLabelImageFilenames
        self._labelImagePaths = additionalLabelImagePaths

        for filename, path in zip(self._labelImageFilenames, self._labelImagePaths):
            assert(self._pluginManager.getImageProvider().getImageShape(filename, path) == self.shape)
            assert(self._pluginManager.getImageProvider().getTimeRange(filename, path) == self.timeRange)
        
        self._labelImageFilenames.insert(0, ilpOptions.labelImageFilename)
        self._labelImagePaths.insert(0, ilpOptions.labelImagePath)
        self._labelImageFrameIdToGlobalId = {} # map from (labelImageFilename, frame, id) to (id)
        
    def fillTraxels(self, usePgmlink=True, ts=None, fs=None, dispyNodeIps=[], turnOffFeatures=[]):
        """
        Compute all the features and predict object count as well as division probabilities.
        Store the resulting information (and all other features) inside `self.TraxelsPerFrame`.

        It also computes which of the segmentation hypotheses overlap and are mutually exclusive, and stores 
        that per traxel, in each traxel's `conflictingTraxelIds` list. (Can only conflict within the timeframe)

        WARNING: usePgmlink is not supported for this derived class, so must be `False`!
        WARNING: distributed computation via Dispy is not supported here, so dispyNodeIps must be an empty list!
        """

        assert(not usePgmlink)
        assert(len(dispyNodeIps) == 0)

        super(ConflictingSegmentsProbabilityGenerator, self).fillTraxels(usePgmlink, ts, fs, dispyNodeIps, turnOffFeatures)

        # find exclusion constraints
        for frame in range(self.timeRange[0], self.timeRange[1]):
            for labelImageIndexA in range(len(self._labelImageFilenames)):
                labelImageA = self._pluginManager.getImageProvider().getLabelImageForFrame(self._labelImageFilenames[labelImageIndexA],
                                                                                            self._labelImagePaths[labelImageIndexA],
                                                                                            frame)
                for labelImageIndexB in range(labelImageIndexA + 1, len(self._labelImageFilenames)):
                    labelImageB = self._pluginManager.getImageProvider().getLabelImageForFrame(self._labelImageFilenames[labelImageIndexB],
                                                                                               self._labelImagePaths[labelImageIndexB],
                                                                                               frame)
                    # check for overlaps - even a 1-pixel overlap is enough to be mutually exclusive!
                    for objectIdA in np.unique(labelImageA):
                        if objectIdA == 0:
                            continue
                        overlapping = set(np.unique(labelImageB[labelImageA == objectIdA])) - set([0])
                        overlappingGlobalIds = [self._labelImageFrameIdToGlobalId[(self._labelImageFilenames[labelImageIndexB], frame, o)] for o in overlapping]
                        globalIdA = self._labelImageFrameIdToGlobalId[(self._labelImageFilenames[labelImageIndexA], frame, objectIdA)]
                        if self.TraxelsPerFrame[frame][globalIdA].conflictingTraxelIds is None:
                            self.TraxelsPerFrame[frame][globalIdA].conflictingTraxelIds = []
                        self.TraxelsPerFrame[frame][globalIdA].conflictingTraxelIds.extend(overlappingGlobalIds)
                        for globalIdB in overlappingGlobalIds:
                            if self.TraxelsPerFrame[frame][globalIdB].conflictingTraxelIds is None:
                                self.TraxelsPerFrame[frame][globalIdB].conflictingTraxelIds = []
                            self.TraxelsPerFrame[frame][globalIdB].conflictingTraxelIds.append(globalIdA)

    def _insertFilenameAndIdToFeatures(self, featureDict, filename):
        """
        For later disambiguation, we store for each row in the feature matrix which file it came from.
        We also store the label image id.  
        """

        # get num elements:
        numElements = None

        for _, v in featureDict.iteritems():
            try:
                currentNumElements = v.shape[0]
            except:
                currentNumElements = len(v)
<commit_msg>Add TODO that states why the test currently fails<commit_after>import numpy as np
import logging
import time
import concurrent.futures

from hytra.core.probabilitygenerator import IlpProbabilityGenerator, computeDivisionFeaturesOnCloud, computeRegionFeaturesOnCloud, DummyExecutor
from hytra.util.progressbar import ProgressBar

def getLogger():
    return logging.getLogger(__name__)

class ConflictingSegmentsProbabilityGenerator(IlpProbabilityGenerator):
    """
    Specialization of the probability generator that computes all the features on its own,
    to have more than one segmentation hypotheses per timeframe.

    First step: make sure that objects from different hypotheses have different IDs

    * do that by adding the maxId of the "previous" segmentation hypothesis for that frame
    * store reference which hypothesis this segment comes from in Traxel, so that we can reconstruct a result from the graph and images

    """

    def __init__(self, 
                 ilpOptions,
                 additionalLabelImageFilenames,
                 additionalLabelImagePaths,
                 turnOffFeatures=[], 
                 useMultiprocessing=True, 
                 pluginPaths=['hytra/plugins'],
                 verbose=False):
        """
        """
        super(ConflictingSegmentsProbabilityGenerator, self).__init__(ilpOptions,
                                                                      turnOffFeatures,
                                                                      useMultiprocessing,
                                                                      pluginPaths,
                                                                      verbose)
                                                                      
        # store the additional segmentation hypotheses and check that they are of the same size
        self._labelImageFilenames = additionalLabelImageFilenames
        self._labelImagePaths = additionalLabelImagePaths

        for filename, path in zip(self._labelImageFilenames, self._labelImagePaths):
            assert(self._pluginManager.getImageProvider().getImageShape(filename, path) == self.shape)
            assert(self._pluginManager.getImageProvider().getTimeRange(filename, path) == self.timeRange)
        
        self._labelImageFilenames.insert(0, ilpOptions.labelImageFilename)
        self._labelImagePaths.insert(0, ilpOptions.labelImagePath)
        self._labelImageFrameIdToGlobalId = {} # map from (labelImageFilename, frame, id) to (id)
        
    def fillTraxels(self, usePgmlink=True, ts=None, fs=None, dispyNodeIps=[], turnOffFeatures=[]):
        """
        Compute all the features and predict object count as well as division probabilities.
        Store the resulting information (and all other features) inside `self.TraxelsPerFrame`.

        It also computes which of the segmentation hypotheses overlap and are mutually exclusive, and stores 
        that per traxel, in each traxel's `conflictingTraxelIds` list. (Can only conflict within the timeframe)

        WARNING: usePgmlink is not supported for this derived class, so must be `False`!
        WARNING: distributed computation via Dispy is not supported here, so dispyNodeIps must be an empty list!
        """

        assert(not usePgmlink)
        assert(len(dispyNodeIps) == 0)

        super(ConflictingSegmentsProbabilityGenerator, self).fillTraxels(usePgmlink, ts, fs, dispyNodeIps, turnOffFeatures)

        # find exclusion constraints
        for frame in range(self.timeRange[0], self.timeRange[1]):
            for labelImageIndexA in range(len(self._labelImageFilenames)):
                labelImageA = self._pluginManager.getImageProvider().getLabelImageForFrame(self._labelImageFilenames[labelImageIndexA],
                                                                                            self._labelImagePaths[labelImageIndexA],
                                                                                            frame)
                for labelImageIndexB in range(labelImageIndexA + 1, len(self._labelImageFilenames)):
                    labelImageB = self._pluginManager.getImageProvider().getLabelImageForFrame(self._labelImageFilenames[labelImageIndexB],
                                                                                               self._labelImagePaths[labelImageIndexB],
                                                                                               frame)
                    # check for overlaps - even a 1-pixel overlap is enough to be mutually exclusive!
                    for objectIdA in np.unique(labelImageA):
                        if objectIdA == 0:
                            continue
                        overlapping = set(np.unique(labelImageB[labelImageA == objectIdA])) - set([0])
                        overlappingGlobalIds = [self._labelImageFrameIdToGlobalId[(self._labelImageFilenames[labelImageIndexB], frame, o)] for o in overlapping]
                        globalIdA = self._labelImageFrameIdToGlobalId[(self._labelImageFilenames[labelImageIndexA], frame, objectIdA)]
                        if self.TraxelsPerFrame[frame][globalIdA].conflictingTraxelIds is None:
                            self.TraxelsPerFrame[frame][globalIdA].conflictingTraxelIds = []
                        self.TraxelsPerFrame[frame][globalIdA].conflictingTraxelIds.extend(overlappingGlobalIds)
                        for globalIdB in overlappingGlobalIds:
                            if self.TraxelsPerFrame[frame][globalIdB].conflictingTraxelIds is None:
                                self.TraxelsPerFrame[frame][globalIdB].conflictingTraxelIds = []
                            self.TraxelsPerFrame[frame][globalIdB].conflictingTraxelIds.append(globalIdA)
        # FIXME: right now an object that overlaps with two objects in an alternative segmentation
        #        adds a constraint that the triplet cannot be active at once, but actually the two others could be active at once!

    def _insertFilenameAndIdToFeatures(self, featureDict, filename):
        """
        For later disambiguation, we store for each row in the feature matrix which file it came from.
        We also store the label image id.  
        """

        # get num elements:
        numElements = None

        for _, v in featureDict.iteritems():
            try:
                currentNumElements = v.shape[0]
            except:
                currentNumElements = len(v)

{'fraction_non_alphanumeric': 0.06073667711598746, 'fraction_numerical': 0.001567398119122257, 'mean_word_length': 1.9181340041161674, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 60, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3517090', 'n_tokens_mistral': 2961, 'n_tokens_neox': 2821, 'n_words': 792}