<commit_before># Multicut Pipeline implemented with luigi
# Taksks for defect and handling
import luigi

from customTargets import HDF5DataTarget, HDF5VolumeTarget
from dataTasks import InputData, ExternalSegmentation, StackedRegionAdjacencyGraph
from defectDetectionTasks import DefectSliceDetection
from featureTasks import EdgeFeatures, RegionFeatures, RegionNodeFeatures

from pipelineParameter import PipelineParameter
from tools import config_logger, run_decorator

import logging
import json

import os
import time
import numpy as np
import vigra
import nifty

from concurrent import futures


# init the workflow logger
workflow_logger = logging.getLogger(__name__)
config_logger(workflow_logger)


# map defect patches to overlapping superpixels
class DefectsToNodes(luigi.Task):

    pathToSeg = luigi.Parameter()

    def requires(self):
        return {'seg' : ExternalSegmentation(self.pathToSeg),
                'defects' : DefectSliceDetection(self.pathToSeg)}

    @run_decorator
    def run(self):
        inp = self.input()
        seg = inp['seg']
        seg.open()
        defects = inp['defects']
        defects.open()

        assert seg.shape == defects.shape

        ny = long(seg.shape[1])
        nx = long(seg.shape[2])

        def defects_to_nodes_z(z):
            slice_begin = [long(z),0L,0L]
            slice_end   = [long(z+1),ny,nx]
            defect_mask = defects.read(slice_begin,slice_end)
            if 1 in defect_mask:
                print z
                seg_z = seg.read(slice_begin,slice_end)
                where_defect = defect_mask == 1
                defect_nodes_slice = np.unique(seg_z[where_defect])
                return list(defect_nodes_slice), len(defect_nodes_slice) * [z]
            else:
                return [], []

        # non-parallel for debugging
        #defect_nodes = []
        #nodes_z = []
        #for z in xrange(seg.shape[0]):
        #    res = defects_to_nodes_z(z)
        #    defect_nodes.extend(res[0])
        #    nodes_z.extend(res[1])

        with futures.ThreadPoolExecutor(max_workers = PipelineParameter().nThreads) as executor:
            tasks = []
            for z in xrange(seg.shape[0]):
                tasks.append(executor.submit(defects_to_nodes_z,z))
            defect_nodes = []
            nodes_z      = []
            for fut in tasks:
                nodes, zz = fut.result()
                if nodes:
                    defect_nodes.extend(nodes)
                    nodes_z.extend(zz)

        assert len(defect_nodes) == len(nodes_z)

        workflow_logger.info("DefectsToNodes: found %i defected nodes" % (len(defect_nodes)))

        self.output().write(np.array(defect_nodes,dtype='uint32'),'defect_nodes')
        self.output().write(np.array(nodes_z,dtype='uint32'),'nodes_z')

    def output(self):
        segFile = os.path.split(self.pathToSeg)[1][:-3]
        save_path = os.path.join( PipelineParameter().cache, "DefectsToNodes_%s_nBins%i_binThreshold%i.h5" % (segFile,
            PipelineParameter().nBinsSliceStatistics,
            PipelineParameter().binThreshold) )
        return HDF5DataTarget(save_path)


# modify the rag to isolate defected superpixels
# introduce skip edges over the defected superpixels in z
class ModifiedAdjacency(luigi.Task):

    pathToSeg = luigi.Parameter()

    def requires(self):
        return {'rag' : StackedRegionAdjacencyGraph(self.pathToSeg),
                'seg' : ExternalSegmentation(self.pathToSeg),
                'defect_nodes' : DefectsToNodes(self.pathToSeg)}

    # compute and save the skip edges as tuples (n1,n2) as well
    # as the resulting modified adjacency (as plain graph)
    @run_decorator
    def run(self):
        inp = self.input()
        rag = inp['rag'].read()
        defect_nodes = inp['defect_nodes'].read('defect_nodes')
        nodes_z = inp['defect_nodes'].read('nodes_z')
        seg = inp['seg']
        seg.open()

        # make sure that z is monotonically increasing (not strictly!)
        assert np.all(np.diff(nodes_z.astype(int)) >= 0), "Defected slice index is not increasing monotonically!"

        edge_offset = rag.totalNumberOfInSliceEdges
        ny = long(seg.shape[1])
        nx = long(seg.shape[2])

        # loop over the defect nodes and find the skip edges,
        # compute the modify adjacency

        delete_edges = [] # the z-edges between defected and non-defected nodes that are deleted from the graph
        ignore_edges = [] # the xy-edges between defected and non-defected nodes, that will be set to maximally repulsive weights

        skip_edges   = [] # the skip edges that run over the defects in z
        skip_ranges  = [] # z-distance of the skip edges
        skip_starts  = [] # starting slices of the skip edges

        # get the original rag adjacency
        uv_ids = rag.uvIds()
        n_nodes = uv_ids.max() + 1

        defect_slices = np.unique(nodes_z)

        # FIXME TODO can't do this here once we have individual defect patches
        consecutive_defect_slices = np.split(defect_slices, np.where(np.diff(defect_slices) != 1)[0] + 1)
        has_lower_defect_list = []
        for consec in consecutive_defect_slices:
            if len(consec) > 1:
                has_lower_defect_list.extend(consec[1:])
        print has_lower_defect_list

        for i, z in enumerate(defect_slices):
            print 'Processing slice', z, ':', i, '/', len(defect_slices)
            has_lower_defect = z in has_lower_defect_list
            delete_edges_z, ignore_edges_z, skip_edges_z, skip_ranges_z = nifty.graph.rag.getSkipEdgesForSlice(
                rag,
                int(z),
                defect_nodes,
                defect_nodes[nodes_z==z],
                has_lower_defect)

            print len(delete_edges_z)
            print len(ignore_edges_z)
            print len(skip_edges_z)
            print len(skip_ranges_z)

            delete_edges.extend(delete_edges_z)
            ignore_edges.extend(ignore_edges_z)

            assert len(skip_edges_z) == len(skip_ranges_z)
            skip_edges.extend(skip_edges_z)
            skip_ranges.extend(skip_ranges_z)
            skip_starts.extend(len(skip_edges_z) * [z])

        delete_edges = np.unique(delete_edges).astype(np.uint32)
        uv_ids = np.delete(uv_ids,delete_edges,axis=0)
        workflow_logger.info("ModifiedAdjacency: deleted %i z-edges due to defects" % len(delete_edges) )

        skip_edges = np.array(skip_edges, dtype = np.uint32)
        skips_before_unique = skip_edges.shape[0]
        print skip_edges.shape

        # make the skip edges unique, keeping rows (see http://stackoverflow.com/questions/16970982/find-unique-rows-in-numpy-array):<commit_msg>Debugged nifty skip edges<commit_after># Multicut Pipeline implemented with luigi
# Taksks for defect and handling
import luigi

from customTargets import HDF5DataTarget, HDF5VolumeTarget
from dataTasks import InputData, ExternalSegmentation, StackedRegionAdjacencyGraph
from defectDetectionTasks import DefectSliceDetection
from featureTasks import EdgeFeatures, RegionFeatures, RegionNodeFeatures

from pipelineParameter import PipelineParameter
from tools import config_logger, run_decorator

import logging
import json

import os
import time
import numpy as np
import vigra
import nifty

from concurrent import futures


# init the workflow logger
workflow_logger = logging.getLogger(__name__)
config_logger(workflow_logger)


# map defect patches to overlapping superpixels
class DefectsToNodes(luigi.Task):

    pathToSeg = luigi.Parameter()

    def requires(self):
        return {'seg' : ExternalSegmentation(self.pathToSeg),
                'defects' : DefectSliceDetection(self.pathToSeg)}

    @run_decorator
    def run(self):
        inp = self.input()
        seg = inp['seg']
        seg.open()
        defects = inp['defects']
        defects.open()

        assert seg.shape == defects.shape

        ny = long(seg.shape[1])
        nx = long(seg.shape[2])

        def defects_to_nodes_z(z):
            slice_begin = [long(z),0L,0L]
            slice_end   = [long(z+1),ny,nx]
            defect_mask = defects.read(slice_begin,slice_end)
            if 1 in defect_mask:
                print z
                seg_z = seg.read(slice_begin,slice_end)
                where_defect = defect_mask == 1
                defect_nodes_slice = np.unique(seg_z[where_defect])
                return list(defect_nodes_slice), len(defect_nodes_slice) * [z]
            else:
                return [], []

        # non-parallel for debugging
        #defect_nodes = []
        #nodes_z = []
        #for z in xrange(seg.shape[0]):
        #    res = defects_to_nodes_z(z)
        #    defect_nodes.extend(res[0])
        #    nodes_z.extend(res[1])

        with futures.ThreadPoolExecutor(max_workers = PipelineParameter().nThreads) as executor:
            tasks = []
            for z in xrange(seg.shape[0]):
                tasks.append(executor.submit(defects_to_nodes_z,z))
            defect_nodes = []
            nodes_z      = []
            for fut in tasks:
                nodes, zz = fut.result()
                if nodes:
                    defect_nodes.extend(nodes)
                    nodes_z.extend(zz)

        assert len(defect_nodes) == len(nodes_z)

        workflow_logger.info("DefectsToNodes: found %i defected nodes" % (len(defect_nodes)))

        self.output().write(np.array(defect_nodes,dtype='uint32'),'defect_nodes')
        self.output().write(np.array(nodes_z,dtype='uint32'),'nodes_z')

    def output(self):
        segFile = os.path.split(self.pathToSeg)[1][:-3]
        save_path = os.path.join( PipelineParameter().cache, "DefectsToNodes_%s_nBins%i_binThreshold%i.h5" % (segFile,
            PipelineParameter().nBinsSliceStatistics,
            PipelineParameter().binThreshold) )
        return HDF5DataTarget(save_path)


# modify the rag to isolate defected superpixels
# introduce skip edges over the defected superpixels in z
class ModifiedAdjacency(luigi.Task):

    pathToSeg = luigi.Parameter()

    def requires(self):
        return {'rag' : StackedRegionAdjacencyGraph(self.pathToSeg),
                'seg' : ExternalSegmentation(self.pathToSeg),
                'defect_nodes' : DefectsToNodes(self.pathToSeg)}

    # compute and save the skip edges as tuples (n1,n2) as well
    # as the resulting modified adjacency (as plain graph)
    @run_decorator
    def run(self):
        inp = self.input()
        rag = inp['rag'].read()
        defect_nodes = inp['defect_nodes'].read('defect_nodes')
        nodes_z = inp['defect_nodes'].read('nodes_z')
        seg = inp['seg']
        seg.open()

        # make sure that z is monotonically increasing (not strictly!)
        assert np.all(np.diff(nodes_z.astype(int)) >= 0), "Defected slice index is not increasing monotonically!"

        edge_offset = rag.totalNumberOfInSliceEdges
        ny = long(seg.shape[1])
        nx = long(seg.shape[2])

        # loop over the defect nodes and find the skip edges,
        # compute the modify adjacency

        delete_edges = [] # the z-edges between defected and non-defected nodes that are deleted from the graph
        ignore_edges = [] # the xy-edges between defected and non-defected nodes, that will be set to maximally repulsive weights

        skip_edges   = [] # the skip edges that run over the defects in z
        skip_ranges  = [] # z-distance of the skip edges
        skip_starts  = [] # starting slices of the skip edges

        # get the original rag adjacency
        uv_ids = rag.uvIds()
        n_nodes = uv_ids.max() + 1

        defect_slices = np.unique(nodes_z)

        defect_node_dict = {int(z) : list(defect_nodes[nodes_z == z].astype(int)) for z in defect_slices}

        # FIXME TODO can't do this here once we have individual defect patches
        consecutive_defect_slices = np.split(defect_slices, np.where(np.diff(defect_slices) != 1)[0] + 1)
        has_lower_defect_list = []
        for consec in consecutive_defect_slices:
            if len(consec) > 1:
                has_lower_defect_list.extend(consec[1:])
        print "Slices with lower defect slice:", has_lower_defect_list

        for i, z in enumerate(defect_slices):
            print 'Processing slice', z, ':', i, '/', len(defect_slices)
            has_lower_defect = z in has_lower_defect_list
            delete_edges_z, ignore_edges_z, skip_edges_z, skip_ranges_z = nifty.graph.rag.getSkipEdgesForSlice(
                rag,
                int(z),
                defect_node_dict,
                has_lower_defect)

            delete_edges.extend(delete_edges_z)
            ignore_edges.extend(ignore_edges_z)

            assert len(skip_edges_z) == len(skip_ranges_z)
            skip_edges.extend(skip_edges_z)
            skip_ranges.extend(skip_ranges_z)
            skip_starts.extend(len(skip_edges_z) * [z-1])

        delete_edges = np.unique(delete_edges).astype(np.uint32)
        uv_ids = np.delete(uv_ids,delete_edges,axis=0)
        workflow_logger.info("ModifiedAdjacency: deleted %i z-edges due to defects" % len(delete_edges) )

        skip_edges = np.array(skip_edges, dtype = np.uint32)
        skips_before_unique = skip_edges.shape[0]
        print skip_edges.shape

        # make the skip edges unique, keeping rows (see http://stackoverflow.com/questions/16970982/find-unique-rows-in-numpy-array):
{'fraction_non_alphanumeric': 0.09400119083060435, 'fraction_numerical': 0.0069217028877642155, 'mean_word_length': 2.5765238221985625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3926723', 'n_tokens_mistral': 4265, 'n_tokens_neox': 4067, 'n_words': 924}