<commit_before>import sys
import numpy as np
import scipy
import scipy.io
import scipy.ndimage
import mahotas
import math
import h5py
import time
import pymaxflow
import timer
import os

Debug = False

job_repeat_attempts = 5

def check_file(filename):
    if not os.path.exists(filename):
        return False
    # verify the file has the expected data
    import h5py
    f = h5py.File(filename, 'r')
    fkeys = f.keys()
    f.close()
    if set(fkeys) != set(['labels']):
        os.unlink(filename)
        return False
    return True

input_labels = sys.argv[1]
input_probs = sys.argv[2]
output_path = sys.argv[3]

repeat_attempt_i = 0
while repeat_attempt_i < job_repeat_attempts and not check_file(output_path):

    repeat_attempt_i += 1

    try:
        
        ## Open the input images
        input_labels_hdf5 = h5py.File(input_labels, 'r')
        label_vol = input_labels_hdf5['labels'][...]
        input_labels_hdf5.close()

        input_probs_hdf5 = h5py.File(input_probs, 'r')
        prob_vol = input_probs_hdf5['probabilities'][...]
        input_probs_hdf5.close()

        # Compress labels to 32 bit
        inverse, packed_vol = np.unique(label_vol, return_inverse=True)
        nlabels = len(inverse)

        if nlabels <= 1:
            print "Cleanup only found {0} segment - nothing to do.".format(nlabels)
            clean_vol = label_vol
        else:

            packed_vol = np.reshape(packed_vol, label_vol.shape)

            print "Cleanup starting with {0} segments.".format(nlabels)

            # Grow labels so there are no boundary pixels
            for image_i in range(packed_vol.shape[2]):
                label_image = packed_vol[:,:,image_i]
                packed_vol[:,:,image_i] = mahotas.cwatershed(np.zeros(label_image.shape, dtype=np.uint32), label_image, return_lines=False)

            if Debug:
                from libtiff import TIFF
                for image_i in range(packed_vol.shape[2]):
                    tif = TIFF.open('preclean_z{0:04}.tif'.format(image_i), mode='w')
                    tif.write_image(np.uint8(packed_vol[:, :, image_i] * 13 % 251))

            # Determine label adjicency and sizes

            borders = np.zeros(packed_vol.shape, dtype=np.bool)

            # Code currently only supports a 3d volume
            assert(packed_vol.ndim == 3)

            with timer.Timer("adjicency matrix construction"):

                full_npix = scipy.sparse.coo_matrix((nlabels, nlabels), dtype=np.uint32)
                full_prob = scipy.sparse.coo_matrix((nlabels, nlabels), dtype=np.float32)

                for axis in range(packed_vol.ndim):

                    for direction in [-1,1]:

                        # Roll the volume to find neighbours
                        shifted_vol = np.roll(packed_vol, direction, axis)

                        # Don't wrap around
                        if axis == 0:
                            shifted_vol[-1 if direction == -1 else 0, :, :] = 0
                        if axis == 1:
                            shifted_vol[:, -1 if direction == -1 else 0, :] = 0
                        if axis == 2:
                            shifted_vol[:, :, -1 if direction == -1 else 0] = 0

                        # Identify neighbours
                        borders = np.logical_and(shifted_vol != 0, packed_vol != shifted_vol)
                        from_labels = packed_vol[borders]
                        to_labels = shifted_vol[borders]

                        direction_npix = scipy.sparse.coo_matrix((np.ones(from_labels.shape, dtype=np.uint32), (from_labels, to_labels)), dtype=np.uint32, shape=(nlabels, nlabels))
                        direction_prob = scipy.sparse.coo_matrix((prob_vol[borders], (from_labels, to_labels)), dtype=np.float32, shape=(nlabels, nlabels))

                        full_npix = full_npix + direction_npix
                        full_prob = full_prob + direction_prob

                full_npix = full_npix + full_npix.transpose()
                full_prob = full_prob + full_prob.transpose()

                #full_npix = full_npix.tocsr()
                #full_prob = full_prob.tocsr()
                full_conn = full_npix / full_npix
                full_mean = full_prob / full_npix

            with timer.Timer("segment size calculation"):
                label_sizes = np.bincount(packed_vol.ravel())

            remap_index = np.arange(nlabels)

            def join_segs(segi, best_seg):
                while best_seg != remap_index[best_seg]:<commit_msg>keep normalized matrices as sparse<commit_after>import sys
import numpy as np
import scipy
import scipy.io
import scipy.ndimage
import mahotas
import math
import h5py
import time
import pymaxflow
import timer
import os

Debug = False

job_repeat_attempts = 5

def check_file(filename):
    if not os.path.exists(filename):
        return False
    # verify the file has the expected data
    import h5py
    f = h5py.File(filename, 'r')
    fkeys = f.keys()
    f.close()
    if set(fkeys) != set(['labels']):
        os.unlink(filename)
        return False
    return True

input_labels = sys.argv[1]
input_probs = sys.argv[2]
output_path = sys.argv[3]

repeat_attempt_i = 0
while repeat_attempt_i < job_repeat_attempts and not check_file(output_path):

    repeat_attempt_i += 1

    try:
        
        ## Open the input images
        input_labels_hdf5 = h5py.File(input_labels, 'r')
        label_vol = input_labels_hdf5['labels'][...]
        input_labels_hdf5.close()

        input_probs_hdf5 = h5py.File(input_probs, 'r')
        prob_vol = input_probs_hdf5['probabilities'][...]
        input_probs_hdf5.close()

        # Compress labels to 32 bit
        inverse, packed_vol = np.unique(label_vol, return_inverse=True)
        nlabels = len(inverse)

        if nlabels <= 1:
            print "Cleanup only found {0} segment - nothing to do.".format(nlabels)
            clean_vol = label_vol
        else:

            packed_vol = np.reshape(packed_vol, label_vol.shape)

            print "Cleanup starting with {0} segments.".format(nlabels)

            # Grow labels so there are no boundary pixels
            for image_i in range(packed_vol.shape[2]):
                label_image = packed_vol[:,:,image_i]
                packed_vol[:,:,image_i] = mahotas.cwatershed(np.zeros(label_image.shape, dtype=np.uint32), label_image, return_lines=False)

            if Debug:
                from libtiff import TIFF
                for image_i in range(packed_vol.shape[2]):
                    tif = TIFF.open('preclean_z{0:04}.tif'.format(image_i), mode='w')
                    tif.write_image(np.uint8(packed_vol[:, :, image_i] * 13 % 251))

            # Determine label adjicency and sizes

            borders = np.zeros(packed_vol.shape, dtype=np.bool)

            # Code currently only supports a 3d volume
            assert(packed_vol.ndim == 3)

            with timer.Timer("adjicency matrix construction"):

                full_npix = scipy.sparse.coo_matrix((nlabels, nlabels), dtype=np.uint32)
                full_prob = scipy.sparse.coo_matrix((nlabels, nlabels), dtype=np.float32)

                for axis in range(packed_vol.ndim):

                    for direction in [-1,1]:

                        # Roll the volume to find neighbours
                        shifted_vol = np.roll(packed_vol, direction, axis)

                        # Don't wrap around
                        if axis == 0:
                            shifted_vol[-1 if direction == -1 else 0, :, :] = 0
                        if axis == 1:
                            shifted_vol[:, -1 if direction == -1 else 0, :] = 0
                        if axis == 2:
                            shifted_vol[:, :, -1 if direction == -1 else 0] = 0

                        # Identify neighbours
                        borders = np.logical_and(shifted_vol != 0, packed_vol != shifted_vol)
                        from_labels = packed_vol[borders]
                        to_labels = shifted_vol[borders]

                        direction_npix = scipy.sparse.coo_matrix((np.ones(from_labels.shape, dtype=np.uint32), (from_labels, to_labels)), dtype=np.uint32, shape=(nlabels, nlabels))
                        direction_prob = scipy.sparse.coo_matrix((prob_vol[borders], (from_labels, to_labels)), dtype=np.float32, shape=(nlabels, nlabels))

                        full_npix = full_npix + direction_npix
                        full_prob = full_prob + direction_prob

                full_npix = full_npix + full_npix.transpose()
                full_prob = full_prob + full_prob.transpose()

                #full_npix = full_npix.tocsr()
                #full_prob = full_prob.tocsr()
                full_conn = scipy.sparse.csr_matrix(full_npix / full_npix)
                full_mean = scipy.sparse.csr_matrix(full_prob / full_npix)

            with timer.Timer("segment size calculation"):
                label_sizes = np.bincount(packed_vol.ravel())

            remap_index = np.arange(nlabels)

            def join_segs(segi, best_seg):
                while best_seg != remap_index[best_seg]:
{'fraction_non_alphanumeric': 0.09634079737848171, 'fraction_numerical': 0.014199890770071, 'mean_word_length': 1.9168525007964319, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3268867', 'n_tokens_mistral': 2912, 'n_tokens_neox': 2712, 'n_words': 619}