<commit_before># -*- coding: utf-8 -*-


import itertools
import logging

import h5py
import math
import numpy as np
from PIL import Image
import pytoml as toml
import requests

from keras.utils.data_utils import get_file

from .config import CONFIG
from .octrees import OctreeVolume
from .regions import DenseRegion
from .util import pad_dims


class Volume(object):
    def __init__(self, image_data, label_data, resolution):
        self.image_data = image_data
        self.label_data = label_data
        self.image_data.flags.writeable = False
        self.label_data.flags.writeable = False
        self.resolution = resolution

    def xyz_coord_to_local(self, a):
        return a

    def xyz_mat_to_local(self, m):
        return m

    @property
    def shape(self):
        return tuple(self.xyz_coord_to_local(np.array(self.image_data.shape)))

    def region_generator(self, subvolume_size):
        subvolumes = self.SubvolumeGenerator(self, subvolume_size, CONFIG.volume.downsample)
        subvolumes = itertools.ifilter(has_uniform_seed_margin, subvolumes)
        return itertools.imap(DenseRegion.from_subvolume, subvolumes)

    class SubvolumeGenerator(object):
        def __init__(self, volume, size_zoom, downsample, partition=None):
            if partition is None:
                partition = (np.array((1, 1, 1)), np.array((0, 0, 0)))
            self.volume = volume
            self.partition = partition
            self.zoom = np.exp2(downsample).astype('uint64')
            self.size_zoom = size_zoom
            self.size_orig = np.multiply(self.size_zoom, self.zoom).astype('uint64')
            self.margin = np.floor_divide(self.size_orig, 2)
            self.partition_size = np.floor_divide(np.array(self.volume.shape), self.partition[0])
            self.ctr_min = (np.multiply(self.partition_size, self.partition[1]) + self.margin).astype('uint64')
            self.ctr_max = (np.multiply(self.partition_size, self.partition[1] + 1) - self.margin - 1).astype('uint64')
            self.random = np.random.RandomState(0)

        @property
        def shape(self):
            return self.size_zoom

        def __iter__(self):
            return self

        def reset(self):
            self.random.seed(0)

        def next(self):
            ctr = np.array([self.random.randint(self.ctr_min[n], self.ctr_max[n]) for n in range(3)]).astype('uint64')
            subvol = (self.volume.xyz_coord_to_local(ctr - self.margin),
                      self.volume.xyz_coord_to_local(ctr + self.margin + np.mod(self.size_orig, 2)))
            image_subvol = self.volume.image_data[
                    subvol[0][0]:subvol[1][0],
                    subvol[0][1]:subvol[1][1],
                    subvol[0][2]:subvol[1][2]]
            label_subvol = self.volume.label_data[
                    subvol[0][0]:subvol[1][0],
                    subvol[0][1]:subvol[1][1],
                    subvol[0][2]:subvol[1][2]]

            image_subvol = self.volume.xyz_mat_to_local(image_subvol.astype('float32')) / 256.0
            label_subvol = self.volume.xyz_mat_to_local(label_subvol)
            subvol_ctr = np.array(label_subvol.shape, dtype='uint64') / 2
            label_id = label_subvol[tuple(subvol_ctr)]
            label_mask = label_subvol == label_id

            if np.any(self.zoom > 1):
                image_subvol = image_subvol.reshape([self.size_zoom[0], self.zoom[0],
                                                     self.size_zoom[1], self.zoom[1],
                                                     self.size_zoom[2], self.zoom[2]]).mean(5).mean(3).mean(1)
                label_mask = label_mask.reshape([self.size_zoom[0], self.zoom[0],
                                                 self.size_zoom[1], self.zoom[1],
                                                 self.size_zoom[2], self.zoom[2]]).all(5).all(3).all(1)<commit_msg>Volumes: ensure subvol seed voxel is uniform<commit_after># -*- coding: utf-8 -*-


import itertools
import logging

import h5py
import math
import numpy as np
from PIL import Image
import pytoml as toml
import requests

from keras.utils.data_utils import get_file

from .config import CONFIG
from .octrees import OctreeVolume
from .regions import DenseRegion
from .util import pad_dims


class Volume(object):
    def __init__(self, image_data, label_data, resolution):
        self.image_data = image_data
        self.label_data = label_data
        self.image_data.flags.writeable = False
        self.label_data.flags.writeable = False
        self.resolution = resolution

    def xyz_coord_to_local(self, a):
        return a

    def xyz_mat_to_local(self, m):
        return m

    @property
    def shape(self):
        return tuple(self.xyz_coord_to_local(np.array(self.image_data.shape)))

    def region_generator(self, subvolume_size):
        subvolumes = self.SubvolumeGenerator(self, subvolume_size, CONFIG.volume.downsample)
        subvolumes = itertools.ifilter(has_uniform_seed_margin, subvolumes)
        return itertools.imap(DenseRegion.from_subvolume, subvolumes)

    class SubvolumeGenerator(object):
        def __init__(self, volume, size_zoom, downsample, partition=None):
            if partition is None:
                partition = (np.array((1, 1, 1)), np.array((0, 0, 0)))
            self.volume = volume
            self.partition = partition
            self.zoom = np.exp2(downsample).astype('uint64')
            self.size_zoom = size_zoom
            self.size_orig = np.multiply(self.size_zoom, self.zoom).astype('uint64')
            self.margin = np.floor_divide(self.size_orig, 2)
            self.partition_size = np.floor_divide(np.array(self.volume.shape), self.partition[0])
            self.ctr_min = (np.multiply(self.partition_size, self.partition[1]) + self.margin).astype('uint64')
            self.ctr_max = (np.multiply(self.partition_size, self.partition[1] + 1) - self.margin - 1).astype('uint64')
            self.random = np.random.RandomState(0)

        @property
        def shape(self):
            return self.size_zoom

        def __iter__(self):
            return self

        def reset(self):
            self.random.seed(0)

        def next(self):
            while True:
                # Only accept subvolumes where the central seed voxel will be
                # of a uniform label after downsampling. For more stringent
                # seed region uniformity filtering, see has_uniform_seed_margin.
                ctr = np.array([self.random.randint(self.ctr_min[n], self.ctr_max[n])
                                for n in range(3)]).astype('uint64')
                seed_min = ctr - np.floor_divide(self.zoom, 2).astype('uint64')
                seed_max = ctr + np.floor_divide(self.zoom, 2).astype('uint64') + np.mod(self.zoom, 2)
                if np.unique(self.volume.label_data[seed_min[0]:seed_max[0],
                                                    seed_min[1]:seed_max[1],
                                                    seed_min[2]:seed_max[2]]).size == 1:
                    break
            subvol = (self.volume.xyz_coord_to_local(ctr - self.margin),
                      self.volume.xyz_coord_to_local(ctr + self.margin + np.mod(self.size_orig, 2)))
            image_subvol = self.volume.image_data[
                    subvol[0][0]:subvol[1][0],
                    subvol[0][1]:subvol[1][1],
                    subvol[0][2]:subvol[1][2]]
            label_subvol = self.volume.label_data[
                    subvol[0][0]:subvol[1][0],
                    subvol[0][1]:subvol[1][1],
                    subvol[0][2]:subvol[1][2]]

            image_subvol = self.volume.xyz_mat_to_local(image_subvol.astype('float32')) / 256.0
            label_subvol = self.volume.xyz_mat_to_local(label_subvol)
            subvol_ctr = np.array(label_subvol.shape, dtype='uint64') / 2
            label_id = label_subvol[tuple(subvol_ctr)]
            label_mask = label_subvol == label_id

            if np.any(self.zoom > 1):
                image_subvol = image_subvol.reshape([self.size_zoom[0], self.zoom[0],
                                                     self.size_zoom[1], self.zoom[1],
                                                     self.size_zoom[2], self.zoom[2]]).mean(5).mean(3).mean(1)
                label_mask = label_mask.reshape([self.size_zoom[0], self.zoom[0],
                                                 self.size_zoom[1], self.zoom[1],
                                                 self.size_zoom[2], self.zoom[2]]).all(5).all(3).all(1)
{'fraction_non_alphanumeric': 0.11489163365182942, 'fraction_numerical': 0.020508040083896527, 'mean_word_length': 2.052275960170697, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '308357', 'n_tokens_mistral': 2689, 'n_tokens_neox': 2584, 'n_words': 462}