<commit_before># @license
# Copyright 2016 Google Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, division, print_function

import collections
import time
import threading

import numpy as np

from .chunks import encode_jpeg, encode_npz, encode_raw
from .token import make_random_token
from . import downsample
from . import downsample_scales

class MeshImplementationNotAvailable(Exception):
    pass

class MeshesNotSupportedForVolume(Exception):
    pass

class InvalidObjectIdForMesh(Exception):
    pass

DownsamplingScaleInfo = collections.namedtuple('DownsamplingScaleInfo', ['key',
                                                                         'downsample_factor',
                                                                         'voxel_size',
                                                                         'shape', ])


def get_scale_key(scale):
    return '%d,%d,%d' % scale


class ServedVolume(object):
    def __init__(self,
                 data,
                 offset=None,
                 voxel_offset=None,
                 voxel_size=(1, 1, 1),
                 encoding='npz',
                 max_voxels_per_chunk_log2=None,
                 volume_type=None,
                 mesh_options=None,
                 downsampling='3d',
                 max_downsampling=downsample_scales.DEFAULT_MAX_DOWNSAMPLING,
                 max_downsampled_size=downsample_scales.DEFAULT_MAX_DOWNSAMPLED_SIZE,
                 max_downsampling_scales=downsample_scales.DEFAULT_MAX_DOWNSAMPLING_SCALES):
        """Initializes a ServedVolume.

        @param data: 3-d [z, y, x] array or 4-d [channel, z, y, x] array.

        @param downsampling: '3d' to use isotropic downsampling, '2d' to downsample separately in
            XY, XZ, and YZ, None to use no downsampling.

        @param max_downsampling: Maximum amount by which on-the-fly downsampling may reduce the
            volume of a chunk.  For example, 4x4x4 downsampling reduces the volume by 64.

        @param volume_type: either 'image' or 'segmentation'.  If not specified, guessed from the
            data type.

        @param voxel_size: Sequence [x, y, z] of floats.  Specifies the voxel size.

        @param mesh_options: A dict with the following keys specifying options for mesh
            simplification for 'segmentation' volumes:

                - max_quadrics_error: float.  Edge collapses with a larger associated quadrics error
                  than this amount are prohibited.  Set this to a negative number to disable mesh
                  simplification, and just use the original mesh produced by the marching cubes
                  algorithm.  Defaults to 1e6.  The effect of this value depends on the voxel_size.

                - max_normal_angle_deviation: float.  Edge collapses that change a triangle normal
                  by more than this angle are prohibited.  The angle is specified in degrees.
                  Defaults to 90.

                - lock_boundary_vertices: bool.  Retain all vertices along mesh surface boundaries,
                  which can only occur at the boundary of the volume.  Defaults to true.
        """
        self.token = make_random_token()
        self.max_voxels_per_chunk_log2 = max_voxels_per_chunk_log2
        self.data = data
        if voxel_offset is not None:
            if offset is not None:
                raise ValueError('Must specify at most one of \'offset\' and \'voxel_offset\'.')
            offset = tuple(voxel_offset * voxel_size)
        if offset is None:
            offset = (0, 0, 0)
        self.offset = offset
        self.data_type = data.dtype.name
        if self.data_type == 'float64':
            self.data_type = 'float32'
        self.encoding = encoding
        if len(data.shape) == 3:
            self.num_channels = 1
            original_shape = data.shape[::-1]
        else:
            if len(data.shape) != 4:
                raise ValueError('data array must be 3- or 4-dimensional.')
            self.num_channels = data.shape[0]
            original_shape = data.shape[1:][::-1]
        original_shape = np.array(original_shape)<commit_msg>fix(python): fix JSON serialization of volume offset.<commit_after># @license
# Copyright 2016 Google Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, division, print_function

import collections
import time
import threading

import numpy as np

from .chunks import encode_jpeg, encode_npz, encode_raw
from .token import make_random_token
from . import downsample
from . import downsample_scales

class MeshImplementationNotAvailable(Exception):
    pass

class MeshesNotSupportedForVolume(Exception):
    pass

class InvalidObjectIdForMesh(Exception):
    pass

DownsamplingScaleInfo = collections.namedtuple('DownsamplingScaleInfo', ['key',
                                                                         'downsample_factor',
                                                                         'voxel_size',
                                                                         'shape', ])


def get_scale_key(scale):
    return '%d,%d,%d' % scale


class ServedVolume(object):
    def __init__(self,
                 data,
                 offset=None,
                 voxel_offset=None,
                 voxel_size=(1, 1, 1),
                 encoding='npz',
                 max_voxels_per_chunk_log2=None,
                 volume_type=None,
                 mesh_options=None,
                 downsampling='3d',
                 max_downsampling=downsample_scales.DEFAULT_MAX_DOWNSAMPLING,
                 max_downsampled_size=downsample_scales.DEFAULT_MAX_DOWNSAMPLED_SIZE,
                 max_downsampling_scales=downsample_scales.DEFAULT_MAX_DOWNSAMPLING_SCALES):
        """Initializes a ServedVolume.

        @param data: 3-d [z, y, x] array or 4-d [channel, z, y, x] array.

        @param downsampling: '3d' to use isotropic downsampling, '2d' to downsample separately in
            XY, XZ, and YZ, None to use no downsampling.

        @param max_downsampling: Maximum amount by which on-the-fly downsampling may reduce the
            volume of a chunk.  For example, 4x4x4 downsampling reduces the volume by 64.

        @param volume_type: either 'image' or 'segmentation'.  If not specified, guessed from the
            data type.

        @param voxel_size: Sequence [x, y, z] of floats.  Specifies the voxel size.

        @param mesh_options: A dict with the following keys specifying options for mesh
            simplification for 'segmentation' volumes:

                - max_quadrics_error: float.  Edge collapses with a larger associated quadrics error
                  than this amount are prohibited.  Set this to a negative number to disable mesh
                  simplification, and just use the original mesh produced by the marching cubes
                  algorithm.  Defaults to 1e6.  The effect of this value depends on the voxel_size.

                - max_normal_angle_deviation: float.  Edge collapses that change a triangle normal
                  by more than this angle are prohibited.  The angle is specified in degrees.
                  Defaults to 90.

                - lock_boundary_vertices: bool.  Retain all vertices along mesh surface boundaries,
                  which can only occur at the boundary of the volume.  Defaults to true.
        """
        self.token = make_random_token()
        self.max_voxels_per_chunk_log2 = max_voxels_per_chunk_log2
        self.data = data
        if voxel_offset is not None:
            if offset is not None:
                raise ValueError('Must specify at most one of \'offset\' and \'voxel_offset\'.')
            offset = tuple(voxel_offset * voxel_size)
        if offset is None:
            offset = (0, 0, 0)
        self.offset = tuple(offset)
        self.data_type = data.dtype.name
        if self.data_type == 'float64':
            self.data_type = 'float32'
        self.encoding = encoding
        if len(data.shape) == 3:
            self.num_channels = 1
            original_shape = data.shape[::-1]
        else:
            if len(data.shape) != 4:
                raise ValueError('data array must be 3- or 4-dimensional.')
            self.num_channels = data.shape[0]
            original_shape = data.shape[1:][::-1]
        original_shape = np.array(original_shape)
{'fraction_non_alphanumeric': 0.0663216812757419, 'fraction_numerical': 0.00929348400042243, 'mean_word_length': 2.0947712418300655, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2242368', 'n_tokens_mistral': 2564, 'n_tokens_neox': 2394, 'n_words': 859}