<commit_before>#! /usr/bin/env python3
#
# Copyright (c) 2016, 2017, Forschungszentrum Juelich GmbH
# Author: <NAME> <<EMAIL>>
#
# This software is made available under the MIT licence, see LICENCE.txt.


import copy
import gzip
import json
import math
import os
import os.path
import sys

import numpy as np


raw_chunk_scheme = "raw/{key}/{0}-{1}/{2}-{3}/{4}-{5}"


def create_next_scale(info, source_scale_index, outside_value=0):
    # Key is the resolution in micrometres
    old_scale_info = info["scales"][source_scale_index]
    new_scale_info = info["scales"][source_scale_index + 1]
    old_chunk_size = old_scale_info["chunk_sizes"][0]
    new_chunk_size = new_scale_info["chunk_sizes"][0]
    old_key = old_scale_info["key"]
    new_key = new_scale_info["key"]
    old_size = old_scale_info["size"]
    new_size = new_scale_info["size"]
    dtype = np.dtype(info["data_type"])
    if dtype.byteorder != "|":
        dtype.byteorder = "<"
    num_channels = info["num_channels"]
    downscaling_factors = [(os + 1) // ns
                           for os, ns in zip(old_size, new_size)]
    half_chunk = [osz // f
                  for osz, f in zip(old_chunk_size, downscaling_factors)]
    chunk_fetch_factor = [nsz // hc
                          for nsz, hc in zip(new_chunk_size, half_chunk)]

    def load_and_downscale_old_chunk(z_idx, y_idx, x_idx):
        xmin = old_chunk_size[0] * x_idx
        xmax = min(old_chunk_size[0] * (x_idx + 1), old_size[0])
        ymin = old_chunk_size[1] * y_idx
        ymax = min(old_chunk_size[1] * (y_idx + 1), old_size[1])
        zmin = old_chunk_size[2] * z_idx
        zmax = min(old_chunk_size[2] * (z_idx + 1), old_size[2])
        chunk_filename = raw_chunk_scheme.format(
            xmin, xmax, ymin, ymax, zmin, zmax, key=old_key)

        try:
            f = open(chunk_filename, "rb")
        except OSError:
            f = gzip.open(chunk_filename + ".gz", "rb")
        with f:
            chunk = np.frombuffer(f.read(), dtype=dtype).reshape(
                [num_channels, zmax - zmin, ymax - ymin, xmax - xmin])
        chunk = chunk.astype(np.float32)  # unbounded type for arithmetic

        if downscaling_factors[2] == 2:
            if chunk.shape[1] % 2 != 0:
                chunk = np.pad(chunk, ((0, 0), (0, 1), (0, 0), (0, 0)),
                               "constant", constant_values=outside_value)
            chunk = (chunk[:, fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b, :, :] + chunk[:, fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b, :, :]) * 0.5

        if downscaling_factors[1] == 2:
            if chunk.shape[2] % 2 != 0:
                chunk = np.pad(chunk, ((0, 0), (0, 0), (0, 1), (0, 0)),
                               "constant", constant_values=outside_value)
            chunk = (chunk[:, :, fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b, :] + chunk[:, :, fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b, :]) * 0.5

        if downscaling_factors[0] == 2:
            if chunk.shape[3] % 2 != 0:
                chunk = np.pad(chunk, ((0, 0), (0, 0), (0, 0), (0, 1)),
                               "constant", constant_values=outside_value)
            chunk = (chunk[:, :, :, ::2] + chunk[:, :, :, 1::2]) * 0.5

        return chunk.astype(dtype)

    for x_idx in range((new_size[0] - 1) // new_chunk_size[0] + 1):
        for y_idx in range((new_size[1] - 1) // new_chunk_size[1] + 1):
            for z_idx in range((new_size[2] - 1) // new_chunk_size[2] + 1):
                xmin = new_chunk_size[0] * x_idx
                xmax = min(new_chunk_size[0] * (x_idx + 1), new_size[0])
                ymin = new_chunk_size[1] * y_idx
                ymax = min(new_chunk_size[1] * (y_idx + 1), new_size[1])
                zmin = new_chunk_size[2] * z_idx
                zmax = min(new_chunk_size[2] * (z_idx + 1), new_size[2])
                new_chunk = np.empty(
                    [num_channels, zmax - zmin, ymax - ymin, xmax - xmin],
                    dtype=dtype
                    )
                new_chunk[:, :half_chunk[2], :half_chunk[1],
                          :half_chunk[0]] = (
                              load_and_downscale_old_chunk(
                                  z_idx * chunk_fetch_factor[2],
                                  y_idx * chunk_fetch_factor[1],
                                  x_idx * chunk_fetch_factor[0]))
                if new_chunk.shape[1] > half_chunk[2]:
                    new_chunk[:, half_chunk[2]:, :half_chunk[1],
                              :half_chunk[0]] = (
                                  load_and_downscale_old_chunk(
                                      z_idx * chunk_fetch_factor[2] + 1,
                                      y_idx * chunk_fetch_factor[1],
                                      x_idx * chunk_fetch_factor[0]))
                if new_chunk.shape[2] > half_chunk[1]:
                    new_chunk[:, :half_chunk[2], half_chunk[1]:,
                              :half_chunk[0]] = (
                                  load_and_downscale_old_chunk(
                                      z_idx * chunk_fetch_factor[2],
                                      y_idx * chunk_fetch_factor[1] + 1,
                                      x_idx * chunk_fetch_factor[0]))
                if (new_chunk.shape[1] > half_chunk[2]
                    and new_chunk.shape[2] > half_chunk[1]):
                    new_chunk[:, half_chunk[2]:, half_chunk[1]:,
                              :half_chunk[0]] = (
                                  load_and_downscale_old_chunk(
                                      z_idx * chunk_fetch_factor[2] + 1,
                                      y_idx * chunk_fetch_factor[1] + 1,
                                      x_idx * chunk_fetch_factor[0]))
                if new_chunk.shape[3] > half_chunk[0]:
                    new_chunk[:, :half_chunk[2], :half_chunk[1],
                              half_chunk[0]:] = (
                                  load_and_downscale_old_chunk(
                                      z_idx * chunk_fetch_factor[2],
                                      y_idx * chunk_fetch_factor[1],
                                      x_idx * chunk_fetch_factor[0] + 1))
                if (new_chunk.shape[1] > half_chunk[2]
                    and new_chunk.shape[3] > half_chunk[0]):
                    new_chunk[:, half_chunk[2]:, :half_chunk[1],
                              half_chunk[0]:] = (
                                  load_and_downscale_old_chunk(
                                      z_idx * chunk_fetch_factor[2] + 1,
                                      y_idx * chunk_fetch_factor[1],
                                      x_idx * chunk_fetch_factor[0] + 1))
                if (new_chunk.shape[2] > half_chunk[1]
                    and new_chunk.shape[3] > half_chunk[0]):
                    new_chunk[:, :half_chunk[2], half_chunk[1]:,
                              half_chunk[0]:] = (load_and_downscale_old_chunk(
                                  z_idx * chunk_fetch_factor[2],
                                  y_idx * chunk_fetch_factor[1] + 1,
                                  x_idx * chunk_fetch_factor[0] + 1))
                if (new_chunk.shape[1] > half_chunk[2]
                    and new_chunk.shape[2] > half_chunk[1]
                    and new_chunk.shape[3] > half_chunk[0]):
                    new_chunk[:, half_chunk[2]:, half_chunk[1]:,
                              half_chunk[0]:] = (
                                  load_and_downscale_old_chunk(
                                      z_idx * chunk_fetch_factor[2] + 1,
                                      y_idx * chunk_fetch_factor[1] + 1,
                                      x_idx * chunk_fetch_factor[0] + 1))

                new_chunk_name = raw_chunk_scheme.format(
                    xmin, xmax, ymin, ymax, zmin, zmax, key=new_key)
                print("Writing", new_chunk_name)
                os.makedirs(os.path.dirname(new_chunk_name), exist_ok=True)
                with gzip.open(new_chunk_name + ".gz", "wb") as f:
                    f.write(new_chunk.astype(dtype).tobytes())


def compute_scales(outside_value):
    """Generate lower scales in Neuroglancer a list of scales from an input JSON file"""
    with open("info") as f:
        info = json.load(f)
    for i in range(len(info["scales"]) - 1):
        create_next_scale(info, i, outside_value=outside_value)
<commit_msg>work in current directory instead of ./raw, clean up code<commit_after>#! /usr/bin/env python3
#
# Copyright (c) 2016, 2017, Forschungszentrum Juelich GmbH
# Author: <NAME> <<EMAIL>>
#
# This software is made available under the MIT licence, see LICENCE.txt.


import gzip
import json
import os
import os.path
import sys

import numpy as np


RAW_CHUNK_PATTERN = "{key}/{0}-{1}/{2}-{3}/{4}-{5}"


def create_next_scale(info, source_scale_index, outside_value=0):
    # Key is the resolution in micrometres
    old_scale_info = info["scales"][source_scale_index]
    new_scale_info = info["scales"][source_scale_index + 1]
    old_chunk_size = old_scale_info["chunk_sizes"][0]
    new_chunk_size = new_scale_info["chunk_sizes"][0]
    old_key = old_scale_info["key"]
    new_key = new_scale_info["key"]
    old_size = old_scale_info["size"]
    new_size = new_scale_info["size"]
    dtype = np.dtype(info["data_type"])
    if dtype.byteorder != "|":
        dtype.byteorder = "<"
    num_channels = info["num_channels"]
    downscaling_factors = [(os + 1) // ns
                           for os, ns in zip(old_size, new_size)]
    half_chunk = [osz // f
                  for osz, f in zip(old_chunk_size, downscaling_factors)]
    chunk_fetch_factor = [nsz // hc
                          for nsz, hc in zip(new_chunk_size, half_chunk)]

    def load_and_downscale_old_chunk(z_idx, y_idx, x_idx):
        xmin = old_chunk_size[0] * x_idx
        xmax = min(old_chunk_size[0] * (x_idx + 1), old_size[0])
        ymin = old_chunk_size[1] * y_idx
        ymax = min(old_chunk_size[1] * (y_idx + 1), old_size[1])
        zmin = old_chunk_size[2] * z_idx
        zmax = min(old_chunk_size[2] * (z_idx + 1), old_size[2])
        chunk_filename = RAW_CHUNK_PATTERN.format(
            xmin, xmax, ymin, ymax, zmin, zmax, key=old_key)

        try:
            f = open(chunk_filename, "rb")
        except OSError:
            f = gzip.open(chunk_filename + ".gz", "rb")
        with f:
            chunk = np.frombuffer(f.read(), dtype=dtype).reshape(
                [num_channels, zmax - zmin, ymax - ymin, xmax - xmin])
        chunk = chunk.astype(np.float32)  # unbounded type for arithmetic

        if downscaling_factors[2] == 2:
            if chunk.shape[1] % 2 != 0:
                chunk = np.pad(chunk, ((0, 0), (0, 1), (0, 0), (0, 0)),
                               "constant", constant_values=outside_value)
            chunk = (chunk[:, fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b, :, :] + chunk[:, fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b, :, :]) * 0.5

        if downscaling_factors[1] == 2:
            if chunk.shape[2] % 2 != 0:
                chunk = np.pad(chunk, ((0, 0), (0, 0), (0, 1), (0, 0)),
                               "constant", constant_values=outside_value)
            chunk = (chunk[:, :, fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b, :] + chunk[:, :, fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b, :]) * 0.5

        if downscaling_factors[0] == 2:
            if chunk.shape[3] % 2 != 0:
                chunk = np.pad(chunk, ((0, 0), (0, 0), (0, 0), (0, 1)),
                               "constant", constant_values=outside_value)
            chunk = (chunk[:, :, :, ::2] + chunk[:, :, :, 1::2]) * 0.5

        return chunk.astype(dtype)

    for x_idx in range((new_size[0] - 1) // new_chunk_size[0] + 1):
        for y_idx in range((new_size[1] - 1) // new_chunk_size[1] + 1):
            for z_idx in range((new_size[2] - 1) // new_chunk_size[2] + 1):
                xmin = new_chunk_size[0] * x_idx
                xmax = min(new_chunk_size[0] * (x_idx + 1), new_size[0])
                ymin = new_chunk_size[1] * y_idx
                ymax = min(new_chunk_size[1] * (y_idx + 1), new_size[1])
                zmin = new_chunk_size[2] * z_idx
                zmax = min(new_chunk_size[2] * (z_idx + 1), new_size[2])
                new_chunk = np.empty(
                    [num_channels, zmax - zmin, ymax - ymin, xmax - xmin],
                    dtype=dtype
                    )
                new_chunk[:, :half_chunk[2], :half_chunk[1],
                          :half_chunk[0]] = (
                              load_and_downscale_old_chunk(
                                  z_idx * chunk_fetch_factor[2],
                                  y_idx * chunk_fetch_factor[1],
                                  x_idx * chunk_fetch_factor[0]))
                if new_chunk.shape[1] > half_chunk[2]:
                    new_chunk[:, half_chunk[2]:, :half_chunk[1],
                              :half_chunk[0]] = (
                                  load_and_downscale_old_chunk(
                                      z_idx * chunk_fetch_factor[2] + 1,
                                      y_idx * chunk_fetch_factor[1],
                                      x_idx * chunk_fetch_factor[0]))
                if new_chunk.shape[2] > half_chunk[1]:
                    new_chunk[:, :half_chunk[2], half_chunk[1]:,
                              :half_chunk[0]] = (
                                  load_and_downscale_old_chunk(
                                      z_idx * chunk_fetch_factor[2],
                                      y_idx * chunk_fetch_factor[1] + 1,
                                      x_idx * chunk_fetch_factor[0]))
                if (new_chunk.shape[1] > half_chunk[2]
                    and new_chunk.shape[2] > half_chunk[1]):
                    new_chunk[:, half_chunk[2]:, half_chunk[1]:,
                              :half_chunk[0]] = (
                                  load_and_downscale_old_chunk(
                                      z_idx * chunk_fetch_factor[2] + 1,
                                      y_idx * chunk_fetch_factor[1] + 1,
                                      x_idx * chunk_fetch_factor[0]))
                if new_chunk.shape[3] > half_chunk[0]:
                    new_chunk[:, :half_chunk[2], :half_chunk[1],
                              half_chunk[0]:] = (
                                  load_and_downscale_old_chunk(
                                      z_idx * chunk_fetch_factor[2],
                                      y_idx * chunk_fetch_factor[1],
                                      x_idx * chunk_fetch_factor[0] + 1))
                if (new_chunk.shape[1] > half_chunk[2]
                    and new_chunk.shape[3] > half_chunk[0]):
                    new_chunk[:, half_chunk[2]:, :half_chunk[1],
                              half_chunk[0]:] = (
                                  load_and_downscale_old_chunk(
                                      z_idx * chunk_fetch_factor[2] + 1,
                                      y_idx * chunk_fetch_factor[1],
                                      x_idx * chunk_fetch_factor[0] + 1))
                if (new_chunk.shape[2] > half_chunk[1]
                    and new_chunk.shape[3] > half_chunk[0]):
                    new_chunk[:, :half_chunk[2], half_chunk[1]:,
                              half_chunk[0]:] = (load_and_downscale_old_chunk(
                                  z_idx * chunk_fetch_factor[2],
                                  y_idx * chunk_fetch_factor[1] + 1,
                                  x_idx * chunk_fetch_factor[0] + 1))
                if (new_chunk.shape[1] > half_chunk[2]
                    and new_chunk.shape[2] > half_chunk[1]
                    and new_chunk.shape[3] > half_chunk[0]):
                    new_chunk[:, half_chunk[2]:, half_chunk[1]:,
                              half_chunk[0]:] = (
                                  load_and_downscale_old_chunk(
                                      z_idx * chunk_fetch_factor[2] + 1,
                                      y_idx * chunk_fetch_factor[1] + 1,
                                      x_idx * chunk_fetch_factor[0] + 1))

                new_chunk_name = RAW_CHUNK_PATTERN.format(
                    xmin, xmax, ymin, ymax, zmin, zmax, key=new_key)
                print("Writing", new_chunk_name)
                os.makedirs(os.path.dirname(new_chunk_name), exist_ok=True)
                with gzip.open(new_chunk_name + ".gz", "wb") as f:
                    f.write(new_chunk.astype(dtype).tobytes())


def compute_scales(outside_value):
    """Generate lower scales following an input info file"""
    with open("info") as f:
        info = json.load(f)
    for i in range(len(info["scales"]) - 1):
        create_next_scale(info, i, outside_value=outside_value)

{'fraction_non_alphanumeric': 0.12080935145078027, 'fraction_numerical': 0.03168575327834807, 'mean_word_length': 1.505425895644418, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 68, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '759897', 'n_tokens_mistral': 5719, 'n_tokens_neox': 5350, 'n_words': 977}