<commit_before>/**
 * @license
 * Copyright 2016 Google Inc.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * @file
 * Support for DVID (https://github.com/janelia-flyem/dvid) servers.
 */

import {ChunkManager} from 'neuroglancer/chunk_manager/frontend';
import {VolumeChunkSourceParameters, volumeSourceToString, TileChunkSourceParameters, tileSourceToString, TileEncoding} from 'neuroglancer/datasource/dvid/base';
import {registerDataSourceFactory, CompletionResult} from 'neuroglancer/datasource/factory';
import {DataType, VolumeType, VolumeChunkSpecification} from 'neuroglancer/sliceview/base';
import {ChunkLayout} from 'neuroglancer/sliceview/chunk_layout';
import {VolumeChunkSource as GenericVolumeChunkSource, MultiscaleVolumeChunkSource as GenericMultiscaleVolumeChunkSource} from 'neuroglancer/sliceview/frontend';
import {StatusMessage} from 'neuroglancer/status';
import {applyCompletionOffset, getPrefixMatchesWithDescriptions} from 'neuroglancer/util/completion';
import {vec3, Vec3} from 'neuroglancer/util/geom';
import {openShardedHttpRequest, sendHttpRequest} from 'neuroglancer/util/http_request';
import {parseFixedLengthArray, parseIntVec, stableStringify, verifyObject, verifyObjectProperty, verifyInt, verifyPositiveInt, verifyString, parseArray, verifyMapKey, verifyFinitePositiveFloat, verifyObjectAsMap} from 'neuroglancer/util/json';
import {CancellablePromise} from 'neuroglancer/util/promise';

let serverDataTypes = new Map<string, DataType>();
serverDataTypes.set('uint8', DataType.UINT8);
serverDataTypes.set('uint32', DataType.UINT32);
serverDataTypes.set('uint64', DataType.UINT64);

export class DataInstanceBaseInfo {
  get typeName(): string { return this.obj['TypeName']; }

  constructor (public obj: any) {
    verifyObject(obj);
    verifyObjectProperty(obj, 'TypeName', verifyString);
  }
};

export class DataInstanceInfo {
  constructor(public obj: any, public name: string, public base: DataInstanceBaseInfo) {}
};

export class VolumeChunkSource extends GenericVolumeChunkSource {
  constructor(
    chunkManager: ChunkManager, spec: VolumeChunkSpecification, public parameters: VolumeChunkSourceParameters) {
    super(chunkManager, spec);
    this.initializeCounterpart(chunkManager.rpc, {
      'type': 'dvid/VolumeChunkSource',
      'parameters': parameters,
    });
  }
  toString () {
    return volumeSourceToString(this.parameters);
  }
};

export class VolumeDataInstanceInfo extends DataInstanceInfo {
  dataType: DataType;
  lowerVoxelBound: Vec3;
  upperVoxelBound: Vec3;
  voxelSize: Vec3;
  numChannels: number;
  constructor(obj: any, name: string, base: DataInstanceBaseInfo, public volumeType: VolumeType) {
    super(obj, name, base);
    let extended = verifyObjectProperty(obj, 'Extended', verifyObject);
    let extendedValues = verifyObjectProperty(extended, 'Values', x => parseArray(x, verifyObject));
    if (extendedValues.length < 1) {
      throw new Error(
          'Expected Extended.Values property to have length >= 1, but received: ${JSON.stringify(extendedValues)}.');
    }
    this.dataType =
        verifyObjectProperty(extendedValues[0], 'DataType', x => verifyMapKey(x, serverDataTypes));
    this.lowerVoxelBound =
        verifyObjectProperty(extended, 'MinPoint', x => parseIntVec(vec3.create(), x));
    this.upperVoxelBound =
        verifyObjectProperty(extended, 'MaxPoint', x => parseIntVec(vec3.create(), x));
    this.voxelSize = verifyObjectProperty(
        extended, 'VoxelSize',
        x => parseFixedLengthArray(vec3.create(), x, verifyFinitePositiveFloat));
    this.numChannels = 1;
  }

  getSources(chunkManager: ChunkManager, parameters: VolumeChunkSourceParameters) {
    return [Array
                .from(VolumeChunkSpecification.getDefaults({
                  voxelSize: this.voxelSize,
                  dataType: this.dataType,
                  numChannels: this.numChannels,
                  lowerVoxelBound: this.lowerVoxelBound,
                  upperVoxelBound: this.upperVoxelBound,
                  volumeType: this.volumeType,
                }))
                .map(spec => {
                  return chunkManager.getChunkSource(
                      VolumeChunkSource, stableStringify(parameters),
                      () => new VolumeChunkSource(chunkManager, spec, parameters));
                })];
  }
};

export class TileLevelInfo {
  /**
   * Resolution of the two downsampled dimensions in the tile plane.  The tile depth is equal to the
   * base voxel size in that dimension.
   */
  resolution: Vec3;
  tileSize: Vec3;

  constructor (obj: any) {
    verifyObject(obj);
    this.resolution = verifyObjectProperty(
      obj, 'Resolution', x => parseFixedLengthArray(vec3.create(), x, verifyFinitePositiveFloat));
    this.tileSize = verifyObjectProperty(
      obj, 'TileSize', x => parseFixedLengthArray(vec3.create(), x, verifyPositiveInt));
  }
};

/**
 * Dimensions for which tiles are computed.
 *
 * FIXME: DVID does not seem to properly indicate which dimensions are available.
 */
const TILE_DIMS = [
  [0, 1],
  // [0, 2],
  // [1, 2],
];

export class TileChunkSource extends GenericVolumeChunkSource {
  constructor(
    chunkManager: ChunkManager, spec: VolumeChunkSpecification, public parameters: TileChunkSourceParameters) {
    super(chunkManager, spec);
    this.initializeCounterpart(chunkManager.rpc, {
      'type': 'dvid/TileChunkSource',
      'parameters': parameters,
    });
  }
  toString () {
    return tileSourceToString(this.parameters);
  }
};

export class TileDataInstanceInfo extends DataInstanceInfo {
  get dataType () { return DataType.UINT8; }
  get volumeType () { return VolumeType.IMAGE; }
  get numChannels () { return 1; }

  encoding: TileEncoding;

  /**
   * Base voxel size (nm).
   */
  voxelSize: Vec3;

  levels: Map<string, TileLevelInfo>;

  lowerVoxelBound: Vec3;
  upperVoxelBound: Vec3;

  constructor (obj: any, name: string, base: DataInstanceBaseInfo) {
    super(obj, name, base);
    let extended = verifyObjectProperty(obj, 'Extended', verifyObject);
    this.levels = verifyObjectProperty(extended, 'Levels', x => verifyObjectAsMap(x, y => new TileLevelInfo(y)));
    let baseLevel = this.levels.get('0');
    if (baseLevel === undefined) {
      throw new Error(`Level 0 is not defined.`);
    }
    this.voxelSize = baseLevel.resolution;
    let minTileCoord = verifyObjectProperty(
        extended, 'MinTileCoord', x => parseFixedLengthArray(vec3.create(), x, verifyInt));
    let maxTileCoord = verifyObjectProperty(
        extended, 'MaxTileCoord', x => parseFixedLengthArray(vec3.create(), x, verifyInt));
    this.lowerVoxelBound = vec3.multiply(vec3.create(), baseLevel.tileSize, minTileCoord);
    this.upperVoxelBound = vec3.multiply(vec3.create(), baseLevel.tileSize, maxTileCoord);

    let encodingNumber = verifyObjectProperty(extended, 'Encoding', x => x);
    switch (encodingNumber) {
      case 2:
        this.encoding = TileEncoding.JPEG;
        break;
      default:
        throw new Error(`Unsupported tile encoding: ${JSON.stringify(encodingNumber)}.`);
    }
  }

  getSources(chunkManager: ChunkManager, parameters: VolumeChunkSourceParameters): GenericVolumeChunkSource[][] {
    let sources: TileChunkSource[][] = [];
    for (let [level, levelInfo] of this.levels) {
      let alternatives = TILE_DIMS.map(dims => {
        let voxelSize = vec3.clone(this.voxelSize);
        let chunkDataSize = vec3.fromValues(1, 1, 1);
        for (let dim of dims) {
          voxelSize[dim] = levelInfo.resolution[dim];
          chunkDataSize[dim] = levelInfo.tileSize[dim];
        }
        let chunkLayout = ChunkLayout.get(vec3.multiply(vec3.create(), voxelSize, chunkDataSize));
        let lowerVoxelBound = vec3.create(), upperVoxelBound = vec3.create();
        for (let i = 0; i < 3; ++i) {
          lowerVoxelBound[i] = Math.floor(this.lowerVoxelBound[i] * (this.voxelSize[i] / voxelSize[i]));
          upperVoxelBound[i] = Math.ceil(this.upperVoxelBound[i] * (this.voxelSize[i] / voxelSize[i]));
        }
        let spec = new VolumeChunkSpecification(
            chunkLayout, chunkDataSize, this.numChannels, this.dataType, lowerVoxelBound,
          upperVoxelBound);
        let tileParameters: TileChunkSourceParameters = {
          'baseUrls': parameters.baseUrls,
          'nodeKey': parameters.nodeKey,
          'dataInstanceKey': parameters.dataInstanceKey,
          'encoding': this.encoding,
          'level': level,
          'dims': `${dims[0]}_${dims[1]}`,
        };<commit_msg>adjusts default tile behavior for dvid source<commit_after>/**
 * @license
 * Copyright 2016 Google Inc.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * @file
 * Support for DVID (https://github.com/janelia-flyem/dvid) servers.
 */

import {ChunkManager} from 'neuroglancer/chunk_manager/frontend';
import {VolumeChunkSourceParameters, volumeSourceToString, TileChunkSourceParameters, tileSourceToString, TileEncoding} from 'neuroglancer/datasource/dvid/base';
import {registerDataSourceFactory, CompletionResult} from 'neuroglancer/datasource/factory';
import {DataType, VolumeType, VolumeChunkSpecification} from 'neuroglancer/sliceview/base';
import {ChunkLayout} from 'neuroglancer/sliceview/chunk_layout';
import {VolumeChunkSource as GenericVolumeChunkSource, MultiscaleVolumeChunkSource as GenericMultiscaleVolumeChunkSource} from 'neuroglancer/sliceview/frontend';
import {StatusMessage} from 'neuroglancer/status';
import {applyCompletionOffset, getPrefixMatchesWithDescriptions} from 'neuroglancer/util/completion';
import {vec3, Vec3} from 'neuroglancer/util/geom';
import {openShardedHttpRequest, sendHttpRequest} from 'neuroglancer/util/http_request';
import {parseFixedLengthArray, parseIntVec, stableStringify, verifyObject, verifyObjectProperty, verifyInt, verifyPositiveInt, verifyString, parseArray, verifyMapKey, verifyFinitePositiveFloat, verifyObjectAsMap} from 'neuroglancer/util/json';
import {CancellablePromise} from 'neuroglancer/util/promise';

let serverDataTypes = new Map<string, DataType>();
serverDataTypes.set('uint8', DataType.UINT8);
serverDataTypes.set('uint32', DataType.UINT32);
serverDataTypes.set('uint64', DataType.UINT64);

export class DataInstanceBaseInfo {
  get typeName(): string { return this.obj['TypeName']; }

  constructor (public obj: any) {
    verifyObject(obj);
    verifyObjectProperty(obj, 'TypeName', verifyString);
  }
};

export class DataInstanceInfo {
  constructor(public obj: any, public name: string, public base: DataInstanceBaseInfo) {}
};

export class VolumeChunkSource extends GenericVolumeChunkSource {
  constructor(
    chunkManager: ChunkManager, spec: VolumeChunkSpecification, public parameters: VolumeChunkSourceParameters) {
    super(chunkManager, spec);
    this.initializeCounterpart(chunkManager.rpc, {
      'type': 'dvid/VolumeChunkSource',
      'parameters': parameters,
    });
  }
  toString () {
    return volumeSourceToString(this.parameters);
  }
};

export class VolumeDataInstanceInfo extends DataInstanceInfo {
  dataType: DataType;
  lowerVoxelBound: Vec3;
  upperVoxelBound: Vec3;
  voxelSize: Vec3;
  numChannels: number;
  constructor(obj: any, name: string, base: DataInstanceBaseInfo, public volumeType: VolumeType) {
    super(obj, name, base);
    let extended = verifyObjectProperty(obj, 'Extended', verifyObject);
    let extendedValues = verifyObjectProperty(extended, 'Values', x => parseArray(x, verifyObject));
    if (extendedValues.length < 1) {
      throw new Error(
          'Expected Extended.Values property to have length >= 1, but received: ${JSON.stringify(extendedValues)}.');
    }
    this.dataType =
        verifyObjectProperty(extendedValues[0], 'DataType', x => verifyMapKey(x, serverDataTypes));
    this.lowerVoxelBound =
        verifyObjectProperty(extended, 'MinPoint', x => parseIntVec(vec3.create(), x));
    this.upperVoxelBound =
        verifyObjectProperty(extended, 'MaxPoint', x => parseIntVec(vec3.create(), x));
    this.voxelSize = verifyObjectProperty(
        extended, 'VoxelSize',
        x => parseFixedLengthArray(vec3.create(), x, verifyFinitePositiveFloat));
    this.numChannels = 1;
  }

  getSources(chunkManager: ChunkManager, parameters: VolumeChunkSourceParameters) {
    return [Array
                .from(VolumeChunkSpecification.getDefaults({
                  voxelSize: this.voxelSize,
                  dataType: this.dataType,
                  numChannels: this.numChannels,
                  lowerVoxelBound: this.lowerVoxelBound,
                  upperVoxelBound: this.upperVoxelBound,
                  volumeType: this.volumeType,
                }))
                .map(spec => {
                  return chunkManager.getChunkSource(
                      VolumeChunkSource, stableStringify(parameters),
                      () => new VolumeChunkSource(chunkManager, spec, parameters));
                })];
  }
};

export class TileLevelInfo {
  /**
   * Resolution of the two downsampled dimensions in the tile plane.  The tile depth is equal to the
   * base voxel size in that dimension.
   */
  resolution: Vec3;
  tileSize: Vec3;

  constructor (obj: any) {
    verifyObject(obj);
    this.resolution = verifyObjectProperty(
      obj, 'Resolution', x => parseFixedLengthArray(vec3.create(), x, verifyFinitePositiveFloat));
    this.tileSize = verifyObjectProperty(
      obj, 'TileSize', x => parseFixedLengthArray(vec3.create(), x, verifyPositiveInt));
  }
};

/**
 * Dimensions for which tiles are computed.
 *
 * DVID does not indicate which dimensions are available but it
 * provides blank tiles if the dimension asked for is not there.
 */
const TILE_DIMS = [
  [0, 1],
  [0, 2],
  [1, 2],
];

export class TileChunkSource extends GenericVolumeChunkSource {
  constructor(
    chunkManager: ChunkManager, spec: VolumeChunkSpecification, public parameters: TileChunkSourceParameters) {
    super(chunkManager, spec);
    this.initializeCounterpart(chunkManager.rpc, {
      'type': 'dvid/TileChunkSource',
      'parameters': parameters,
    });
  }
  toString () {
    return tileSourceToString(this.parameters);
  }
};

export class TileDataInstanceInfo extends DataInstanceInfo {
  get dataType () { return DataType.UINT8; }
  get volumeType () { return VolumeType.IMAGE; }
  get numChannels () { return 1; }

  encoding: TileEncoding;

  /**
   * Base voxel size (nm).
   */
  voxelSize: Vec3;

  levels: Map<string, TileLevelInfo>;

  lowerVoxelBound: Vec3;
  upperVoxelBound: Vec3;

  constructor (obj: any, name: string, base: DataInstanceBaseInfo) {
    super(obj, name, base);
    let extended = verifyObjectProperty(obj, 'Extended', verifyObject);
    this.levels = verifyObjectProperty(extended, 'Levels', x => verifyObjectAsMap(x, y => new TileLevelInfo(y)));
    let baseLevel = this.levels.get('0');
    if (baseLevel === undefined) {
      throw new Error(`Level 0 is not defined.`);
    }
    this.voxelSize = baseLevel.resolution;
    let minTileCoord = verifyObjectProperty(
        extended, 'MinTileCoord', x => parseFixedLengthArray(vec3.create(), x, verifyInt));
    let maxTileCoord = verifyObjectProperty(
        extended, 'MaxTileCoord', x => parseFixedLengthArray(vec3.create(), x, verifyInt));
    this.lowerVoxelBound = vec3.multiply(vec3.create(), baseLevel.tileSize, minTileCoord);
    this.upperVoxelBound = vec3.multiply(vec3.create(), baseLevel.tileSize, maxTileCoord);

    let encodingNumber = verifyObjectProperty(extended, 'Encoding', x => x);
    switch (encodingNumber) {
      case 2:
        this.encoding = TileEncoding.JPEG;
        break;
      default:
        throw new Error(`Unsupported tile encoding: ${JSON.stringify(encodingNumber)}.`);
    }
  }

  getSources(chunkManager: ChunkManager, parameters: VolumeChunkSourceParameters): GenericVolumeChunkSource[][] {
    let sources: TileChunkSource[][] = [];
    for (let [level, levelInfo] of this.levels) {
      let alternatives = TILE_DIMS.map(dims => {
        let voxelSize = vec3.clone(this.voxelSize);
        let chunkDataSize = vec3.fromValues(1, 1, 1);
        // tiles are always NxMx1
        for (let i = 0; i < 2; ++i) {
          voxelSize[dim] = levelInfo.resolution[dim];
          chunkDataSize[dim] = levelInfo.tileSize[dim];
        }

        let chunkLayout = ChunkLayout.get(vec3.multiply(vec3.create(), voxelSize, chunkDataSize));
        let lowerVoxelBound = vec3.create(), upperVoxelBound = vec3.create();
        for (let i = 0; i < 3; ++i) {
          lowerVoxelBound[i] = Math.floor(this.lowerVoxelBound[i] * (this.voxelSize[i] / voxelSize[i]));
          upperVoxelBound[i] = Math.ceil(this.upperVoxelBound[i] * (this.voxelSize[i] / voxelSize[i]));
        }
        let spec = new VolumeChunkSpecification(
            chunkLayout, chunkDataSize, this.numChannels, this.dataType, lowerVoxelBound,
          upperVoxelBound);
        let tileParameters: TileChunkSourceParameters = {
          'baseUrls': parameters.baseUrls,
          'nodeKey': parameters.nodeKey,
          'dataInstanceKey': parameters.dataInstanceKey,
          'encoding': this.encoding,
          'level': level,
          'dims': `${dims[0]}_${dims[1]}`,
        };
{'fraction_non_alphanumeric': 0.11106238351057998, 'fraction_numerical': 0.007510141431860542, 'mean_word_length': 3.859616409163559, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 39, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2357703', 'n_tokens_mistral': 5365, 'n_tokens_neox': 5220, 'n_words': 1293}