<commit_before>import { defined, defaultValue } from "../Source/Cesium.js";
import concatTypedArrays from "./concatTypedArrays.js";
import MetadataTester from "./MetadataTester.js";

/**
 * Class to generate implicit subtrees for implicit tiling unit tests
 * @private
 */
export default function ImplicitTilingTester() {}

/**
 * Description of a single availability bitstream
 * @typedef {Object} AvailabilityDescription
 * @property {String|Number} descriptor Either a string of <code>0</code>s and <code>1</code>s representing the bitstream values, or an integer <code>0</code> or <code>1</code> to indicate a constant value.
 * @property {Number} lengthBits How many bits are in the bitstream. This must be specified, even if descriptor is a string of <code>0</code>s and <code>1</code>s
 * @property {Boolean} isInternal <code>true</code> if an internal bufferView should be created. <code>false</code> indicates the bufferview is stored in an external buffer instead.
 * @property {Boolean} [shareBuffer=false] This is only used for content availability. If <code>true</code>, then the content availability will share the same buffer as the tile availaibility, as this is a common optimization
 * @property {Boolean} [includeAvailableCount=false] If true, set availableCount
 */

/**
 * A description of 3DTILES_metadata properties stored in the subtree.
 * @typedef {Object} MetadataDescription
 * @property {Boolean} isInternal True if the metadata should be stored in the subtree file, false if the metadata should be stored in an external buffer.
 * @property {Array} propertyTables Array of property table objects to pass into {@link MetadataTester.createPropertyTables} in order to create the property table buffer views.
 * @private
 */

/**
 * A JSON description of a subtree file for easier generation
 * @typedef {Object} SubtreeDescription
 * @property {Boolean} [useLegacySchema] If true, the resulting JSON chunk will use the legacy schema for subtrees and metadata (e.g. use bufferViews rather than bitstream, use 3DTILES_metadata extension rather than tileMetadata or contentMetadata). Used to test backwards compatibility.
 * @property {AvailabilityDescription} tileAvailability A description of the tile availability bitstream to generate
 * @property {AvailabilityDescription} contentAvailability A description of the content availability bitstream to generate
 * @property {Boolean} [useMultipleContentsExtension] If true, use the 3DTILES_multiple_contents extension. Used to test backwards compatibility.
 * @property {AvailabilityDescription} childSubtreeAvailability A description of the child subtree availability bitstream to generate
 * @property {AvailabilityDescription} other A description of another bitstream. This is not used for availability, but rather to simulate extra buffer views.
 * @property {MetadataDescription} [metadata] For testing 3DTILES_metadata, additional options can be passed in here.
 * @property {Boolean} [json] If true, return the result as a JSON with external buffers. Should not be true if any of the availability buffers are internal.
 * @private
 */

/**
 * Results of procedurally generating a subtree.
 * @typedef {Object} GeneratedSubtree
 * @property {Uint8Array} [subtreeJson] The JSON portion of the subtree file. Mutually exclusive with subtreeBuffer.
 * @property {Uint8Array} [subtreeBuffer] A typed array storing the contents of the subtree file (including the internal buffer). Mutually exclusive with subtreeJson.
 * @property {Uint8Array} externalBuffer A typed array representing an external .bin file. This is always returned, but it may be an empty typed array.

 */

/**
 * Generate a subtree buffer
 * @param {SubtreeDescription} subtreeDescription A JSON description of the subtree's structure and values
 * @param {Boolean} constantOnly true if all the bitstreams are constant, i.e. no buffers/bufferViews are needed.
 * @return {GeneratedSubtree} The procedurally generated subtree and an external buffer.
 *
 * @example
 *  const subtreeDescription = {
 *    tileAvailability: {
 *      descriptor: 1,
 *      lengthBits: 5,
 *      isInternal: true,
 *    },
 *    contentAvailability: {
 *      descriptor: "11000",
 *      lengthBits: 5,
 *      isInternal: true,
 *    },
 *    childSubtreeAvailability: {
 *      descriptor: "1111000010100000",
 *      lengthBits: 16,
 *      isInternal: true,
 *    },
 *    other: {
 *      descriptor: "101010",
 *      lengthBits: 6,
 *      isInternal: false,
 *    },
 *  };
 *  const results = ImplicitTilingTester.generateSubtreeBuffers(
 *    subtreeDescription
 *  );
 *
 * @private
 */
ImplicitTilingTester.generateSubtreeBuffers = function (
  subtreeDescription,
  constantOnly
) {
  constantOnly = defaultValue(constantOnly, false);

  // This will be populated by makeBufferViews() and makeBuffers()
  let subtreeJson = {};
  if (!constantOnly) {
    subtreeJson = {
      buffers: [],
      bufferViews: [],
    };
  }

  const bufferViewsU8 = makeBufferViews(subtreeDescription, subtreeJson);
  const buffersU8 = makeBuffers(bufferViewsU8, subtreeJson);

  if (subtreeDescription.json) {
    return {
      subtreeJson: subtreeJson,
      externalBuffer: buffersU8.external,
    };
  }

  const jsonChunk = makeJsonChunk(subtreeJson);
  const binaryChunk = buffersU8.internal;
  const header = makeSubtreeHeader(jsonChunk.length, binaryChunk.length);
  const subtreeBuffer = concatTypedArrays([header, jsonChunk, binaryChunk]);

  return {
    subtreeBuffer: subtreeBuffer,
    externalBuffer: buffersU8.external,
  };
};

function makeBufferViews(subtreeDescription, subtreeJson) {
  // Content availability is optional.
  const hasContent = defined(subtreeDescription.contentAvailability);

  // pass 1: parse availability -------------------------------------------
  const parsedAvailability = {
    tileAvailability: parseAvailability(subtreeDescription.tileAvailability),
    childSubtreeAvailability: parseAvailability(
      subtreeDescription.childSubtreeAvailability
    ),
  };

  if (hasContent) {
    parsedAvailability.contentAvailability = subtreeDescription.contentAvailability.map(
      parseAvailability
    );
  }

  // to simulate additional buffer views for metadata or other purposes.
  if (defined(subtreeDescription.other)) {
    parsedAvailability.other = parseAvailability(subtreeDescription.other);
  }

  // pass 2: create buffer view JSON and gather typed arrays ------------------
  const bufferViewsU8 = {
    internal: [],
    external: [],
    count: 0,
  };

  const useLegacySchema = subtreeDescription.useLegacySchema;
  const bufferViewJsonArray = [];
  gatherBufferViews(
    bufferViewsU8,
    bufferViewJsonArray,
    parsedAvailability.tileAvailability,
    useLegacySchema
  );
<commit_msg>Ensure useLegacySchema defaults to false<commit_after>import { defined, defaultValue } from "../Source/Cesium.js";
import concatTypedArrays from "./concatTypedArrays.js";
import MetadataTester from "./MetadataTester.js";

/**
 * Class to generate implicit subtrees for implicit tiling unit tests
 * @private
 */
export default function ImplicitTilingTester() {}

/**
 * Description of a single availability bitstream
 * @typedef {Object} AvailabilityDescription
 * @property {String|Number} descriptor Either a string of <code>0</code>s and <code>1</code>s representing the bitstream values, or an integer <code>0</code> or <code>1</code> to indicate a constant value.
 * @property {Number} lengthBits How many bits are in the bitstream. This must be specified, even if descriptor is a string of <code>0</code>s and <code>1</code>s
 * @property {Boolean} isInternal <code>true</code> if an internal bufferView should be created. <code>false</code> indicates the bufferview is stored in an external buffer instead.
 * @property {Boolean} [shareBuffer=false] This is only used for content availability. If <code>true</code>, then the content availability will share the same buffer as the tile availaibility, as this is a common optimization
 * @property {Boolean} [includeAvailableCount=false] If true, set availableCount
 */

/**
 * A description of 3DTILES_metadata properties stored in the subtree.
 * @typedef {Object} MetadataDescription
 * @property {Boolean} isInternal True if the metadata should be stored in the subtree file, false if the metadata should be stored in an external buffer.
 * @property {Array} propertyTables Array of property table objects to pass into {@link MetadataTester.createPropertyTables} in order to create the property table buffer views.
 * @private
 */

/**
 * A JSON description of a subtree file for easier generation
 * @typedef {Object} SubtreeDescription
 * @property {Boolean} [useLegacySchema=false] If true, the resulting JSON chunk will use the legacy schema for subtrees and metadata (e.g. use bufferViews rather than bitstream, use 3DTILES_metadata extension rather than tileMetadata or contentMetadata). Used to test backwards compatibility.
 * @property {AvailabilityDescription} tileAvailability A description of the tile availability bitstream to generate
 * @property {AvailabilityDescription} contentAvailability A description of the content availability bitstream to generate
 * @property {Boolean} [useMultipleContentsExtension] If true, use the 3DTILES_multiple_contents extension. Used to test backwards compatibility.
 * @property {AvailabilityDescription} childSubtreeAvailability A description of the child subtree availability bitstream to generate
 * @property {AvailabilityDescription} other A description of another bitstream. This is not used for availability, but rather to simulate extra buffer views.
 * @property {MetadataDescription} [metadata] For testing 3DTILES_metadata, additional options can be passed in here.
 * @property {Boolean} [json] If true, return the result as a JSON with external buffers. Should not be true if any of the availability buffers are internal.
 * @private
 */

/**
 * Results of procedurally generating a subtree.
 * @typedef {Object} GeneratedSubtree
 * @property {Uint8Array} [subtreeJson] The JSON portion of the subtree file. Mutually exclusive with subtreeBuffer.
 * @property {Uint8Array} [subtreeBuffer] A typed array storing the contents of the subtree file (including the internal buffer). Mutually exclusive with subtreeJson.
 * @property {Uint8Array} externalBuffer A typed array representing an external .bin file. This is always returned, but it may be an empty typed array.

 */

/**
 * Generate a subtree buffer
 * @param {SubtreeDescription} subtreeDescription A JSON description of the subtree's structure and values
 * @param {Boolean} constantOnly true if all the bitstreams are constant, i.e. no buffers/bufferViews are needed.
 * @return {GeneratedSubtree} The procedurally generated subtree and an external buffer.
 *
 * @example
 *  const subtreeDescription = {
 *    tileAvailability: {
 *      descriptor: 1,
 *      lengthBits: 5,
 *      isInternal: true,
 *    },
 *    contentAvailability: {
 *      descriptor: "11000",
 *      lengthBits: 5,
 *      isInternal: true,
 *    },
 *    childSubtreeAvailability: {
 *      descriptor: "1111000010100000",
 *      lengthBits: 16,
 *      isInternal: true,
 *    },
 *    other: {
 *      descriptor: "101010",
 *      lengthBits: 6,
 *      isInternal: false,
 *    },
 *  };
 *  const results = ImplicitTilingTester.generateSubtreeBuffers(
 *    subtreeDescription
 *  );
 *
 * @private
 */
ImplicitTilingTester.generateSubtreeBuffers = function (
  subtreeDescription,
  constantOnly
) {
  constantOnly = defaultValue(constantOnly, false);

  // This will be populated by makeBufferViews() and makeBuffers()
  let subtreeJson = {};
  if (!constantOnly) {
    subtreeJson = {
      buffers: [],
      bufferViews: [],
    };
  }

  const bufferViewsU8 = makeBufferViews(subtreeDescription, subtreeJson);
  const buffersU8 = makeBuffers(bufferViewsU8, subtreeJson);

  if (subtreeDescription.json) {
    return {
      subtreeJson: subtreeJson,
      externalBuffer: buffersU8.external,
    };
  }

  const jsonChunk = makeJsonChunk(subtreeJson);
  const binaryChunk = buffersU8.internal;
  const header = makeSubtreeHeader(jsonChunk.length, binaryChunk.length);
  const subtreeBuffer = concatTypedArrays([header, jsonChunk, binaryChunk]);

  return {
    subtreeBuffer: subtreeBuffer,
    externalBuffer: buffersU8.external,
  };
};

function makeBufferViews(subtreeDescription, subtreeJson) {
  // Content availability is optional.
  const hasContent = defined(subtreeDescription.contentAvailability);

  // pass 1: parse availability -------------------------------------------
  const parsedAvailability = {
    tileAvailability: parseAvailability(subtreeDescription.tileAvailability),
    childSubtreeAvailability: parseAvailability(
      subtreeDescription.childSubtreeAvailability
    ),
  };

  if (hasContent) {
    parsedAvailability.contentAvailability = subtreeDescription.contentAvailability.map(
      parseAvailability
    );
  }

  // to simulate additional buffer views for metadata or other purposes.
  if (defined(subtreeDescription.other)) {
    parsedAvailability.other = parseAvailability(subtreeDescription.other);
  }

  // pass 2: create buffer view JSON and gather typed arrays ------------------
  const bufferViewsU8 = {
    internal: [],
    external: [],
    count: 0,
  };

  const useLegacySchema = defaultValue(
    subtreeDescription.useLegacySchema,
    false
  );
  const bufferViewJsonArray = [];
  gatherBufferViews(
    bufferViewsU8,
    bufferViewJsonArray,
    parsedAvailability.tileAvailability,
    useLegacySchema
  );

{'fraction_non_alphanumeric': 0.09102921744237263, 'fraction_numerical': 0.008368815151960064, 'mean_word_length': 4.215543644716692, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1657612', 'n_tokens_mistral': 3622, 'n_tokens_neox': 3466, 'n_words': 1276}