<commit_before>// import * as Pool from "../util/Pool.js";
import * as Recyclable from "../util/Recyclable.js";
import * as ValueMax from "../util/ValueMax.js";
import * as AsyncWorker from "../util/AsyncWorker.js";
import * as ValueDesc from "../Unpacker/ValueDesc.js";
import * as Weights from "../Unpacker/Weights.js";
import * as NeuralNet from "../Conv/NeuralNet.js";
import { tensorflowJsURL } from "./NeuralWorker/NeuralWorker_Common.js";

importScripts( tensorflowJsURL ); // Load tensorflow.js library in global scope.

/**
 * The implementation of a neural network web worker. It may own one or two neural
 * network.
 *
 */
class NeuralWorker_Body extends AsyncWorker.Body {

  /** */
  constructor() {
    super(); // register callback for handling messages sent from NeuralWorker_Proxy.
  }

  /** @override */
  async* disposeResources() {
    if ( this.alignmentMarkValueArray ) {
      this.alignmentMarkValueArray.disposeResources_and_recycleToPool();
      this.alignmentMarkValueArray = null;
    }

    this.NeuralNetArray_dispose();
    this.workerId = undefined;
    yield *super.disposeResources();
  }

  /** Release the neural network. */
  NeuralNetArray_dispose() {
    if ( this.neuralNetArray ) {
      this.neuralNetArray.disposeResources_and_recycleToPool();
      this.neuralNetArray = null;
    }
  }

  /**
   *
   * @param {number} workerId
   *   A non-negative integer represents this worker's id. The id of the first worker
   * should be 0.
   */
  async* initWorker( workerId ) {
    this.workerId = workerId;

//!!! ...unfinished... (2022/09/15)
// What if failed when:
//   - library (tensorflow.js) downloading
//   - worker starting (also a kind of library downloading)
//   - versus downloading
//   - versus result sending
//
// Perhaps, needs a life-cycle manager to handle them gracefully.

    return { value: true };
  }

  /**
   * @param {Object[]} neuralNetParamsBaseArray
   *   An array of object. Every element is an object looks like NeuralNet.ParamsBase.
   *
   * @param {ArrayBuffer[]} weightArrayBufferArray
   *   An array of every neural network's weights. Every element  will be interpreted
   * as Float32Array.
   *
   * @yield {boolean}
   *   - Yield { done: true, value: { value: true } }, if success.
   *   - Yield { done: true, value: { value: false } }, if failed.
   */
  async* NeuralNetArray_create( neuralNetParamsBaseArray, weightArrayBufferArray ) {

    // 0. Prepare container for all neural networks.
    {
      if ( this.neuralNetArray )
        this.neuralNetArray.clear();
      else
        this.neuralNetArray = Recyclable.OwnerArray.Pool.get_or_create_by();

      this.neuralNetArray.length = neuralNetParamsBaseArray.length;
    }

    // 2. Create every neural network.
    let progress;
    try {
      let bAllOk = true;
      for ( let i = 0; i < neuralNetParamsBaseArray.length; ++i ) {
        let neuralNetParamsBase = neuralNetParamsBaseArray[ i ];
        let weightArrayBuffer = weightArrayBufferArray[ i ];

        let inputWeightArray;
        {
          let weightElementOffsetBegin = 0;
          let byteOffset
            = weightElementOffsetBegin * Float32Array.BYTES_PER_ELEMENT;

          let byteLength = Math.floor(
            weightArrayBuffer.byteLength / Float32Array.BYTES_PER_ELEMENT );

          let aFloat32Array
            = new Float32Array( weightArrayBuffer, byteOffset, byteLength );

          // Ensure there is no NaN value in the weight array. (Force NaN to 0.)<commit_msg>Improve comments.<commit_after>// import * as Pool from "../util/Pool.js";
import * as Recyclable from "../util/Recyclable.js";
import * as ValueMax from "../util/ValueMax.js";
import * as AsyncWorker from "../util/AsyncWorker.js";
import * as ValueDesc from "../Unpacker/ValueDesc.js";
import * as Weights from "../Unpacker/Weights.js";
import * as NeuralNet from "../Conv/NeuralNet.js";
import { tensorflowJsURL } from "./NeuralWorker/NeuralWorker_Common.js";

importScripts( tensorflowJsURL ); // Load tensorflow.js library in global scope.

/**
 * The implementation of a neural network web worker. It may own one or two neural
 * network.
 *
 */
class NeuralWorker_Body extends AsyncWorker.Body {

  /** */
  constructor() {
    super(); // register callback for handling messages sent from NeuralWorker_Proxy.
  }

  /** @override */
  async* disposeResources() {
    if ( this.alignmentMarkValueArray ) {
      this.alignmentMarkValueArray.disposeResources_and_recycleToPool();
      this.alignmentMarkValueArray = null;
    }

    this.NeuralNetArray_dispose();
    this.workerId = undefined;
    yield *super.disposeResources();
  }

  /** Release the neural network. */
  NeuralNetArray_dispose() {
    if ( this.neuralNetArray ) {
      this.neuralNetArray.disposeResources_and_recycleToPool();
      this.neuralNetArray = null;
    }
  }

  /**
   *
   * @param {number} workerId
   *   A non-negative integer represents this worker's id. The id of the first worker
   * should be 0.
   */
  async* initWorker( workerId ) {
    this.workerId = workerId;

//!!! ...unfinished... (2022/09/15)
// What if failed when:
//   - library (tensorflow.js) downloading
//   - worker starting (also a kind of library downloading)
//   - versus downloading
//   - versus result sending
//
// Perhaps, needs a life-cycle manager to handle them gracefully.

    return { value: true };
  }

  /**
   * @param {Object[]} neuralNetParamsBaseArray
   *   An array of object. Every element is an object looks like NeuralNet.ParamsBase.
   *
   * @param {ArrayBuffer[]} weightArrayBufferArray
   *   An array of every neural network's weights. Every element  will be interpreted
   * as Float32Array.
   *
   * @yield {boolean}
   *   - Yield { done: true, value: { value: true } }, if success.
   *   - Yield { done: true, value: { value: false } }, if failed.
   */
  async* NeuralNetArray_create( neuralNetParamsBaseArray, weightArrayBufferArray ) {

    // 0. Prepare container for all neural networks.
    {
      if ( this.neuralNetArray )
        this.neuralNetArray.clear(); // Release old neural networks.
      else
        this.neuralNetArray = Recyclable.OwnerArray.Pool.get_or_create_by();

      this.neuralNetArray.length = neuralNetParamsBaseArray.length;
    }

    // 2. Create every neural network.
    let progress;
    try {
      let bAllOk = true;
      for ( let i = 0; i < neuralNetParamsBaseArray.length; ++i ) {
        let neuralNetParamsBase = neuralNetParamsBaseArray[ i ];
        let weightArrayBuffer = weightArrayBufferArray[ i ];

        let inputWeightArray;
        {
          let weightElementOffsetBegin = 0;
          let byteOffset
            = weightElementOffsetBegin * Float32Array.BYTES_PER_ELEMENT;

          let byteLength = Math.floor(
            weightArrayBuffer.byteLength / Float32Array.BYTES_PER_ELEMENT );

          let aFloat32Array
            = new Float32Array( weightArrayBuffer, byteOffset, byteLength );

          // Ensure there is no NaN value in the weight array. (Force NaN to 0.)
{'fraction_non_alphanumeric': 0.10890527509926262, 'fraction_numerical': 0.006806579693703914, 'mean_word_length': 2.8062601187263896, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2540025', 'n_tokens_mistral': 2120, 'n_tokens_neox': 2030, 'n_words': 582}