<commit_before>const Worker = require('tiny-worker');

/**
 * Encapuslates the underlying Web Worker through asynchronous calls.
 */
exports.LMLayer = class LMLayer {
  constructor() {
    // Worker state
    this._worker = new Worker('lmlayer.js');
    this._worker.onmessage = this._onmessage.bind(this);

    // Keep track of individual requests to make a nice async/await API.
    this._promises = new PromiseStore;

    // Keep track of tokens.
    this._currentToken = Number.MIN_SAFE_INTEGER;
  }

  /**
   * [async] Sends a context, transform, and token to the LMLayer.
   */
  predictWithContext({transform, context, customToken}) {
    let token = customToken || this._nextToken();

    return new Promise((resolve, reject) => {
      this._promises.record(token, resolve, reject);
      this._cast('predict', {
        token, transform, context
      });
    });
  }


  /**
   * Throw an asynchronous method call (a "cast") over to the Web Worker.
   */
  _cast(method, payload) {
    this._worker.postMessage({ method, ...payload });
  }

  /**
   * Handles the wrapped worker's onmessage events.
   */
  _onmessage(event) {
    const {method, token} = event.data;

    let accept = this._promises.keep(token);

    if (method === 'suggestions') {
      accept(event.data);
    } else {
      throw new Error(`Unknown message: ${method}`);
    }
  }

  /**
   * Returns the next token. Note: mutates state.
   */
  _nextToken() {
    let token = this._currentToken++;
    if (!Number.isSafeInteger(token)) {
      throw new RangeError('Ran out of usable tokens');
    }
    return token;
  }
};


/**
 * Associate tokens with promises.
 *
 * You can .track() them, and then .keep() them. You may also .break() them.
 */
class PromiseStore {
  constructor() {
    this._promises = new Map();
  }

  /**
   * Associate a token with its respective resolve and reject callbacks.
   */
  record(token, resolve, reject) {
    if (this._promises.has(token)) {
      reject(`Existing request with token ${token}`);
    }
    this._promises.set(token, resolve);
  }

  /**
   * Fetch a promise's resolution function.
   *
   * Calling the resolution function will stop tracking the promise.
   */
  keep(token) {
    let accept = this._promises.get(token);

    if (!accept) {
      throw new Error(`No promise associated with token: ${token}`);
    }

    // This acts like the resolve function, BUT, it removes the promise from
    // the store -- because it's resolved!
    return (resolvedValue) => {
      this._promises.delete(token);
      return accept(resolvedValue);<commit_msg>Rename method: record() -> track().<commit_after>const Worker = require('tiny-worker');

/**
 * Encapuslates the underlying Web Worker through asynchronous calls.
 */
exports.LMLayer = class LMLayer {
  constructor() {
    // Worker state
    this._worker = new Worker('lmlayer.js');
    this._worker.onmessage = this._onmessage.bind(this);

    // Keep track of individual requests to make a nice async/await API.
    this._promises = new PromiseStore;

    // Keep track of tokens.
    this._currentToken = Number.MIN_SAFE_INTEGER;
  }

  /**
   * [async] Sends a context, transform, and token to the LMLayer.
   */
  predictWithContext({transform, context, customToken}) {
    let token = customToken || this._nextToken();

    return new Promise((resolve, reject) => {
      this._promises.track(token, resolve, reject);
      this._cast('predict', {
        token, transform, context
      });
    });
  }


  /**
   * Throw an asynchronous method call (a "cast") over to the Web Worker.
   */
  _cast(method, payload) {
    this._worker.postMessage({ method, ...payload });
  }

  /**
   * Handles the wrapped worker's onmessage events.
   */
  _onmessage(event) {
    const {method, token} = event.data;

    let accept = this._promises.keep(token);

    if (method === 'suggestions') {
      accept(event.data);
    } else {
      throw new Error(`Unknown message: ${method}`);
    }
  }

  /**
   * Returns the next token. Note: mutates state.
   */
  _nextToken() {
    let token = this._currentToken++;
    if (!Number.isSafeInteger(token)) {
      throw new RangeError('Ran out of usable tokens');
    }
    return token;
  }
};


/**
 * Associate tokens with promises.
 *
 * You can .track() them, and then .keep() them. You may also .break() them.
 */
class PromiseStore {
  constructor() {
    this._promises = new Map();
  }

  /**
   * Associate a token with its respective resolve and reject callbacks.
   */
  track(token, resolve, reject) {
    if (this._promises.has(token)) {
      reject(`Existing request with token ${token}`);
    }
    this._promises.set(token, resolve);
  }

  /**
   * Fetch a promise's resolution function.
   *
   * Calling the resolution function will stop tracking the promise.
   */
  keep(token) {
    let accept = this._promises.get(token);

    if (!accept) {
      throw new Error(`No promise associated with token: ${token}`);
    }

    // This acts like the resolve function, BUT, it removes the promise from
    // the store -- because it's resolved!
    return (resolvedValue) => {
      this._promises.delete(token);
      return accept(resolvedValue);
{'fraction_non_alphanumeric': 0.13297463489623368, 'fraction_numerical': 0.0, 'mean_word_length': 2.7690079652425776, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3556318', 'n_tokens_mistral': 1587, 'n_tokens_neox': 1540, 'n_words': 458}