<commit_before>export class Variable {
  constructor(value) {
    this.value = value;
  }
}

export class Token {
  constructor() {}
}

export class FunctionToken extends Token {
  constructor(fn) {
    super();
    this.fn = fn;
  }

  __call__(args) {
    return defaultCall(this, args);
  }
}

export function defaultCall(fnToken, args) {
  return fnToken.fn(args);
}

export function call(fn, args) {
  return fn['__call__'](args);
};
<commit_msg>lib.js comments<commit_after>// Variable class -------------------------------------------------------------
// * this should never *ever* be accessed through anywhere except set/get
//   variable functions
// * takes one paramater, value, which is stored in inst.value and represents
//   the value of the variable
export class Variable {
  constructor(value) {
    this.value = value;
  }
}

// Base token class -----------------------------------------------------------
// * doesn't do anything
// * use x instanceof Token to check if x is any kind of token

export class Token {
  constructor() {}
}

// Function token class -------------------------------------------------------
// * takes one paramater, fn, which is stored in inst.fn and represents the
//   function that will be called
// * use inst.__call__ to call the function (with optional arguments)

export class FunctionToken extends Token {
  constructor(fn) {
    super();
    this.fn = fn;
  }

  __call__(args) {
    // Call this function. By default uses defaultCall, but can be overriden
    // by subclasses.
    return defaultCall(this, args);
  }
}

// Call function --------------------------------------------------------------

export function call(fn, args) {
  return fn['__call__'](args);
}

export function defaultCall(fnToken, args) {
  return fnToken.fn(args);
}
{'fraction_non_alphanumeric': 0.2273485269594219, 'fraction_numerical': 0.0, 'mean_word_length': 1.9900332225913622, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4339091', 'n_tokens_mistral': 477, 'n_tokens_neox': 470, 'n_words': 165}