<commit_before>/// <reference path="lexical-model-compiler/lexical-model.ts" />

import * as TypeScript from 'typescript';
import * as fs from 'fs';

import LexicalModelCompiler from "./";

/**
 * Exit codes defined in <sysexits.h>:
 * https://www.freebsd.org/cgi/man.cgi?query=sysexits&apropos=0&sektion=0&manpath=FreeBSD+4.3-RELEASE&format=html
 */
const enum SysExits {
  EX_USAGE = 64,
  EX_DATAERR = 65,
};

/**
 * Loads a lexical model's source module from the given filename.
 * @param filename path to the model source file.
 */
function loadFromFilename(filename: string): LexicalModelSource {
  let sourceCode = fs.readFileSync(filename, 'utf8');

  // Compile the module to JavaScript code.
  // NOTE: transpile module does a very simple TS to JS compilation.
  // It DOES NOT check for types!
  let compilationOutput = TypeScript.transpile(sourceCode, {
    // Our runtime should support ES6 with Node/CommonJS modules.
    target: TypeScript.ScriptTarget.ES2015,
    module: TypeScript.ModuleKind.CommonJS,
  });

  // Turn the module into a function in which we can inject a global.
  let moduleCode = '(function(exports){' + compilationOutput + '})';

  // Run the module; its exports will be assigned to `moduleExports`.
  let moduleExports = {};
  let module = eval(moduleCode);
  module(moduleExports);

  if (!moduleExports['__esModule'] || !moduleExports['default']) {
    console.error(`Model source '${filename}' does have a default export. Did you remember to write \`export default source;\`?`);
    process.exit(SysExits.EX_DATAERR);
  }

  return moduleExports['default'] as LexicalModelSource;
}

if (process.argv.length < 3) {
  console.error('Must provide a lexical model source file.');
  process.exit(SysExits.EX_USAGE);<commit_msg>Rename TypeScript to ts, because even the internal ts does it.<commit_after>/// <reference path="lexical-model-compiler/lexical-model.ts" />

import * as ts from 'typescript';
import * as fs from 'fs';

import LexicalModelCompiler from "./";

/**
 * Exit codes defined in <sysexits.h>:
 * https://www.freebsd.org/cgi/man.cgi?query=sysexits&apropos=0&sektion=0&manpath=FreeBSD+4.3-RELEASE&format=html
 */
const enum SysExits {
  EX_USAGE = 64,
  EX_DATAERR = 65,
};

/**
 * Loads a lexical model's source module from the given filename.
 * @param filename path to the model source file.
 */
function loadFromFilename(filename: string): LexicalModelSource {
  let sourceCode = fs.readFileSync(filename, 'utf8');

  // Compile the module to JavaScript code.
  // NOTE: transpile module does a very simple TS to JS compilation.
  // It DOES NOT check for types!
  let compilationOutput = ts.transpile(sourceCode, {
    // Our runtime should support ES6 with Node/CommonJS modules.
    target: ts.ScriptTarget.ES2015,
    module: ts.ModuleKind.CommonJS,
  });

  // Turn the module into a function in which we can inject a global.
  let moduleCode = '(function(exports){' + compilationOutput + '})';

  // Run the module; its exports will be assigned to `moduleExports`.
  let moduleExports = {};
  let module = eval(moduleCode);
  module(moduleExports);

  if (!moduleExports['__esModule'] || !moduleExports['default']) {
    console.error(`Model source '${filename}' does have a default export. Did you remember to write \`export default source;\`?`);
    process.exit(SysExits.EX_DATAERR);
  }

  return moduleExports['default'] as LexicalModelSource;
}

if (process.argv.length < 3) {
  console.error('Must provide a lexical model source file.');
  process.exit(SysExits.EX_USAGE);
{'fraction_non_alphanumeric': 0.12475303415184871, 'fraction_numerical': 0.00846740050804403, 'mean_word_length': 4.166180758017493, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1660027', 'n_tokens_mistral': 1124, 'n_tokens_neox': 1075, 'n_words': 331}