<commit_before>"use strict";

module.exports = File;

var SHEBANG_REGEX = /^\#\!.*/;

var isFunction = require("lodash/lang/isFunction");
var transform  = require("./index");
var generate   = require("../generation");
var defaults   = require("lodash/object/defaults");
var includes   = require("lodash/collection/includes");
var clone      = require("../helpers/clone");
var parse      = require("../helpers/parse");
var Scope      = require("../traversal/scope");
var util       = require("../util");
var path       = require("path");
var each       = require("lodash/collection/each");
var t          = require("../types");

function File(opts) {
  this.dynamicImportIds = {};
  this.dynamicImported  = [];
  this.dynamicImports   = [];

  this.dynamicData      = {};
  this.data             = {};

  this.lastStatements   = [];
  this.opts             = this.normalizeOptions(opts);
  this.ast              = {};

  this.buildTransformers();
}

File.helpers = [
  "inherits",
  "defaults",
  "prototype-properties",
  "apply-constructor",
  "tagged-template-literal",
  "tagged-template-literal-loose",
  "interop-require",
  "to-array",
  "to-consumable-array",
  "sliced-to-array",
  "object-without-properties",
  "has-own",
  "slice",
  "bind",
  "define-property",
  "async-to-generator",
  "interop-require-wildcard",
  "typeof",
  "extends",
  "get",
  "set",
  "class-call-check",
  "object-destructuring-empty",
  "temporal-undefined",
  "temporal-assert-defined",
  "tail-call"
];

File.validOptions = [
  "filename",
  "filenameRelative",
  "blacklist",
  "whitelist",
  "loose",
  "optional",
  "modules",
  "sourceMap",
  "sourceMapName",
  "sourceFileName",
  "sourceRoot",
  "moduleRoot",
  "moduleIds",
  "comments",
  "reactCompat",
  "keepModuleIdExtensions",
  "code",
  "ast",
  "format",
  "playground",
  "experimental",
  "resolveModuleSource",
  "runtime",

  // these are used by plugins
  "ignore",
  "only",
  "extensions",
  "accept"
];

File.prototype.normalizeOptions = function (opts) {
  opts = clone(opts);

  for (var key in opts) {
    if (key[0] !== "_" && File.validOptions.indexOf(key) < 0) {
      throw new ReferenceError("Unknown option: " + key);
    }
  }

  defaults(opts, {
    keepModuleIdExtensions: false,
    resolveModuleSource:    null,
    experimental:           false,
    reactCompat:            false,
    playground:             false,
    whitespace:             true,
    moduleIds:              false,
    blacklist:              [],
    whitelist:              [],
    sourceMap:              false,
    optional:               [],
    comments:               true,
    filename:               "unknown",
    modules:                "common",
    runtime:                false,
    loose:                  [],
    code:                   true,
    ast:                    true
  });

  // normalize windows path separators to unix
  opts.filename = util.normalisePathSeparator(opts.filename);
  if (opts.sourceRoot) {
    opts.sourceRoot = util.normalisePathSeparator(opts.sourceRoot);
  }

  opts.basename = path.basename(opts.filename, path.extname(opts.filename));

  opts.blacklist = util.arrayify(opts.blacklist);
  opts.whitelist = util.arrayify(opts.whitelist);
  opts.optional  = util.arrayify(opts.optional);
  opts.loose     = util.arrayify(opts.loose);

  if (includes(opts.loose, "all")) {
    opts.loose = Object.keys(transform.transformers);
  }

  defaults(opts, {
    moduleRoot: opts.sourceRoot
  });

  defaults(opts, {
    sourceRoot: opts.moduleRoot
  });

  defaults(opts, {
    filenameRelative: opts.filename
  });

  defaults(opts, {
    sourceFileName: opts.filenameRelative,
    sourceMapName:  opts.filenameRelative
  });

  if (opts.playground) {
    opts.experimental = true;
  }

  if (opts.runtime) {
    this.set("runtimeIdentifier", t.identifier("to5Runtime"));
  }

  opts.blacklist = transform._ensureTransformerNames("blacklist", opts.blacklist);
  opts.whitelist = transform._ensureTransformerNames("whitelist", opts.whitelist);
  opts.optional  = transform._ensureTransformerNames("optional", opts.optional);
  opts.loose     = transform._ensureTransformerNames("loose", opts.loose);

  if (opts.reactCompat) {
    opts.optional.push("reactCompat");
    console.error("The reactCompat option has been moved into the optional transformer " +
                  "`reactCompat` - backwards compatibility will be removed in v4.0.0");
  }

  return opts;
};

File.prototype.isLoose = function (key) {
  return includes(this.opts.loose, key);
};

File.prototype.buildTransformers = function () {
  var file = this;

  var transformers = {};

  var secondaryStack = [];<commit_msg>force experimental and playground flag when whitelisting/optionalising transformers<commit_after>"use strict";

module.exports = File;

var SHEBANG_REGEX = /^\#\!.*/;

var isFunction = require("lodash/lang/isFunction");
var transform  = require("./index");
var generate   = require("../generation");
var defaults   = require("lodash/object/defaults");
var includes   = require("lodash/collection/includes");
var clone      = require("../helpers/clone");
var parse      = require("../helpers/parse");
var Scope      = require("../traversal/scope");
var util       = require("../util");
var path       = require("path");
var each       = require("lodash/collection/each");
var t          = require("../types");

function File(opts) {
  this.dynamicImportIds = {};
  this.dynamicImported  = [];
  this.dynamicImports   = [];

  this.dynamicData      = {};
  this.data             = {};

  this.lastStatements   = [];
  this.opts             = this.normalizeOptions(opts);
  this.ast              = {};

  this.buildTransformers();
}

File.helpers = [
  "inherits",
  "defaults",
  "prototype-properties",
  "apply-constructor",
  "tagged-template-literal",
  "tagged-template-literal-loose",
  "interop-require",
  "to-array",
  "to-consumable-array",
  "sliced-to-array",
  "object-without-properties",
  "has-own",
  "slice",
  "bind",
  "define-property",
  "async-to-generator",
  "interop-require-wildcard",
  "typeof",
  "extends",
  "get",
  "set",
  "class-call-check",
  "object-destructuring-empty",
  "temporal-undefined",
  "temporal-assert-defined",
  "tail-call"
];

File.validOptions = [
  "filename",
  "filenameRelative",
  "blacklist",
  "whitelist",
  "loose",
  "optional",
  "modules",
  "sourceMap",
  "sourceMapName",
  "sourceFileName",
  "sourceRoot",
  "moduleRoot",
  "moduleIds",
  "comments",
  "reactCompat",
  "keepModuleIdExtensions",
  "code",
  "ast",
  "format",
  "playground",
  "experimental",
  "resolveModuleSource",
  "runtime",

  // these are used by plugins
  "ignore",
  "only",
  "extensions",
  "accept"
];

File.prototype.normalizeOptions = function (opts) {
  opts = clone(opts);

  for (var key in opts) {
    if (key[0] !== "_" && File.validOptions.indexOf(key) < 0) {
      throw new ReferenceError("Unknown option: " + key);
    }
  }

  defaults(opts, {
    keepModuleIdExtensions: false,
    resolveModuleSource:    null,
    experimental:           false,
    reactCompat:            false,
    playground:             false,
    whitespace:             true,
    moduleIds:              false,
    blacklist:              [],
    whitelist:              [],
    sourceMap:              false,
    optional:               [],
    comments:               true,
    filename:               "unknown",
    modules:                "common",
    runtime:                false,
    loose:                  [],
    code:                   true,
    ast:                    true
  });

  // normalize windows path separators to unix
  opts.filename = util.normalisePathSeparator(opts.filename);
  if (opts.sourceRoot) {
    opts.sourceRoot = util.normalisePathSeparator(opts.sourceRoot);
  }

  opts.basename = path.basename(opts.filename, path.extname(opts.filename));

  opts.blacklist = util.arrayify(opts.blacklist);
  opts.whitelist = util.arrayify(opts.whitelist);
  opts.optional  = util.arrayify(opts.optional);
  opts.loose     = util.arrayify(opts.loose);

  if (includes(opts.loose, "all")) {
    opts.loose = Object.keys(transform.transformers);
  }

  defaults(opts, {
    moduleRoot: opts.sourceRoot
  });

  defaults(opts, {
    sourceRoot: opts.moduleRoot
  });

  defaults(opts, {
    filenameRelative: opts.filename
  });

  defaults(opts, {
    sourceFileName: opts.filenameRelative,
    sourceMapName:  opts.filenameRelative
  });

  if (opts.playground) {
    opts.experimental = true;
  }

  if (opts.runtime) {
    this.set("runtimeIdentifier", t.identifier("to5Runtime"));
  }

  opts.blacklist = transform._ensureTransformerNames("blacklist", opts.blacklist);
  opts.whitelist = transform._ensureTransformerNames("whitelist", opts.whitelist);
  opts.optional  = transform._ensureTransformerNames("optional", opts.optional);
  opts.loose     = transform._ensureTransformerNames("loose", opts.loose);

  if (opts.reactCompat) {
    opts.optional.push("reactCompat");
    console.error("The reactCompat option has been moved into the optional transformer " +
                  "`reactCompat` - backwards compatibility will be removed in v4.0.0");
  }

  var ensureEnabled = function (key) {
    var namespace = transform.namespaces[key];
    if (namespace === "playground") opts.playground = true;
    if (namespace === "experimental") opts.experimental = true;
  };

  each(opts.whitelist, ensureEnabled);
  each(opts.optional, ensureEnabled);

  return opts;
};

File.prototype.isLoose = function (key) {
  return includes(this.opts.loose, key);
};

File.prototype.buildTransformers = function () {
  var file = this;

  var transformers = {};

  var secondaryStack = [];
{'fraction_non_alphanumeric': 0.14820913956360643, 'fraction_numerical': 0.0012350761630300535, 'mean_word_length': 2.836162652980655, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5978449', 'n_tokens_mistral': 3059, 'n_tokens_neox': 2979, 'n_words': 560}