<commit_before>#!/usr/bin/env node

/*global require, global, process*/
const parseArgs = require('../deps/minimist.js'),
      { string, arr } = (typeof lively !== "undefined" && lively.lang) || require("../deps/lively.lang.min.js"),
      { resource } = (typeof lively !== "undefined" && lively.resources) || require("../deps/lively.resources.js"),
      { join: j, isAbsolute, normalize } = require("path"),
      fs = require("fs"),
      { spawn } = require("child_process"),
      flatn = require("../flatn-cjs.js"),
      args = process.argv.slice(2);

if (printHelp(args)) process.exit(0);

let activeDir = process.cwd(),
    command,
    cmdIndex = args.findIndex(arg =>
      ["list", "install", "node", "env"]
        .includes(arg) && (command = arg));

processFlatnConfig();

switch (command) {
  case 'list':    doList(   args.slice(0, cmdIndex), args.slice(cmdIndex+1)); break;
  case 'install': doInstall(args.slice(0, cmdIndex), args.slice(cmdIndex+1)); break;
  case 'node':    doNode(   args.slice(0, cmdIndex), args.slice(cmdIndex+1)); break;
  case 'env':     doEnv(    args.slice(0, cmdIndex), args.slice(cmdIndex+1)); break;
  default: strangeInvocation();
}

// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// processing arguments

function readGenericArgs(args, requireAtLeastOnePackageDir = true) {
  return currentPackages(
    parseArgs(args, {
      alias: {
        packages: "C",
        "dev-package": "D",
        package: "P"
      }
    }), requireAtLeastOnePackageDir);
  // let {packageCollectionDirs, individualPackageDirs, devPackageDirs} = currentPackages(args);
}

function currentPackages(args, requireAtLeastOnePackageDir) {
  let fromArgs = packagesFromArgs(args),
      fromEnv = flatn.packageDirsFromEnv(),
      packageCollectionDirs = uniq(fromArgs.packageCollectionDirs.concat(fromEnv.packageCollectionDirs)),
      individualPackageDirs = uniq(fromArgs.individualPackageDirs.concat(fromEnv.individualPackageDirs)),
      devPackageDirs = uniq(fromArgs.devPackageDirs.concat(fromEnv.devPackageDirs));

  if (process.env.FLATN_PACKAGE_DIRS) {
    for (let dir of process.env.FLATN_PACKAGE_DIRS.split(":"))
      if (!packageCollectionDirs.includes(dir)) packageCollectionDirs.push(dir);
  }
  return checkPackages(
    packageCollectionDirs,
    individualPackageDirs,
    devPackageDirs,
    requireAtLeastOnePackageDir
  );
}

function packagesFromArgs(args) {
  return {
    packageCollectionDirs: normalizeDirs(args["packages"]),
    individualPackageDirs: normalizeDirs(args["package"]),
    devPackageDirs: normalizeDirs(args["dev-package"])
  };
}

function normalizeDirs(dirs) {
  return (dirs && typeof dirs === "string" ? [dirs] : dirs || [])
     .map(ea => isAbsolute(ea) ? ea : normalize(j(activeDir, ea)))
     .filter(Boolean);
}

function checkPackages(
  packageCollectionDirs,
  individualPackageDirs,
  devPackageDirs,
  requireAtLeastOnePackageDir
) {
  [].concat(packageCollectionDirs)
    .concat(individualPackageDirs)
    .concat(devPackageDirs)
    .forEach(p => {
      if (fs.existsSync(p)) return;
      console.error(`[flatn] package path ${p} does not exist!`);
      process.exit(1);
    });

  if (requireAtLeastOnePackageDir) {
    if (![].concat(packageCollectionDirs)
           .concat(individualPackageDirs)
           .concat(devPackageDirs).length) {
      console.error("No package directories specified.\n"
                  + "Use one or multiple --packages/-C, --package/-P, and --dev-package/-D\n"
                  + "to specify package directories.");
      return process.exit(1);
    }
  }

  return {packageCollectionDirs, individualPackageDirs, devPackageDirs};
}

function unexpectedArgs(args, expected, commandName) {
  expected = expected.concat("_");
  let unexpected = Object.keys(args).filter(ea => !expected.includes(ea));
  if (unexpected.length) {
    console.error(`command "${commandName}" does not expect arguments ${unexpected.map(JSON.stringify).join(", ")}`);
    process.exit(1);
  }
}


function processFlatnConfig() {
  let configFile = j(activeDir, ".flatn.json"),
      globalConfigFile = j(process.env.HOME, ".flatn.json"),
      config = fs.existsSync(configFile) ? JSON.parse(fs.readFileSync(configFile)) : null,
      globalConfig = fs.existsSync(globalConfigFile) ? JSON.parse(fs.readFileSync(globalConfigFile)) : null;
  if (!config && !globalConfig) return;
  let fullConfig = {};

  if (globalConfig) {
    // console.log(`Using config from ${globalConfigFile}`);
    fullConfig.FLATN_PACKAGE_COLLECTION_DIRS = (globalConfig.FLATN_PACKAGE_COLLECTION_DIRS || []).map(makeAbsolute(process.env.HOME));
    fullConfig.FLATN_PACKAGE_DIRS =            (globalConfig.FLATN_PACKAGE_DIRS || []).map(makeAbsolute(process.env.HOME));
    fullConfig.FLATN_DEV_PACKAGE_DIRS =        (globalConfig.FLATN_DEV_PACKAGE_DIRS || []).map(makeAbsolute(process.env.HOME));
  }

  if (config) {
    // console.log(`Using config from ${configFile}`)
    fullConfig.FLATN_PACKAGE_COLLECTION_DIRS = (config.FLATN_PACKAGE_COLLECTION_DIRS || []).map(makeAbsolute(activeDir));
    fullConfig.FLATN_PACKAGE_DIRS =            (config.FLATN_PACKAGE_DIRS || []).map(makeAbsolute(activeDir));
    fullConfig.FLATN_DEV_PACKAGE_DIRS =        (config.FLATN_DEV_PACKAGE_DIRS || []).map(makeAbsolute(activeDir));
  }

  process.env.FLATN_PACKAGE_COLLECTION_DIRS = arr.uniq(fullConfig.FLATN_PACKAGE_COLLECTION_DIRS.concat((process.env.FLATN_PACKAGE_COLLECTION_DIRS || "").split(":"))).join(":");
  process.env.FLATN_PACKAGE_DIRS = arr.uniq(fullConfig.FLATN_PACKAGE_DIRS.concat((process.env.FLATN_PACKAGE_DIRS || "").split(":"))).join(":");
  process.env.FLATN_DEV_PACKAGE_DIRS = arr.uniq(fullConfig.FLATN_DEV_PACKAGE_DIRS.concat((process.env.FLATN_DEV_PACKAGE_DIRS || "").split(":"))).join(":");
  
  function makeAbsolute(baseDir) {
    return path => isAbsolute(path) ? path : j(baseDir, path);
  }
}

// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// commands

function doNode(genericArgs, args) {
  // starts a node process that uses the package dirs from arguments and from
  // the process env to resolve package names

  let cmdArgs = parseArgs(args, {"--": true});
  unexpectedArgs(cmdArgs, ["--"], "node");

  // so node knows about the package dirs passed by arguments....
  let {packageCollectionDirs, individualPackageDirs, devPackageDirs} = readGenericArgs(genericArgs);
  flatn.setPackageDirsOfEnv(packageCollectionDirs, individualPackageDirs, devPackageDirs);

  if (!fs.existsSync(j(activeDir, "package.json"))) {
    console.error(`No package.json is present in ${activeDir}, cannot install!`)
    process.exit(1);
  }

  var tty = require('tty'),
      spawn = require('child_process').spawn;

  var child = spawn("node", cmdArgs["--"], {stdio: [0,1,2]});
  process.stdin.resume();  
  child.on("exit", function(code, signal) {
    process.stdin.pause();
    process.exit(code);
  });

  // setTimeout(() => process.exit(0), 3000);
}




function doList(genericArgs, args) {
  let cmdArgs = parseArgs(args, {alias: {}});
  unexpectedArgs(cmdArgs, [], "list");

  let {packageCollectionDirs, individualPackageDirs, devPackageDirs} = readGenericArgs(genericArgs);
  flatn.setPackageDirsOfEnv(packageCollectionDirs, individualPackageDirs, devPackageDirs);

  return Promise.resolve(
    flatn
      .buildPackageMap(packageCollectionDirs, individualPackageDirs, devPackageDirs)
      .allPackages())
    .then(packageSpecs =>
      console.log(
        string.printTable(
          packageSpecs.map(({config, location}) => [
            `${config.name}@${config.version}`,
            `${location}`
          ]))))
    .then(() => process.exit(0))
    .catch(err => {
      console.error(err.stack);
      process.exit(1);
    });
}


<commit_msg>fix flatn list<commit_after>#!/usr/bin/env node

/*global require, global, process*/
const parseArgs = require('../deps/minimist.js'),
      { string, arr } = (typeof lively !== "undefined" && lively.lang) || require("../deps/lively.lang.min.js"),
      { resource } = (typeof lively !== "undefined" && lively.resources) || require("../deps/lively.resources.js"),
      { join: j, isAbsolute, normalize } = require("path"),
      fs = require("fs"),
      { spawn } = require("child_process"),
      flatn = require("../flatn-cjs.js"),
      args = process.argv.slice(2);

if (printHelp(args)) process.exit(0);

let activeDir = process.cwd(),
    command,
    cmdIndex = args.findIndex(arg =>
      ["list", "install", "node", "env"]
        .includes(arg) && (command = arg));

processFlatnConfig();

switch (command) {
  case 'list':    doList(   args.slice(0, cmdIndex), args.slice(cmdIndex+1)); break;
  case 'install': doInstall(args.slice(0, cmdIndex), args.slice(cmdIndex+1)); break;
  case 'node':    doNode(   args.slice(0, cmdIndex), args.slice(cmdIndex+1)); break;
  case 'env':     doEnv(    args.slice(0, cmdIndex), args.slice(cmdIndex+1)); break;
  default: strangeInvocation();
}

// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// processing arguments

function readGenericArgs(args, requireAtLeastOnePackageDir = true) {
  return currentPackages(
    parseArgs(args, {
      alias: {
        packages: "C",
        "dev-package": "D",
        package: "P"
      }
    }), requireAtLeastOnePackageDir);
  // let {packageCollectionDirs, individualPackageDirs, devPackageDirs} = currentPackages(args);
}

function currentPackages(args, requireAtLeastOnePackageDir) {
  let fromArgs = packagesFromArgs(args),
      fromEnv = flatn.packageDirsFromEnv(),
      packageCollectionDirs = uniq(fromArgs.packageCollectionDirs.concat(fromEnv.packageCollectionDirs)),
      individualPackageDirs = uniq(fromArgs.individualPackageDirs.concat(fromEnv.individualPackageDirs)),
      devPackageDirs = uniq(fromArgs.devPackageDirs.concat(fromEnv.devPackageDirs));

  if (process.env.FLATN_PACKAGE_DIRS) {
    for (let dir of process.env.FLATN_PACKAGE_DIRS.split(":"))
      if (!packageCollectionDirs.includes(dir)) packageCollectionDirs.push(dir);
  }
  return checkPackages(
    packageCollectionDirs,
    individualPackageDirs,
    devPackageDirs,
    requireAtLeastOnePackageDir
  );
}

function packagesFromArgs(args) {
  return {
    packageCollectionDirs: normalizeDirs(args["packages"]),
    individualPackageDirs: normalizeDirs(args["package"]),
    devPackageDirs: normalizeDirs(args["dev-package"])
  };
}

function normalizeDirs(dirs) {
  return (dirs && typeof dirs === "string" ? [dirs] : dirs || [])
     .map(ea => isAbsolute(ea) ? ea : normalize(j(activeDir, ea)))
     .filter(Boolean);
}

function checkPackages(
  packageCollectionDirs,
  individualPackageDirs,
  devPackageDirs,
  requireAtLeastOnePackageDir
) {
  [].concat(packageCollectionDirs)
    .concat(individualPackageDirs)
    .concat(devPackageDirs)
    .forEach(p => {
      if (fs.existsSync(p)) return;
      console.error(`[flatn] package path ${p} does not exist!`);
      process.exit(1);
    });

  if (requireAtLeastOnePackageDir) {
    if (![].concat(packageCollectionDirs)
           .concat(individualPackageDirs)
           .concat(devPackageDirs).length) {
      console.error("No package directories specified.\n"
                  + "Use one or multiple --packages/-C, --package/-P, and --dev-package/-D\n"
                  + "to specify package directories.");
      return process.exit(1);
    }
  }

  return {packageCollectionDirs, individualPackageDirs, devPackageDirs};
}

function unexpectedArgs(args, expected, commandName) {
  expected = expected.concat("_");
  let unexpected = Object.keys(args).filter(ea => !expected.includes(ea));
  if (unexpected.length) {
    console.error(`command "${commandName}" does not expect arguments ${unexpected.map(JSON.stringify).join(", ")}`);
    process.exit(1);
  }
}


function processFlatnConfig() {
  let configFile = j(activeDir, ".flatn.json"),
      globalConfigFile = j(process.env.HOME, ".flatn.json"),
      config = fs.existsSync(configFile) ? JSON.parse(fs.readFileSync(configFile)) : null,
      globalConfig = fs.existsSync(globalConfigFile) ? JSON.parse(fs.readFileSync(globalConfigFile)) : null;
  if (!config && !globalConfig) return;
  let fullConfig = {};

  if (globalConfig) {
    // console.log(`Using config from ${globalConfigFile}`);
    fullConfig.FLATN_PACKAGE_COLLECTION_DIRS = (globalConfig.FLATN_PACKAGE_COLLECTION_DIRS || []).map(makeAbsolute(process.env.HOME));
    fullConfig.FLATN_PACKAGE_DIRS =            (globalConfig.FLATN_PACKAGE_DIRS || []).map(makeAbsolute(process.env.HOME));
    fullConfig.FLATN_DEV_PACKAGE_DIRS =        (globalConfig.FLATN_DEV_PACKAGE_DIRS || []).map(makeAbsolute(process.env.HOME));
  }

  if (config) {
    // console.log(`Using config from ${configFile}`)
    fullConfig.FLATN_PACKAGE_COLLECTION_DIRS = (config.FLATN_PACKAGE_COLLECTION_DIRS || []).map(makeAbsolute(activeDir));
    fullConfig.FLATN_PACKAGE_DIRS =            (config.FLATN_PACKAGE_DIRS || []).map(makeAbsolute(activeDir));
    fullConfig.FLATN_DEV_PACKAGE_DIRS =        (config.FLATN_DEV_PACKAGE_DIRS || []).map(makeAbsolute(activeDir));
  }

  process.env.FLATN_PACKAGE_COLLECTION_DIRS = arr.uniq(fullConfig.FLATN_PACKAGE_COLLECTION_DIRS.concat((process.env.FLATN_PACKAGE_COLLECTION_DIRS || "").split(":"))).join(":");
  process.env.FLATN_PACKAGE_DIRS = arr.uniq(fullConfig.FLATN_PACKAGE_DIRS.concat((process.env.FLATN_PACKAGE_DIRS || "").split(":"))).join(":");
  process.env.FLATN_DEV_PACKAGE_DIRS = arr.uniq(fullConfig.FLATN_DEV_PACKAGE_DIRS.concat((process.env.FLATN_DEV_PACKAGE_DIRS || "").split(":"))).join(":");
  
  function makeAbsolute(baseDir) {
    return path => isAbsolute(path) ? path : j(baseDir, path);
  }
}

// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// commands

function doNode(genericArgs, args) {
  // starts a node process that uses the package dirs from arguments and from
  // the process env to resolve package names

  let cmdArgs = parseArgs(args, {"--": true});
  unexpectedArgs(cmdArgs, ["--"], "node");

  // so node knows about the package dirs passed by arguments....
  let {packageCollectionDirs, individualPackageDirs, devPackageDirs} = readGenericArgs(genericArgs);
  flatn.setPackageDirsOfEnv(packageCollectionDirs, individualPackageDirs, devPackageDirs);

  if (!fs.existsSync(j(activeDir, "package.json"))) {
    console.error(`No package.json is present in ${activeDir}, cannot install!`)
    process.exit(1);
  }

  var tty = require('tty'),
      spawn = require('child_process').spawn;

  var child = spawn("node", cmdArgs["--"], {stdio: [0,1,2]});
  process.stdin.resume();  
  child.on("exit", function(code, signal) {
    process.stdin.pause();
    process.exit(code);
  });

  // setTimeout(() => process.exit(0), 3000);
}




function doList(genericArgs, args) {
  let cmdArgs = parseArgs(args, {alias: {}});
  unexpectedArgs(cmdArgs, [], "list");

  let {packageCollectionDirs, individualPackageDirs, devPackageDirs} = readGenericArgs(genericArgs);
  flatn.setPackageDirsOfEnv(packageCollectionDirs, individualPackageDirs, devPackageDirs);

  return Promise.resolve(
    flatn
      .buildPackageMap(packageCollectionDirs, individualPackageDirs, devPackageDirs)
      .allPackages())
    .then(packageSpecs =>
      console.log(
        string.printTable(
          packageSpecs.map(({name, version, location}) => [
            `${name}@${version}`,
            `${location}`
          ]))))
    .then(() => process.exit(0))
    .catch(err => {
      console.error(err.stack);
      process.exit(1);
    });
}



{'fraction_non_alphanumeric': 0.16407054582904224, 'fraction_numerical': 0.003089598352214212, 'mean_word_length': 3.992609254498715, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2032991', 'n_tokens_mistral': 5158, 'n_tokens_neox': 5022, 'n_words': 833}