<commit_before>/* @flow weak */

const path = require('path');
const execa = require('execa');
const chalk = require('chalk');
const fs = require('fs-extra');
const readPkg = require('read-pkg');

const needle = require('@knit/needle');
const depchek = require('@knit/depcheck');

exports.isScoped = (module) => module[0] === '@';

exports.getModuleName = (module) => module.replace(/@.*?\//, '');

exports.readPkg = (dir) => {
  try {
    return readPkg.sync(path.join(needle.paths.modules, dir));
  } catch (err) {
    throw {
      message: `could not find a \`package.json\` in ${path.join(needle.paths.modulesStub, dir)}.`,
      stderr: `All directories under ${needle.paths.modulesStub} are expected to be node modules. To ignore a directory add a 'ignore: true' to the package.json.`,
    };
  }
};

exports.findDependencies = (dir) => (
   depchek(path.join(needle.paths.modules, dir))
    .then(res => Object.keys(res.using))
    .then(using => {
      const pkg = exports.readPkg(dir);
      return using.concat(Object.keys((pkg || {}).dependencies || {}));
    })
);

exports.findMissingDependencies = (using, modules) => {
  const installed = Object.keys(needle.pkg.dependencies || {});

  return using.filter(m => !installed.includes(m) && !modules.includes(m));
};

exports.findUnusedDependencies = (using, modules) => {
  const installed = Object.keys(needle.pkg.dependencies || {});

  return installed.filter(m => !using.includes(m) && !modules.includes(m));
};

exports.findInternalDependencies = (using, modules) => (
   using.filter(m => (
    modules.includes(m)
  ))
);

exports.findModules = () => {
  try {
    return fs.readdirSync(needle.paths.modules)
      .reduce((acc, d) => {
        if (exports.isScoped(d)) {
          return acc.concat(fs.readdirSync(path.join(needle.paths.modules, d)).map(dir => path.join(d, dir)));
        }

        return acc.concat(d);
      }, []).filter(d => fs.statSync(path.join(needle.paths.modules, d)).isDirectory());
  } catch (err) {
    throw {
      message: 'Modules directory could not be read.',
      stderr: `Make sure your modules are in \`${needle.paths.modulesStub}\``,
    };
  }
};

exports.findPublicModules = () => (
   exports.findModules().filter(m => {
     const pkg = exports.readPkg(m);
     return pkg && !pkg.ignore;
   })
);

exports.findModifiedSince = (modules, tag) => {
  const output = execa.sync('git', ['diff', '--dirstat=files,0', tag, '--', needle.paths.modulesStub]);
  const lines = (output.stdout || '').split('\n');

  const modified = lines.map(l => {
    const [scope, name] = l.split(`${needle.paths.modulesStub}/`)[1].split('/');
    return exports.isScoped(scope) ? [scope, name].join('/') : scope;
  }).filter(
    Boolean
  ).reduce(
    (acc, m) =>
       acc.includes(m) ? acc : acc.concat(m)

  , []).filter(
    m => modules.includes(m)
  );

  return modified;
};

exports.findAllDependencies = (modules) => (
   modules.reduce((acc, m) =>
     Promise.all([acc, exports.findDependencies(m)]).then(
      vals => (
        vals[0].concat(vals[1], modules)
      )<commit_msg>:bug: find updated modules in new projects without crashing<commit_after>/* @flow weak */

const path = require('path');
const execa = require('execa');
const chalk = require('chalk');
const fs = require('fs-extra');
const readPkg = require('read-pkg');

const needle = require('@knit/needle');
const depchek = require('@knit/depcheck');

exports.isScoped = (module) => module[0] === '@';

exports.getModuleName = (module) => module.replace(/@.*?\//, '');

exports.readPkg = (dir) => {
  try {
    return readPkg.sync(path.join(needle.paths.modules, dir));
  } catch (err) {
    throw {
      message: `could not find a \`package.json\` in ${path.join(needle.paths.modulesStub, dir)}.`,
      stderr: `All directories under ${needle.paths.modulesStub} are expected to be node modules. To ignore a directory add a 'ignore: true' to the package.json.`,
    };
  }
};

exports.findDependencies = (dir) => (
   depchek(path.join(needle.paths.modules, dir))
    .then(res => Object.keys(res.using))
    .then(using => {
      const pkg = exports.readPkg(dir);
      return using.concat(Object.keys((pkg || {}).dependencies || {}));
    })
);

exports.findMissingDependencies = (using, modules) => {
  const installed = Object.keys(needle.pkg.dependencies || {});

  return using.filter(m => !installed.includes(m) && !modules.includes(m));
};

exports.findUnusedDependencies = (using, modules) => {
  const installed = Object.keys(needle.pkg.dependencies || {});

  return installed.filter(m => !using.includes(m) && !modules.includes(m));
};

exports.findInternalDependencies = (using, modules) => (
   using.filter(m => (
    modules.includes(m)
  ))
);

exports.findModules = () => {
  try {
    return fs.readdirSync(needle.paths.modules)
      .reduce((acc, d) => {
        if (exports.isScoped(d)) {
          return acc.concat(fs.readdirSync(path.join(needle.paths.modules, d)).map(dir => path.join(d, dir)));
        }

        return acc.concat(d);
      }, []).filter(d => fs.statSync(path.join(needle.paths.modules, d)).isDirectory());
  } catch (err) {
    throw {
      message: 'Modules directory could not be read.',
      stderr: `Make sure your modules are in \`${needle.paths.modulesStub}\``,
    };
  }
};

exports.findPublicModules = () => (
   exports.findModules().filter(m => {
     const pkg = exports.readPkg(m);
     return pkg && !pkg.ignore;
   })
);

exports.findModifiedSince = (modules, tag) => {
  const output = execa.sync('git', ['diff', '--dirstat=files,0', tag, '--', needle.paths.modulesStub]);
  const lines = (output.stdout || '').split('\n').filter(x => x.length);

  const modified = lines.map(l => {
    const [scope, name] = l.split(`${needle.paths.modulesStub}/`)[1].split('/');
    return exports.isScoped(scope) ? [scope, name].join('/') : scope;
  }).filter(
    Boolean
  ).reduce(
    (acc, m) =>
       acc.includes(m) ? acc : acc.concat(m)

  , []).filter(
    m => modules.includes(m)
  );

  return modified;
};

exports.findAllDependencies = (modules) => (
   modules.reduce((acc, m) =>
     Promise.all([acc, exports.findDependencies(m)]).then(
      vals => (
        vals[0].concat(vals[1], modules)
      )
{'fraction_non_alphanumeric': 0.19552, 'fraction_numerical': 0.0016, 'mean_word_length': 3.4554526015680684, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2392868', 'n_tokens_mistral': 2107, 'n_tokens_neox': 2076, 'n_words': 409}