<commit_before>import path = require("path")
import os = require("os")
import cluster = require("cluster")
import Bluebird = require("bluebird")
import fs = require("fs")
import unixify = require("unixify")
import _ = require("lodash")
import linez = require("linez")
import spinner = require("cli-spinner")
import logger = require("./logger")
import util = require("./util")
import log_helper = require("./plugin/log_helper")

Bluebird.promisifyAll(fs)

const log = logger.create("plugin")

const FILE_EXT = /\.[^\.]*$/

const is_plugin = (name : string) : boolean =>
  !!/^vile-/.test(name)

const valid_plugin = (api : vile.Plugin) : boolean =>
  !!(api && typeof api.punish == "function")

const is_array = (list : any[]) : boolean =>
  !!(list && typeof list.forEach == "function")

const is_promise = (list : Bluebird<any>) : boolean =>
  !!(list && typeof list.then == "function")

const require_plugin = (name : string) : vile.Plugin | null => {
  let cwd_node_modules = path.join(process.cwd(), "node_modules")

  let plugin : vile.Plugin

  try {
    plugin = require(`${cwd_node_modules}/@forthright/vile-${name}`)
  } catch (error) {
    log.error(_.get(error, "stack", error))
  }

  return plugin
}

const map_plugin_name_to_issues = (name : string) => (issues : vile.Issue[]) =>
  _.map(issues, (issue : vile.Issue) =>
    (issue.plugin = name, issue))

const run_plugin = (
  name : string,
  config : vile.PluginConfig = {
    config: {},
    ignore: []
  }
) : Bluebird<vile.IssueList> =>
  new Bluebird((
    resolve : (i : vile.IssueList) => void,
    reject : (e : string) => void
  ) => {
    let api : vile.Plugin = require_plugin(name)

    if (!valid_plugin(api)) reject(`invalid plugin API: ${name}`)

    let issues : any = api.punish(config)

    if (is_promise(issues)) {
      issues
        .then(map_plugin_name_to_issues(name))
        .then(resolve)
        .catch(reject) // TODO: keep running other plugins?
    } else if (is_array(issues)) {
      resolve(map_plugin_name_to_issues(name)(issues))
    } else {
      console.warn(`${name} plugin did not return [] or Promise<[]>`)
      resolve(<any>[]) // TODO: ?
    }
  })

const log_plugins_finished = (pkg_names : string[]) => {
  _.each(pkg_names, (plugin_name : string) => {
    log.info(`${ plugin_name.replace("vile-", "") }:finish`)
  })
}

const run_plugins_in_fork = (
  plugins : vile.PluginList,
  config  : vile.YMLConfig,
  worker  : cluster.Worker
) : Bluebird<vile.IssueList> =>
  new Bluebird((
    resolve : (issue : vile.IssueList) => void,
    reject : (error : string) => void
  ) => {
    worker.on("message", (issues : vile.IssueList) => {
      if (issues) {
        worker.disconnect()
        resolve(issues)
      } else {
        worker.send(<vile.Lib.PluginWorkerData>{
          plugins: plugins,
          config: config
        })
      }
    })

    worker.on("exit", (code, signal) => {
      let name = plugins.join(",")

      if (signal || code !== 0) {
        reject(`${name} worker exited [code = ${code} | sig = ${signal}]`)
      }
    })
  })

const normalize_paths = (issues : vile.IssueList) =>
  _.each(issues, (issue) => {
    if (_.has(issue, "path")) {
      issue.path = unixify(issue.path)
        .replace(process.cwd(), "")

      // HACK: see above todo
      if (!/windows/i.test(os.type())) {
        issue.path = unixify(issue.path)
          .replace(/^\.\/?/, "")
          .replace(/^\/?/, "")
      }
    }
  })

const check_for_uninstalled_plugins = (
  allowed : string[],
  plugins : vile.PluginList
) => {
  let errors = false

  _.each(allowed, (name : string) => {
    if (!_.some(plugins, (plugin : string) =>
      plugin.replace("vile-", "") == name
    )) {
      errors = true
      log.error(`${name} is not installed`)
    }
  })

  // Hack: should be doing this in cli modules
  if (errors) process.exit(1)
}<commit_msg>Let's try being more simple.<commit_after>import path = require("path")
import os = require("os")
import cluster = require("cluster")
import Bluebird = require("bluebird")
import fs = require("fs")
import unixify = require("unixify")
import _ = require("lodash")
import linez = require("linez")
import spinner = require("cli-spinner")
import logger = require("./logger")
import util = require("./util")
import log_helper = require("./plugin/log_helper")

Bluebird.promisifyAll(fs)

const log = logger.create("plugin")

const FILE_EXT = /\.[^\.]*$/

const is_plugin = (name : string) : boolean =>
  !!/^vile-/.test(name)

const valid_plugin = (api : vile.Plugin) : boolean =>
  !!(api && typeof api.punish == "function")

const is_array = (list : any[]) : boolean =>
  !!(list && typeof list.forEach == "function")

const is_promise = (list : Bluebird<any>) : boolean =>
  !!(list && typeof list.then == "function")

const require_plugin = (name : string) : vile.Plugin | null => {
  let cwd_node_modules = path.join(process.cwd(), "node_modules")

  let plugin : vile.Plugin

  try {
    plugin = require(`${cwd_node_modules}/@forthright/vile-${name}`)
  } catch (error) {
    log.error(_.get(error, "stack", error))
  }

  return plugin
}

const map_plugin_name_to_issues = (name : string) => (issues : vile.Issue[]) =>
  _.map(issues, (issue : vile.Issue) =>
    (issue.plugin = name, issue))

const run_plugin = (
  name : string,
  config : vile.PluginConfig = {
    config: {},
    ignore: []
  }
) : Bluebird<vile.IssueList> =>
  new Bluebird((
    resolve : (i : vile.IssueList) => void,
    reject : (e : string) => void
  ) => {
    let api : vile.Plugin = require_plugin(name)

    if (!valid_plugin(api)) reject(`invalid plugin API: ${name}`)

    let issues : any = api.punish(config)

    if (is_promise(issues)) {
      issues
        .then(map_plugin_name_to_issues(name))
        .then(resolve)
        .catch(reject) // TODO: keep running other plugins?
    } else if (is_array(issues)) {
      resolve(map_plugin_name_to_issues(name)(issues))
    } else {
      console.warn(`${name} plugin did not return [] or Promise<[]>`)
      resolve(<any>[]) // TODO: ?
    }
  })

const log_plugins_finished = (pkg_names : string[]) => {
  _.each(pkg_names, (plugin_name : string) => {
    log.info(`${ plugin_name.replace("vile-", "") }:finish`)
  })
}

const run_plugins_in_fork = (
  plugins : vile.PluginList,
  config  : vile.YMLConfig,
  worker  : cluster.Worker
) : Bluebird<vile.IssueList> =>
  new Bluebird((
    resolve : (issue : vile.IssueList) => void,
    reject : (error : string) => void
  ) => {
    worker.on("message", (issues : vile.IssueList) => {
      if (issues) {
        worker.disconnect()
        resolve(issues)
      } else {
        worker.send(<vile.Lib.PluginWorkerData>{
          plugins: plugins,
          config: config
        })
      }
    })

    worker.on("exit", (code, signal) => {
      let name = plugins.join(",")

      if (signal || code !== 0) {
        reject(`${name} worker exited [code = ${code} | sig = ${signal}]`)
      }
    })
  })

const normalize_paths = (issues : vile.IssueList) =>
  _.each(issues, (issue) => {
    if (_.has(issue, "path")) {
      issue.path = unixify(issue.path)

      if (process.cwd() !== ".") {
        issue.path = issue.path
          .replace(process.cwd(), "")
      }

      issue.path = issue.path.replace(/^\.\//, "")
    }
  })

const check_for_uninstalled_plugins = (
  allowed : string[],
  plugins : vile.PluginList
) => {
  let errors = false

  _.each(allowed, (name : string) => {
    if (!_.some(plugins, (plugin : string) =>
      plugin.replace("vile-", "") == name
    )) {
      errors = true
      log.error(`${name} is not installed`)
    }
  })

  // Hack: should be doing this in cli modules
  if (errors) process.exit(1)
}
{'fraction_non_alphanumeric': 0.1779704892570541, 'fraction_numerical': 0.0005177323323841574, 'mean_word_length': 2.9524296675191817, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1593039', 'n_tokens_mistral': 2641, 'n_tokens_neox': 2573, 'n_words': 566}