<commit_before>#!/usr/bin/env node
'use strict'
const fs = require('fs')
const path = require('path')
const Queue = require('work').Queue
const commandLineArgs = require('command-line-args')
const tool = require('command-line-tool')
const command = require('./command')

const cli = commandLineArgs([
  { name: 'folders', type: String, multiple: true }
])

const options = cli.parse()

function * makeFolderList () {
  const folders = [ 'amd', 'commonjs', 'global' ]
  for (let one of folders) {
    for (let two of fs.readdirSync(one)) {
      yield path.join(one, two)
    }
  }
}

const queue = new Queue()
const folderList = options.folders || makeFolderList()

for (let folder of folderList) {
  queue.push(command.jsdoc(folder))
  queue.push(command.jsdocParse(folder))
  queue.push(command.dmd(folder))
}

queue
  .on('shift', task => console.log(task.name))
  .process()

// process.on('unhandledRejection', (err, p) => {
//   console.log('UNHANDLED', p)
//   tool.stop(err.stack || err, 1)
// })

// process.on('handledRejection', p => {
//   console.log('HANDLED', p)
// })<commit_msg>concurrency<commit_after>#!/usr/bin/env node
'use strict'
const fs = require('fs')
const path = require('path')
const Queue = require('work').Queue
const commandLineArgs = require('command-line-args')
const tool = require('command-line-tool')
const command = require('./command')
const Task = require('work').Task

const cli = commandLineArgs([
  { name: 'folders', type: String, multiple: true }
])

const options = cli.parse()

function * makeFolderList () {
  const folders = [ 'amd', 'commonjs', 'global' ]
  for (let one of folders) {
    for (let two of fs.readdirSync(one)) {
      yield path.join(one, two)
    }
  }
}

const mainQueue = new Queue({ maxConcurrent: 10 })
const folderList = options.folders || makeFolderList()

for (let folder of folderList) {
  const task = new Task(function (deferred) {
    const queue = new Queue()
    queue
      .push(command.jsdoc(folder))
      .push(command.jsdocParse(folder))
      .push(command.dmd(folder))
      .on('shift', task => console.log(task.name))
      .on('complete', deferred.resolve)
      .on('error', deferred.reject)
      .process()
  })
  mainQueue.push(task)
}

mainQueue.process()

// process.on('unhandledRejection', (err, p) => {
//   console.log('UNHANDLED', p)
//   tool.stop(err.stack || err, 1)
// })

// process.on('handledRejection', p => {
//   console.log('HANDLED', p)
// })
{'fraction_non_alphanumeric': 0.1771172638436482, 'fraction_numerical': 0.0016286644951140066, 'mean_word_length': 3.761627906976744, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5947367', 'n_tokens_mistral': 837, 'n_tokens_neox': 824, 'n_words': 194}