<commit_before>'use strict'

const async = require('async')
const path  = require('path')
const mm    = require('micromatch')
const fse   = require('fs-extra')
const log   = require('./log')

/**
 * Run multiple route functions parallel.
 * @public
 * @param {Array} routes - Array of route configurations.
 * @param {String} srcPath - Path to the source folder.
 * @param {String} distPath - Path to the destination folder.
 * @param {Function} next - The callback that handles the response. Receives the following properties: err.
 */
module.exports = function(routes, srcPath, distPath, next) {

	const handlers = []

	const addFile = (file) => {

		routes.forEach((route, index) => {

			// Absolute path to the requested file
			const filePath = file.path

			// Set file path relative to the src path as route paths are relative, too
			const fileRoute = path.relative(srcPath, filePath)

			// Only add handler to fn queue when fileRoute and route path matches
			if (mm.isMatch(fileRoute, route.path)===false) return false

			const fn = (next) => {

				log(`{cyan:Starting handler: {magenta:${ route.name } {grey:${ fileRoute }`)

				const processHandler = ({ data, savePath }) => {

					log(`{cyan:Finished handler: {magenta:${ route.name } {grey:${ fileRoute }`)

					if (data==null) {
						next(new Error(`Handler of route '${ route.name }' returned without data`))
						return false
					}

					if (savePath==null || savePath==='') {
						next(new Error(`File of route '${ route.name }' could not be saved as no path has been specified by the handler`))
						return false
					}

					fse.outputFile(savePath, data, next)
					return true

				}

				route
					.handler(filePath, srcPath, distPath, route)
					.then(processHandler, next)

			}

			handlers.push(fn)

		})

	}

	const executeHandlers = () => {

		// Run each file handler and continue with next
		async.parallel(handlers, next)

	}

	fse
		.walk(srcPath)
		.on('data', addFile)
		.on('end', executeHandlers)

}<commit_msg>Reorganized run.js to be more like rewrite.js<commit_after>'use strict'

const async = require('async')
const path  = require('path')
const mm    = require('micromatch')
const fse   = require('fs-extra')
const log   = require('./log')

/**
 * Run multiple route functions parallel.
 * @public
 * @param {Array} routes - Array of route configurations.
 * @param {String} srcPath - Path to the source folder.
 * @param {String} distPath - Path to the destination folder.
 * @param {Function} next - The callback that handles the response. Receives the following properties: err.
 */
module.exports = function(routes, srcPath, distPath, next) {

	const handlers = []

	const addFile = (file) => {

		// Absolute path to the requested file
		const filePath = file.path

		// Set file path relative to the src path as route paths are relative, too
		const fileRoute = path.relative(srcPath, filePath)

		// Generate an array of matching routes
		const matches = routes.filter((route) => mm.isMatch(fileRoute, route.path))

		// Only add handler to fn queue when fileRoute and route path matches
		if (matches.length===0) return false

		// Rewrite request using the first matching route only
		const route = matches[0]

		const fn = (next) => {

			log(`{cyan:Starting handler: {magenta:${ route.name } {grey:${ fileRoute }`)

			const processHandler = ({ data, savePath }) => {

				log(`{cyan:Finished handler: {magenta:${ route.name } {grey:${ fileRoute }`)

				if (data==null) {
					return next(new Error(`Handler of route '${ route.name }' returned without data`))
				}

				if (savePath==null || savePath==='') {
					return next(new Error(`File of route '${ route.name }' could not be saved as no path has been specified by the handler`))
				}

				fse.outputFile(savePath, data, next)

			}

			route
				.handler(filePath, srcPath, distPath, route)
				.then(processHandler, next)

		}

		// Add fn to final list of handlers
		handlers.push(fn)

	}

	const executeHandlers = () => {

		// Run each file handler and continue with next
		async.parallel(handlers, next)

	}

	fse
		.walk(srcPath)
		.on('data', addFile)
		.on('end', executeHandlers)

}
{'fraction_non_alphanumeric': 0.1285680019189254, 'fraction_numerical': 0.0004797313504437515, 'mean_word_length': 3.339229968782518, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1291571', 'n_tokens_mistral': 1450, 'n_tokens_neox': 1241, 'n_words': 430}