<commit_before>var path          = require('path'),
	fs            = require('fs'),
	child_process = require('child_process'),
	os            = require('os'),

	// Resolve directories
	rhinoDir      = path.join( path.dirname( fs.realpathSync( __filename )), '../../..' ),
	canDir        = path.join( rhinoDir, 'can' ),
	docsDir       = path.join( canDir, "docs" ),
	doccoDir      = path.join( canDir, "util/docco" ),
	doccoOutDir   = path.join( doccoDir, "docs" ),
	sourceDir     = path.join( doccoDir, 'standalone' ),
	makePath      = path.join( doccoDir, "makestandalone.js" ),
	genCommand;

function execCommandWithOutput( command, cwd, callback ) {

	var spawn, parts;

	parts = command.split(" ");
	console.log( parts );
	spawn = child_process.spawn( parts.shift(), parts, cwd || {
		cwd : cwd
	});

	["stdout", "stderr"].forEach( function( stream ) {
		spawn[stream].setEncoding("utf-8");
		spawn[stream].pipe( process[stream] );
	});

	spawn.on("exit", callback );

}

function runDocco() {

	fs.mkdir(docsDir, function() {

		fs.readdir("temp", function( err, files ) {
		
			var command = os.platform() != "win32" ?
				"docco " :
				"sh docco ";

			// Prepend temp to each file
			files = files.map(function( file ) {
				return path.join( "temp", file );
			});

			console.log( "Generating docco annotated source..." );

			execCommandWithOutput( command + files.join(" "), doccoDir, function() {
				fs.readdir( doccoOutDir, function( err, files ) {
					console.log("Moving files into place...");
					files.forEach(function( file ) {
						console.log( "\t" + file );
						fs.renameSync( "docs/" + file, "../../docs/" + file );
					});
					console.log("Cleaning up...");
					["temp", "standalone", "docs"].forEach(function( dir ) {
						fs.readdir( dir, function( e, files ) {
							files.forEach(function( file ) {
								fs.unlinkSync( path.join( dir, file ));
							});
							fs.rmdir( dir );
						});
					});
					console.log("Done!");
				});
			})

		});

	});

}

function format( exitCode ) {

	if ( exitCode != 0 ) {
		console.log("Error generating unminified sources.");
		return
	}

	fs.readdir( sourceDir, function( err, files ) {

		var count = 0;

		if ( ! files.length ) {
			console.log("Error - Source directory is empty");
		}

		// Only annotate full srcs
		files = files.filter(function( file ) {
			return file.indexOf(".min.") < 0;
		});

		// Create the temp directory for stripped code
		fs.mkdir("temp", function() {

			// Generate source for all standalones
			console.log( "Stripping multiline comments and converting tabs to four spaces:" );

			files.forEach(function( file ) {
				fs.readFile( path.join( sourceDir, file ), "utf-8", function( err, code ) {
					console.log( "\t" + file );
					code = code.replace( /\/\*(?:.*)(?:\n\s+\*.*)*/gim, "");
					code = code.replace( /\t/gim, "    ");
					fs.writeFile( path.join( "temp", file ), code, "utf-8", function() {
						if ( ++count == files.length ) {
							runDocco();
						}
					});
				});
			});
		});
	});
}

console.log("Generating unminified sources...");
if ( os.platform() != "win32" ) {
	genCommand = "./js " + makePath;
} else {
	genCommand = "js.bat " + makePath;
}
execCommandWithOutput( genCommand, rhinoDir, format );
<commit_msg>hopefully the last windows compat commit for docco.<commit_after>var path          = require('path'),
	fs            = require('fs'),
	child_process = require('child_process'),
	os            = require('os'),

	// Resolve directories
	rhinoDir      = path.join( path.dirname( fs.realpathSync( __filename )), '../../..' ),
	canDir        = path.join( rhinoDir, 'can' ),
	docsDir       = path.join( canDir, "docs" ),
	doccoDir      = path.join( canDir, "util/docco" ),
	doccoOutDir   = path.join( doccoDir, "docs" ),
	sourceDir     = path.join( doccoDir, 'standalone' ),
	makePath      = path.join( doccoDir, "makestandalone.js" ),
	genCommand;

function execCommandWithOutput( command, cwd, callback ) {

	var spawn, parts;

	parts = command.split(" ");
	spawn = child_process.spawn( parts.shift(), parts, {
		cwd : cwd,
		env : process.env
	});

	["stdout", "stderr"].forEach( function( stream ) {
		spawn[stream].setEncoding("utf-8");
		spawn[stream].pipe( process[stream] );
	});

	spawn.on("exit", callback );

}

function runDocco() {

	fs.mkdir(docsDir, function() {

		fs.readdir("temp", function( err, files ) {
		
			var command = os.platform() != "win32" ?
				"docco " :
				"sh docco ";

			// Prepend temp to each file
			files = files.map(function( file ) {
				return path.join( "temp", file );
			});

			console.log( "Generating docco annotated source..." );

			execCommandWithOutput( command + files.join(" "), doccoDir, function() {
				fs.readdir( doccoOutDir, function( err, files ) {
					console.log("Moving files into place...");
					files.forEach(function( file ) {
						console.log( "\t" + file );
						fs.renameSync( "docs/" + file, "../../docs/" + file );
					});
					console.log("Cleaning up...");
					["temp", "standalone", "docs"].forEach(function( dir ) {
						fs.readdir( dir, function( e, files ) {
							files.forEach(function( file ) {
								fs.unlinkSync( path.join( dir, file ));
							});
							fs.rmdir( dir );
						});
					});
					console.log("Done!");
				});
			})

		});

	});

}

function format( exitCode ) {

	if ( exitCode != 0 ) {
		console.log("Error generating unminified sources.");
		return
	}

	fs.readdir( sourceDir, function( err, files ) {

		var count = 0;

		if ( ! files.length ) {
			console.log("Error - Source directory is empty");
		}

		// Only annotate full srcs
		files = files.filter(function( file ) {
			return file.indexOf(".min.") < 0;
		});

		// Create the temp directory for stripped code
		fs.mkdir("temp", function() {

			// Generate source for all standalones
			console.log( "Stripping multiline comments and converting tabs to four spaces:" );

			files.forEach(function( file ) {
				fs.readFile( path.join( sourceDir, file ), "utf-8", function( err, code ) {
					console.log( "\t" + file );
					code = code.replace( /\/\*(?:.*)(?:\n\s+\*.*)*/gim, "");
					code = code.replace( /\t/gim, "    ");
					fs.writeFile( path.join( "temp", file ), code, "utf-8", function() {
						if ( ++count == files.length ) {
							runDocco();
						}
					});
				});
			});
		});
	});
}

console.log("Generating unminified sources...");

if ( os.platform() != "win32" ) {
	genCommand = "./js " + makePath;
} else {
	genCommand = "js.bat " + makePath;
}

execCommandWithOutput( genCommand, rhinoDir, format );
{'fraction_non_alphanumeric': 0.1848124428179323, 'fraction_numerical': 0.0030497102775236353, 'mean_word_length': 2.720363017583664, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1966303', 'n_tokens_mistral': 2592, 'n_tokens_neox': 2173, 'n_words': 510}