<commit_before>//#Fetch
//Fetch can be used to automate the downloading of resources into your project
//it accepts the following types of operations:
//
//1.	**file** - download a file and copy it to the target
//1.	**clone** - clone a git repository and copy all or parts of its
//contents to specified targets
//1.	**zip** - download a zip file, extract it and copy all or parts of its
//contents to specified targets
//
//**_Basic use:_**
//
//_single fetch:_
//
//		fetch({
//			file: "http://code.jquery.com/jquery.js",
//			target: "javascripts/vendor/jquery.js"
//		}, function callback( errors ){
//			//if any errors it will be an array
//		});
//_batch fetch:_
//
//		fetch([
//			file: "http://code.jquery.com/jquery.js",
//			target: "jquery.js"
//		},{
//			"clone": "https://code.google.com/p/dat-gui/",
//			"target": {
//				"src/dat": "dat/",
//				"build": "datgui/build"
//			}
//		},{
//			"zip": "https://github.com/twitter/bootstrap/zipball/master",
//			"target": {
//				"js/": "bootstap/js",
//				"less/": "bootstrap/less"
//			}
//		}], function callback( errors ){
//			//if any errors it will be an array
//		});

var request = require('request'),
	AdmZip = require('adm-zip'),
	mkdirp = require('mkdirp'),
	async = require('async'),
    _ = require('underscore'),
	fs = require('fs-extra'),
    path = require('path'),
	os = require('os'),
	spawn = require('child_process').spawn,
    createOperationCollection = require('./fetch/createOperationCollection'),
    writeOperations = require('./fetch/writeOperation'),
    fromFile = require('./fetch/fromFile');




//###fetch() module
function fetch( resource, options, callback ){
	if( arguments.length === 2 && typeof arguments[1] === 'function' ){
		callback = options;
		options = {};
	}

	if( Array.isArray(resource) ){
		//create an obj of fetch requests for individual resources,
		//so we can load them in parallel
		var fetches = resource.map(function( res ){
			return function( next ){
				fetch( res, next);
			};
		});
		async.parallel( fetches, callback );
	} else {
		//if its a single resource, fetch it
		if( resource.file ) fromFile( resource, options, exit);
		else if( resource.clone ) fromGit( resource, options, exit );
		else if( resource.zip ) fromZip( resource, options, exit );
		else {
			exit(Error("Invalid resource, did not find method of retrieval in object"));
		}
	}
	function exit( err ){
		callback( err, resource );
	}
}

function getTmp(){ return os.tmpDir() + 'sketchplate/'; }
function getFilename( src ){ return src.substr(src.lastIndexOf('/')+1, src.length); }

//process the targets supplied
function processTargets( contentsFolder, resource, callback ){
    createOperationCollection( contentsFolder, resource, function( err, operations ){
        //operations: [ { source:String, destination:String, isDirectory:Boolean } ]
        writeOperations( operations, callback );
    });
}


//The resource is a git repo
var reposCloned = 0;
//retrieve a git repository and process its contents for targets
//
//-	{Object} [resource]
//-	{String} [resource.clone] the repository address to clone
//- {String || Object} [resource.target] if string entire contents will be copied
// otherwise should be an object of "source":"target" relationships
//-	{Function} [callback] receive callback when completed
function fromGit( resource, options, callback ){
	reposCloned++;
	//if options specified a tmpDir, use that instead
	var tmpStorage = (options.tmpDir ? options.tmpDir : getTmp() ) + 'repo'+reposCloned;
	var repoSourceFolder = tmpStorage +'/';

	var clean = function( next ) {
        fs.remove(tmpStorage, next);
	};

	async.waterfall([
		function prepare( next ){
			if( fs.existsSync( tmpStorage ) ){
				clean( next );
			} else {
				next( null );
			}
		},
		function mk( next ){
			mkdirp( tmpStorage, function(err){
				next( err );
			});
		},
		function cloneRepo( next ){
			var clone = spawn('git', [ 'clone', resource.clone, tmpStorage ]);
			clone.stdout.on('data', function ( /*data*/ ) {});
			clone.on('exit', function( code ){
				if( code === 0 ){
					next();
				} else {
					next( new Error("Error fetching git repo: "+resource.clone+", exited with code: "+code) );<commit_msg>fixes linux issue with mkdirp creating already existing directory<commit_after>//#Fetch
//Fetch can be used to automate the downloading of resources into your project
//it accepts the following types of operations:
//
//1.	**file** - download a file and copy it to the target
//1.	**clone** - clone a git repository and copy all or parts of its
//contents to specified targets
//1.	**zip** - download a zip file, extract it and copy all or parts of its
//contents to specified targets
//
//**_Basic use:_**
//
//_single fetch:_
//
//		fetch({
//			file: "http://code.jquery.com/jquery.js",
//			target: "javascripts/vendor/jquery.js"
//		}, function callback( errors ){
//			//if any errors it will be an array
//		});
//_batch fetch:_
//
//		fetch([
//			file: "http://code.jquery.com/jquery.js",
//			target: "jquery.js"
//		},{
//			"clone": "https://code.google.com/p/dat-gui/",
//			"target": {
//				"src/dat": "dat/",
//				"build": "datgui/build"
//			}
//		},{
//			"zip": "https://github.com/twitter/bootstrap/zipball/master",
//			"target": {
//				"js/": "bootstap/js",
//				"less/": "bootstrap/less"
//			}
//		}], function callback( errors ){
//			//if any errors it will be an array
//		});

var request = require('request'),
	AdmZip = require('adm-zip'),
	async = require('async'),
    _ = require('underscore'),
	fs = require('fs-extra'),
    path = require('path'),
	os = require('os'),
	spawn = require('child_process').spawn,
    createOperationCollection = require('./fetch/createOperationCollection'),
    writeOperations = require('./fetch/writeOperation'),
    fromFile = require('./fetch/fromFile');




//###fetch() module
function fetch( resource, options, callback ){
	if( arguments.length === 2 && typeof arguments[1] === 'function' ){
		callback = options;
		options = {};
	}

	if( Array.isArray(resource) ){
		//create an obj of fetch requests for individual resources,
		//so we can load them in parallel
		var fetches = resource.map(function( res ){
			return function( next ){
				fetch( res, next);
			};
		});
		async.parallel( fetches, callback );
	} else {
		//if its a single resource, fetch it
		if( resource.file ) fromFile( resource, options, exit);
		else if( resource.clone ) fromGit( resource, options, exit );
		else if( resource.zip ) fromZip( resource, options, exit );
		else {
			exit(Error("Invalid resource, did not find method of retrieval in object"));
		}
	}
	function exit( err ){
		callback( err, resource );
	}
}

function getTmp(){ return path.join(os.tmpDir(),'sketchplate') + '/'; }
function getFilename( src ){ return src.substr(src.lastIndexOf('/')+1, src.length); }

//process the targets supplied
function processTargets( contentsFolder, resource, callback ){
    createOperationCollection( contentsFolder, resource, function( err, operations ){
        //operations: [ { source:String, destination:String, isDirectory:Boolean } ]
        writeOperations( operations, callback );
    });
}


//The resource is a git repo
var reposCloned = 0;
//retrieve a git repository and process its contents for targets
//
//-	{Object} [resource]
//-	{String} [resource.clone] the repository address to clone
//- {String || Object} [resource.target] if string entire contents will be copied
// otherwise should be an object of "source":"target" relationships
//-	{Function} [callback] receive callback when completed
function fromGit( resource, options, callback ){
	reposCloned++;
	//if options specified a tmpDir, use that instead
	var tmpStorage = (options.tmpDir ? options.tmpDir : getTmp() ) + 'repo'+reposCloned;
	var repoSourceFolder = tmpStorage +'/';

	var clean = function( next ) {
        fs.remove(tmpStorage, next);
	};

	async.waterfall([
		function prepare( next ){
			if( fs.existsSync( tmpStorage ) ){
				clean( next );
			} else {
				next( null );
			}
		},
		function mk( next ){
            fs.ensureDir(tmpStorage, function(err){
                next(err);
            });
		},
		function cloneRepo( next ){
			var clone = spawn('git', [ 'clone', resource.clone, tmpStorage ]);
			clone.stdout.on('data', function ( /*data*/ ) {});
			clone.on('exit', function( code ){
				if( code === 0 ){
					next();
				} else {
					next( new Error("Error fetching git repo: "+resource.clone+", exited with code: "+code) );
{'fraction_non_alphanumeric': 0.16533490011750882, 'fraction_numerical': 0.0018801410105757932, 'mean_word_length': 3.2640280561122244, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '447426', 'n_tokens_mistral': 2811, 'n_tokens_neox': 2596, 'n_words': 805}