<commit_before>'use strict';

var host = 'github.com', owner = 'aurelia', protocol = 'https';

module.exports = function download(options) {		
var async = require("async"),
	unzip = require("unzip"),
	fs = require("fs"), 
	path = require('path'),
	Promise = require('promise'),
	endOfLine = require('os').EOL,
	ps = require("event-stream").pause(),
	request = require("request");
	
	function ensureDirectories() {
		console.info('create directories structure ...');
		// https://github.com/smhanov/node-promise-sequence
		var dirs = [options.zips, options.zipsMaster, options.unZip, options.unZipMaster];
	
		/**
		 * NOTE: 
		 * 
		 * If there are no error, callback will only be called once.
		 * 
		 * If there are multiple errors, callback will be called 
		 * exactly as many time as errors occur. 
		 * 
		 * Sometimes, this behavior maybe useful, but users 
		 * should be aware of this and handle errors in callback. 
		 * 
		 */
		function rmfile(dir, file, callback){
		  var p = path.join(dir, file)
		  fs.lstat(p, function(err, stat){
			if(err) callback.call(null,err)
			else if(stat.isDirectory()) rmdir(p, callback)
			else fs.unlink(p, callback)
		  })
		}

		function rmdir(dir, callback) {
		  fs.readdir(dir, function(err, files){
			if(err) callback.call(null,err)
			else if( files.length ){
			  var i,j;
			  for(i=j=files.length; i--; ){
				rmfile(dir, files[i], function(err) {
				  if (err) callback.call(null, err)
				  else if(--j === 0) fs.rmdir(dir,callback)
				})
			  }
			}
			else fs.rmdir(dir, callback)
		  })
		}
	
		function __dir(dir) {
			return new Promise(function (resolve, reject) {
				rmdir(dir, function(e, result) {
					if (e && e.code != 'ENOENT') {
						reject(e);
					} 
					else {
						fs.mkdir(dir, function(e, result){
							if (e) {
								reject(e);
							}
							else 
								resolve();
						});
					}
				});
			});
		}
		
		for (var i in dirs) {
			dirs[i] = __dir(dirs[i]);
		}

		return Promise.all(dirs).then(function() {
			console.info('ok');
		});
	}
	
	function extract_version(repo, body) {
		var matches, 
			versions = [], 
			regEx = new RegExp('href="/' + owner + '/' + repo + '/archive/(\\d{1,2}\\.\\d{1,2}\\.\\d{1,2}).zip"', 'ig'),
			ver, mod, i0, i1, versionInfo;
			
		while (matches = regEx.exec(body))
			versions.push(matches[1]);
		
		if (versions.length === 1 || versions.length === 0) {
			versionInfo = {name: repo, version: (versions[0] || '')};
		}
		else {
			versions = versions.map(function(item){
				item = item.split('.');
				return parseInt(item[0],10)*1000000 +
					parseInt(item[1],10)*1000 +
					parseInt(item[2],10);
			}).sort();
			
			ver = versions[versions.length -1]
			mod = ver % 1000000;
			i0 = (ver - mod) / 1000000;

			ver = mod;
			mod = ver % 1000;

			i1 = (ver - mod) / 1000;

			versionInfo = {name: repo, version: (i0 + "." + i1 + "." + mod)};
		}
		return versionInfo;
	}
	
	function get_repo_release(repo, callback) {
		console.info('request version for ' + repo + ' ...');
		
		var url = protocol + '://' + host + '/' +owner+ '/' +repo+ '/tags';
		
		request.get(url, function(error, response, body) {
		  if (!error && response.statusCode == 200) {
			var versionInfo = extract_version(repo, body);
			console.info(versionInfo.version || 'none');
			callback(null, versionInfo);
		  }
		  else {
			  callback(error || ('Request for [' +url+ '] ended with status code: '+response.statusCode+'.'));
		  }
		})
		.on('error', function(e) {
			callback(e, null);
		});
	}
	
	function download_repo_release(repo, callback) {
		var url = protocol + '://' + host + '/' +owner+ '/' +repo.name+ '/archive/' + repo.version + ".zip",
			dest = (repo.version === 'master' ? options.zipsMaster : options.zips) + '/' + repo.name + '.zip';
		
		console.info('downloading ... ' + repo.name +' ('+ repo.version + ')');
		
		var file = fs.createWriteStream(dest);
		
		request(url).pipe(file);
			
		file.on('finish', function () {
			console.info('ok');
			file.close(function() {
				repo.dest = dest;
				callback(null, repo);
			}); // close() is async, call callback after close completes.
		});
		
		file.on('error', function (e) {
			fs.unlink(dest); // Delete the file async. (But we don't check the result)
			callback(e, null);
		});<commit_msg>correct error handling<commit_after>'use strict';

var host = 'github.com', owner = 'aurelia', protocol = 'https';

module.exports = function download(options) {		
var async = require("async"),
	unzip = require("unzip"),
	fs = require("fs"), 
	path = require('path'),
	Promise = require('promise'),
	endOfLine = require('os').EOL,
	ps = require("event-stream").pause(),
	request = require("request");
	
	function ensureDirectories() {
		console.info('create directories structure ...');
		// https://github.com/smhanov/node-promise-sequence
		var dirs = [options.zips, options.zipsMaster, options.unZip, options.unZipMaster];
	
		/**
		 * NOTE: 
		 * 
		 * If there are no error, callback will only be called once.
		 * 
		 * If there are multiple errors, callback will be called 
		 * exactly as many time as errors occur. 
		 * 
		 * Sometimes, this behavior maybe useful, but users 
		 * should be aware of this and handle errors in callback. 
		 * 
		 */
		function rmfile(dir, file, callback){
		  var p = path.join(dir, file)
		  fs.lstat(p, function(err, stat){
			if(err) callback.call(null,err)
			else if(stat.isDirectory()) rmdir(p, callback)
			else fs.unlink(p, callback)
		  })
		}

		function rmdir(dir, callback) {
		  fs.readdir(dir, function(err, files){
			if(err) callback.call(null,err)
			else if( files.length ){
			  var i,j;
			  for(i=j=files.length; i--; ){
				rmfile(dir, files[i], function(err) {
				  if (err) callback.call(null, err)
				  else if(--j === 0) fs.rmdir(dir,callback)
				})
			  }
			}
			else fs.rmdir(dir, callback)
		  })
		}
	
		function __dir(dir) {
			return new Promise(function (resolve, reject) {
				rmdir(dir, function(e, result) {
					if (e && e.code != 'ENOENT') {
						reject(e);
					} 
					else {
						fs.mkdir(dir, function(e, result){
							if (e) {
								reject(e);
							}
							else 
								resolve();
						});
					}
				});
			});
		}
		
		for (var i in dirs) {
			dirs[i] = __dir(dirs[i]);
		}

		return Promise.all(dirs).then(function() {
			console.info('ok');
		});
	}
	
	function extract_version(repo, body) {
		var matches, 
			versions = [], 
			regEx = new RegExp('href="/' + owner + '/' + repo + '/archive/(\\d{1,2}\\.\\d{1,2}\\.\\d{1,2}).zip"', 'ig'),
			ver, mod, i0, i1, versionInfo;
			
		while (matches = regEx.exec(body))
			versions.push(matches[1]);
		
		if (versions.length === 1 || versions.length === 0) {
			versionInfo = {name: repo, version: (versions[0] || '')};
		}
		else {
			versions = versions.map(function(item){
				item = item.split('.');
				return parseInt(item[0],10)*1000000 +
					parseInt(item[1],10)*1000 +
					parseInt(item[2],10);
			}).sort();
			
			ver = versions[versions.length -1]
			mod = ver % 1000000;
			i0 = (ver - mod) / 1000000;

			ver = mod;
			mod = ver % 1000;

			i1 = (ver - mod) / 1000;

			versionInfo = {name: repo, version: (i0 + "." + i1 + "." + mod)};
		}
		return versionInfo;
	}
	
	function get_repo_release(repo, callback) {
		console.info('request version for ' + repo + ' ...');
		
		var url = protocol + '://' + host + '/' +owner+ '/' +repo+ '/tags';
		
		request.get(url, function(error, response, body) {
		  if (!error && response.statusCode == 200) {
			var versionInfo = extract_version(repo, body);
			console.info(versionInfo.version || 'none');
			callback(null, versionInfo);
		  }
		  else {
			callback(error || ('Request for [' +url+ '] ended with status code: '+response.statusCode+'.'));
		  }
		});
	}
	
	function download_repo_release(repo, callback) {
		var url = protocol + '://' + host + '/' +owner+ '/' +repo.name+ '/archive/' + repo.version + ".zip",
			dest = (repo.version === 'master' ? options.zipsMaster : options.zips) + '/' + repo.name + '.zip';
		
		console.info('downloading ... ' + repo.name +' ('+ repo.version + ')');
		
		var file = fs.createWriteStream(dest);
		
		request(url).pipe(file);
			
		file.on('finish', function () {
			console.info('ok');
			file.close(function() {
				repo.dest = dest;
				callback(null, repo);
			}); // close() is async, call callback after close completes.
		});
		
		file.on('error', function (e) {
			fs.unlink(dest); // Delete the file async. (But we don't check the result)
			callback(e, null);
		});
{'fraction_non_alphanumeric': 0.18408369577994593, 'fraction_numerical': 0.014811331844363465, 'mean_word_length': 2.81353653070372, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6677183', 'n_tokens_mistral': 3587, 'n_tokens_neox': 2887, 'n_words': 680}