<commit_before>var exec = require('child_process').exec;
var fs = require('fs');
var path = require('path');
var mkdirp = require('mkdirp');
var yaml = require('js-yaml');
var walk = require('walkdir');
var _ = require('underscore');

module.exports.git_cmd = function(git_dir, command, options, args, callback) {
	var bash, _ref, _ref1;
	if (!callback) {
		_ref = [args, callback], callback = _ref[0], args = _ref[1];
	}
	if (!callback) {
		_ref1 = [options, callback], callback = _ref1[0], options = _ref1[1];
	}
	if (options == null) {
		options = {};
	}
	options = this.options_to_argv(options);
	options = options.join(" ");
	if (args == null) {
		args = [];
	}
	if (args instanceof Array) {
		args = args.join(" ");
	}
	bash = "" + "git" + " " + command + " " + options + " " + args;
	console.log(      bash = "" + "git" + " " + command + " " + options + " " + args);
	exec(bash, {
		cwd: git_dir
	}, callback);
	return bash;
};

module.exports.options_to_argv = function(options) {
	var key, val;
	var argv = [];
	for (key in options) {
		val = options[key];
		if (key.length === 1) {
			if (val === true) {
				argv.push("-" + key);
			} else if (val === false) {

			} else {
				argv.push("-" + key);
				argv.push(val);
			}
		} else {
			if (val === true) {
				argv.push("--" + key);
			} else if (val === false) {

			} else {
				argv.push("--" + key + "=" + val);
			}
		}
	}
	return argv;
};

module.exports.createExtendsDirectory = function(startdir, extendDir, callback){
    console.log("  extending appstack " + extendDir)
    module.exports.git_cmd(startdir, "submodule add", {}, ['--force', extendDir, "extends"], function(err, data){
	if(err) {
	    console.log(err);
	}
    });
    //updateRepoSubmodules(startdir)
};

module.exports.createScriptDirectories = function(startdir, callback){
    var operational_scripts_dir = path.join(startdir, "operational-scripts");
    fs.exists(operational_scripts_dir, function(exists) {
	if ( ! exists ) {
            mkdirp.sync(operational_scripts_dir);
	}
        createReadmeMd(operational_scripts_dir, "## This file was auto-generated by CloudCoreo CLI\n" +
		       "Scripts contained in this directory will be exposed to the CloudCoreo UI and can be run on an\n" +
		       "adhoc basis.\n" +
		       "", function(err, data){
			   if (err) { console.log("error creating readme in operational scripts directory"); }
		       });
    });
    var shutdown_scripts_dir = path.join(startdir, "shutdown-scripts");
    fs.exists(shutdown_scripts_dir, function(exists){
	if ( ! exists ) {
	    mkdirp.sync(shutdown_scripts_dir);
	}
    });
    createReadmeMd(shutdown_scripts_dir, "## This file was auto-generated by CloudCoreo CLI\n" +
		   "Scripts contained in this directory will be proccessed (if possible) when an instance\n" +
		   "is asked to shut down. The order in which these scripts will be run is defined by the\n" +
		   "order.yaml\n" +
		   "", function(err, data) { if (err) { console.log('error creating readme in shutdown scripts dir' + err); }});
    shutdownOrderFile = path.join(shutdown_scripts_dir, "order.yaml");
    writeIfNeeded(shutdownOrderFile, "## This file was auto-generated by CloudCoreo CLI\n" +
		  "## this yaml file dictates the order in which the scripts will run. i.e:\n" +
		  "## script-order:\n" +
		  "##   - deregister_dns.sh\n" +
		  "##   - delete_backups.sh\n" +
		  "\n" +
		  "", function(err, data) { if (err) { console.log('error creating order.yaml file in shutdown scripts dir' + err); } });
    boot_scripts_dir = path.join(startdir, "boot-scripts");
    fs.exists(boot_scripts_dir, function(exists){
	if (! exists) {
	    mkdirp.sync(boot_scripts_dir);
	}
    });
    createReadmeMd(boot_scripts_dir, "## This file was auto-generated by CloudCoreo CLI\n" +
		   "Scripts contained in this directory will be proccessed when an instance is booting. \n" +
		   "The order in which these scripts will be run is defined by the order.yaml\n" +
		   "", function(err, data) { if (err) { console.log('error creating readme in the bootscripts dir: ' + err); } });
    var bootOrderYaml = path.join(boot_scripts_dir,"order.yaml");
    writeIfNeeded(bootOrderYaml,"## This file was auto-generated by CloudCoreo CLI\n" +
		  "## this yaml file dictates the order in which the scripts will run. i.e:\n" +
		  "## script-order:\n" +
		  "##   - install_chef.sh\n" +
		  "##   - run_chef.sh\n" +
		  "\n" +
		  "", function(err, data) { if (err) { console.log('error creating order.yaml in the bootscripts dir: ' + err); } });
};

getServicefiles = function(rootDir, callback){
    var servicefiles = []
    console.log("walking rootDir: %s" % rootDir)
    
    var emitter = walk(rootDir);
    emitter.on('file',function(filename, stat){
	// only do this stuff if it is not the git directory
	if ( ! /.git$/.test(filename) && /services[\/|\\]config.rb/.test(filename)) {
	    console.log('  filename passed: ', filename);
	    servicefiles.push(filename)
	}
    });
    emitter.on('end',function(filename, stat){
	servicefiles.sort(function(a, b){
	    return b.length - a.length;
	});
	console.log('got servicefiles: ' + servicefiles);
	callback(null, servicefiles);
    });
}

getConfigVariableFiles = function(rootDir, callback) {
    var order_files = []
    console.log("walking rootDir: " + rootDir)

    var emitter = walk(rootDir);
    emitter.on('file',function(filename, stat){
	// only do this stuff if it is not the git directory
	if ( ! /.git$/.test(filename) && /\/config.y(a)?ml/.test(filename)) {
	    console.log('filename passed: ', filename);
	    order_files.push(filename)
	}
    });
    emitter.on('end',function(filename, stat){
	order_files.sort(function(a, b){
	    return b.length - a.length;
	});
	callback(null, order_files);
    });
};

getVariablesFromConfigFiles = function(variableFiles, callback){
    var combinedDictionary = {}
    variableFiles.forEach(function(f){
        try {
            console.log("loading file " + path.normalize(f));
	    
	    var my_doc;
	    fs.readFile(path.normalize(f), function(err, data){
		yaml.safeLoadAll(data, function(my_doc){
		    console.log("my_doc: " + JSON.stringify(my_doc['variables']));
		    if( (! my_doc) || (! my_doc.variables)) {
			return;
		    }
		    for ( var m_var in my_doc['variables'] ) {
			console.log("found variable: " + m_var)
			combinedDictionary[m_var] = my_doc['variables'][m_var]
		    };
		});
	    });
	    
        } catch (ex) {
            console.log("Error parsing file " + f + " - perhaps invalid yaml?");
	    process.exit(1);
	}
	console.log("variablefile - returning vars: " + JSON.stringify(combinedDictionary));
	callback(null, combinedDictionary);
    });
};

getVariablesFromServicefiles = function(servicefiles, callback){
    var variables = {}
    for (var f in servicefiles) {

	fs.readFile(f, function(err, contents){
	    var myRegexp = /\$\{(.*?)\}/;
	    var match = myRegexp.exec(contents);
            while( match != null ) {
		if ( /STACK::/.test(match) ) { 
		    continue; 
		}
		variables[match] = ""
	    };	    
	});
	console.log("servicefile - returning vars: " + JSON.stringify(variables));
	callback(null, variables);
    }
};

parseConfigVariablesFiles = function(basedir, callback){
    getConfigVariableFiles(basedir, function(err, variable_files){
	if (err) {
	    console.log(err);
	    process.exit(1);
	} else {
	    console.log('  get service files now');
	    getServicefiles(basedir, function(err, service_files){
		if (err){
		    console.log(err);
		    process.exit(1);
		} else {
		    getVariablesFromConfigFiles(variable_files, function(err, configVariables){
			if (err){
			    console.log(err);
			    process.exit(1);
			} else {
			    getVariablesFromServicefiles(service_files, function(err, scriptVariables){
				console.log('merging:');
				console.log('   scriptVariables: ' + JSON.stringify(scriptVariables));
				console.log('   configVariables: ' + JSON.stringify(configVariables));
				
				var target = _.extend(configVariables, scriptVariables);
				console.log('returning combined config files: ' + JSON.stringify(target));
				callback(null, target);
			    });
			}
		    });
		}
	    });
	}
    });
}

generateVariablesFile = function(startdir, showMissing, callback){
    var variables = parseConfigVariablesFiles(startdir, function(err, data){});
    var finalDoc = {"variables": {}}
    for (var item in variables) {
	for (var key in item ) {
	    if (item.hasOwnProperty(key)) {
		alert(key + " -> " + p[key]);
	    }
	}
    }
    // for key, value in sorted(variables.items()):
    //     debug("  key: %s | value: %s" % (key, value))
    //     if showMissing == True:
    //         if value:
    //             continue
    //     if not value:
    //         if finalDoc['variables'] == None:
    //             finalDoc['variables'] = {}
    //         value = {}
    //         value['default'] = ''
    //         value['description'] = ''
    //         value['overrides'] = ''
    //         value['type'] = ''
    //         value['required'] = True
    //     debug("finalDoc['variables']: %s" % finalDoc['variables'])
    //     debug("  key: %s | value: %s" % (key, value))
    //     finalDoc['variables'][key] = value
    // debug(finalDoc)
    // return finalDoc
    callback(null, 'hi');
}

initBaseDirectory = function(startdir, callback){
    console.log('initing base dir');
    var yml = yaml.dump(generateVariablesFile(startdir, true, function(err, data){
	if (err) {
	    console.log(err);
	    process.exit(1);
	} else {
	    console.log('yaml dumped');
	}
    }))
//     with open(os.path.join(startdir, "config.yaml"), 'w') as var_file:
//         var_file.write(
//                   """## This file was auto-generated by CloudCoreo CLI
// ## this yaml file contains variables, their defaults, overrides, type and namespaces. i.e:
// ## variables:
// ##   PUBLIC_ROUTE_NAME:
// ##     default: my-public-route
// ##     description: 'the name to give to the public route'
// ##     required: true
// ##     type: string

// %s
//                   """ % yml)
}

createReadmeMd = function(directory, content, callback) {
    var readmeFile = path.join(directory, "README.md");
    writeIfNeeded(readmeFile, content, callback);
};

writeIfNeeded = function(file_path, content, callback){
    console.log('file path: ' + file_path);
    fs.exists(file_path, function(exists) {
	if ( ! exists ) {
	    content.toString().split('\n').forEach(function (line) { 
		fs.appendFileSync(file_path, line.toString() + "\n");
	    });
	} else {
	    callback(null, true);
	}
    });
};

module.exports.createOverridesDirectory = function(startdir, callback) {
    override_dir = path.join(startdir, "overrides");
    mkdirp(override_dir, function(err) { 
	if (err) {
	    callback(err, false);
	} else {
	    createReadmeMd(override_dir, "## This file was auto-generated by CloudCoreo CLI\n" +
			   "Anything contained in this directory will act as an override for the stack in which it is contained.\n" +
			   "\n" +
			   "The paths should be considered relative to the parent of this directory.\n" +
			   "\n" +
			   "For example, if you have a directory structure like this, \n" +
			   "```\n" +
			   "+-- parent\n" +
			   "|   +-- overrides\n" +
			   "|   |   +-- stack-a\n" +
			   "|   |   |   +-- boot-scripts\n" +
			   "|   |   |   |   +-- order.yaml\n" +
			   "|   +-- stack-a\n" +
			   "|   |   +-- boot-scripts\n" +
			   "|   |   |   +-- order.yaml\n" +
			   "```\n" +
			   "Because the directory structure within the override directory matches the structure of the parent, \n" +
			   "the 'order.yaml' file will be ignored in the stack-a directory and instead the overrides/stack-a order.yaml\n" +
			   "file will be used.\n" +
			   "", function(err, created) {
			      if(err) {
				   callback(err, false);
			       } else {
				   callback(null, true);
			       }
			   });
	}
    });
};

module.exports.createConfigYamlFile = function(dir, variables, callback) {
    var configFilename = path.join(dir, "config.yaml");
    mkdirp(dir, function(err) { 
	writeIfNeeded(configFilename, "## This file was auto-generated by CloudCoreo CLI\n" +
		      "## this yaml file contains variables, their defaults, overrides, type and namespaces. i.e:\n" +
		      "## variables:\n" +
		      "##   PUBLIC_ROUTE_NAME:\n" +
		      "##     default: my-public-route\n" +
		      "##     description: 'the name to give to the public route'\n" +
		      "##     required: true\n" +
		      "##     type: string\n" +
		      "", function(err, data) { if (err) { console.log('error');callback(err, false); } });
    });
};

module.exports.createServicesDirectory = function(startdir, callback) {
    var services_dir = path.join(startdir, "services");
    mkdirp(services_dir, function(err) { 
	var configFilename = path.join(services_dir, "config.rb");
        writeIfNeeded(configFilename,"## This file was auto-generated by CloudCoreo CLI\n" +
		      "## This file was automatically generated using the CloudCoreo CLI\n" +
		      "##\n" +
		      "## This config.rb file exists to create and maintain services not related to compute.\n" +
		      "## for example, a VPC might be maintained using:\n" +
		      "##\n" +
		      "## coreo_aws_vpc_vpc \"my-vpc\" do\n" +
		      "##   action :sustain\n" +
		      "##   cidr \"172.16.17.32/16\"\n" +
		      "##   internet_gateway true\n" +
		      "## end\n" +
		      "##\n" +
		      "\n" +
		      "", function(err, data) { if (err) { console.log('error');callback(err, false); } });
	createReadmeMd(services_dir, "## This file was auto-generated by CloudCoreo CLI\n" +
		       "This is your services directory. Place a config.rb file in here containing CloudCoreo service\n" +
		       "syntax. For example, your config.rb might contain the following in order to create a VPC\n" +
		       "```\n" +
		       "coreo_aws_vpc_vpc \"my-vpc\" do\n" +
		       "  action :sustain\n" +
		       "  cidr \"172.16.17.32/16\"\n" +
		       "  internet_gateway true\n" +
		       "end\n" +
		       "\n" +
		       "coreo_aws_vpc_routetable \"my-public-route\" do\n" +
		       "  action :create\n" +
		       "  vpc \"my-vpc\"\n" +
		       "  number_of_tables 3\n" +
		       "  routes [\n" +
		       "         { :from => \"0.0.0.0/0\", :to => \"my-vpc\", :type => :igw },\n" +
		       "        ]\n" +
		       "end\n" +
		       "\n" +
		       "coreo_aws_vpc_subnet \"my-public-subnet\" do\n" +
		       "  action :create\n" +
		       "  number_of_zones 3\n" +
		       "  percent_of_vpc_allocated 50\n" +
		       "  route_table \"my-public-route\"\n" +
		       "  vpc \"my-vpc\"\n" +
		       "  map_public_ip_on_launch true\n" +
		       "end\n" +
		       "```\n" +
		       "", function(err, created) {
			   if(err) {
			       callback(err, false);
			   } else {
			       callback(null, true);
			   }
		       });
    });
};

    
<commit_msg>adding config and homedir stuff<commit_after>var exec = require('child_process').exec;
var fs = require('fs');
var path = require('path');
var mkdirp = require('mkdirp');
var yaml = require('js-yaml');
var walk = require('walkdir');
var _ = require('underscore');
var jsonfile = require('jsonfile');

module.exports.getConfigFile = function() {
    mkdirp.sync(path.join(this.getUserHome(), '.cloudcoreo'));
    var file = path.join(this.getUserHome(), '.cloudcoreo', 'config');
    return file;
};

module.exports.addConfig = function() {
    var file = this.getConfigFile();
    var configArray = this.getConfigArray();
    configArray.push(bo);
    jsonfile.writeFileSync(file, configArray);
};

module.exports.getConfigArray = function() {
    var file = this.getConfigFile();
    var config = []
    if (fs.existsSync(file)) {
	config = jsonfile.readFileSync(file);
    }
    var configArray = []
    // handle user modified screwups array vs. not array etc.
    if ( Array.isArray(config) ) {
	for ( var i in config ) {
	    var c = config[i];
	    if (c["username"] == bo["username"]) {
		// just skip this one because we are going to add it anyway
		continue;
	    }
	    configArray.push(c);
	}
    } else if (config) { // or its there but NOT an array
	// dont add it if it is the same username
	if ( ! config["username"] == bo["username"] ) {
	    configArray.push(config);
	}
    }
    return configArray;
};

module.exports.getUserHome = function() {
  return process.env[(process.platform == 'win32') ? 'USERPROFILE' : 'HOME'];
};

module.exports.git_cmd = function(git_dir, command, options, args, callback) {
	var bash, _ref, _ref1;
	if (!callback) {
		_ref = [args, callback], callback = _ref[0], args = _ref[1];
	}
	if (!callback) {
		_ref1 = [options, callback], callback = _ref1[0], options = _ref1[1];
	}
	if (options == null) {
		options = {};
	}
	options = this.options_to_argv(options);
	options = options.join(" ");
	if (args == null) {
		args = [];
	}
	if (args instanceof Array) {
		args = args.join(" ");
	}
	bash = "" + "git" + " " + command + " " + options + " " + args;
	console.log(      bash = "" + "git" + " " + command + " " + options + " " + args);
	exec(bash, {
		cwd: git_dir
	}, callback);
	return bash;
};

module.exports.options_to_argv = function(options) {
	var key, val;
	var argv = [];
	for (key in options) {
		val = options[key];
		if (key.length === 1) {
			if (val === true) {
				argv.push("-" + key);
			} else if (val === false) {

			} else {
				argv.push("-" + key);
				argv.push(val);
			}
		} else {
			if (val === true) {
				argv.push("--" + key);
			} else if (val === false) {

			} else {
				argv.push("--" + key + "=" + val);
			}
		}
	}
	return argv;
};

module.exports.createExtendsDirectory = function(startdir, extendDir, callback){
    console.log("  extending appstack " + extendDir)
    module.exports.git_cmd(startdir, "submodule add", {}, ['--force', extendDir, "extends"], function(err, data){
	if(err) {
	    console.log(err);
	}
    });
    //updateRepoSubmodules(startdir)
};

module.exports.createScriptDirectories = function(startdir, callback){
    var operational_scripts_dir = path.join(startdir, "operational-scripts");
    fs.exists(operational_scripts_dir, function(exists) {
	if ( ! exists ) {
            mkdirp.sync(operational_scripts_dir);
	}
        createReadmeMd(operational_scripts_dir, "## This file was auto-generated by CloudCoreo CLI\n" +
		       "Scripts contained in this directory will be exposed to the CloudCoreo UI and can be run on an\n" +
		       "adhoc basis.\n" +
		       "", function(err, data){
			   if (err) { console.log("error creating readme in operational scripts directory"); }
		       });
    });
    var shutdown_scripts_dir = path.join(startdir, "shutdown-scripts");
    fs.exists(shutdown_scripts_dir, function(exists){
	if ( ! exists ) {
	    mkdirp.sync(shutdown_scripts_dir);
	}
    });
    createReadmeMd(shutdown_scripts_dir, "## This file was auto-generated by CloudCoreo CLI\n" +
		   "Scripts contained in this directory will be proccessed (if possible) when an instance\n" +
		   "is asked to shut down. The order in which these scripts will be run is defined by the\n" +
		   "order.yaml\n" +
		   "", function(err, data) { if (err) { console.log('error creating readme in shutdown scripts dir' + err); }});
    shutdownOrderFile = path.join(shutdown_scripts_dir, "order.yaml");
    writeIfNeeded(shutdownOrderFile, "## This file was auto-generated by CloudCoreo CLI\n" +
		  "## this yaml file dictates the order in which the scripts will run. i.e:\n" +
		  "## script-order:\n" +
		  "##   - deregister_dns.sh\n" +
		  "##   - delete_backups.sh\n" +
		  "\n" +
		  "", function(err, data) { if (err) { console.log('error creating order.yaml file in shutdown scripts dir' + err); } });
    boot_scripts_dir = path.join(startdir, "boot-scripts");
    fs.exists(boot_scripts_dir, function(exists){
	if (! exists) {
	    mkdirp.sync(boot_scripts_dir);
	}
    });
    createReadmeMd(boot_scripts_dir, "## This file was auto-generated by CloudCoreo CLI\n" +
		   "Scripts contained in this directory will be proccessed when an instance is booting. \n" +
		   "The order in which these scripts will be run is defined by the order.yaml\n" +
		   "", function(err, data) { if (err) { console.log('error creating readme in the bootscripts dir: ' + err); } });
    var bootOrderYaml = path.join(boot_scripts_dir,"order.yaml");
    writeIfNeeded(bootOrderYaml,"## This file was auto-generated by CloudCoreo CLI\n" +
		  "## this yaml file dictates the order in which the scripts will run. i.e:\n" +
		  "## script-order:\n" +
		  "##   - install_chef.sh\n" +
		  "##   - run_chef.sh\n" +
		  "\n" +
		  "", function(err, data) { if (err) { console.log('error creating order.yaml in the bootscripts dir: ' + err); } });
};

getServicefiles = function(rootDir, callback){
    var servicefiles = []
    console.log("walking rootDir: %s" % rootDir)
    
    var emitter = walk(rootDir);
    emitter.on('file',function(filename, stat){
	// only do this stuff if it is not the git directory
	if ( ! /.git$/.test(filename) && /services[\/|\\]config.rb/.test(filename)) {
	    console.log('  filename passed: ', filename);
	    servicefiles.push(filename)
	}
    });
    emitter.on('end',function(filename, stat){
	servicefiles.sort(function(a, b){
	    return b.length - a.length;
	});
	console.log('got servicefiles: ' + servicefiles);
	callback(null, servicefiles);
    });
}

getConfigVariableFiles = function(rootDir, callback) {
    var order_files = []
    console.log("walking rootDir: " + rootDir)

    var emitter = walk(rootDir);
    emitter.on('file',function(filename, stat){
	// only do this stuff if it is not the git directory
	if ( ! /.git$/.test(filename) && /\/config.y(a)?ml/.test(filename)) {
	    console.log('filename passed: ', filename);
	    order_files.push(filename)
	}
    });
    emitter.on('end',function(filename, stat){
	order_files.sort(function(a, b){
	    return b.length - a.length;
	});
	callback(null, order_files);
    });
};

getVariablesFromConfigFiles = function(variableFiles, callback){
    var combinedDictionary = {}
    variableFiles.forEach(function(f){
        try {
            console.log("loading file " + path.normalize(f));
	    
	    var my_doc;
	    fs.readFile(path.normalize(f), function(err, data){
		yaml.safeLoadAll(data, function(my_doc){
		    console.log("my_doc: " + JSON.stringify(my_doc['variables']));
		    if( (! my_doc) || (! my_doc.variables)) {
			return;
		    }
		    for ( var m_var in my_doc['variables'] ) {
			console.log("found variable: " + m_var)
			combinedDictionary[m_var] = my_doc['variables'][m_var]
		    };
		});
	    });
	    
        } catch (ex) {
            console.log("Error parsing file " + f + " - perhaps invalid yaml?");
	    process.exit(1);
	}
	console.log("variablefile - returning vars: " + JSON.stringify(combinedDictionary));
	callback(null, combinedDictionary);
    });
};

getVariablesFromServicefiles = function(servicefiles, callback){
    var variables = {}
    for (var f in servicefiles) {

	fs.readFile(f, function(err, contents){
	    var myRegexp = /\$\{(.*?)\}/;
	    var match = myRegexp.exec(contents);
            while( match != null ) {
		if ( /STACK::/.test(match) ) { 
		    continue; 
		}
		variables[match] = ""
	    };	    
	});
	console.log("servicefile - returning vars: " + JSON.stringify(variables));
	callback(null, variables);
    }
};

parseConfigVariablesFiles = function(basedir, callback){
    getConfigVariableFiles(basedir, function(err, variable_files){
	if (err) {
	    console.log(err);
	    process.exit(1);
	} else {
	    console.log('  get service files now');
	    getServicefiles(basedir, function(err, service_files){
		if (err){
		    console.log(err);
		    process.exit(1);
		} else {
		    getVariablesFromConfigFiles(variable_files, function(err, configVariables){
			if (err){
			    console.log(err);
			    process.exit(1);
			} else {
			    getVariablesFromServicefiles(service_files, function(err, scriptVariables){
				console.log('merging:');
				console.log('   scriptVariables: ' + JSON.stringify(scriptVariables));
				console.log('   configVariables: ' + JSON.stringify(configVariables));
				
				var target = _.extend(configVariables, scriptVariables);
				console.log('returning combined config files: ' + JSON.stringify(target));
				callback(null, target);
			    });
			}
		    });
		}
	    });
	}
    });
}

generateVariablesFile = function(startdir, showMissing, callback){
    var variables = parseConfigVariablesFiles(startdir, function(err, data){});
    var finalDoc = {"variables": {}}
    for (var item in variables) {
	for (var key in item ) {
	    if (item.hasOwnProperty(key)) {
		alert(key + " -> " + p[key]);
	    }
	}
    }
    // for key, value in sorted(variables.items()):
    //     debug("  key: %s | value: %s" % (key, value))
    //     if showMissing == True:
    //         if value:
    //             continue
    //     if not value:
    //         if finalDoc['variables'] == None:
    //             finalDoc['variables'] = {}
    //         value = {}
    //         value['default'] = ''
    //         value['description'] = ''
    //         value['overrides'] = ''
    //         value['type'] = ''
    //         value['required'] = True
    //     debug("finalDoc['variables']: %s" % finalDoc['variables'])
    //     debug("  key: %s | value: %s" % (key, value))
    //     finalDoc['variables'][key] = value
    // debug(finalDoc)
    // return finalDoc
    callback(null, 'hi');
}

initBaseDirectory = function(startdir, callback){
    console.log('initing base dir');
    var yml = yaml.dump(generateVariablesFile(startdir, true, function(err, data){
	if (err) {
	    console.log(err);
	    process.exit(1);
	} else {
	    console.log('yaml dumped');
	}
    }))
//     with open(os.path.join(startdir, "config.yaml"), 'w') as var_file:
//         var_file.write(
//                   """## This file was auto-generated by CloudCoreo CLI
// ## this yaml file contains variables, their defaults, overrides, type and namespaces. i.e:
// ## variables:
// ##   PUBLIC_ROUTE_NAME:
// ##     default: my-public-route
// ##     description: 'the name to give to the public route'
// ##     required: true
// ##     type: string

// %s
//                   """ % yml)
}

createReadmeMd = function(directory, content, callback) {
    var readmeFile = path.join(directory, "README.md");
    writeIfNeeded(readmeFile, content, callback);
};

writeIfNeeded = function(file_path, content, callback){
    console.log('file path: ' + file_path);
    fs.exists(file_path, function(exists) {
	if ( ! exists ) {
	    content.toString().split('\n').forEach(function (line) { 
		fs.appendFileSync(file_path, line.toString() + "\n");
	    });
	} else {
	    callback(null, true);
	}
    });
};

module.exports.createOverridesDirectory = function(startdir, callback) {
    override_dir = path.join(startdir, "overrides");
    mkdirp(override_dir, function(err) { 
	if (err) {
	    callback(err, false);
	} else {
	    createReadmeMd(override_dir, "## This file was auto-generated by CloudCoreo CLI\n" +
			   "Anything contained in this directory will act as an override for the stack in which it is contained.\n" +
			   "\n" +
			   "The paths should be considered relative to the parent of this directory.\n" +
			   "\n" +
			   "For example, if you have a directory structure like this, \n" +
			   "```\n" +
			   "+-- parent\n" +
			   "|   +-- overrides\n" +
			   "|   |   +-- stack-a\n" +
			   "|   |   |   +-- boot-scripts\n" +
			   "|   |   |   |   +-- order.yaml\n" +
			   "|   +-- stack-a\n" +
			   "|   |   +-- boot-scripts\n" +
			   "|   |   |   +-- order.yaml\n" +
			   "```\n" +
			   "Because the directory structure within the override directory matches the structure of the parent, \n" +
			   "the 'order.yaml' file will be ignored in the stack-a directory and instead the overrides/stack-a order.yaml\n" +
			   "file will be used.\n" +
			   "", function(err, created) {
			      if(err) {
				   callback(err, false);
			       } else {
				   callback(null, true);
			       }
			   });
	}
    });
};

module.exports.createConfigYamlFile = function(dir, variables, callback) {
    var configFilename = path.join(dir, "config.yaml");
    mkdirp(dir, function(err) { 
	writeIfNeeded(configFilename, "## This file was auto-generated by CloudCoreo CLI\n" +
		      "## this yaml file contains variables, their defaults, overrides, type and namespaces. i.e:\n" +
		      "## variables:\n" +
		      "##   PUBLIC_ROUTE_NAME:\n" +
		      "##     default: my-public-route\n" +
		      "##     description: 'the name to give to the public route'\n" +
		      "##     required: true\n" +
		      "##     type: string\n" +
		      "", function(err, data) { if (err) { console.log('error');callback(err, false); } });
    });
};

module.exports.createServicesDirectory = function(startdir, callback) {
    var services_dir = path.join(startdir, "services");
    mkdirp(services_dir, function(err) { 
	var configFilename = path.join(services_dir, "config.rb");
        writeIfNeeded(configFilename,"## This file was auto-generated by CloudCoreo CLI\n" +
		      "## This file was automatically generated using the CloudCoreo CLI\n" +
		      "##\n" +
		      "## This config.rb file exists to create and maintain services not related to compute.\n" +
		      "## for example, a VPC might be maintained using:\n" +
		      "##\n" +
		      "## coreo_aws_vpc_vpc \"my-vpc\" do\n" +
		      "##   action :sustain\n" +
		      "##   cidr \"172.16.17.32/16\"\n" +
		      "##   internet_gateway true\n" +
		      "## end\n" +
		      "##\n" +
		      "\n" +
		      "", function(err, data) { if (err) { console.log('error');callback(err, false); } });
	createReadmeMd(services_dir, "## This file was auto-generated by CloudCoreo CLI\n" +
		       "This is your services directory. Place a config.rb file in here containing CloudCoreo service\n" +
		       "syntax. For example, your config.rb might contain the following in order to create a VPC\n" +
		       "```\n" +
		       "coreo_aws_vpc_vpc \"my-vpc\" do\n" +
		       "  action :sustain\n" +
		       "  cidr \"172.16.17.32/16\"\n" +
		       "  internet_gateway true\n" +
		       "end\n" +
		       "\n" +
		       "coreo_aws_vpc_routetable \"my-public-route\" do\n" +
		       "  action :create\n" +
		       "  vpc \"my-vpc\"\n" +
		       "  number_of_tables 3\n" +
		       "  routes [\n" +
		       "         { :from => \"0.0.0.0/0\", :to => \"my-vpc\", :type => :igw },\n" +
		       "        ]\n" +
		       "end\n" +
		       "\n" +
		       "coreo_aws_vpc_subnet \"my-public-subnet\" do\n" +
		       "  action :create\n" +
		       "  number_of_zones 3\n" +
		       "  percent_of_vpc_allocated 50\n" +
		       "  route_table \"my-public-route\"\n" +
		       "  vpc \"my-vpc\"\n" +
		       "  map_public_ip_on_launch true\n" +
		       "end\n" +
		       "```\n" +
		       "", function(err, created) {
			   if(err) {
			       callback(err, false);
			   } else {
			       callback(null, true);
			   }
		       });
    });
};

    

{'fraction_non_alphanumeric': 0.16202945990180032, 'fraction_numerical': 0.0029524084592920637, 'mean_word_length': 2.6429740472293664, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7410820', 'n_tokens_mistral': 11082, 'n_tokens_neox': 10139, 'n_words': 2346}