<commit_before>/*///////////////////////////////////////////////////////////////////////////

 For Best results use the an 'app-config.json' file. This File should be
 located in the same directory as this file. If you choose not to use it
 then your application must run out of the same directory as this file
 and you entry point must be a file named 'start.js' located in this
 directory.

 'app-config.json' allows you to configure following properties for your
 application:


 "awsUpdates:" <Check for and get this latest AWS updates on a pull>
 "applicationName:" <name of the application>
 "applicationDirectory": <set this to the directory you application lives in>
 "applicationPath": <the path to the application directory>
 "appEntry": <set this to the name of the 'js' file that is your entry point>
 "commandArguments": <command line arguments you would like pass to the application>
 "appEnvironmentVariables": <{<key>:<pair>}, key pair environment variables that need to be se for the application >
 "appURL": <https://myapp> used for manual webhooks
 "dependencies" : <dependencies within the package.json that need to be pulled in addition to the application directory>


 "pullPort": <set this to the port for a pull requests> - defaults to 8000

 // The key and cert files are only necessary if you want to listen for a pull
 // request securely. If they are omitted and HTTP server is start instead. Beware
 // as someone could be snooping and then start sending your servers pull requests
 "pullKey": <path to a ssh key file for the HTTPS Server>
 "pullCert": <path to a ssh cert file for the HTTPS Server>
 "pullCa": <array of paths to the certificate authority files> (optional)
 "pullPassphrase" : <string - phrase that the certificate was generated with> (optional if certificate was not generated with a passphrase)

 // This is a secret key that is configured here and passed in via a webhook in
 // response to a pull request. This is to prevent unauthorized requests from causing
 // pulls. If no pull secret is configure then all pull request are valid
 "pullSecret": <secret phrase>

 "pullBranch": <git branch to use for the pull>
 "pullField": <field that contains the branch information on a post by the web hook (defaults to 'ref')>

 // In theory you can put an cloud vendor specific params in here. You just have to have support in cloud.js for them.
 // Curently AWS is the only cloud platform supported
 // Put your AWS config params in here. Example:
 "accessKeyId": "<KEY>",
 "secretAccessKey": "<KEY>",
 "region": "us-east-1"

 "sudo": <Mac and Linux only, should we prefix all shell commands with sudo?> - defaults to no sudo

 // This file must conform the following interface: It must have a start function that is exported that excepts a callback
 // as a parameter and calls the callback when compete to start the application.
 "preLaunch": <a javascript file to execute relative to the application directory before starting the app>

 // To Trigger a pull and restart


 *////////////////////////////////////////////////////////////////////////////

(function (){
    var fs = require ('fs');
    var exec = require('child_process').exec;
    var path = require('path');
    var url = require('url');
    var qs = require ('querystring');
    var cluster = require ('cluster');
    var async = require ('async');
    var config_file, config, error, pull_error = "";
    var package_json, package_copy, parsed_package, parsed_copy;
    var restart = false;
    var need_restart = false;
    var http = require('http');
    var https = require('https');
    // This an abstraction layer for different cloud services
    // all AWS specific code or other cloud vendor stuff should go in here
    var cloud = require ("./cloud.js");
    var updating_on = false;
    var sudo;
    var instance_data;
    var secure_post;
    var pull_field;
    var pull_list;
    var homePath;
    var appDir;

    // remove non-standard quotation marks and replace them with the standard ones
    function conditionString (str){
        var open = String.fromCharCode(147);
        var close = String.fromCharCode(148);
        return str && str.replace(open,'"').replace(close,'"');
    }


    // post a command out
    function post (_url, body, port, secure, path, cb){
        var qs = require('querystring');
        body = qs.stringify (body);
        var options = {
            host: _url,
            port: port,
            path: path,
            method: 'POST',
            headers: {
                'Content-Type': 'application/x-www-form-urlencoded',
                'Content-Length': body.length
            },
            rejectUnauthorized:false // add this so if someone uses a local certificate on the machine the post will still go through
        };

        var server = (secure) ? https : http;
        var req = server.request(options, function(res) {
            var result = "";
            res.setEncoding('utf8');
            res.on('data', function (chunk) {result += chunk;});
            res.on('end', function (){
                cb && cb (null, result);
            });
        });
        req.on('error', function(e) {
            console.log('problem with request: ' + e.message);
            cb && cb (e.message);
        });

        // write data to request body
        req.write(body);
        req.end();
    }

    // pull the latest source
    function pull(cb, master, req, res){
        master = (master === true || master === "true");
        pull_error = "";
        need_restart = false;
        if (!pull_list){
            pull_list = [appDir];
            if (config.dependencies){
                config.dependencies.forEach (function (proj){
                    pull_list.push (homePath + "/" + proj + "/");
                });
            }
        }
        function _pull (cb){
            console.log ("Pulling the latest code from remote repository");
            async.eachSeries (pull_list, function (proj, cb){
                // get the latest code
                console.log ("\nPulling " + proj);
                var child = exec ("cd " + proj + " ; " + sudo + "git pull", function (err, std, ster){
                    if (err){
                        console.log ("\n\nError pulling repository. Error" + ster);
                        pull_error += "\n\nError pulling repository. Error" + ster;
                    }
                    else{
                        console.log ("\n\n" + std);
                        if (std && std.search ("Already up-to-date") === -1){
                            need_restart = true;
                        }
                    }
                    cb ();
                });
            }, function (err){
                cb && cb ();
            });
        }
        // only pull the latest if in the cloud. For local development don't do anything. The developer must manually pull
        if (cloud.isCloud ()){
            if (!master){
                _pull (function (){
                    if (need_restart && restart){
                        if (pull_error){
                            res && res.send('They were errors pulling. Errors:' + pull_error);
                        }
                        else{
                            res && res.send('Pull Successful!');
                        }
                        res.send('hello world');
                        process.exit(0);
                    }
                    cb && cb ();
                });
            }
            else { // else we are the master so find all the other AWS instances to pull from
                // get other instances that our are same type and already running
                req.query.master = false;
                cloud.getInstances (function (err, instances){
                    if (instances && instances.length){
                        console.log ("Found " + instances.length + " instances, re-posting.");
                        instances.forEach (function (instance){
                            if (instance.dns && instance.id !== cloud.getInstanceId ()){ // don't signal ourselves
                                post (instance.dns, req.body, config.pullPort, secure_post,
                                    url.format ({pathname:"/pull", query:req.query}));
                            }
                        });
                        // now pull and restart ourselves
                        _pull (function (){
                            if (need_restart && restart){
                                if (pull_error){
                                    res && res.send('They were errors pulling. Errors:' + pull_error);
                                }
                                else{
                                    res && res.send('Pull Successful!');
                                }
                            }
                            cb && cb ();
                        });
                    }
                    else{
                        console.log ("No instances found");
                        cb && cb ();
                    }
                });
            }
        }
        else {cb && cb ();}
    }
    // check if any NPM dependencies changed
    function checkNodeDependencies (cb){
        // read in our files
        console.log ("\nChecking for Node version changes");
        try { package_copy = fs.readFileSync ("package.copy");}
        catch (e) { package_copy = null;}
        try {package_json = fs.readFileSync ("package.json");}
        catch (e) { package_json = null;}
        parsed_package = (package_json) ? JSON.parse (package_json) : null;
        parsed_copy = (package_copy) ? JSON.parse (package_copy) : null;
        // see if our node versions match
        if (parsed_package && parsed_package.nodeVersion){
            var version = parsed_package.nodeVersion.replace ('v', "");
            var node_version = process.version.replace ('v', "");
            if (version !== node_version){
                console.log ("	Upgrading Node");
                console.log ("		current version:" + node_version);
                console.log ("		requested version:" + version);
                // upgrade node using 'n'
                var child = exec (sudo + "n " + version, function (err, std, ster){
                    if (err){
                        console.log ("		Node upgrade failed. Error:" + ster);
                        pull_error += "\nNode upgrade failed. Error:" + ster;
                    }
                    else{
                        console.log ("		Node upgrade success, restarting");
                        process.exit (0); // exit so we get restarted
                    }
                    cb & cb ();
                });
            }
            else{
                console.log ("  Node is up to date");
                console.log ("      current version:" + node_version);
                cb & cb ();
            }
        }
        else{
            cb & cb ();
        }
    }

    // check AWS dependencies
    function checkAWSDependencies (cb){
        if (!config.local && instance_data.deploy){
            var child = exec("sudo yum -y update", function (err, std, ster){
                console.log ("\nChecking for AWS Updates\n" + std);
                if (err) {cb (ster);}
                else {cb ();}
            });
        }
        else {cb ();}
    }

    // check for any node module changes and reinstall the associated packages.
    // NPM doesn't do a good job of keeping track. So keep a copy of the last successful
    // update and compare it. Find ones that have changed and delete them and then
    // re-install.
    function checkNPMDependencies (cb, path){
        function deleteRecursiveSync(itemPath) {
            if (fs.existsSync(itemPath)){
                if (fs.existsSync(itemPath) && fs.statSync(itemPath).isDirectory()) {
                    fs.readdirSync(itemPath).forEach(function(childItemName) {
                        deleteRecursiveSync (path.join (itemPath, childItemName));
                    });
                    try {fs.rmdirSync(itemPath);}
                    catch (e){}
                } else {
                    fs.unlinkSync(itemPath);
                }
            }
        }

        var _package_json, _parsed_copy, _parsed_package, _package_copy;
        if (!path){
            _package_json = package_json;
            _parsed_copy = parsed_copy;
            _parsed_package = parsed_package;
            _package_copy = package_copy;
            path = '';
        }
        else{
            try { _package_copy = fs.readFileSync (path + "package.copy");}
            catch (e) { _package_copy = null;}
            try {_package_json = fs.readFileSync (path + "package.json");}
            catch (e) { _package_json = null;}
            _parsed_package = (_package_json) ? JSON.parse (_package_json) : null;
            _parsed_copy = (_package_copy) ? JSON.parse (_package_copy) : null;
        }

        console.log ("\nChecking for Node Module dependency changes");
        if (!_package_json){
            console.log ("WARNING Your Application has no 'package.json' . It is highly" +
                " recommended that you use one to manage your NPM dependencies");
        }
        else{ // delete the modules that have changed and re-install with new versions
            if (!package_copy || package_copy.toString() !== _package_json.toString ()){
                console.log ("\tNPM dependency changes detected");
                if (_parsed_package && _parsed_package.dependencies){
                    for (var package_name in _parsed_package.dependencies){
                        var copy_version = (_parsed_copy && _parsed_copy.dependencies) ? _parsed_copy.dependencies[package_name] : "";
                        if (copy_version !== _parsed_package.dependencies[package_name]){
                            deleteRecursiveSync (path + "node_modules/" + package_name);
                        }
                    }
                }
                if (_parsed_copy && _parsed_copy.dependencies){
                    for (package_name in _parsed_copy.dependencies){
                        copy_version = (_parsed_package && _parsed_package.dependencies) ? _parsed_package.dependencies[package_name] : "";
                        if (copy_version !== _parsed_copy.dependencies[package_name]){
                            deleteRecursiveSync (path + "node_modules/" + package_name);
                        }
                    }
                }
                console.log ("\tInstalling new Node Modules");
                var cmd_str = (path) ? "cd " + path + " ; " + sudo + "npm install -d" : sudo + "npm install -d";
                var child = exec (cmd_str, function (err, std, ster){
                    if (err){
                        console.log ("\t\tError installing Node modules. Error:" + ster);
                        pull_error += "\nError installing Node modules. Error:" + ster;
                    }
                    else{
                        console.log ("\t\tSuccessfully updated Node Modules: " + std);
                        fs.writeFileSync (path + "package.copy", _package_json);
                    }
                    cb && cb ();
                });
            }
            else{
                console.log ("\tNo node module changes detected");
                cb && cb ();
            }
        }
    }

    function checkAllNPMDependencies (cb){
        // first check the local ones
        checkNPMDependencies (function (){
            // now check the dependencies of any dependent projects
            var dependencies = (config && config.dependencies) || [];
            async.eachSeries (dependencies, function (dependency, done){
                var path = homePath + "/" + dependency + "/";
                checkNPMDependencies (function (){
                    done ();
                }, path);
            // other dependency directories are linked in using symbolic links
            // If we deleted them, add them back in
            }, function createNPMLinks (){
                console.log ("creating NPM Links");
                async.eachSeries (dependencies, function (proj, cb){
                    var cmd_str = " cd " + appDir + " ; sudo npm link " + proj;
                    var child = exec (cmd_str, function (err, std, ster){
                        if (err){<commit_msg>Improved support for dependency directories<commit_after>/*///////////////////////////////////////////////////////////////////////////

 For Best results use the an 'app-config.json' file. This File should be
 located in the same directory as this file. If you choose not to use it
 then your application must run out of the same directory as this file
 and you entry point must be a file named 'start.js' located in this
 directory.

 'app-config.json' allows you to configure following properties for your
 application:


 "awsUpdates:" <Check for and get this latest AWS updates on a pull>
 "applicationName:" <name of the application>
 "applicationDirectory": <set this to the directory you application lives in>
 "applicationPath": <the path to the application directory>
 "appEntry": <set this to the name of the 'js' file that is your entry point>
 "commandArguments": <command line arguments you would like pass to the application>
 "appEnvironmentVariables": <{<key>:<pair>}, key pair environment variables that need to be se for the application >
 "appURL": <https://myapp> used for manual webhooks
 "dependencies" : <dependencies within the package.json that need to be pulled in addition to the application directory>


 "pullPort": <set this to the port for a pull requests> - defaults to 8000

 // The key and cert files are only necessary if you want to listen for a pull
 // request securely. If they are omitted and HTTP server is start instead. Beware
 // as someone could be snooping and then start sending your servers pull requests
 "pullKey": <path to a ssh key file for the HTTPS Server>
 "pullCert": <path to a ssh cert file for the HTTPS Server>
 "pullCa": <array of paths to the certificate authority files> (optional)
 "pullPassphrase" : <string - phrase that the certificate was generated with> (optional if certificate was not generated with a passphrase)

 // This is a secret key that is configured here and passed in via a webhook in
 // response to a pull request. This is to prevent unauthorized requests from causing
 // pulls. If no pull secret is configure then all pull request are valid
 "pullSecret": <secret phrase>

 "pullBranch": <git branch to use for the pull>
 "pullField": <field that contains the branch information on a post by the web hook (defaults to 'ref')>

 // In theory you can put an cloud vendor specific params in here. You just have to have support in cloud.js for them.
 // Curently AWS is the only cloud platform supported
 // Put your AWS config params in here. Example:
 "accessKeyId": "<KEY>",
 "secretAccessKey": "<KEY>",
 "region": "us-east-1"

 "sudo": <Mac and Linux only, should we prefix all shell commands with sudo?> - defaults to no sudo

 // This file must conform the following interface: It must have a start function that is exported that excepts a callback
 // as a parameter and calls the callback when compete to start the application.
 "preLaunch": <a javascript file to execute relative to the application directory before starting the app>

 // To Trigger a pull and restart


 *////////////////////////////////////////////////////////////////////////////

(function (){
    var fs = require ('fs');
    var exec = require('child_process').exec;
    var path = require('path');
    var url = require('url');
    var qs = require ('querystring');
    var cluster = require ('cluster');
    var async = require ('async');
    var config_file, config, error, pull_error = "";
    var package_json, package_copy, parsed_package, parsed_copy;
    var restart = false;
    var need_restart = false;
    var http = require('http');
    var https = require('https');
    // This an abstraction layer for different cloud services
    // all AWS specific code or other cloud vendor stuff should go in here
    var cloud = require ("./cloud.js");
    var updating_on = false;
    var sudo;
    var instance_data;
    var secure_post;
    var pull_field;
    var pull_list;
    var homePath;
    var appDir;

    // remove non-standard quotation marks and replace them with the standard ones
    function conditionString (str){
        var open = String.fromCharCode(147);
        var close = String.fromCharCode(148);
        return str && str.replace(open,'"').replace(close,'"');
    }


    // post a command out
    function post (_url, body, port, secure, path, cb){
        var qs = require('querystring');
        body = qs.stringify (body);
        var options = {
            host: _url,
            port: port,
            path: path,
            method: 'POST',
            headers: {
                'Content-Type': 'application/x-www-form-urlencoded',
                'Content-Length': body.length
            },
            rejectUnauthorized:false // add this so if someone uses a local certificate on the machine the post will still go through
        };

        var server = (secure) ? https : http;
        var req = server.request(options, function(res) {
            var result = "";
            res.setEncoding('utf8');
            res.on('data', function (chunk) {result += chunk;});
            res.on('end', function (){
                cb && cb (null, result);
            });
        });
        req.on('error', function(e) {
            console.log('problem with request: ' + e.message);
            cb && cb (e.message);
        });

        // write data to request body
        req.write(body);
        req.end();
    }

    // pull the latest source
    function pull(cb, master, req, res){
        master = (master === true || master === "true");
        pull_error = "";
        need_restart = false;
        if (!pull_list){
            pull_list = [appDir];
            if (config.dependencies){
                config.dependencies.forEach (function (proj){
                    pull_list.push (homePath + "/" + proj + "/");
                });
            }
        }
        function _pull (cb){
            console.log ("Pulling the latest code from remote repository");
            async.eachSeries (pull_list, function (proj, cb){
                // get the latest code
                console.log ("\nPulling " + proj);
                var child = exec ("cd " + proj + " ; " + sudo + "git pull", function (err, std, ster){
                    if (err){
                        console.log ("\n\nError pulling repository. Error" + ster);
                        pull_error += "\n\nError pulling repository. Error" + ster;
                    }
                    else{
                        console.log ("\n\n" + std);
                        if (std && std.search ("Already up-to-date") === -1){
                            need_restart = true;
                        }
                    }
                    cb ();
                });
            }, function (err){
                cb && cb ();
            });
        }
        // only pull the latest if in the cloud. For local development don't do anything. The developer must manually pull
        if (cloud.isCloud ()){
            if (!master){
                _pull (function (){
                    if (need_restart && restart){
                        if (pull_error){
                            res && res.send('They were errors pulling. Errors:' + pull_error);
                        }
                        else{
                            res && res.send('Pull Successful!');
                        }
                        res.send('hello world');
                        process.exit(0);
                    }
                    cb && cb ();
                });
            }
            else { // else we are the master so find all the other AWS instances to pull from
                // get other instances that our are same type and already running
                req.query.master = false;
                cloud.getInstances (function (err, instances){
                    if (instances && instances.length){
                        console.log ("Found " + instances.length + " instances, re-posting.");
                        instances.forEach (function (instance){
                            if (instance.dns && instance.id !== cloud.getInstanceId ()){ // don't signal ourselves
                                post (instance.dns, req.body, config.pullPort, secure_post,
                                    url.format ({pathname:"/pull", query:req.query}));
                            }
                        });
                        // now pull and restart ourselves
                        _pull (function (){
                            if (need_restart && restart){
                                if (pull_error){
                                    res && res.send('They were errors pulling. Errors:' + pull_error);
                                }
                                else{
                                    res && res.send('Pull Successful!');
                                }
                            }
                            cb && cb ();
                        });
                    }
                    else{
                        console.log ("No instances found");
                        cb && cb ();
                    }
                });
            }
        }
        else {cb && cb ();}
    }
    // check if any NPM dependencies changed
    function checkNodeDependencies (cb){
        // read in our files
        console.log ("\nChecking for Node version changes");
        try { package_copy = fs.readFileSync ("package.copy");}
        catch (e) { package_copy = null;}
        try {package_json = fs.readFileSync ("package.json");}
        catch (e) { package_json = null;}
        parsed_package = (package_json) ? JSON.parse (package_json) : null;
        parsed_copy = (package_copy) ? JSON.parse (package_copy) : null;
        // see if our node versions match
        if (parsed_package && parsed_package.nodeVersion){
            var version = parsed_package.nodeVersion.replace ('v', "");
            var node_version = process.version.replace ('v', "");
            if (version !== node_version){
                console.log ("	Upgrading Node");
                console.log ("		current version:" + node_version);
                console.log ("		requested version:" + version);
                // upgrade node using 'n'
                var child = exec (sudo + "n " + version, function (err, std, ster){
                    if (err){
                        console.log ("		Node upgrade failed. Error:" + ster);
                        pull_error += "\nNode upgrade failed. Error:" + ster;
                    }
                    else{
                        console.log ("		Node upgrade success, restarting");
                        process.exit (0); // exit so we get restarted
                    }
                    cb & cb ();
                });
            }
            else{
                console.log ("  Node is up to date");
                console.log ("      current version:" + node_version);
                cb & cb ();
            }
        }
        else{
            cb & cb ();
        }
    }

    // check AWS dependencies
    function checkAWSDependencies (cb){
        if (!config.local && instance_data.deploy){
            var child = exec("sudo yum -y update", function (err, std, ster){
                console.log ("\nChecking for AWS Updates\n" + std);
                if (err) {cb (ster);}
                else {cb ();}
            });
        }
        else {cb ();}
    }

    // check for any node module changes and reinstall the associated packages.
    // NPM doesn't do a good job of keeping track. So keep a copy of the last successful
    // update and compare it. Find ones that have changed and delete them and then
    // re-install.
    function checkNPMDependencies (cb, projPath){
        function deleteRecursiveSync(itemPath) {
            if (fs.existsSync(itemPath)){
                if (fs.existsSync(itemPath) && fs.statSync(itemPath).isDirectory()) {
                    fs.readdirSync(itemPath).forEach(function(childItemName) {
                        deleteRecursiveSync (path.join (itemPath, childItemName));
                    });
                    try {fs.rmdirSync(itemPath);}
                    catch (e){}
                } else {
                    fs.unlinkSync(itemPath);
                }
            }
        }

        var _package_json, _parsed_copy, _parsed_package, _package_copy;
        if (!projPath){
            _package_json = package_json;
            _parsed_copy = parsed_copy;
            _parsed_package = parsed_package;
            _package_copy = package_copy;
            projPath = '';
        }
        else{
            try { _package_copy = fs.readFileSync (projPath + "package.copy");}
            catch (e) { _package_copy = null;}
            try {_package_json = fs.readFileSync (projPath + "package.json");}
            catch (e) { _package_json = null;}
            _parsed_package = (_package_json) ? JSON.parse (_package_json) : null;
            _parsed_copy = (_package_copy) ? JSON.parse (_package_copy) : null;
        }

        console.log ("\nChecking for Node Module dependency changes");
        if (!_package_json){
            console.log ("WARNING Your Application has no 'package.json' . It is highly" +
                " recommended that you use one to manage your NPM dependencies");
        }
        else{ // delete the modules that have changed and re-install with new versions
            if (!package_copy || package_copy.toString() !== _package_json.toString ()){
                console.log ("\tNPM dependency changes detected");
                if (_parsed_package && _parsed_package.dependencies){
                    for (var package_name in _parsed_package.dependencies){
                        var copy_version = (_parsed_copy && _parsed_copy.dependencies) ? _parsed_copy.dependencies[package_name] : "";
                        if (copy_version !== _parsed_package.dependencies[package_name]){
                            deleteRecursiveSync (projPath + "node_modules/" + package_name);
                        }
                    }
                }
                if (_parsed_copy && _parsed_copy.dependencies){
                    for (package_name in _parsed_copy.dependencies){
                        copy_version = (_parsed_package && _parsed_package.dependencies) ? _parsed_package.dependencies[package_name] : "";
                        if (copy_version !== _parsed_copy.dependencies[package_name]){
                            deleteRecursiveSync (projPath + "node_modules/" + package_name);
                        }
                    }
                }
                console.log ("\tInstalling new Node Modules");
                var cmd_str = (projPath) ? "cd " + projPath + " ; " + sudo + "npm install -d" : sudo + "npm install -d";
                var child = exec (cmd_str, function (err, std, ster){
                    if (err){
                        console.log ("\t\tError installing Node modules. Error:" + ster);
                        pull_error += "\nError installing Node modules. Error:" + ster;
                    }
                    else{
                        console.log ("\t\tSuccessfully updated Node Modules: " + std);
                        fs.writeFileSync (projPath + "package.copy", _package_json);
                    }
                    cb && cb ();
                });
            }
            else{
                console.log ("\tNo node module changes detected");
                cb && cb ();
            }
        }
    }

    function checkAllNPMDependencies (cb){
        // first check the local ones
        checkNPMDependencies (function (){
            // now check the dependencies of any dependent projects
            var dependencies = (config && config.dependencies) || [];
            async.eachSeries (dependencies, function (dependency, done){
                var path = homePath + "/" + dependency + "/";
                checkNPMDependencies (function (){
                    done ();
                }, path);
            // other dependency directories are linked in using symbolic links
            // If we deleted them, add them back in
            }, function createNPMLinks (){
                console.log ("creating NPM Links");
                async.eachSeries (dependencies, function (proj, cb){
                    var cmd_str = " cd " + appDir + " ; sudo npm link " + proj;
                    var child = exec (cmd_str, function (err, std, ster){
                        if (err){
{'fraction_non_alphanumeric': 0.10880860318994684, 'fraction_numerical': 0.0009062348960850652, 'mean_word_length': 1.5629016025392892, 'pattern_counts': {'":': 36, '<': 49, '<?xml version=': 0, '>': 49, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '37817', 'n_tokens_mistral': 8226, 'n_tokens_neox': 7813, 'n_words': 2805}