<commit_before>//
FIRE.isnode = !!(typeof(process) !== 'undefined' && process.versions && process.versions.node);
FIRE.isnw = !!(FIRE.isnode && 'node-webkit' in process.versions);   // node-webkit
FIRE.isas = !!(FIRE.isnode && 'atom-shell' in process.versions);    // atom-shell
FIRE.isapp = FIRE.isnw || FIRE.isas;                                // native client
FIRE.isweb = !FIRE.isnode && !FIRE.isapp;                           // common web browser

if (FIRE.isnode) {
    FIRE.isdarwin = process.platform === 'darwin';
    FIRE.iswin32 = process.platform === 'win32';
}
else {
    // http://stackoverflow.com/questions/19877924/what-is-the-list-of-possible-values-for-navigator-platform-as-of-today
    var platform = window.navigator.platform;
    FIRE.isdarwin = platform.substring(0, 3) === 'Mac';
    FIRE.iswin32 = platform.substring(0, 3) === 'Win';
}

if (FIRE.isnode) {
    var Fs = require('fs');
    var Path = require('path');

    FIRE.Path = Path;

    var _readDirRecursively = function ( dirPath, resultArray ) {
        var fileList = Fs.readdirSync(dirPath);
        var stat = null;
        var path = '';
        for (var i = 0; i < fileList.length; i++) {
            path = Path.join(dirPath, fileList[i]);
            try {
                stat = Fs.statSync(path);
            }
            catch (e) {
                continue;
            }
            if (stat.isFile()) {
                resultArray.push(path);
            }
            else if (stat.isDirectory()) {
                _readDirRecursively(path, resultArray);
            }
        }
    };
    FIRE.readDirRecursively = function ( path ) {
        var retval = [];
        if (Array.isArray(path)) {
            for (var i = 0; i < path.length; i++) {
                retval = retval.concat(FIRE.readDirRecursively(path[i]));
            }
            return retval;
        }
        var stat = null;
        //var entry = event.dataTransfer.items[i].webkitGetAsEntry();
        try {
            stat = Fs.statSync(path);
        }
        catch (e) {
            return [];
        }
        if (stat.isFile()) {
            retval.push(path);
        }
        else if (stat.isDirectory()) {
            _readDirRecursively(path, retval);
        }
        return retval;
    };
}
else {
    var error = function () {
        throw "This function can only be used in node-webkit or server";
    };
    FIRE.readDirRecursively = error;

    FIRE.Path = {};
    FIRE.Path.basename = function (path) {
        return path.replace(/^.*(\\|\/|\:)/, '');
    };
    FIRE.Path.extname = function (path) {
        return path.substring((~-path.lastIndexOf(".") >>> 0) + 1);
    };
}

FIRE.Path.setExtension = function (path, newExtension) {
    // if (FIRE.isnode) return Path.join(Path.dirname(path), Path.basename(path, Path.extname(path))) + newExtension;
    var dotIndex = (~-path.lastIndexOf(".") >>> 0) + 1;
    return path.substring(0, dotIndex) + newExtension;
};

if (FIRE.isnw) {
    FIRE.getSavePath = function (defaultFilename, preserveDirKey, callback) {
        var persistentId = 'SaveFileDialog';
        var chooser = document.getElementById(persistentId);
        if (chooser) {
            // remove the old one or change event may not fired
            document.body.removeChild(chooser);
        }
        chooser = document.createElement('input');
        document.body.appendChild(chooser);
        chooser.id = persistentId;
        chooser.style.display = 'none';
        chooser.type = 'file';

        chooser.nwsaveas = defaultFilename;
        var defaultDir = localStorage[preserveDirKey];
        chooser.nwworkingdir = defaultDir || '';
        
        chooser.onchange = function (evt) {
            document.body.removeChild(chooser);
            //chooser.removeEventListener("change", arguments.callee);
            chooser.onchange = null;
            //console.log('value ' + this.value);
            localStorage[preserveDirKey] = this.value;
            callback(this.value);
        };
        chooser.click();
    };
    var nwgui = require('nw.gui');
    FIRE.showItemInFolder = nwgui.Shell.showItemInFolder;
}
else if (FIRE.isas) {
    FIRE.getSavePath = function (defaultFilename, preserveDirKey, callback, title, browserWindow) {
        var defaultDir = localStorage[preserveDirKey];
        var defaultPath = null;
        if (defaultDir && typeof defaultDir === 'string') {
            defaultDir = FIRE.Path.dirname(defaultDir);
            defaultPath = FIRE.Path.join(defaultDir, defaultFilename);
        }
        var remote = require('remote');
        var dialog = remote.require('dialog');
        var options = {
            title: title,
            defaultPath: defaultPath,
        };
        if (!browserWindow) {
            browserWindow = remote.getCurrentWindow();
        }
        dialog.showSaveDialog(browserWindow, options, function (path) {
            if (path) {
                localStorage[preserveDirKey] = path;
            }
            callback(path);
        });
    };
    var shell = require('shell');
    FIRE.showItemInFolder = shell.showItemInFolder;
}
else {
    var error = function () {
        throw "This function can only be used in node-webkit or atom-shell";
    };
    FIRE.getSavePath = error;
    FIRE.showItemInFolder = error;
}<commit_msg>support nodejs context<commit_after>//
FIRE.isnode = !!(typeof process !== 'undefined' && process.versions && process.versions.node);
FIRE.isviewctx = (typeof __dirname === 'undefined' || __dirname === null);  // window's render context in node-webkit or atom-shell
FIRE.isnw = !!(FIRE.isnode && 'node-webkit' in process.versions);           // node-webkit
FIRE.isas = !!(FIRE.isnode && 'atom-shell' in process.versions);            // atom-shell
FIRE.isapp = FIRE.isnw || FIRE.isas;                                        // native client
FIRE.isweb = !FIRE.isnode && !FIRE.isapp;                                   // common web browser

if (FIRE.isnode) {
    FIRE.isdarwin = process.platform === 'darwin';
    FIRE.iswin32 = process.platform === 'win32';
}
else {
    // http://stackoverflow.com/questions/19877924/what-is-the-list-of-possible-values-for-navigator-platform-as-of-today
    var platform = window.navigator.platform;
    FIRE.isdarwin = platform.substring(0, 3) === 'Mac';
    FIRE.iswin32 = platform.substring(0, 3) === 'Win';
}

if (FIRE.isnode) {
    var Fs = require('fs');
    var Path = require('path');

    FIRE.Path = Path;

    var _readDirRecursively = function ( dirPath, resultArray ) {
        var fileList = Fs.readdirSync(dirPath);
        var stat = null;
        var path = '';
        for (var i = 0; i < fileList.length; i++) {
            path = Path.join(dirPath, fileList[i]);
            try {
                stat = Fs.statSync(path);
            }
            catch (e) {
                continue;
            }
            if (stat.isFile()) {
                resultArray.push(path);
            }
            else if (stat.isDirectory()) {
                _readDirRecursively(path, resultArray);
            }
        }
    };
    FIRE.readDirRecursively = function ( path ) {
        var retval = [];
        if (Array.isArray(path)) {
            for (var i = 0; i < path.length; i++) {
                retval = retval.concat(FIRE.readDirRecursively(path[i]));
            }
            return retval;
        }
        var stat = null;
        //var entry = event.dataTransfer.items[i].webkitGetAsEntry();
        try {
            stat = Fs.statSync(path);
        }
        catch (e) {
            return [];
        }
        if (stat.isFile()) {
            retval.push(path);
        }
        else if (stat.isDirectory()) {
            _readDirRecursively(path, retval);
        }
        return retval;
    };
}
else {
    var error = function () {
        throw "This function can only be used in node-webkit or server";
    };
    FIRE.readDirRecursively = error;

    FIRE.Path = {};
    FIRE.Path.basename = function (path) {
        return path.replace(/^.*(\\|\/|\:)/, '');
    };
    FIRE.Path.extname = function (path) {
        return path.substring((~-path.lastIndexOf(".") >>> 0) + 1);
    };
}

FIRE.Path.setExtension = function (path, newExtension) {
    // if (FIRE.isnode) return Path.join(Path.dirname(path), Path.basename(path, Path.extname(path))) + newExtension;
    var dotIndex = (~-path.lastIndexOf(".") >>> 0) + 1;
    return path.substring(0, dotIndex) + newExtension;
};

if (FIRE.isviewctx) {
    if (FIRE.isnw) {
        FIRE.getSavePath = function (defaultFilename, preserveDirKey, callback) {
            var persistentId = 'SaveFileDialog';
            var chooser = document.getElementById(persistentId);
            if (chooser) {
                // remove the old one or change event may not fired
                document.body.removeChild(chooser);
            }
            chooser = document.createElement('input');
            document.body.appendChild(chooser);
            chooser.id = persistentId;
            chooser.style.display = 'none';
            chooser.type = 'file';

            chooser.nwsaveas = defaultFilename;
            var defaultDir = localStorage[preserveDirKey];
            chooser.nwworkingdir = defaultDir || '';
        
            chooser.onchange = function (evt) {
                document.body.removeChild(chooser);
                //chooser.removeEventListener("change", arguments.callee);
                chooser.onchange = null;
                //console.log('value ' + this.value);
                localStorage[preserveDirKey] = this.value;
                callback(this.value);
            };

            //chooser.addEventListener("change", function (evt) {
            //    console.log('change');
            //}, false);
            //chooser.addEventListener("blur", function (evt) {
            //    console.log('blur');
            //}, false);
            //chooser.addEventListener("load", function (evt) {
            //    console.log('load');
            //}, false);
            //chooser.addEventListener("unload", function (evt) {
            //    console.log('unload');
            //}, false);
            //chooser.addEventListener("abort", function (evt) {
            //    console.log('abort');
            //}, false);
            //chooser.addEventListener("click", function (evt) {
            //    console.log('click');
            //}, false);
            //chooser.addEventListener("error", function (evt) {
            //    console.log('error');
            //}, false);
            //chooser.addEventListener("resize", function (evt) {
            //    console.log('resize');
            //}, false);
            //chooser.addEventListener("select", function (evt) {
            //    console.log('select');
            //}, false);
            //chooser.addEventListener("submit", function (evt) {
            //    console.log('submit');
            //}, false);
            //chooser.addEventListener("reset", function (evt) {
            //    console.log('reset');
            //}, false);
            //chooser.addEventListener("focus", function (evt) {
            //    console.log('focus');
            //}, false);
            //chooser.addEventListener("focusin", function (evt) {
            //    console.log('focusin');
            //}, false);
            //chooser.addEventListener("focusout", function (evt) {
            //    console.log('focusout');
            //}, false);
            //chooser.addEventListener("error", function (evt) {
            //    console.log('error');
            //}, false);


            //chooser.addEventListener("change", function (evt) {
            //    console.log('change');
            //}, true);
            //chooser.addEventListener("blur", function (evt) {
            //    console.log('blur');
            //}, true);
            //chooser.addEventListener("load", function (evt) {
            //    console.log('load');
            //}, true);
            //chooser.addEventListener("unload", function (evt) {
            //    console.log('unload');
            //}, true);
            //chooser.addEventListener("abort", function (evt) {
            //    console.log('abort');
            //}, true);
            //chooser.addEventListener("click", function (evt) {
            //    console.log('click');
            //}, true);
            //chooser.addEventListener("error", function (evt) {
            //    console.log('error');
            //}, true);
            //chooser.addEventListener("resize", function (evt) {
            //    console.log('resize');
            //}, true);
            //chooser.addEventListener("select", function (evt) {
            //    console.log('select');
            //}, true);
            //chooser.addEventListener("submit", function (evt) {
            //    console.log('submit');
            //}, true);
            //chooser.addEventListener("reset", function (evt) {
            //    console.log('reset');
            //}, true);
            //chooser.addEventListener("focus", function (evt) {
            //    console.log('focus');
            //}, true);
            //chooser.addEventListener("focusin", function (evt) {
            //    console.log('focusin');
            //}, true);
            //chooser.addEventListener("focusout", function (evt) {
            //    console.log('focusout');
            //}, true);
            //chooser.addEventListener("error", function (evt) {
            //    console.log('error');
            //}, true);
            chooser.click();
            //chooser.value = '';
            //chooser.files = new FileList();
            //chooser.files.append(new File('',''));
            //chooser.files = null;
        };
        var nwgui = require('nw.gui');
        FIRE.showItemInFolder = nwgui.Shell.showItemInFolder;
    }
    else if (FIRE.isas) {
        FIRE.getSavePath = function (defaultFilename, preserveDirKey, callback, title, browserWindow) {
            var defaultDir = localStorage[preserveDirKey];
            var defaultPath = null;
            if (defaultDir && typeof defaultDir === 'string') {
                defaultDir = FIRE.Path.dirname(defaultDir);
                defaultPath = FIRE.Path.join(defaultDir, defaultFilename);
            }
            var remote = require('remote');
            var dialog = remote.require('dialog');
            var options = {
                title: title,
                defaultPath: defaultPath,
            };
            if (!browserWindow) {
                browserWindow = remote.getCurrentWindow();
            }
            dialog.showSaveDialog(browserWindow, options, function (path) {
                if (path) {
                    localStorage[preserveDirKey] = path;
                }
                callback(path);
            });
        };
        var shell = require('shell');
        FIRE.showItemInFolder = shell.showItemInFolder;
    }
    else {
        var error = function () {
            throw "This function can only be used in node-webkit or atom-shell";
        };
        FIRE.getSavePath = error;
        FIRE.showItemInFolder = error;
    }
}
{'fraction_non_alphanumeric': 0.14624843821924113, 'fraction_numerical': 0.0032879594923390545, 'mean_word_length': 1.7269141115294961, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1965882', 'n_tokens_mistral': 4299, 'n_tokens_neox': 4233, 'n_words': 822}