<commit_before>"use strict";

var cfg        = {},
    _          = require("lodash"),
    fs         = require("graceful-fs"),
    mkdirp     = require("mkdirp"),
    path       = require("path"),
    configFile = require("./paths.js").get().cfg,
    defaults   = {
        "listeners" : [
            {
                "host"     : "0.0.0.0",
                "port"     : 8989,
                "protocol" : "http"
            }
        ],
        "debug"        : false,
        "keepAlive"    : 20000,
        "linkLength"   : 3,
        "logLevel"     : 2,
        "maxFileSize"  : 0,
        "maxOpen"      : 256,
        "public"       : false,
        "readInterval" : 250,
        "timestamps"   : true,
        "zipLevel"     : 1
    };

cfg.init = function (config, callback) {
    if (typeof config === "object" && config !== null) {
        config = _.defaults(config, defaults); // Add missing options
        callback(null, config);
    } else if (process.env.NODE_ENV === "droppydemo") {
        config = _.defaults({
            "listeners" : [
                {
                    "host"     : "0.0.0.0",
                    "port"     : process.env.PORT,
                    "protocol" : "http"
                }
            ],
            "public"     : true,
            "timestamps" : false
        }, defaults);
        callback(null, config);
    } else {
        fs.stat(configFile, function (err) {
            if (err) {
                if (err.code === "ENOENT") {
                    config = defaults;
                    mkdirp(path.dirname(configFile), function () {
                        write(config, function (err) {
                            callback(err || null, config);
                        });
                    });
                } else {
                    callback(err);
                }
            } else {
                fs.readFile(configFile, function (err, data) {
                    if (err) return callback(err);

                    try {
                        config = JSON.parse(String(data));
                    } catch (error) {
                        return callback(err);
                    }

                    if (!config) config = {};
                    config = _.defaults(config, defaults);

                    // Remove options no longer present
                    Object.keys(config).forEach(function (key) {
                        if (typeof defaults[key] === "undefined") {
                            delete config[key];
                        }
                    });

                    write(config, function (err) {
                        callback(err || null, config);
                    });
                });
            }
        });
    }
};

function write(config, callback) {
    fs.writeFile(configFile, JSON.stringify(config, null, 4), callback);
}

exports = module.exports = cfg;
<commit_msg>Config: Add migration from old format<commit_after>"use strict";

var cfg        = {},
    _          = require("lodash"),
    fs         = require("graceful-fs"),
    mkdirp     = require("mkdirp"),
    path       = require("path"),
    configFile = require("./paths.js").get().cfg,
    defaults   = {
        "listeners" : [
            {
                "host"     : "0.0.0.0",
                "port"     : 8989,
                "protocol" : "http"
            }
        ],
        "debug"        : false,
        "keepAlive"    : 20000,
        "linkLength"   : 3,
        "logLevel"     : 2,
        "maxFileSize"  : 0,
        "maxOpen"      : 256,
        "public"       : false,
        "readInterval" : 250,
        "timestamps"   : true,
        "zipLevel"     : 1
    };

cfg.init = function (config, callback) {
    if (typeof config === "object" && config !== null) {
        config = migrate(config);
        config = _.defaults(config, defaults); // Add missing options
        callback(null, config);
    } else if (process.env.NODE_ENV === "droppydemo") {
        config = _.defaults({
            "listeners" : [
                {
                    "host"     : "0.0.0.0",
                    "port"     : process.env.PORT,
                    "protocol" : "http"
                }
            ],
            "public"     : true,
            "timestamps" : false
        }, defaults);
        callback(null, config);
    } else {
        fs.stat(configFile, function (err) {
            if (err) {
                if (err.code === "ENOENT") {
                    config = defaults;
                    mkdirp(path.dirname(configFile), function () {
                        write(config, function (err) {
                            callback(err || null, config);
                        });
                    });
                } else {
                    callback(err);
                }
            } else {
                fs.readFile(configFile, function (err, data) {
                    if (err) return callback(err);

                    try {
                        config = JSON.parse(String(data));
                    } catch (error) {
                        return callback(err);
                    }

                    if (!config) config = {};

                    config = migrate(config);
                    config = _.defaults(config, defaults);

                    // Remove options no longer present
                    Object.keys(config).forEach(function (key) {
                        if (typeof defaults[key] === "undefined") {
                            delete config[key];
                        }
                    });

                    write(config, function (err) {
                        callback(err || null, config);
                    });
                });
            }
        });
    }
};

function write(config, callback) {
    fs.writeFile(configFile, JSON.stringify(config, null, 4), callback);
}

function migrate(config) {
    var oldProps = ["host", "port", "useTLS", "useSPDY", "useHSTS"];

    var needToMigrate = oldProps.every(function (prop) {
        return config.hasOwnProperty(prop);
    });

    if (needToMigrate && !config.listeners) {
        config.listeners = [{
            "host"     : config.host,
            "port"     : config.port,
            "protocol" : config.useSPDY ? "spdy" : config.useTLS ? "https" : "http",
            "hsts"     : config.useHSTS ? 31536000 : 0
        }];
        oldProps.forEach(function (prop) {
            delete config[prop];
        });
        return config;
    } else {
        return config;
    }
}

exports = module.exports = cfg;

{'fraction_non_alphanumeric': 0.12940461725394897, 'fraction_numerical': 0.009872417982989065, 'mean_word_length': 1.0971337579617835, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2710584', 'n_tokens_mistral': 1826, 'n_tokens_neox': 1682, 'n_words': 355}