<commit_before>#!/usr/bin/env node

var util = require('util'),
    path = require('path'),
    fs = require('fs'),
    step = require('./3rdparty/step'),
    _ = require('underscore'),
    fileUtils = require('./fileUtils'),
    assets = require('./assets'),
    resolvers = require('./loaders/Fs/resolvers'),
    SiteGraph = require('./SiteGraph'),
    FsLoader = require('./loaders/Fs'),
    error = require('./error'),
    options = {};

process.on('uncaughtException', function (e) {
    console.log("Uncaught exception: " + sys.inspect(e.msg) + "\n" + e.stack);
});

_.each(require('optimist').usage('FIXME').demand(['root', 'fixup-url']).argv,
    function (value, optionName) {
        options[optionName.replace(/-([a-z])/g, function ($0, $1) {
            return $1.toUpperCase();
        })] = value;
    }
);

var siteGraph = new SiteGraph(),
    loader = new FsLoader({
        siteGraph: siteGraph,
        root: options.root
    }),
    templates = [];

step(
    function () {
        var group = this.group();
        (options.label || []).forEach(function (labelDefinition) {
            var keyValue = labelDefinition.split('=');
            if (keyValue.length != 2) {
                throw "Invalid label syntax: " + labelDefinition;
            }
            var labelName = keyValue[0],
                labelValue = keyValue[1],
                callback = group(),
                matchSenchaJSBuilder = labelValue.match(/\.jsb(\d)$/);
            if (matchSenchaJSBuilder) {
                var url = path.dirname(labelValue) || '',
                    version = parseInt(matchSenchaJSBuilder[1], 10);
                fs.readFile(path.join(loader.root, labelValue), 'utf8', error.logAndExit(function (fileBody) {
                    loader.addLabelResolver(labelName, resolvers.SenchaJSBuilder, {
                        url: url,
                        version: version,
                        body: JSON.parse(fileBody)
                    });
                    callback();
                }));
            } else {
                path.exists(path.join(loader.root, labelValue), function (exists) {
                    if (!exists) {
                        throw new Error("Label " + labelName + ": Dir not found: " + labelValue);
                    }
                    loader.addLabelResolver(labelName, resolvers.Directory, {url: labelValue});
                    callback();
                });
            }
        });
        process.nextTick(group());
        if ('fixupUrl' in options) {
            fileUtils.mkpath(path.join(loader.root, options.fixupUrl), this.parallel());
        }
    },
    function () {
        var group = this.group();
        options._.forEach(function (templateUrl) {
            var template = loader.loadAsset({type: 'HTML', url: templateUrl});
            templates.push(template);
            loader.populate(template, ['htmlScript', 'jsStaticInclude'], group());
        });
    },
    error.logAndExit(function () {
        function makeHumanReadableFileName (asset) {
            return (asset.originalUrl || asset.url).replace(/[^a-z0-9_\-\.]/g, '_');
        }
        templates.forEach(function (template) {
            var document = template.parseTree, // Egh, gotta do it async?
                seenAssets = {};
            function makeTemplateRelativeUrl(url) {
                return fileUtils.buildRelativeUrl(path.dirname(template.url), url);
            }<commit_msg>More logAndExit<commit_after>#!/usr/bin/env node

var util = require('util'),
    path = require('path'),
    fs = require('fs'),
    step = require('./3rdparty/step'),
    _ = require('underscore'),
    fileUtils = require('./fileUtils'),
    assets = require('./assets'),
    resolvers = require('./loaders/Fs/resolvers'),
    SiteGraph = require('./SiteGraph'),
    FsLoader = require('./loaders/Fs'),
    error = require('./error'),
    options = {};

process.on('uncaughtException', function (e) {
    console.log("Uncaught exception: " + sys.inspect(e.msg) + "\n" + e.stack);
});

_.each(require('optimist').usage('FIXME').demand(['root', 'fixup-url']).argv,
    function (value, optionName) {
        options[optionName.replace(/-([a-z])/g, function ($0, $1) {
            return $1.toUpperCase();
        })] = value;
    }
);

var siteGraph = new SiteGraph(),
    loader = new FsLoader({
        siteGraph: siteGraph,
        root: options.root
    }),
    templates = [];

step(
    function () {
        var group = this.group();
        (options.label || []).forEach(function (labelDefinition) {
            var keyValue = labelDefinition.split('=');
            if (keyValue.length != 2) {
                throw "Invalid label syntax: " + labelDefinition;
            }
            var labelName = keyValue[0],
                labelValue = keyValue[1],
                callback = group(),
                matchSenchaJSBuilder = labelValue.match(/\.jsb(\d)$/);
            if (matchSenchaJSBuilder) {
                var url = path.dirname(labelValue) || '',
                    version = parseInt(matchSenchaJSBuilder[1], 10);
                fs.readFile(path.join(loader.root, labelValue), 'utf8', error.logAndExit(function (fileBody) {
                    loader.addLabelResolver(labelName, resolvers.SenchaJSBuilder, {
                        url: url,
                        version: version,
                        body: JSON.parse(fileBody)
                    });
                    callback();
                }));
            } else {
                path.exists(path.join(loader.root, labelValue), function (exists) {
                    if (!exists) {
                        callback(new Error("Label " + labelName + ": Dir not found: " + labelValue));
                    } else {
                        loader.addLabelResolver(labelName, resolvers.Directory, {url: labelValue});
                        callback();
                    }
                });
            }
        });
        process.nextTick(group());
        if ('fixupUrl' in options) {
            fileUtils.mkpath(path.join(loader.root, options.fixupUrl), this.parallel());
        }
    },
    error.logAndExit(function () {
        var group = this.group();
        options._.forEach(function (templateUrl) {
            var template = loader.loadAsset({type: 'HTML', url: templateUrl});
            templates.push(template);
            loader.populate(template, ['htmlScript', 'jsStaticInclude'], group());
        });
    }),
    error.logAndExit(function () {
        function makeHumanReadableFileName (asset) {
            return (asset.originalUrl || asset.url).replace(/[^a-z0-9_\-\.]/g, '_');
        }
        templates.forEach(function (template) {
            var document = template.parseTree, // Egh, gotta do it async?
                seenAssets = {};
            function makeTemplateRelativeUrl(url) {
                return fileUtils.buildRelativeUrl(path.dirname(template.url), url);
            }
{'fraction_non_alphanumeric': 0.14920452916726387, 'fraction_numerical': 0.003726530027232335, 'mean_word_length': 1.8563241915677446, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6136638', 'n_tokens_mistral': 1929, 'n_tokens_neox': 1850, 'n_words': 353}