<commit_before>var Pagelet = require('./pagelet.js');
var status = Pagelet.status;
var mode = Pagelet.mode;
var _ = require('./util.js');
var Readable = require('stream').Readable;
var util = require('util');
var Promise = require('bluebird');

var BigPipe = module.exports = function BigPipe(options) {
    this.options = _.mixin(_.mixin({}, BigPipe.options), options);
    this.map = {};
    this.pagelets = [];
    this.pipelines = [];
    this.rendered = [];
    this.sources = {};
    this.state = status.pending;
    this.quicklings = [];
    this.parentQuicklings = [];
    this.Pagelet = Pagelet;
    this.pageletData = {};

    this.on('pagelet:after', this._onPageletDone);
    Readable.call(this, null);
};

// default options
BigPipe.options = {
    skipAnalysis: true,
    // configure output template.
    // you can use the following varible.
    // 
    // - this.id            pagelet id
    // - this.js            pagelet js
    // - this.css           pagelet css
    // - this.scripts       pagelet inline scripts
    // - this.styles        pagelet inline css
    // - this.html          pagelet html content
    // - this.container     pagelet container if specified
    // - this.json          stringfied json data.
    tpl: {
        _default: '<script type="text/javascript">' +
            'BigPipe.onPageletArrive(<%= this.json %>);' +
            '</script>',

        quickling: 'BigPipe.onPageletArrive(<%= this.json %>);'
    }
};

util.inherits(BigPipe, Readable);

// don't call this directly.
BigPipe.prototype._read = function (n) {
    if (this.state === status.pending) {
        this.render();
    }
};

// bind pagelet data source.
// accept fucntion and model name.
BigPipe.prototype.bind = function (id, fn) {
    if (fn.length === 1) {
        fn = Promise.promisify(fn);
    }
    this.sources[id] = fn;
    return this;
};

// 当为 quickling 模式的时候调用。
// 设置后，只有在此列表中的 pagelet 才会渲染。
BigPipe.prototype.addQuicklingPagelets = function (pagelets) {
    var me = this;
    pagelets.forEach(function (id) {
        var pageletIDSplit = id.split('@');
        var pageletID = pageletIDSplit.shift();
        me.quicklings.push(pageletID);
        me.parentQuicklings.push.apply(me.parentQuicklings, pageletIDSplit);
    });
};

// 判断是否是 quickling 模式。
BigPipe.prototype.isQuicklingMode = function () {
    return !!(this.quicklings && this.quicklings.length);
};

// 拼写兼容
BigPipe.prototype.isQuickingMode = BigPipe.prototype.isQuicklingMode;

// 添加 pagelet.
BigPipe.prototype.addPagelet = function (obj) {
    // 已经晚了了，不处理。
    if (this.state === status.fulfilled) {
        return false;
    }
    else if (!obj.id) {
        this.emit('error', new Error('Id is required when add pagelet'));
        return;
    }

    if (!this.isQuicklingMode() && obj.mode !== mode.pipeline && obj.mode !== mode.async) {
        // 非 quickling 请求，只接收 pipeline 和 async 模式的 pagelet.
        return false;
    }
    else if (this.isQuicklingMode() && !~this.quicklings.indexOf(obj.id)) {
        // 以下几种情况均需要继续执行
        // 1. 被声明为父Pagelet
        // 2. 先祖Pagelet被请求了

        if (!~this.parentQuicklings.indexOf(obj.id)) {
            var parent, founded;
            // 如果没有被声明为父Pagelet，则检查是否属于先祖Pagelet被请求了的情况

            // 如果是lazy模式，则不寻找先祖Pagelet
            if (obj.lazy) {
                return false;
            }
            // 支持嵌套pagelet时，一次性返回所有pagelet
            parent = obj;
            founded = false;
            // 如果指定的pagelet不在quicklings列表中，检查先祖节点，查看先祖节点是否在列表中，如果在则加载
            // 以此实现Pagelet A contain Pagelet B时，会用一个请求同时返回A与B
            while (parent && parent.parentId) {
                parent = this.map[parent.parentId];
                if (this.isQuicklingWidget(parent) && !!~this.quicklings.indexOf(parent.id)) {
                    founded = true;
                    // 插入为需要输出的Pagelet
                    this.quicklings.push(obj.id);
                    break;
                }
            }
            if (!founded) {
                return false;
            }
        }
    }

    obj.reqID = this.reqID;
    obj.skipAnalysis = this.options.skipAnalysis;

    var pagelet = new Pagelet(obj);
    var self = this;

    this.pagelets.push(pagelet);
    this.map[pagelet.id] = pagelet;
    pagelet.mode === mode.pipeline && this.pipelines.push(pagelet);

    // 转发事件。
    pagelet.on('error', this.emit.bind(this, 'error'));
    [
        'before', 'after',
        'render:before', 'render:after',
        'analyse:before', 'analyse:after'
    ].forEach(function (name) {
        pagelet.on(name, self.emit.bind(self, 'pagelet:' + name, pagelet));
    });

    if (this.state === status.rendering) {
        this.renderPagelet(pagelet);
    }
};

BigPipe.prototype.isQuicklingWidget = function (item) {
    return (item.mode === mode.quickling || item.mode === mode.async || item.mode === mode.pipeline);
};

BigPipe.prototype.render = function () {
    if (!this.pagelets) {
        return;
    }
    var pagelets = this.pagelets.concat();

    this.state = status.rendering;

    pagelets.forEach(this.renderPagelet.bind(this));
    this._checkFinish();
};

BigPipe.prototype.prepareAllSources = function (cb) {
    var sources = this.sources;
    var promiseSources = {};
    var me = this;
    for (var id in sources) {
        if (sources.hasOwnProperty(id)) {
            // 屏蔽所有异常
            promiseSources[id] = sources[id]().catch(function (err) {
                return {};
            });
        }
    }
    return Promise.props(promiseSources).then(function (data) {
        me.pageletData = data;
    });
};

BigPipe.prototype.renderPagelet = function (pagelet) {
    var sources = this.sources;
    var source = sources[pagelet.id];

    // 屏蔽无法调整为同步的 bigpipe 绑定策略
    // if (!source && typeof sources.all === 'function') {
    //     source = sources.all.bind(null, pagelet.id);
    // }

    // // hook
    // this.emit('pagelet:source', pagelet.id, function (_source) {
    //     source = _source;
    // });
<commit_msg>add error msg in spider mode<commit_after>var Pagelet = require('./pagelet.js');
var status = Pagelet.status;
var mode = Pagelet.mode;
var _ = require('./util.js');
var Readable = require('stream').Readable;
var util = require('util');
var Promise = require('bluebird');

var BigPipe = module.exports = function BigPipe(options) {
    this.options = _.mixin(_.mixin({}, BigPipe.options), options);
    this.map = {};
    this.pagelets = [];
    this.pipelines = [];
    this.rendered = [];
    this.sources = {};
    this.state = status.pending;
    this.quicklings = [];
    this.parentQuicklings = [];
    this.Pagelet = Pagelet;
    this.pageletData = {};

    this.on('pagelet:after', this._onPageletDone);
    Readable.call(this, null);
};

// default options
BigPipe.options = {
    skipAnalysis: true,
    // configure output template.
    // you can use the following varible.
    // 
    // - this.id            pagelet id
    // - this.js            pagelet js
    // - this.css           pagelet css
    // - this.scripts       pagelet inline scripts
    // - this.styles        pagelet inline css
    // - this.html          pagelet html content
    // - this.container     pagelet container if specified
    // - this.json          stringfied json data.
    tpl: {
        _default: '<script type="text/javascript">' +
            'BigPipe.onPageletArrive(<%= this.json %>);' +
            '</script>',

        quickling: 'BigPipe.onPageletArrive(<%= this.json %>);'
    }
};

util.inherits(BigPipe, Readable);

// don't call this directly.
BigPipe.prototype._read = function (n) {
    if (this.state === status.pending) {
        this.render();
    }
};

// bind pagelet data source.
// accept fucntion and model name.
BigPipe.prototype.bind = function (id, fn) {
    if (fn.length === 1) {
        fn = Promise.promisify(fn);
    }
    this.sources[id] = fn;
    return this;
};

// 当为 quickling 模式的时候调用。
// 设置后，只有在此列表中的 pagelet 才会渲染。
BigPipe.prototype.addQuicklingPagelets = function (pagelets) {
    var me = this;
    pagelets.forEach(function (id) {
        var pageletIDSplit = id.split('@');
        var pageletID = pageletIDSplit.shift();
        me.quicklings.push(pageletID);
        me.parentQuicklings.push.apply(me.parentQuicklings, pageletIDSplit);
    });
};

// 判断是否是 quickling 模式。
BigPipe.prototype.isQuicklingMode = function () {
    return !!(this.quicklings && this.quicklings.length);
};

// 拼写兼容
BigPipe.prototype.isQuickingMode = BigPipe.prototype.isQuicklingMode;

// 添加 pagelet.
BigPipe.prototype.addPagelet = function (obj) {
    // 已经晚了了，不处理。
    if (this.state === status.fulfilled) {
        return false;
    }
    else if (!obj.id) {
        this.emit('error', new Error('Id is required when add pagelet'));
        return;
    }

    if (!this.isQuicklingMode() && obj.mode !== mode.pipeline && obj.mode !== mode.async) {
        // 非 quickling 请求，只接收 pipeline 和 async 模式的 pagelet.
        return false;
    }
    else if (this.isQuicklingMode() && !~this.quicklings.indexOf(obj.id)) {
        // 以下几种情况均需要继续执行
        // 1. 被声明为父Pagelet
        // 2. 先祖Pagelet被请求了

        if (!~this.parentQuicklings.indexOf(obj.id)) {
            var parent, founded;
            // 如果没有被声明为父Pagelet，则检查是否属于先祖Pagelet被请求了的情况

            // 如果是lazy模式，则不寻找先祖Pagelet
            if (obj.lazy) {
                return false;
            }
            // 支持嵌套pagelet时，一次性返回所有pagelet
            parent = obj;
            founded = false;
            // 如果指定的pagelet不在quicklings列表中，检查先祖节点，查看先祖节点是否在列表中，如果在则加载
            // 以此实现Pagelet A contain Pagelet B时，会用一个请求同时返回A与B
            while (parent && parent.parentId) {
                parent = this.map[parent.parentId];
                if (this.isQuicklingWidget(parent) && !!~this.quicklings.indexOf(parent.id)) {
                    founded = true;
                    // 插入为需要输出的Pagelet
                    this.quicklings.push(obj.id);
                    break;
                }
            }
            if (!founded) {
                return false;
            }
        }
    }

    obj.reqID = this.reqID;
    obj.skipAnalysis = this.options.skipAnalysis;

    var pagelet = new Pagelet(obj);
    var self = this;

    this.pagelets.push(pagelet);
    this.map[pagelet.id] = pagelet;
    pagelet.mode === mode.pipeline && this.pipelines.push(pagelet);

    // 转发事件。
    pagelet.on('error', this.emit.bind(this, 'error'));
    [
        'before', 'after',
        'render:before', 'render:after',
        'analyse:before', 'analyse:after'
    ].forEach(function (name) {
        pagelet.on(name, self.emit.bind(self, 'pagelet:' + name, pagelet));
    });

    if (this.state === status.rendering) {
        this.renderPagelet(pagelet);
    }
};

BigPipe.prototype.isQuicklingWidget = function (item) {
    return (item.mode === mode.quickling || item.mode === mode.async || item.mode === mode.pipeline);
};

BigPipe.prototype.render = function () {
    if (!this.pagelets) {
        return;
    }
    var pagelets = this.pagelets.concat();

    this.state = status.rendering;

    pagelets.forEach(this.renderPagelet.bind(this));
    this._checkFinish();
};

BigPipe.prototype.prepareAllSources = function (cb) {
    var sources = this.sources;
    var promiseSources = {};
    var me = this;
    for (var id in sources) {
        if (sources.hasOwnProperty(id)) {
            // 屏蔽所有异常
            promiseSources[id] = sources[id]().catch(function (err) {
                return {
                    BigPipeFailed: true,
                    err: err
                };
            });
        }
    }
    return Promise.props(promiseSources).then(function (data) {
        me.pageletData = data;
    });
};

BigPipe.prototype.renderPagelet = function (pagelet) {
    var sources = this.sources;
    var source = sources[pagelet.id];

    // 屏蔽无法调整为同步的 bigpipe 绑定策略
    // if (!source && typeof sources.all === 'function') {
    //     source = sources.all.bind(null, pagelet.id);
    // }

    // // hook
    // this.emit('pagelet:source', pagelet.id, function (_source) {
    //     source = _source;
    // });

{'fraction_non_alphanumeric': 0.14249132661490171, 'fraction_numerical': 0.0004956220056170494, 'mean_word_length': 2.122775341759092, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2170750', 'n_tokens_mistral': 4113, 'n_tokens_neox': 4150, 'n_words': 749}