<commit_before>var needle = require('needle');
var cheerio = require('cheerio');
var config = require('./config');

var Spider = function(opts) {
    this.rules = opts.rules || config;
    this.callback = opts.callback;
    if (opts.run) this.run();
    return this;
};

Spider.prototype.run = function(rules) {
    this.once(rules || this.rules, function(once) {
        this.get(once.url, function(jQuery) {
            this.once(once.rules, function(one) {
                if (one.list) {
                    if (one.rule.url) {
                        this.list(one, jQuery, function(rule) {
                            this.run(rule);
                        });
                    } else {
                        console.error("列表中不含网址规则,无法继续操作!");
                    }
                } else {
                    this.one(one, jQuery, function(data) {
                        this.callback(data);
                    });
                }
            });
        });
    });
};

Spider.prototype.list = function(spider, $, cb) {
    $(spider.list).each(function(i, v) {
        var one = {};
        for (var k in spider.rule) {
            switch (spider.rule[k].type) {
                case 'text':
                    one[k] = v.find(spider.rule[k].text).text();
                    break;

                case 'html':
                    one[k] = v.find(spider.rule[k].text).html();
                    break;

                case 'val':
                    one[k] = v.find(spider.rule[k].text).val();
                    break;

                default:
                    one[k] = v.find(spider.rule[k].text).attr(spider.rule[k].type);
                    break;
            }
        }
        this.once(spider.link, function(once) {
            once.url = one.url;
            cb(once);
        });
    });
};

Spider.prototype.one = function(spider, $, cb) {
    var one = {};
    for (var k in spider.rule) {
        switch (spider.rule[k].type) {
            case 'text':
                one[k] = $(spider.rule[k].text).text();
                break;

            case 'html':
                one[k] = $(spider.rule[k].text).html();
                break;

            case 'val':
                one[k] = $(spider.rule[k].text).val();
                break;

            default:
                one[k] = $(spider.rule[k].text).attr(spider.rule[k].type);
                break;
        }
    }
    cb(one);
};

Spider.prototype.get = function(url, cb) {
    needle.get(url, function(err, res) {
        if (!err) {
            cb(cheerio.load(res.body));
        } else {
            console.log(err);
        }
    });
};

Spider.prototype.more = function(list, cb) {

};

Spider.prototype.once = function(more, cb) {
    if (this.isArray(more)) {
        more.forEach(function(once) {
            cb(once);
        });
    } else {
        cb(more);
    }
};

Spider.prototype.isArray = function(object) {
    return object && typeof object === 'object' && Array == object.constructor;
};


exports = module.exports = Spider;<commit_msg>添加调试输出<commit_after>var needle = require('needle');
var cheerio = require('cheerio');
var config = require('./config');

var Spider = function(opts) {
    this.rules = opts.rules || config;
    this.callback = opts.callback;
    if (opts.run) this.run();
    return this;
};

Spider.prototype.run = function(rules) {
    this.once(rules || this.rules, function(once) {
        this.get(once.url, function(jQuery) {
            this.once(once.rules, function(one) {
                if (one.list) {
                    if (one.rule.url) {
                        this.list(one, jQuery, function(rule) {
                            console.info("[+] [" + once.url + "]运行规则: ", rule);
                            this.run(rule);
                        });
                    } else {
                        console.error("[-] [" + once.url + "]列表中不含网址规则,无法继续操作!");
                    }
                } else {
                    this.one(one, jQuery, function(data) {
                        console.info("[+] [" + once.url + "]获取一条数据: ", data);
                        this.callback(data);
                    });
                }
            });
        });
    });
};

Spider.prototype.list = function(spider, $, cb) {
    $(spider.list).each(function(i, v) {
        var one = {};
        for (var k in spider.rule) {
            switch (spider.rule[k].type) {
                case 'text':
                    one[k] = v.find(spider.rule[k].text).text();
                    break;

                case 'html':
                    one[k] = v.find(spider.rule[k].text).html();
                    break;

                case 'val':
                    one[k] = v.find(spider.rule[k].text).val();
                    break;

                default:
                    one[k] = v.find(spider.rule[k].text).attr(spider.rule[k].type);
                    break;
            }
        }
        this.once(spider.link, function(once) {
            once.url = one.url;
            cb(once);
        });
    });
};

Spider.prototype.one = function(spider, $, cb) {
    var one = {};
    for (var k in spider.rule) {
        switch (spider.rule[k].type) {
            case 'text':
                one[k] = $(spider.rule[k].text).text();
                break;

            case 'html':
                one[k] = $(spider.rule[k].text).html();
                break;

            case 'val':
                one[k] = $(spider.rule[k].text).val();
                break;

            default:
                one[k] = $(spider.rule[k].text).attr(spider.rule[k].type);
                break;
        }
    }
    cb(one);
};

Spider.prototype.get = function(url, cb) {
    needle.get(url, function(err, res) {
        if (!err) {
            try {
                cb(cheerio.load(res.body));
            } catch (e) {
                console.error("[-] [" + url + "]页面处理失败: ", e);
            }
        } else {
            console.log("[-] [" + url + "]网页访问错误: ", err);
        }
    });
};

Spider.prototype.more = function(list, cb) {

};

Spider.prototype.once = function(more, cb) {
    if (this.isArray(more)) {
        more.forEach(function(once) {
            cb(once);
        });
    } else {
        cb(more);
    }
};

Spider.prototype.isArray = function(object) {
    return object && typeof object === 'object' && Array == object.constructor;
};


exports = module.exports = Spider;
{'fraction_non_alphanumeric': 0.160755768932941, 'fraction_numerical': 0.0, 'mean_word_length': 1.3039600428112736, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5338102', 'n_tokens_mistral': 2017, 'n_tokens_neox': 1968, 'n_words': 321}