<commit_before>var http    = require('http'),
    cheerio = require('cheerio'),
    _       = require('lodash'),
    async   = require('async');

function ScraperBoknett() {}

_.extend(ScraperBoknett.prototype, {
    getReviewPage: function(offset, callback) {
        offset   = offset || 0;
        callback = callback || function() {};

        var data = '';

        var options = {
            host: 'www.boknett.no',
            path: '/Anmeldelser/(offset)/' + offset
        };

        http.get(options, function(res) {
            res.on('data', function(chunk) {
                data += chunk;
            }).on('end', function() {
                callback(null, data);
            }).on('error', function(err) {
                callback(true, []);
            });
        });
    },

    normalizeSource: function(source) {
        return source.toLowerCase().replace(" ", "-");
    },

    getReviewLinksFromPage: function(html) {
        var $       = cheerio.load(html),
            reviews = $('.reviews .review');

        if (!reviews.length) {
            return false;
        }

        return _.map(reviews, function(review) {
            var review = $(review);

            return {
                url: review.find('a').attr('href'),
                author: review.find('h2').text(),
                title: review.find('h1').text(),
                source: this.normalizeSource(review.find('.source em').text())
            };
        }.bind(this));
    },

    getReviewUrls: function(limit, sources, callback) {
        var highestOffset = 0,
            result        = [],
            resultLength  = 0,
            stop          = false;

        // Set up the queue and worker
        var queue = async.queue(function(task, done) {
            this.getReviewPage(task.offset, function(err, res) {
                var reviews = this.getReviewLinksFromPage(res);

                if (!reviews) {
                    stop = true;
                    queue.empty();
                    done();

                    return;
                }

                result = result.concat(_.filter(reviews, function(review) {
                    return _.contains(sources, review.source);
                }));

                resultLength = result.length;

                done();
            }.bind(this));
        }.bind(this), 10);

        // Callback called when queue is emptied by the worker
        queue.empty = function() {
            if (stop || resultLength >= limit) {
                if (stop) {
                    queue.kill();
                }

                if (resultLength >= limit) {
                    result = result.slice(0, limit);
                }

                callback(null, result);
            } else if (resultLength < limit) {
                for (var i = 0; i < 10; i++) {
                    queue.push({ offset: (highestOffset += 30)});
                }
            }
        };

        // Add ten tasks to the queue
        for (var i = 0; i < 10; i++) {
            queue.push({ offset: (highestOffset += 30)});
        }
    }
});

module.exports = ScraperBoknett;
<commit_msg>Revert "Rewrote the logic. No longer using stone age timeout, but the empty callback triggered by async.queue".<commit_after>var http    = require('http'),
    cheerio = require('cheerio'),
    _       = require('lodash'),
    async   = require('async');

function ScraperBoknett() {}

_.extend(ScraperBoknett.prototype, {
    getReviewPage: function(offset, callback) {
        offset   = offset || 0;
        callback = callback || function() {};

        var data = '';

        var options = {
            host: 'www.boknett.no',
            path: '/Anmeldelser/(offset)/' + offset
        };

        http.get(options, function(res) {
            res.on('data', function(chunk) {
                data += chunk;
            }).on('end', function() {
                callback(null, data);
            }).on('error', function(err) {
                callback(true, []);
            });
        });
    },

    normalizeSource: function(source) {
        return source.toLowerCase().replace(" ", "-");
    },

    getReviewLinksFromPage: function(html) {
        var $       = cheerio.load(html),
            reviews = $('.reviews .review');

        if (!reviews.length) {
            return false;
        }

        return _.map(reviews, function(review) {
            var review = $(review);

            return {
                url: review.find('a').attr('href'),
                author: review.find('h2').text(),
                title: review.find('h1').text(),
                source: this.normalizeSource(review.find('.source em').text())
            };
        }.bind(this));
    },

    getReviewUrls: function(limit, sources, callback) {
        var highestOffset = 0,
            result        = [],
            resultLength  = 0,
            stop          = false;

        var queue = async.queue(function(task, done) {
            this.getReviewPage(task.offset, function(err, res) {
                var reviews = this.getReviewLinksFromPage(res);

                if (!reviews) {
                    stop = true;
                    queue.kill();
                    done();

                    return;
                }

                result = result.concat(_.filter(reviews, function(review) {
                    return _.contains(sources, review.source);
                }));

                resultLength = result.length;

                done();
            }.bind(this));
        }.bind(this), 10);

        var intervalId = setInterval(function() {
            if (!queue.idle()) {
                return;
            }

            if (stop) {
                clearInterval(intervalId);
                callback(null, result);
            } else if (resultLength >= limit) {
                clearInterval(intervalId);
                callback(null, result.slice(0, limit));
            } else if (resultLength < limit) {
                for (var i = 0; i < 10; i++) {
                    queue.push({ offset: (highestOffset += 30)});
                }
            }
        }, 50);
    }
});

module.exports = ScraperBoknett;

{'fraction_non_alphanumeric': 0.13673008706868753, 'fraction_numerical': 0.005320864237342793, 'mean_word_length': 1.3748085758039816, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5965902', 'n_tokens_mistral': 1715, 'n_tokens_neox': 1591, 'n_words': 349}