<commit_before>var Knox = require("knox");
var Async = require("async");

/**
 * Useful for overriding `knox` client methods.
 */
var noop = function(source, destination, headers, callback) {
    process.nextTick(function() {
        callback(null, {});
    });
};

/**
 * Returns a rate-limited `knox` client for `bucket`. If `dryRun` is passed,
 * return a non-client.
 */
module.exports = function(options) {
    var client = Knox.createClient({
        bucket: options.bucket,
        key: options.key,
        secret: options.secret,
    });

    // @jb: With Node v0.10.x, I've seen the 'error' event fire from `knox` requests
    //      after the error handler has been cleaned up. Bind our own handler to
    //      to track the trouble.
    var clientRequest = client.request;
    client.request = function(method, filename, headers) {
        var req = clientRequest.apply(client, arguments);
        req.on('error', function(err) {
            console.log('ERROR: %s %s %s', method, filename, err);
        });
        return req;
    };

    if (options.dryRun) {
        client.putFile = client.copyFile = noop;
        return client;
    }

    // Limit `putFile` to avoid opening too many files.
    var clientPutFile = client.putFile;

    var putFileQueue = Async.queue(function(task, callback) {
        var done = function(err, res) {
            return callback(err, res);
        };

        var request = clientPutFile.apply(client, [task.src, task.filename, task.headers, done]);
    }, 2);

    client.putFile = function(src, filename, headers, callback) {
        putFileQueue.push({
            src: src,
            filename: filename,
            headers: headers,
            callback: callback
        });
    };
<commit_msg>putFileQueue correctly calls the passed callback.<commit_after>var Knox = require("knox");
var Async = require("async");

/**
 * Useful for overriding `knox` client methods.
 */
var noop = function(source, destination, headers, callback) {
    process.nextTick(function() {
        callback(null, {});
    });
};

/**
 * Returns a rate-limited `knox` client for `bucket`. If `dryRun` is passed,
 * return a non-client.
 */
module.exports = function(options) {
    var client = Knox.createClient({
        bucket: options.bucket,
        key: options.key,
        secret: options.secret,
    });

    // @jb: With Node v0.10.x, I've seen the 'error' event fire from `knox` requests
    //      after the error handler has been cleaned up. Bind our own handler to
    //      to track the trouble.
    var clientRequest = client.request;
    client.request = function(method, filename, headers) {
        var req = clientRequest.apply(client, arguments);
        req.on('error', function(err) {
            console.log('ERROR: %s %s %s', method, filename, err);
        });
        return req;
    };

    if (options.dryRun) {
        client.putFile = client.copyFile = noop;
        return client;
    }

    // Limit `putFile` to avoid opening too many files.
    var clientPutFile = client.putFile;

    var putFileQueue = Async.queue(function(task, callback) {
        var done = function(err, res) {
            callback();
            task.callback(err, res)
        };

        var request = clientPutFile.apply(client, [task.src, task.filename, task.headers, done]);
    }, 2);

    client.putFile = function(src, filename, headers, callback) {
        putFileQueue.push({
            src: src,
            filename: filename,
            headers: headers,
            callback: callback
        });
    };

{'fraction_non_alphanumeric': 0.1271352562307477, 'fraction_numerical': 0.0022402688322598714, 'mean_word_length': 2.338317757009346, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1819301', 'n_tokens_mistral': 1043, 'n_tokens_neox': 1008, 'n_words': 301}