<commit_before>var unzip = require('unzip');
var request = require('request');
var archiver = require('archiver');

function factory(root, options) {

    if (!root || typeof root !== 'string') {
        throw new Error('root path required (first argument, e.g.: app.use(github2cocoon("/ludei_zips/", { username: "andyperlitch" }));');
    }

    options = options || {};

    // Store predefined username
    var predefined_user = options.username || false;

    // Ensure that root starts and ends with a slash
    root = root.replace(/^\/*/, '/').replace(/\/*$/, '/');

    // RegEx to test paths
    var exp = predefined_user
        ? new RegExp('^' + root + '([^\/]+)/([^\/]+)') 
        : new RegExp('^' + root + '([^\/]+)/([^\/]+)/([^\/]+)');

    return function(req, res, next) {

        var url = req.url;

        var result = exp.exec(url);

        if (!result) {
            return next();
        }

        // username, repo, branch
        var username, repo, branch;
        if (predefined_user) {
            username = predefined_user;
            repo = result[1];
            branch = result[2];
        } else {
            username = result[1];
            repo = result[2];
            branch = result[3];
        }

        // The archive stream
        var archive = archiver('zip', {
            // Pass options to underlying
            // zlib library
            zlib: {
                // Set this higher than default,
                // cocoon launcher complains a lot
                // otherwise
                chunkSize: 32*1024,
                windowBits: 14, 
                memLevel: 7
            }
        });

        // Will hold name of parent directory
        var rootDirName;

        // Get input, pipe to unzip utility
        var input = request('https://github.com/' + username + '/' + repo + '/archive/' + branch).pipe(unzip.Parse());

        // Pass all but our parent directory to
        // archiver.
        input.on('entry', function(entry) {
            var filepath = entry.path;
            if (!rootDirName) {
                rootDirName = filepath;
                entry.autodrain();    
            } else {
                // Strip name of parent dir
                archive.append(entry, { name: entry.path.replace(rootDirName, '') });
            }
        });

        // Finalize the archive when input has closed.
        input.on('close', function() {
            archive.finalize();
        });

        // Emit to response
        archive.pipe(res);
    }

}
exports = module.exports = factory;<commit_msg>more tweaks to zlib options<commit_after>var unzip = require('unzip');
var request = require('request');
var archiver = require('archiver');

function factory(root, options) {

    if (!root || typeof root !== 'string') {
        throw new Error('root path required (first argument, e.g.: app.use(github2cocoon("/ludei_zips/", { username: "andyperlitch" }));');
    }

    options = options || {};

    // Store predefined username
    var predefined_user = options.username || false;

    // Ensure that root starts and ends with a slash
    root = root.replace(/^\/*/, '/').replace(/\/*$/, '/');

    // RegEx to test paths
    var exp = predefined_user
        ? new RegExp('^' + root + '([^\/]+)/([^\/]+)') 
        : new RegExp('^' + root + '([^\/]+)/([^\/]+)/([^\/]+)');

    return function(req, res, next) {

        var url = req.url;

        var result = exp.exec(url);

        if (!result) {
            return next();
        }

        // username, repo, branch
        var username, repo, branch;
        if (predefined_user) {
            username = predefined_user;
            repo = result[1];
            branch = result[2];
        } else {
            username = result[1];
            repo = result[2];
            branch = result[3];
        }

        // The archive stream
        var archive = archiver('zip', {
            // Pass options to underlying
            // zlib library
            zlib: {
                // Set this higher than default, cocoon launcher complains a lot
                // otherwise. This could use some tweaking
                windowBits: 14, 
                memLevel: 7
            }
        });

        // Will hold name of parent directory
        var rootDirName;

        // Get input, pipe to unzip utility
        var input = request('https://github.com/' + username + '/' + repo + '/archive/' + branch).pipe(unzip.Parse());

        // Pass all but our parent directory to
        // archiver.
        input.on('entry', function(entry) {
            var filepath = entry.path;
            if (!rootDirName) {
                rootDirName = filepath;
                entry.autodrain();    
            } else {
                // Strip name of parent dir
                archive.append(entry, { name: entry.path.replace(rootDirName, '') });
            }
        });

        // Finalize the archive when input has closed.
        input.on('close', function() {
            archive.finalize();
        });

        // Emit to response
        archive.pipe(res);
    }

}
exports = module.exports = factory;
{'fraction_non_alphanumeric': 0.14426992437463643, 'fraction_numerical': 0.004653868528214078, 'mean_word_length': 1.5841683366733468, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1125571', 'n_tokens_mistral': 1453, 'n_tokens_neox': 1368, 'n_words': 392}