<commit_before>var express = require('express'),
    http = require('http'),
    path = require('path'),
    git = require('git-rev'),
    spawn = require('child_process').spawn,
    crypto = require('crypto'),
    fs = require('fs');

var app = express(),
    tag;

app.configure(function(){
    app.set('host', process.env.HOST || '127.0.0.1');
    app.set('port', process.env.PORT || 2652);
    app.use(express.logger());
    app.use(express.compress());
});

git.tag(function (str) {
    tag = str;
});

var bundleBuilder = function(req, res, min){
    var version = req.params.version || tag;
    var query = req.query;

    var filename;
    if (Object.keys(query).length > 0) {
        // User want a custom bundle
        var patterns = Object.keys(query).sort().join('&'),
            shasum = crypto.createHash('sha1');
        shasum.update(tag + '?' + patterns);

        var hex = shasum.digest('hex').substr(0,8);
        filename = 'patterns-' + version + '-' + hex + (min?'.min':'') + '.js';
    } else {
        filename = 'patterns-' + version + (min?'.min':'') + '.js';
    }

    // Generate args that are passed to jam
    var fullname = path.join(__dirname, 'bundles', filename);

    // The CLI interface is:
    // jam compile -i pat1 [-i pat2 [-i pat3]] -o outfile
    var args = [ 'compile' ].concat(Object.keys(query).sort())
        .join(' -i ').split(' ').concat(['-o', fullname, '-v']);

    if (!min)
        args.push('--no-minify');

    if (!fs.existsSync(fullname)) {
        // Not in cache, so generate
        var p = spawn('jam', args);

        p.on('exit', function(code) {
            if (code === 0) {
                return res.download(fullname);
            } else {
                res.send('Something went wrong build the package');
            }
        });
        return;
    }
    return res.download(fullname);
};

app.get('/getBundle', function(req, res){
    var minify = req.query.minify === 'minify';
    delete req.query.minify;
    bundleBuilder(req, res, minify);
});

app.get('/bundles/:name', function(req, res){
    var fullname = path.join(__dirname, 'bundles', req.params.name);
    return res.download(fullname);
});

app.use(express.static(path.join(__dirname)));

http.createServer(app).listen(app.get('port'), app.get('host'), function(){<commit_msg>bundler: take jam from node_modules<commit_after>var express = require('express'),
    http = require('http'),
    path = require('path'),
    git = require('git-rev'),
    spawn = require('child_process').spawn,
    crypto = require('crypto'),
    fs = require('fs');

var app = express(),
    tag;

app.configure(function(){
    app.set('host', process.env.HOST || '127.0.0.1');
    app.set('port', process.env.PORT || 2652);
    app.use(express.logger());
    app.use(express.compress());
});

git.tag(function (str) {
    tag = str;
});

var bundleBuilder = function(req, res, min){
    var version = req.params.version || tag;
    var query = req.query;

    var filename;
    if (Object.keys(query).length > 0) {
        // User want a custom bundle
        var patterns = Object.keys(query).sort().join('&'),
            shasum = crypto.createHash('sha1');
        shasum.update(tag + '?' + patterns);

        var hex = shasum.digest('hex').substr(0,8);
        filename = 'patterns-' + version + '-' + hex + (min?'.min':'') + '.js';
    } else {
        filename = 'patterns-' + version + (min?'.min':'') + '.js';
    }

    // Generate args that are passed to jam
    var fullname = path.join(__dirname, 'bundles', filename);

    // The CLI interface is:
    // jam compile -i pat1 [-i pat2 [-i pat3]] -o outfile
    var args = [ 'compile' ].concat(Object.keys(query).sort())
        .join(' -i ').split(' ').concat(['-o', fullname, '-v']);

    if (!min)
        args.push('--no-minify');

    if (!fs.existsSync(fullname)) {
        // Not in cache, so generate
        var p = spawn('./node_modules/.bin/jam', args);

        p.on('exit', function(code) {
            if (code === 0) {
                return res.download(fullname);
            } else {
                res.send('Something went wrong build the package');
            }
        });
        return;
    }
    return res.download(fullname);
};

app.get('/getBundle', function(req, res){
    var minify = req.query.minify === 'minify';
    delete req.query.minify;
    bundleBuilder(req, res, minify);
});

app.get('/bundles/:name', function(req, res){
    var fullname = path.join(__dirname, 'bundles', req.params.name);
    return res.download(fullname);
});

app.use(express.static(path.join(__dirname)));

http.createServer(app).listen(app.get('port'), app.get('host'), function(){
{'fraction_non_alphanumeric': 0.19179837676206749, 'fraction_numerical': 0.007689021785561726, 'mean_word_length': 2.5857580398162328, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '598055', 'n_tokens_mistral': 1527, 'n_tokens_neox': 1484, 'n_words': 341}