<commit_before>#!/usr/bin/env node
"use strict";

var fs = require('fs'),
    path = require('path'),
    http = require('http'),
    os = require('os'),
    exec = require('child_process').exec,
    execFile = require('child_process').execFile,
    spawn = require('child_process').spawn,
    zlib = require('zlib'),

    express = require('express'),
    winston = module.exports.logger = require('winston'),
    async = require('async'),
    unzip = require('unzip'),
    semver = require('semver'),
    uuid = require('node-uuid'),
    jsonParser = require('body-parser').json(),
    gitBackend = require('git-http-backend'),
    tmp = require('tmp'),
    requestModule = require('request'),
    mime = require('mime'),

    Git = require('./lib/git');

var servicePath = '/emergence/services/sencha-cmd',
    distPath = path.join(servicePath, 'dist'),
    buildsRepoPath = path.join(servicePath, 'builds.git'),

    cmdVersion = '6.2.2',

    parallelWorkers = 4,

    app = express(),
    port = process.argv[2],
    logRe = /^([a-f0-9]{40}) (.*)$/,
    buildBranchRe = /^builds\/([a-f0-9]{40})$/,
    refsRe = /^([a-f0-9]{40}) ([a-z]+)\t(.*)$/,
    urlRe = /^https?:\/\//,
    longTreeRe = /^(\d+) blob ([a-f0-9]{40}) +(\d+)\t(.*)$/,
    gitEnv = {},
    builds = {},
    buildsRepo,
    cmdPath;

if (!port) {
    console.log('port required');
    process.exit(1);
}


// setup git environment variables
gitEnv.GIT_AUTHOR_NAME = gitEnv.GIT_COMMITTER_NAME = 'Sencha Cmd';
gitEnv.GIT_AUTHOR_EMAIL = gitEnv.GIT_COMMITTER_EMAIL = '<EMAIL>@'+os.<EMAIL>();


// register routes
app.get('/builds', function(request, response) {
    response.send(builds);
});

app.get('/builds/:buildId', function(request, response) {
    var buildId = request.params.buildId;

    if (!builds.hasOwnProperty(buildId)) {
        response.status(404).send('build not found');
        return;
    }

    response.send(builds[request.params.buildId]);
});

app.post('/builds', jsonParser, function(request, response) {
    var buildId = uuid.v4();

    winston.info('queued build', buildId);
    builds[buildId] = { status: 'pending', options: request.body };

    response.send(buildId);
});

// setup git backend queue
var gitBackendQueue = async.queue(function(task, callback) {
    var request = task.request,
        response = task.response,
        requestEncoding = request.headers['content-encoding'],
        requestStream;

    winston.info('Passing request to git backend', request.url);

    if (requestEncoding == 'gzip') {
        requestStream = request.pipe(zlib.createGunzip());
    } else if (requestEncoding == 'deflate') {
        requestStream = request.pipe(zlib.createInflate());
    } else {
        requestStream = request;
    }

    requestStream.pipe(gitBackend(request.url, function(error, service) {
        if (error) {
            response.end(error + '\n');
            return callback(error);
        }

        var branch = service.fields.branch,
            action = service.action;

        winston.verbose('handling', action, service.fields);

        response.setHeader('Content-Type', service.type);

        var ps = spawn(service.cmd, service.args.concat(buildsRepoPath));

        ps.on('exit', function(code) {
            if (action != 'push' || !service.fields.refname) {
                return callback();
            }

            winston.info('finished receiving %s: %s -> %s', service.fields.refname, service.fields.last, service.fields.head);

            if (branch && (branch = buildBranchRe.exec(branch))) {
                buildTree(branch[1], callback);
            } else {
                callback();
            }
        });

        ps.stdout.pipe(service.createStream()).pipe(ps.stdin);
    })).pipe(response);
});

app.all('/.git/*', function(request, response) {
    console.log("Receiving git request", request.url);
    gitBackendQueue.push({ request: request, response: response }, function() {
        winston.info('Finished request', request.url);
    });
});


// setup sencha CMD queue
var cmdQueue = async.queue(function(task, callback) {
    winston.info('executing', cmdPath, task.args.join(' '));

    execFile(
        cmdPath,
        task.args,
        task,
        function(error, stdout, stderr) {
            if (error || stderr) {
                return callback({stderr: stderr, stdout: stdout, error: error});
            }

            callback(null, stdout);
        }
    );<commit_msg>Use winston.info instead of console.log<commit_after>#!/usr/bin/env node
"use strict";

var fs = require('fs'),
    path = require('path'),
    http = require('http'),
    os = require('os'),
    exec = require('child_process').exec,
    execFile = require('child_process').execFile,
    spawn = require('child_process').spawn,
    zlib = require('zlib'),

    express = require('express'),
    winston = module.exports.logger = require('winston'),
    async = require('async'),
    unzip = require('unzip'),
    semver = require('semver'),
    uuid = require('node-uuid'),
    jsonParser = require('body-parser').json(),
    gitBackend = require('git-http-backend'),
    tmp = require('tmp'),
    requestModule = require('request'),
    mime = require('mime'),

    Git = require('./lib/git');

var servicePath = '/emergence/services/sencha-cmd',
    distPath = path.join(servicePath, 'dist'),
    buildsRepoPath = path.join(servicePath, 'builds.git'),

    cmdVersion = '6.2.2',

    parallelWorkers = 4,

    app = express(),
    port = process.argv[2],
    logRe = /^([a-f0-9]{40}) (.*)$/,
    buildBranchRe = /^builds\/([a-f0-9]{40})$/,
    refsRe = /^([a-f0-9]{40}) ([a-z]+)\t(.*)$/,
    urlRe = /^https?:\/\//,
    longTreeRe = /^(\d+) blob ([a-f0-9]{40}) +(\d+)\t(.*)$/,
    gitEnv = {},
    builds = {},
    buildsRepo,
    cmdPath;

if (!port) {
    winston.error('port required');
    process.exit(1);
}


// setup git environment variables
gitEnv.GIT_AUTHOR_NAME = gitEnv.GIT_COMMITTER_NAME = 'Sencha Cmd';
gitEnv.GIT_AUTHOR_EMAIL = gitEnv.GIT_COMMITTER_EMAIL = '<EMAIL>();


// register routes
app.get('/builds', function(request, response) {
    response.send(builds);
});

app.get('/builds/:buildId', function(request, response) {
    var buildId = request.params.buildId;

    if (!builds.hasOwnProperty(buildId)) {
        response.status(404).send('build not found');
        return;
    }

    response.send(builds[request.params.buildId]);
});

app.post('/builds', jsonParser, function(request, response) {
    var buildId = uuid.v4();

    winston.info('queued build', buildId);
    builds[buildId] = { status: 'pending', options: request.body };

    response.send(buildId);
});

// setup git backend queue
var gitBackendQueue = async.queue(function(task, callback) {
    var request = task.request,
        response = task.response,
        requestEncoding = request.headers['content-encoding'],
        requestStream;

    winston.info('Passing request to git backend', request.url);

    if (requestEncoding == 'gzip') {
        requestStream = request.pipe(zlib.createGunzip());
    } else if (requestEncoding == 'deflate') {
        requestStream = request.pipe(zlib.createInflate());
    } else {
        requestStream = request;
    }

    requestStream.pipe(gitBackend(request.url, function(error, service) {
        if (error) {
            response.end(error + '\n');
            return callback(error);
        }

        var branch = service.fields.branch,
            action = service.action;

        winston.verbose('handling', action, service.fields);

        response.setHeader('Content-Type', service.type);

        var ps = spawn(service.cmd, service.args.concat(buildsRepoPath));

        ps.on('exit', function(code) {
            if (action != 'push' || !service.fields.refname) {
                return callback();
            }

            winston.info('finished receiving %s: %s -> %s', service.fields.refname, service.fields.last, service.fields.head);

            if (branch && (branch = buildBranchRe.exec(branch))) {
                buildTree(branch[1], callback);
            } else {
                callback();
            }
        });

        ps.stdout.pipe(service.createStream()).pipe(ps.stdin);
    })).pipe(response);
});

app.all('/.git/*', function(request, response) {
    winston.info("Receiving git request", request.url);
    gitBackendQueue.push({ request: request, response: response }, function() {
        winston.info('Finished request', request.url);
    });
});


// setup sencha CMD queue
var cmdQueue = async.queue(function(task, callback) {
    winston.info('executing', cmdPath, task.args.join(' '));

    execFile(
        cmdPath,
        task.args,
        task,
        function(error, stdout, stderr) {
            if (error || stderr) {
                return callback({stderr: stderr, stdout: stdout, error: error});
            }

            callback(null, stdout);
        }
    );
{'fraction_non_alphanumeric': 0.17088465845464726, 'fraction_numerical': 0.006047032474804031, 'mean_word_length': 2.7493702770780857, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1925940', 'n_tokens_mistral': 2826, 'n_tokens_neox': 2760, 'n_words': 530}