<commit_before>#! /usr/bin/env node
"use strict";
// this is expected to run in nodejs
/* global require, module */
/* jshint esnext:true */

const express = require('express')
  , http = require('http')
  , socketio = require('socket.io')
  , rethinkdbdash = require('rethinkdbdash')
  , path = require('path')
  , bodyParser = require('body-parser')
  , StringDecoder = require('string_decoder').StringDecoder
  // https://github.com/squaremo/amqp.node
  , amqplib = require('amqplib')
  ;

var ROOT_PATH = __dirname.split(path.sep).slice(0, -1).join(path.sep);

function Logging(loglevel) {
    this._numericLoglevel = this._levels[loglevel];
}
Logging.prototype._levels = {};

([
  , ['DEBUG', 10, console.info]
  , ['INFO', 20, console.info]
  , ['WARNING', 30, console.warn]
  , ['ERROR', 40, console.error]
  , ['CRITICAL', 50, console.error]
]).forEach(function(setup) {
    // method names: debug, info, warning, etc..
    var loglevel = setup[0]
      , method = loglevel.toLowerCase()
      , numeric = setup[1]
      , log = setup[2]
      ;
    Logging.prototype._levels[loglevel] = numeric;
    Logging.prototype[method] = function() {
        if(numeric < this._numericLoglevel)
            return;
        var args = [loglevel], i, l;
        for(i=0,l=arguments.length;i<l;i++)
            args.push(arguments[i]);
        log.apply(null, args);
    };
});

/**
 * Using a class here, so state variables are managed not on module level
 * Initialization starts the server.
 *
 */
function Server(logging, portNum,  amqpSetup, rethinkSetup) {
    this._log = logging;
    this._portNum = portNum;
    this._dbName = rethinkSetup.db;
    this._dbTable = 'draganddrop';
    this._app = express();
    this._server = http.createServer(this._app);
    this._sio = socketio(this._server);

    this._r = rethinkdbdash(rethinkSetup);

    this._amqpConnection = null;
    this._queueName = 'drag_and_drop_queue';
    // Start serving when the database and rabbitmq queue is ready
    Promise.all([
                 this._initDB()
               , this._initAmqp()
               ])
    .then(this._listen.bind(this))
    .catch(function(err) {
        this._log.error('Can\'t initialize server.', err);
        process.exit(1);
    }.bind(this));

    this._app.get('/', this.fbDNDIndex.bind(this));
    this._app.use('/static', express.static('lib/static'));
    this._app.get('/report/:docid', this.fbDNDReport.bind(this));

    this._app.use('/runchecks', bodyParser.raw(
                    {type: 'application/octet-stream', limit: '15mb'}));
    this._app.post('/runchecks', this.fbDNDReceive.bind(this));
    this._sio.on('connection', this.fbDNDSocketConnect.bind(this));
}

var _p = Server.prototype;

_p._initDB = function() {
    var createDatabase, createTable;

    createDatabase = function() {
        return this._r.dbCreate(this._dbName)
            .run()
            //.then(function(response) {/* pass */})
            .error(function(err){
                if (err.message.indexOf('already exists') !== -1)
                    return;
                throw err;
            });

    }.bind(this);

    createTable = function() {
        return this._r.tableCreate(this._dbTable)
            .run()
            //.then(function(response) {/* pass */})
            .error(function(err){
            if (err.message.indexOf('already exists') !== -1)
                return;
            throw err;
        });
    }.bind(this);

    return createDatabase()
        .then(createTable)
        .error(function(err) {
            // It's not an error if the table already exists
            this._log.warning('Error while initializing database.', err);
            throw err;
        }.bind(this));
};

_p._initAmqp = function() {
    return amqplib.connect('amqp://' + amqpSetup.host)
            .then(function(connection) {
                process.once('SIGINT', connection.close.bind(connection));

                this._amqpConnection = connection;
            }.bind(this))
            .catch(function(err) {
                this._log.error('Error while connecting to queue.', err);
                throw err;
            }.bind(this));
};

_p._listen = function() {
    this._server.listen(this._portNum);
    this._log.info('Listening to port', this._portNum);
};

/**
 * GET
 *
 * CLIENT
 *  : say hello
 *  : initiate the d'n'd interface to post fonts
 *        on success
 *        change url to docid url
 *        init socketio connection for docid (with answer from post)
 */
_p.fbDNDIndex = function(req, res) {
    return res.sendFile('static/html/drag-and-drop-client.html',
                                                    { root: ROOT_PATH});
};


/**
 * This is almost a copy of the clients Controller.unpack method
 */
_p.unpack = function(data) {
    var offset = 0, head, json, font, decoder
        , result = []
        ;

    while(offset < data.byteLength) {
        head = new DataView(data.buffer, offset, 8);
        head = [head.getUint32(0, true), head.getUint32(4, true)];
        offset += 8;

        json = new Uint8Array(data.buffer, offset, head[0]);
        offset += json.byteLength;
        decoder = new StringDecoder('utf8');
        json = JSON.parse(decoder.write(Buffer.from(json)));

        font = new Uint8Array(data.buffer, offset, head[1]);
        offset += font.byteLength;

        result.push([json, font]);
    }
    return result;
};

_p._query = function() {
    return this._r.table(this._dbTable);
};

/**
 * GET docid
 *
 * Loads the report page client, which receives its information via socket.io
 *
 * SERVER
 *   : test if docid exists
 * CLIENT
 *   : same as fbDNDIndex on success
 */
_p.fbDNDReport = function(req, res, next) {
    var docid = req.param('docid');
    return this._query().getAll(docid).count()
        .then(function(found) {
            if(found)
                // This is the same client as the index page,
                // after the docid was returned
                return this.fbDNDIndex.bind(this, req, res, next);
            // answer 404: NotFound
            return res.status(404).send('Not found');
        }).catch(next);
};

/**
 * Almost straight copy from the clients Controller.js
 */
function mergeArrays(arrays) {
    var jobSize = arrays.reduce(function(prev, cur) {
                                return prev + cur.byteLength; }, 0)
      , result, i, l, offset
      ;
    result = new Buffer(jobSize);
    for(i=0,l=arrays.length,offset=0; i<l;offset+=arrays[i].byteLength, i++)
        result.set(new Buffer(arrays[i].buffer), offset);
    return result;
}
<commit_msg>Fix fbDNDReport<commit_after>#! /usr/bin/env node
"use strict";
// this is expected to run in nodejs
/* global require, module */
/* jshint esnext:true */

const express = require('express')
  , http = require('http')
  , socketio = require('socket.io')
  , rethinkdbdash = require('rethinkdbdash')
  , path = require('path')
  , bodyParser = require('body-parser')
  , StringDecoder = require('string_decoder').StringDecoder
  // https://github.com/squaremo/amqp.node
  , amqplib = require('amqplib')
  ;

var ROOT_PATH = __dirname.split(path.sep).slice(0, -1).join(path.sep);

function Logging(loglevel) {
    this._numericLoglevel = this._levels[loglevel];
}
Logging.prototype._levels = {};

([
  , ['DEBUG', 10, console.info]
  , ['INFO', 20, console.info]
  , ['WARNING', 30, console.warn]
  , ['ERROR', 40, console.error]
  , ['CRITICAL', 50, console.error]
]).forEach(function(setup) {
    // method names: debug, info, warning, etc..
    var loglevel = setup[0]
      , method = loglevel.toLowerCase()
      , numeric = setup[1]
      , log = setup[2]
      ;
    Logging.prototype._levels[loglevel] = numeric;
    Logging.prototype[method] = function() {
        if(numeric < this._numericLoglevel)
            return;
        var args = [loglevel], i, l;
        for(i=0,l=arguments.length;i<l;i++)
            args.push(arguments[i]);
        log.apply(null, args);
    };
});

/**
 * Using a class here, so state variables are managed not on module level
 * Initialization starts the server.
 *
 */
function Server(logging, portNum,  amqpSetup, rethinkSetup) {
    this._log = logging;
    this._portNum = portNum;
    this._dbName = rethinkSetup.db;
    this._dbTable = 'draganddrop';
    this._app = express();
    this._server = http.createServer(this._app);
    this._sio = socketio(this._server);

    this._r = rethinkdbdash(rethinkSetup);

    this._amqpConnection = null;
    this._queueName = 'drag_and_drop_queue';
    // Start serving when the database and rabbitmq queue is ready
    Promise.all([
                 this._initDB()
               , this._initAmqp()
               ])
    .then(this._listen.bind(this))
    .catch(function(err) {
        this._log.error('Can\'t initialize server.', err);
        process.exit(1);
    }.bind(this));

    this._app.get('/', this.fbDNDIndex.bind(this));
    this._app.use('/static', express.static('lib/static'));
    this._app.get('/report/:docid', this.fbDNDReport.bind(this));

    this._app.use('/runchecks', bodyParser.raw(
                    {type: 'application/octet-stream', limit: '15mb'}));
    this._app.post('/runchecks', this.fbDNDReceive.bind(this));
    this._sio.on('connection', this.fbDNDSocketConnect.bind(this));
}

var _p = Server.prototype;

_p._initDB = function() {
    var createDatabase, createTable;

    createDatabase = function() {
        return this._r.dbCreate(this._dbName)
            .run()
            //.then(function(response) {/* pass */})
            .error(function(err){
                if (err.message.indexOf('already exists') !== -1)
                    return;
                throw err;
            });

    }.bind(this);

    createTable = function() {
        return this._r.tableCreate(this._dbTable)
            .run()
            //.then(function(response) {/* pass */})
            .error(function(err){
            if (err.message.indexOf('already exists') !== -1)
                return;
            throw err;
        });
    }.bind(this);

    return createDatabase()
        .then(createTable)
        .error(function(err) {
            // It's not an error if the table already exists
            this._log.warning('Error while initializing database.', err);
            throw err;
        }.bind(this));
};

_p._initAmqp = function() {
    return amqplib.connect('amqp://' + amqpSetup.host)
            .then(function(connection) {
                process.once('SIGINT', connection.close.bind(connection));

                this._amqpConnection = connection;
            }.bind(this))
            .catch(function(err) {
                this._log.error('Error while connecting to queue.', err);
                throw err;
            }.bind(this));
};

_p._listen = function() {
    this._server.listen(this._portNum);
    this._log.info('Listening to port', this._portNum);
};

/**
 * GET
 *
 * CLIENT
 *  : say hello
 *  : initiate the d'n'd interface to post fonts
 *        on success
 *        change url to docid url
 *        init socketio connection for docid (with answer from post)
 */
_p.fbDNDIndex = function(req, res) {
    return res.sendFile('static/html/drag-and-drop-client.html',
                                                    { root: ROOT_PATH});
};


/**
 * This is almost a copy of the clients Controller.unpack method
 */
_p.unpack = function(data) {
    var offset = 0, head, json, font, decoder
        , result = []
        ;

    while(offset < data.byteLength) {
        head = new DataView(data.buffer, offset, 8);
        head = [head.getUint32(0, true), head.getUint32(4, true)];
        offset += 8;

        json = new Uint8Array(data.buffer, offset, head[0]);
        offset += json.byteLength;
        decoder = new StringDecoder('utf8');
        json = JSON.parse(decoder.write(Buffer.from(json)));

        font = new Uint8Array(data.buffer, offset, head[1]);
        offset += font.byteLength;

        result.push([json, font]);
    }
    return result;
};

_p._query = function() {
    return this._r.table(this._dbTable);
};

/**
 * GET docid
 *
 * Loads the report page client, which receives its information via socket.io
 *
 * SERVER
 *   : test if docid exists
 * CLIENT
 *   : same as fbDNDIndex on success
 */
_p.fbDNDReport = function(req, res, next) {
    var docid = req.param('docid');
    return this._query().getAll(docid).count()
        .then(function(found) {
            if(found)
                // This is the same client as the index page,
                // after the docid was returned
                return this.fbDNDIndex(req, res, next);
            // answer 404: NotFound
            return res.status(404).send('Not found');
        }.bind(this))
        .catch(next);
};

/**
 * Almost straight copy from the clients Controller.js
 */
function mergeArrays(arrays) {
    var jobSize = arrays.reduce(function(prev, cur) {
                                return prev + cur.byteLength; }, 0)
      , result, i, l, offset
      ;
    result = new Buffer(jobSize);
    for(i=0,l=arrays.length,offset=0; i<l;offset+=arrays[i].byteLength, i++)
        result.set(new Buffer(arrays[i].buffer), offset);
    return result;
}

{'fraction_non_alphanumeric': 0.14702547433668456, 'fraction_numerical': 0.006652052309320432, 'mean_word_length': 2.3174523570712138, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1073523', 'n_tokens_mistral': 4168, 'n_tokens_neox': 4012, 'n_words': 895}