<commit_before>/**
 * Test run for scanner app
 * @type {*|exports|module.exports}
 */
var fs =  require('fs');
var Q = require('q');
var path = require('path');
//console.log("HEEEEYYY", process.env.OPENSHIFT_DATA_DIR + "nmap-7.00/nmap");
//TODO: Move to Core utils
var ajv = require('ajv')();
ajv.addSchema(require('../../schemas/nmap.json'), 'nmap');

var App = require('../App');

//Data is chunked into 1024 targets, then 4 sets of 256 which each of is split into sets of 16
var nmap = require('libnmap');
//App.Store.kue.app.listen(8080);
App.Store.queue.process('418b3a196f9a8a9fb526e9be3f5e6b35', function(job, done){
    job.on('failed', function() {
        job.state('inactive').save();
    });

    var targets = job.data.targets;
    var total = targets.length;
    targets = App.util.chunk(targets, 256);
    var indx = 0;
    App.util.async.eachSeries(targets, function(midTargetBlock, targetsCB){
        //console.log(targetBlock);


        midTargetBlock = App.util.chunk(midTargetBlock, 16);
        console.log('Large Block');
        App.util.async.eachSeries(midTargetBlock, function(targetBlock, midTargetBlockCB){

            isItUp(targetBlock).then(function(report){
                //console.log(report);
                var targetblocksuccess = 0;
                for (var addr in report) {
                    job.progress(indx, total);
                    indx++;
                    //TODO: Add validation
                    App.Store.conn.targets.merge(addr, report[addr], function(err, res){
                        if(err){
                            job.log(addr,report[addr]);<commit_msg>Deploy<commit_after>/**
 * Test run for scanner app
 * @type {*|exports|module.exports}
 */
var fs =  require('fs');
var cradle = require('cradle');
var Q = require('q');
var path = require('path');
//console.log("HEEEEYYY", process.env.OPENSHIFT_DATA_DIR + "nmap-7.00/nmap");
//TODO: Move to Core utils
var ajv = require('ajv')();
ajv.addSchema(require('../../schemas/nmap.json'), 'nmap');
var connection = new(cradle.Connection)('https://phearnet.cloudant.com', 443, {
    cache: true,
    raw: false,
    forceSave: true,
    auth: {
        username: process.env.COUCH_KEY,
        password: <PASSWORD>
    }
});

var db = c.database('targets');
var App = require('../App');

//Data is chunked into 1024 targets, then 4 sets of 256 which each of is split into sets of 16
var nmap = require('libnmap');
//App.Store.kue.app.listen(8080);
App.Store.queue.process('418b3a196f9a8a9fb526e9be3f5e6b35', function(job, done){
    job.on('failed', function() {
        job.state('inactive').save();
    });

    var targets = job.data.targets;
    var total = targets.length;
    targets = App.util.chunk(targets, 256);
    var indx = 0;
    App.util.async.eachSeries(targets, function(midTargetBlock, targetsCB){
        //console.log(targetBlock);


        midTargetBlock = App.util.chunk(midTargetBlock, 16);
        console.log('Large Block');
        App.util.async.eachSeries(midTargetBlock, function(targetBlock, midTargetBlockCB){

            isItUp(targetBlock).then(function(report){
                //console.log(report);
                var targetblocksuccess = 0;
                for (var addr in report) {
                    job.progress(indx, total);
                    indx++;
                    //TODO: Add validation
                    db.merge(addr, report[addr], function(err, res){
                        if(err){
                            job.log(addr,report[addr]);
{'fraction_non_alphanumeric': 0.1517476555839727, 'fraction_numerical': 0.02529127593066212, 'mean_word_length': 2.52, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2979890', 'n_tokens_mistral': 1174, 'n_tokens_neox': 1080, 'n_words': 238}