<commit_before>"use strict";

var dummy
  , path = require('path')
  , hotplate = require('hotplate')
  , declare = require('simpledeclare')

  , SimpleDbLayer = require( 'simpledblayer' )
  , SimpleSchema = require( 'simpleschema' )
  , JsonRestStores = require( 'jsonreststores' )

  , hotCoreJsonRestStores = require( 'hotCoreJsonRestStores' )
  , hotCoreStoreRegistry = require( 'hotCoreStoreRegistry' )
  , hotCoreServerLogger = require( 'hotCoreServerLogger' )
;

/**
Provides tranport functionalities (SMS an email) to hotplate modules

@module hotCoreTransport
@main hotCoreTransport
@class hotCoreTransport
@static
*/

// Set some sane defaults


hotplate.config.set('hotCoreTransport', {
  subsystems: {

  }
 
});


// var strategyMaker = require( './auth/' + strategyName );


/**
  Initialisation hook. It creates the basic authentication stores
  (Users, UsersStrategies, AuthStrategies, Logins) 

  @event init
  @requires hotCoreJsonRestStores hotCoreStoreRegistry hotCoreServerLogger
*/

hotplate.hotEvents.on( 'stores', 'hotCoreTransport', hotplate.cachable( function( done ){

  var stores = {}

  hotCoreJsonRestStores.get( function( err, s ){
    if( err ) return done( err );

    var HotStore = s.HotStore;
    var HotSchema = s.HotSchema;
    var BasicDbStore = s.BasicDbStore;
    var BasicSchema = s.BasicSchema;
 
    var Messages = declare( [ BasicDbStore ],  {

      schema: new HotSchema({
        id          : { type: 'id',     searchable: true },

        type        : { type: 'string', searchable: true, required: true,  notEmpty: true, trim: 5 },        
        from        : { type: 'string', searchable: true, required: true,  notEmpty: true, trim: 255 },        
        subject     : { type: 'string', searchable: true, required: true,  notEmpty: false, trim: 1024 },
        body        : { type: 'string', searchable: true, required: true,  notEmpty: true, trim: 256000 },
     }),

      idProperty: 'id',

      storeName:  'messages',

      nested: [
        {
          layer: 'messagesTo',
          join: { 'messageId': 'id' },
          type: 'multiple'
        },

        {
          layer: 'messagesToDeliveryLog',
          join: { 'messageId': 'id' },
          type: 'multiple'
        },
      ],

      hotExpose: false,

    });
    stores.messages = new Messages();

    // Status can be: delivering, delivered, undeliverable, dontdeliver
    var MessagesTo = declare( [ BasicDbStore ],  {

      schema: new HotSchema({
        id        : { type: 'id', searchable: true },
        messageId : { type: 'id', searchable: true },

        to        : { type: 'string', searchable: true, required: true, notEmpty: true, trim: 255 },        
        status    : { type: 'string', searchable: true, required: true, notEmpty: true, trim: 15 },
        attempts  : { type: 'number' },
      }),

      idProperty: 'id',

      storeName:  'messagesTo',

      hotExpose: false,

      nested: [
        {
          layer: 'messages',
          join: { 'id' : 'messageId' },
          type: 'lookup',
          parentField: 'messageId',
        },
      ],

    });
    stores.messagesTo = new MessagesTo();

    var MessagesToDeliveryLog = declare( [ BasicDbStore ],  {

      schema: new HotSchema({
        id          : { type: 'id', searchable: true },

        messageToId   : { type: 'id', searchable: true },
        messageId     : { type: 'id', searchable: true }, // NN

        date          : { type: 'date', default: function() { return new Date() } },
        level         : { type: 'number' },
        line          : { type: 'from', searchable: true, required: true, notEmpty: true, trim: 255 },        
      }),

      onlineSearchSchema: new HotSchema({
      }),

      nested: [
        {
          layer: 'messages',
          join: { 'id' : 'messageId' },
          type: 'lookup',
          parentField: 'messageId',
        },
        {
          layer: 'messagesTo',
          join: { 'id' : 'messageToId' },
          type: 'lookup',
          parentField: 'messageToId',
        },
      ],

      idProperty: 'id',

      storeName:  'messagesToDeliveryLog',

      hotExpose: false,

    });
    stores.messagesToDeliveryLog = new MessagesToDeliveryLog();

    done( null, stores );

  });

}))

/**

@event setRoutes
*/
hotplate.hotEvents.on( 'setRoutes', hotplate.cachable( function( app, done ){
  done( null );
}))


<commit_msg>Implemente most of hotCoreTransport, empty stubs for protocol-specific transports<commit_after>"use strict";

var dummy
  , path = require('path')
  , hotplate = require('hotplate')
  , declare = require('simpledeclare')
  , async = require('async')

  , SimpleDbLayer = require( 'simpledblayer' )
  , SimpleSchema = require( 'simpleschema' )
  , JsonRestStores = require( 'jsonreststores' )

  , hotCoreJsonRestStores = require( 'hotCoreJsonRestStores' )
  , hotCoreStoreRegistry = require( 'hotCoreStoreRegistry' )
  , hotCoreServerLogger = require( 'hotCoreServerLogger' )
;

/**
Provides tranport functionalities (SMS an email) to hotplate modules

@module hotCoreTransport
@main hotCoreTransport
@class hotCoreTransport
@static
*/

// Set some sane defaults


var stores = {}

hotplate.config.set('hotCoreTransport', {
  activeTransports: {
    'sms-twilio': true,
    'email-default': true,
    'sms-plivo': true
  } 
});


/**
  @event setRoutes
*/
hotplate.hotEvents.on( 'setRoutes', 'hotCoreTransport', function( app, done ){
  async.eachSeries(
    Object.keys( hotplate.config.get('hotCoreTransport').activeTransports),
    function( transport, cb ){
      var transportModule = require('./transport/' + transport + '.js' );
      // Run setRoutes for relevant module
      transportModule.setRoutes( app, getAllTransportConfig, cb );
    },
    function( err ){
      if( err ) return done( err );
      done( null );
    }
  );
});


/**
  @event run
*/
hotplate.hotEvents.on( 'run', 'hotCoreTransport', function( done ){

  async.eachSeries(
    Object.keys( hotplate.config.get('hotCoreTransport').activeTransports ),
    function( transport, cb ){
      var transportModule = require('./transport/' + transport + '.js' );
      // Run setRoutes for relevant module
      transportModule.run( getAllTransportConfig, cb );
    },
    function( err ){
      if( err ) return done( err );  

      startTransportCron(); // Set the main cron for transport
      done( null );
    }
  );
});


/**
  Initialisation hook. It creates the basic authentication stores
  (Users, UsersStrategies, AuthStrategies, Logins) 

  @event init
  @requires hotCoreJsonRestStores hotCoreStoreRegistry hotCoreServerLogger
*/

function getAllTransportConfig( transport, done ){

  hotplate.hotEvents.emit( 'allTransportConfig', transport, function( err, results ) { 
    if( err ) return done( err );

    // No configuration available: return null
    if( ! Array.isArray( results ) || results.length === 0 ) return done( err, [ ] );

  
    // Return flattened results
    done( null, Array.prototype.concat.apply([], results.onlyResults() ));
    
  });
};

function getTransportConfig( messageTo, done ){

  hotplate.hotEvents.emit( 'transportConfig', messageTo, function( err, results ) { 
    if( err ) return done( err );

    // No configuration available: return null
    if( ! Array.isArray( results ) || results.length === 0 ) return done( err, null )

    // If more than one, issue a warning
    if( results.length > 1 ) hotplate.log("WARNING: more than 1 results returned for ", messageTo );

    // Return the first result
    done( null, results[ 0 ].result );
  });

};


/*

  Input: messageTo

  Where messageTo.message.type can be:

  'sms'
  -----

  config: {
    apiId,
    accountName,
  },  

  'email'
  -------

  config: {
    smtp:{
      systemName,
      systemEmail,
      server,
      port,
      login,
      password
    },
    imap: {
      server
      port,        
      login,
      password,
      pollInterval,
    }
  }
*/


// This function will load the right delivery layer and will pass it the message
function dealWithMessage(  messageTo, config, done ){
  
  var transportModule = require('./transport/' + config.transport + '.js' );

  stores.messagesTo.dbLayer.update( { conditions: { and: [ { field: 'id', type: 'is', value: messageTo.id } ]  } }, { status: 'delivering' }, {}, function( err ){
    if( err ) return done( err );

    // If there was an error, at least try to revert the status.
    // If there is an error from this attempt, it will be added to the original one as err.errReverting      
    if( err ){
      stores.messagesTo.dbLayer.update( { conditions: { and: [ { field: 'id', type:'is', value: messageTo.id } ]  } }, { status: 'todeliver' }, function( errReverting ){
        if( errReverting) err.errReverting = errReverting;
        return done( err );
      });
    }

    // Actually attempts to send the message
    transportModule.sendMessage( messageTo, config, done );

  });
}

// Runs through the message queue, attempting to send messages
function startTransportCron(){

  var inCycle = false;
  setInterval( function(){
    console.log("*********************************************************");
    console.log("*********************************************************");
    console.log("*********************************************************");
    console.log("*********************************************************");
    console.log("*********************************************************");
    console.log("*********************************************************");
    console.log("*********************************************************");
    console.log("*********************************************************");
    console.log("***************************** CRON STARTED **************");
    console.log("*********************************************************");
    console.log("*********************************************************");
    console.log("*********************************************************");
    console.log("*********************************************************");
    console.log("*********************************************************");
    console.log("*********************************************************");
    console.log("*********************************************************");
    console.log("*********************************************************");
    console.log("*********************************************************");

    // Do not run cron twice
    if( inCycle) return;
    inCycle = true;

    hotCoreStoreRegistry.getAllStores( function( err, storeRegistry){
      if( err ) {
        inCycle = false; 
        hotplate.log("Error getting cursor from messagesTo:", err );
        return;
      };

      // Get the cursor for elements in messagesTo...
      stores.messagesTo.dbLayer.select( {conditions: { and: [ { field: 'status', type: 'is', value: 'todeliver' } ]  } }, { useCursor: true , delete: false, children: true }, function( err, cursor, total, grandTotal ){
        if( err ) {
          inCycle = false; 
          hotplate.log("Error getting cursor from messagesTo:", err );
          return;
        }; 

        // ...and go through the cursor asynchromously using async
        var messageTo;
        async.doWhilst(

          function( callback ){
            cursor.next( function( err, i ){
              if( err ) return callback( err );

              messageTo = i;
              if( messageTo !== null ){

                // ******************************************
                // THIS IS WHERE EACH MESSAGE IS EVALUATED
                // ******************************************                  

                // ...get the config for that message from the application
                getTransportConfig( messageTo, function( err, config ){
                  if( err ) { return callback( err ); };

                  // Deal with the message!

                  dealWithMessage( messageTo, config, function( err){
                    if( err ) { return callback( err ); };

                    // All done!
                    callback( null );
                  });


                });

              } else {
                callback( null );                
              }

            });
          },

          function(){ return messageTo != null; },

          function( err ) {

            if( err ) {
              inCycle = false; 
              hotplate.log("Error getting cursor from messagesTo:", err );
              return;
            };

            console.log( "CRON DONE!" );

            inCycle = false;

            return; 
          }
        );


      });

    })    

    // Set inCycle to false to allow this function to work in next timeout()
    
  }, 3000 );
}



hotplate.hotEvents.on( 'stores', 'hotCoreTransport', hotplate.cachable( function( done ){

  hotCoreJsonRestStores.get( function( err, s ){
    if( err ) return done( err );

    var HotStore = s.HotStore;
    var HotSchema = s.HotSchema;
    var BasicDbStore = s.BasicDbStore;
    var BasicSchema = s.BasicSchema;
 
    var Messages = declare( [ BasicDbStore ],  {

      schema: new HotSchema({
        id          : { type: 'id',     searchable: true },

        type        : { type: 'string', searchable: true, required: true,  notEmpty: true, trim: 5 },        
        from        : { type: 'string', searchable: true, required: true,  notEmpty: false, trim: 255 },        
        subject     : { type: 'string', searchable: true, required: false,  notEmpty: false, trim: 1024 },
        body        : { type: 'string', searchable: true, required: true,  notEmpty: true, trim: 256000 },
     }),

      idProperty: 'id',

      storeName:  'messages',

      nested: [
        {
          layer: 'messagesTo',
          join: { 'messageId': 'id' },
          type: 'multiple'
        },

        {
          layer: 'messagesToDeliveryLog',
          join: { 'messageId': 'id' },
          type: 'multiple'
        },
      ],

      hotExpose: false,

    });
    stores.messages = new Messages();

    // Status can be: todeliver, delivering, delivered, undeliverable, dontdeliver
    var MessagesTo = declare( [ BasicDbStore ],  {

      schema: new BasicSchema({
        id        : { type: 'id', searchable: true },
        messageId : { type: 'id', searchable: true },

        to        : { type: 'string', searchable: true, required: true, notEmpty: true, trim: 255 },        
        status    : { type: 'string', searchable: true, required: true, notEmpty: true, trim: 15 },
        attempts  : { type: 'number' },
        added     : { type: 'date', protected: true, default: function() { return new Date() } },
        lastChange: { type: 'date', protected: false, default: function() { return new Date() } },
      }),

      idProperty: 'id',

      storeName:  'messagesTo',

      hotExpose: false,

      nested: [
        {
          layer: 'messages',
          join: { 'id' : 'messageId' },
          type: 'lookup',
          parentField: 'messageId',
        },
      ],

    });
    stores.messagesTo = new MessagesTo();

    var MessagesToDeliveryLog = declare( [ BasicDbStore ],  {

      schema: new BasicSchema({
        id          : { type: 'id', searchable: true },

        messageToId   : { type: 'id', searchable: true },
        messageId     : { type: 'id', searchable: true }, // NN

        date          : { type: 'date', protected: true, default: function() { return new Date() } },
        level         : { type: 'number' },
        line          : { type: 'from', searchable: true, required: true, notEmpty: true, trim: 255 },        
      }),

      onlineSearchSchema: new HotSchema({
      }),

      nested: [
        {
          layer: 'messages',
          join: { 'id' : 'messageId' },
          type: 'lookup',
          parentField: 'messageId',
        },
        {
          layer: 'messagesTo',
          join: { 'id' : 'messageToId' },
          type: 'lookup',
          parentField: 'messageToId',
        },
      ],

      idProperty: 'id',

      storeName:  'messagesToDeliveryLog',

      hotExpose: false,

    });
    stores.messagesToDeliveryLog = new MessagesToDeliveryLog();

    done( null, stores );
  });

}))


{'fraction_non_alphanumeric': 0.192583073251331, 'fraction_numerical': 0.003243375558411358, 'mean_word_length': 2.3764462809917357, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4259654', 'n_tokens_mistral': 4719, 'n_tokens_neox': 4434, 'n_words': 1210}