<commit_before>/*

https://docs.aws.amazon.com/ses/latest/DeveloperGuide/mailbox-simulator.html
<EMAIL> (Successful email)
<EMAIL> (Soft bounce)
<EMAIL> (Out of office response from ISP)
<EMAIL> (Complaint from ISP)
<EMAIL> (Hard bounce as target email is on Amazon's suppression list)

API endpoints
US East (N. Virginia)	us-east-1
US West (Oregon)	us-west-2
EU (Ireland)	eu-west-1

Successful response has form:
{ ResponseMetadata: { RequestId: 'e8a3d6b4-94fd-11z6-afac-757cax279ap5' },
  MessageId: '01020157a1261241-90a5e1cd-3a5z-4sb7-1r41-957a4cae8e58-000000' }
Throttling error:
{ Throttling: Maximum sending rate exceeded.
    at ...
  message: 'Maximum sending rate exceeded.',
  code: 'Throttling',
  time: 2016-10-18T07:50:11.286Z,
  requestId: '7f2d1781-9507-11e6-8885-675506266ba7',
  statusCode: 400,
  retryable: true }

*/

const AWS = require('aws-sdk');
const Promise = require('bluebird'); // Bluebird is signficantly faster than native promises

const wrapLink = require('./analytics').wrapLink;
const insertUnsubscribeLink = require('./analytics').insertUnsubscribeLink;
const insertTrackingPixel = require('./analytics').insertTrackingPixel;
const mailMerge = require('./mail-merge');

const db = require('../../../../models');
const AmazonEmail = require('./amazon');
const CampaignAnalyticsLink = require('../../../../models').campaignanalyticslink;
const CampaignAnalyticsOpen = require('../../../../models').campaignanalyticsopen;

module.exports = (generator, listSubscriberModel, redis, campaignAndListInfo, amazonAccountInfo) => {
  /*
    # Send emails via Amazon SES' API #
    This function focuses on a tradeoff between speed and memory usage. It does the following:
      1. Get all ids (and only ids) of listsubscriber rows containing emails to be sent. We don't want any more info
         than this else memory use will be very large.
      2. Prepare emails to be sent, creating entries in various tables and configuring the email itself. Do this in
         batches.
      3. Sen
  */

  // Unpack args
  const {
    campaignInfo, // See object passed by send-campaign.js, contains info about the campaigns table
    // totalListSubscribers // num - the total number of list subscribers to be emailed
  } = campaignAndListInfo;

  const {
    accessKey, // Amazon access key - str
    secretKey, // Amazon secret key - str
    region, // Amazon region - str
    whiteLabelUrl, // URL of a service that serves this app - str
    quotas // Amazon limits - obj { AvailableToday, MaxSendRate }
  } = amazonAccountInfo;

  // Constants
  const isDevMode = process.env.NODE_ENV === 'development';
  const rateLimit = process.env.DEV_SEND_RATE || quotas.MaxSendRate; // No. of email we can send p/s as established by Amazon
  const rateLimitTimesFive = rateLimit * 5; // Arbitrary quantity of emails to store in buffer
  const emailBuffer = []; // This stores the current buffer of emails to be sent

  const ses = isDevMode
    ? new AWS.SES({  // Dev mode
        apiVersion: '2010-12-01',
        // convertResponseTypes: false,
        maxRetries: 0,
        accessKeyId: accessKey,
        secretAccessKey:
        secretKey,
        region,
        endpoint: 'http://localhost:9999'
      })
    : new AWS.SES({ // Prod mode
        apiVersion: '2010-12-01',
        accessKeyId: accessKey,
        secretAccessKey: secretKey,
        region
      });

  // Vars
  let databaseIsWorking = false; // This is a flag that will ensure that the db does not get assigned too much work
  let shouldSend = true; // Another flag that indicates whether or not a campaign should send

  const Producer = require('../message-queue/amazon/producer')();
  const Receiver = require('../message-queue/amazon/receiver')(ses, rateLimit, campaignInfo);

  (async function sendEmailsViaAmazon() {
    // 1. First we'll get an array containing the ids of all users we're going to email.
    //// This way we have the certainty of knowing who to email without the memory footprint
    //// associated with including all other columns.
    console.log(`\nEmail campaign send initiated. Sending ${rateLimit} per second.\n`); // eslint-disable-line
    console.log('Preparing to send email campaign. Getting list subscribers ids ...'); // eslint-disable-line
    const arrayOfIds = await getListSubscriberIds(); // Returns ids from listsubscribers [1,2,3, ... etc]
    const initialBuffer = await fillInitialBuffer(arrayOfIds); // Initialise the buffer
    emailBuffer.push(...initialBuffer);
    if (isDevMode) console.log(`Filled emailBuffer with ${emailBuffer.length} emails`); // eslint-disable-line
    // 2. Listen for cancellation event
    redis.subscriber.on('message', (channel, campaignId) => {
      if (campaignId == campaignInfo.campaignId) {
        cancel();
      }
    });
    redis.subscriber.subscribe('stop-campaign-sending');
    // 3. Fill the buffer with an initial set of emails
    emailProducer(arrayOfIds, rateLimit);
  })();

  function getListSubscriberIds() {
    return listSubscriberModel.findAll({
      where: {
        listId: campaignInfo.listId,
        subscribed: true
      },
      include: [{
        model: db.campaignsubscriber,
        where: { sent: false }
      }],
      attributes: [
        'id'
      ],
      raw: true
    })
    .then(instances => { // [{id: 1}, {id: 2}, etc]
      const plainArrayOfIdNumbers = instances.map(x => x.id);
      return plainArrayOfIdNumbers;
    })
    .catch(err => {
       console.log(err); //eslint-disable-line
    });
  }

  function getEmailsAndCampaignInfo(arrayOfIds, rateLimit) {
    // This function is tasked with getting and therefore preparing all information needed to send<commit_msg>Fix campaign status not being updated to sending<commit_after>/*

https://docs.aws.amazon.com/ses/latest/DeveloperGuide/mailbox-simulator.html
<EMAIL> (Successful email)
<EMAIL> (Soft bounce)
<EMAIL> (Out of office response from ISP)
<EMAIL> (Complaint from ISP)
<EMAIL> (Hard bounce as target email is on Amazon's suppression list)

API endpoints
US East (N. Virginia)	us-east-1
US West (Oregon)	us-west-2
EU (Ireland)	eu-west-1

Successful response has form:
{ ResponseMetadata: { RequestId: 'e8a3d6b4-94fd-11z6-afac-757cax279ap5' },
  MessageId: '01020157a1261241-90a5e1cd-3a5z-4sb7-1r41-957a4cae8e58-000000' }
Throttling error:
{ Throttling: Maximum sending rate exceeded.
    at ...
  message: 'Maximum sending rate exceeded.',
  code: 'Throttling',
  time: 2016-10-18T07:50:11.286Z,
  requestId: '7f2d1781-9507-11e6-8885-675506266ba7',
  statusCode: 400,
  retryable: true }

*/

const AWS = require('aws-sdk');
const Promise = require('bluebird'); // Bluebird is signficantly faster than native promises

const wrapLink = require('./analytics').wrapLink;
const insertUnsubscribeLink = require('./analytics').insertUnsubscribeLink;
const insertTrackingPixel = require('./analytics').insertTrackingPixel;
const mailMerge = require('./mail-merge');

const db = require('../../../../models');
const AmazonEmail = require('./amazon');
const CampaignAnalyticsLink = require('../../../../models').campaignanalyticslink;
const CampaignAnalyticsOpen = require('../../../../models').campaignanalyticsopen;

module.exports = (generator, listSubscriberModel, redis, campaignAndListInfo, amazonAccountInfo) => {
  /*
    # Send emails via Amazon SES' API #
    This function focuses on a tradeoff between speed and memory usage. It does the following:
      1. Get all ids (and only ids) of listsubscriber rows containing emails to be sent. We don't want any more info
         than this else memory use will be very large.
      2. Prepare emails to be sent, creating entries in various tables and configuring the email itself. Do this in
         batches.
      3. Sen
  */

  // Unpack args
  const {
    campaignInfo, // See object passed by send-campaign.js, contains info about the campaigns table
    // totalListSubscribers // num - the total number of list subscribers to be emailed
  } = campaignAndListInfo;

  const {
    accessKey, // Amazon access key - str
    secretKey, // Amazon secret key - str
    region, // Amazon region - str
    whiteLabelUrl, // URL of a service that serves this app - str
    quotas // Amazon limits - obj { AvailableToday, MaxSendRate }
  } = amazonAccountInfo;

  // Constants
  const isDevMode = process.env.NODE_ENV === 'development';
  const rateLimit = process.env.DEV_SEND_RATE || quotas.MaxSendRate; // No. of email we can send p/s as established by Amazon
  const rateLimitTimesFive = rateLimit * 5; // Arbitrary quantity of emails to store in buffer
  const emailBuffer = []; // This stores the current buffer of emails to be sent

  const ses = isDevMode
    ? new AWS.SES({  // Dev mode
        apiVersion: '2010-12-01',
        // convertResponseTypes: false,
        maxRetries: 0,
        accessKeyId: accessKey,
        secretAccessKey:
        secretKey,
        region,
        endpoint: 'http://localhost:9999'
      })
    : new AWS.SES({ // Prod mode
        apiVersion: '2010-12-01',
        accessKeyId: accessKey,
        secretAccessKey: secretKey,
        region
      });

  // Vars
  let databaseIsWorking = false; // This is a flag that will ensure that the db does not get assigned too much work
  let shouldSend = true; // Another flag that indicates whether or not a campaign should send

  const Producer = require('../message-queue/amazon/producer')();
  const Receiver = require('../message-queue/amazon/receiver')(ses, rateLimit, campaignInfo);

  (async function sendEmailsViaAmazon() {
    // 1. First we'll get an array containing the ids of all users we're going to email.
    //// This way we have the certainty of knowing who to email without the memory footprint
    //// associated with including all other columns.
    console.log(`\nEmail campaign send initiated. Sending ${rateLimit} per second.\n`); // eslint-disable-line
    updateCampaignStatus();
    console.log('Preparing to send email campaign. Getting list subscribers ids ...'); // eslint-disable-line
    const arrayOfIds = await getListSubscriberIds(); // Returns ids from listsubscribers [1,2,3, ... etc]
    const initialBuffer = await fillInitialBuffer(arrayOfIds); // Initialise the buffer
    emailBuffer.push(...initialBuffer);
    if (isDevMode) console.log(`Filled emailBuffer with ${emailBuffer.length} emails`); // eslint-disable-line
    // 2. Listen for cancellation event
    redis.subscriber.on('message', (channel, campaignId) => {
      if (campaignId == campaignInfo.campaignId) {
        cancel();
      }
    });
    redis.subscriber.subscribe('stop-campaign-sending');
    // 3. Fill the buffer with an initial set of emails
    emailProducer(arrayOfIds, rateLimit);
  })();

  function updateCampaignStatus() {
    db.campaign.update({
      status: 'sending'
    }, {
      where: {
        id: campaignInfo.campaignId
      }
    }).catch(err => {throw err;});
  }

  function getListSubscriberIds() {
    return listSubscriberModel.findAll({
      where: {
        listId: campaignInfo.listId,
        subscribed: true
      },
      include: [{
        model: db.campaignsubscriber,
        where: { sent: false }
      }],
      attributes: [
        'id'
      ],
      raw: true
    })
    .then(instances => { // [{id: 1}, {id: 2}, etc]
      const plainArrayOfIdNumbers = instances.map(x => x.id);
      return plainArrayOfIdNumbers;
    })
    .catch(err => {
       console.log(err); //eslint-disable-line
    });
  }

  function getEmailsAndCampaignInfo(arrayOfIds, rateLimit) {
    // This function is tasked with getting and therefore preparing all information needed to send
{'fraction_non_alphanumeric': 0.11504804392587509, 'fraction_numerical': 0.02367879203843514, 'mean_word_length': 3.090175438596491, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 24, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4439015', 'n_tokens_mistral': 3599, 'n_tokens_neox': 3357, 'n_words': 1078}