<commit_before>/*!
 * Copyright 2014 Apereo Foundation (AF) Licensed under the
 * Educational Community License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License. You may
 * obtain a copy of the License at
 *
 *     http://opensource.org/licenses/ECL-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an "AS IS"
 * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

var _ = require('underscore');
var crypto = require('crypto');
var events = require('events');
var fs = require('fs');
var nodemailer = require('nodemailer');
var path = require('path');
var redback = require('redback');
var util = require('util');

var IO = require('oae-util/lib/io');
var Locking = require('oae-util/lib/locking');
var log = require('oae-logger').logger('oae-email');
var OaeModules = require('oae-util/lib/modules');
var Redis = require('oae-util/lib/redis');
var Telemetry = require('oae-telemetry').telemetry('oae-email');
var Validator = require('oae-util/lib/validator').Validator;

var EmailConfig = require('oae-config').config('oae-email');
var EmailRateLimiter = null;

// These hashes cache both the base templates (in the source tree) and the tenant override templates
// (provided in a directory by a system configuration property).
var baseCompiledTemplates = {};
var overrideCompiledTemplates = {};

// The configuration value that specifies where the override templates lay
var customEmailTemplatesDir = null;

/*!
 * Whether or not the server is in debug mode. If true, no emails will ever be sent, instead the email
 * data will be logged. This is equivalent to "disabling" emails.
 */
var debug = true;

// The cached connection pool with the configured mail values. This can be smtp, sendmail, ..
var emailTransport = null;

// The interval in which the same email can't be sent out multiple times
var deduplicationInterval = null;

// The configuration for e-mail throttling
var throttleConfig = {
    'timespan': null,
    'count': null
};


/**
 * ## EmailAPI
 *
 * ### Events
 *
 * * `debugSent(message)` - If `debug` is enabled, this event is fired and indicates an email was sent from the system. The `message` object, which is a MailComposer-compatable object is provided
 *
 * ### Templates
 *
 * All emails that are sent are based on an internationalizable template. To load a new template for the system, you must
 * create a directory in your OAE module called `emailTemplates`. The directory structure looks like this (using oae-content
 * as an example):
 *
 *  * oae-content/                              (module directory)
 *      ** emailTemplates/                      (directory that is scanned by oae-email)
 *          *** default/                        (the default templates, chosen if there is no locale)
 *              **** templateId.meta.json.jst   (the "meta" template for template with id "templateId")
 *              **** templateId.html.jst        (the "html" template for template with id "templateId")
 *              **** templateId.txt.jst         (the "text" template for template with id "templateId")
 *          *** en_CA/                          (the en_CA locale templates, used if the receiving user has locale en_CA)
 *              **** templateId.meta.json.jst
 *              **** templateId.html.jst
 *              **** templateId.txt.jst
 *
 * **The 'default' locale:** The default locale is chosen if the user's locale does not have a template provided for it.
 * **The 'meta.json' template:** This template should produce a JSON object that specifies email metadata. This template **must** exist, and should at least provide the "subject" of the email.
 * **The 'html' template:** This template provides an HTML-formatted version of the email content. One of HTML and TXT templates must be provided.
 * **The 'txt' template:** This template provides a plain-text version of the email content. If this is not provided, the HTML version will be converted to plain-text in replacement. One of HTML and TXT must be provided.
 *
 * Template overrides can be specified on a per-tenant basis. The overrides directory can be specified with the config property
 * `customEmailTemplatesDir`. If a template is overridden (e.g., exists in the override directory), it will be selected over
 * the base templates. The template directory structure is the same as the base templates, except each module directory should
 * exist in a folder named after the tenant alias that is overridden.
 *
 * ### JST Files
 *
 * All templates: meta.json.jst, html.jst and txt.jst are JavaScriptTemplates, and are compiled and rendered using UnderscoreJS:
 * http://underscorejs.org/#template
 */
var EmailAPI = module.exports = new events.EventEmitter();

/**
 * Initialize the email module.
 *
 * @param  {Object}     emailSystemConfig   The `email` config object from the system `config.js`. Refer to that file for the configuration options
 * @param  {Function}   callback            Invoked when emails have been initialized
 * @param  {Object}     callback.err        An error that occurred, if any
 */
var init = module.exports.init = function(emailSystemConfig, callback) {
    // Email configuration
    customEmailTemplatesDir = emailSystemConfig.customEmailTemplatesDir;
    debug = (emailSystemConfig.debug !== false);
    deduplicationInterval = emailSystemConfig.deduplicationInterval || (7 * 24 * 60 * 60);
    emailSystemConfig.throttling = emailSystemConfig.throttling || {};
    throttleConfig.count = emailSystemConfig.throttling.count || 10;
    throttleConfig.timespan = emailSystemConfig.throttling.timespan || 2 * 60;

    // Create the Redback rate limiter for emails
    var EmailRedback = redback.use(Redis.getClient(), {'namespace': 'oae-email:redback'});

    // For robust unit tests, we need to try and let the timespan cover 2 buckets so that
    // the rate-limit test doesn't roll over to the next interval, thus resetting the frequency
    // to 0 and having an intermittently failing test
    var bucketInterval = Math.ceil(throttleConfig.timespan / 2);
    EmailRateLimiter = EmailRedback.createRateLimit('email', {
        // The rate limiter seems to need at least 3 buckets to work. So we give exactly 3 buckets
        'bucket_span': bucketInterval * 3,
        'bucket_interval': bucketInterval,
        'subject_expiry': throttleConfig.timespan
    });

    // If there was an existing email transport, we close it.
    if (emailTransport) {
        emailTransport.close();
        emailTransport = null;
    }

    // Open an email transport if we're not in debug mode
    if (!debug) {
        if (emailSystemConfig.transport === 'SMTP') {
            log().info({'data': emailSystemConfig.smtpTransport}, 'Configuring SMTP email transport.');
            emailTransport = nodemailer.createTransport('SMTP', emailSystemConfig.smtpTransport);
        } else if (emailSystemConfig.transport === 'sendmail') {
            log().info({'data': emailSystemConfig.sendmailTransport}, 'Configuring Sendmail email transport.');
            emailTransport = nodemailer.createTransport('Sendmail', emailSystemConfig.sendmailTransport.path);
        } else {
            return callback({'code': 400, 'msg': 'Misconfigured mail transport'});
        }
    }

    return refreshTemplates(callback);<commit_msg>(Issue 818) Increase rate-limiting bucket count in redback to 5 to fix intermittent test failures<commit_after>/*!
 * Copyright 2014 Apereo Foundation (AF) Licensed under the
 * Educational Community License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License. You may
 * obtain a copy of the License at
 *
 *     http://opensource.org/licenses/ECL-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an "AS IS"
 * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

var _ = require('underscore');
var crypto = require('crypto');
var events = require('events');
var fs = require('fs');
var nodemailer = require('nodemailer');
var path = require('path');
var redback = require('redback');
var util = require('util');

var IO = require('oae-util/lib/io');
var Locking = require('oae-util/lib/locking');
var log = require('oae-logger').logger('oae-email');
var OaeModules = require('oae-util/lib/modules');
var Redis = require('oae-util/lib/redis');
var Telemetry = require('oae-telemetry').telemetry('oae-email');
var Validator = require('oae-util/lib/validator').Validator;

var EmailConfig = require('oae-config').config('oae-email');
var EmailRateLimiter = null;

// These hashes cache both the base templates (in the source tree) and the tenant override templates
// (provided in a directory by a system configuration property).
var baseCompiledTemplates = {};
var overrideCompiledTemplates = {};

// The configuration value that specifies where the override templates lay
var customEmailTemplatesDir = null;

/*!
 * Whether or not the server is in debug mode. If true, no emails will ever be sent, instead the email
 * data will be logged. This is equivalent to "disabling" emails.
 */
var debug = true;

// The cached connection pool with the configured mail values. This can be smtp, sendmail, ..
var emailTransport = null;

// The interval in which the same email can't be sent out multiple times
var deduplicationInterval = null;

// The configuration for e-mail throttling
var throttleConfig = {
    'timespan': null,
    'count': null
};


/**
 * ## EmailAPI
 *
 * ### Events
 *
 * * `debugSent(message)` - If `debug` is enabled, this event is fired and indicates an email was sent from the system. The `message` object, which is a MailComposer-compatable object is provided
 *
 * ### Templates
 *
 * All emails that are sent are based on an internationalizable template. To load a new template for the system, you must
 * create a directory in your OAE module called `emailTemplates`. The directory structure looks like this (using oae-content
 * as an example):
 *
 *  * oae-content/                              (module directory)
 *      ** emailTemplates/                      (directory that is scanned by oae-email)
 *          *** default/                        (the default templates, chosen if there is no locale)
 *              **** templateId.meta.json.jst   (the "meta" template for template with id "templateId")
 *              **** templateId.html.jst        (the "html" template for template with id "templateId")
 *              **** templateId.txt.jst         (the "text" template for template with id "templateId")
 *          *** en_CA/                          (the en_CA locale templates, used if the receiving user has locale en_CA)
 *              **** templateId.meta.json.jst
 *              **** templateId.html.jst
 *              **** templateId.txt.jst
 *
 * **The 'default' locale:** The default locale is chosen if the user's locale does not have a template provided for it.
 * **The 'meta.json' template:** This template should produce a JSON object that specifies email metadata. This template **must** exist, and should at least provide the "subject" of the email.
 * **The 'html' template:** This template provides an HTML-formatted version of the email content. One of HTML and TXT templates must be provided.
 * **The 'txt' template:** This template provides a plain-text version of the email content. If this is not provided, the HTML version will be converted to plain-text in replacement. One of HTML and TXT must be provided.
 *
 * Template overrides can be specified on a per-tenant basis. The overrides directory can be specified with the config property
 * `customEmailTemplatesDir`. If a template is overridden (e.g., exists in the override directory), it will be selected over
 * the base templates. The template directory structure is the same as the base templates, except each module directory should
 * exist in a folder named after the tenant alias that is overridden.
 *
 * ### JST Files
 *
 * All templates: meta.json.jst, html.jst and txt.jst are JavaScriptTemplates, and are compiled and rendered using UnderscoreJS:
 * http://underscorejs.org/#template
 */
var EmailAPI = module.exports = new events.EventEmitter();

/**
 * Initialize the email module.
 *
 * @param  {Object}     emailSystemConfig   The `email` config object from the system `config.js`. Refer to that file for the configuration options
 * @param  {Function}   callback            Invoked when emails have been initialized
 * @param  {Object}     callback.err        An error that occurred, if any
 */
var init = module.exports.init = function(emailSystemConfig, callback) {
    // Email configuration
    customEmailTemplatesDir = emailSystemConfig.customEmailTemplatesDir;
    debug = (emailSystemConfig.debug !== false);
    deduplicationInterval = emailSystemConfig.deduplicationInterval || (7 * 24 * 60 * 60);
    emailSystemConfig.throttling = emailSystemConfig.throttling || {};
    throttleConfig.count = emailSystemConfig.throttling.count || 10;
    throttleConfig.timespan = emailSystemConfig.throttling.timespan || 2 * 60;

    // Create the Redback rate limiter for emails
    var EmailRedback = redback.use(Redis.getClient(), {'namespace': 'oae-email:redback'});

    /*!
     * For robust unit tests, any provided timespan needs to cover at least 2 buckets so that when
     * we do a count on the rate, we don't risk rolling over to a new interval and miss the emails
     * we just sent, resetting the frequency to 0 and intermittently failing the test. Therefore
     * we set the bucket interval to be (timespan / 2).
     *
     * Additionally, when a bucket is incremented in redback, the following 2 buckets are cleared.
     * Therefore in order to ensure we don't roll over to a new bucket while incrementing and risking
     * our previous bucket getting cleared, we must ensure we have at least 5 buckets so that the
     * clearing of the "next 2" buckets does not impact the counting of the "previous 2". (e.g., if
     * the current time bucket is 2, redback will clear buckets 3 and 4 while we count back from 0,
     * 1 and 2).
     */
    var bucketInterval = Math.ceil(throttleConfig.timespan / 2);
    EmailRateLimiter = EmailRedback.createRateLimit('email', {
        // The rate limiter seems to need at least 5 buckets to work, so lets give it exactly 5 (there are exactly bucket_span / bucket_interval buckets)
        'bucket_span': bucketInterval * 5,
        'bucket_interval': bucketInterval,
        'subject_expiry': throttleConfig.timespan
    });

    // If there was an existing email transport, we close it.
    if (emailTransport) {
        emailTransport.close();
        emailTransport = null;
    }

    // Open an email transport if we're not in debug mode
    if (!debug) {
        if (emailSystemConfig.transport === 'SMTP') {
            log().info({'data': emailSystemConfig.smtpTransport}, 'Configuring SMTP email transport.');
            emailTransport = nodemailer.createTransport('SMTP', emailSystemConfig.smtpTransport);
        } else if (emailSystemConfig.transport === 'sendmail') {
            log().info({'data': emailSystemConfig.sendmailTransport}, 'Configuring Sendmail email transport.');
            emailTransport = nodemailer.createTransport('Sendmail', emailSystemConfig.sendmailTransport.path);
        } else {
            return callback({'code': 400, 'msg': 'Misconfigured mail transport'});
        }
    }

    return refreshTemplates(callback);
{'fraction_non_alphanumeric': 0.0961720375859242, 'fraction_numerical': 0.004603645077883585, 'mean_word_length': 3.3940149625935163, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3497632', 'n_tokens_mistral': 4184, 'n_tokens_neox': 4061, 'n_words': 1721}