<commit_before>/**
 * Copyright (C) 2017 Hurence
 * <p>
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * <p>
 * http://www.apache.org/licenses/LICENSE-2.0
 * <p>
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.hurence.logisland.processor.mailer;

import com.hurence.logisland.annotation.documentation.CapabilityDescription;
import com.hurence.logisland.annotation.documentation.Tags;
import com.hurence.logisland.component.PropertyDescriptor;
import com.hurence.logisland.processor.*;
import com.hurence.logisland.record.Field;
import com.hurence.logisland.record.FieldDictionary;
import com.hurence.logisland.record.FieldType;
import com.hurence.logisland.record.Record;
import com.hurence.logisland.util.string.JsonUtil;
import com.hurence.logisland.validator.StandardValidators;
import com.hurence.logisland.validator.ValidationContext;
import com.hurence.logisland.validator.ValidationResult;

import org.apache.commons.mail.EmailException;
import org.apache.commons.mail.ImageHtmlEmail;
import org.apache.commons.mail.SimpleEmail;
import org.apache.commons.mail.resolver.DataSourceClassPathResolver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Mailer Processor.
 * This processor is able to send mails from the incoming records.
 * A minimum processor configuration is required (mainly smtp server info and some mails)
 * Only records with either FIELD_MAIL_TEXT or FIELD_MAIL_HTML fields set will generate a mail.
 * - If FIELD_MAIL_TEXT is set, it holds the text content of the mail to be sent
 * - If FIELD_MAIL_HTML is set, it informs the processor to use the HTML template of its configuration for sending the
 *   mail. Any parameter defined in the template will be retrieved from the fields with the same name embedded in the
 *   record.
 * 
 * If it is enabled in processor configuration, it is possible to overwrite some config properties directly in the
 * record with some specific fields (FIELD_MAIL_TO, FIELD_MAIL_SUBJECT...)
 * 
 * Note: you can use an html template using embedded image. Images should then be in the class path (in a jar file)
 * and the format of src attribute in the img html tag should be like:
 * <img src="path/in/classpath/spectacular_image.gif" alt="Spectacular Image">
 */
@Tags({"smtp", "email", "e-mail", "mail", "mailer", "message", "alert", "html"})
@CapabilityDescription(
        "The Mailer processor is aimed at sending an email (like for instance an alert email) from an incoming record."
        + " To generate an email and trigger an email sending, an incoming record must have a mail_text field with the"
        + " content of the mail as value or a mail_html field to send an emaile from an HTML template."
        + " Other optional mail_* fields may be used to customize the Mailer processor upon reception of the record.")
public class MailerProcessor extends AbstractProcessor {

    private static Logger logger = LoggerFactory.getLogger(MailerProcessor.class);
    
    // Easy trick to not allow debugging without changing the logger level but instead using a configuration key
    private boolean debug = false;

    private String smtpServer = null;
    private int smtpPort = 25;
    private String smtpSecurityUsername = null;
    private String smtpSecurityPassword = <PASSWORD>;
    private boolean smtpSecuritySsl = false;

    private static final String DEFAULT_FROM_NAME = "Logisland";
    private static final String DEFAULT_SUBJECT = "[LOGISLAND] Automatic email";

    private String[] mailTos = new String[]{};
    private String mailFromAddress = null;
    private String mailFromName = DEFAULT_FROM_NAME;
    private String mailBounceAddress = null;
    private String mailReplyToAddress = null;
    private String mailSubject = DEFAULT_SUBJECT;
    private boolean allowFieldsOverwriting = true;
    
    // HTML template as stated in the configuration
    private String htmlTemplate = null;
    
    // HTML form derived from htmlTemplate, ready for parameters injection
    private MessageFormat htmlForm = null;
    
    // List of parameters for the HTML template. They must be present in the record as fields.
    private List<String> parameterNames = new ArrayList<String>();

    /**
     * Definitions for the fields of the incoming record
     */
    
    private static final String FIELD_PREFIX = "mail";
    
    /**
     * Either FIELD_MAIL_TEXT or FIELD_MAIL_HTML is mandatory for a mail to be sent.
     * If both fields are present, HTML mode is used and FIELD_MAIL_TEXT content is used as alternative text message.
     */
    // This holds the content of the mail to be sent as text.
    public static final String FIELD_MAIL_TEXT = FIELD_PREFIX + "_text";
    // If this (boolean) field is present and set to true, the HTML template is used to send the mail instead of the text.
    public static final String FIELD_MAIL_HTML = FIELD_PREFIX + "_html";

    // May be used to overwrite mail.to configured in processor
    public static final String FIELD_MAIL_TO = FIELD_PREFIX + "_to";
    // May be used to overwrite mail.from.address configured in processor 
    public static final String FIELD_MAIL_FROM_ADDRESS = FIELD_PREFIX + "_from_address";    
    // May be used to overwrite mail.from.name configured in processor
    public static final String FIELD_MAIL_FROM_NAME = FIELD_PREFIX + "_from_name";
    // May be used to overwrite mail.bounce.address configured in processor 
    public static final String FIELD_MAIL_BOUNCE_ADDRESS = FIELD_PREFIX + "_bounce_address"; 
    // May be used to overwrite mail.replyto.address configured in processor 
    public static final String FIELD_MAIL_REPLYTO_ADDRESS = FIELD_PREFIX + "_replyto_address";
    // May be used to overwrite mail.subject configured in processor 
    public static final String FIELD_MAIL_SUBJECT = FIELD_PREFIX + "_subject";

    /**
     * Configuration keys
     */

    // Easy trick to not allow debugging without changing the logger level but instead using a configuration key
    private static final String KEY_DEBUG = "debug";

    private static final String KEY_SMTP_SERVER = "smtp.server";
    private static final String KEY_SMTP_PORT = "smtp.port";
    private static final String KEY_SMTP_SECURITY_USERNAME = "smtp.security.username";
    private static final String KEY_SMTP_SECURITY_PASSWORD = "<PASSWORD>";
    private static final String KEY_SMTP_SECURITY_SSL = "smtp.security.ssl";

    private static final String KEY_MAIL_TO = "mail.to";
    private static final String KEY_MAIL_FROM_ADDRESS = "mail.from.address";
    private static final String KEY_MAIL_FROM_NAME = "mail.from.name";
    private static final String KEY_MAIL_BOUNCE_ADDRESS = "mail.bounce.address";
    private static final String KEY_MAIL_REPLYTO_ADDRESS = "mail.replyto.address";
    private static final String KEY_MAIL_SUBJECT = "mail.subject";
    
    private static final String KEY_ALLOW_OVERWRITE = "allow_overwrite";
    
    private static final String KEY_HTML_TEMPLATE = "html.template";
    
    public static final PropertyDescriptor DEBUG = new PropertyDescriptor.Builder()
            .name(KEY_DEBUG)
            .description("Enable debug. If enabled, debug information are written into stdout.")
            .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
            .required(false)
            .defaultValue("false")
            .build();
    
    public static final PropertyDescriptor SMTP_SERVER = new PropertyDescriptor.Builder()
            .name(KEY_SMTP_SERVER)
            .description("Hostname or IP address of the SMTP server to use.")
            .required(true)
            .build();
    
    public static final PropertyDescriptor SMTP_PORT = new PropertyDescriptor.Builder()
            .name(KEY_SMTP_PORT)
            .description("TCP port number of the SMTP server to use.")
            .addValidator(StandardValidators.PORT_VALIDATOR)
            .defaultValue("25")
            .required(false)
            .build();
    
    public static final PropertyDescriptor SMTP_SECURITY_USERNAME = new PropertyDescriptor.Builder()
            .name(KEY_SMTP_SECURITY_USERNAME)
            .description("SMTP username.")
            .required(false)
            .build();
    
    public static final PropertyDescriptor SMTP_SECURITY_PASSWORD = new PropertyDescriptor.Builder()
            .name(KEY_SMTP_SECURITY_PASSWORD)
            .description("SMTP password.")
            .required(false)
            .build();
    
    public static final PropertyDescriptor SMTP_SECURITY_SSL = new PropertyDescriptor.Builder()
            .name(KEY_SMTP_SECURITY_SSL)
            .description("Use SSL under SMPT or not. Default is false.")
            .required(false)
            .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
            .defaultValue("false")
            .build();
    
    public static final PropertyDescriptor MAIL_FROM_ADDRESS = new PropertyDescriptor.Builder()
            .name(KEY_MAIL_FROM_ADDRESS)
            .description("Valid mail sender email address.")
            .required(true)
            .build();
    
    public static final PropertyDescriptor MAIL_FROM_NAME = new PropertyDescriptor.Builder()
            .name(KEY_MAIL_FROM_NAME)
            .description("Mail sender name.")
            .required(false)
            .build();
    
    public static final PropertyDescriptor MAIL_BOUNCE_ADDRESS = new PropertyDescriptor.Builder()
            .name(KEY_MAIL_BOUNCE_ADDRESS)
            .description("Valid bounce email address (where error mail is sent if the mail is refused by the recipient"
                    + " server).")
            .required(true)
            .build();
    
    public static final PropertyDescriptor MAIL_REPLYTO_ADDRESS = new PropertyDescriptor.Builder()
            .name(KEY_MAIL_REPLYTO_ADDRESS)
            .description("Reply to email address.")
            .required(false)
            .build();
    
    public static final PropertyDescriptor MAIL_SUBJECT = new PropertyDescriptor.Builder()
            .name(KEY_MAIL_SUBJECT)
            .description("Mail subject.")
            .required(false)
            .defaultValue(DEFAULT_SUBJECT)
            .build();
    
    public static final PropertyDescriptor MAIL_TO = new PropertyDescriptor.Builder()
            .name(KEY_MAIL_TO)
            .description("Comma separated list of email recipients. If not set, the record must have a "
            + FIELD_MAIL_TO + " field and " + KEY_ALLOW_OVERWRITE + " configuration key should be true.")
            .required(false)
            .build();
    
    public static final PropertyDescriptor ALLOW_OVERWRITE = new PropertyDescriptor.Builder()
            .name(KEY_ALLOW_OVERWRITE)
            .description("If true, allows to overwrite processor configuration with special record fields (" +
                    FIELD_MAIL_TO + ", " + FIELD_MAIL_FROM_ADDRESS + ", " + FIELD_MAIL_SUBJECT +" etc). If false, special record fields"
                    + " are ignored and only processor configuration keys are used.")
            .required(false)
            .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
            .defaultValue("true")
            .build();
    
    public static final PropertyDescriptor HTML_TEMPLATE = new PropertyDescriptor.Builder()
            .name(KEY_HTML_TEMPLATE)
            .description("HTML template to use.")
            .required(false)
            .build();

    @Override
    public void init(final ProcessContext context)<commit_msg>More documentation for the mailer processor.<commit_after>/**
 * Copyright (C) 2017 Hurence
 * <p>
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * <p>
 * http://www.apache.org/licenses/LICENSE-2.0
 * <p>
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.hurence.logisland.processor.mailer;

import com.hurence.logisland.annotation.documentation.CapabilityDescription;
import com.hurence.logisland.annotation.documentation.Tags;
import com.hurence.logisland.component.PropertyDescriptor;
import com.hurence.logisland.processor.*;
import com.hurence.logisland.record.Field;
import com.hurence.logisland.record.FieldDictionary;
import com.hurence.logisland.record.FieldType;
import com.hurence.logisland.record.Record;
import com.hurence.logisland.util.string.JsonUtil;
import com.hurence.logisland.validator.StandardValidators;
import com.hurence.logisland.validator.ValidationContext;
import com.hurence.logisland.validator.ValidationResult;

import org.apache.commons.mail.EmailException;
import org.apache.commons.mail.ImageHtmlEmail;
import org.apache.commons.mail.SimpleEmail;
import org.apache.commons.mail.resolver.DataSourceClassPathResolver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Mailer Processor.
 * This processor is able to send mails from the incoming records.
 * A minimum processor configuration is required (mainly smtp server info and some mail sender info)
 * Only records with either FIELD_MAIL_TEXT or FIELD_MAIL_HTML fields set will generate a mail.
 * - If FIELD_MAIL_TEXT is set, it holds the text content of the mail to be sent
 * - If FIELD_MAIL_HTML is set, it informs the processor to use the HTML template of its configuration for sending the
 *   mail. Any parameter defined in the template will be retrieved from the fields with the same name embedded in the
 *   record. Every parameters defined in the template with the form ${xxx} must be present in the record (the record
 *   must hold a xxx field).
 * 
 * If it is enabled in processor configuration (allow_overwrite), it is possible to overwrite some config properties
 * directly in the record with some specific fields (FIELD_MAIL_TO, FIELD_MAIL_SUBJECT...)
 * 
 * Note: you can use an html template using embedded image. Images should then be in the class path (in a jar file)
 * and the format of src attribute in the img html tag should be like:
 * <img src="path/in/classpath/spectacular_image.gif" alt="Spectacular Image">
 */
@Tags({"smtp", "email", "e-mail", "mail", "mailer", "message", "alert", "html"})
@CapabilityDescription(
        "The Mailer processor is aimed at sending an email (like for instance an alert email) from an incoming record."
        + " To generate an email and trigger an email sending, an incoming record must have a mail_text field with the"
        + " content of the mail as value or a mail_html field to send an email from the HTML template of the configuration."
        + " Other optional mail_* fields may be used to customize the Mailer processor upon reception of the record."
        + " Part from error records when he is enabled to process the incoming record or to send the mail, this processor"
        + " is not expected to produce any output records.")
public class MailerProcessor extends AbstractProcessor {

    private static Logger logger = LoggerFactory.getLogger(MailerProcessor.class);
    
    // Easy trick to not allow debugging without changing the logger level but instead using a configuration key
    private boolean debug = false;

    private String smtpServer = null;
    private int smtpPort = 25;
    private String smtpSecurityUsername = null;
    private String smtpSecurityPassword = <PASSWORD>;
    private boolean smtpSecuritySsl = false;

    private static final String DEFAULT_FROM_NAME = "Logisland";
    private static final String DEFAULT_SUBJECT = "[LOGISLAND] Automatic email";

    private String[] mailTos = new String[]{};
    private String mailFromAddress = null;
    private String mailFromName = DEFAULT_FROM_NAME;
    private String mailBounceAddress = null;
    private String mailReplyToAddress = null;
    private String mailSubject = DEFAULT_SUBJECT;
    private boolean allowFieldsOverwriting = true;
    
    // HTML template as stated in the configuration
    private String htmlTemplate = null;
    
    // HTML form derived from htmlTemplate, ready for parameters injection
    private MessageFormat htmlForm = null;
    
    // List of parameters for the HTML template. They must be present in the record as fields.
    private List<String> parameterNames = new ArrayList<String>();

    /**
     * Definitions for the fields of the incoming record
     */
    
    private static final String FIELD_PREFIX = "mail";
    
    /**
     * Either FIELD_MAIL_TEXT or FIELD_MAIL_HTML is mandatory for a mail to be sent.
     * If both fields are present, HTML mode is used and FIELD_MAIL_TEXT content is used as alternative text message.
     */
    // This holds the content of the mail to be sent as text.
    public static final String FIELD_MAIL_TEXT = FIELD_PREFIX + "_text";
    // If this (boolean) field is present and set to true, the HTML template is used to send the mail instead of the text.
    public static final String FIELD_MAIL_HTML = FIELD_PREFIX + "_html";

    // May be used to overwrite mail.to configured in processor
    public static final String FIELD_MAIL_TO = FIELD_PREFIX + "_to";
    // May be used to overwrite mail.from.address configured in processor 
    public static final String FIELD_MAIL_FROM_ADDRESS = FIELD_PREFIX + "_from_address";    
    // May be used to overwrite mail.from.name configured in processor
    public static final String FIELD_MAIL_FROM_NAME = FIELD_PREFIX + "_from_name";
    // May be used to overwrite mail.bounce.address configured in processor 
    public static final String FIELD_MAIL_BOUNCE_ADDRESS = FIELD_PREFIX + "_bounce_address"; 
    // May be used to overwrite mail.replyto.address configured in processor 
    public static final String FIELD_MAIL_REPLYTO_ADDRESS = FIELD_PREFIX + "_replyto_address";
    // May be used to overwrite mail.subject configured in processor 
    public static final String FIELD_MAIL_SUBJECT = FIELD_PREFIX + "_subject";

    /**
     * Configuration keys
     */

    // Easy trick to not allow debugging without changing the logger level but instead using a configuration key
    private static final String KEY_DEBUG = "debug";

    private static final String KEY_SMTP_SERVER = "smtp.server";
    private static final String KEY_SMTP_PORT = "smtp.port";
    private static final String KEY_SMTP_SECURITY_USERNAME = "smtp.security.username";
    private static final String KEY_SMTP_SECURITY_PASSWORD = "<PASSWORD>";
    private static final String KEY_SMTP_SECURITY_SSL = "smtp.security.ssl";

    private static final String KEY_MAIL_TO = "mail.to";
    private static final String KEY_MAIL_FROM_ADDRESS = "mail.from.address";
    private static final String KEY_MAIL_FROM_NAME = "mail.from.name";
    private static final String KEY_MAIL_BOUNCE_ADDRESS = "mail.bounce.address";
    private static final String KEY_MAIL_REPLYTO_ADDRESS = "mail.replyto.address";
    private static final String KEY_MAIL_SUBJECT = "mail.subject";
    
    private static final String KEY_ALLOW_OVERWRITE = "allow_overwrite";
    
    private static final String KEY_HTML_TEMPLATE = "html.template";
    
    public static final PropertyDescriptor DEBUG = new PropertyDescriptor.Builder()
            .name(KEY_DEBUG)
            .description("Enable debug. If enabled, debug information are written into stdout.")
            .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
            .required(false)
            .defaultValue("false")
            .build();
    
    public static final PropertyDescriptor SMTP_SERVER = new PropertyDescriptor.Builder()
            .name(KEY_SMTP_SERVER)
            .description("Hostname or IP address of the SMTP server to use.")
            .required(true)
            .build();
    
    public static final PropertyDescriptor SMTP_PORT = new PropertyDescriptor.Builder()
            .name(KEY_SMTP_PORT)
            .description("TCP port number of the SMTP server to use.")
            .addValidator(StandardValidators.PORT_VALIDATOR)
            .defaultValue("25")
            .required(false)
            .build();
    
    public static final PropertyDescriptor SMTP_SECURITY_USERNAME = new PropertyDescriptor.Builder()
            .name(KEY_SMTP_SECURITY_USERNAME)
            .description("SMTP username.")
            .required(false)
            .build();
    
    public static final PropertyDescriptor SMTP_SECURITY_PASSWORD = new PropertyDescriptor.Builder()
            .name(KEY_SMTP_SECURITY_PASSWORD)
            .description("SMTP password.")
            .required(false)
            .build();
    
    public static final PropertyDescriptor SMTP_SECURITY_SSL = new PropertyDescriptor.Builder()
            .name(KEY_SMTP_SECURITY_SSL)
            .description("Use SSL under SMPT or not. Default is false.")
            .required(false)
            .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
            .defaultValue("false")
            .build();
    
    public static final PropertyDescriptor MAIL_FROM_ADDRESS = new PropertyDescriptor.Builder()
            .name(KEY_MAIL_FROM_ADDRESS)
            .description("Valid mail sender email address.")
            .required(true)
            .build();
    
    public static final PropertyDescriptor MAIL_FROM_NAME = new PropertyDescriptor.Builder()
            .name(KEY_MAIL_FROM_NAME)
            .description("Mail sender name.")
            .required(false)
            .build();
    
    public static final PropertyDescriptor MAIL_BOUNCE_ADDRESS = new PropertyDescriptor.Builder()
            .name(KEY_MAIL_BOUNCE_ADDRESS)
            .description("Valid bounce email address (where error mail is sent if the mail is refused by the recipient"
                    + " server).")
            .required(true)
            .build();
    
    public static final PropertyDescriptor MAIL_REPLYTO_ADDRESS = new PropertyDescriptor.Builder()
            .name(KEY_MAIL_REPLYTO_ADDRESS)
            .description("Reply to email address.")
            .required(false)
            .build();
    
    public static final PropertyDescriptor MAIL_SUBJECT = new PropertyDescriptor.Builder()
            .name(KEY_MAIL_SUBJECT)
            .description("Mail subject.")
            .required(false)
            .defaultValue(DEFAULT_SUBJECT)
            .build();
    
    public static final PropertyDescriptor MAIL_TO = new PropertyDescriptor.Builder()
            .name(KEY_MAIL_TO)
            .description("Comma separated list of email recipients. If not set, the record must have a "
            + FIELD_MAIL_TO + " field and " + KEY_ALLOW_OVERWRITE + " configuration key should be true.")
            .required(false)
            .build();
    
    public static final PropertyDescriptor ALLOW_OVERWRITE = new PropertyDescriptor.Builder()
            .name(KEY_ALLOW_OVERWRITE)
            .description("If true, allows to overwrite processor configuration with special record fields (" +
                    FIELD_MAIL_TO + ", " + FIELD_MAIL_FROM_ADDRESS + ", " + FIELD_MAIL_SUBJECT +" etc). If false,"
                            + " special record fields are ignored and only processor configuration keys are used.")
            .required(false)
            .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
            .defaultValue("true")
            .build();
    
    public static final PropertyDescriptor HTML_TEMPLATE = new PropertyDescriptor.Builder()
            .name(KEY_HTML_TEMPLATE)
            .description("HTML template to use. It is used when the incoming record contains a " + FIELD_MAIL_HTML
                    + " field. The template may contain some parameters. The parameter format in the template is of the"
                    + " form ${xxx}. For instance ${param_user} in the template means that a field named"
                    + " param_user must be present in the record and its value will replace the ${param_user} string"
                    + " in the HTML template when the mail will be sent. If some parameters are declared in the template"
                    + ", everyone of them must be present in the record as fields, otherwise the record will generate"
                    + " an error record. If an incoming record contains a " + FIELD_MAIL_HTML + " field, a template"
                    + " must be present in the configuration and the HTML mail format will be preferred. If the record"
                    + " also contains a " + FIELD_MAIL_TEXT + " field, its content will be used as an alternative text"
                    + " message to be used in the mail reader program of the recipient if it does not supports HTML.")
            .required(false)
            .build();

    @Override
    public void init(final ProcessContext context)
{'fraction_non_alphanumeric': 0.0719368811881188, 'fraction_numerical': 0.001082920792079208, 'mean_word_length': 3.0867709815078235, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4050791', 'n_tokens_mistral': 6668, 'n_tokens_neox': 6600, 'n_words': 2409}