<commit_before>/*
 * #%L
 * Bitmagasin integrationstest
 * 
 * $Id$
 * $HeadURL$
 * %%
 * Copyright (C) 2010 The State and University Library, The Royal Library and The State Archives, Denmark
 * %%
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as 
 * published by the Free Software Foundation, either version 2.1 of the 
 * License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Lesser Public License for more details.
 * 
 * You should have received a copy of the GNU General Lesser Public 
 * License along with this program.  If not, see
 * <http://www.gnu.org/licenses/lgpl-2.1.html>.
 * #L%
 */
package org.bitrepository.protocol.activemq;

import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

import javax.jms.Connection;
import javax.jms.DeliveryMode;
import javax.jms.Destination;
import javax.jms.ExceptionListener;
import javax.jms.JMSException;
import javax.jms.MessageConsumer;
import javax.jms.MessageProducer;
import javax.jms.Session;
import javax.jms.TextMessage;

import org.apache.activemq.ActiveMQConnectionFactory;
import org.apache.activemq.util.ByteArrayInputStream;
import org.bitrepository.bitrepositorymessages.AlarmMessage;
import org.bitrepository.bitrepositorymessages.DeleteFileFinalResponse;
import org.bitrepository.bitrepositorymessages.DeleteFileProgressResponse;
import org.bitrepository.bitrepositorymessages.DeleteFileRequest;
import org.bitrepository.bitrepositorymessages.GetChecksumsFinalResponse;
import org.bitrepository.bitrepositorymessages.GetChecksumsProgressResponse;
import org.bitrepository.bitrepositorymessages.GetChecksumsRequest;
import org.bitrepository.bitrepositorymessages.GetFileFinalResponse;
import org.bitrepository.bitrepositorymessages.GetFileIDsFinalResponse;
import org.bitrepository.bitrepositorymessages.GetFileIDsProgressResponse;
import org.bitrepository.bitrepositorymessages.GetFileIDsRequest;
import org.bitrepository.bitrepositorymessages.GetFileProgressResponse;
import org.bitrepository.bitrepositorymessages.GetFileRequest;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForDeleteFileRequest;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForDeleteFileResponse;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForGetChecksumsRequest;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForGetChecksumsResponse;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForGetFileIDsRequest;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForGetFileIDsResponse;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForGetFileRequest;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForGetFileResponse;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForPutFileRequest;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForPutFileResponse;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForReplaceFileRequest;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForReplaceFileResponse;
import org.bitrepository.bitrepositorymessages.Message;
import org.bitrepository.bitrepositorymessages.PutFileFinalResponse;
import org.bitrepository.bitrepositorymessages.PutFileProgressResponse;
import org.bitrepository.bitrepositorymessages.PutFileRequest;
import org.bitrepository.bitrepositorymessages.ReplaceFileFinalResponse;
import org.bitrepository.bitrepositorymessages.ReplaceFileProgressResponse;
import org.bitrepository.bitrepositorymessages.ReplaceFileRequest;
import org.bitrepository.common.JaxbHelper;
import org.bitrepository.protocol.CoordinationLayerException;
import org.bitrepository.protocol.messagebus.MessageBus;
import org.bitrepository.protocol.messagebus.MessageListener;
import org.bitrepository.protocol.security.MessageAuthenticationException;
import org.bitrepository.protocol.security.OperationAuthorizationException;
import org.bitrepository.protocol.security.SecurityManager;
import org.bitrepository.settings.collectionsettings.MessageBusConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;

/**
 * Contains the basic functionality for connection and communicating with the
 * coordination layer over JMS through active MQ.
 *
 * TODO add retries for whenever a JMS exception is thrown. Currently it is
 * very unstable to connection issues.
 *
 * TODO currently creates only topics.
 */
public class ActiveMQMessageBus implements MessageBus {
    /** The Log. */
    private final Logger log = LoggerFactory.getLogger(getClass());

    /** The key for storing the message type in a string property in the message headers. */
    public static final String MESSAGE_TYPE_KEY = "org.bitrepository.messages.type";
    /** The key for storing the BitRepositoryCollectionID in a string property in the message headers. */
    public static final String COLLECTION_ID_KEY = "org.bitrepository.messages.collectionid";
    /** The key for storing the message type in a string property in the message headers. */
    public static final String MESSAGE_SIGNATURE_KEY = "org.bitrepository.messages.signature";
    /** The default acknowledge mode. */
    public static final int ACKNOWLEDGE_MODE = Session.AUTO_ACKNOWLEDGE;
    /** Default transacted. */
    public static final boolean TRANSACTED = false;

    /** The variable to separate the parts of the consumer key. */
    private static final String CONSUMER_KEY_SEPARATOR = "#";

    /** The session for sending messages. Should not be the same as the consumer session, 
     * as sessions are not thread safe. This also means the session should be used in a synchronized manor.
     * TODO Switch to use a session pool/producer poll to allow multithreaded message sending, see 
     * https://sbforge.org/jira/browse/BITMAG-357.
     */
    private final Session producerSession;
    
    /** The session for receiving messages. */
    private final Session consumerSession;

    /**
     * Map of the consumers, mapping from a hash of "destinations and listener" to consumer.
     * Used to identify if a listener is already registered.
     */
    private final Map<String, MessageConsumer> consumers = Collections
            .synchronizedMap(new HashMap<String, MessageConsumer>());
    /** Map of destinations, mapping from ID to destination. */
    private final Map<String, Destination> destinations = new HashMap<String, Destination>();
    /** The configuration for the connection to the activeMQ. */
    private final MessageBusConfiguration configuration;
    private String schemaLocation = "BitRepositoryMessages.xsd";
    private final JaxbHelper jaxbHelper;
    private final Connection connection;
    private final SecurityManager securityManager;
    
    /**
     * Use the {@link org.bitrepository.protocol.ProtocolComponentFactory} to get a handle on a instance of
     * MessageBusConnections. This constructor is for the
     * <code>ProtocolComponentFactory</code> eyes only.
     *
     * @param messageBusConfigurations The properties for the connection.
     */
    public ActiveMQMessageBus(MessageBusConfiguration messageBusConfiguration, SecurityManager securityManager) {
        log.debug("Initializing ActiveMQConnection to '" + messageBusConfiguration + "'.");
        this.configuration = messageBusConfiguration;
        this.securityManager = securityManager;
        jaxbHelper = new JaxbHelper("xsd/", schemaLocation); 
        // Retrieve factory for connection
        ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory(configuration.getURL());

        try {
            connection = connectionFactory.createConnection();
            connection.setExceptionListener(new MessageBusExceptionListener());
            
            producerSession = connection.createSession(TRANSACTED, ACKNOWLEDGE_MODE);
            consumerSession = connection.createSession(TRANSACTED, ACKNOWLEDGE_MODE);
            
            startListeningForMessages();
            
        } catch (JMSException e) {
            throw new CoordinationLayerException("Unable to initialise connection to message bus", e);
        }
        log.debug("ActiveMQConnection initialized for '" + configuration + "'.");
    }
    
    /**
     * Start to listen for message on the message bus. This is done in a separate thread to avoid blocking, 
     * so the main thread can continue without having to wait for the messagebus listening to start.
     */
    private void startListeningForMessages() {
        Thread connectionStarter = new Thread(new Runnable() {
            @Override
            public void run() {
                try {
                    connection.start();
                } catch (Exception e) {
                    log.error("Unable to start listening on the message bus", e);
                }
            }               
        });
        connectionStarter.start();

    }

    @Override
    public synchronized void addListener(String destinationID, final MessageListener listener) {
        log.debug("Adding listener '{}' to destination: '{}' on message-bus '{}'.", 
                new Object[] {listener, destinationID, configuration.getName()});
        MessageConsumer consumer = getMessageConsumer(destinationID, listener);
        try {
            consumer.setMessageListener(new ActiveMQMessageListener(listener));
        } catch (JMSException e) {
            throw new CoordinationLayerException(
                    "Unable to add listener '" + listener + "' to destinationID '" + destinationID + "'", e);
        }
    }

    @Override
    public synchronized void removeListener(String destinationID, MessageListener listener) {
        log.debug("Removing listener '" + listener + "' from destination: '" + destinationID + "' " +
        		"on message-bus '" + configuration + "'.");
        MessageConsumer consumer = getMessageConsumer(destinationID, listener);
        try {
            // We need to set the listener to null to have the removeListerer take effect at once. 
            // If this isn't done the listener will continue to receive messages. Do we have a memory leak here? 
            consumer.setMessageListener(null);
            consumer.close();
        } catch (JMSException e) {
            throw new CoordinationLayerException(
                    "Unable to remove listener '" + listener + "' from destinationID '" + destinationID + "'", e);
        }
        consumers.remove(getConsumerHash(destinationID, listener));<commit_msg>BITMAG-432 Alarm service should use manually acknowledged, durable subscriber: Started work.<commit_after>/*
 * #%L
 * Bitmagasin integrationstest
 * 
 * $Id$
 * $HeadURL$
 * %%
 * Copyright (C) 2010 The State and University Library, The Royal Library and The State Archives, Denmark
 * %%
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as 
 * published by the Free Software Foundation, either version 2.1 of the 
 * License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Lesser Public License for more details.
 * 
 * You should have received a copy of the GNU General Lesser Public 
 * License along with this program.  If not, see
 * <http://www.gnu.org/licenses/lgpl-2.1.html>.
 * #L%
 */
package org.bitrepository.protocol.activemq;

import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

import javax.jms.Connection;
import javax.jms.DeliveryMode;
import javax.jms.Destination;
import javax.jms.ExceptionListener;
import javax.jms.JMSException;
import javax.jms.MessageConsumer;
import javax.jms.MessageProducer;
import javax.jms.Session;
import javax.jms.TextMessage;

import org.apache.activemq.ActiveMQConnectionFactory;
import org.apache.activemq.util.ByteArrayInputStream;
import org.bitrepository.bitrepositorymessages.AlarmMessage;
import org.bitrepository.bitrepositorymessages.DeleteFileFinalResponse;
import org.bitrepository.bitrepositorymessages.DeleteFileProgressResponse;
import org.bitrepository.bitrepositorymessages.DeleteFileRequest;
import org.bitrepository.bitrepositorymessages.GetChecksumsFinalResponse;
import org.bitrepository.bitrepositorymessages.GetChecksumsProgressResponse;
import org.bitrepository.bitrepositorymessages.GetChecksumsRequest;
import org.bitrepository.bitrepositorymessages.GetFileFinalResponse;
import org.bitrepository.bitrepositorymessages.GetFileIDsFinalResponse;
import org.bitrepository.bitrepositorymessages.GetFileIDsProgressResponse;
import org.bitrepository.bitrepositorymessages.GetFileIDsRequest;
import org.bitrepository.bitrepositorymessages.GetFileProgressResponse;
import org.bitrepository.bitrepositorymessages.GetFileRequest;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForDeleteFileRequest;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForDeleteFileResponse;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForGetChecksumsRequest;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForGetChecksumsResponse;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForGetFileIDsRequest;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForGetFileIDsResponse;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForGetFileRequest;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForGetFileResponse;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForPutFileRequest;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForPutFileResponse;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForReplaceFileRequest;
import org.bitrepository.bitrepositorymessages.IdentifyPillarsForReplaceFileResponse;
import org.bitrepository.bitrepositorymessages.Message;
import org.bitrepository.bitrepositorymessages.PutFileFinalResponse;
import org.bitrepository.bitrepositorymessages.PutFileProgressResponse;
import org.bitrepository.bitrepositorymessages.PutFileRequest;
import org.bitrepository.bitrepositorymessages.ReplaceFileFinalResponse;
import org.bitrepository.bitrepositorymessages.ReplaceFileProgressResponse;
import org.bitrepository.bitrepositorymessages.ReplaceFileRequest;
import org.bitrepository.common.JaxbHelper;
import org.bitrepository.protocol.CoordinationLayerException;
import org.bitrepository.protocol.messagebus.MessageBus;
import org.bitrepository.protocol.messagebus.MessageListener;
import org.bitrepository.protocol.security.MessageAuthenticationException;
import org.bitrepository.protocol.security.OperationAuthorizationException;
import org.bitrepository.protocol.security.SecurityManager;
import org.bitrepository.settings.collectionsettings.MessageBusConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;

/**
 * Contains the basic functionality for connection and communicating with the
 * coordination layer over JMS through active MQ.
 *
 * TODO add retries for whenever a JMS exception is thrown. Currently it is
 * very unstable to connection issues.
 *
 * TODO currently creates only topics.
 */
public class ActiveMQMessageBus implements MessageBus {
    /** The Log. */
    private final Logger log = LoggerFactory.getLogger(getClass());

    /** The key for storing the message type in a string property in the message headers. */
    public static final String MESSAGE_TYPE_KEY = "org.bitrepository.messages.type";
    /** The key for storing the BitRepositoryCollectionID in a string property in the message headers. */
    public static final String COLLECTION_ID_KEY = "org.bitrepository.messages.collectionid";
    /** The key for storing the message type in a string property in the message headers. */
    public static final String MESSAGE_SIGNATURE_KEY = "org.bitrepository.messages.signature";
    /** Default transacted. */
    public static final boolean TRANSACTED = false;

    /** The variable to separate the parts of the consumer key. */
    private static final String CONSUMER_KEY_SEPARATOR = "#";

    /** The session for sending messages. Should not be the same as the consumer session, 
     * as sessions are not thread safe. This also means the session should be used in a synchronized manor.
     * TODO Switch to use a session pool/producer poll to allow multithreaded message sending, see 
     * https://sbforge.org/jira/browse/BITMAG-357.
     */
    private final Session producerSession;
    
    /** The session for receiving messages. */
    private final Session consumerSession;

    /** The session for receiving messages. */
    private final Session manAckConsumerSession;

    /**
     * Map of the consumers, mapping from a hash of "destinations and listener" to consumer.
     * Used to identify if a listener is already registered.
     */
    private final Map<String, MessageConsumer> consumers = Collections
            .synchronizedMap(new HashMap<String, MessageConsumer>());
    /** Map of destinations, mapping from ID to destination. */
    private final Map<String, Destination> destinations = new HashMap<String, Destination>();
    /** The configuration for the connection to the activeMQ. */
    private final MessageBusConfiguration configuration;
    private String schemaLocation = "BitRepositoryMessages.xsd";
    private final JaxbHelper jaxbHelper;
    private final Connection connection;
    private final SecurityManager securityManager;
    
    /**
     * Use the {@link org.bitrepository.protocol.ProtocolComponentFactory} to get a handle on a instance of
     * MessageBusConnections. This constructor is for the
     * <code>ProtocolComponentFactory</code> eyes only.
     *
     * @param messageBusConfigurations The properties for the connection.
     */
    public ActiveMQMessageBus(MessageBusConfiguration messageBusConfiguration, SecurityManager securityManager) {
        log.debug("Initializing ActiveMQConnection to '" + messageBusConfiguration + "'.");
        this.configuration = messageBusConfiguration;
        this.securityManager = securityManager;
        jaxbHelper = new JaxbHelper("xsd/", schemaLocation); 
        // Retrieve factory for connection
        ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory(configuration.getURL());

        try {
            connection = connectionFactory.createConnection();
            connection.setExceptionListener(new MessageBusExceptionListener());
            
            producerSession = connection.createSession(TRANSACTED, Session.AUTO_ACKNOWLEDGE);
            consumerSession = connection.createSession(TRANSACTED, Session.AUTO_ACKNOWLEDGE);
            manAckConsumerSession = connection.createSession(TRANSACTED, Session.CLIENT_ACKNOWLEDGE);
            
            startListeningForMessages();
            
        } catch (JMSException e) {
            throw new CoordinationLayerException("Unable to initialise connection to message bus", e);
        }
        log.debug("ActiveMQConnection initialized for '" + configuration + "'.");
    }
    
    /**
     * Start to listen for message on the message bus. This is done in a separate thread to avoid blocking, 
     * so the main thread can continue without having to wait for the messagebus listening to start.
     */
    private void startListeningForMessages() {
        Thread connectionStarter = new Thread(new Runnable() {
            @Override
            public void run() {
                try {
                    connection.start();
                } catch (Exception e) {
                    log.error("Unable to start listening on the message bus", e);
                }
            }               
        });
        connectionStarter.start();

    }

    @Override
    public synchronized void addListener(String destinationID, final MessageListener listener) {
        log.debug("Adding listener '{}' to destination: '{}' on message-bus '{}'.", 
                new Object[] {listener, destinationID, configuration.getName()});
        MessageConsumer consumer = getMessageConsumer(destinationID, listener);
        try {
            consumer.setMessageListener(new ActiveMQMessageListener(listener));
        } catch (JMSException e) {
            throw new CoordinationLayerException(
                    "Unable to add listener '" + listener + "' to destinationID '" + destinationID + "'", e);
        }
    }

    @Override
    public synchronized void addDurableListener(String destinationID, final MessageListener listener) {
        log.debug("Adding durable listener '{}' to destination: '{}' on message-bus '{}'.",
                new Object[] {listener, destinationID, configuration.getName()});
        MessageConsumer consumer = getDurableMessageConsumer(destinationID, listener);
        try {
            consumer.setMessageListener(new ActiveMQMessageListener(listener));
        } catch (JMSException e) {
            throw new CoordinationLayerException(
                    "Unable to add durable listener '" + listener + "' to destinationID '" + destinationID + "'", e);
        }
    }

    @Override
    public synchronized void removeListener(String destinationID, MessageListener listener) {
        log.debug("Removing listener '" + listener + "' from destination: '" + destinationID + "' " +
        		"on message-bus '" + configuration + "'.");
        MessageConsumer consumer = getMessageConsumer(destinationID, listener);
        try {
            // We need to set the listener to null to have the removeListerer take effect at once. 
            // If this isn't done the listener will continue to receive messages. Do we have a memory leak here? 
            consumer.setMessageListener(null);
            consumer.close();
        } catch (JMSException e) {
            throw new CoordinationLayerException(
                    "Unable to remove listener '" + listener + "' from destinationID '" + destinationID + "'", e);
        }
        consumers.remove(getConsumerHash(destinationID, listener));
{'fraction_non_alphanumeric': 0.07535677766416218, 'fraction_numerical': 0.0012892900013337482, 'mean_word_length': 3.9579016971567116, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2567058', 'n_tokens_mistral': 5478, 'n_tokens_neox': 5347, 'n_words': 1625}