<commit_before>/*******************************************************************************
 * This file is part of OpenNMS(R).
 *
 * Copyright (C) 2010-2014 The OpenNMS Group, Inc.
 * OpenNMS(R) is Copyright (C) 1999-2014 The OpenNMS Group, Inc.
 *
 * OpenNMS(R) is a registered trademark of The OpenNMS Group, Inc.
 *
 * OpenNMS(R) is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published
 * by the Free Software Foundation, either version 3 of the License,
 * or (at your option) any later version.
 *
 * OpenNMS(R) is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with OpenNMS(R).  If not, see:
 *      http://www.gnu.org/licenses/
 *
 * For more information contact:
 *     OpenNMS(R) Licensing <<EMAIL>>
 *     http://www.opennms.org/
 *     http://www.opennms.com/
 *******************************************************************************/

package org.opennms.netmgt.syslogd;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.time.Month;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.Locale;

import org.apache.commons.io.IOUtils;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.opennms.core.test.ConfigurationTestUtils;
import org.opennms.core.test.MockLogAppender;
import org.opennms.core.time.ZonedDateTimeBuilder;
import org.opennms.netmgt.config.SyslogdConfigFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class SyslogMessageTest {
    private static final Logger LOG = LoggerFactory.getLogger(SyslogMessageTest.class);

    private final SyslogdConfigFactory m_config;

    public SyslogMessageTest() throws Exception {
        InputStream stream = null;
        try {
            stream = ConfigurationTestUtils.getInputStreamForResource(this, "/etc/syslogd-configuration.xml");
            m_config = new SyslogdConfigFactory(stream);
        } finally {
            if (stream != null) {
                IOUtils.closeQuietly(stream);
            }
        }
    }
    
    @Before
    public void setUp() {
        MockLogAppender.setupLogging(true, "TRACE");
    }

    @Test
    public void testCustomParserWithProcess() throws Exception {
        final SyslogParser parser = new CustomSyslogParser(m_config, SyslogdTestUtils.toByteBuffer("<6>test: 2007-01-01 127.0.0.1 OpenNMS[1234]: A SyslogNG style message"));
        assertTrue(parser.find());
        final SyslogMessage message = parser.parse();

        assertEquals(SyslogFacility.KERNEL, message.getFacility());
        assertEquals(SyslogSeverity.INFORMATIONAL, message.getSeverity());
        assertEquals("test", message.getMessageID());
        assertEquals("127.0.0.1", message.getHostName());
        assertEquals("OpenNMS", message.getProcessName());
        assertEquals("1234", message.getProcessId());
        assertEquals("A SyslogNG style message", message.getMessage());
    }

    @Test
    public void testCustomParserWithSimpleForwardingRegexAndSyslog21Message() throws Exception {
        // see: http://searchdatacenter.techtarget.com/tip/Turn-aggregated-syslog-messages-into-OpenNMS-events

        final InputStream stream = new ByteArrayInputStream(("<syslogd-configuration>" +
                                                        "<configuration " +
                                                        "syslog-port=\"10514\" " +
                                                        "new-suspect-on-message=\"false\" " +
                                                        "forwarding-regexp=\"^((.+?) (.*))\\r?\\n?$\" " +
                                                        "matching-group-host=\"2\" " +
                                                        "matching-group-message=\"3\" " +
                                                        "discard-uei=\"DISCARD-MATCHING-MESSAGES\" " +
                                                        "/></syslogd-configuration>").getBytes());
        final SyslogdConfigFactory config = new SyslogdConfigFactory(stream);

        final SyslogParser parser = new CustomSyslogParser(config, SyslogdTestUtils.toByteBuffer("<173>Dec  7 12:02:06 10.13.110.116 mgmtd[8326]: [mgmtd.NOTICE]: Configuration saved to database initial"));
        assertTrue(parser.find());
        final SyslogMessage message = parser.parse();
        final Calendar calendar = new GregorianCalendar();
        calendar.set(Calendar.YEAR, ZonedDateTimeBuilder.getBestYearForMonth(Month.DECEMBER.getValue()));
        calendar.set(Calendar.MONTH, Calendar.DECEMBER);
        calendar.set(Calendar.DATE, 7);
        calendar.set(Calendar.HOUR_OF_DAY, 12);
        calendar.set(Calendar.MINUTE, 2);
        calendar.set(Calendar.SECOND, 6);
        calendar.set(Calendar.MILLISECOND, 0);
        final Date date = calendar.getTime();

        LOG.debug("got message: {}", message);

        assertEquals(SyslogFacility.LOCAL5, message.getFacility());
        assertEquals(SyslogSeverity.NOTICE, message.getSeverity());
        assertEquals(null, message.getMessageID());
        assertEquals(date, message.getDate());
        assertEquals("10.13.110.116", message.getHostName());
        assertEquals("mgmtd", message.getProcessName());
        assertEquals("8326", message.getProcessId());
        assertEquals("[mgmtd.NOTICE]: Configuration saved to database initial", message.getMessage());
    }
    
    @Test
    public void testCustomParserNms5242() throws Exception {
        final Locale startLocale = Locale.getDefault();
        try {
            Locale.setDefault(Locale.FRANCE);
            final InputStream stream = new ByteArrayInputStream(
               (
                   "<?xml version=\"1.0\"?>\n" +
                   "<syslogd-configuration>\n" +
                   "    <configuration\n" +
                   "            syslog-port=\"10514\"\n" +
                   "            new-suspect-on-message=\"false\"\n" +
                   "            parser=\"org.opennms.netmgt.syslogd.CustomSyslogParser\"\n" +
                   "            forwarding-regexp=\"^((.+?) (.*))\\n?$\"\n" +
                   "            matching-group-host=\"2\"\n" +
                   "            matching-group-message=\"3\"\n" +
                   "            discard-uei=\"DISCARD-MATCHING-MESSAGES\"\n" +
                   "            />\n" +
                   "\n" +
                   "    <hideMessage>\n" +
                   "        <hideMatch>\n" +
                   "            <match type=\"substr\" expression=\"TEST\"/>\n" +
                   "        </hideMatch>\n" +
                   "    </hideMessage>\n" +
                   "</syslogd-configuration>\n"
                ).getBytes()
            );
            final SyslogdConfigFactory config = new SyslogdConfigFactory(stream);

            final SyslogParser parser = new CustomSyslogParser(config, SyslogdTestUtils.toByteBuffer("<0>Mar 14 17:10:25 petrus sudo:  cyrille : user NOT in sudoers ; TTY=pts/2 ; PWD=/home/cyrille ; USER=root ; COMMAND=/usr/bin/vi /etc/aliases"));
            assertTrue(parser.find());
            final SyslogMessage message = parser.parse();
            LOG.debug("message = {}", message);
            final Calendar cal = Calendar.getInstance();
            cal.set(Calendar.YEAR, ZonedDateTimeBuilder.getBestYearForMonth(Calendar.MARCH));
            cal.set(Calendar.MONTH, Calendar.MARCH);
            cal.set(Calendar.DAY_OF_MONTH, 14);
            cal.set(Calendar.HOUR_OF_DAY, 17);
            cal.set(Calendar.MINUTE, 10);
            cal.set(Calendar.SECOND, 25);
            cal.set(Calendar.MILLISECOND, 0);
            assertEquals(SyslogFacility.KERNEL, message.getFacility());
            assertEquals(SyslogSeverity.EMERGENCY, message.getSeverity());
            assertNull(message.getMessageID());
            assertEquals(cal.getTime(), message.getDate());
            assertEquals("petrus", message.getHostName());
            assertEquals("sudo", message.getProcessName());
            assertEquals(null, message.getProcessId());
            assertEquals("cyrille : user NOT in sudoers ; TTY=pts/2 ; PWD=/home/cyrille ; USER=root ; COMMAND=/usr/bin/vi /etc/aliases", message.getMessage());
        } finally {
            Locale.setDefault(startLocale);
        }
    }
    
    @Test
    public void testSyslogNGParserWithProcess() throws Exception {
        final SyslogParser parser = new SyslogNGParser(m_config, SyslogdTestUtils.toByteBuffer("<6>test: 2007-01-01 127.0.0.1 OpenNMS[1234]: A SyslogNG style message"));
        assertTrue(parser.find());
        final SyslogMessage message = parser.parse();<commit_msg>[bamboo] Automated branch merge (from release-21.0.4:6463d1e7ea156bea22a84b87c010ab0fcbae43d1)<commit_after>/*******************************************************************************
 * This file is part of OpenNMS(R).
 *
 * Copyright (C) 2010-2014 The OpenNMS Group, Inc.
 * OpenNMS(R) is Copyright (C) 1999-2014 The OpenNMS Group, Inc.
 *
 * OpenNMS(R) is a registered trademark of The OpenNMS Group, Inc.
 *
 * OpenNMS(R) is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published
 * by the Free Software Foundation, either version 3 of the License,
 * or (at your option) any later version.
 *
 * OpenNMS(R) is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with OpenNMS(R).  If not, see:
 *      http://www.gnu.org/licenses/
 *
 * For more information contact:
 *     OpenNMS(R) Licensing <<EMAIL>>
 *     http://www.opennms.org/
 *     http://www.opennms.com/
 *******************************************************************************/

package org.opennms.netmgt.syslogd;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.time.Month;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.Locale;

import org.apache.commons.io.IOUtils;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.opennms.core.test.ConfigurationTestUtils;
import org.opennms.core.test.MockLogAppender;
import org.opennms.core.time.ZonedDateTimeBuilder;
import org.opennms.netmgt.config.SyslogdConfigFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class SyslogMessageTest {
    private static final Logger LOG = LoggerFactory.getLogger(SyslogMessageTest.class);

    private final SyslogdConfigFactory m_config;

    public SyslogMessageTest() throws Exception {
        InputStream stream = null;
        try {
            stream = ConfigurationTestUtils.getInputStreamForResource(this, "/etc/syslogd-configuration.xml");
            m_config = new SyslogdConfigFactory(stream);
        } finally {
            if (stream != null) {
                IOUtils.closeQuietly(stream);
            }
        }
    }
    
    @Before
    public void setUp() {
        MockLogAppender.setupLogging(true, "TRACE");
    }

    @Test
    public void testCustomParserWithProcess() throws Exception {
        final SyslogParser parser = new CustomSyslogParser(m_config, SyslogdTestUtils.toByteBuffer("<6>test: 2007-01-01 127.0.0.1 OpenNMS[1234]: A SyslogNG style message"));
        assertTrue(parser.find());
        final SyslogMessage message = parser.parse();

        assertEquals(SyslogFacility.KERNEL, message.getFacility());
        assertEquals(SyslogSeverity.INFORMATIONAL, message.getSeverity());
        assertEquals("test", message.getMessageID());
        assertEquals("127.0.0.1", message.getHostName());
        assertEquals("OpenNMS", message.getProcessName());
        assertEquals("1234", message.getProcessId());
        assertEquals("A SyslogNG style message", message.getMessage());
    }

    @Test
    public void testCustomParserWithSimpleForwardingRegexAndSyslog21Message() throws Exception {
        // see: http://searchdatacenter.techtarget.com/tip/Turn-aggregated-syslog-messages-into-OpenNMS-events

        final InputStream stream = new ByteArrayInputStream(("<syslogd-configuration>" +
                                                        "<configuration " +
                                                        "syslog-port=\"10514\" " +
                                                        "new-suspect-on-message=\"false\" " +
                                                        "forwarding-regexp=\"^((.+?) (.*))\\r?\\n?$\" " +
                                                        "matching-group-host=\"2\" " +
                                                        "matching-group-message=\"3\" " +
                                                        "discard-uei=\"DISCARD-MATCHING-MESSAGES\" " +
                                                        "/></syslogd-configuration>").getBytes());
        final SyslogdConfigFactory config = new SyslogdConfigFactory(stream);

        final SyslogParser parser = new CustomSyslogParser(config, SyslogdTestUtils.toByteBuffer("<173>Dec  7 12:02:06 10.13.110.116 mgmtd[8326]: [mgmtd.NOTICE]: Configuration saved to database initial"));
        assertTrue(parser.find());
        final SyslogMessage message = parser.parse();
        final Calendar calendar = new GregorianCalendar();
        calendar.set(Calendar.YEAR, ZonedDateTimeBuilder.getBestYearForMonth(Month.DECEMBER.getValue()));
        calendar.set(Calendar.MONTH, Calendar.DECEMBER);
        calendar.set(Calendar.DATE, 7);
        calendar.set(Calendar.HOUR_OF_DAY, 12);
        calendar.set(Calendar.MINUTE, 2);
        calendar.set(Calendar.SECOND, 6);
        calendar.set(Calendar.MILLISECOND, 0);
        final Date date = calendar.getTime();

        LOG.debug("got message: {}", message);

        assertEquals(SyslogFacility.LOCAL5, message.getFacility());
        assertEquals(SyslogSeverity.NOTICE, message.getSeverity());
        assertEquals(null, message.getMessageID());
        assertEquals(date, message.getDate());
        assertEquals("10.13.110.116", message.getHostName());
        assertEquals("mgmtd", message.getProcessName());
        assertEquals("8326", message.getProcessId());
        assertEquals("[mgmtd.NOTICE]: Configuration saved to database initial", message.getMessage());
    }
    
    @Test
    public void testCustomParserNms5242() throws Exception {
        final Locale startLocale = Locale.getDefault();
        try {
            Locale.setDefault(Locale.FRANCE);
            final InputStream stream = new ByteArrayInputStream(
               (
                   "<?xml version=\"1.0\"?>\n" +
                   "<syslogd-configuration>\n" +
                   "    <configuration\n" +
                   "            syslog-port=\"10514\"\n" +
                   "            new-suspect-on-message=\"false\"\n" +
                   "            parser=\"org.opennms.netmgt.syslogd.CustomSyslogParser\"\n" +
                   "            forwarding-regexp=\"^((.+?) (.*))\\n?$\"\n" +
                   "            matching-group-host=\"2\"\n" +
                   "            matching-group-message=\"3\"\n" +
                   "            discard-uei=\"DISCARD-MATCHING-MESSAGES\"\n" +
                   "            />\n" +
                   "\n" +
                   "    <hideMessage>\n" +
                   "        <hideMatch>\n" +
                   "            <match type=\"substr\" expression=\"TEST\"/>\n" +
                   "        </hideMatch>\n" +
                   "    </hideMessage>\n" +
                   "</syslogd-configuration>\n"
                ).getBytes()
            );
            final SyslogdConfigFactory config = new SyslogdConfigFactory(stream);

            final SyslogParser parser = new CustomSyslogParser(config, SyslogdTestUtils.toByteBuffer("<0>Mar 14 17:10:25 petrus sudo:  cyrille : user NOT in sudoers ; TTY=pts/2 ; PWD=/home/cyrille ; USER=root ; COMMAND=/usr/bin/vi /etc/aliases"));
            assertTrue(parser.find());
            final SyslogMessage message = parser.parse();
            LOG.debug("message = {}", message);
            final Calendar cal = Calendar.getInstance();
            cal.set(Calendar.YEAR, ZonedDateTimeBuilder.getBestYearForMonth(Month.MARCH.getValue()));
            cal.set(Calendar.MONTH, Calendar.MARCH);
            cal.set(Calendar.DAY_OF_MONTH, 14);
            cal.set(Calendar.HOUR_OF_DAY, 17);
            cal.set(Calendar.MINUTE, 10);
            cal.set(Calendar.SECOND, 25);
            cal.set(Calendar.MILLISECOND, 0);
            assertEquals(SyslogFacility.KERNEL, message.getFacility());
            assertEquals(SyslogSeverity.EMERGENCY, message.getSeverity());
            assertNull(message.getMessageID());
            assertEquals(cal.getTime(), message.getDate());
            assertEquals("petrus", message.getHostName());
            assertEquals("sudo", message.getProcessName());
            assertEquals(null, message.getProcessId());
            assertEquals("cyrille : user NOT in sudoers ; TTY=pts/2 ; PWD=/home/cyrille ; USER=root ; COMMAND=/usr/bin/vi /etc/aliases", message.getMessage());
        } finally {
            Locale.setDefault(startLocale);
        }
    }
    
    @Test
    public void testSyslogNGParserWithProcess() throws Exception {
        final SyslogParser parser = new SyslogNGParser(m_config, SyslogdTestUtils.toByteBuffer("<6>test: 2007-01-01 127.0.0.1 OpenNMS[1234]: A SyslogNG style message"));
        assertTrue(parser.find());
        final SyslogMessage message = parser.parse();
{'fraction_non_alphanumeric': 0.134535666218035, 'fraction_numerical': 0.017981157469717363, 'mean_word_length': 2.314775160599572, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 2, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 4}, 'pii_count': 10, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3440742', 'n_tokens_mistral': 5294, 'n_tokens_neox': 4935, 'n_words': 1014}