<commit_before>/*
*  Copyright (c) 2016, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
*
*  WSO2 Inc. licenses this file to you under the Apache License,
*  Version 2.0 (the "License"); you may not use this file except
*  in compliance with the License.
*  You may obtain a copy of the License at
*
*    http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied.  See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.wso2.analytics.apim.integration.tests.apim.analytics;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import org.wso2.carbon.event.simulator.stub.types.EventDto;

public class AbnormalRequestCountTestCase extends APIMAnalyticsBaseTestCase {
    private static final Log log = LogFactory.getLog(AbnormalRequestCountTestCase.class);

    private final String STREAM_NAME = "org.wso2.apimgt.statistics.request";
    private final String STREAM_VERSION = "1.1.0";
    private final String TEST_RESOURCE_PATH = "abnormalRequestCount";
    private final String PUBLISHER_FILE = "logger_abnormalRequestCount.xml";
    private final String SPARK_SCRIPT = "org_wso2_analytics_apim_request_stat_generator";
    private final String REQUEST_PERCENTILE_TABLE = "ORG_WSO2_ANALYTICS_APIM_REQUESTPERCENTILE";
    private final String REQUEST_COUNT_PER_MINUTE_TABLE = "org_wso2_analytics_apim_store_requestPerMinPerApiStream";
    private final int MAX_TRIES = 20;

    @BeforeClass(alwaysRun = true)
    public void setup() throws Exception {
        super.init();
        if (isTableExist(-1234, STREAM_NAME.replace('.', '_'))) {
            deleteData(-1234, STREAM_NAME.replace('.', '_'));
        }
        if (isTableExist(-1234, REQUEST_PERCENTILE_TABLE)) {
            deleteData(-1234, REQUEST_PERCENTILE_TABLE);
        }
        if (isTableExist(-1234, REQUEST_COUNT_PER_MINUTE_TABLE)) {
            deleteData(-1234, REQUEST_COUNT_PER_MINUTE_TABLE);
        }
        // deploy the publisher xml files
        deployPublisher(TEST_RESOURCE_PATH, PUBLISHER_FILE);
    }

    @AfterClass(alwaysRun = true)
    public void cleanup() throws Exception {
        if (isTableExist(-1234, STREAM_NAME.replace('.', '_'))) {
            deleteData(-1234, STREAM_NAME.replace('.', '_'));
        }
        if (isTableExist(-1234, REQUEST_PERCENTILE_TABLE)) {
            deleteData(-1234, REQUEST_PERCENTILE_TABLE);
        }
        if (isTableExist(-1234, REQUEST_COUNT_PER_MINUTE_TABLE)) {
            deleteData(-1234, REQUEST_COUNT_PER_MINUTE_TABLE);
        }
        // undeploy the publishers
        undeployPublisher(PUBLISHER_FILE);
    }

    @Test(groups = "wso2.analytics.apim", description = "Tests if the Spark script is deployed")
    public void testRequestStatGeneratorSparkScriptDeployment() throws Exception {
        Assert.assertTrue(isSparkScriptExists(SPARK_SCRIPT), "org_wso2_analytics_apim_request_stat_generator spark script is not deployed!");
    }

    @Test(groups = "wso2.analytics.apim", description = "Test if the Simulation data has been published"
            , dependsOnMethods = "testRequestStatGeneratorSparkScriptDeployment")
    public void testRequestSimulationDataSent() throws Exception {

        //publish events
        pubishEventsFromCSV(TEST_RESOURCE_PATH, "sim.csv", getStreamId(STREAM_NAME, STREAM_VERSION), 100);
        Thread.sleep(70000);
        pubishEventsFromCSV(TEST_RESOURCE_PATH, "sim.csv", getStreamId(STREAM_NAME, STREAM_VERSION), 100);
        Thread.sleep(60000);
        int i = 0;
        boolean eventsPublished = false;
        while (i < MAX_TRIES) {
            long requestPerMinuteEventCount = getRecordCount(-1234, STREAM_NAME.replace('.', '_'));
            eventsPublished = (requestPerMinuteEventCount >= 4);
            if (eventsPublished) {
                break;
            }
            i++;
            Thread.sleep(10000);
        }

        Assert.assertTrue(eventsPublished, "Simulation events did not get published!");
    }

    @Test(groups = "wso2.analytics.apim", description = "Test org_wso2_analytics_apim_request_stat_generator Spark Script execution"
            , dependsOnMethods = "testRequestSimulationDataSent")
    public void testRequestStatGeneratorSparkScriptExecution() throws Exception {
        //run the script
        executeSparkScript(SPARK_SCRIPT);
        int i = 0;
        boolean scriptExecuted = false;
        while (i < MAX_TRIES) {
            Thread.sleep(10000);
            long percentileTableCount = getRecordCount(-1234, REQUEST_PERCENTILE_TABLE);
            scriptExecuted = (percentileTableCount == 2);
            if (scriptExecuted) {
                break;
            }
            i++;
        }
        Assert.assertTrue(scriptExecuted, "Spark script did not execute as expected!");
    }

    @Test(groups = "wso2.analytics.apim", description = "Test Abnormal ResponseTime Alert",
            dependsOnMethods = "testRequestStatGeneratorSparkScriptExecution")
    public void testAbnormalResponseTimeAlert() throws Exception {
        int initialCount = logViewerClient.getAllRemoteSystemLogs().length;

        EventDto eventDto = new EventDto();
        eventDto.setEventStreamId(getStreamId(STREAM_NAME, STREAM_VERSION));
        eventDto.setAttributeValues(
                new String[]{"external","s8SWbnmzQEgzMIsol7AHt9cjhEsa","/calc/1.0","CalculatorAPI:v1.0","CalculatorAPI",
                        "/add?x=12&y=3","/add","GET","1.0","1","1456894602313","<EMAIL>","carbon.super","192.168.66.1",
                        "<EMAIL>","DefaultApplication","1","Mozilla/5.0","Unlimited","False","127.0.01"}
        );
        publishEvent(eventDto);

        eventDto = new EventDto();
        eventDto.setEventStreamId(getStreamId(STREAM_NAME, STREAM_VERSION));
        eventDto.setAttributeValues(
                new String[]{"external","s8SWbnmzQEgzMIsol7AHt9cjhEsa","/calc/1.0","CalculatorAPI:v1.0","CalculatorAPI",
                        "/add?x=12&y=3","/add","GET","1.0","1","1456894603940","<EMAIL>","carbon.super","192.168.66.1",
                        "<EMAIL>","DefaultApplication","1","Mozilla/5.0","Unlimited","False","127.0.01"}
        );
        publishEvent(eventDto);

        boolean abnormalRequestCountAlertTriggered = isAlertReceived(initialCount, "Unique ID: logger_abnormalRequestCount", 5 ,5000);
        Assert.assertTrue(abnormalRequestCountAlertTriggered, "Abnormal Response Count Alert event not received!");
    }<commit_msg>Improving AbnormalRequestCountTestCase<commit_after>/*
*  Copyright (c) 2016, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
*
*  WSO2 Inc. licenses this file to you under the Apache License,
*  Version 2.0 (the "License"); you may not use this file except
*  in compliance with the License.
*  You may obtain a copy of the License at
*
*    http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied.  See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.wso2.analytics.apim.integration.tests.apim.analytics;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import org.wso2.carbon.event.simulator.stub.types.EventDto;

public class AbnormalRequestCountTestCase extends APIMAnalyticsBaseTestCase {
    private static final Log log = LogFactory.getLog(AbnormalRequestCountTestCase.class);

    private final String STREAM_NAME = "org.wso2.apimgt.statistics.request";
    private final String STREAM_VERSION = "1.1.0";
    private final String TEST_RESOURCE_PATH = "abnormalRequestCount";
    private final String PUBLISHER_FILE = "logger_abnormalRequestCount.xml";
    private final String SPARK_SCRIPT = "org_wso2_analytics_apim_request_stat_generator";
    private final String REQUEST_PERCENTILE_TABLE = "ORG_WSO2_ANALYTICS_APIM_REQUESTPERCENTILE";
    private final String REQUEST_COUNT_PER_MINUTE_TABLE = "org_wso2_analytics_apim_store_requestPerMinPerApiStream";
    private final int MAX_TRIES = 20;

    @BeforeClass(alwaysRun = true)
    public void setup() throws Exception {
        super.init();
        if (isTableExist(-1234, STREAM_NAME.replace('.', '_'))) {
            deleteData(-1234, STREAM_NAME.replace('.', '_'));
        }
        if (isTableExist(-1234, REQUEST_PERCENTILE_TABLE)) {
            deleteData(-1234, REQUEST_PERCENTILE_TABLE);
        }
        if (isTableExist(-1234, REQUEST_COUNT_PER_MINUTE_TABLE)) {
            deleteData(-1234, REQUEST_COUNT_PER_MINUTE_TABLE);
        }
        // deploy the publisher xml files
        deployPublisher(TEST_RESOURCE_PATH, PUBLISHER_FILE);
    }

    @AfterClass(alwaysRun = true)
    public void cleanup() throws Exception {
        if (isTableExist(-1234, STREAM_NAME.replace('.', '_'))) {
            deleteData(-1234, STREAM_NAME.replace('.', '_'));
        }
        if (isTableExist(-1234, REQUEST_PERCENTILE_TABLE)) {
            deleteData(-1234, REQUEST_PERCENTILE_TABLE);
        }
        if (isTableExist(-1234, REQUEST_COUNT_PER_MINUTE_TABLE)) {
            deleteData(-1234, REQUEST_COUNT_PER_MINUTE_TABLE);
        }
        // undeploy the publishers
        undeployPublisher(PUBLISHER_FILE);
    }

    @Test(groups = "wso2.analytics.apim", description = "Tests if the Spark script is deployed")
    public void testRequestStatGeneratorSparkScriptDeployment() throws Exception {
        Assert.assertTrue(isSparkScriptExists(SPARK_SCRIPT), "org_wso2_analytics_apim_request_stat_generator spark script is not deployed!");
    }

    @Test(groups = "wso2.analytics.apim", description = "Test if the Simulation data has been published"
            , dependsOnMethods = "testRequestStatGeneratorSparkScriptDeployment")
    public void testRequestSimulationDataSent() throws Exception {

        //publish events
        pubishEventsFromCSV(TEST_RESOURCE_PATH, "sim.csv", getStreamId(STREAM_NAME, STREAM_VERSION), 100);
        Thread.sleep(70000);
        pubishEventsFromCSV(TEST_RESOURCE_PATH, "sim.csv", getStreamId(STREAM_NAME, STREAM_VERSION), 100);
        Thread.sleep(60000);
        int i = 0;
        boolean eventsPublished = false;
        while (i < MAX_TRIES) {
            long requestPerMinuteEventCount = getRecordCount(-1234, STREAM_NAME.replace('.', '_'));
            eventsPublished = (requestPerMinuteEventCount >= 4);
            if (eventsPublished) {
                break;
            }
            i++;
            Thread.sleep(10000);
        }

        Assert.assertTrue(eventsPublished, "Simulation events did not get published!");
    }

    @Test(groups = "wso2.analytics.apim", description = "Test org_wso2_analytics_apim_request_stat_generator Spark Script execution"
            , dependsOnMethods = "testRequestSimulationDataSent")
    public void testRequestStatGeneratorSparkScriptExecution() throws Exception {
        //run the script
        executeSparkScript(SPARK_SCRIPT);
        int i = 0;
        boolean scriptExecuted = false;
        while (i < MAX_TRIES) {
            Thread.sleep(10000);
            long percentileTableCount = getRecordCount(-1234, REQUEST_PERCENTILE_TABLE);
            scriptExecuted = (percentileTableCount == 2);
            if (scriptExecuted) {
                break;
            }
            i++;
        }
        Assert.assertTrue(scriptExecuted, "Spark script did not execute as expected!");
    }

    @Test(groups = "wso2.analytics.apim", description = "Test Abnormal ResponseTime Alert",
            dependsOnMethods = "testRequestStatGeneratorSparkScriptExecution")
    public void testAbnormalRequestCountAlert() throws Exception {
        int initialCount = logViewerClient.getAllRemoteSystemLogs().length;

        EventDto eventDto = new EventDto();
        eventDto.setEventStreamId(getStreamId(STREAM_NAME, STREAM_VERSION));
        eventDto.setAttributeValues(
                new String[]{"external","s8SWbnmzQEgzMIsol7AHt9cjhEsa","/calc/1.0","CalculatorAPI:v1.0","CalculatorAPI",
                        "/add?x=12&y=3","/add","GET","1.0","1","1456894602313","<EMAIL>","carbon.super","192.168.66.1",
                        "<EMAIL>","DefaultApplication","1","Mozilla/5.0","Unlimited","False","127.0.01"}
        );

        for (int i=0;i<15;i++){
            publishEvent(eventDto);
        }

        boolean abnormalRequestCountAlertTriggered = isAlertReceived(initialCount, "Unique ID: logger_abnormalRequestCount", 5 ,5000);
        Assert.assertTrue(abnormalRequestCountAlertTriggered, "Abnormal Response Count Alert event not received!");
    }
{'fraction_non_alphanumeric': 0.10504867557165497, 'fraction_numerical': 0.028148818957059844, 'mean_word_length': 3.350623768877216, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 4}, 'pii_count': 3, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2022562', 'n_tokens_mistral': 4141, 'n_tokens_neox': 3845, 'n_words': 799}