<commit_before>/*
*  Copyright (c) 2015, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
*
*  WSO2 Inc. licenses this file to you under the Apache License,
*  Version 2.0 (the "License"); you may not use this file except
*  in compliance with the License.
*  You may obtain a copy of the License at
*
*    http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied.  See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.wso2.carbon.analytics.eventsink.internal;

import com.google.gson.Gson;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.wso2.carbon.analytics.datasource.commons.AnalyticsSchema;
import org.wso2.carbon.analytics.datasource.commons.ColumnDefinition;
import org.wso2.carbon.analytics.datasource.commons.Record;
import org.wso2.carbon.analytics.datasource.commons.exception.AnalyticsException;
import org.wso2.carbon.analytics.datasource.core.util.GenericUtils;
import org.wso2.carbon.analytics.eventsink.internal.jmx.EventCounter;
import org.wso2.carbon.analytics.eventsink.internal.util.AnalyticsEventSinkConstants;
import org.wso2.carbon.analytics.eventsink.internal.util.AnalyticsEventSinkUtil;
import org.wso2.carbon.analytics.eventsink.internal.util.ServiceHolder;
import org.wso2.carbon.databridge.commons.Attribute;
import org.wso2.carbon.databridge.commons.Event;
import org.wso2.carbon.databridge.commons.StreamDefinition;
import org.wso2.carbon.databridge.core.definitionstore.AbstractStreamDefinitionStore;
import org.wso2.carbon.databridge.core.exception.StreamDefinitionStoreException;
import org.wso2.carbon.utils.CarbonUtils;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * Analytics data service connector which actually makes the interacts with DS.
 */
public class AnalyticsDSConnector {
    private static final Log log = LogFactory.getLog(AnalyticsDSConnector.class);

    private Gson gson;

    private AtomicInteger recordsPersisted;
    private AtomicInteger totalRecordCounter;
    private long startTime;
    private boolean isProfilePersistence;

    public AnalyticsDSConnector() {
        gson = new Gson();
        String profileReceiver = System.getProperty("profilePersistence");
        if (profileReceiver != null && profileReceiver.equalsIgnoreCase("true")) {
            isProfilePersistence = true;
            recordsPersisted = new AtomicInteger();
            totalRecordCounter = new AtomicInteger();
            startTime = 0;
        }
    }

    public void insertEvents(int tenantId, List<Event> events) throws StreamDefinitionStoreException,
            AnalyticsException {
        if (!ServiceHolder.getEventPublisherManagementService().isDrop()) {
            //In CEP HA setup the same event will be sent along the cluster, and hence only the leader
            // will need to store the event to avoid the duplicate events stored..
            List<Record> records = this.convertEventsToRecord(tenantId, events);
            EventCounter.incrementAndGet(events.size());
            startTimeMeasurement();
            ServiceHolder.getAnalyticsDataAPI().put(records);
            endTimeMeasurement(records.size());
        }
    }

    private void endTimeMeasurement(int recordCount) {
        if (isProfilePersistence) {
            recordsPersisted.addAndGet(recordCount);
            if (recordsPersisted.get() > 100000) {
                synchronized (this) {
                    if (recordsPersisted.get() > 100000) {
                        DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
                        Date date = new Date();

                        long endTime = System.currentTimeMillis();
                        int currentBatchSize = recordsPersisted.getAndSet(0);
                        totalRecordCounter.addAndGet(currentBatchSize);

                        String line = "[" + dateFormat.format(date) + "] # of records : " + currentBatchSize +
                                " start timestamp : " + startTime +
                                " end time stamp : " + endTime + " Throughput is (records / sec) : " +
                                (currentBatchSize * 1000) / (endTime - startTime) + " Total Record Count : " +
                                totalRecordCounter + " \n";
                        File file = new File(CarbonUtils.getCarbonHome() + File.separator + "persistence-perf.txt");
                        if (!file.exists()) {
                            log.info("Creating the performance measurement file at : " + file.getAbsolutePath());
                        }
                        try {
                            appendToFile(IOUtils.toInputStream(line), file);
                        } catch (IOException e) {
                            log.error(e.getMessage(), e);
                        }
                        startTime = 0;
                    }
                }
            }
        }

    }

    public void appendToFile(final InputStream in, final File f) throws IOException {
        OutputStream stream = null;
        try {
            stream = new BufferedOutputStream(new FileOutputStream(f, true));
            IOUtils.copy(in, stream);
        } finally {
            IOUtils.closeQuietly(stream);
        }
    }

    private void startTimeMeasurement() {
        if (isProfilePersistence) {
            if (startTime == 0) {
                synchronized (this) {
                    if (startTime == 0) {
                        startTime = System.currentTimeMillis();<commit_msg>Making persistence stats cutoff point configurable<commit_after>/*
*  Copyright (c) 2015, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
*
*  WSO2 Inc. licenses this file to you under the Apache License,
*  Version 2.0 (the "License"); you may not use this file except
*  in compliance with the License.
*  You may obtain a copy of the License at
*
*    http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied.  See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.wso2.carbon.analytics.eventsink.internal;

import com.google.gson.Gson;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.wso2.carbon.analytics.datasource.commons.AnalyticsSchema;
import org.wso2.carbon.analytics.datasource.commons.ColumnDefinition;
import org.wso2.carbon.analytics.datasource.commons.Record;
import org.wso2.carbon.analytics.datasource.commons.exception.AnalyticsException;
import org.wso2.carbon.analytics.datasource.core.util.GenericUtils;
import org.wso2.carbon.analytics.eventsink.internal.jmx.EventCounter;
import org.wso2.carbon.analytics.eventsink.internal.util.AnalyticsEventSinkConstants;
import org.wso2.carbon.analytics.eventsink.internal.util.AnalyticsEventSinkUtil;
import org.wso2.carbon.analytics.eventsink.internal.util.ServiceHolder;
import org.wso2.carbon.databridge.commons.Attribute;
import org.wso2.carbon.databridge.commons.Event;
import org.wso2.carbon.databridge.commons.StreamDefinition;
import org.wso2.carbon.databridge.core.definitionstore.AbstractStreamDefinitionStore;
import org.wso2.carbon.databridge.core.exception.StreamDefinitionStoreException;
import org.wso2.carbon.utils.CarbonUtils;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * Analytics data service connector which actually makes the interacts with DS.
 */
public class AnalyticsDSConnector {
    private static final Log log = LogFactory.getLog(AnalyticsDSConnector.class);

    private Gson gson;

    private AtomicInteger recordsPersisted;
    private AtomicInteger totalRecordCounter;
    private long startTime;
    private boolean isProfilePersistence;
    private int cutoff = 100000;

    public AnalyticsDSConnector() {
        gson = new Gson();
        String profileReceiver = System.getProperty("profilePersistence");
        if (profileReceiver != null && profileReceiver.equalsIgnoreCase("true")) {
            isProfilePersistence = true;
            recordsPersisted = new AtomicInteger();
            totalRecordCounter = new AtomicInteger();
            startTime = 0;
            String cutoffInput = System.getProperty("persistenceStatsCutoff");
            if (cutoffInput != null && StringUtils.isNumeric(cutoffInput)) {
                this.cutoff = Integer.parseInt(cutoffInput);
            }
        }
    }

    public void insertEvents(int tenantId, List<Event> events) throws StreamDefinitionStoreException,
            AnalyticsException {
        if (!ServiceHolder.getEventPublisherManagementService().isDrop()) {
            //In CEP HA setup the same event will be sent along the cluster, and hence only the leader
            // will need to store the event to avoid the duplicate events stored..
            List<Record> records = this.convertEventsToRecord(tenantId, events);
            EventCounter.incrementAndGet(events.size());
            startTimeMeasurement();
            ServiceHolder.getAnalyticsDataAPI().put(records);
            endTimeMeasurement(records.size());
        }
    }

    private void endTimeMeasurement(int recordCount) {
        if (isProfilePersistence) {
            recordsPersisted.addAndGet(recordCount);
            if (recordsPersisted.get() > cutoff) {
                synchronized (this) {
                    if (recordsPersisted.get() > cutoff) {
                        DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
                        Date date = new Date();

                        long endTime = System.currentTimeMillis();
                        int currentBatchSize = recordsPersisted.getAndSet(0);
                        totalRecordCounter.addAndGet(currentBatchSize);

                        String line = "[" + dateFormat.format(date) + "] # of records : " + currentBatchSize +
                                " start timestamp : " + startTime +
                                " end time stamp : " + endTime + " Throughput is (records / sec) : " +
                                (currentBatchSize * 1000) / (endTime - startTime) + " Total Record Count : " +
                                totalRecordCounter + " \n";
                        File file = new File(CarbonUtils.getCarbonHome() + File.separator + "persistence-perf.txt");
                        if (!file.exists()) {
                            log.info("Creating the data persistence performance measurement log at : " + file.getAbsolutePath());
                        }
                        try {
                            appendToFile(IOUtils.toInputStream(line), file);
                        } catch (IOException e) {
                            log.error(e.getMessage(), e);
                        }
                        startTime = 0;
                    }
                }
            }
        }

    }

    public void appendToFile(final InputStream in, final File f) throws IOException {
        OutputStream stream = null;
        try {
            stream = new BufferedOutputStream(new FileOutputStream(f, true));
            IOUtils.copy(in, stream);
        } finally {
            IOUtils.closeQuietly(stream);
        }
    }

    private void startTimeMeasurement() {
        if (isProfilePersistence) {
            if (startTime == 0) {
                synchronized (this) {
                    if (startTime == 0) {
                        startTime = System.currentTimeMillis();
{'fraction_non_alphanumeric': 0.09152781037315184, 'fraction_numerical': 0.007197058593444419, 'mean_word_length': 2.479586281981492, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6547525', 'n_tokens_mistral': 3431, 'n_tokens_neox': 3276, 'n_words': 799}