<commit_before>package communicator;

/*
*  Copyright (c) 2016, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
*
*  WSO2 Inc. licenses this file to you under the Apache License,
*  Version 2.0 (the "License"); you may not use this file except
*  in compliance with the License.
*  You may obtain a copy of the License at
*
*    http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied.  See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import org.wso2.carbon.databridge.agent.AgentHolder;
import org.wso2.carbon.databridge.agent.DataPublisher;
import org.wso2.carbon.databridge.agent.exception.DataEndpointAgentConfigurationException;
import org.wso2.carbon.databridge.agent.exception.DataEndpointAuthenticationException;
import org.wso2.carbon.databridge.agent.exception.DataEndpointConfigurationException;
import org.wso2.carbon.databridge.agent.exception.DataEndpointException;
import org.wso2.carbon.databridge.commons.Event;
import org.wso2.carbon.databridge.commons.exception.TransportException;
import org.wso2.carbon.databridge.commons.utils.DataBridgeCommonsUtils;

import java.io.File;
import java.net.*;
import java.util.Enumeration;

public class HttpdLogAgent {
    private static final String HTTPD_LOG_STREAM = "testS";
    private static final String VERSION = "1.0.0";
    private static final int defaultThriftPort = 7611;
    private static final int defaultBinaryPort = 9611;
    private static final String pid = "14005";

    public static void main(String[] args) throws DataEndpointAuthenticationException,
            DataEndpointAgentConfigurationException,
            TransportException,
            DataEndpointException,
            DataEndpointConfigurationException,
            SocketException,
            UnknownHostException {

        System.out.println("Starting DAS HttpLog Agent");
        String currentDir = System.getProperty("user.dir");
        System.setProperty("javax.net.ssl.trustStore", currentDir + "/jvm-monitor-agent/src/main/resources/client-truststore.jks");
        System.setProperty("javax.net.ssl.trustStorePassword", "<PASSWORD>");

        AgentHolder.setConfigPath(getDataAgentConfigPath());
        String host = getLocalAddress().getHostAddress();

        String type = getProperty("type", "Thrift");
        int receiverPort = defaultThriftPort;
        if (type.equals("Binary")) {
            receiverPort = defaultBinaryPort;
        }
        int securePort = receiverPort + 100;

        String url = getProperty("url", "tcp://" + host + ":" + receiverPort);
        String authURL = getProperty("authURL", "ssl://" + host + ":" + securePort);
        String username = getProperty("username", "admin");
        String password = getProperty("password", "<PASSWORD>");

        DataPublisher dataPublisher = new DataPublisher(type, url, authURL, username, password);

        String streamId = DataBridgeCommonsUtils.generateStreamId(HTTPD_LOG_STREAM, VERSION);
        publishLogEvents(dataPublisher, streamId);
        dataPublisher.shutdown();
    }

    public static String getDataAgentConfigPath() {
        File filePath = new File("jvm-monitor-agent" + File.separator + "src" + File.separator + "main" + File.separator + "resources");
        if (!filePath.exists()) {
            filePath = new File("test" + File.separator + "resources");
        }
        if (!filePath.exists()) {
            filePath = new File("resources");
        }
        if (!filePath.exists()) {
            filePath = new File("test" + File.separator + "resources");
        }
        return filePath.getAbsolutePath() + File.separator + "data-agent-conf.xml";
    }

    private static void publishLogEvents(DataPublisher dataPublisher, String streamId) {

        int i = 1;
        while (true) {
            System.out.println("Publish Memory data : " + i);

//            MemoryMonitor agentObj = new MemoryMonitor();
//            MemoryMXBean mxb = agentObj.getMemoryBean(pid);
//            ArrayList gcData = agentObj.getRamInfoHtml(mxb);

//            Event event = new Event(streamId, System.currentTimeMillis(), new Object[]{"external"}, null,
//                    new Object[]{gcData.get(0), gcData.get(1), gcData.get(2), gcData.get(3), gcData.get(4),
//                            gcData.get(5), gcData.get(6)});
//            dataPublisher.publish(event);

            Event event = new Event( streamId, System.currentTimeMillis(), null, null,
                    new Object[]{1, "testNew8"} );
            dataPublisher.publish(event);

            i++;

            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }


    }

    public static InetAddress getLocalAddress() throws SocketException, UnknownHostException {
        Enumeration<NetworkInterface> ifaces = NetworkInterface.getNetworkInterfaces();
        while (ifaces.hasMoreElements()) {
            NetworkInterface iface = ifaces.nextElement();
            Enumeration<InetAddress> addresses = iface.getInetAddresses();

            while (addresses.hasMoreElements()) {
                InetAddress addr = addresses.nextElement();
                if (addr instanceof Inet4Address && !addr.isLoopbackAddress()) {<commit_msg>Create usage data to send to DAS<commit_after>package communicator;

/*
*  Copyright (c) 2016, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
*
*  WSO2 Inc. licenses this file to you under the Apache License,
*  Version 2.0 (the "License"); you may not use this file except
*  in compliance with the License.
*  You may obtain a copy of the License at
*
*    http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied.  See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import com.sun.tools.attach.AttachNotSupportedException;
import jvmmonitor.LogManager;
import jvmmonitor.exceptions.MonitoringNotStartedException;
import jvmmonitor.management.GarbageCollectionLog;
import jvmmonitor.management.MemoryUsageLog;
import org.wso2.carbon.databridge.agent.AgentHolder;
import org.wso2.carbon.databridge.agent.DataPublisher;
import org.wso2.carbon.databridge.agent.exception.DataEndpointAgentConfigurationException;
import org.wso2.carbon.databridge.agent.exception.DataEndpointAuthenticationException;
import org.wso2.carbon.databridge.agent.exception.DataEndpointConfigurationException;
import org.wso2.carbon.databridge.agent.exception.DataEndpointException;
import org.wso2.carbon.databridge.commons.Event;
import org.wso2.carbon.databridge.commons.exception.TransportException;
import org.wso2.carbon.databridge.commons.utils.DataBridgeCommonsUtils;

import javax.management.MalformedObjectNameException;
import java.io.File;
import java.io.IOException;
import java.net.*;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.Map;

public class HttpdLogAgent {
    private static final String HTTPD_LOG_STREAM = "testS";
    private static final String VERSION = "1.0.0";
    private static final int defaultThriftPort = 7611;
    private static final int defaultBinaryPort = 9611;
    private static final String pid = "14005";

    public static void main(String[] args) throws DataEndpointAuthenticationException,
            DataEndpointAgentConfigurationException,
            TransportException,
            DataEndpointException,
            DataEndpointConfigurationException,
            SocketException,
            UnknownHostException {

        System.out.println("Starting DAS HttpLog Agent");
        String currentDir = System.getProperty("user.dir");
        System.setProperty("javax.net.ssl.trustStore", currentDir + "/jvm-monitor-agent/src/main/resources/client-truststore.jks");
        System.setProperty("javax.net.ssl.trustStorePassword", "<PASSWORD>");

        AgentHolder.setConfigPath(getDataAgentConfigPath());
        String host = getLocalAddress().getHostAddress();

        String type = getProperty("type", "Thrift");
        int receiverPort = defaultThriftPort;
        if (type.equals("Binary")) {
            receiverPort = defaultBinaryPort;
        }
        int securePort = receiverPort + 100;

        String url = getProperty("url", "tcp://" + host + ":" + receiverPort);
        String authURL = getProperty("authURL", "ssl://" + host + ":" + securePort);
        String username = getProperty("username", "admin");
        String password = getProperty("password", "<PASSWORD>");

        DataPublisher dataPublisher = new DataPublisher(type, url, authURL, username, password);

        String streamId = DataBridgeCommonsUtils.generateStreamId(HTTPD_LOG_STREAM, VERSION);
        publishLogEvents(dataPublisher, streamId);
        dataPublisher.shutdown();
    }

    public static String getDataAgentConfigPath() {
        File filePath = new File("jvm-monitor-agent" + File.separator + "src" + File.separator + "main" + File.separator + "resources");
        if (!filePath.exists()) {
            filePath = new File("test" + File.separator + "resources");
        }
        if (!filePath.exists()) {
            filePath = new File("resources");
        }
        if (!filePath.exists()) {
            filePath = new File("test" + File.separator + "resources");
        }
        return filePath.getAbsolutePath() + File.separator + "data-agent-conf.xml";
    }

    private static void publishLogEvents(DataPublisher dataPublisher, String streamId) {

        LogManager logObj = null;

        try {
            logObj = new LogManager(pid);
            logObj.stratMonitoring();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (AttachNotSupportedException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        } catch (MalformedObjectNameException e) {
            e.printStackTrace();
        }

        int i = 1;
        while (true) {
            System.out.println("Publish Memory data : " + i++);

            try {

                Map<String, Object> usagesData = logObj.getUsageLog();
                ArrayList<Map<String, String>> gcLog = (ArrayList<Map<String, String>>) usagesData.get(LogManager.GARBAGE_COLLECTION_LOG);
                Map<String, Long> memoryUL = (Map<String, Long>) usagesData.get(LogManager.MEMORY_USAGE_LOG);

                if (gcLog.isEmpty()) {

                    Event event = new Event(streamId, System.currentTimeMillis(), null, null,
                            new Object[]{memoryUL.get(MemoryUsageLog.MAX_HEAP_MEMORY)
                                    , memoryUL.get(MemoryUsageLog.ALLOCATED_HEAP_MEMORY)
                                    , memoryUL.get(MemoryUsageLog.USED_HEAP_MEMORY)
                                    , memoryUL.get(MemoryUsageLog.MAX_NON_HEAP_MEMORY)
                                    , memoryUL.get(MemoryUsageLog.ALLOCATED_NON_HEAP_MEMORY)
                                    , memoryUL.get(MemoryUsageLog.USED_NON_HEAP_MEMORY)
                                    , memoryUL.get(MemoryUsageLog.PENDING_FINALIZATIONS)
                            });
                    dataPublisher.publish(event);

                } else {
                    for (Map<String, String> gcmap : gcLog) {

                        Event event = new Event(streamId, System.currentTimeMillis(), null, null,
                                new Object[]{gcmap.get(GarbageCollectionLog.GC_TYPE)
                                        , gcmap.get(GarbageCollectionLog.GC_DURATION)
                                        , gcmap.get(GarbageCollectionLog.GC_START_TIME)
                                        , gcmap.get(GarbageCollectionLog.GC_CAUSE)
                                        , gcmap.get(GarbageCollectionLog.EDEN_SPACE_USED_MEMORY_AFTER_GC)
                                        , gcmap.get(GarbageCollectionLog.EDEN_SPACE_USED_MEMORY_BEFORE_GC)
                                        , gcmap.get(GarbageCollectionLog.SURVIVOR_SPACE_USED_MEMORY_AFTER_GC)
                                        , gcmap.get(GarbageCollectionLog.SURVIVOR_SPACE_USED_MEMORY_BEFORE_GC)
                                        , gcmap.get(GarbageCollectionLog.OLD_GEN_USED_MEMORY_AFTER_GC)
                                        , gcmap.get(GarbageCollectionLog.OLD_GEN_USED_MEMORY_BEFORE_GC)
                                        , gcmap.get(GarbageCollectionLog.EDEN_SPACE_COMMITTED_MEMORY_AFTER_GC)
                                        , gcmap.get(GarbageCollectionLog.EDEN_SPACE_COMMITTED_MEMORY_BEFORE_GC)
                                        , gcmap.get(GarbageCollectionLog.SURVIVOR_SPACE_COMMITTED_MEMORY_AFTER_GC)
                                        , gcmap.get(GarbageCollectionLog.SURVIVOR_SPACE_COMMITTED_MEMORY_BEFORE_GC)
                                        , gcmap.get(GarbageCollectionLog.OLD_GEN_COMMITTED_MEMORY_AFTER_GC)
                                        , gcmap.get(GarbageCollectionLog.OLD_GEN_COMMITTED_MEMORY_BEFORE_GC)
                                        , gcmap.get(GarbageCollectionLog.EDEN_SPACE_MAX_MEMORY_AFTER_GC)
                                        , gcmap.get(GarbageCollectionLog.EDEN_SPACE_MAX_MEMORY_BEFORE_GC)
                                        , gcmap.get(GarbageCollectionLog.SURVIVOR_SPACE_MAX_MEMORY_AFTER_GC)
                                        , gcmap.get(GarbageCollectionLog.SURVIVOR_SPACE_MAX_MEMORY_BEFORE_GC)
                                        , gcmap.get(GarbageCollectionLog.OLD_GEN_MAX_MEMORY_AFTER_GC)
                                        , gcmap.get(GarbageCollectionLog.OLD_GEN_MAX_MEMORY_BEFORE_GC)
                                        , memoryUL.get(MemoryUsageLog.MAX_HEAP_MEMORY)
                                        , memoryUL.get(MemoryUsageLog.ALLOCATED_HEAP_MEMORY)
                                        , memoryUL.get(MemoryUsageLog.USED_HEAP_MEMORY)
                                        , memoryUL.get(MemoryUsageLog.MAX_NON_HEAP_MEMORY)
                                        , memoryUL.get(MemoryUsageLog.ALLOCATED_NON_HEAP_MEMORY)
                                        , memoryUL.get(MemoryUsageLog.USED_NON_HEAP_MEMORY)
                                        , memoryUL.get(MemoryUsageLog.PENDING_FINALIZATIONS)
                                });
                        dataPublisher.publish(event);

                    }
                }


            } catch (MonitoringNotStartedException e) {
                e.printStackTrace();
            }

            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }


    }

    public static InetAddress getLocalAddress() throws SocketException, UnknownHostException {
        Enumeration<NetworkInterface> ifaces = NetworkInterface.getNetworkInterfaces();
        while (ifaces.hasMoreElements()) {
            NetworkInterface iface = ifaces.nextElement();
            Enumeration<InetAddress> addresses = iface.getInetAddresses();

            while (addresses.hasMoreElements()) {
                InetAddress addr = addresses.nextElement();
                if (addr instanceof Inet4Address && !addr.isLoopbackAddress()) {
{'fraction_non_alphanumeric': 0.09319771137952956, 'fraction_numerical': 0.006293706293706294, 'mean_word_length': 2.34062433637715, 'pattern_counts': {'":': 4, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 63, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4669172', 'n_tokens_mistral': 4298, 'n_tokens_neox': 4079, 'n_words': 807}