<commit_before>/*
 * Copyright (c) 2017, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
 *
 * WSO2 Inc. licenses this file to you under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.wso2.carbon.siddhi.editor.core.util;

import org.apache.log4j.Logger;
import org.wso2.carbon.siddhi.editor.core.commons.metadata.AttributeMetaData;
import org.wso2.carbon.siddhi.editor.core.commons.metadata.MetaData;
import org.wso2.carbon.siddhi.editor.core.commons.metadata.ParameterMetaData;
import org.wso2.carbon.siddhi.editor.core.commons.metadata.ProcessorMetaData;
import org.wso2.siddhi.annotation.Extension;
import org.wso2.siddhi.annotation.Parameter;
import org.wso2.siddhi.annotation.ReturnAttribute;
import org.wso2.siddhi.core.ExecutionPlanRuntime;
import org.wso2.siddhi.core.SiddhiManager;
import org.wso2.siddhi.query.api.definition.AbstractDefinition;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.jar.JarEntry;
import java.util.jar.JarInputStream;

/**
 * Utility class for getting the meta data for the in built and extension processors in siddhi
 */
public class SourceEditorUtils {
    static final Logger LOGGER = Logger.getLogger(SourceEditorUtils.class);

    private SourceEditorUtils() {

    }

    /**
     * Validate the execution plan string using the Siddhi Manager
     * Will return a valid executionPlanRuntime
     *
     * @param executionPlan Execution plan string
     * @return Valid execution plan runtime
     */
    public static ExecutionPlanRuntime validateExecutionPlan(String executionPlan) {
        SiddhiManager siddhiManager = new SiddhiManager();
        ExecutionPlanRuntime executionPlanRuntime = null;
        try {
            executionPlanRuntime = siddhiManager.createExecutionPlanRuntime(executionPlan);
            executionPlanRuntime.start();
        } finally {
            if (executionPlanRuntime != null) {
                executionPlanRuntime.shutdown();
            }
        }
        return executionPlanRuntime;
    }

    /**
     * Get the definition of the inner streams in the partitions
     * Inner streams will be separated based on the partition
     *
     * @param executionPlanRuntime              Execution plan runtime created after validating
     * @param partitionsWithMissingInnerStreams Required inner stream names separated based on partition it belongs to
     * @return The inner stream definitions separated base on the partition it belongs to
     */
    public static List<List<AbstractDefinition>> getInnerStreamDefinitions(ExecutionPlanRuntime executionPlanRuntime,
                                                                           List<List<String>>
                                                                                   partitionsWithMissingInnerStreams) {
        List<List<AbstractDefinition>> innerStreamDefinitions = new ArrayList<>();

        // Transforming the element ID to partition inner streams map to element ID no to partition inner streams map
        Map<Integer, Map<String, AbstractDefinition>> innerStreamsMap = new ConcurrentHashMap<>();
        executionPlanRuntime.getPartitionedInnerStreamDefinitionMap().entrySet().parallelStream().forEach(
                entry -> innerStreamsMap.put(
                        Integer.valueOf(entry.getKey().split("-")[1]),
                        entry.getValue()
                )
        );

        // Creating an ordered list of partition inner streams based on partition element ID
        // This is important since the client sends the missing inner streams 2D list
        // with partitions in the order they are in the execution plan
        List<Map<String, AbstractDefinition>> rankedPartitionsWithInnerStreams = new ArrayList<>();
        List<Integer> rankedPartitionElementIds = new ArrayList<>();
        for (Map.Entry<Integer, Map<String, AbstractDefinition>> entry :
                innerStreamsMap.entrySet()) {
            int i = 0;
            for (; i < rankedPartitionsWithInnerStreams.size(); i++) {
                if (entry.getKey() < rankedPartitionElementIds.get(i)) {
                    break;
                }
            }
            rankedPartitionsWithInnerStreams.add(i, entry.getValue());
            rankedPartitionElementIds.add(i, entry.getKey());
        }

        // Extracting the requested stream definitions from based on the order
        // in rankedPartitionsWithInnerStreams and partitionsWithMissingInnerStreams
        // The inner stream definitions 2D list fetched from the Siddhi Manager
        // and the missing inner streams 2D list are now in the same order
        // Therefore the outer loops in both lists can be looped together
        for (int i = 0; i < partitionsWithMissingInnerStreams.size(); i++) {
            List<String> partitionWithMissingInnerStreams = partitionsWithMissingInnerStreams.get(i);
            Map<String, AbstractDefinition> partitionWithInnerStreams = rankedPartitionsWithInnerStreams.get(i);
            List<AbstractDefinition> innerStreamDefinition = new ArrayList<>();

            for (String missingInnerStream : partitionWithMissingInnerStreams) {
                AbstractDefinition streamDefinition = partitionWithInnerStreams.get(missingInnerStream);
                if (streamDefinition != null) {
                    innerStreamDefinition.add(streamDefinition);
                }
            }
            innerStreamDefinitions.add(innerStreamDefinition);
        }

        return innerStreamDefinitions;
    }

    /**
     * Get the definitions of the streams that are requested
     * used for fetching the definitions of streams that queries output into without defining them first
     *
     * @param executionPlanRuntime Execution plan runtime created after validating
     * @param missingStreams       Required stream names
     * @return The stream definitions
     */
    public static List<AbstractDefinition> getStreamDefinitions(ExecutionPlanRuntime executionPlanRuntime,
                                                                List<String> missingStreams) {
        List<AbstractDefinition> streamDefinitions = new ArrayList<>();
        Map<String, AbstractDefinition> streamDefinitionMap = executionPlanRuntime.getStreamDefinitionMap();
        for (String stream : missingStreams) {
            AbstractDefinition streamDefinition = streamDefinitionMap.get(stream);
            if (streamDefinition != null) {
                streamDefinitions.add(streamDefinition);
            }
        }
        return streamDefinitions;
    }

    /**
     * Returns the in built processor meta data
     * Scans for all classes in all jars in the classpath
     *
     * @return In-built processor meta data
     */
    public static MetaData getInBuiltProcessorMetaData() {
        Map<String, Set<Class<?>>> processorClassMap = getClassesInClassPathFromPackages();
        return generateInBuiltMetaData(processorClassMap);
    }

    /**
     * Returns the extension processor meta data
     * Gets the meta data from the siddhi manager
     *
     * @return Extension processor meta data
     */
    public static Map<String, MetaData> getExtensionProcessorMetaData() {
        SiddhiManager siddhiManager = new SiddhiManager();
        Map<String, Class> extensionsMap = siddhiManager.getExtensions();<commit_msg>Committing possible fixes to avoid failures when releasing repo<commit_after>/*
 * Copyright (c) 2017, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
 *
 * WSO2 Inc. licenses this file to you under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.wso2.carbon.siddhi.editor.core.util;

import org.apache.log4j.Logger;
import org.wso2.carbon.siddhi.editor.core.commons.metadata.AttributeMetaData;
import org.wso2.carbon.siddhi.editor.core.commons.metadata.MetaData;
import org.wso2.carbon.siddhi.editor.core.commons.metadata.ParameterMetaData;
import org.wso2.carbon.siddhi.editor.core.commons.metadata.ProcessorMetaData;
import org.wso2.siddhi.annotation.Extension;
import org.wso2.siddhi.annotation.Parameter;
import org.wso2.siddhi.annotation.ReturnAttribute;
import org.wso2.siddhi.core.ExecutionPlanRuntime;
import org.wso2.siddhi.core.SiddhiManager;
import org.wso2.siddhi.query.api.definition.AbstractDefinition;
import org.wso2.siddhi.query.api.definition.StreamDefinition;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.jar.JarEntry;
import java.util.jar.JarInputStream;

/**
 * Utility class for getting the meta data for the in built and extension processors in siddhi
 */
public class SourceEditorUtils {
    static final Logger LOGGER = Logger.getLogger(SourceEditorUtils.class);

    private SourceEditorUtils() {

    }

    /**
     * Validate the execution plan string using the Siddhi Manager
     * Will return a valid executionPlanRuntime
     *
     * @param executionPlan Execution plan string
     * @return Valid execution plan runtime
     */
    public static ExecutionPlanRuntime validateExecutionPlan(String executionPlan) {
        SiddhiManager siddhiManager = new SiddhiManager();
        ExecutionPlanRuntime executionPlanRuntime = null;
        try {
            executionPlanRuntime = siddhiManager.createExecutionPlanRuntime(executionPlan);
            executionPlanRuntime.start();
        } finally {
            if (executionPlanRuntime != null) {
                executionPlanRuntime.shutdown();
            }
        }
        return executionPlanRuntime;
    }

    /**
     * Get the definition of the inner streams in the partitions
     * Inner streams will be separated based on the partition
     *
     * @param executionPlanRuntime              Execution plan runtime created after validating
     * @param partitionsWithMissingInnerStreams Required inner stream names separated based on partition it belongs to
     * @return The inner stream definitions separated base on the partition it belongs to
     */
    public static List<List<AbstractDefinition>> getInnerStreamDefinitions(ExecutionPlanRuntime executionPlanRuntime,
                                                                           List<List<String>>
                                                                                   partitionsWithMissingInnerStreams) {
        List<List<AbstractDefinition>> innerStreamDefinitions = new ArrayList<>();

        // Transforming the element ID to partition inner streams map to element ID no to partition inner streams map
        Map<Integer, Map<String, AbstractDefinition>> innerStreamsMap = new ConcurrentHashMap<>();
        executionPlanRuntime.getPartitionedInnerStreamDefinitionMap().entrySet().parallelStream().forEach(
                entry -> innerStreamsMap.put(
                        Integer.valueOf(entry.getKey().split("-")[1]),
                        entry.getValue()
                )
        );

        // Creating an ordered list of partition inner streams based on partition element ID
        // This is important since the client sends the missing inner streams 2D list
        // with partitions in the order they are in the execution plan
        List<Map<String, AbstractDefinition>> rankedPartitionsWithInnerStreams = new ArrayList<>();
        List<Integer> rankedPartitionElementIds = new ArrayList<>();
        for (Map.Entry<Integer, Map<String, AbstractDefinition>> entry :
                innerStreamsMap.entrySet()) {
            int i = 0;
            for (; i < rankedPartitionsWithInnerStreams.size(); i++) {
                if (entry.getKey() < rankedPartitionElementIds.get(i)) {
                    break;
                }
            }
            rankedPartitionsWithInnerStreams.add(i, entry.getValue());
            rankedPartitionElementIds.add(i, entry.getKey());
        }

        // Extracting the requested stream definitions from based on the order
        // in rankedPartitionsWithInnerStreams and partitionsWithMissingInnerStreams
        // The inner stream definitions 2D list fetched from the Siddhi Manager
        // and the missing inner streams 2D list are now in the same order
        // Therefore the outer loops in both lists can be looped together
        for (int i = 0; i < partitionsWithMissingInnerStreams.size(); i++) {
            List<String> partitionWithMissingInnerStreams = partitionsWithMissingInnerStreams.get(i);
            Map<String, AbstractDefinition> partitionWithInnerStreams = rankedPartitionsWithInnerStreams.get(i);
            List<AbstractDefinition> innerStreamDefinition = new ArrayList<>();

            for (String missingInnerStream : partitionWithMissingInnerStreams) {
                AbstractDefinition streamDefinition = partitionWithInnerStreams.get(missingInnerStream);
                if (streamDefinition != null) {
                    innerStreamDefinition.add(streamDefinition);
                }
            }
            innerStreamDefinitions.add(innerStreamDefinition);
        }

        return innerStreamDefinitions;
    }

    /**
     * Get the definitions of the streams that are requested
     * used for fetching the definitions of streams that queries output into without defining them first
     *
     * @param executionPlanRuntime Execution plan runtime created after validating
     * @param missingStreams       Required stream names
     * @return The stream definitions
     */
    public static List<AbstractDefinition> getStreamDefinitions(ExecutionPlanRuntime executionPlanRuntime,
                                                                List<String> missingStreams) {
        List<AbstractDefinition> streamDefinitions = new ArrayList<>();
        Map<String, StreamDefinition> streamDefinitionMap = executionPlanRuntime.getStreamDefinitionMap();
        for (String stream : missingStreams) {
            AbstractDefinition streamDefinition = streamDefinitionMap.get(stream);
            if (streamDefinition != null) {
                streamDefinitions.add(streamDefinition);
            }
        }
        return streamDefinitions;
    }

    /**
     * Returns the in built processor meta data
     * Scans for all classes in all jars in the classpath
     *
     * @return In-built processor meta data
     */
    public static MetaData getInBuiltProcessorMetaData() {
        Map<String, Set<Class<?>>> processorClassMap = getClassesInClassPathFromPackages();
        return generateInBuiltMetaData(processorClassMap);
    }

    /**
     * Returns the extension processor meta data
     * Gets the meta data from the siddhi manager
     *
     * @return Extension processor meta data
     */
    public static Map<String, MetaData> getExtensionProcessorMetaData() {
        SiddhiManager siddhiManager = new SiddhiManager();
        Map<String, Class> extensionsMap = siddhiManager.getExtensions();
{'fraction_non_alphanumeric': 0.07433011134222439, 'fraction_numerical': 0.003609445735959868, 'mean_word_length': 2.791927627000696, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7327694', 'n_tokens_mistral': 3938, 'n_tokens_neox': 3840, 'n_words': 1195}