<commit_before>/*
 * Copyright (c) 2017, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
 *
 * WSO2 Inc. licenses this file to you under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.wso2.siddhi.core.query.selector.attribute.aggregator.incremental;

import org.wso2.siddhi.core.event.ComplexEventChunk;
import org.wso2.siddhi.core.event.stream.MetaStreamEvent;
import org.wso2.siddhi.core.event.stream.StreamEvent;
import org.wso2.siddhi.core.event.stream.StreamEventPool;
import org.wso2.siddhi.core.executor.ExpressionExecutor;
import org.wso2.siddhi.core.query.selector.GroupByKeyGenerator;
import org.wso2.siddhi.core.query.selector.attribute.processor.executor.GroupByAggregationAttributeExecutor;
import org.wso2.siddhi.core.table.Table;
import org.wso2.siddhi.core.util.IncrementalTimeConverterUtil;
import org.wso2.siddhi.query.api.aggregation.TimePeriod;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * This class implements the logic to aggregate data that is in-memory or in tables, in incremental data processing.
 * 
 * In-memory data is required to be aggregated when retrieving values from an aggregate.
 * Table data (tables used to persist incremental aggregates) needs to be aggregated when querying aggregate data from
 * a different server (apart from the server, which was used to define the aggregation).
 */
public class IncrementalDataAggregator {
    private final List<TimePeriod.Duration> incrementalDurations;
    private final TimePeriod.Duration aggregateForDuration;
    private final ExpressionExecutor timestampExecutor;
    private final ExpressionExecutor timeZoneExecutor;
    private final BaseIncrementalValueStore baseIncrementalValueStore;

    private final Map<Long, BaseIncrementalValueStore> baseIncrementalValueStoreMap;
    private final Map<Long, Map<String, BaseIncrementalValueStore>> baseIncrementalValueGroupByStoreMap;

    public IncrementalDataAggregator(List<TimePeriod.Duration> incrementalDurations,
            TimePeriod.Duration aggregateForDuration, List<ExpressionExecutor> baseExecutors,
            ExpressionExecutor timestampExecutor, MetaStreamEvent metaStreamEvent) {
        this.incrementalDurations = incrementalDurations;
        this.aggregateForDuration = aggregateForDuration;
        this.timestampExecutor = timestampExecutor;
        this.timeZoneExecutor = baseExecutors.get(0);
        StreamEventPool streamEventPool = new StreamEventPool(metaStreamEvent, 10);
        this.baseIncrementalValueStore = new BaseIncrementalValueStore(-1, baseExecutors, streamEventPool);

        this.baseIncrementalValueStoreMap = new HashMap<>();
        this.baseIncrementalValueGroupByStoreMap = new HashMap<>();
    }

    public ComplexEventChunk<StreamEvent> aggregateInMemoryData(
            Map<TimePeriod.Duration, IncrementalExecutor> incrementalExecutorMap) {
        for (TimePeriod.Duration duration : incrementalDurations) {
            IncrementalExecutor incrementalExecutor = incrementalExecutorMap.get(duration);

            ArrayList<HashMap<String, BaseIncrementalValueStore>> baseIncrementalValueGroupByStoreList =
                    incrementalExecutor.getBaseIncrementalValueGroupByStoreList();
            Map<String, BaseIncrementalValueStore> baseIncrementalValueStoreMap = incrementalExecutor
                    .getBaseIncrementalValueStoreMap();
            ArrayList<BaseIncrementalValueStore> baseIncrementalValueStoreList = incrementalExecutor
                    .getBaseIncrementalValueStoreList();
            BaseIncrementalValueStore baseIncrementalValueStore = incrementalExecutor.getBaseIncrementalValueStore();

            if (baseIncrementalValueGroupByStoreList != null) {
                for (HashMap<String, BaseIncrementalValueStore> aBaseIncrementalValueGroupByStoreList :
                        baseIncrementalValueGroupByStoreList) {
                    for (Map.Entry<String, BaseIncrementalValueStore> entry : aBaseIncrementalValueGroupByStoreList
                            .entrySet()) {
                        processInMemoryAggregates(entry.getValue().createStreamEvent(), entry.getValue().getTimestamp(),
                                entry.getKey());
                    }
                }
            } else if (baseIncrementalValueStoreMap != null) {
                for (Map.Entry<String, BaseIncrementalValueStore> entry : baseIncrementalValueStoreMap.entrySet()) {
                    processInMemoryAggregates(entry.getValue().createStreamEvent(), entry.getValue().getTimestamp(),
                            entry.getKey());
                }
            } else if (baseIncrementalValueStoreList != null) {
                for (BaseIncrementalValueStore aBaseIncrementalValueStoreList : baseIncrementalValueStoreList) {
                    processInMemoryAggregates(aBaseIncrementalValueStoreList.createStreamEvent(),
                            aBaseIncrementalValueStoreList.getTimestamp(), null);
                }
            } else {
                processInMemoryAggregates(baseIncrementalValueStore.createStreamEvent(),
                        baseIncrementalValueStore.getTimestamp(), null);
            }

            if (duration == aggregateForDuration) {
                break;
            }
        }

        return createEventChunkFromAggregatedData();
    }

    public ComplexEventChunk<StreamEvent> aggregateTableData(Map<TimePeriod.Duration, Table> aggregationTables,
            GroupByKeyGenerator groupByKeyGenerator) {<commit_msg>Correct in-memory data aggregation logic The aggregation must be done only if inmemory data exists. Hence we need to check whether the in-memory store 'baseIncrementalValueStore' is already processed. Otherwise ignore it.<commit_after>/*
 * Copyright (c) 2017, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
 *
 * WSO2 Inc. licenses this file to you under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.wso2.siddhi.core.query.selector.attribute.aggregator.incremental;

import org.wso2.siddhi.core.event.ComplexEventChunk;
import org.wso2.siddhi.core.event.stream.MetaStreamEvent;
import org.wso2.siddhi.core.event.stream.StreamEvent;
import org.wso2.siddhi.core.event.stream.StreamEventPool;
import org.wso2.siddhi.core.executor.ExpressionExecutor;
import org.wso2.siddhi.core.query.selector.GroupByKeyGenerator;
import org.wso2.siddhi.core.query.selector.attribute.processor.executor.GroupByAggregationAttributeExecutor;
import org.wso2.siddhi.core.table.Table;
import org.wso2.siddhi.core.util.IncrementalTimeConverterUtil;
import org.wso2.siddhi.query.api.aggregation.TimePeriod;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * This class implements the logic to aggregate data that is in-memory or in tables, in incremental data processing.
 * 
 * In-memory data is required to be aggregated when retrieving values from an aggregate.
 * Table data (tables used to persist incremental aggregates) needs to be aggregated when querying aggregate data from
 * a different server (apart from the server, which was used to define the aggregation).
 */
public class IncrementalDataAggregator {
    private final List<TimePeriod.Duration> incrementalDurations;
    private final TimePeriod.Duration aggregateForDuration;
    private final ExpressionExecutor timestampExecutor;
    private final ExpressionExecutor timeZoneExecutor;
    private final BaseIncrementalValueStore baseIncrementalValueStore;

    private final Map<Long, BaseIncrementalValueStore> baseIncrementalValueStoreMap;
    private final Map<Long, Map<String, BaseIncrementalValueStore>> baseIncrementalValueGroupByStoreMap;

    public IncrementalDataAggregator(List<TimePeriod.Duration> incrementalDurations,
            TimePeriod.Duration aggregateForDuration, List<ExpressionExecutor> baseExecutors,
            ExpressionExecutor timestampExecutor, MetaStreamEvent metaStreamEvent) {
        this.incrementalDurations = incrementalDurations;
        this.aggregateForDuration = aggregateForDuration;
        this.timestampExecutor = timestampExecutor;
        this.timeZoneExecutor = baseExecutors.get(0);
        StreamEventPool streamEventPool = new StreamEventPool(metaStreamEvent, 10);
        this.baseIncrementalValueStore = new BaseIncrementalValueStore(-1, baseExecutors, streamEventPool);

        this.baseIncrementalValueStoreMap = new HashMap<>();
        this.baseIncrementalValueGroupByStoreMap = new HashMap<>();
    }

    public ComplexEventChunk<StreamEvent> aggregateInMemoryData(
            Map<TimePeriod.Duration, IncrementalExecutor> incrementalExecutorMap) {
        for (TimePeriod.Duration duration : incrementalDurations) {
            IncrementalExecutor incrementalExecutor = incrementalExecutorMap.get(duration);

            ArrayList<HashMap<String, BaseIncrementalValueStore>> baseIncrementalValueGroupByStoreList =
                    incrementalExecutor.getBaseIncrementalValueGroupByStoreList();
            Map<String, BaseIncrementalValueStore> baseIncrementalValueStoreMap = incrementalExecutor
                    .getBaseIncrementalValueStoreMap();
            ArrayList<BaseIncrementalValueStore> baseIncrementalValueStoreList = incrementalExecutor
                    .getBaseIncrementalValueStoreList();
            BaseIncrementalValueStore baseIncrementalValueStore = incrementalExecutor.getBaseIncrementalValueStore();

            if (baseIncrementalValueGroupByStoreList != null) {
                for (HashMap<String, BaseIncrementalValueStore> aBaseIncrementalValueGroupByStoreList :
                        baseIncrementalValueGroupByStoreList) {
                    for (Map.Entry<String, BaseIncrementalValueStore> entry : aBaseIncrementalValueGroupByStoreList
                            .entrySet()) {
                        processInMemoryAggregates(entry.getValue().createStreamEvent(), entry.getValue().getTimestamp(),
                                entry.getKey());
                    }
                }
            } else if (baseIncrementalValueStoreMap != null) {
                for (Map.Entry<String, BaseIncrementalValueStore> entry : baseIncrementalValueStoreMap.entrySet()) {
                    processInMemoryAggregates(entry.getValue().createStreamEvent(), entry.getValue().getTimestamp(),
                            entry.getKey());
                }
            } else if (baseIncrementalValueStoreList != null) {
                for (BaseIncrementalValueStore aBaseIncrementalValueStoreList : baseIncrementalValueStoreList) {
                    processInMemoryAggregates(aBaseIncrementalValueStoreList.createStreamEvent(),
                            aBaseIncrementalValueStoreList.getTimestamp(), null);
                }
            } else if (baseIncrementalValueStore.isProcessed()) {
                processInMemoryAggregates(baseIncrementalValueStore.createStreamEvent(),
                        baseIncrementalValueStore.getTimestamp(), null);
            }

            if (duration == aggregateForDuration) {
                break;
            }
        }

        return createEventChunkFromAggregatedData();
    }

    public ComplexEventChunk<StreamEvent> aggregateTableData(Map<TimePeriod.Duration, Table> aggregationTables,
            GroupByKeyGenerator groupByKeyGenerator) {
{'fraction_non_alphanumeric': 0.07148043565954014, 'fraction_numerical': 0.0041952400161355385, 'mean_word_length': 3.438238453276047, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3409666', 'n_tokens_mistral': 3151, 'n_tokens_neox': 2963, 'n_words': 714}