<commit_before>package org.apache.kylin.storage.gridtable;

import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.BitSet;
import java.util.Comparator;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.SortedMap;

public class GTAggregateScanner implements IGTScanner {

    private static final Logger logger = LoggerFactory.getLogger(GTAggregateScanner.class);
    final GTInfo info;
    final BitSet dimensions; // dimensions to return, can be more than group by
    final BitSet groupBy;
    final BitSet metrics;
    final String[] metricsAggrFuncs;
    final IGTScanner inputScanner;


    public GTAggregateScanner(IGTScanner inputScanner, GTScanRequest req) {
        if (req.hasAggregation() == false)
            throw new IllegalStateException();
        
        this.info = inputScanner.getInfo();
        this.dimensions = (BitSet) req.getColumns().clone();
        this.dimensions.andNot(req.getAggrMetrics());
        this.groupBy = req.getAggrGroupBy();
        this.metrics = req.getAggrMetrics();
        this.metricsAggrFuncs = req.getAggrMetricsFuncs();
        this.inputScanner = inputScanner;
    }

    @Override
    public GTInfo getInfo() {
        return info;
    }
    
    @Override
    public int getScannedRowCount() {
        return inputScanner.getScannedRowCount();
    }

    @Override
    public int getScannedRowBlockCount() {
        return inputScanner.getScannedRowBlockCount();
    }

    @Override
    public void close() throws IOException {
        inputScanner.close();
    }

    @Override
    public Iterator<GTRecord> iterator() {
        AggregationCacheWithBytesKey aggregationCacheWithBytesKey = new AggregationCacheWithBytesKey();
        for (GTRecord r : inputScanner) {
            aggregationCacheWithBytesKey.aggregate(r);
            MemoryChecker.checkMemory();
        }
        return aggregationCacheWithBytesKey.iterator();
    }

    class AggregationCacheWithBytesKey {
        final SortedMap<byte[], MeasureAggregator[]> aggBufMap;

        public AggregationCacheWithBytesKey() {
            aggBufMap = Maps.newTreeMap(new Comparator<byte[]>() {
                @Override
                public int compare(byte[] o1, byte[] o2) {
                    int result = 0;
                    Preconditions.checkArgument(o1.length == o2.length);
                    final int length = o1.length;
                    for (int i = 0; i < length; ++i) {
                        result = o1[i] - o2[i];
                        if (result == 0) {
                            continue;
                        } else {
                            return result;
                        }
                    }
                    return result;
                }
            });
        }

        private byte[] createKey(GTRecord record) {
            byte[] result = new byte[info.getMaxColumnLength(groupBy)];
            int offset = 0;
            for (int i = groupBy.nextSetBit(0); i >= 0; i = groupBy.nextSetBit(i + 1)) {
                final ByteArray byteArray = record.cols[i];
                final int columnLength = info.codeSystem.maxCodeLength(i);
                System.arraycopy(byteArray.array(), byteArray.offset(), result, offset, columnLength);
                offset += columnLength;
            }
            assert offset == result.length;
            return result;
        }

        void aggregate(GTRecord r) {
            final byte[] key = createKey(r);
            MeasureAggregator[] aggrs = aggBufMap.get(key);
            if (aggrs == null) {<commit_msg>fix UT<commit_after>package org.apache.kylin.storage.gridtable;

import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.BitSet;
import java.util.Comparator;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.SortedMap;

public class GTAggregateScanner implements IGTScanner {

    private static final Logger logger = LoggerFactory.getLogger(GTAggregateScanner.class);
    final GTInfo info;
    final BitSet dimensions; // dimensions to return, can be more than group by
    final BitSet groupBy;
    final BitSet metrics;
    final String[] metricsAggrFuncs;
    final IGTScanner inputScanner;


    public GTAggregateScanner(IGTScanner inputScanner, GTScanRequest req) {
        if (req.hasAggregation() == false)
            throw new IllegalStateException();
        
        this.info = inputScanner.getInfo();
        this.dimensions = (BitSet) req.getColumns().clone();
        this.dimensions.andNot(req.getAggrMetrics());
        this.groupBy = req.getAggrGroupBy();
        this.metrics = req.getAggrMetrics();
        this.metricsAggrFuncs = req.getAggrMetricsFuncs();
        this.inputScanner = inputScanner;
    }

    @Override
    public GTInfo getInfo() {
        return info;
    }
    
    @Override
    public int getScannedRowCount() {
        return inputScanner.getScannedRowCount();
    }

    @Override
    public int getScannedRowBlockCount() {
        return inputScanner.getScannedRowBlockCount();
    }

    @Override
    public void close() throws IOException {
        inputScanner.close();
    }

    @Override
    public Iterator<GTRecord> iterator() {
        AggregationCacheWithBytesKey aggregationCacheWithBytesKey = new AggregationCacheWithBytesKey();
        for (GTRecord r : inputScanner) {
            aggregationCacheWithBytesKey.aggregate(r);
            MemoryChecker.checkMemory();
        }
        return aggregationCacheWithBytesKey.iterator();
    }

    class AggregationCacheWithBytesKey {
        final SortedMap<byte[], MeasureAggregator[]> aggBufMap;

        public AggregationCacheWithBytesKey() {
            aggBufMap = Maps.newTreeMap(new Comparator<byte[]>() {
                @Override
                public int compare(byte[] o1, byte[] o2) {
                    int result = 0;
                    Preconditions.checkArgument(o1.length == o2.length);
                    final int length = o1.length;
                    for (int i = 0; i < length; ++i) {
                        result = o1[i] - o2[i];
                        if (result == 0) {
                            continue;
                        } else {
                            return result;
                        }
                    }
                    return result;
                }
            });
        }

        private byte[] createKey(GTRecord record) {
            byte[] result = new byte[info.getMaxColumnLength(groupBy)];
            int offset = 0;
            for (int i = groupBy.nextSetBit(0); i >= 0; i = groupBy.nextSetBit(i + 1)) {
                final ByteArray byteArray = record.cols[i];
                final int columnLength = info.codeSystem.maxCodeLength(i);
                System.arraycopy(byteArray.array(), byteArray.offset(), result, offset, byteArray.length());
                offset += columnLength;
            }
            assert offset == result.length;
            return result;
        }

        void aggregate(GTRecord r) {
            final byte[] key = createKey(r);
            MeasureAggregator[] aggrs = aggBufMap.get(key);
            if (aggrs == null) {
{'fraction_non_alphanumeric': 0.09495896834701055, 'fraction_numerical': 0.004168294906864661, 'mean_word_length': 2.099717400080743, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6206018', 'n_tokens_mistral': 2169, 'n_tokens_neox': 2095, 'n_words': 463}