<commit_before>import {
    _,
    Bean,
    IRowNodeStage,
    Autowired,
    GridOptionsWrapper,
    ColumnController,
    ValueService,
    RowNode,
    Column,
    Utils,
    StageExecuteParams,
    IAggFunc,
    GroupValueService
} from "ag-grid/main";
import {PivotStage} from "./pivotStage";
import {AggFuncService} from "../aggregation/aggFuncService";

@Bean('aggregationStage')
export class AggregationStage implements IRowNodeStage {

    @Autowired('gridOptionsWrapper') private gridOptionsWrapper: GridOptionsWrapper;
    @Autowired('columnController') private columnController: ColumnController;
    @Autowired('valueService') private valueService: ValueService;
    @Autowired('groupValueService') private groupValueService: GroupValueService;
    @Autowired('pivotStage') private pivotStage: PivotStage;
    @Autowired('aggFuncService') private aggFuncService: AggFuncService;

    // it's possible to recompute the aggregate without doing the other parts
    // + gridApi.recomputeAggregates()
    public execute(params: StageExecuteParams): any {
        let rootNode = params.rowNode;

        // we don't do aggregation if user provided the groups
        let rowsAlreadyGrouped = _.exists(this.gridOptionsWrapper.getNodeChildDetailsFunc());
        if (rowsAlreadyGrouped) {
            return;
        }

        let pivotActive = this.columnController.isPivotActive();

        let measureColumns = this.columnController.getValueColumns();
        let pivotColumns = pivotActive ? this.columnController.getPivotColumns() : [];

        this.recursivelyCreateAggData(rootNode, measureColumns, pivotColumns);
    }

    private recursivelyCreateAggData(rowNode: RowNode, measureColumns: Column[], pivotColumns: Column[]) {

        // aggregate all children first, as we use the result in this nodes calculations
        rowNode.childrenAfterFilter.forEach( child => {
            if (child.group) {
                this.recursivelyCreateAggData(child, measureColumns, pivotColumns);
            }
        });

        //Optionally prevent the aggregation at the root Node
        //https://ag-grid.atlassian.net/browse/AG-388
        let notPivoting = !this.columnController.isPivotMode();
        let suppressAggAtRootLevel = this.gridOptionsWrapper.isSuppressAggAtRootLevel();
        let isRootNode = rowNode.level === -1;
        if (isRootNode && suppressAggAtRootLevel && notPivoting) return;
        this.aggregateRowNode(rowNode, measureColumns, pivotColumns);
    }

    private aggregateRowNode(rowNode: RowNode, measureColumns: Column[], pivotColumns: Column[]): void {

        let measureColumnsMissing = measureColumns.length === 0;
        let pivotColumnsMissing = pivotColumns.length === 0;
        let userProvidedGroupRowAggNodes = this.gridOptionsWrapper.getGroupRowAggNodesFunc();

        let aggResult: any;
        if (rowNode.group && userProvidedGroupRowAggNodes) {
            aggResult = userProvidedGroupRowAggNodes(rowNode.childrenAfterFilter);
        } else if (measureColumnsMissing) {
            aggResult = null;
        } else if (rowNode.group && pivotColumnsMissing) {
            aggResult = this.aggregateRowNodeUsingValuesOnly(rowNode, measureColumns);
        } else {
            aggResult = this.aggregateRowNodeUsingValuesAndPivot(rowNode);
        }

        rowNode.setAggData(aggResult);

        // if we are grouping, then it's possible there is a sibling footer
        // to the group, so update the data here also if there is one
        if (rowNode.sibling) {
            rowNode.sibling.setAggData(aggResult);
        }
    }

    private aggregateRowNodeUsingValuesAndPivot(rowNode: RowNode): any {
        let result: any = {};
        let pivotColumnDefs = this.pivotStage.getPivotColumnDefs();

        // Step 1: process value columns
        pivotColumnDefs
            .filter(v => !Utils.exists(v.pivotTotalColumnIds)) // only process pivot value columns
            .forEach(valueColDef => {
                let keys: string[] = valueColDef.pivotKeys;
                let values: any[];
                let valueColumn: Column = valueColDef.pivotValueColumn;

                if (rowNode.leafGroup) {
                    // lowest level group, get the values from the mapped set
                    values = this.getValuesFromMappedSet(rowNode.childrenMapped, keys, valueColumn);
                } else {
                    // value columns and pivot columns, non-leaf group
                    values = this.getValuesPivotNonLeaf(rowNode, valueColDef.colId);
                }

                result[valueColDef.colId] = this.aggregateValues(values, valueColumn.getAggFunc());
            });

        // Step 2: process total columns
        pivotColumnDefs
            .filter(v => Utils.exists(v.pivotTotalColumnIds)) // only process pivot total columns
            .forEach(totalColDef => {
                let aggResults: any[] = [];

                //retrieve results for colIds associated with this pivot total column
                totalColDef.pivotTotalColumnIds.forEach((colId: string) => {
                    aggResults.push(result[colId]);
                });

                result[totalColDef.colId] = this.aggregateValues(aggResults, totalColDef.aggFunc);
            });

        return result;
    }

    private aggregateRowNodeUsingValuesOnly(rowNode: RowNode, valueColumns: Column[]): any {
        let result: any = {};

        let values2d = this.getValuesNormal(rowNode, valueColumns);

        valueColumns.forEach( (valueColumn: Column, index: number) => {
            result[valueColumn.getId()] = this.aggregateValues(values2d[index], valueColumn.getAggFunc());
        });

        return result;
    }

    private getValuesPivotNonLeaf(rowNode: RowNode, colId: string): any[] {
        let values: any[] = [];
        rowNode.childrenAfterFilter.forEach( rowNode => {
            let value = rowNode.aggData[colId];
            values.push(value);
        });
        return values;
    }

    private getValuesFromMappedSet(mappedSet: any, keys: string[], valueColumn: Column): any[] {
        let mapPointer = mappedSet;
        keys.forEach( key => mapPointer = mapPointer ? mapPointer[key] : null );

        if (!mapPointer) {
            return [];
        }

        let values: any = [];
        mapPointer.forEach( (rowNode: RowNode) => {
            let value = this.valueService.getValue(valueColumn, rowNode);
            values.push(value);
        });
<commit_msg>AG-560 iteration of change detection and refresh<commit_after>import {
    _,
    Bean,
    IRowNodeStage,
    Autowired,
    GridOptionsWrapper,
    ColumnController,
    ValueService,
    RowNode,
    Column,
    Utils,
    StageExecuteParams,
    IAggFunc,
    GroupValueService,
    ChangedPath
} from "ag-grid/main";
import {PivotStage} from "./pivotStage";
import {AggFuncService} from "../aggregation/aggFuncService";

@Bean('aggregationStage')
export class AggregationStage implements IRowNodeStage {

    @Autowired('gridOptionsWrapper') private gridOptionsWrapper: GridOptionsWrapper;
    @Autowired('columnController') private columnController: ColumnController;
    @Autowired('valueService') private valueService: ValueService;
    @Autowired('groupValueService') private groupValueService: GroupValueService;
    @Autowired('pivotStage') private pivotStage: PivotStage;
    @Autowired('aggFuncService') private aggFuncService: AggFuncService;

    // it's possible to recompute the aggregate without doing the other parts
    // + gridApi.recomputeAggregates()
    public execute(params: StageExecuteParams): any {
        let {rowNode, changedPath} = params;

        // we don't do aggregation if user provided the groups
        let rowsAlreadyGrouped = _.exists(this.gridOptionsWrapper.getNodeChildDetailsFunc());
        if (rowsAlreadyGrouped) {
            return;
        }

        let pivotActive = this.columnController.isPivotActive();

        let measureColumns = this.columnController.getValueColumns();
        let pivotColumns = pivotActive ? this.columnController.getPivotColumns() : [];

        this.recursivelyCreateAggData(rowNode, changedPath, measureColumns, pivotColumns);
    }

    private recursivelyCreateAggData(rowNode: RowNode, changedPath: ChangedPath, measureColumns: Column[], pivotColumns: Column[]) {

        // aggregate all children first, as we use the result in this nodes calculations
        rowNode.childrenAfterFilter.forEach( child => {
            if (child.group) {
                this.recursivelyCreateAggData(child, changedPath, measureColumns, pivotColumns);
            }
        });

        //Optionally prevent the aggregation at the root Node
        //https://ag-grid.atlassian.net/browse/AG-388
        let isRootNode = rowNode.level === -1;
        if (isRootNode) {
            let notPivoting = !this.columnController.isPivotMode();
            let suppressAggAtRootLevel = this.gridOptionsWrapper.isSuppressAggAtRootLevel();
            if (suppressAggAtRootLevel && notPivoting) { return; }
        }

        let skipBecauseNoChangedPath = changedPath && !changedPath[rowNode.id];
        if (skipBecauseNoChangedPath) { return; }

        this.aggregateRowNode(rowNode, changedPath, measureColumns, pivotColumns);
    }

    private aggregateRowNode(rowNode: RowNode, changedPath: ChangedPath, measureColumns: Column[], pivotColumns: Column[]): void {

        let measureColumnsMissing = measureColumns.length === 0;
        let pivotColumnsMissing = pivotColumns.length === 0;
        let userProvidedGroupRowAggNodes = this.gridOptionsWrapper.getGroupRowAggNodesFunc();

        let aggResult: any;
        if (rowNode.group && userProvidedGroupRowAggNodes) {
            aggResult = userProvidedGroupRowAggNodes(rowNode.childrenAfterFilter);
            if (changedPath) { this.pullUpChangedPath(rowNode, changedPath); }
        } else if (measureColumnsMissing) {
            aggResult = null;
        } else if (rowNode.group && pivotColumnsMissing) {
            aggResult = this.aggregateRowNodeUsingValuesOnly(rowNode, changedPath, measureColumns);
            if (changedPath) { this.pullUpChangedPath(rowNode, changedPath); }
        } else {
            aggResult = this.aggregateRowNodeUsingValuesAndPivot(rowNode);
            if (changedPath) { this.pullUpChangedPath(rowNode, changedPath); }
        }

        rowNode.setAggData(aggResult);

        // if we are grouping, then it's possible there is a sibling footer
        // to the group, so update the data here also if there is one
        if (rowNode.sibling) {
            rowNode.sibling.setAggData(aggResult);
        }
    }

    private pullUpChangedPath(rowNode: RowNode, changedPath: ChangedPath): void {
        if (!rowNode.parent) { return; }
        let parentId = rowNode.parent.id;
        let colIds: {[id:string]:boolean} = changedPath[rowNode.id];
        if (colIds) {
            if (!changedPath[parentId]) {
                changedPath[parentId] = {};
            }
            let parentColIds = changedPath[parentId];
            Object.keys(colIds).forEach( key => parentColIds[key] = true );
        }
    }

    private aggregateRowNodeUsingValuesAndPivot(rowNode: RowNode): any {
        let result: any = {};
        let pivotColumnDefs = this.pivotStage.getPivotColumnDefs();

        // Step 1: process value columns
        pivotColumnDefs
            .filter(v => !Utils.exists(v.pivotTotalColumnIds)) // only process pivot value columns
            .forEach(valueColDef => {
                let keys: string[] = valueColDef.pivotKeys;
                let values: any[];
                let valueColumn: Column = valueColDef.pivotValueColumn;

                if (rowNode.leafGroup) {
                    // lowest level group, get the values from the mapped set
                    values = this.getValuesFromMappedSet(rowNode.childrenMapped, keys, valueColumn);
                } else {
                    // value columns and pivot columns, non-leaf group
                    values = this.getValuesPivotNonLeaf(rowNode, valueColDef.colId);
                }

                result[valueColDef.colId] = this.aggregateValues(values, valueColumn.getAggFunc());
            });

        // Step 2: process total columns
        pivotColumnDefs
            .filter(v => Utils.exists(v.pivotTotalColumnIds)) // only process pivot total columns
            .forEach(totalColDef => {
                let aggResults: any[] = [];

                //retrieve results for colIds associated with this pivot total column
                totalColDef.pivotTotalColumnIds.forEach((colId: string) => {
                    aggResults.push(result[colId]);
                });

                result[totalColDef.colId] = this.aggregateValues(aggResults, totalColDef.aggFunc);
            });

        return result;
    }

    private aggregateRowNodeUsingValuesOnly(rowNode: RowNode, changedPath: ChangedPath, valueColumns: Column[]): any {
        let result: any = {};

        let changedPathThisRowNode = changedPath ? changedPath[rowNode.id] : null;

        // if a change path was provided, we only gather values for the change columns
        let changedValueColumns = changedPathThisRowNode ?
            valueColumns.filter( col => changedPathThisRowNode[col.getId()])
            : valueColumns;

        let notChangedValueColumns = changedPathThisRowNode ?
            valueColumns.filter( col => !changedPathThisRowNode[col.getId()])
            : null;

        let values2d = this.getValuesNormal(rowNode, changedValueColumns);
        let oldValues = rowNode.aggData;

        changedValueColumns.forEach( (valueColumn: Column, index: number) => {
            result[valueColumn.getId()] = this.aggregateValues(values2d[index], valueColumn.getAggFunc());
        });

        if (notChangedValueColumns && oldValues) {
            notChangedValueColumns.forEach( (valueColumn: Column) => {
                result[valueColumn.getId()] = oldValues[valueColumn.getId()];
            });
        }

        return result;
    }

    private getValuesPivotNonLeaf(rowNode: RowNode, colId: string): any[] {
        let values: any[] = [];
        rowNode.childrenAfterFilter.forEach( rowNode => {
            let value = rowNode.aggData[colId];
            values.push(value);
        });
        return values;
    }

    private getValuesFromMappedSet(mappedSet: any, keys: string[], valueColumn: Column): any[] {
        let mapPointer = mappedSet;
        keys.forEach( key => mapPointer = mapPointer ? mapPointer[key] : null );

        if (!mapPointer) {
            return [];
        }

        let values: any = [];
        mapPointer.forEach( (rowNode: RowNode) => {
            let value = this.valueService.getValue(valueColumn, rowNode);
            values.push(value);
        });

{'fraction_non_alphanumeric': 0.09723155489853515, 'fraction_numerical': 0.0015454911974197016, 'mean_word_length': 2.591457528957529, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2994705', 'n_tokens_mistral': 4059, 'n_tokens_neox': 3861, 'n_words': 902}