<commit_before>/*
// This software is subject to the terms of the Eclipse Public License v1.0
// Agreement, available at the following URL:
// http://www.eclipse.org/legal/epl-v10.html.
// You must accept the terms of that agreement to use this software.
//
// Copyright (C) 2004-2005 <NAME>
// Copyright (C) 2005-2012 Pentaho and others
// All Rights Reserved.
*/
package mondrian.rolap;

import mondrian.olap.*;
import mondrian.rolap.agg.*;
import mondrian.rolap.aggmatcher.AggGen;
import mondrian.rolap.aggmatcher.AggStar;
import mondrian.rolap.cache.SegmentCacheIndex;
import mondrian.rolap.cache.SegmentCacheIndexImpl;
import mondrian.server.Execution;
import mondrian.server.Locus;
import mondrian.spi.*;
import mondrian.util.*;

import org.apache.log4j.Logger;
import org.apache.log4j.MDC;

import java.util.*;
import java.util.concurrent.Future;

/**
 * A <code>FastBatchingCellReader</code> doesn't really Read cells: when asked
 * to look up the values of stored measures, it lies, and records the fact
 * that the value was asked for.  Later, we can look over the values which
 * are required, fetch them in an efficient way, and re-run the evaluation
 * with a real evaluator.
 *
 * <p>NOTE: When it doesn't know the answer, it lies by returning an error
 * object.  The calling code must be able to deal with that.</p>
 *
 * <p>This class tries to minimize the amount of storage needed to record the
 * fact that a cell was requested.</p>
 */
public class FastBatchingCellReader implements CellReader {
    private static final Logger LOGGER =
        Logger.getLogger(FastBatchingCellReader.class);

    private final RolapCube cube;

    /**
     * Records the number of requests. The field is used for correctness: if
     * the request count stays the same during an operation, you know that the
     * FastBatchingCellReader has not told any lies during that operation, and
     * therefore the result is true. The field is also useful for debugging.
     */
    private int missCount;

    /**
     * Number of occasions that a requested cell was already in cache.
     */
    private int hitCount;

    /**
     * Number of occasions that requested cell was in the process of being
     * loaded into cache but not ready.
     */
    private int pendingCount;

    private final AggregationManager aggMgr;

    private final boolean cacheEnabled;

    private final SegmentCacheManager cacheMgr;

    private final RolapAggregationManager.PinSet pinnedSegments;

    /**
     * Indicates that the reader has given incorrect results.
     */
    private boolean dirty;

    private final List<CellRequest> cellRequests = new ArrayList<CellRequest>();

    /**
     * Creates a FastBatchingCellReader.
     *
     * @param execution Execution that calling statement belongs to. Allows us
     *                  to check for cancel
     * @param cube      Cube that requests belong to
     * @param aggMgr    Aggregation manager
     */
    public FastBatchingCellReader(
        Execution execution,
        RolapCube cube,
        AggregationManager aggMgr)
    {
        assert cube != null;
        assert execution != null;
        this.cube = cube;
        this.aggMgr = aggMgr;
        cacheMgr = aggMgr.cacheMgr;
        pinnedSegments = this.aggMgr.createPinSet();
        cacheEnabled = !MondrianProperties.instance().DisableCaching.get();
    }

    public Object get(RolapEvaluator evaluator) {
        final CellRequest request =
            RolapAggregationManager.makeRequest(evaluator);

        if (request == null || request.isUnsatisfiable()) {
            return Util.nullValue; // request not satisfiable.
        }

        // Try to retrieve a cell and simultaneously pin the segment which
        // contains it.
        final Object o = aggMgr.getCellFromCache(request, pinnedSegments);

        assert o != Boolean.TRUE : "getCellFromCache no longer returns TRUE";
        if (o != null) {
            ++hitCount;
            return o;
        }

        // If this query has not had any cache misses, it's worth doing a
        // synchronous request for the cell segment. If it is in the cache, it
        // will be worth the wait, because we can avoid the effort of batching
        // up requests that could have been satisfied by the same segment.
        if (cacheEnabled
            && missCount == 0)
        {
            SegmentWithData segmentWithData = cacheMgr.peek(request);
            if (segmentWithData != null) {
                segmentWithData.getStar().register(segmentWithData);
                final Object o2 =
                    aggMgr.getCellFromCache(request, pinnedSegments);
                if (o2 != null) {
                    ++hitCount;
                    return o2;
                }
            }
        }

        // if there is no such cell, record that we need to fetch it, and
        // return 'error'
        recordCellRequest(request);
        return RolapUtil.valueNotReadyException;
    }

    public int getMissCount() {
        return missCount;
    }

    public int getHitCount() {
        return hitCount;
    }

    public int getPendingCount() {
        return pendingCount;
    }

    public final void recordCellRequest(CellRequest request) {
        assert !request.isUnsatisfiable();
        ++missCount;
        cellRequests.add(request);
        int limit =
            MondrianProperties.instance().CellBatchSize.get();
        if (limit <= 0) {
            limit = 5000; // TODO Make this logic into a pluggable algorithm.
        }
        if (cellRequests.size() % limit == 0) {
            // Signal that it's time to ask the cache manager if it has cells
            // we need in the cache. Not really an exception.
            throw CellRequestQuantumExceededException.INSTANCE;
        }
    }<commit_msg>Augments the default size of CellBatchSize. According from the overwhelming feedback from services & customers, 5k was a ludicruously low default value. I'm cranking it up to 50k, which is a reasonable and safe value. Most users set it to 100k or so without problems related to memory consumption or concurrency issues.<commit_after>/*
// This software is subject to the terms of the Eclipse Public License v1.0
// Agreement, available at the following URL:
// http://www.eclipse.org/legal/epl-v10.html.
// You must accept the terms of that agreement to use this software.
//
// Copyright (C) 2004-2005 <NAME>
// Copyright (C) 2005-2012 Pentaho and others
// All Rights Reserved.
*/
package mondrian.rolap;

import mondrian.olap.*;
import mondrian.rolap.agg.*;
import mondrian.rolap.aggmatcher.AggGen;
import mondrian.rolap.aggmatcher.AggStar;
import mondrian.rolap.cache.SegmentCacheIndex;
import mondrian.rolap.cache.SegmentCacheIndexImpl;
import mondrian.server.Execution;
import mondrian.server.Locus;
import mondrian.spi.*;
import mondrian.util.*;

import org.apache.log4j.Logger;
import org.apache.log4j.MDC;

import java.util.*;
import java.util.concurrent.Future;

/**
 * A <code>FastBatchingCellReader</code> doesn't really Read cells: when asked
 * to look up the values of stored measures, it lies, and records the fact
 * that the value was asked for.  Later, we can look over the values which
 * are required, fetch them in an efficient way, and re-run the evaluation
 * with a real evaluator.
 *
 * <p>NOTE: When it doesn't know the answer, it lies by returning an error
 * object.  The calling code must be able to deal with that.</p>
 *
 * <p>This class tries to minimize the amount of storage needed to record the
 * fact that a cell was requested.</p>
 */
public class FastBatchingCellReader implements CellReader {
    private static final Logger LOGGER =
        Logger.getLogger(FastBatchingCellReader.class);

    private final RolapCube cube;

    /**
     * Records the number of requests. The field is used for correctness: if
     * the request count stays the same during an operation, you know that the
     * FastBatchingCellReader has not told any lies during that operation, and
     * therefore the result is true. The field is also useful for debugging.
     */
    private int missCount;

    /**
     * Number of occasions that a requested cell was already in cache.
     */
    private int hitCount;

    /**
     * Number of occasions that requested cell was in the process of being
     * loaded into cache but not ready.
     */
    private int pendingCount;

    private final AggregationManager aggMgr;

    private final boolean cacheEnabled;

    private final SegmentCacheManager cacheMgr;

    private final RolapAggregationManager.PinSet pinnedSegments;

    /**
     * Indicates that the reader has given incorrect results.
     */
    private boolean dirty;

    private final List<CellRequest> cellRequests = new ArrayList<CellRequest>();

    /**
     * Creates a FastBatchingCellReader.
     *
     * @param execution Execution that calling statement belongs to. Allows us
     *                  to check for cancel
     * @param cube      Cube that requests belong to
     * @param aggMgr    Aggregation manager
     */
    public FastBatchingCellReader(
        Execution execution,
        RolapCube cube,
        AggregationManager aggMgr)
    {
        assert cube != null;
        assert execution != null;
        this.cube = cube;
        this.aggMgr = aggMgr;
        cacheMgr = aggMgr.cacheMgr;
        pinnedSegments = this.aggMgr.createPinSet();
        cacheEnabled = !MondrianProperties.instance().DisableCaching.get();
    }

    public Object get(RolapEvaluator evaluator) {
        final CellRequest request =
            RolapAggregationManager.makeRequest(evaluator);

        if (request == null || request.isUnsatisfiable()) {
            return Util.nullValue; // request not satisfiable.
        }

        // Try to retrieve a cell and simultaneously pin the segment which
        // contains it.
        final Object o = aggMgr.getCellFromCache(request, pinnedSegments);

        assert o != Boolean.TRUE : "getCellFromCache no longer returns TRUE";
        if (o != null) {
            ++hitCount;
            return o;
        }

        // If this query has not had any cache misses, it's worth doing a
        // synchronous request for the cell segment. If it is in the cache, it
        // will be worth the wait, because we can avoid the effort of batching
        // up requests that could have been satisfied by the same segment.
        if (cacheEnabled
            && missCount == 0)
        {
            SegmentWithData segmentWithData = cacheMgr.peek(request);
            if (segmentWithData != null) {
                segmentWithData.getStar().register(segmentWithData);
                final Object o2 =
                    aggMgr.getCellFromCache(request, pinnedSegments);
                if (o2 != null) {
                    ++hitCount;
                    return o2;
                }
            }
        }

        // if there is no such cell, record that we need to fetch it, and
        // return 'error'
        recordCellRequest(request);
        return RolapUtil.valueNotReadyException;
    }

    public int getMissCount() {
        return missCount;
    }

    public int getHitCount() {
        return hitCount;
    }

    public int getPendingCount() {
        return pendingCount;
    }

    public final void recordCellRequest(CellRequest request) {
        assert !request.isUnsatisfiable();
        ++missCount;
        cellRequests.add(request);
        int limit =
            MondrianProperties.instance().CellBatchSize.get();
        if (limit <= 0) {
            limit = 50000; // TODO Make this logic into a pluggable algorithm.
        }
        if (cellRequests.size() % limit == 0) {
            // Signal that it's time to ask the cache manager if it has cells
            // we need in the cache. Not really an exception.
            throw CellRequestQuantumExceededException.INSTANCE;
        }
    }
{'fraction_non_alphanumeric': 0.0771871075763918, 'fraction_numerical': 0.005943909585600669, 'mean_word_length': 2.4111936036550543, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6883438', 'n_tokens_mistral': 3231, 'n_tokens_neox': 3106, 'n_words': 1178}