<commit_before>/*-
 * ========================LICENSE_START=================================
 * restheart-mongodb
 * %%
 * Copyright (C) 2014 - 2020 SoftInstigate
 * %%
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 * =========================LICENSE_END==================================
 */
package org.restheart.mongodb.db;

import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.mongodb.client.FindIterable;
import static java.lang.Thread.MIN_PRIORITY;
import java.util.Comparator;
import java.util.Objects;
import java.util.Optional;
import java.util.TreeMap;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Executors;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.bson.BsonDocument;
import static org.fusesource.jansi.Ansi.Color.GREEN;
import static org.fusesource.jansi.Ansi.Color.RED;
import static org.fusesource.jansi.Ansi.Color.YELLOW;
import static org.fusesource.jansi.Ansi.ansi;
import org.restheart.cache.Cache;
import org.restheart.cache.CacheFactory;
import org.restheart.cache.LoadingCache;
import org.restheart.exchange.ExchangeKeys.EAGER_CURSOR_ALLOCATION_POLICY;
import org.restheart.mongodb.MongoServiceConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 *
 * @author <NAME> {@literal <<EMAIL>>}
 */
public class CursorPool {

    private static final Logger LOGGER
            = LoggerFactory.getLogger(CursorPool.class);

    /**
     * Cursor in the pool won't be used if<br>REQUESTED_SKIPS - POOL_SKIPS &gt;
     * MIN_SKIP_DISTANCE_PERCENTAGE * REQUESTED_SKIPS.<br>The cursor from the
     * pool need to be iterated via the next() method (REQUESTED_SKIPS -
     * POOL_SKIPS) times to reach the requested page; since skip() is more
     * efficient than next(), using the cursor in the pool is worthwhile only if
     * next() has to be used less than MIN_SKIP_DISTANCE_PERCENTAGE *
     * REQUESTED_SKIPS times.
     */
    public static final double MIN_SKIP_DISTANCE_PERCENTAGE = 10 / 100f; // 10%
    // MUST BE < 10 since this 10 the TTL of the default cursor in mongodb
    private static final long TTL = 8 * 60 * 1000;
    private static final long POOL_SIZE
            = MongoServiceConfiguration.get().getEagerPoolSize();
    private static final ThreadPoolExecutor POOL_POPULATOR
            = new ThreadPoolExecutor(
                    1, 2,
                    1, TimeUnit.MINUTES,
                    new ArrayBlockingQueue<>(1),
                    new ThreadFactoryBuilder()
                            .setDaemon(true)
                            .setNameFormat("cursor-pool-populator-%d")
                            .setPriority(MIN_PRIORITY)
                            .build()
            );

    /**
     *
     * @return
     */
    public static CursorPool getInstance() {
        return DBCursorPoolSingletonHolder.INSTANCE;
    }

    private final DatabaseImpl dbsDAO;

    private final int SKIP_SLICE_LINEAR_DELTA
            = MongoServiceConfiguration.get().getEagerLinearSliceDelta();

    private final int SKIP_SLICE_LINEAR_WIDTH
            = MongoServiceConfiguration.get().getEagerLinearSliceWidht();

    private final int[] SKIP_SLICES_HEIGHTS
            = MongoServiceConfiguration.get().getEagerLinearSliceHeights();

    private final int SKIP_SLICE_RND_MIN_WIDTH
            = MongoServiceConfiguration.get().getEagerRndSliceMinWidht();

    private final int SKIP_SLICE_RND_MAX_CURSORS
            = MongoServiceConfiguration.get().getEagerRndMaxCursors();

    private final Cache<CursorPoolEntryKey, FindIterable<BsonDocument>> cache;
    private final LoadingCache<CursorPoolEntryKey, Long> collSizes;

    private CursorPool(DatabaseImpl dbsDAO) {
        this.dbsDAO = dbsDAO;

        cache = CacheFactory.createLocalCache(
                POOL_SIZE,
                Cache.EXPIRE_POLICY.AFTER_READ,
                TTL);

        collSizes = CacheFactory.createLocalLoadingCache(100,
                org.restheart.cache.Cache.EXPIRE_POLICY.AFTER_WRITE,
                60 * 1000,
                (CursorPoolEntryKey key) -> {
                    return dbsDAO.getCollectionSize(
                            key.getSession(),
                            key.getCollection(),
                            key.getFilter());
                }
        );

        if (LOGGER.isDebugEnabled()) {
            // print stats every 1 minute
            Executors.newSingleThreadScheduledExecutor()
                    .scheduleAtFixedRate(() -> {
                        getCacheSizes().forEach((s, c) -> {
                            LOGGER.debug("db cursor pool size: {}\t{}", s, c);
                        });

                        LOGGER.trace(""
                                + "db cursor pool entries: {}",
                                cache.asMap().keySet());
                    }, 1, 1, TimeUnit.MINUTES);
        }
    }

    /**
     *
     * @param key
     * @param allocationPolicy
     * @return
     */
    public synchronized SkippedFindIterable get(
            CursorPoolEntryKey key,
            EAGER_CURSOR_ALLOCATION_POLICY allocationPolicy) {
        if (key.getSkipped() < SKIP_SLICE_LINEAR_WIDTH) {<commit_msg>:loud_sound: Print cursor pool stats on TRACE level<commit_after>/*-
 * ========================LICENSE_START=================================
 * restheart-mongodb
 * %%
 * Copyright (C) 2014 - 2020 SoftInstigate
 * %%
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 * =========================LICENSE_END==================================
 */
package org.restheart.mongodb.db;

import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.mongodb.client.FindIterable;
import static java.lang.Thread.MIN_PRIORITY;
import java.util.Comparator;
import java.util.Objects;
import java.util.Optional;
import java.util.TreeMap;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Executors;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.bson.BsonDocument;
import static org.fusesource.jansi.Ansi.Color.GREEN;
import static org.fusesource.jansi.Ansi.Color.RED;
import static org.fusesource.jansi.Ansi.Color.YELLOW;
import static org.fusesource.jansi.Ansi.ansi;
import org.restheart.cache.Cache;
import org.restheart.cache.CacheFactory;
import org.restheart.cache.LoadingCache;
import org.restheart.exchange.ExchangeKeys.EAGER_CURSOR_ALLOCATION_POLICY;
import org.restheart.mongodb.MongoServiceConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 *
 * @author <NAME> {@literal <<EMAIL>>}
 */
public class CursorPool {

    private static final Logger LOGGER
            = LoggerFactory.getLogger(CursorPool.class);

    /**
     * Cursor in the pool won't be used if<br>REQUESTED_SKIPS - POOL_SKIPS &gt;
     * MIN_SKIP_DISTANCE_PERCENTAGE * REQUESTED_SKIPS.<br>The cursor from the
     * pool need to be iterated via the next() method (REQUESTED_SKIPS -
     * POOL_SKIPS) times to reach the requested page; since skip() is more
     * efficient than next(), using the cursor in the pool is worthwhile only if
     * next() has to be used less than MIN_SKIP_DISTANCE_PERCENTAGE *
     * REQUESTED_SKIPS times.
     */
    public static final double MIN_SKIP_DISTANCE_PERCENTAGE = 10 / 100f; // 10%
    // MUST BE < 10 since this 10 the TTL of the default cursor in mongodb
    private static final long TTL = 8 * 60 * 1000;
    private static final long POOL_SIZE
            = MongoServiceConfiguration.get().getEagerPoolSize();
    private static final ThreadPoolExecutor POOL_POPULATOR
            = new ThreadPoolExecutor(
                    1, 2,
                    1, TimeUnit.MINUTES,
                    new ArrayBlockingQueue<>(1),
                    new ThreadFactoryBuilder()
                            .setDaemon(true)
                            .setNameFormat("cursor-pool-populator-%d")
                            .setPriority(MIN_PRIORITY)
                            .build()
            );

    /**
     *
     * @return
     */
    public static CursorPool getInstance() {
        return DBCursorPoolSingletonHolder.INSTANCE;
    }

    private final DatabaseImpl dbsDAO;

    private final int SKIP_SLICE_LINEAR_DELTA
            = MongoServiceConfiguration.get().getEagerLinearSliceDelta();

    private final int SKIP_SLICE_LINEAR_WIDTH
            = MongoServiceConfiguration.get().getEagerLinearSliceWidht();

    private final int[] SKIP_SLICES_HEIGHTS
            = MongoServiceConfiguration.get().getEagerLinearSliceHeights();

    private final int SKIP_SLICE_RND_MIN_WIDTH
            = MongoServiceConfiguration.get().getEagerRndSliceMinWidht();

    private final int SKIP_SLICE_RND_MAX_CURSORS
            = MongoServiceConfiguration.get().getEagerRndMaxCursors();

    private final Cache<CursorPoolEntryKey, FindIterable<BsonDocument>> cache;
    private final LoadingCache<CursorPoolEntryKey, Long> collSizes;

    private CursorPool(DatabaseImpl dbsDAO) {
        this.dbsDAO = dbsDAO;

        cache = CacheFactory.createLocalCache(
                POOL_SIZE,
                Cache.EXPIRE_POLICY.AFTER_READ,
                TTL);

        collSizes = CacheFactory.createLocalLoadingCache(100,
                org.restheart.cache.Cache.EXPIRE_POLICY.AFTER_WRITE,
                60 * 1000,
                (CursorPoolEntryKey key) -> {
                    return dbsDAO.getCollectionSize(
                            key.getSession(),
                            key.getCollection(),
                            key.getFilter());
                }
        );

        if (LOGGER.isTraceEnabled()) {
            // print stats every 1 minute
            Executors.newSingleThreadScheduledExecutor()
                    .scheduleAtFixedRate(() -> {
                        getCacheSizes().forEach((s, c) -> {
                            LOGGER.debug("db cursor pool size: {}\t{}", s, c);
                        });

                        LOGGER.trace("db cursor pool entries: {}",
                                cache.asMap().keySet());
                    }, 1, 1, TimeUnit.MINUTES);
        }
    }

    /**
     *
     * @param key
     * @param allocationPolicy
     * @return
     */
    public synchronized SkippedFindIterable get(
            CursorPoolEntryKey key,
            EAGER_CURSOR_ALLOCATION_POLICY allocationPolicy) {
        if (key.getSkipped() < SKIP_SLICE_LINEAR_WIDTH) {
{'fraction_non_alphanumeric': 0.10234773164007017, 'fraction_numerical': 0.007519425181719442, 'mean_word_length': 2.522660388463802, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '936304', 'n_tokens_mistral': 3464, 'n_tokens_neox': 3272, 'n_words': 820}