<commit_before>package com.hazelcast.simulator.tests.map;

import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.IList;
import com.hazelcast.core.IMap;
import com.hazelcast.logging.ILogger;
import com.hazelcast.logging.Logger;
import com.hazelcast.query.EntryObject;
import com.hazelcast.query.PagingPredicate;
import com.hazelcast.query.Predicate;
import com.hazelcast.query.PredicateBuilder;
import com.hazelcast.query.Predicates;
import com.hazelcast.query.SqlPredicate;
import com.hazelcast.simulator.test.TestContext;
import com.hazelcast.simulator.test.annotations.RunWithWorker;
import com.hazelcast.simulator.test.annotations.Setup;
import com.hazelcast.simulator.test.annotations.Verify;
import com.hazelcast.simulator.test.annotations.Warmup;
import com.hazelcast.simulator.tests.map.helpers.Employee;
import com.hazelcast.simulator.tests.map.helpers.PredicateOperationCounter;
import com.hazelcast.simulator.worker.loadsupport.MapStreamer;
import com.hazelcast.simulator.worker.loadsupport.MapStreamerFactory;
import com.hazelcast.simulator.worker.selector.OperationSelectorBuilder;
import com.hazelcast.simulator.worker.tasks.AbstractWorker;

import java.util.Collection;

import static java.lang.String.format;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.junit.Assert.assertTrue;

/**
 * In this test we are using different predicate methods to execute a query on a map of {@link Employee} objects.
 *
 * This test also concurrently updates and modifies the employee objects in the map while the predicate queries are executing. The
 * test may also destroy the map while while predicate are executing. We verify the result of every query to ensure that the
 * objects returned fit the requirements of the query.
 */
public class MapPredicateTest {

    private enum Operation {
        PREDICATE_BUILDER,
        SQL_STRING,
        PAGING_PREDICATE,
        UPDATE_EMPLOYEE,
        DESTROY_MAP
    }

    private static final ILogger LOGGER = Logger.getLogger(MapPredicateTest.class);

    public String basename = this.getClass().getSimpleName();
    public int threadCount = 3;
    public int keyCount = 100;
    public int pageSize = 5;

    public double predicateBuilderProb = 0.2;
    public double sqlStringProb = 0.2;
    public double pagePredicateProb = 0.2;
    public double updateEmployeeProb = 0.3;
    public double destroyProb = 0.1;

    private final OperationSelectorBuilder<Operation> operationSelectorBuilder = new OperationSelectorBuilder<Operation>();

    private IMap<Integer, Employee> map;
    private IList<PredicateOperationCounter> operationCounterList;

    @Setup
    public void setUp(TestContext testContext) throws Exception {
        HazelcastInstance targetInstance = testContext.getTargetInstance();
        map = targetInstance.getMap(basename);
        operationCounterList = targetInstance.getList(basename + "OperationCounter");

        operationSelectorBuilder.addOperation(Operation.PREDICATE_BUILDER, predicateBuilderProb)
                .addOperation(Operation.SQL_STRING, sqlStringProb)
                .addOperation(Operation.PAGING_PREDICATE, pagePredicateProb)
                .addOperation(Operation.UPDATE_EMPLOYEE, updateEmployeeProb)
                .addOperation(Operation.DESTROY_MAP, destroyProb);
    }

    @Warmup(global = true)
    public void globalWarmup() {
        initMap();
    }

    private void initMap() {
        MapStreamer<Integer, Employee> streamer = MapStreamerFactory.getInstance(map);
        for (int i = 0; i < keyCount; i++) {
            Employee employee = new Employee(i);
            streamer.pushEntry(employee.getId(), employee);
        }
        streamer.await();
    }

    @Verify(global = true)
    public void globalVerify() throws Exception {
        PredicateOperationCounter total = new PredicateOperationCounter();
        for (PredicateOperationCounter operationCounter : operationCounterList) {
            total.add(operationCounter);
        }
        LOGGER.info(format("Operation counters from %s: %s", basename, total));
    }

    @RunWithWorker
    public Worker createWorker() {
        return new Worker();
    }

    private class Worker extends AbstractWorker<Operation> {
        private final PredicateOperationCounter operationCounter = new PredicateOperationCounter();
        private long lastUpdateMs = System.currentTimeMillis();
        private long iterationsLastMinute = 0;
        private long maxLastMinute = Long.MIN_VALUE;
        private long minLastMinute = Long.MAX_VALUE;
        private long spendTimeMs = 0;

        public Worker() {
            super(operationSelectorBuilder);
        }

        @Override
        public void timeStep(Operation operation) {
            long startMs = System.currentTimeMillis();

            switch (operation) {
                case PREDICATE_BUILDER:
                    predicateBuilder();
                    break;
                case SQL_STRING:
                    sqlString();
                    break;
                case PAGING_PREDICATE:
                    pagingPredicate();
                    break;
                case UPDATE_EMPLOYEE:
                    updateEmployee();
                    break;
                case DESTROY_MAP:
                    destroyMap();
                    break;
                default:
                    throw new UnsupportedOperationException();
            }

            long nowMs = System.currentTimeMillis();
            long durationMs = nowMs - startMs;
            maxLastMinute = Math.max(durationMs, maxLastMinute);
            minLastMinute = Math.min(durationMs, minLastMinute);
            iterationsLastMinute++;
            spendTimeMs += durationMs;

            if (lastUpdateMs + SECONDS.toMillis(60) < nowMs) {
                double avg = (iterationsLastMinute * 1000d) / spendTimeMs;

                LOGGER.info("last minute: iterations=" + iterationsLastMinute
                        + " min=" + minLastMinute + " ms"
                        + " max=" + maxLastMinute + " ms"
                        + " avg=" + avg + " predicates/second");

                maxLastMinute = Long.MIN_VALUE;<commit_msg>Minor tweaks predicate logging<commit_after>package com.hazelcast.simulator.tests.map;

import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.IList;
import com.hazelcast.core.IMap;
import com.hazelcast.logging.ILogger;
import com.hazelcast.logging.Logger;
import com.hazelcast.query.EntryObject;
import com.hazelcast.query.PagingPredicate;
import com.hazelcast.query.Predicate;
import com.hazelcast.query.PredicateBuilder;
import com.hazelcast.query.Predicates;
import com.hazelcast.query.SqlPredicate;
import com.hazelcast.simulator.test.TestContext;
import com.hazelcast.simulator.test.annotations.RunWithWorker;
import com.hazelcast.simulator.test.annotations.Setup;
import com.hazelcast.simulator.test.annotations.Verify;
import com.hazelcast.simulator.test.annotations.Warmup;
import com.hazelcast.simulator.tests.map.helpers.Employee;
import com.hazelcast.simulator.tests.map.helpers.PredicateOperationCounter;
import com.hazelcast.simulator.worker.loadsupport.MapStreamer;
import com.hazelcast.simulator.worker.loadsupport.MapStreamerFactory;
import com.hazelcast.simulator.worker.selector.OperationSelectorBuilder;
import com.hazelcast.simulator.worker.tasks.AbstractWorker;

import java.util.Collection;

import static java.lang.String.format;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.junit.Assert.assertTrue;

/**
 * In this test we are using different predicate methods to execute a query on a map of {@link Employee} objects.
 *
 * This test also concurrently updates and modifies the employee objects in the map while the predicate queries are executing. The
 * test may also destroy the map while while predicate are executing. We verify the result of every query to ensure that the
 * objects returned fit the requirements of the query.
 */
public class MapPredicateTest {

    private enum Operation {
        PREDICATE_BUILDER,
        SQL_STRING,
        PAGING_PREDICATE,
        UPDATE_EMPLOYEE,
        DESTROY_MAP
    }

    private static final ILogger LOGGER = Logger.getLogger(MapPredicateTest.class);

    public String basename = this.getClass().getSimpleName();
    public int threadCount = 3;
    public int keyCount = 100;
    public int pageSize = 5;

    public double predicateBuilderProb = 0.2;
    public double sqlStringProb = 0.2;
    public double pagePredicateProb = 0.2;
    public double updateEmployeeProb = 0.3;
    public double destroyProb = 0.1;

    private final OperationSelectorBuilder<Operation> operationSelectorBuilder = new OperationSelectorBuilder<Operation>();

    private IMap<Integer, Employee> map;
    private IList<PredicateOperationCounter> operationCounterList;

    @Setup
    public void setUp(TestContext testContext) throws Exception {
        HazelcastInstance targetInstance = testContext.getTargetInstance();
        map = targetInstance.getMap(basename);
        operationCounterList = targetInstance.getList(basename + "OperationCounter");

        operationSelectorBuilder.addOperation(Operation.PREDICATE_BUILDER, predicateBuilderProb)
                .addOperation(Operation.SQL_STRING, sqlStringProb)
                .addOperation(Operation.PAGING_PREDICATE, pagePredicateProb)
                .addOperation(Operation.UPDATE_EMPLOYEE, updateEmployeeProb)
                .addOperation(Operation.DESTROY_MAP, destroyProb);
    }

    @Warmup(global = true)
    public void globalWarmup() {
        initMap();
    }

    private void initMap() {
        MapStreamer<Integer, Employee> streamer = MapStreamerFactory.getInstance(map);
        for (int i = 0; i < keyCount; i++) {
            Employee employee = new Employee(i);
            streamer.pushEntry(employee.getId(), employee);
        }
        streamer.await();
    }

    @Verify(global = true)
    public void globalVerify() throws Exception {
        PredicateOperationCounter total = new PredicateOperationCounter();
        for (PredicateOperationCounter operationCounter : operationCounterList) {
            total.add(operationCounter);
        }
        LOGGER.info(format("Operation counters from %s: %s", basename, total));
    }

    @RunWithWorker
    public Worker createWorker() {
        return new Worker();
    }

    private class Worker extends AbstractWorker<Operation> {
        private final PredicateOperationCounter operationCounter = new PredicateOperationCounter();
        private long lastUpdateMs = System.currentTimeMillis();
        private long iterationsLastMinute = 0;
        private long maxLastMinute = Long.MIN_VALUE;
        private long minLastMinute = Long.MAX_VALUE;
        private long spendTimeMs = 0;

        public Worker() {
            super(operationSelectorBuilder);
        }

        @Override
        public void timeStep(Operation operation) {
            long startMs = System.currentTimeMillis();

            switch (operation) {
                case PREDICATE_BUILDER:
                    predicateBuilder();
                    break;
                case SQL_STRING:
                    sqlString();
                    break;
                case PAGING_PREDICATE:
                    pagingPredicate();
                    break;
                case UPDATE_EMPLOYEE:
                    updateEmployee();
                    break;
                case DESTROY_MAP:
                    destroyMap();
                    break;
                default:
                    throw new UnsupportedOperationException();
            }

            long nowMs = System.currentTimeMillis();
            long durationMs = nowMs - startMs;
            maxLastMinute = Math.max(durationMs, maxLastMinute);
            minLastMinute = Math.min(durationMs, minLastMinute);
            iterationsLastMinute++;
            spendTimeMs += durationMs;

            if (lastUpdateMs + SECONDS.toMillis(60) < nowMs) {
                double avg = spendTimeMs/iterationsLastMinute;
                double perf = (iterationsLastMinute * 1000d) / spendTimeMs;

                LOGGER.info("last minute: iterations=" + iterationsLastMinute
                        + " min=" + minLastMinute + " ms"
                        + " max=" + maxLastMinute + " ms"
                        + " avg=" + avg + " ms"
                        + " perf=" + perf + " predicates/second");

                maxLastMinute = Long.MIN_VALUE;
{'fraction_non_alphanumeric': 0.08003504021661224, 'fraction_numerical': 0.003822569084972525, 'mean_word_length': 2.6902732882750513, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7662643', 'n_tokens_mistral': 3498, 'n_tokens_neox': 3321, 'n_words': 774}