<commit_before>/*
 * Copyright (c) 2008-2017, Hazelcast, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.hazelcast.client;

import com.hazelcast.client.config.ClientConfig;
import com.hazelcast.client.impl.HazelcastClientInstanceImpl;
import com.hazelcast.client.spi.ClientInvocationService;
import com.hazelcast.client.spi.impl.ClientInvocation;
import com.hazelcast.client.spi.impl.ClientSmartInvocationServiceImpl;
import com.hazelcast.client.test.bounce.MultiSocketClientDriverFactory;
import com.hazelcast.config.Config;
import com.hazelcast.core.ExecutionCallback;
import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.HazelcastOverloadException;
import com.hazelcast.core.IMap;
import com.hazelcast.internal.util.ThreadLocalRandomProvider;
import com.hazelcast.test.HazelcastParametersRunnerFactory;
import com.hazelcast.test.HazelcastTestSupport;
import com.hazelcast.test.annotation.SlowTest;
import com.hazelcast.test.bounce.BounceMemberRule;
import org.junit.After;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameter;
import org.junit.runners.Parameterized.Parameters;
import org.junit.runners.Parameterized.UseParametersRunnerFactory;

import java.lang.reflect.Field;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;

import static com.hazelcast.client.impl.ClientTestUtil.getHazelcastClientInstanceImpl;
import static com.hazelcast.client.spi.properties.ClientProperty.BACKPRESSURE_BACKOFF_TIMEOUT_MILLIS;
import static com.hazelcast.client.spi.properties.ClientProperty.MAX_CONCURRENT_INVOCATIONS;
import static com.hazelcast.util.ExceptionUtil.rethrow;
import static java.lang.Math.max;
import static java.lang.String.format;
import static java.lang.String.valueOf;
import static java.util.Arrays.asList;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

@RunWith(Parameterized.class)
@UseParametersRunnerFactory(HazelcastParametersRunnerFactory.class)
@Category(SlowTest.class)
public class ClientBackpressureBouncingTest extends HazelcastTestSupport {

    private static final long TEST_DURATION_SECONDS = 240; // 4 minutes
    private static final long TEST_TIMEOUT_MILLIS = 10 * 60 * 1000; // 10 minutes

    private static final int MAX_CONCURRENT_INVOCATION_CONFIG = 100;
    private static final int WORKER_THREAD_COUNT = 5;

    @Parameters(name = "backoffTimeoutMillis:{0}")
    public static Iterable<Object[]> parameters() {
        return asList(new Object[][]{
                {-1},
                {60000},
        });
    }

    @Parameter
    public long backoff;

    @Rule
    public BounceMemberRule bounceMemberRule = BounceMemberRule
            .with(new Config())
            .driverFactory(new MultiSocketClientDriverFactory(
                    new ClientConfig()
                            .setProperty(MAX_CONCURRENT_INVOCATIONS.getName(), valueOf(MAX_CONCURRENT_INVOCATION_CONFIG))
                            .setProperty(BACKPRESSURE_BACKOFF_TIMEOUT_MILLIS.getName(), valueOf(backoff))
            )).build();

    private InvocationCheckingThread checkingThread;

    @After
    public void tearDown() {
        if (checkingThread != null) {
            checkingThread.shutdown();
        }
    }

    @Test(timeout = TEST_TIMEOUT_MILLIS)
    public void testInFlightInvocationCountIsNotGrowing() {
        HazelcastInstance driver = bounceMemberRule.getNextTestDriver();
        checkingThread = new InvocationCheckingThread(driver);
        checkingThread.start();

        IMap<Integer, Integer> map = driver.getMap(randomMapName());
        Runnable[] tasks = createTasks(map);

        bounceMemberRule.testRepeatedly(tasks, TEST_DURATION_SECONDS);
        System.out.println("Finished bouncing");

        checkingThread.shutdown();
        checkingThread.assertInFlightInvocationsWereNotGrowing();
    }

    private Runnable[] createTasks(final IMap<Integer, Integer> map) {
        Runnable[] tasks = new Runnable[WORKER_THREAD_COUNT];
        for (int i = 0; i < WORKER_THREAD_COUNT; i++) {
            tasks[i] = new MyRunnable(map, i);
        }
        return tasks;
    }

    private static class InvocationCheckingThread extends Thread {

        private final long warmUpDeadline;
        private final long deadLine;
        private final ConcurrentMap<Long, ClientInvocation> callIdMap;

        private int maxInvocationCountObserved;
        private int maxInvocationCountObservedDuringWarmup;

        private volatile boolean running = true;

        private InvocationCheckingThread(HazelcastInstance client) {
            long durationMillis = TimeUnit.SECONDS.toMillis(TEST_DURATION_SECONDS);
            this.warmUpDeadline = System.currentTimeMillis() + (durationMillis / 5);
            this.deadLine = System.currentTimeMillis() + durationMillis;
            this.callIdMap = extraCallIdMap(client);
        }

        @Override
        public void run() {
            while (System.currentTimeMillis() < deadLine && running) {
                int currentSize = callIdMap.size();
                maxInvocationCountObserved = max(currentSize, maxInvocationCountObserved);
                if (System.currentTimeMillis() < warmUpDeadline) {
                    maxInvocationCountObservedDuringWarmup = max(currentSize, maxInvocationCountObservedDuringWarmup);
                }
                sleepAtLeastMillis(100);
            }
        }

        private void shutdown() {
            running = false;
            interrupt();
            assertJoinable(this);
        }

        private void assertInFlightInvocationsWereNotGrowing() {
            assertTrue("There are no invocations to be observed!", maxInvocationCountObserved > 0);

            long maximumTolerableInvocationCount = (long) (maxInvocationCountObservedDuringWarmup * 1.2);
            assertTrue("Apparently number of in-flight invocations is growing."
                    + " Max. number of in-flight invocation during first fifth of test duration: "
                    + maxInvocationCountObservedDuringWarmup
                    + " Max. number of in-flight invocation in total: "
                    + maxInvocationCountObserved, maxInvocationCountObserved <= maximumTolerableInvocationCount);
        }

        @SuppressWarnings("unchecked")
        private ConcurrentMap<Long, ClientInvocation> extraCallIdMap(HazelcastInstance client) {
            try {
                HazelcastClientInstanceImpl clientImpl = getHazelcastClientInstanceImpl(client);
                ClientInvocationService invocationService = clientImpl.getInvocationService();
                ClientSmartInvocationServiceImpl smartInvocationService = (ClientSmartInvocationServiceImpl) invocationService;
                Field callIdMapField = ClientSmartInvocationServiceImpl.class.getSuperclass().getDeclaredField("callIdMap");
                callIdMapField.setAccessible(true);
                return (ConcurrentMap<Long, ClientInvocation>) callIdMapField.get(smartInvocationService);
            } catch (Exception e) {
                throw rethrow(e);
            }
        }
    }

    private class MyRunnable implements Runnable {

        private final AtomicLong progressCounter = new AtomicLong();
        private final AtomicLong failureCounter = new AtomicLong();
        private final AtomicLong backpressureCounter = new AtomicLong();
        private final ExecutionCallback<Integer> callback = new CountingCallback();

        private final IMap<Integer, Integer> map;
        private final int workerNo;

        MyRunnable(IMap<Integer, Integer> map, int workerNo) {<commit_msg>Cleanup of ClientBackpressureBouncingTest<commit_after>/*
 * Copyright (c) 2008-2017, Hazelcast, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.hazelcast.client;

import com.hazelcast.client.config.ClientConfig;
import com.hazelcast.client.impl.HazelcastClientInstanceImpl;
import com.hazelcast.client.spi.ClientInvocationService;
import com.hazelcast.client.spi.impl.ClientInvocation;
import com.hazelcast.client.spi.impl.ClientSmartInvocationServiceImpl;
import com.hazelcast.client.test.bounce.MultiSocketClientDriverFactory;
import com.hazelcast.config.Config;
import com.hazelcast.core.ExecutionCallback;
import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.HazelcastOverloadException;
import com.hazelcast.core.IMap;
import com.hazelcast.internal.util.ThreadLocalRandomProvider;
import com.hazelcast.test.HazelcastParametersRunnerFactory;
import com.hazelcast.test.HazelcastTestSupport;
import com.hazelcast.test.annotation.SlowTest;
import com.hazelcast.test.bounce.BounceMemberRule;
import org.junit.After;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameter;
import org.junit.runners.Parameterized.Parameters;
import org.junit.runners.Parameterized.UseParametersRunnerFactory;

import java.lang.reflect.Field;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;

import static com.hazelcast.client.impl.ClientTestUtil.getHazelcastClientInstanceImpl;
import static com.hazelcast.client.spi.properties.ClientProperty.BACKPRESSURE_BACKOFF_TIMEOUT_MILLIS;
import static com.hazelcast.client.spi.properties.ClientProperty.MAX_CONCURRENT_INVOCATIONS;
import static com.hazelcast.util.ExceptionUtil.rethrow;
import static java.lang.Math.max;
import static java.lang.String.format;
import static java.lang.String.valueOf;
import static java.util.Arrays.asList;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

@RunWith(Parameterized.class)
@UseParametersRunnerFactory(HazelcastParametersRunnerFactory.class)
@Category(SlowTest.class)
public class ClientBackpressureBouncingTest extends HazelcastTestSupport {

    private static final long TEST_DURATION_SECONDS = 240; // 4 minutes
    private static final long TEST_TIMEOUT_MILLIS = 10 * 60 * 1000; // 10 minutes

    private static final int MAX_CONCURRENT_INVOCATION_CONFIG = 100;
    private static final int WORKER_THREAD_COUNT = 5;

    @Parameters(name = "backoffTimeoutMillis:{0}")
    public static Iterable<Object[]> parameters() {
        return asList(new Object[][]{
                {-1},
                {60000},
        });
    }

    @Parameter
    public long backoff;

    @Rule
    public BounceMemberRule bounceMemberRule = BounceMemberRule
            .with(new Config())
            .driverFactory(new MultiSocketClientDriverFactory(
                    new ClientConfig()
                            .setProperty(MAX_CONCURRENT_INVOCATIONS.getName(), valueOf(MAX_CONCURRENT_INVOCATION_CONFIG))
                            .setProperty(BACKPRESSURE_BACKOFF_TIMEOUT_MILLIS.getName(), valueOf(backoff))
            )).build();

    private InvocationCheckingThread checkingThread;

    @After
    public void tearDown() {
        if (checkingThread != null) {
            checkingThread.shutdown();
        }
    }

    @Test(timeout = TEST_TIMEOUT_MILLIS)
    public void testInFlightInvocationCountIsNotGrowing() {
        HazelcastInstance driver = bounceMemberRule.getNextTestDriver();
        IMap<Integer, Integer> map = driver.getMap(randomMapName());
        Runnable[] tasks = createTasks(map);

        checkingThread = new InvocationCheckingThread(driver);
        checkingThread.start();

        bounceMemberRule.testRepeatedly(tasks, TEST_DURATION_SECONDS);
        System.out.println("Finished bouncing");

        checkingThread.shutdown();
        checkingThread.assertInFlightInvocationsWereNotGrowing();
    }

    private Runnable[] createTasks(final IMap<Integer, Integer> map) {
        Runnable[] tasks = new Runnable[WORKER_THREAD_COUNT];
        for (int i = 0; i < WORKER_THREAD_COUNT; i++) {
            tasks[i] = new MyRunnable(map, i);
        }
        return tasks;
    }

    private static class InvocationCheckingThread extends Thread {

        private final long warmUpDeadline;
        private final long deadLine;
        private final ConcurrentMap<Long, ClientInvocation> callIdMap;

        private int maxInvocationCountObserved;
        private int maxInvocationCountObservedDuringWarmup;

        private volatile boolean running = true;

        private InvocationCheckingThread(HazelcastInstance client) {
            long durationMillis = TimeUnit.SECONDS.toMillis(TEST_DURATION_SECONDS);
            long now = System.currentTimeMillis();

            this.warmUpDeadline = now + (durationMillis / 5);
            this.deadLine = now + durationMillis;
            this.callIdMap = extraCallIdMap(client);
        }

        @Override
        public void run() {
            while (System.currentTimeMillis() < deadLine && running) {
                int currentSize = callIdMap.size();
                maxInvocationCountObserved = max(currentSize, maxInvocationCountObserved);
                if (System.currentTimeMillis() < warmUpDeadline) {
                    maxInvocationCountObservedDuringWarmup = max(currentSize, maxInvocationCountObservedDuringWarmup);
                }
                sleepAtLeastMillis(100);
            }
        }

        private void shutdown() {
            running = false;
            interrupt();
            assertJoinable(this);
        }

        private void assertInFlightInvocationsWereNotGrowing() {
            assertTrue("There are no invocations to be observed!", maxInvocationCountObserved > 0);

            long maximumTolerableInvocationCount = (long) (maxInvocationCountObservedDuringWarmup * 1.2);
            assertTrue("Apparently number of in-flight invocations is growing."
                    + " Max. number of in-flight invocation during first fifth of test duration: "
                    + maxInvocationCountObservedDuringWarmup
                    + " Max. number of in-flight invocation in total: "
                    + maxInvocationCountObserved, maxInvocationCountObserved <= maximumTolerableInvocationCount);
        }

        @SuppressWarnings("unchecked")
        private ConcurrentMap<Long, ClientInvocation> extraCallIdMap(HazelcastInstance client) {
            try {
                HazelcastClientInstanceImpl clientImpl = getHazelcastClientInstanceImpl(client);
                ClientInvocationService invocationService = clientImpl.getInvocationService();
                ClientSmartInvocationServiceImpl smartInvocationService = (ClientSmartInvocationServiceImpl) invocationService;
                Field callIdMapField = ClientSmartInvocationServiceImpl.class.getSuperclass().getDeclaredField("callIdMap");
                callIdMapField.setAccessible(true);
                return (ConcurrentMap<Long, ClientInvocation>) callIdMapField.get(smartInvocationService);
            } catch (Exception e) {
                throw rethrow(e);
            }
        }
    }

    private class MyRunnable implements Runnable {

        private final ExecutionCallback<Integer> callback = new CountingCallback();
        private final AtomicLong backpressureCounter = new AtomicLong();
        private final AtomicLong progressCounter = new AtomicLong();
        private final AtomicLong failureCounter = new AtomicLong();

        private final IMap<Integer, Integer> map;
        private final int workerNo;

        MyRunnable(IMap<Integer, Integer> map, int workerNo) {
{'fraction_non_alphanumeric': 0.07944927881749911, 'fraction_numerical': 0.005364167362021695, 'mean_word_length': 3.345765345765346, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2774217', 'n_tokens_mistral': 4691, 'n_tokens_neox': 4452, 'n_words': 999}