<commit_before>/*
 * Copyright (c) 2008-2016, Hazelcast, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.hazelcast.map.impl.query;

import com.hazelcast.config.Config;
import com.hazelcast.config.MapIndexConfig;
import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.IMap;
import com.hazelcast.query.Predicate;
import com.hazelcast.query.SampleObjects.Employee;
import com.hazelcast.query.SampleObjects.Value;
import com.hazelcast.query.SqlPredicate;
import com.hazelcast.spi.properties.GroupProperty;
import com.hazelcast.test.AssertTask;
import com.hazelcast.test.HazelcastParallelClassRunner;
import com.hazelcast.test.HazelcastTestSupport;
import com.hazelcast.test.TestHazelcastInstanceFactory;
import com.hazelcast.test.annotation.ParallelTest;
import com.hazelcast.test.annotation.QuickTest;
import com.hazelcast.util.Clock;
import com.hazelcast.util.IterableUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import static com.hazelcast.query.Predicates.equal;
import static com.hazelcast.test.TimeConstants.MINUTE;
import static java.lang.Thread.interrupted;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

@RunWith(HazelcastParallelClassRunner.class)
@Category({QuickTest.class, ParallelTest.class})
public class QueryIndexMigrationTest extends HazelcastTestSupport {

    private Random random = new Random();

    private TestHazelcastInstanceFactory nodeFactory;
    private ExecutorService executor;

    @Before
    public void createFactory() {
        nodeFactory = createHazelcastInstanceFactory(6);
    }

    @After
    public void shutdown() throws Exception {
        if (executor != null) {
            executor.shutdownNow();
            executor.awaitTermination(ASSERT_TRUE_EVENTUALLY_TIMEOUT, SECONDS);
        }
        shutdownNodeFactory();
    }

    @Test(timeout = MINUTE)
    public void testQueryDuringAndAfterMigration() throws Exception {
        HazelcastInstance instance = nodeFactory.newHazelcastInstance();
        int count = 500;
        IMap<String, Employee> map = instance.getMap("employees");
        for (int i = 0; i < count; i++) {
            map.put(String.valueOf(i), new Employee("joe" + i, i % 60, ((i & 1) == 1), (double) i));
        }

        nodeFactory.newInstances(new Config(), 3);

        final IMap<String, Employee> employees = instance.getMap("employees");
        assertTrueAllTheTime(new AssertTask() {
            @Override
            public void run() throws Exception {
                Collection<Employee> values = employees.values(new SqlPredicate("active and name LIKE 'joe15%'"));
                for (Employee employee : values) {
                    assertTrue(employee.isActive());
                }
                assertEquals(6, values.size());
            }
        }, 3);
    }

    @Test
    public void testQueryDuringAndAfterMigrationWithIndex() throws Exception {
        Config config = new Config();
        HazelcastInstance instance = nodeFactory.newHazelcastInstance(config);

        IMap<String, Employee> map = instance.getMap("employees");
        map.addIndex("name", false);
        map.addIndex("active", false);

        int size = 500;
        for (int i = 0; i < size; i++) {
            map.put(String.valueOf(i), new Employee("joe" + i, i % 60, ((i & 1) == 1), (double) i));
        }

        nodeFactory.newInstances(config, 3);

        final IMap<String, Employee> employees = instance.getMap("employees");
        assertTrueAllTheTime(new AssertTask() {
            @Override
            public void run() throws Exception {
                Collection<Employee> values = employees.values(new SqlPredicate("active and name LIKE 'joe15%'"));
                for (Employee employee : values) {
                    assertTrue(employee.isActive() && employee.getName().startsWith("joe15"));
                }
                assertEquals(6, values.size());
            }
        }, 3);
    }

    @Test
    public void testQueryWithIndexesWhileMigrating() throws Exception {
        HazelcastInstance instance = nodeFactory.newHazelcastInstance();
        IMap<String, Employee> map = instance.getMap("employees");
        map.addIndex("age", true);
        map.addIndex("active", false);

        for (int i = 0; i < 500; i++) {
            map.put("e" + i, new Employee("name" + i, i % 50, ((i & 1) == 1), (double) i));
        }
        assertEquals(500, map.size());
        Set<Map.Entry<String, Employee>> entries = map.entrySet(new SqlPredicate("active=true and age>44"));
        assertEquals(30, entries.size());

        nodeFactory.newInstances(new Config(), 3);

        long startNow = Clock.currentTimeMillis();
        while ((Clock.currentTimeMillis() - startNow) < 10000) {
            entries = map.entrySet(new SqlPredicate("active=true and age>44"));
            assertEquals(30, entries.size());
        }
    }

    /**
     * test for issue #359
     */
    @Test(timeout = 2 * MINUTE)
    public void testIndexCleanupOnMigration() throws Exception {
        int nodeCount = 6;
        final int runCount = 500;
        final Config config = newConfigWithIndex("testMap", "name");
        executor = Executors.newFixedThreadPool(nodeCount);
        List<Future<?>> futures = new ArrayList<Future<?>>();

        for (int i = 0; i < nodeCount; i++) {
            sleepMillis(random.nextInt((i + 1) * 100) + 10);
            futures.add(executor.submit(new Runnable() {
                public void run() {
                    HazelcastInstance hz = nodeFactory.newHazelcastInstance(config);
                    IMap<Object, Value> map = hz.getMap("testMap");
                    updateMapAndRunQuery(map, runCount);
                }
            }));
        }<commit_msg>QueryIndexMigrationTest moved to SlowTest category<commit_after>/*
 * Copyright (c) 2008-2016, Hazelcast, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.hazelcast.map.impl.query;

import com.hazelcast.config.Config;
import com.hazelcast.config.MapIndexConfig;
import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.IMap;
import com.hazelcast.query.Predicate;
import com.hazelcast.query.SampleObjects.Employee;
import com.hazelcast.query.SampleObjects.Value;
import com.hazelcast.query.SqlPredicate;
import com.hazelcast.spi.properties.GroupProperty;
import com.hazelcast.test.AssertTask;
import com.hazelcast.test.HazelcastParallelClassRunner;
import com.hazelcast.test.HazelcastTestSupport;
import com.hazelcast.test.TestHazelcastInstanceFactory;
import com.hazelcast.test.annotation.ParallelTest;
import com.hazelcast.test.annotation.SlowTest;
import com.hazelcast.util.Clock;
import com.hazelcast.util.IterableUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import static com.hazelcast.query.Predicates.equal;
import static com.hazelcast.test.TimeConstants.MINUTE;
import static java.lang.Thread.interrupted;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

@RunWith(HazelcastParallelClassRunner.class)
@Category({SlowTest.class, ParallelTest.class})
public class QueryIndexMigrationTest extends HazelcastTestSupport {

    private Random random = new Random();

    private TestHazelcastInstanceFactory nodeFactory;
    private ExecutorService executor;

    @Before
    public void createFactory() {
        nodeFactory = createHazelcastInstanceFactory(6);
    }

    @After
    public void shutdown() throws Exception {
        if (executor != null) {
            executor.shutdownNow();
            executor.awaitTermination(ASSERT_TRUE_EVENTUALLY_TIMEOUT, SECONDS);
        }
        shutdownNodeFactory();
    }

    @Test(timeout = MINUTE)
    public void testQueryDuringAndAfterMigration() throws Exception {
        HazelcastInstance instance = nodeFactory.newHazelcastInstance();
        int count = 500;
        IMap<String, Employee> map = instance.getMap("employees");
        for (int i = 0; i < count; i++) {
            map.put(String.valueOf(i), new Employee("joe" + i, i % 60, ((i & 1) == 1), (double) i));
        }

        nodeFactory.newInstances(new Config(), 3);

        final IMap<String, Employee> employees = instance.getMap("employees");
        assertTrueAllTheTime(new AssertTask() {
            @Override
            public void run() throws Exception {
                Collection<Employee> values = employees.values(new SqlPredicate("active and name LIKE 'joe15%'"));
                for (Employee employee : values) {
                    assertTrue(employee.isActive());
                }
                assertEquals(6, values.size());
            }
        }, 3);
    }

    @Test
    public void testQueryDuringAndAfterMigrationWithIndex() throws Exception {
        Config config = new Config();
        HazelcastInstance instance = nodeFactory.newHazelcastInstance(config);

        IMap<String, Employee> map = instance.getMap("employees");
        map.addIndex("name", false);
        map.addIndex("active", false);

        int size = 500;
        for (int i = 0; i < size; i++) {
            map.put(String.valueOf(i), new Employee("joe" + i, i % 60, ((i & 1) == 1), (double) i));
        }

        nodeFactory.newInstances(config, 3);

        final IMap<String, Employee> employees = instance.getMap("employees");
        assertTrueAllTheTime(new AssertTask() {
            @Override
            public void run() throws Exception {
                Collection<Employee> values = employees.values(new SqlPredicate("active and name LIKE 'joe15%'"));
                for (Employee employee : values) {
                    assertTrue(employee.isActive() && employee.getName().startsWith("joe15"));
                }
                assertEquals(6, values.size());
            }
        }, 3);
    }

    @Test
    public void testQueryWithIndexesWhileMigrating() throws Exception {
        HazelcastInstance instance = nodeFactory.newHazelcastInstance();
        IMap<String, Employee> map = instance.getMap("employees");
        map.addIndex("age", true);
        map.addIndex("active", false);

        for (int i = 0; i < 500; i++) {
            map.put("e" + i, new Employee("name" + i, i % 50, ((i & 1) == 1), (double) i));
        }
        assertEquals(500, map.size());
        Set<Map.Entry<String, Employee>> entries = map.entrySet(new SqlPredicate("active=true and age>44"));
        assertEquals(30, entries.size());

        nodeFactory.newInstances(new Config(), 3);

        long startNow = Clock.currentTimeMillis();
        while ((Clock.currentTimeMillis() - startNow) < 10000) {
            entries = map.entrySet(new SqlPredicate("active=true and age>44"));
            assertEquals(30, entries.size());
        }
    }

    /**
     * test for issue #359
     */
    @Test(timeout = 4 * MINUTE)
    public void testIndexCleanupOnMigration() throws Exception {
        int nodeCount = 6;
        final int runCount = 500;
        final Config config = newConfigWithIndex("testMap", "name");
        executor = Executors.newFixedThreadPool(nodeCount);
        List<Future<?>> futures = new ArrayList<Future<?>>();

        for (int i = 0; i < nodeCount; i++) {
            sleepMillis(random.nextInt((i + 1) * 100) + 10);
            futures.add(executor.submit(new Runnable() {
                public void run() {
                    HazelcastInstance hz = nodeFactory.newHazelcastInstance(config);
                    IMap<Object, Value> map = hz.getMap("testMap");
                    updateMapAndRunQuery(map, runCount);
                }
            }));
        }
{'fraction_non_alphanumeric': 0.11332312404287902, 'fraction_numerical': 0.011813607525705535, 'mean_word_length': 2.98546934030805, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6081474', 'n_tokens_mistral': 4087, 'n_tokens_neox': 3874, 'n_words': 948}