<commit_before>/*
 * Copyright 2016 Crown Copyright
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package stroom.volume.server;

import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import stroom.entity.server.util.StroomEntityManager;
import stroom.entity.shared.BaseResultList;
import stroom.node.server.MockStroomPropertyService;
import stroom.node.server.NodeCache;
import stroom.node.server.StroomPropertyService;
import stroom.node.shared.FindVolumeCriteria;
import stroom.node.shared.Node;
import stroom.node.shared.Rack;
import stroom.node.shared.Volume;
import stroom.node.shared.Volume.VolumeType;
import stroom.node.shared.VolumeState;
import stroom.statistics.internal.InternalStatisticsReceiver;
import stroom.util.config.StroomProperties;
import stroom.util.io.FileUtil;
import stroom.util.spring.StroomBeanStore;
import stroom.util.test.StroomJUnit4ClassRunner;
import stroom.util.test.StroomUnitTest;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;

@RunWith(StroomJUnit4ClassRunner.class)
public class TestVolumeServiceImpl extends StroomUnitTest {
    private static final Path DEFAULT_VOLUMES_PATH = StroomProperties.getConfigDir().resolve(VolumeServiceImpl.DEFAULT_VOLUMES_SUBDIR);

    private static final Path DEFAULT_INDEX_VOLUME_PATH = DEFAULT_VOLUMES_PATH.resolve(VolumeServiceImpl.DEFAULT_INDEX_VOLUME_SUBDIR);
    private static final Path DEFAULT_STREAM_VOLUME_PATH = DEFAULT_VOLUMES_PATH.resolve(VolumeServiceImpl.DEFAULT_STREAM_VOLUME_SUBDIR);
    final MockStroomPropertyService mockStroomPropertyService = new MockStroomPropertyService();
    private final Rack rack1 = Rack.create("rack1");
    private final Rack rack2 = Rack.create("rack2");
    private final Node node1a = Node.create(rack1, "1a");
    private final Node node1b = Node.create(rack1, "1b");
    private final Node node1c = Node.create(rack1, "1c");
    private final Node node2a = Node.create(rack2, "2a");
    private final Node node2b = Node.create(rack2, "2b");
    private final Volume public1a = Volume.create(node1a, FileUtil.getCanonicalPath(FileUtil.getTempDir().resolve("PUBLIC_1A")), VolumeType.PUBLIC,
            VolumeState.create(0, 1000));
    private final Volume public1b = Volume.create(node1b, FileUtil.getCanonicalPath(FileUtil.getTempDir().resolve("PUBLIC_1B")), VolumeType.PUBLIC,
            VolumeState.create(0, 1000));
    private final Volume public2a = Volume.create(node2a, FileUtil.getCanonicalPath(FileUtil.getTempDir().resolve("PUBLIC_2A")), VolumeType.PUBLIC,
            VolumeState.create(0, 1000));
    private final Volume public2b = Volume.create(node2b, FileUtil.getCanonicalPath(FileUtil.getTempDir().resolve("PUBLIC_2B")), VolumeType.PUBLIC,
            VolumeState.create(0, 1000));
    private List<Volume> volumeList = null;
    private MockVolumeService volumeServiceImpl = null;
    @Mock
    private StroomEntityManager stroomEntityManager;

    @Before
    public void init() throws IOException {
        MockitoAnnotations.initMocks(this);
        deleteDefaultVolumesDir();

        volumeList = new ArrayList<>();
//        volumeList.clear();
        volumeList.add(public1a);
        volumeList.add(public1b);
        volumeList.add(public2a);
        volumeList.add(public2b);

        mockStroomPropertyService.setProperty(VolumeServiceImpl.PROP_RESILIENT_REPLICATION_COUNT, "2");

        volumeServiceImpl = new MockVolumeService(stroomEntityManager, new NodeCache(node1a), mockStroomPropertyService, null, null);
        volumeServiceImpl.volumeList = volumeList;
    }

    @Test
    public void testNode1aNodeWithCacheAndSomePuckerLocalStorage() {
        final Set<Volume> call1 = volumeServiceImpl.getStreamVolumeSet(node1a);
        final Set<Volume> call2 = volumeServiceImpl.getStreamVolumeSet(node1a);
        Assert.assertEquals(2, call1.size());
        Assert.assertEquals(2, call2.size());

        // Check that we only write once in a rack
        Assert.assertTrue(call1.contains(public1a) ^ call1.contains(public1b));
        Assert.assertTrue(call1.contains(public2a) ^ call1.contains(public2b));
        Assert.assertTrue(call2.contains(public1a) ^ call2.contains(public1b));
        Assert.assertTrue(call2.contains(public2a) ^ call2.contains(public2b));

        // Check that we round robin OK
        Assert.assertTrue(call1.contains(public2a) ^ call2.contains(public2a));
        Assert.assertTrue(call1.contains(public2b) ^ call2.contains(public2b));

    }

    @Test
    public void testNode1cNodeWithNoStorage() {
        final Set<Volume> call1 = volumeServiceImpl.getStreamVolumeSet(node1c);
        final Set<Volume> call2 = volumeServiceImpl.getStreamVolumeSet(node1c);
        Assert.assertEquals(2, call1.size());
        Assert.assertEquals(2, call2.size());

        // Check that we only write once in a rack
        Assert.assertTrue(call1.contains(public1a) ^ call1.contains(public1b));
        Assert.assertTrue(call1.contains(public2a) ^ call1.contains(public2b));
        Assert.assertTrue(call2.contains(public1a) ^ call2.contains(public1b));
        Assert.assertTrue(call2.contains(public2a) ^ call2.contains(public2b));

        // Check that we round robin OK
        Assert.assertTrue(call1.contains(public1a) ^ call2.contains(public1a));
        Assert.assertTrue(call1.contains(public1b) ^ call2.contains(public1b));
        Assert.assertTrue(call1.contains(public2a) ^ call2.contains(public2a));
        Assert.assertTrue(call1.contains(public2b) ^ call2.contains(public2b));
    }

    @Test
    public void testNode2aNodeWithNoCache() {
        final Set<Volume> call1 = volumeServiceImpl.getStreamVolumeSet(node2a);
        final Set<Volume> call2 = volumeServiceImpl.getStreamVolumeSet(node2a);
        Assert.assertEquals(2, call1.size());
        Assert.assertEquals(2, call2.size());

        // Check that we only write once in a rack
        Assert.assertTrue(call1.contains(public1a) ^ call1.contains(public1b));
        Assert.assertTrue(call1.contains(public2a) ^ call1.contains(public2b));
        Assert.assertTrue(call2.contains(public1a) ^ call2.contains(public1b));
        Assert.assertTrue(call2.contains(public2a) ^ call2.contains(public2b));

        // Check that we round robin OK on rack 1
        Assert.assertTrue(call1.contains(public1a) ^ call2.contains(public1a));
        Assert.assertTrue(call1.contains(public1b) ^ call2.contains(public1b));
    }

    @Test
    public void testStartup_Disabled() {
        mockStroomPropertyService.setProperty(VolumeServiceImpl.PROP_CREATE_DEFAULT_VOLUME_ON_STARTUP, "false");

        volumeServiceImpl.startup();

        Assert.assertFalse(volumeServiceImpl.saveCalled);
        Assert.assertFalse(Files.exists(DEFAULT_INDEX_VOLUME_PATH));
        Assert.assertFalse(Files.exists(DEFAULT_STREAM_VOLUME_PATH));
    }

    @Test
    public void testStartup_EnabledExistingVolumes() {
        mockStroomPropertyService.setProperty(VolumeServiceImpl.PROP_CREATE_DEFAULT_VOLUME_ON_STARTUP, "true");

        volumeServiceImpl.startup();

        Assert.assertFalse(volumeServiceImpl.saveCalled);
        Assert.assertFalse(Files.exists(DEFAULT_INDEX_VOLUME_PATH));
        Assert.assertFalse(Files.exists(DEFAULT_STREAM_VOLUME_PATH));
    }

    @Test
    public void testStartup_EnabledNoExistingVolumes() {
        mockStroomPropertyService.setProperty(VolumeServiceImpl.PROP_CREATE_DEFAULT_VOLUME_ON_STARTUP, "true");
        volumeServiceImpl.volumeList.clear();
        volumeServiceImpl.startup();

        Assert.assertTrue(volumeServiceImpl.saveCalled);
        //make sure both paths have been saved
        Assert.assertEquals(2, volumeServiceImpl.savedVolumes.stream()
                .map(vol -> vol.getPath())
                .filter(path -> path.equals(FileUtil.getCanonicalPath(DEFAULT_INDEX_VOLUME_PATH)) ||
                        path.equals(FileUtil.getCanonicalPath(DEFAULT_STREAM_VOLUME_PATH)))
                .count());
        Assert.assertTrue(Files.exists(DEFAULT_INDEX_VOLUME_PATH));
        Assert.assertTrue(Files.exists(DEFAULT_STREAM_VOLUME_PATH));
    }

    private void deleteDefaultVolumesDir() throws IOException {
        FileUtil.deleteDir(DEFAULT_INDEX_VOLUME_PATH);
        FileUtil.deleteDir(DEFAULT_STREAM_VOLUME_PATH);
        FileUtil.deleteDir(DEFAULT_VOLUMES_PATH);
    }

    private static class MockVolumeService extends VolumeServiceImpl {

        public List<Volume> volumeList = null;
        public boolean saveCalled;
        public List<Volume> savedVolumes = new ArrayList<>();

        public MockVolumeService(final StroomEntityManager stroomEntityManager, final NodeCache nodeCache,
                                 final StroomPropertyService stroomPropertyService, final StroomBeanStore stroomBeanStore,
                                 final InternalStatisticsReceiver internalStatisticsReceiver) {
            super(stroomEntityManager, nodeCache, stroomPropertyService, stroomBeanStore, internalStatisticsReceiver);
        }

        @Override
        public BaseResultList<Volume> find(final FindVolumeCriteria criteria) {
            return BaseResultList.createUnboundedList(volumeList);
        }

        @Override
        VolumeState saveVolumeState(final VolumeState volumeState) {
            return volumeState;
        }

        @Override
        public Volume save(Volume entity) throws RuntimeException {
            super.save(entity);
            saveCalled = true;
            savedVolumes.add(entity);
            return entity;
        }
    }

    ;


}
<commit_msg>gh-907 Default volumes are now created as soon as they are needed<commit_after>/*
 * Copyright 2016 Crown Copyright
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package stroom.volume.server;

import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import stroom.entity.server.util.StroomEntityManager;
import stroom.entity.shared.BaseResultList;
import stroom.node.server.MockStroomPropertyService;
import stroom.node.server.NodeCache;
import stroom.node.server.StroomPropertyService;
import stroom.node.shared.FindVolumeCriteria;
import stroom.node.shared.Node;
import stroom.node.shared.Rack;
import stroom.node.shared.Volume;
import stroom.node.shared.Volume.VolumeType;
import stroom.node.shared.VolumeState;
import stroom.statistics.internal.InternalStatisticsReceiver;
import stroom.util.config.StroomProperties;
import stroom.util.io.FileUtil;
import stroom.util.test.StroomJUnit4ClassRunner;
import stroom.util.test.StroomUnitTest;

import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;

@RunWith(StroomJUnit4ClassRunner.class)
public class TestVolumeServiceImpl extends StroomUnitTest {
    private static final Path DEFAULT_VOLUMES_PATH = StroomProperties.getConfigDir().resolve(VolumeServiceImpl.DEFAULT_VOLUMES_SUBDIR);

    private static final Path DEFAULT_INDEX_VOLUME_PATH = DEFAULT_VOLUMES_PATH.resolve(VolumeServiceImpl.DEFAULT_INDEX_VOLUME_SUBDIR);
    private static final Path DEFAULT_STREAM_VOLUME_PATH = DEFAULT_VOLUMES_PATH.resolve(VolumeServiceImpl.DEFAULT_STREAM_VOLUME_SUBDIR);
    private final MockStroomPropertyService mockStroomPropertyService = new MockStroomPropertyService();
    private final Rack rack1 = Rack.create("rack1");
    private final Rack rack2 = Rack.create("rack2");
    private final Node node1a = Node.create(rack1, "1a");
    private final Node node1b = Node.create(rack1, "1b");
    private final Node node1c = Node.create(rack1, "1c");
    private final Node node2a = Node.create(rack2, "2a");
    private final Node node2b = Node.create(rack2, "2b");
    private final Volume public1a = Volume.create(node1a, FileUtil.getCanonicalPath(FileUtil.getTempDir().resolve("PUBLIC_1A")), VolumeType.PUBLIC,
            VolumeState.create(0, 1000));
    private final Volume public1b = Volume.create(node1b, FileUtil.getCanonicalPath(FileUtil.getTempDir().resolve("PUBLIC_1B")), VolumeType.PUBLIC,
            VolumeState.create(0, 1000));
    private final Volume public2a = Volume.create(node2a, FileUtil.getCanonicalPath(FileUtil.getTempDir().resolve("PUBLIC_2A")), VolumeType.PUBLIC,
            VolumeState.create(0, 1000));
    private final Volume public2b = Volume.create(node2b, FileUtil.getCanonicalPath(FileUtil.getTempDir().resolve("PUBLIC_2B")), VolumeType.PUBLIC,
            VolumeState.create(0, 1000));
    private MockVolumeService volumeServiceImpl = null;
    @Mock
    private StroomEntityManager stroomEntityManager;

    @Before
    public void init() {
        MockitoAnnotations.initMocks(this);
        deleteDefaultVolumesDir();

        final List<Volume> volumeList = new ArrayList<>();
        volumeList.add(public1a);
        volumeList.add(public1b);
        volumeList.add(public2a);
        volumeList.add(public2b);

        mockStroomPropertyService.setProperty(VolumeServiceImpl.PROP_RESILIENT_REPLICATION_COUNT, "2");

        volumeServiceImpl = new MockVolumeService(stroomEntityManager, new NodeCache(node1a), mockStroomPropertyService, null);
        volumeServiceImpl.volumeList = volumeList;
    }

    @Test
    public void testNode1aNodeWithCacheAndSomePuckerLocalStorage() {
        final Set<Volume> call1 = volumeServiceImpl.getStreamVolumeSet(node1a);
        final Set<Volume> call2 = volumeServiceImpl.getStreamVolumeSet(node1a);
        Assert.assertEquals(2, call1.size());
        Assert.assertEquals(2, call2.size());

        // Check that we only write once in a rack
        Assert.assertTrue(call1.contains(public1a) ^ call1.contains(public1b));
        Assert.assertTrue(call1.contains(public2a) ^ call1.contains(public2b));
        Assert.assertTrue(call2.contains(public1a) ^ call2.contains(public1b));
        Assert.assertTrue(call2.contains(public2a) ^ call2.contains(public2b));

        // Check that we round robin OK
        Assert.assertTrue(call1.contains(public2a) ^ call2.contains(public2a));
        Assert.assertTrue(call1.contains(public2b) ^ call2.contains(public2b));

    }

    @Test
    public void testNode1cNodeWithNoStorage() {
        final Set<Volume> call1 = volumeServiceImpl.getStreamVolumeSet(node1c);
        final Set<Volume> call2 = volumeServiceImpl.getStreamVolumeSet(node1c);
        Assert.assertEquals(2, call1.size());
        Assert.assertEquals(2, call2.size());

        // Check that we only write once in a rack
        Assert.assertTrue(call1.contains(public1a) ^ call1.contains(public1b));
        Assert.assertTrue(call1.contains(public2a) ^ call1.contains(public2b));
        Assert.assertTrue(call2.contains(public1a) ^ call2.contains(public1b));
        Assert.assertTrue(call2.contains(public2a) ^ call2.contains(public2b));

        // Check that we round robin OK
        Assert.assertTrue(call1.contains(public1a) ^ call2.contains(public1a));
        Assert.assertTrue(call1.contains(public1b) ^ call2.contains(public1b));
        Assert.assertTrue(call1.contains(public2a) ^ call2.contains(public2a));
        Assert.assertTrue(call1.contains(public2b) ^ call2.contains(public2b));
    }

    @Test
    public void testNode2aNodeWithNoCache() {
        final Set<Volume> call1 = volumeServiceImpl.getStreamVolumeSet(node2a);
        final Set<Volume> call2 = volumeServiceImpl.getStreamVolumeSet(node2a);
        Assert.assertEquals(2, call1.size());
        Assert.assertEquals(2, call2.size());

        // Check that we only write once in a rack
        Assert.assertTrue(call1.contains(public1a) ^ call1.contains(public1b));
        Assert.assertTrue(call1.contains(public2a) ^ call1.contains(public2b));
        Assert.assertTrue(call2.contains(public1a) ^ call2.contains(public1b));
        Assert.assertTrue(call2.contains(public2a) ^ call2.contains(public2b));

        // Check that we round robin OK on rack 1
        Assert.assertTrue(call1.contains(public1a) ^ call2.contains(public1a));
        Assert.assertTrue(call1.contains(public1b) ^ call2.contains(public1b));
    }

    @Test
    public void testStartup_Disabled() {
        mockStroomPropertyService.setProperty(VolumeServiceImpl.PROP_CREATE_DEFAULT_VOLUME_ON_STARTUP, "false");

        volumeServiceImpl.startup();

        Assert.assertFalse(volumeServiceImpl.saveCalled);
        Assert.assertFalse(Files.exists(DEFAULT_INDEX_VOLUME_PATH));
        Assert.assertFalse(Files.exists(DEFAULT_STREAM_VOLUME_PATH));
    }

    @Test
    public void testStartup_EnabledExistingVolumes() {
        mockStroomPropertyService.setProperty(VolumeServiceImpl.PROP_CREATE_DEFAULT_VOLUME_ON_STARTUP, "true");

        volumeServiceImpl.startup();

        Assert.assertFalse(volumeServiceImpl.saveCalled);
        Assert.assertFalse(Files.exists(DEFAULT_INDEX_VOLUME_PATH));
        Assert.assertFalse(Files.exists(DEFAULT_STREAM_VOLUME_PATH));
    }

    @Test
    public void testStartup_EnabledNoExistingVolumes() {
        mockStroomPropertyService.setProperty(VolumeServiceImpl.PROP_CREATE_DEFAULT_VOLUME_ON_STARTUP, "true");
        volumeServiceImpl.volumeList.clear();
        volumeServiceImpl.startup();

        Assert.assertTrue(volumeServiceImpl.saveCalled);
        //make sure both paths have been saved
        Assert.assertEquals(2, volumeServiceImpl.savedVolumes.stream()
                .map(Volume::getPath)
                .filter(path -> path.equals(FileUtil.getCanonicalPath(DEFAULT_INDEX_VOLUME_PATH)) ||
                        path.equals(FileUtil.getCanonicalPath(DEFAULT_STREAM_VOLUME_PATH)))
                .count());
        Assert.assertTrue(Files.exists(DEFAULT_INDEX_VOLUME_PATH));
        Assert.assertTrue(Files.exists(DEFAULT_STREAM_VOLUME_PATH));
    }

    private void deleteDefaultVolumesDir() {
        FileUtil.deleteDir(DEFAULT_INDEX_VOLUME_PATH);
        FileUtil.deleteDir(DEFAULT_STREAM_VOLUME_PATH);
        FileUtil.deleteDir(DEFAULT_VOLUMES_PATH);
    }

    private static class MockVolumeService extends VolumeServiceImpl {

        List<Volume> volumeList = null;
        boolean saveCalled;
        List<Volume> savedVolumes = new ArrayList<>();

        MockVolumeService(final StroomEntityManager stroomEntityManager,
                          final NodeCache nodeCache,
                          final StroomPropertyService stroomPropertyService,
                          final InternalStatisticsReceiver internalStatisticsReceiver) {
            super(stroomEntityManager, nodeCache, stroomPropertyService, internalStatisticsReceiver);
        }

        @Override
        public BaseResultList<Volume> find(final FindVolumeCriteria criteria) {
            return BaseResultList.createUnboundedList(volumeList);
        }

        @Override
        VolumeState saveVolumeState(final VolumeState volumeState) {
            return volumeState;
        }

        @Override
        public Volume save(Volume entity) throws RuntimeException {
            super.save(entity);
            saveCalled = true;
            savedVolumes.add(entity);
            return entity;
        }
    }
}

{'fraction_non_alphanumeric': 0.09500638946230218, 'fraction_numerical': 0.01744814705593237, 'mean_word_length': 3.854927225005965, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '143060', 'n_tokens_mistral': 5886, 'n_tokens_neox': 5787, 'n_words': 1081}