<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.jackrabbit.oak.segment.standby;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeFalse;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Random;

import com.google.common.io.ByteStreams;
import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.CIHelper;
import org.apache.jackrabbit.oak.commons.junit.TemporaryPort;
import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync;
import org.apache.jackrabbit.oak.segment.standby.server.StandbyServerSync;
import org.apache.jackrabbit.oak.segment.test.proxy.NetworkErrorProxy;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public abstract class DataStoreTestBase extends TestBase {

    private static final Logger logger = LoggerFactory.getLogger(DataStoreTestBase.class);

    private static final long GB = 1024 * 1024 * 1024;

    @Rule
    public TemporaryPort serverPort = new TemporaryPort();

    @Rule
    public TemporaryPort proxyPort = new TemporaryPort();

    @Rule
    public TestName testName = new TestName();

    abstract FileStore getPrimary();

    abstract FileStore getSecondary();

    abstract boolean storesShouldBeDifferent();

    private InputStream newRandomInputStream(final long size, final int seed) {
        return new InputStream() {

            private final Random random = new Random(seed);

            private long count = 0;

            @Override
            public int read() throws IOException {
                if (count >= size) {
                    return -1;
                }
                count++;
                return Math.abs(random.nextInt());
            }

        };
    }

    protected byte[] addTestContent(NodeStore store, String child, int size)
        throws CommitFailedException, IOException {
        NodeBuilder builder = store.getRoot().builder();
        builder.child(child).setProperty("ts", System.currentTimeMillis());

        byte[] data = new byte[size];
        new Random().nextBytes(data);
        Blob blob = store.createBlob(new ByteArrayInputStream(data));

        builder.child(child).setProperty("testBlob", blob);

        store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
        return data;
    }

    private void addTestContentOnTheFly(NodeStore store, String child, long size, int seed) throws CommitFailedException, IOException {
        NodeBuilder builder = store.getRoot().builder();
        builder.child(child).setProperty("ts", System.currentTimeMillis());

        InputStream randomInputStream = newRandomInputStream(size, seed);
        Blob blob = store.createBlob(randomInputStream);

        builder.child(child).setProperty("testBlob", blob);

        store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
    }

    @Before
    public void before() {
        logger.info("Test begin: {}", testName.getMethodName());
    }

    @After
    public void after() {
        logger.info("Test end: {}", testName.getMethodName());
    }

    @Test
    public void testResilientSync() throws Exception {
        final int blobSize = 5 * MB;
        FileStore primary = getPrimary();
        FileStore secondary = getSecondary();

        NodeStore store = SegmentNodeStoreBuilders.builder(primary).build();
        byte[] data = addTestContent(store, "server", blobSize);

        // run 1: unsuccessful
        try (
            StandbyServerSync serverSync = new StandbyServerSync(serverPort.getPort(), primary, MB);
            StandbyClientSync cl = newStandbyClientSync(secondary, serverPort.getPort(), 4_000)
        ) {
            serverSync.start();
            // no persisted head on primary
            // sync shouldn't be successful, but shouldn't throw exception either,
            // timeout too low for TarMK flush thread to kick-in
            cl.run();
            assertNotEquals(primary.getHead(), secondary.getHead());
        }

        // run 2: successful
        try (
            StandbyServerSync serverSync = new StandbyServerSync(serverPort.getPort(), primary, MB);
            StandbyClientSync cl = newStandbyClientSync(secondary, serverPort.getPort(), 4_000)
        ) {
            serverSync.start();
            // this time persisted head will be available on primary
            // waited at least 4s + 4s > 5s (TarMK flush thread run frequency)
            cl.run();
            assertEquals(primary.getHead(), secondary.getHead());
        }

        assertTrue(primary.getStats().getApproximateSize() < MB);
        assertTrue(secondary.getStats().getApproximateSize() < MB);

        PropertyState ps = secondary.getHead().getChildNode("root")
            .getChildNode("server").getProperty("testBlob");
        assertNotNull(ps);
        assertEquals(Type.BINARY.tag(), ps.getType().tag());
        Blob b = ps.getValue(Type.BINARY);
        assertEquals(blobSize, b.length());
        byte[] testData = new byte[blobSize];
        try (
            InputStream blobInputStream = b.getNewStream()
        ) {
            ByteStreams.readFully(blobInputStream, testData);
            assertArrayEquals(data, testData);
        }
    }

    @Test
    public void testSync() throws Exception {
        final int blobSize = 5 * MB;
        FileStore primary = getPrimary();
        FileStore secondary = getSecondary();

        NodeStore store = SegmentNodeStoreBuilders.builder(primary).build();
        byte[] data = addTestContent(store, "server", blobSize);
        try (
            StandbyServerSync serverSync = new StandbyServerSync(serverPort.getPort(), primary, MB);
            StandbyClientSync cl = newStandbyClientSync(secondary, serverPort.getPort())
        ) {
            serverSync.start();
            primary.flush();
            cl.run();
            assertEquals(primary.getHead(), secondary.getHead());
        }

        assertTrue(primary.getStats().getApproximateSize() < MB);
        assertTrue(secondary.getStats().getApproximateSize() < MB);

        PropertyState ps = secondary.getHead().getChildNode("root")
            .getChildNode("server").getProperty("testBlob");
        assertNotNull(ps);
        assertEquals(Type.BINARY.tag(), ps.getType().tag());
        Blob b = ps.getValue(Type.BINARY);
        assertEquals(blobSize, b.length());
        byte[] testData = new byte[blobSize];
        try (
            InputStream blobInputStream = b.getNewStream()
        ) {
            ByteStreams.readFully(blobInputStream, testData);
            assertArrayEquals(data, testData);
        }
    }

    /*
     * See OAK-5902.
     */
    @Test
    public void testSyncBigBlob() throws Exception {
        assumeFalse(CIHelper.windows());
        
        final long blobSize = GB;
        final int seed = 13;

        FileStore primary = getPrimary();
        FileStore secondary = getSecondary();

        NodeStore store = SegmentNodeStoreBuilders.builder(primary).build();
        addTestContentOnTheFly(store, "server", blobSize, seed);

        try (
            StandbyServerSync serverSync = new StandbyServerSync(serverPort.getPort(), primary, 8 * MB);
            StandbyClientSync cl = newStandbyClientSync(secondary, serverPort.getPort(), 2 * 60 * 1000)
        ) {
            serverSync.start();
            primary.flush();
            cl.run();
            assertEquals(primary.getHead(), secondary.getHead());
        }

        assertTrue(primary.getStats().getApproximateSize() < MB);
        assertTrue(secondary.getStats().getApproximateSize() < MB);

        PropertyState ps = secondary.getHead().getChildNode("root")
            .getChildNode("server").getProperty("testBlob");
        assertNotNull(ps);
        assertEquals(Type.BINARY.tag(), ps.getType().tag());
        Blob b = ps.getValue(Type.BINARY);
        assertEquals(blobSize, b.length());<commit_msg>OAK-6829: testSyncBigBlob enabled on Windows again<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.jackrabbit.oak.segment.standby;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Random;

import com.google.common.io.ByteStreams;
import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.junit.TemporaryPort;
import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync;
import org.apache.jackrabbit.oak.segment.standby.server.StandbyServerSync;
import org.apache.jackrabbit.oak.segment.test.proxy.NetworkErrorProxy;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public abstract class DataStoreTestBase extends TestBase {

    private static final Logger logger = LoggerFactory.getLogger(DataStoreTestBase.class);

    private static final long GB = 1024 * 1024 * 1024;

    @Rule
    public TemporaryPort serverPort = new TemporaryPort();

    @Rule
    public TemporaryPort proxyPort = new TemporaryPort();

    @Rule
    public TestName testName = new TestName();

    abstract FileStore getPrimary();

    abstract FileStore getSecondary();

    abstract boolean storesShouldBeDifferent();

    private InputStream newRandomInputStream(final long size, final int seed) {
        return new InputStream() {

            private final Random random = new Random(seed);

            private long count = 0;

            @Override
            public int read() throws IOException {
                if (count >= size) {
                    return -1;
                }
                count++;
                return Math.abs(random.nextInt());
            }

        };
    }

    protected byte[] addTestContent(NodeStore store, String child, int size)
        throws CommitFailedException, IOException {
        NodeBuilder builder = store.getRoot().builder();
        builder.child(child).setProperty("ts", System.currentTimeMillis());

        byte[] data = new byte[size];
        new Random().nextBytes(data);
        Blob blob = store.createBlob(new ByteArrayInputStream(data));

        builder.child(child).setProperty("testBlob", blob);

        store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
        return data;
    }

    private void addTestContentOnTheFly(NodeStore store, String child, long size, int seed) throws CommitFailedException, IOException {
        NodeBuilder builder = store.getRoot().builder();
        builder.child(child).setProperty("ts", System.currentTimeMillis());

        InputStream randomInputStream = newRandomInputStream(size, seed);
        Blob blob = store.createBlob(randomInputStream);

        builder.child(child).setProperty("testBlob", blob);

        store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
    }

    @Before
    public void before() {
        logger.info("Test begin: {}", testName.getMethodName());
    }

    @After
    public void after() {
        logger.info("Test end: {}", testName.getMethodName());
    }

    @Test
    public void testResilientSync() throws Exception {
        final int blobSize = 5 * MB;
        FileStore primary = getPrimary();
        FileStore secondary = getSecondary();

        NodeStore store = SegmentNodeStoreBuilders.builder(primary).build();
        byte[] data = addTestContent(store, "server", blobSize);

        // run 1: unsuccessful
        try (
            StandbyServerSync serverSync = new StandbyServerSync(serverPort.getPort(), primary, MB);
            StandbyClientSync cl = newStandbyClientSync(secondary, serverPort.getPort(), 4_000)
        ) {
            serverSync.start();
            // no persisted head on primary
            // sync shouldn't be successful, but shouldn't throw exception either,
            // timeout too low for TarMK flush thread to kick-in
            cl.run();
            assertNotEquals(primary.getHead(), secondary.getHead());
        }

        // run 2: successful
        try (
            StandbyServerSync serverSync = new StandbyServerSync(serverPort.getPort(), primary, MB);
            StandbyClientSync cl = newStandbyClientSync(secondary, serverPort.getPort(), 4_000)
        ) {
            serverSync.start();
            // this time persisted head will be available on primary
            // waited at least 4s + 4s > 5s (TarMK flush thread run frequency)
            cl.run();
            assertEquals(primary.getHead(), secondary.getHead());
        }

        assertTrue(primary.getStats().getApproximateSize() < MB);
        assertTrue(secondary.getStats().getApproximateSize() < MB);

        PropertyState ps = secondary.getHead().getChildNode("root")
            .getChildNode("server").getProperty("testBlob");
        assertNotNull(ps);
        assertEquals(Type.BINARY.tag(), ps.getType().tag());
        Blob b = ps.getValue(Type.BINARY);
        assertEquals(blobSize, b.length());
        byte[] testData = new byte[blobSize];
        try (
            InputStream blobInputStream = b.getNewStream()
        ) {
            ByteStreams.readFully(blobInputStream, testData);
            assertArrayEquals(data, testData);
        }
    }

    @Test
    public void testSync() throws Exception {
        final int blobSize = 5 * MB;
        FileStore primary = getPrimary();
        FileStore secondary = getSecondary();

        NodeStore store = SegmentNodeStoreBuilders.builder(primary).build();
        byte[] data = addTestContent(store, "server", blobSize);
        try (
            StandbyServerSync serverSync = new StandbyServerSync(serverPort.getPort(), primary, MB);
            StandbyClientSync cl = newStandbyClientSync(secondary, serverPort.getPort())
        ) {
            serverSync.start();
            primary.flush();
            cl.run();
            assertEquals(primary.getHead(), secondary.getHead());
        }

        assertTrue(primary.getStats().getApproximateSize() < MB);
        assertTrue(secondary.getStats().getApproximateSize() < MB);

        PropertyState ps = secondary.getHead().getChildNode("root")
            .getChildNode("server").getProperty("testBlob");
        assertNotNull(ps);
        assertEquals(Type.BINARY.tag(), ps.getType().tag());
        Blob b = ps.getValue(Type.BINARY);
        assertEquals(blobSize, b.length());
        byte[] testData = new byte[blobSize];
        try (
            InputStream blobInputStream = b.getNewStream()
        ) {
            ByteStreams.readFully(blobInputStream, testData);
            assertArrayEquals(data, testData);
        }
    }

    /*
     * See OAK-5902.
     */
    @Test
    public void testSyncBigBlob() throws Exception {
        final long blobSize = GB;
        final int seed = 13;

        FileStore primary = getPrimary();
        FileStore secondary = getSecondary();

        NodeStore store = SegmentNodeStoreBuilders.builder(primary).build();
        addTestContentOnTheFly(store, "server", blobSize, seed);

        try (
            StandbyServerSync serverSync = new StandbyServerSync(serverPort.getPort(), primary, 8 * MB);
            StandbyClientSync cl = newStandbyClientSync(secondary, serverPort.getPort(), 2 * 60 * 1000)
        ) {
            serverSync.start();
            primary.flush();
            cl.run();
            assertEquals(primary.getHead(), secondary.getHead());
        }

        assertTrue(primary.getStats().getApproximateSize() < MB);
        assertTrue(secondary.getStats().getApproximateSize() < MB);

        PropertyState ps = secondary.getHead().getChildNode("root")
            .getChildNode("server").getProperty("testBlob");
        assertNotNull(ps);
        assertEquals(Type.BINARY.tag(), ps.getType().tag());
        Blob b = ps.getValue(Type.BINARY);
        assertEquals(blobSize, b.length());
{'fraction_non_alphanumeric': 0.10650083612040134, 'fraction_numerical': 0.005330267558528428, 'mean_word_length': 2.892007321537523, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3025940', 'n_tokens_mistral': 5200, 'n_tokens_neox': 4990, 'n_words': 1238}