<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.jackrabbit.oak.plugins.blob.datastore;

import java.io.ByteArrayInputStream;
import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ScheduledFuture;

import ch.qos.logback.classic.Level;
import com.google.common.base.StandardSystemProperty;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.commons.FileIOUtils;
import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
import org.apache.jackrabbit.oak.plugins.blob.BlobTrackingStore;
import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector;
import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore;
import org.apache.jackrabbit.oak.plugins.document.DocumentBlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.document.DocumentMKBuilderProvider;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
import org.apache.jackrabbit.oak.plugins.document.TestUtils;
import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector;
import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
import org.apache.jackrabbit.oak.spi.cluster.ClusterRepositoryInfo;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.stats.Clock;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Sets.newHashSet;
import static com.google.common.collect.Sets.union;
import static java.lang.String.valueOf;
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static org.apache.commons.io.FileUtils.forceDelete;
import static org.apache.commons.io.IOUtils.closeQuietly;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.readStringsAsSet;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.writeStrings;
import static org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils.createFDS;
import static org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils.getBlobStore;
import static org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils
    .SharedStoreRecordType.REPOSITORY;
import static org.apache.jackrabbit.oak.plugins.document.Revision.getCurrentTimestamp;
import static org.apache.jackrabbit.oak.spi.commit.CommitInfo.EMPTY;
import static org.apache.jackrabbit.oak.spi.commit.EmptyHook.INSTANCE;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeNoException;
import static org.junit.Assume.assumeThat;

/**
 */
public class DataStoreTrackerGCTest {
    private Clock clock;
    private File blobStoreRoot;

    @Rule
    public TemporaryFolder folder = new TemporaryFolder(new File("target"));

    @Rule
    public DocumentMKBuilderProvider builderProvider = new DocumentMKBuilderProvider();

    @BeforeClass
    public static void check() {
        File root = new File("./target/blobstore");
        try {
            BlobStore store = getBlobStore(root);
            assumeThat(store, instanceOf(BlobTrackingStore.class));
        } catch (Exception e) {
            assumeNoException(e);
        } finally {
            try {
                forceDelete(root);
            } catch (IOException e) {
            }
        }
    }

    @Before
    public void setup() throws IOException, InterruptedException {
        this.clock = getTestClock();
        TestUtils.setRevisionClock(clock);
        this.blobStoreRoot = folder.newFolder("blobstore");
    }

    @AfterClass
    public static void resetClock() {
        TestUtils.resetRevisionClockToDefault();
    }

    @Test
    public void gc() throws Exception {
        Cluster cluster = new Cluster("cluster1");
        BlobStore s = cluster.blobStore;
        BlobIdTracker tracker = (BlobIdTracker) ((BlobTrackingStore) s).getTracker();

        DataStoreState state = init(cluster.nodeStore, 0);
        ScheduledFuture<?> scheduledFuture = newSingleThreadScheduledExecutor()
            .schedule(tracker.new SnapshotJob(), 0, MILLISECONDS);
        scheduledFuture.get();
        // All blobs added should be tracked now
        assertEquals(state.blobsAdded, retrieveTracked(tracker));

        cluster.gc.collectGarbage(false);
        Set<String> existingAfterGC = iterate(s);
        // Check the state of the blob store after gc
        assertEquals(state.blobsPresent, existingAfterGC);
        // Tracked blobs should reflect deletions after gc
        assertEquals(state.blobsPresent, retrieveTracked(tracker));
    }

    @Test
    public void gcReconcileActiveDeletion() throws Exception {
        Cluster cluster = new Cluster("cluster1");
        BlobStore s = cluster.blobStore;
        BlobIdTracker tracker = (BlobIdTracker) ((BlobTrackingStore) s).getTracker();
        DataStoreState state = init(cluster.nodeStore, 0);

        // Simulate creation and active deletion after init without version gc to enable references to hang around
        List<String> addlAdded = doActiveDelete(cluster.nodeStore,
            (DataStoreBlobStore) cluster.blobStore, tracker, folder,0, 2);
        List<String> addlPresent = Lists.newArrayList(addlAdded.get(2), addlAdded.get(3));
        List<String> activeDeleted = Lists.newArrayList(addlAdded.get(0), addlAdded.get(1));
        state.blobsPresent.addAll(addlPresent);
        state.blobsAdded.addAll(addlPresent);

        cluster.gc.collectGarbage(false);

        Set<String> existingAfterGC = iterate(s);
        // Check the state of the blob store after gc
        assertEquals(state.blobsPresent, existingAfterGC);
        // Tracked blobs should reflect deletions after gc
        assertEquals(state.blobsPresent, retrieveTracked(tracker));
        // Check that the delete tracker is refreshed
        assertEquals(Sets.newHashSet(activeDeleted), retrieveActiveDeleteTracked(tracker, folder));
    }

    @Test
    public void gcReconcileActiveDeletionMarkCleared() throws Exception {
        Cluster cluster = new Cluster("cluster1");
        BlobStore s = cluster.blobStore;
        BlobIdTracker tracker = (BlobIdTracker) ((BlobTrackingStore) s).getTracker();
        // Simulate active deletion before the init to ensure that the references also cleared
        List<String> addlAdded = doActiveDelete(cluster.nodeStore,
            (DataStoreBlobStore) cluster.blobStore, tracker, folder,0, 2);
        DataStoreState state = init(cluster.nodeStore, 0);

        // Force a snapshot of the tracker to refresh
        File f = folder.newFile();
        tracker.remove(f, BlobTracker.Options.ACTIVE_DELETION);

        List<String> addlPresent = Lists.newArrayList(addlAdded.get(2), addlAdded.get(3));
        List<String> activeDeleted = Lists.newArrayList(addlAdded.get(0), addlAdded.get(1));
        state.blobsPresent.addAll(addlPresent);
        state.blobsAdded.addAll(addlPresent);

        cluster.gc.collectGarbage(false);
        Set<String> existingAfterGC = iterate(s);
        // Check the state of the blob store after gc
        assertEquals(state.blobsPresent, existingAfterGC);
        // Tracked blobs should reflect deletions after gc
        assertEquals(state.blobsPresent, retrieveTracked(tracker));
        // Check that the delete tracker is refreshed
        assertEquals(Sets.newHashSet(), retrieveActiveDeleteTracked(tracker, folder));
    }

    @Test
    public void consistencyCheckNoActiveDeletion() throws Exception {
        File tmpFolder = folder.newFolder();
        System.setProperty(StandardSystemProperty.JAVA_IO_TMPDIR.key(), tmpFolder.getAbsolutePath());

        try {
            Cluster cluster = new Cluster("cluster1");
            BlobStore s = cluster.blobStore;
            BlobIdTracker tracker = (BlobIdTracker) ((BlobTrackingStore) s).getTracker();
            DataStoreState state = init(cluster.nodeStore, 0);

            // Since datastore in consistent state and only active deletions the missing list should be empty
            assertEquals(0, cluster.gc.checkConsistency());
            assertTrue(FileUtils.listFiles(tmpFolder, null, true).size() == 0);
        } finally {
            System.clearProperty(StandardSystemProperty.JAVA_IO_TMPDIR.key());
        }
    }
<commit_msg>OAK-7093: ActiveDelete synchronization with BlobTracker leaves temp files<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.jackrabbit.oak.plugins.blob.datastore;

import java.io.ByteArrayInputStream;
import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ScheduledFuture;

import ch.qos.logback.classic.Level;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
import org.apache.jackrabbit.oak.plugins.blob.BlobTrackingStore;
import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector;
import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore;
import org.apache.jackrabbit.oak.plugins.document.DocumentBlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.document.DocumentMKBuilderProvider;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
import org.apache.jackrabbit.oak.plugins.document.TestUtils;
import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector;
import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
import org.apache.jackrabbit.oak.spi.cluster.ClusterRepositoryInfo;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.stats.Clock;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

import static com.google.common.base.StandardSystemProperty.JAVA_IO_TMPDIR;
import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Sets.newHashSet;
import static com.google.common.collect.Sets.union;
import static java.lang.String.valueOf;
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static org.apache.commons.io.FileUtils.forceDelete;
import static org.apache.commons.io.IOUtils.closeQuietly;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.readStringsAsSet;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.writeStrings;
import static org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils.createFDS;
import static org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils.getBlobStore;
import static org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils
    .SharedStoreRecordType.REPOSITORY;
import static org.apache.jackrabbit.oak.plugins.document.Revision.getCurrentTimestamp;
import static org.apache.jackrabbit.oak.spi.commit.CommitInfo.EMPTY;
import static org.apache.jackrabbit.oak.spi.commit.EmptyHook.INSTANCE;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeNoException;
import static org.junit.Assume.assumeThat;

/**
 */
public class DataStoreTrackerGCTest {
    private Clock clock;
    private File blobStoreRoot;

    @Rule
    public TemporaryFolder folder = new TemporaryFolder(new File("target"));

    @Rule
    public DocumentMKBuilderProvider builderProvider = new DocumentMKBuilderProvider();

    @BeforeClass
    public static void check() {
        File root = new File("./target/blobstore");
        try {
            BlobStore store = getBlobStore(root);
            assumeThat(store, instanceOf(BlobTrackingStore.class));
        } catch (Exception e) {
            assumeNoException(e);
        } finally {
            try {
                forceDelete(root);
            } catch (IOException e) {
            }
        }
    }

    @Before
    public void setup() throws IOException, InterruptedException {
        this.clock = getTestClock();
        TestUtils.setRevisionClock(clock);
        this.blobStoreRoot = folder.newFolder("blobstore");
    }

    @AfterClass
    public static void resetClock() {
        TestUtils.resetRevisionClockToDefault();
    }

    @Test
    public void gc() throws Exception {
        Cluster cluster = new Cluster("cluster1");
        BlobStore s = cluster.blobStore;
        BlobIdTracker tracker = (BlobIdTracker) ((BlobTrackingStore) s).getTracker();

        DataStoreState state = init(cluster.nodeStore, 0);
        ScheduledFuture<?> scheduledFuture = newSingleThreadScheduledExecutor()
            .schedule(tracker.new SnapshotJob(), 0, MILLISECONDS);
        scheduledFuture.get();
        // All blobs added should be tracked now
        assertEquals(state.blobsAdded, retrieveTracked(tracker));

        cluster.gc.collectGarbage(false);
        Set<String> existingAfterGC = iterate(s);
        // Check the state of the blob store after gc
        assertEquals(state.blobsPresent, existingAfterGC);
        // Tracked blobs should reflect deletions after gc
        assertEquals(state.blobsPresent, retrieveTracked(tracker));
    }

    @Test
    public void gcReconcileActiveDeletion() throws Exception {
        Cluster cluster = new Cluster("cluster1");
        BlobStore s = cluster.blobStore;
        BlobIdTracker tracker = (BlobIdTracker) ((BlobTrackingStore) s).getTracker();
        DataStoreState state = init(cluster.nodeStore, 0);

        // Simulate creation and active deletion after init without version gc to enable references to hang around
        List<String> addlAdded = doActiveDelete(cluster.nodeStore,
            (DataStoreBlobStore) cluster.blobStore, tracker, folder,0, 2);
        List<String> addlPresent = Lists.newArrayList(addlAdded.get(2), addlAdded.get(3));
        List<String> activeDeleted = Lists.newArrayList(addlAdded.get(0), addlAdded.get(1));
        state.blobsPresent.addAll(addlPresent);
        state.blobsAdded.addAll(addlPresent);

        cluster.gc.collectGarbage(false);

        Set<String> existingAfterGC = iterate(s);
        // Check the state of the blob store after gc
        assertEquals(state.blobsPresent, existingAfterGC);
        // Tracked blobs should reflect deletions after gc
        assertEquals(state.blobsPresent, retrieveTracked(tracker));
        // Check that the delete tracker is refreshed
        assertEquals(Sets.newHashSet(activeDeleted), retrieveActiveDeleteTracked(tracker, folder));
    }

    @Test
    public void gcReconcileActiveDeletionMarkCleared() throws Exception {
        Cluster cluster = new Cluster("cluster1");
        BlobStore s = cluster.blobStore;
        BlobIdTracker tracker = (BlobIdTracker) ((BlobTrackingStore) s).getTracker();
        // Simulate active deletion before the init to ensure that the references also cleared
        List<String> addlAdded = doActiveDelete(cluster.nodeStore,
            (DataStoreBlobStore) cluster.blobStore, tracker, folder,0, 2);
        DataStoreState state = init(cluster.nodeStore, 0);

        // Force a snapshot of the tracker to refresh
        File f = folder.newFile();
        tracker.remove(f, BlobTracker.Options.ACTIVE_DELETION);

        List<String> addlPresent = Lists.newArrayList(addlAdded.get(2), addlAdded.get(3));
        List<String> activeDeleted = Lists.newArrayList(addlAdded.get(0), addlAdded.get(1));
        state.blobsPresent.addAll(addlPresent);
        state.blobsAdded.addAll(addlPresent);

        cluster.gc.collectGarbage(false);
        Set<String> existingAfterGC = iterate(s);
        // Check the state of the blob store after gc
        assertEquals(state.blobsPresent, existingAfterGC);
        // Tracked blobs should reflect deletions after gc
        assertEquals(state.blobsPresent, retrieveTracked(tracker));
        // Check that the delete tracker is refreshed
        assertEquals(Sets.newHashSet(), retrieveActiveDeleteTracked(tracker, folder));
    }

    @Test
    public void consistencyCheckNoActiveDeletion() throws Exception {
        File tmpFolder = folder.newFolder();
        String previousTmp = System.setProperty(JAVA_IO_TMPDIR.key(), tmpFolder.getAbsolutePath());

        try {
            Cluster cluster = new Cluster("cluster1");
            BlobStore s = cluster.blobStore;
            BlobIdTracker tracker = (BlobIdTracker) ((BlobTrackingStore) s).getTracker();
            DataStoreState state = init(cluster.nodeStore, 0);

            // Since datastore in consistent state and only active deletions the missing list should be empty
            assertEquals(0, cluster.gc.checkConsistency());
            assertTrue(FileUtils.listFiles(tmpFolder, null, true).size() == 0);
        } finally {
            if (previousTmp != null) {
                System.setProperty(JAVA_IO_TMPDIR.key(), previousTmp);
            } else {
                System.clearProperty(JAVA_IO_TMPDIR.key());
            }
        }
    }

{'fraction_non_alphanumeric': 0.09428417495224568, 'fraction_numerical': 0.0028407699466131164, 'mean_word_length': 3.815566037735849, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4560669', 'n_tokens_mistral': 5816, 'n_tokens_neox': 5623, 'n_words': 1306}