<commit_before>/*
 * Copyright 2015 herd contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.finra.herd.dao.impl;

import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;

import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

import com.amazonaws.AmazonServiceException;
import com.amazonaws.retry.PredefinedRetryPolicies;
import com.amazonaws.retry.RetryPolicy;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.Headers;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.GetObjectTaggingRequest;
import com.amazonaws.services.s3.model.GetObjectTaggingResult;
import com.amazonaws.services.s3.model.ListVersionsRequest;
import com.amazonaws.services.s3.model.MultiObjectDeleteException;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.RestoreObjectRequest;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.model.S3VersionSummary;
import com.amazonaws.services.s3.model.SetObjectTaggingRequest;
import com.amazonaws.services.s3.model.SetObjectTaggingResult;
import com.amazonaws.services.s3.model.StorageClass;
import com.amazonaws.services.s3.model.Tag;
import com.amazonaws.services.s3.model.Tier;
import com.amazonaws.services.s3.model.VersionListing;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpStatus;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;

import org.finra.herd.core.helper.LogLevel;
import org.finra.herd.dao.AbstractDaoTest;
import org.finra.herd.dao.AwsClientFactory;
import org.finra.herd.dao.S3Operations;
import org.finra.herd.dao.helper.AwsHelper;
import org.finra.herd.dao.helper.JavaPropertiesHelper;
import org.finra.herd.model.dto.S3FileTransferRequestParamsDto;

/**
 * This class tests functionality within the S3 DAO implementation.
 */
public class S3DaoImplTest extends AbstractDaoTest
{

    private static final String OTHER_EXCEPTION_MESSAGE = "OtherExceptionMessage";

    private static final String RESTORE_ALREADY_IN_PROGRESS_EXCEPTION_MESSAGE = "RestoreAlreadyInProgress";

    private static final String TEST_FILE = "UT_S3DaoImplTest_Test_File";

    @Mock
    private AwsHelper awsHelper;

    @Mock
    private JavaPropertiesHelper javaPropertiesHelper;

    @InjectMocks
    private S3DaoImpl s3DaoImpl;

    @Mock
    private S3Operations s3Operations;

    @Mock
    private AmazonS3Client s3client;

    @Mock
    private AwsClientFactory awsClientFactory;

    @Before
    public void before()
    {
        MockitoAnnotations.initMocks(this);
    }

    @Test
    public void testDeleteDirectoryMultiObjectDeleteException()
    {
        // Create an S3 file transfer request parameters DTO to access S3 objects.
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
        s3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX);

        // Create a retry policy.
        RetryPolicy retryPolicy =
            new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true);

        // Create an S3 version summary.
        S3VersionSummary s3VersionSummary = new S3VersionSummary();
        s3VersionSummary.setKey(S3_KEY);
        s3VersionSummary.setVersionId(S3_VERSION_ID);

        // Create a version listing.
        VersionListing versionListing = new VersionListing();
        versionListing.setVersionSummaries(Collections.singletonList(s3VersionSummary));

        // Create a delete error.
        MultiObjectDeleteException.DeleteError deleteError = new MultiObjectDeleteException.DeleteError();
        deleteError.setKey(S3_KEY);
        deleteError.setVersionId(S3_VERSION_ID);
        deleteError.setCode(ERROR_CODE);
        deleteError.setMessage(ERROR_MESSAGE);

        // Create a multi object delete exception.
        MultiObjectDeleteException multiObjectDeleteException = new MultiObjectDeleteException(Collections.singletonList(deleteError), new ArrayList<>());

        // Mock the external calls.
        when(awsClientFactory.getAmazonS3Client(any(S3FileTransferRequestParamsDto.class))).thenReturn(s3client);
        when(s3Operations.listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class))).thenReturn(versionListing);
        when(s3Operations.deleteObjects(any(DeleteObjectsRequest.class), any(AmazonS3Client.class))).thenThrow(multiObjectDeleteException);

        // Try to call the method under test.
        try
        {
            s3DaoImpl.deleteDirectory(s3FileTransferRequestParamsDto);
        }
        catch (IllegalStateException e)
        {
            assertEquals(String.format(
                "Failed to delete keys/key versions with prefix \"%s\" from bucket \"%s\". Reason: One or more objects could not be deleted " +
                    "(Service: null; Status Code: 0; Error Code: null; Request ID: null; S3 Extended Request ID: null; Proxy: null)", S3_KEY_PREFIX,
                S3_BUCKET_NAME), e.getMessage());
        }

        // Verify the external calls.
        //verify()
        //verify(retryPolicyFactory, times(2)).getRetryPolicy();
        verify(awsClientFactory, times(2)).getAmazonS3Client(any(S3FileTransferRequestParamsDto.class));
        verify(s3Operations).listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class));
        verify(s3Operations).deleteObjects(any(DeleteObjectsRequest.class), any(AmazonS3Client.class));
        verifyNoMoreInteractionsHelper();
    }

    @Test
    public void testRestoreObjectsInDeepArchiveWithExpeditedArchiveRetrievalOption()
    {
        List<File> files = Collections.singletonList(new File(TEST_FILE));

        // Create an S3 file transfer request parameters DTO to access S3 objects.
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
        s3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX);
        s3FileTransferRequestParamsDto.setFiles(files);

        // Create a retry policy.
        RetryPolicy retryPolicy =
            new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true);

        // Create an Object Metadata with DeepArchive storage class.
        ObjectMetadata objectMetadata = new ObjectMetadata();
        objectMetadata.setOngoingRestore(false);
        objectMetadata.setHeader(Headers.STORAGE_CLASS, StorageClass.DeepArchive);

        ArgumentCaptor<AmazonS3Client> s3ClientCaptor = ArgumentCaptor.forClass(AmazonS3Client.class);
        ArgumentCaptor<String> s3BucketNameCaptor = ArgumentCaptor.forClass(String.class);
        ArgumentCaptor<String> keyCaptor = ArgumentCaptor.forClass(String.class);<commit_msg>DM-15154: Enhancement - Add batch mode of operation to BData Restore endpoint<commit_after>/*
 * Copyright 2015 herd contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.finra.herd.dao.impl;

import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;

import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

import com.amazonaws.AmazonServiceException;
import com.amazonaws.retry.PredefinedRetryPolicies;
import com.amazonaws.retry.RetryPolicy;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.Headers;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.GetObjectTaggingRequest;
import com.amazonaws.services.s3.model.GetObjectTaggingResult;
import com.amazonaws.services.s3.model.ListVersionsRequest;
import com.amazonaws.services.s3.model.MultiObjectDeleteException;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.RestoreObjectRequest;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.model.S3VersionSummary;
import com.amazonaws.services.s3.model.SetObjectTaggingRequest;
import com.amazonaws.services.s3.model.SetObjectTaggingResult;
import com.amazonaws.services.s3.model.StorageClass;
import com.amazonaws.services.s3.model.Tag;
import com.amazonaws.services.s3.model.Tier;
import com.amazonaws.services.s3.model.VersionListing;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpStatus;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;

import org.finra.herd.core.helper.LogLevel;
import org.finra.herd.dao.AbstractDaoTest;
import org.finra.herd.dao.AwsClientFactory;
import org.finra.herd.dao.S3Operations;
import org.finra.herd.dao.helper.AwsHelper;
import org.finra.herd.dao.helper.JavaPropertiesHelper;
import org.finra.herd.model.dto.S3FileTransferRequestParamsDto;

/**
 * This class tests functionality within the S3 DAO implementation.
 */
public class S3DaoImplTest extends AbstractDaoTest
{

    private static final String OTHER_EXCEPTION_MESSAGE = "OtherExceptionMessage";

    private static final String RESTORE_ALREADY_IN_PROGRESS_EXCEPTION_MESSAGE = "RestoreAlreadyInProgress";

    private static final String TEST_FILE = "UT_S3DaoImplTest_Test_File";

    @Mock
    private AwsHelper awsHelper;

    @Mock
    private JavaPropertiesHelper javaPropertiesHelper;

    @InjectMocks
    private S3DaoImpl s3DaoImpl;

    @Mock
    private S3Operations s3Operations;

    @Mock
    private AmazonS3Client s3client;

    @Mock
    private AwsClientFactory awsClientFactory;

    @Before
    public void before()
    {
        MockitoAnnotations.initMocks(this);
    }

    @Test
    public void testDeleteDirectoryMultiObjectDeleteException()
    {
        // Create an S3 file transfer request parameters DTO to access S3 objects.
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
        s3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX);

        // Create a retry policy.
        RetryPolicy retryPolicy =
            new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true);

        // Create an S3 version summary.
        S3VersionSummary s3VersionSummary = new S3VersionSummary();
        s3VersionSummary.setKey(S3_KEY);
        s3VersionSummary.setVersionId(S3_VERSION_ID);

        // Create a version listing.
        VersionListing versionListing = new VersionListing();
        versionListing.setVersionSummaries(Collections.singletonList(s3VersionSummary));

        // Create a delete error.
        MultiObjectDeleteException.DeleteError deleteError = new MultiObjectDeleteException.DeleteError();
        deleteError.setKey(S3_KEY);
        deleteError.setVersionId(S3_VERSION_ID);
        deleteError.setCode(ERROR_CODE);
        deleteError.setMessage(ERROR_MESSAGE);

        // Create a multi object delete exception.
        MultiObjectDeleteException multiObjectDeleteException = new MultiObjectDeleteException(Collections.singletonList(deleteError), new ArrayList<>());

        // Mock the external calls.
        when(awsClientFactory.getAmazonS3Client(any(S3FileTransferRequestParamsDto.class))).thenReturn(s3client);
        when(s3Operations.listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class))).thenReturn(versionListing);
        when(s3Operations.deleteObjects(any(DeleteObjectsRequest.class), any(AmazonS3Client.class))).thenThrow(multiObjectDeleteException);

        // Try to call the method under test.
        try
        {
            s3DaoImpl.deleteDirectory(s3FileTransferRequestParamsDto);
        }
        catch (IllegalStateException e)
        {
            assertEquals(String.format(
                "Failed to delete keys/key versions with prefix \"%s\" from bucket \"%s\". Reason: One or more objects could not be deleted " +
                    "(Service: null; Status Code: 0; Error Code: null; Request ID: null; S3 Extended Request ID: null; Proxy: null)", S3_KEY_PREFIX,
                S3_BUCKET_NAME), e.getMessage());
        }

        // Verify the external calls.
        verify(awsClientFactory, times(2)).getAmazonS3Client(any(S3FileTransferRequestParamsDto.class));
        verify(s3Operations).listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class));
        verify(s3Operations).deleteObjects(any(DeleteObjectsRequest.class), any(AmazonS3Client.class));
        verifyNoMoreInteractionsHelper();
    }

    @Test
    public void testRestoreObjectsInDeepArchiveWithExpeditedArchiveRetrievalOption()
    {
        List<File> files = Collections.singletonList(new File(TEST_FILE));

        // Create an S3 file transfer request parameters DTO to access S3 objects.
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
        s3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX);
        s3FileTransferRequestParamsDto.setFiles(files);

        // Create a retry policy.
        RetryPolicy retryPolicy =
            new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true);

        // Create an Object Metadata with DeepArchive storage class.
        ObjectMetadata objectMetadata = new ObjectMetadata();
        objectMetadata.setOngoingRestore(false);
        objectMetadata.setHeader(Headers.STORAGE_CLASS, StorageClass.DeepArchive);

        ArgumentCaptor<AmazonS3Client> s3ClientCaptor = ArgumentCaptor.forClass(AmazonS3Client.class);
        ArgumentCaptor<String> s3BucketNameCaptor = ArgumentCaptor.forClass(String.class);
        ArgumentCaptor<String> keyCaptor = ArgumentCaptor.forClass(String.class);
{'fraction_non_alphanumeric': 0.08226378193374893, 'fraction_numerical': 0.012834616795012835, 'mean_word_length': 4.350882930019621, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2279127', 'n_tokens_mistral': 4680, 'n_tokens_neox': 4702, 'n_words': 954}