<commit_before>package org.ihtsdo.buildcloud.dao.s3;

import com.amazonaws.AmazonClientException;
import com.amazonaws.services.s3.model.*;

import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.StreamUtils;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;

public class OfflineS3ClientImplTest {
	
	private static final Logger LOGGER = LoggerFactory.getLogger(OfflineS3ClientImplTest.class);

	public static final String TEST_FILE_TXT = "testFile.txt";
	private S3Client s3Client;
	private List<InputStream> streamsToClose;

	private static final String TEST_BUCKET = "test-bucket";

	@Before
	public void setup() throws IOException {
		streamsToClose = new ArrayList<>();
		Path tempDirectory = Files.createTempDirectory(getClass().getName());
		Assert.assertTrue(new File(tempDirectory.toFile(), TEST_BUCKET).mkdirs());
		s3Client = new OfflineS3ClientImpl(tempDirectory.toFile());
		Assert.assertNotNull(getTestFileStream());
	}

	@Test
	public void testListObjects() {
		String buildDir = "builds/123/";
		Assert.assertEquals(0, s3Client.listObjects(TEST_BUCKET, buildDir).getObjectSummaries().size());

		s3Client.putObject(TEST_BUCKET, buildDir + "execA/file1.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "exec1/file1.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "exec1/file2.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "execZ/file1.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "exec2/file2.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "exec2/file1.txt", getTestFileStream(), null);

		List<S3ObjectSummary> objectSummaries = s3Client.listObjects(TEST_BUCKET, buildDir).getObjectSummaries();

		Assert.assertEquals(6, objectSummaries.size());
		Assert.assertEquals("builds/123/exec1/file1.txt", objectSummaries.get(0).getKey());
		Assert.assertEquals("builds/123/exec1/file2.txt", objectSummaries.get(1).getKey());
		Assert.assertEquals("builds/123/exec2/file1.txt", objectSummaries.get(2).getKey());
		Assert.assertEquals("builds/123/exec2/file2.txt", objectSummaries.get(3).getKey());
		Assert.assertEquals("builds/123/execA/file1.txt", objectSummaries.get(4).getKey());
		Assert.assertEquals("builds/123/execZ/file1.txt", objectSummaries.get(5).getKey());
	}

	@Test
	public void testListObjectsUsingListObjectsRequest() {
		String buildDir = "builds/123/";
		Assert.assertEquals(0, s3Client.listObjects(TEST_BUCKET, buildDir).getObjectSummaries().size());

		s3Client.putObject(TEST_BUCKET, buildDir + "execA/file1.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "exec1/file1.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "exec1/file2.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "execZ/file1.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "exec2/file2.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "exec2/file1.txt", getTestFileStream(), null);

		List<S3ObjectSummary> objectSummaries = s3Client.listObjects(new ListObjectsRequest(TEST_BUCKET, buildDir, null, null, null)).getObjectSummaries();

		Assert.assertEquals(6, objectSummaries.size());
		Assert.assertEquals("builds/123/exec1/file1.txt", objectSummaries.get(0).getKey());
		Assert.assertEquals("builds/123/exec1/file2.txt", objectSummaries.get(1).getKey());
		Assert.assertEquals("builds/123/exec2/file1.txt", objectSummaries.get(2).getKey());
		Assert.assertEquals("builds/123/exec2/file2.txt", objectSummaries.get(3).getKey());
		Assert.assertEquals("builds/123/execA/file1.txt", objectSummaries.get(4).getKey());
		Assert.assertEquals("builds/123/execZ/file1.txt", objectSummaries.get(5).getKey());
	}

	@Test
	public void testPutObjectGetObject() throws IOException {
		String buildDir = "builds/123/";
		String key = buildDir + "execA/file1.txt";

		s3Client.putObject(TEST_BUCKET, key, getTestFileStream(), null);

		List<S3ObjectSummary> objectSummaries = s3Client.listObjects(TEST_BUCKET, "").getObjectSummaries();
		Assert.assertEquals(1, objectSummaries.size());
		Assert.assertEquals("builds/123/execA/file1.txt", objectSummaries.get(0).getKey());

		S3Object s3Object = s3Client.getObject(TEST_BUCKET, key);
		Assert.assertEquals(TEST_BUCKET, s3Object.getBucketName());
		Assert.assertEquals(key, s3Object.getKey());
		S3ObjectInputStream objectContent = s3Object.getObjectContent();
		Assert.assertNotNull(objectContent);
		Assert.assertTrue(objectContent.available() > 0);
		String content = StreamUtils.copyToString(objectContent, Charset.defaultCharset());
		Assert.assertEquals("Some content\n", content);
	}

	@Test
	public void testPutObjectByFile() throws IOException {
		String buildDir = "builds/123/";
		String key = buildDir + "execA/file1.txt";

		s3Client.putObject(TEST_BUCKET, key, getTestFile());

		S3Object s3Object = s3Client.getObject(TEST_BUCKET, key);
		Assert.assertEquals(TEST_BUCKET, s3Object.getBucketName());
		Assert.assertEquals(key, s3Object.getKey());
		S3ObjectInputStream objectContent = s3Object.getObjectContent();
		Assert.assertNotNull(objectContent);
		Assert.assertTrue(objectContent.available() > 0);
		String content = StreamUtils.copyToString(objectContent, Charset.defaultCharset());
		Assert.assertEquals("Some content\n", content);
	}

	@Test
	public void testPutObjectByPutRequest() throws IOException {
		String buildDir = "builds/123/";
		String key = buildDir + "execA/file1.txt";

		ObjectMetadata metadata = new ObjectMetadata();
		PutObjectRequest putRequest = new PutObjectRequest(TEST_BUCKET, key, getTestFileStream(), metadata);
		metadata.setContentLength(13);

		s3Client.putObject(putRequest);

		S3Object s3Object = s3Client.getObject(TEST_BUCKET, key);
		Assert.assertEquals(TEST_BUCKET, s3Object.getBucketName());
		Assert.assertEquals(key, s3Object.getKey());
		S3ObjectInputStream objectContent = s3Object.getObjectContent();
		Assert.assertNotNull(objectContent);
		Assert.assertTrue(objectContent.available() > 0);
		String content = StreamUtils.copyToString(objectContent, Charset.defaultCharset());
		Assert.assertEquals("Some content\n", content);
	}

	@Test
	public void testPutObjectKeyCharacters() throws IOException {
		String buildDir = "builds/123/";
		String key = buildDir + "execA/file1_01:01:01.txt";

		s3Client.putObject(TEST_BUCKET, key, getTestFileStream(), null);

		List<S3ObjectSummary> objectSummaries = s3Client.listObjects(TEST_BUCKET, "").getObjectSummaries();
		Assert.assertEquals(1, objectSummaries.size());
		Assert.assertEquals("builds/123/execA/file1_01:01:01.txt", objectSummaries.get(0).getKey());
	}

	@Test
	public void testPutObjectNoInput() {
		try {
			s3Client.putObject(TEST_BUCKET, "123", null, null);
			Assert.fail("Should have thrown Exception");
		} catch (AmazonClientException e) {
			// pass
		}
	}

	@Test
	public void testDeleteNonExistentObject() {
		// should not throw an error
		s3Client.deleteObject(TEST_BUCKET, "file-does-not-exist.txt");
	}

	@Test
	public void testDeleteObject() throws IOException {
		String buildDir = "builds/123/";
		String key = buildDir + "execA/file1.txt";
		s3Client.putObject(TEST_BUCKET, key, getTestFileStream(), null);
		List<S3ObjectSummary> objectSummaries = s3Client.listObjects(TEST_BUCKET, "").getObjectSummaries();
		Assert.assertEquals(1, objectSummaries.size());
		Assert.assertEquals("builds/123/execA/file1.txt", objectSummaries.get(0).getKey());

		s3Client.deleteObject(TEST_BUCKET, key);

		Assert.assertEquals(0, s3Client.listObjects(TEST_BUCKET, "").getObjectSummaries().size());
	}<commit_msg>Removed S3 Offline special characters test as not multiplatform.<commit_after>package org.ihtsdo.buildcloud.dao.s3;

import com.amazonaws.AmazonClientException;
import com.amazonaws.services.s3.model.*;

import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.StreamUtils;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;

public class OfflineS3ClientImplTest {
	
	private static final Logger LOGGER = LoggerFactory.getLogger(OfflineS3ClientImplTest.class);

	public static final String TEST_FILE_TXT = "testFile.txt";
	private S3Client s3Client;
	private List<InputStream> streamsToClose;

	private static final String TEST_BUCKET = "test-bucket";

	@Before
	public void setup() throws IOException {
		streamsToClose = new ArrayList<>();
		Path tempDirectory = Files.createTempDirectory(getClass().getName());
		Assert.assertTrue(new File(tempDirectory.toFile(), TEST_BUCKET).mkdirs());
		s3Client = new OfflineS3ClientImpl(tempDirectory.toFile());
		Assert.assertNotNull(getTestFileStream());
	}

	@Test
	public void testListObjects() {
		String buildDir = "builds/123/";
		Assert.assertEquals(0, s3Client.listObjects(TEST_BUCKET, buildDir).getObjectSummaries().size());

		s3Client.putObject(TEST_BUCKET, buildDir + "execA/file1.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "exec1/file1.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "exec1/file2.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "execZ/file1.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "exec2/file2.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "exec2/file1.txt", getTestFileStream(), null);

		List<S3ObjectSummary> objectSummaries = s3Client.listObjects(TEST_BUCKET, buildDir).getObjectSummaries();

		Assert.assertEquals(6, objectSummaries.size());
		Assert.assertEquals("builds/123/exec1/file1.txt", objectSummaries.get(0).getKey());
		Assert.assertEquals("builds/123/exec1/file2.txt", objectSummaries.get(1).getKey());
		Assert.assertEquals("builds/123/exec2/file1.txt", objectSummaries.get(2).getKey());
		Assert.assertEquals("builds/123/exec2/file2.txt", objectSummaries.get(3).getKey());
		Assert.assertEquals("builds/123/execA/file1.txt", objectSummaries.get(4).getKey());
		Assert.assertEquals("builds/123/execZ/file1.txt", objectSummaries.get(5).getKey());
	}

	@Test
	public void testListObjectsUsingListObjectsRequest() {
		String buildDir = "builds/123/";
		Assert.assertEquals(0, s3Client.listObjects(TEST_BUCKET, buildDir).getObjectSummaries().size());

		s3Client.putObject(TEST_BUCKET, buildDir + "execA/file1.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "exec1/file1.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "exec1/file2.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "execZ/file1.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "exec2/file2.txt", getTestFileStream(), null);
		s3Client.putObject(TEST_BUCKET, buildDir + "exec2/file1.txt", getTestFileStream(), null);

		List<S3ObjectSummary> objectSummaries = s3Client.listObjects(new ListObjectsRequest(TEST_BUCKET, buildDir, null, null, null)).getObjectSummaries();

		Assert.assertEquals(6, objectSummaries.size());
		Assert.assertEquals("builds/123/exec1/file1.txt", objectSummaries.get(0).getKey());
		Assert.assertEquals("builds/123/exec1/file2.txt", objectSummaries.get(1).getKey());
		Assert.assertEquals("builds/123/exec2/file1.txt", objectSummaries.get(2).getKey());
		Assert.assertEquals("builds/123/exec2/file2.txt", objectSummaries.get(3).getKey());
		Assert.assertEquals("builds/123/execA/file1.txt", objectSummaries.get(4).getKey());
		Assert.assertEquals("builds/123/execZ/file1.txt", objectSummaries.get(5).getKey());
	}

	@Test
	public void testPutObjectGetObject() throws IOException {
		String buildDir = "builds/123/";
		String key = buildDir + "execA/file1.txt";

		s3Client.putObject(TEST_BUCKET, key, getTestFileStream(), null);

		List<S3ObjectSummary> objectSummaries = s3Client.listObjects(TEST_BUCKET, "").getObjectSummaries();
		Assert.assertEquals(1, objectSummaries.size());
		Assert.assertEquals("builds/123/execA/file1.txt", objectSummaries.get(0).getKey());

		S3Object s3Object = s3Client.getObject(TEST_BUCKET, key);
		Assert.assertEquals(TEST_BUCKET, s3Object.getBucketName());
		Assert.assertEquals(key, s3Object.getKey());
		S3ObjectInputStream objectContent = s3Object.getObjectContent();
		Assert.assertNotNull(objectContent);
		Assert.assertTrue(objectContent.available() > 0);
		String content = StreamUtils.copyToString(objectContent, Charset.defaultCharset());
		Assert.assertEquals("Some content\n", content);
	}

	@Test
	public void testPutObjectByFile() throws IOException {
		String buildDir = "builds/123/";
		String key = buildDir + "execA/file1.txt";

		s3Client.putObject(TEST_BUCKET, key, getTestFile());

		S3Object s3Object = s3Client.getObject(TEST_BUCKET, key);
		Assert.assertEquals(TEST_BUCKET, s3Object.getBucketName());
		Assert.assertEquals(key, s3Object.getKey());
		S3ObjectInputStream objectContent = s3Object.getObjectContent();
		Assert.assertNotNull(objectContent);
		Assert.assertTrue(objectContent.available() > 0);
		String content = StreamUtils.copyToString(objectContent, Charset.defaultCharset());
		Assert.assertEquals("Some content\n", content);
	}

	@Test
	public void testPutObjectByPutRequest() throws IOException {
		String buildDir = "builds/123/";
		String key = buildDir + "execA/file1.txt";

		ObjectMetadata metadata = new ObjectMetadata();
		PutObjectRequest putRequest = new PutObjectRequest(TEST_BUCKET, key, getTestFileStream(), metadata);
		metadata.setContentLength(13);

		s3Client.putObject(putRequest);

		S3Object s3Object = s3Client.getObject(TEST_BUCKET, key);
		Assert.assertEquals(TEST_BUCKET, s3Object.getBucketName());
		Assert.assertEquals(key, s3Object.getKey());
		S3ObjectInputStream objectContent = s3Object.getObjectContent();
		Assert.assertNotNull(objectContent);
		Assert.assertTrue(objectContent.available() > 0);
		String content = StreamUtils.copyToString(objectContent, Charset.defaultCharset());
		Assert.assertEquals("Some content\n", content);
	}

	@Test
	public void testPutObjectNoInput() {
		try {
			s3Client.putObject(TEST_BUCKET, "123", null, null);
			Assert.fail("Should have thrown Exception");
		} catch (AmazonClientException e) {
			// pass
		}
	}

	@Test
	public void testDeleteNonExistentObject() {
		// should not throw an error
		s3Client.deleteObject(TEST_BUCKET, "file-does-not-exist.txt");
	}

	@Test
	public void testDeleteObject() throws IOException {
		String buildDir = "builds/123/";
		String key = buildDir + "execA/file1.txt";
		s3Client.putObject(TEST_BUCKET, key, getTestFileStream(), null);
		List<S3ObjectSummary> objectSummaries = s3Client.listObjects(TEST_BUCKET, "").getObjectSummaries();
		Assert.assertEquals(1, objectSummaries.size());
		Assert.assertEquals("builds/123/execA/file1.txt", objectSummaries.get(0).getKey());

		s3Client.deleteObject(TEST_BUCKET, key);

		Assert.assertEquals(0, s3Client.listObjects(TEST_BUCKET, "").getObjectSummaries().size());
	}
{'fraction_non_alphanumeric': 0.13750482811896486, 'fraction_numerical': 0.026908716364104544, 'mean_word_length': 6.41527446300716, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5155968', 'n_tokens_mistral': 5371, 'n_tokens_neox': 5131, 'n_words': 813}