<commit_before>package org.sagebionetworks;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;

import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.sagebionetworks.client.SynapseAdminClient;
import org.sagebionetworks.client.SynapseAdminClientImpl;
import org.sagebionetworks.client.SynapseClient;
import org.sagebionetworks.client.SynapseClientImpl;
import org.sagebionetworks.client.exceptions.SynapseClientException;
import org.sagebionetworks.client.exceptions.SynapseException;
import org.sagebionetworks.client.exceptions.SynapseNotFoundException;
import org.sagebionetworks.repo.manager.S3TestUtils;
import org.sagebionetworks.repo.model.EntityHeader;
import org.sagebionetworks.repo.model.Project;
import org.sagebionetworks.repo.model.file.ExternalFileHandle;
import org.sagebionetworks.repo.model.file.ExternalUploadDestination;
import org.sagebionetworks.repo.model.file.FileHandle;
import org.sagebionetworks.repo.model.file.FileHandleResults;
import org.sagebionetworks.repo.model.file.PreviewFileHandle;
import org.sagebionetworks.repo.model.file.S3FileHandle;
import org.sagebionetworks.repo.model.file.S3UploadDestination;
import org.sagebionetworks.repo.model.file.UploadDestination;
import org.sagebionetworks.repo.model.file.UploadDestinationLocation;
import org.sagebionetworks.repo.model.file.UploadType;
import org.sagebionetworks.repo.model.project.ExternalS3StorageLocationSetting;
import org.sagebionetworks.repo.model.project.ExternalStorageLocationSetting;
import org.sagebionetworks.repo.model.project.ExternalSyncSetting;
import org.sagebionetworks.repo.model.project.ProjectSetting;
import org.sagebionetworks.repo.model.project.ProjectSettingsType;
import org.sagebionetworks.repo.model.project.S3StorageLocationSetting;
import org.sagebionetworks.repo.model.project.StorageLocationSetting;
import org.sagebionetworks.repo.model.project.UploadDestinationListSetting;
import org.sagebionetworks.schema.adapter.JSONObjectAdapterException;
import org.sagebionetworks.util.TimedAssert;
import org.sagebionetworks.utils.MD5ChecksumHelper;

import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
import com.google.common.collect.Lists;

public class IT049FileHandleTest {

	private static SynapseAdminClient adminSynapse;
	private static SynapseClient synapse;
	private static Long userToDelete;
	private static AmazonS3Client s3Client;
	
	private static final String LARGE_FILE_PATH_PROP_KEY = "org.sagebionetworks.test.large.file.path";
	private static final long MAX_WAIT_MS = 1000*10; // 10 sec
	private static final String FILE_NAME = "LittleImage.png";

	private List<FileHandle> toDelete = null;
	private File imageFile;
	private Project project;

	@BeforeClass
	public static void beforeClass() throws Exception {
		// Create a user
		adminSynapse = new SynapseAdminClientImpl();
		SynapseClientHelper.setEndpoints(adminSynapse);
		adminSynapse.setUserName(StackConfiguration.getMigrationAdminUsername());
		adminSynapse.setApiKey(StackConfiguration.getMigrationAdminAPIKey());
		adminSynapse.clearAllLocks();
		
		synapse = new SynapseClientImpl();
		userToDelete = SynapseClientHelper.createUser(adminSynapse, synapse);
		s3Client = new AmazonS3Client(new BasicAWSCredentials(StackConfiguration.getIAMUserId(), StackConfiguration.getIAMUserKey()));
		s3Client.createBucket(StackConfiguration.singleton().getExternalS3TestBucketName());
	}
	
	@Before
	public void before() throws SynapseException {
		toDelete = new ArrayList<FileHandle>();
		// Get the image file from the classpath.
		URL url = IT049FileHandleTest.class.getClassLoader().getResource("images/"+FILE_NAME);
		imageFile = new File(url.getFile().replaceAll("%20", " "));
		project = new Project();
		project = synapse.createEntity(project);
	}

	@After
	public void after() throws Exception {
		for (FileHandle handle: toDelete) {
			try {
				synapse.deleteFileHandle(handle.getId());
			} catch (SynapseNotFoundException e) {
			} catch (SynapseClientException e) { }
		}
		synapse.deleteEntity(project, true);
		S3TestUtils.doDeleteAfter(s3Client);
	}
	
	@AfterClass
	public static void afterClass() throws Exception {
		try {
			adminSynapse.deleteUser(userToDelete);
		} catch (SynapseException e) { }
	}
	
	@Test
	public void testImageFileRoundTrip() throws SynapseException, IOException, InterruptedException{
		assertNotNull(imageFile);
		assertTrue(imageFile.exists());
		String expectedMD5 = MD5ChecksumHelper.getMD5Checksum(imageFile);
		// Create the image
		List<File> list = new LinkedList<File>();
		list.add(imageFile);
		FileHandleResults results = synapse.createFileHandles(list, project.getId());
		assertNotNull(results);
		// We should have one image on the list
		assertNotNull(results.getList());
		assertEquals(1, results.getList().size());
		S3FileHandle handle = (S3FileHandle) results.getList().get(0);
		toDelete.add(handle);
		System.out.println(handle);
		assertEquals("image/png", handle.getContentType());
		assertEquals(FILE_NAME, handle.getFileName());
		assertEquals(new Long(imageFile.length()), handle.getContentSize());
		assertEquals(expectedMD5, handle.getContentMd5());
		
		// Now wait for the preview to be created.
		long start = System.currentTimeMillis();
		while(handle.getPreviewId() == null){
			System.out.println("Waiting for a preview to be created...");
			Thread.sleep(1000);
			assertTrue("Timed out waiting for a preview image to be created.", (System.currentTimeMillis()-start) < MAX_WAIT_MS);
			handle = (S3FileHandle) synapse.getRawFileHandle(handle.getId());
		}
		// Get the preview file handle.
		PreviewFileHandle preview = (PreviewFileHandle) synapse.getRawFileHandle(handle.getPreviewId());
		assertNotNull(preview);
		System.out.println(preview);
		toDelete.add(preview);
		
		//clear the preview and wait for it to be recreated
		synapse.clearPreview(handle.getId());
		handle = (S3FileHandle) synapse.getRawFileHandle(handle.getId());
		while(handle.getPreviewId() == null){
			System.out.println("Waiting for a preview to be recreated...");
			Thread.sleep(1000);
			assertTrue("Timed out waiting for a preview image to be created.", (System.currentTimeMillis()-start) < MAX_WAIT_MS);
			handle = (S3FileHandle) synapse.getRawFileHandle(handle.getId());
		}
		preview = (PreviewFileHandle) synapse.getRawFileHandle(handle.getPreviewId());
		assertNotNull(preview);
		toDelete.add(preview);
		
		// Now delete the root file handle.
		synapse.deleteFileHandle(handle.getId());
		// The main handle and the preview should get deleted.
		try{
			synapse.getRawFileHandle(handle.getId());
			fail("The handle should be deleted.");
		}catch(SynapseNotFoundException e){
			// expected.
		}
		try{
			synapse.getRawFileHandle(handle.getPreviewId());
			fail("The handle should be deleted.");
		}catch(SynapseNotFoundException e){
			// expected.
		}
	}
	
	@Test
	public void testSingleFileRoundTrip() throws SynapseException, IOException, InterruptedException{
		assertNotNull(imageFile);
		assertTrue(imageFile.exists());
		String expectedMD5 = MD5ChecksumHelper.getMD5Checksum(imageFile);
		// Create the image
		String myContentType = "test/content-type";
		FileHandle result = synapse.createFileHandle(imageFile, myContentType, project.getId());
		assertNotNull(result);
		S3FileHandle handle = (S3FileHandle) result;
		toDelete.add(handle);
		System.out.println(handle);
		assertEquals(myContentType, handle.getContentType());
		assertEquals(FILE_NAME, handle.getFileName());
		assertEquals(new Long(imageFile.length()), handle.getContentSize());
		assertEquals(expectedMD5, handle.getContentMd5());
		
		//preview will not be created for our test content type

		// Now delete the root file handle.
		synapse.deleteFileHandle(handle.getId());
		// The main handle and the preview should get deleted.
		try{
			synapse.getRawFileHandle(handle.getId());
			fail("The handle should be deleted.");
		}catch(SynapseNotFoundException e){
			// expected.
		}
	}
	
	@Test
	public void testSingleFileDeprecatedRoundTrip() throws SynapseException, IOException, InterruptedException {
		assertNotNull(imageFile);
		assertTrue(imageFile.exists());
		String expectedMD5 = MD5ChecksumHelper.getMD5Checksum(imageFile);
		// Create the image
		String myContentType = "test/content-type";
		@SuppressWarnings("deprecation")
		FileHandle result = synapse.createFileHandle(imageFile, myContentType);
		assertNotNull(result);
		S3FileHandle handle = (S3FileHandle) result;
		toDelete.add(handle);
		System.out.println(handle);
		assertEquals(myContentType, handle.getContentType());
		assertEquals(FILE_NAME, handle.getFileName());
		assertEquals(new Long(imageFile.length()), handle.getContentSize());
		assertEquals(expectedMD5, handle.getContentMd5());

		// preview will not be created for our test content type

		// Now delete the root file handle.
		synapse.deleteFileHandle(handle.getId());
		// The main handle and the preview should get deleted.
		try {
			synapse.getRawFileHandle(handle.getId());
			fail("The handle should be deleted.");
		} catch (SynapseNotFoundException e) {
			// expected.
		}
	}
	
	@Test
	public void testExternalRoundTrip() throws JSONObjectAdapterException, SynapseException{
		ExternalFileHandle efh = new ExternalFileHandle();
		efh.setContentType("text/plain");
		efh.setFileName("foo.bar");
		efh.setExternalURL("http://google.com");
		// Save it
		ExternalFileHandle clone = synapse.createExternalFileHandle(efh);
		assertNotNull(clone);
		toDelete.add(clone);
		assertNotNull(clone.getId());
		assertNotNull(clone.getCreatedBy());
		assertNotNull(clone.getCreatedOn());
		assertNotNull(clone.getEtag());
		assertEquals(efh.getFileName(), clone.getFileName());
		assertEquals(efh.getExternalURL(), clone.getExternalURL());
		assertEquals(efh.getContentType(), clone.getContentType());
	}
	
	@Test
	public void testExternalRoundTripWithNulls() throws JSONObjectAdapterException, SynapseException{
		// Us a null name and content type
		ExternalFileHandle efh = new ExternalFileHandle();
		efh.setContentType(null);
		efh.setFileName(null);
		efh.setExternalURL("http://google.com");
		// Save it
		ExternalFileHandle clone = synapse.createExternalFileHandle(efh);
		assertNotNull(clone);
		toDelete.add(clone);
		assertNotNull(clone.getId());
		assertNotNull(clone.getCreatedBy());
		assertNotNull(clone.getCreatedOn());
		assertNotNull(clone.getEtag());
		assertEquals("NOT_SET", clone.getFileName());
		assertEquals(efh.getExternalURL(), clone.getExternalURL());
		assertEquals("NOT_SET", clone.getContentType());
	}
	
	@Test
	public void testProjectSettingsCrud() throws SynapseException, IOException, InterruptedException {
		// create an project setting
		UploadDestinationListSetting projectSetting = new UploadDestinationListSetting();
		projectSetting.setProjectId(project.getId());
		projectSetting.setSettingsType(ProjectSettingsType.upload);

		ExternalStorageLocationSetting externalDestination = new ExternalStorageLocationSetting();
		externalDestination.setUploadType(UploadType.HTTPS);
		externalDestination.setUrl("https://not-valid");
		externalDestination.setBanner("warning, at institute");
		externalDestination.setDescription("not in synapse, this is");

		List<StorageLocationSetting> settings = synapse.getMyStorageLocationSettings();
		assertFalse(settings.contains(externalDestination));

		externalDestination = synapse.createStorageLocationSetting(externalDestination);

		settings = synapse.getMyStorageLocationSettings();
		assertTrue(settings.contains(externalDestination));
		StorageLocationSetting settingsClone = synapse.getMyStorageLocationSetting(externalDestination.getStorageLocationId());
		assertEquals(externalDestination, settingsClone);

		projectSetting.setLocations(Lists.newArrayList(externalDestination.getStorageLocationId()));
		ProjectSetting created = synapse.createProjectSetting(projectSetting);
		assertEquals(project.getId(), created.getProjectId());
		assertEquals(ProjectSettingsType.upload, created.getSettingsType());
		assertEquals(UploadDestinationListSetting.class, created.getClass());
		assertEquals(projectSetting.getLocations(), ((UploadDestinationListSetting) created).getLocations());

		ProjectSetting clone = synapse.getProjectSetting(project.getId(), ProjectSettingsType.upload);
		assertEquals(created, clone);

		UploadDestination uploadDestination = synapse.getUploadDestination(project.getId(), externalDestination.getStorageLocationId());
		assertEquals(externalDestination.getUploadType(), uploadDestination.getUploadType());
		assertEquals(externalDestination.getBanner(), uploadDestination.getBanner());
		assertEquals(externalDestination.getStorageLocationId(), uploadDestination.getStorageLocationId());

		synapse.deleteProjectSetting(created.getId());

		assertNull(synapse.getProjectSetting(project.getId(), ProjectSettingsType.upload));
	}

	@Test
	public void testExternalUploadDestinationUploadAndModifyRoundTrip() throws Exception {
		String baseKey = "test-" + UUID.randomUUID();

		// we need to create a authentication object
		String username = synapse.getUserSessionData().getProfile().getUserName();
		S3TestUtils.createObjectFromString(StackConfiguration.singleton().getExternalS3TestBucketName(), baseKey + "owner.txt", username,
				s3Client);

		// create setting
		ExternalS3StorageLocationSetting externalS3Destination = new ExternalS3StorageLocationSetting();
		externalS3Destination.setUploadType(UploadType.S3);
		externalS3Destination.setEndpointUrl(null);
		externalS3Destination.setBucket(StackConfiguration.singleton().getExternalS3TestBucketName());
		externalS3Destination.setBaseKey(baseKey);
		externalS3Destination.setBanner("warning, at institute");
		externalS3Destination.setDescription("not in synapse, this is");
		externalS3Destination = synapse.createStorageLocationSetting(externalS3Destination);

		UploadDestinationListSetting projectSetting = new UploadDestinationListSetting();
		projectSetting.setProjectId(project.getId());
		projectSetting.setSettingsType(ProjectSettingsType.upload);
		projectSetting.setLocations(Lists.newArrayList(externalS3Destination.getStorageLocationId()));
		synapse.createProjectSetting(projectSetting);

		String myContentType = "test/content-type";
		FileHandle result = synapse.createFileHandle(imageFile, myContentType, project.getId());
		toDelete.add(result);

		assertEquals(S3FileHandle.class, result.getClass());
		assertEquals(externalS3Destination.getStorageLocationId(), result.getStorageLocationId());

		File tmpFile = File.createTempFile(imageFile.getName(), ".tmp");
		synapse.downloadFromFileHandleTemporaryUrl(result.getId(), tmpFile);

		FileHandle result2 = synapse.createFileHandle(imageFile, myContentType, false, project.getId(), result.getStorageLocationId());
		toDelete.add(result2);

		assertEquals(S3FileHandle.class, result2.getClass());
		assertEquals(externalS3Destination.getStorageLocationId(), result2.getStorageLocationId());
	}

	@Test
	public void testAutoSyncRoundTrip() throws Exception {
		String baseKey = "test-" + UUID.randomUUID();

		// we need to create a authentication object
		String username = synapse.getUserSessionData().getProfile().getUserName();
		S3TestUtils.createObjectFromString(StackConfiguration.singleton().getExternalS3TestBucketName(), baseKey + "owner.txt", username,
				s3Client);

		final String md5 = S3TestUtils.createObjectFromString(StackConfiguration.singleton().getExternalS3TestBucketName(), baseKey
				+ "file1.txt", UUID.randomUUID().toString(), s3Client);

		// create setting
		ExternalS3StorageLocationSetting externalS3Destination = new ExternalS3StorageLocationSetting();
		externalS3Destination.setUploadType(UploadType.S3);
		externalS3Destination.setEndpointUrl(null);
		externalS3Destination.setBucket(StackConfiguration.singleton().getExternalS3TestBucketName());
		externalS3Destination.setBaseKey(baseKey);
		externalS3Destination.setBanner("warning, at institute");
		externalS3Destination.setDescription("not in synapse, this is");
		externalS3Destination = synapse.createStorageLocationSetting(externalS3Destination);

		ExternalSyncSetting externalSyncSetting = new ExternalSyncSetting();
		externalSyncSetting.setLocationId(externalS3Destination.getStorageLocationId());
		externalSyncSetting.setAutoSync(true);
		externalSyncSetting.setProjectId(project.getId());
		externalSyncSetting.setSettingsType(ProjectSettingsType.external_sync);
		synapse.createProjectSetting(externalSyncSetting);

		TimedAssert.waitForAssert(30000, 500, new Runnable() {
			@Override
			public void run() {
				try {
					List<EntityHeader> entityHeaderByMd5 = synapse.getEntityHeaderByMd5(md5);
					assertEquals(1, entityHeaderByMd5.size());
					assertEquals("file1.txt", entityHeaderByMd5.get(0).getName());
				} catch (SynapseNotFoundException e) {
					fail();
				} catch (SynapseException e) {
					throw new RuntimeException(e);
				}
			}
		});
	}

	@Test
	public void testExternalUploadDestinationChoice() throws SynapseException, IOException, InterruptedException {
		// create an project setting
		ExternalStorageLocationSetting externalDestination = new ExternalStorageLocationSetting();
		externalDestination.setUploadType(UploadType.SFTP);
		externalDestination.setUrl("sftp://somewhere");
		externalDestination.setBanner("warning, at institute");
		externalDestination.setDescription("not in synapse, this is");
		externalDestination = synapse.createStorageLocationSetting(externalDestination);

		S3StorageLocationSetting internalS3Destination = new S3StorageLocationSetting();
		internalS3Destination.setUploadType(UploadType.S3);
		internalS3Destination.setBanner("warning, not at institute");<commit_msg>test fix, cannot use md5<commit_after>package org.sagebionetworks;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;

import org.json.JSONObject;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.sagebionetworks.client.SynapseAdminClient;
import org.sagebionetworks.client.SynapseAdminClientImpl;
import org.sagebionetworks.client.SynapseClient;
import org.sagebionetworks.client.SynapseClientImpl;
import org.sagebionetworks.client.exceptions.SynapseClientException;
import org.sagebionetworks.client.exceptions.SynapseException;
import org.sagebionetworks.client.exceptions.SynapseNotFoundException;
import org.sagebionetworks.repo.manager.S3TestUtils;
import org.sagebionetworks.repo.model.EntityHeader;
import org.sagebionetworks.repo.model.Project;
import org.sagebionetworks.repo.model.QueryResults;
import org.sagebionetworks.repo.model.entity.query.EntityQueryResults;
import org.sagebionetworks.repo.model.file.ExternalFileHandle;
import org.sagebionetworks.repo.model.file.ExternalUploadDestination;
import org.sagebionetworks.repo.model.file.FileHandle;
import org.sagebionetworks.repo.model.file.FileHandleResults;
import org.sagebionetworks.repo.model.file.PreviewFileHandle;
import org.sagebionetworks.repo.model.file.S3FileHandle;
import org.sagebionetworks.repo.model.file.S3UploadDestination;
import org.sagebionetworks.repo.model.file.UploadDestination;
import org.sagebionetworks.repo.model.file.UploadDestinationLocation;
import org.sagebionetworks.repo.model.file.UploadType;
import org.sagebionetworks.repo.model.project.ExternalS3StorageLocationSetting;
import org.sagebionetworks.repo.model.project.ExternalStorageLocationSetting;
import org.sagebionetworks.repo.model.project.ExternalSyncSetting;
import org.sagebionetworks.repo.model.project.ProjectSetting;
import org.sagebionetworks.repo.model.project.ProjectSettingsType;
import org.sagebionetworks.repo.model.project.S3StorageLocationSetting;
import org.sagebionetworks.repo.model.project.StorageLocationSetting;
import org.sagebionetworks.repo.model.project.UploadDestinationListSetting;
import org.sagebionetworks.schema.adapter.JSONObjectAdapter;
import org.sagebionetworks.schema.adapter.JSONObjectAdapterException;
import org.sagebionetworks.schema.adapter.org.json.JSONObjectAdapterImpl;
import org.sagebionetworks.util.TimedAssert;
import org.sagebionetworks.utils.MD5ChecksumHelper;

import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
import com.google.common.collect.Lists;

public class IT049FileHandleTest {

	private static SynapseAdminClient adminSynapse;
	private static SynapseClient synapse;
	private static Long userToDelete;
	private static AmazonS3Client s3Client;
	
	private static final String LARGE_FILE_PATH_PROP_KEY = "org.sagebionetworks.test.large.file.path";
	private static final long MAX_WAIT_MS = 1000*10; // 10 sec
	private static final String FILE_NAME = "LittleImage.png";

	private List<FileHandle> toDelete = null;
	private File imageFile;
	private Project project;

	@BeforeClass
	public static void beforeClass() throws Exception {
		// Create a user
		adminSynapse = new SynapseAdminClientImpl();
		SynapseClientHelper.setEndpoints(adminSynapse);
		adminSynapse.setUserName(StackConfiguration.getMigrationAdminUsername());
		adminSynapse.setApiKey(StackConfiguration.getMigrationAdminAPIKey());
		adminSynapse.clearAllLocks();
		
		synapse = new SynapseClientImpl();
		userToDelete = SynapseClientHelper.createUser(adminSynapse, synapse);
		s3Client = new AmazonS3Client(new BasicAWSCredentials(StackConfiguration.getIAMUserId(), StackConfiguration.getIAMUserKey()));
		s3Client.createBucket(StackConfiguration.singleton().getExternalS3TestBucketName());
	}
	
	@Before
	public void before() throws SynapseException {
		toDelete = new ArrayList<FileHandle>();
		// Get the image file from the classpath.
		URL url = IT049FileHandleTest.class.getClassLoader().getResource("images/"+FILE_NAME);
		imageFile = new File(url.getFile().replaceAll("%20", " "));
		project = new Project();
		project = synapse.createEntity(project);
	}

	@After
	public void after() throws Exception {
		for (FileHandle handle: toDelete) {
			try {
				synapse.deleteFileHandle(handle.getId());
			} catch (SynapseNotFoundException e) {
			} catch (SynapseClientException e) { }
		}
		synapse.deleteEntity(project, true);
		S3TestUtils.doDeleteAfter(s3Client);
	}
	
	@AfterClass
	public static void afterClass() throws Exception {
		try {
			adminSynapse.deleteUser(userToDelete);
		} catch (SynapseException e) { }
	}
	
	@Test
	public void testImageFileRoundTrip() throws SynapseException, IOException, InterruptedException{
		assertNotNull(imageFile);
		assertTrue(imageFile.exists());
		String expectedMD5 = MD5ChecksumHelper.getMD5Checksum(imageFile);
		// Create the image
		List<File> list = new LinkedList<File>();
		list.add(imageFile);
		FileHandleResults results = synapse.createFileHandles(list, project.getId());
		assertNotNull(results);
		// We should have one image on the list
		assertNotNull(results.getList());
		assertEquals(1, results.getList().size());
		S3FileHandle handle = (S3FileHandle) results.getList().get(0);
		toDelete.add(handle);
		System.out.println(handle);
		assertEquals("image/png", handle.getContentType());
		assertEquals(FILE_NAME, handle.getFileName());
		assertEquals(new Long(imageFile.length()), handle.getContentSize());
		assertEquals(expectedMD5, handle.getContentMd5());
		
		// Now wait for the preview to be created.
		long start = System.currentTimeMillis();
		while(handle.getPreviewId() == null){
			System.out.println("Waiting for a preview to be created...");
			Thread.sleep(1000);
			assertTrue("Timed out waiting for a preview image to be created.", (System.currentTimeMillis()-start) < MAX_WAIT_MS);
			handle = (S3FileHandle) synapse.getRawFileHandle(handle.getId());
		}
		// Get the preview file handle.
		PreviewFileHandle preview = (PreviewFileHandle) synapse.getRawFileHandle(handle.getPreviewId());
		assertNotNull(preview);
		System.out.println(preview);
		toDelete.add(preview);
		
		//clear the preview and wait for it to be recreated
		synapse.clearPreview(handle.getId());
		handle = (S3FileHandle) synapse.getRawFileHandle(handle.getId());
		while(handle.getPreviewId() == null){
			System.out.println("Waiting for a preview to be recreated...");
			Thread.sleep(1000);
			assertTrue("Timed out waiting for a preview image to be created.", (System.currentTimeMillis()-start) < MAX_WAIT_MS);
			handle = (S3FileHandle) synapse.getRawFileHandle(handle.getId());
		}
		preview = (PreviewFileHandle) synapse.getRawFileHandle(handle.getPreviewId());
		assertNotNull(preview);
		toDelete.add(preview);
		
		// Now delete the root file handle.
		synapse.deleteFileHandle(handle.getId());
		// The main handle and the preview should get deleted.
		try{
			synapse.getRawFileHandle(handle.getId());
			fail("The handle should be deleted.");
		}catch(SynapseNotFoundException e){
			// expected.
		}
		try{
			synapse.getRawFileHandle(handle.getPreviewId());
			fail("The handle should be deleted.");
		}catch(SynapseNotFoundException e){
			// expected.
		}
	}
	
	@Test
	public void testSingleFileRoundTrip() throws SynapseException, IOException, InterruptedException{
		assertNotNull(imageFile);
		assertTrue(imageFile.exists());
		String expectedMD5 = MD5ChecksumHelper.getMD5Checksum(imageFile);
		// Create the image
		String myContentType = "test/content-type";
		FileHandle result = synapse.createFileHandle(imageFile, myContentType, project.getId());
		assertNotNull(result);
		S3FileHandle handle = (S3FileHandle) result;
		toDelete.add(handle);
		System.out.println(handle);
		assertEquals(myContentType, handle.getContentType());
		assertEquals(FILE_NAME, handle.getFileName());
		assertEquals(new Long(imageFile.length()), handle.getContentSize());
		assertEquals(expectedMD5, handle.getContentMd5());
		
		//preview will not be created for our test content type

		// Now delete the root file handle.
		synapse.deleteFileHandle(handle.getId());
		// The main handle and the preview should get deleted.
		try{
			synapse.getRawFileHandle(handle.getId());
			fail("The handle should be deleted.");
		}catch(SynapseNotFoundException e){
			// expected.
		}
	}
	
	@Test
	public void testSingleFileDeprecatedRoundTrip() throws SynapseException, IOException, InterruptedException {
		assertNotNull(imageFile);
		assertTrue(imageFile.exists());
		String expectedMD5 = MD5ChecksumHelper.getMD5Checksum(imageFile);
		// Create the image
		String myContentType = "test/content-type";
		@SuppressWarnings("deprecation")
		FileHandle result = synapse.createFileHandle(imageFile, myContentType);
		assertNotNull(result);
		S3FileHandle handle = (S3FileHandle) result;
		toDelete.add(handle);
		System.out.println(handle);
		assertEquals(myContentType, handle.getContentType());
		assertEquals(FILE_NAME, handle.getFileName());
		assertEquals(new Long(imageFile.length()), handle.getContentSize());
		assertEquals(expectedMD5, handle.getContentMd5());

		// preview will not be created for our test content type

		// Now delete the root file handle.
		synapse.deleteFileHandle(handle.getId());
		// The main handle and the preview should get deleted.
		try {
			synapse.getRawFileHandle(handle.getId());
			fail("The handle should be deleted.");
		} catch (SynapseNotFoundException e) {
			// expected.
		}
	}
	
	@Test
	public void testExternalRoundTrip() throws JSONObjectAdapterException, SynapseException{
		ExternalFileHandle efh = new ExternalFileHandle();
		efh.setContentType("text/plain");
		efh.setFileName("foo.bar");
		efh.setExternalURL("http://google.com");
		// Save it
		ExternalFileHandle clone = synapse.createExternalFileHandle(efh);
		assertNotNull(clone);
		toDelete.add(clone);
		assertNotNull(clone.getId());
		assertNotNull(clone.getCreatedBy());
		assertNotNull(clone.getCreatedOn());
		assertNotNull(clone.getEtag());
		assertEquals(efh.getFileName(), clone.getFileName());
		assertEquals(efh.getExternalURL(), clone.getExternalURL());
		assertEquals(efh.getContentType(), clone.getContentType());
	}
	
	@Test
	public void testExternalRoundTripWithNulls() throws JSONObjectAdapterException, SynapseException{
		// Us a null name and content type
		ExternalFileHandle efh = new ExternalFileHandle();
		efh.setContentType(null);
		efh.setFileName(null);
		efh.setExternalURL("http://google.com");
		// Save it
		ExternalFileHandle clone = synapse.createExternalFileHandle(efh);
		assertNotNull(clone);
		toDelete.add(clone);
		assertNotNull(clone.getId());
		assertNotNull(clone.getCreatedBy());
		assertNotNull(clone.getCreatedOn());
		assertNotNull(clone.getEtag());
		assertEquals("NOT_SET", clone.getFileName());
		assertEquals(efh.getExternalURL(), clone.getExternalURL());
		assertEquals("NOT_SET", clone.getContentType());
	}
	
	@Test
	public void testProjectSettingsCrud() throws SynapseException, IOException, InterruptedException {
		// create an project setting
		UploadDestinationListSetting projectSetting = new UploadDestinationListSetting();
		projectSetting.setProjectId(project.getId());
		projectSetting.setSettingsType(ProjectSettingsType.upload);

		ExternalStorageLocationSetting externalDestination = new ExternalStorageLocationSetting();
		externalDestination.setUploadType(UploadType.HTTPS);
		externalDestination.setUrl("https://not-valid");
		externalDestination.setBanner("warning, at institute");
		externalDestination.setDescription("not in synapse, this is");

		List<StorageLocationSetting> settings = synapse.getMyStorageLocationSettings();
		assertFalse(settings.contains(externalDestination));

		externalDestination = synapse.createStorageLocationSetting(externalDestination);

		settings = synapse.getMyStorageLocationSettings();
		assertTrue(settings.contains(externalDestination));
		StorageLocationSetting settingsClone = synapse.getMyStorageLocationSetting(externalDestination.getStorageLocationId());
		assertEquals(externalDestination, settingsClone);

		projectSetting.setLocations(Lists.newArrayList(externalDestination.getStorageLocationId()));
		ProjectSetting created = synapse.createProjectSetting(projectSetting);
		assertEquals(project.getId(), created.getProjectId());
		assertEquals(ProjectSettingsType.upload, created.getSettingsType());
		assertEquals(UploadDestinationListSetting.class, created.getClass());
		assertEquals(projectSetting.getLocations(), ((UploadDestinationListSetting) created).getLocations());

		ProjectSetting clone = synapse.getProjectSetting(project.getId(), ProjectSettingsType.upload);
		assertEquals(created, clone);

		UploadDestination uploadDestination = synapse.getUploadDestination(project.getId(), externalDestination.getStorageLocationId());
		assertEquals(externalDestination.getUploadType(), uploadDestination.getUploadType());
		assertEquals(externalDestination.getBanner(), uploadDestination.getBanner());
		assertEquals(externalDestination.getStorageLocationId(), uploadDestination.getStorageLocationId());

		synapse.deleteProjectSetting(created.getId());

		assertNull(synapse.getProjectSetting(project.getId(), ProjectSettingsType.upload));
	}

	@Test
	public void testExternalUploadDestinationUploadAndModifyRoundTrip() throws Exception {
		String baseKey = "test-" + UUID.randomUUID();

		// we need to create a authentication object
		String username = synapse.getUserSessionData().getProfile().getUserName();
		S3TestUtils.createObjectFromString(StackConfiguration.singleton().getExternalS3TestBucketName(), baseKey + "owner.txt", username,
				s3Client);

		// create setting
		ExternalS3StorageLocationSetting externalS3Destination = new ExternalS3StorageLocationSetting();
		externalS3Destination.setUploadType(UploadType.S3);
		externalS3Destination.setEndpointUrl(null);
		externalS3Destination.setBucket(StackConfiguration.singleton().getExternalS3TestBucketName());
		externalS3Destination.setBaseKey(baseKey);
		externalS3Destination.setBanner("warning, at institute");
		externalS3Destination.setDescription("not in synapse, this is");
		externalS3Destination = synapse.createStorageLocationSetting(externalS3Destination);

		UploadDestinationListSetting projectSetting = new UploadDestinationListSetting();
		projectSetting.setProjectId(project.getId());
		projectSetting.setSettingsType(ProjectSettingsType.upload);
		projectSetting.setLocations(Lists.newArrayList(externalS3Destination.getStorageLocationId()));
		synapse.createProjectSetting(projectSetting);

		String myContentType = "test/content-type";
		FileHandle result = synapse.createFileHandle(imageFile, myContentType, project.getId());
		toDelete.add(result);

		assertEquals(S3FileHandle.class, result.getClass());
		assertEquals(externalS3Destination.getStorageLocationId(), result.getStorageLocationId());

		File tmpFile = File.createTempFile(imageFile.getName(), ".tmp");
		synapse.downloadFromFileHandleTemporaryUrl(result.getId(), tmpFile);

		FileHandle result2 = synapse.createFileHandle(imageFile, myContentType, false, project.getId(), result.getStorageLocationId());
		toDelete.add(result2);

		assertEquals(S3FileHandle.class, result2.getClass());
		assertEquals(externalS3Destination.getStorageLocationId(), result2.getStorageLocationId());
	}

	@Test
	public void testAutoSyncRoundTrip() throws Exception {
		String baseKey = "test-" + UUID.randomUUID();

		// we need to create a authentication object
		String username = synapse.getUserSessionData().getProfile().getUserName();
		S3TestUtils.createObjectFromString(StackConfiguration.singleton().getExternalS3TestBucketName(), baseKey + "owner.txt", username,
				s3Client);

		final String md5 = S3TestUtils.createObjectFromString(StackConfiguration.singleton().getExternalS3TestBucketName(), baseKey
				+ "file1.txt", UUID.randomUUID().toString(), s3Client);

		// create setting
		ExternalS3StorageLocationSetting externalS3Destination = new ExternalS3StorageLocationSetting();
		externalS3Destination.setUploadType(UploadType.S3);
		externalS3Destination.setEndpointUrl(null);
		externalS3Destination.setBucket(StackConfiguration.singleton().getExternalS3TestBucketName());
		externalS3Destination.setBaseKey(baseKey);
		externalS3Destination.setBanner("warning, at institute");
		externalS3Destination.setDescription("not in synapse, this is");
		externalS3Destination = synapse.createStorageLocationSetting(externalS3Destination);

		ExternalSyncSetting externalSyncSetting = new ExternalSyncSetting();
		externalSyncSetting.setLocationId(externalS3Destination.getStorageLocationId());
		externalSyncSetting.setAutoSync(true);
		externalSyncSetting.setProjectId(project.getId());
		externalSyncSetting.setSettingsType(ProjectSettingsType.external_sync);
		synapse.createProjectSetting(externalSyncSetting);

		TimedAssert.waitForAssert(30000, 500, new Runnable() {
			@Override
			public void run() {
				try {
					JSONObject query = synapse.query("select name from entity where parentId == '" + project.getId() + "' LIMIT_1_OFFSET_1");
					assertEquals(1L, query.getInt("totalNumberOfResults"));
					assertEquals("file1.txt", query.getJSONArray("results").getJSONObject(0).getString("entity.name"));
				} catch (SynapseNotFoundException e) {
					fail();
				} catch (Exception e) {
					throw new RuntimeException(e);
				}
			}
		});
	}

	@Test
	public void testExternalUploadDestinationChoice() throws SynapseException, IOException, InterruptedException {
		// create an project setting
		ExternalStorageLocationSetting externalDestination = new ExternalStorageLocationSetting();
		externalDestination.setUploadType(UploadType.SFTP);
		externalDestination.setUrl("sftp://somewhere");
		externalDestination.setBanner("warning, at institute");
		externalDestination.setDescription("not in synapse, this is");
		externalDestination = synapse.createStorageLocationSetting(externalDestination);

		S3StorageLocationSetting internalS3Destination = new S3StorageLocationSetting();
		internalS3Destination.setUploadType(UploadType.S3);
		internalS3Destination.setBanner("warning, not at institute");
{'fraction_non_alphanumeric': 0.10733248019168459, 'fraction_numerical': 0.007297083889236802, 'mean_word_length': 6.588429752066116, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 14, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2146011', 'n_tokens_mistral': 11162, 'n_tokens_neox': 10475, 'n_words': 1901}