<commit_before>package org.sagebionetworks;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.commons.io.IOUtils;
import org.apache.http.client.ClientProtocolException;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.sagebionetworks.client.SynapseAdministration;
import org.sagebionetworks.client.exceptions.SynapseException;
import org.sagebionetworks.repo.model.Entity;
import org.sagebionetworks.repo.model.MigrationType;
import org.sagebionetworks.repo.model.PrincipalBackup;
import org.sagebionetworks.repo.model.Project;
import org.sagebionetworks.repo.model.UserGroup;
import org.sagebionetworks.repo.model.UserProfile;
import org.sagebionetworks.repo.model.daemon.BackupRestoreStatus;
import org.sagebionetworks.repo.model.daemon.BackupSubmission;
import org.sagebionetworks.repo.model.daemon.DaemonStatus;
import org.sagebionetworks.repo.model.daemon.DaemonType;
import org.sagebionetworks.repo.model.daemon.RestoreSubmission;
import org.sagebionetworks.repo.model.search.Document;
import org.sagebionetworks.repo.model.status.StackStatus;
import org.sagebionetworks.schema.adapter.JSONObjectAdapterException;
import org.sagebionetworks.schema.adapter.org.json.EntityFactory;
import org.sagebionetworks.tool.migration.PrincipalRetriever012;
import org.sagebionetworks.tool.migration.dao.EntityData;

import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.PutObjectResult;
import com.amazonaws.services.s3.model.S3Object;

/**
 * This test will push data from a backup into synapse.
 * 
 * @author jmhill
 * 
 */
public class IT100BackupRestoration {

	public static final long TEST_TIME_OUT = 1000 * 60 * 4; // Currently 4 mins

	public static final String BACKUP_FILE_NAME = "Backup-staging-A-66004-4066545524488105200.zip";
	public static final String PRINCIPALS_BACKUP_FILE_NAME = "Backup-principals.zip";
	private static final String S3_DOMAIN = "https://s3.amazonaws.com/";
	private static String S3_WORKFLOW_BUCKET = StackConfiguration.getS3WorkflowBucket();
	private static final String S3_WORKFLOW_URL_PREFIX = S3_DOMAIN + S3_WORKFLOW_BUCKET + "/";
			
	
	private static SynapseAdministration synapse;
	private static AmazonS3Client s3Client;
	private static String bucket;
	
	private List<Entity> toDelete = null;

	@BeforeClass
	public static void beforeClass() throws Exception {
		// Use the synapse client to do some of the work for us.
		synapse = new SynapseAdministration();
		synapse.setAuthEndpoint(StackConfiguration
				.getAuthenticationServicePrivateEndpoint());
		synapse.setRepositoryEndpoint(StackConfiguration
				.getRepositoryServiceEndpoint());
		synapse.login(StackConfiguration.getIntegrationTestUserAdminName(),
				StackConfiguration.getIntegrationTestUserAdminPassword());

		System.out.println(StackConfiguration.getPortalEndpoint());
		String iamId = StackConfiguration.getIAMUserId();
		String iamKey = StackConfiguration.getIAMUserKey();
		if (iamId == null)
			throw new IllegalArgumentException("IAM id cannot be null");
		if (iamKey == null)
			throw new IllegalArgumentException("IAM key cannot be null");
		bucket = StackConfiguration.getSharedS3BackupBucket();
		if (bucket == null)
			throw new IllegalArgumentException("Bucket cannot be null null");
		AWSCredentials creds = new BasicAWSCredentials(iamId, iamKey);
		s3Client = new AmazonS3Client(creds);
	}
	
	@After
	public void after() throws Exception {
		if(synapse != null && toDelete != null){
			for(Entity e: toDelete){
				synapse.deleteEntity(e);
			}
		}
	}
	
	@Before
	public void before()throws Exception {
		synapse.login(StackConfiguration.getIntegrationTestUserAdminName(),
				StackConfiguration.getIntegrationTestUserAdminPassword());
		toDelete = new ArrayList<Entity>();
	}

	// This was used to create the backup used for the restoration step.
	@Ignore
	@Test
	public void createSnapshot() throws Exception {
		// Start the daemon
		BackupSubmission submission = new BackupSubmission();
		BackupRestoreStatus status = synapse.startBackupDaemon(submission, MigrationType.ENTITY);
		assertNotNull(status);
		assertNotNull(status.getStatus());
		assertFalse(DaemonStatus.FAILED == status.getStatus());
		assertEquals(DaemonType.BACKUP, status.getType());
		String backupId = status.getId();
		assertNotNull(backupId);
		String backupUri = "/daemon/" + backupId;
		long start = System.currentTimeMillis();
		// Wait for the backup to finish.
		while (true) {
			long now = System.currentTimeMillis();
			assertTrue("Timed out waiting for a backup to complete", now
					- start < TEST_TIME_OUT);
			status = synapse.getDaemonStatus(backupId);
			assertNotNull(status);
			assertNotNull(status.getStatus());
			// We are done if it failed.
			assertFalse(DaemonStatus.FAILED == status.getStatus());
			// Are we done?
			if (DaemonStatus.COMPLETED == status.getStatus()) {
				System.out.println("Backup Complete. Message: "
						+ status.getProgresssMessage());
				System.out.println("Backup File: "
						+ status.getBackupUrl());
				break;
			} else {
				long current = status.getProgresssCurrent();
				long total = status.getProgresssTotal();
				if (total <= 0) {
					total = 1000000;
				}
				String message = status.getProgresssMessage();
				double percent = ((double) current / (double) total) * 100.0;
				System.out.println("Backup progresss: " + percent
						+ " % Message: " + message);
			}
			// Wait.
			Thread.sleep(1000);
		}
	}
	
	@Test
	public void createPrincipalsSnapshot012() throws Exception {
		String repoEndpoint = "https://repo-staging.sagebase.org/repo/v1";
		String authEndpoint = "https://auth-staging.sagebase.org/auth/v1";
		PrincipalRetriever012 pr = new PrincipalRetriever012(authEndpoint, repoEndpoint);
		String adminUser = "<EMAIL>";
		String adminPw = "<PASSWORD>";
		pr.login(adminUser, adminPw);
		Collection<EntityData> principalData = pr.getPrincipalData();
		// validate it
		Collection<PrincipalBackup> pbs = pr.getPrincipals();
		assertEquals(principalData.size(), pbs.size());
		for (PrincipalBackup pb : pbs) {
			UserGroup group = pb.getUserGroup();
			assertNotNull(group);
			UserProfile userProfile = pb.getUserProfile();
			assertNotNull(group.getIsIndividual());
			if (group.getIsIndividual()) assertNotNull(userProfile);
			if (!group.getIsIndividual()) assertNull(userProfile);
			assertNotNull(group.getId());
			assertNotNull(group.getName());
			if (userProfile!=null) assertEquals(group.getId(), userProfile.getOwnerId());
			if (userProfile!=null) assertNotNull(userProfile.getEtag());
		}
		File file = new File(PRINCIPALS_BACKUP_FILE_NAME);
		try {<commit_msg>ignoring v0.12 migration integration test<commit_after>package org.sagebionetworks;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.commons.io.IOUtils;
import org.apache.http.client.ClientProtocolException;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.sagebionetworks.client.SynapseAdministration;
import org.sagebionetworks.client.exceptions.SynapseException;
import org.sagebionetworks.repo.model.Entity;
import org.sagebionetworks.repo.model.MigrationType;
import org.sagebionetworks.repo.model.PrincipalBackup;
import org.sagebionetworks.repo.model.Project;
import org.sagebionetworks.repo.model.UserGroup;
import org.sagebionetworks.repo.model.UserProfile;
import org.sagebionetworks.repo.model.daemon.BackupRestoreStatus;
import org.sagebionetworks.repo.model.daemon.BackupSubmission;
import org.sagebionetworks.repo.model.daemon.DaemonStatus;
import org.sagebionetworks.repo.model.daemon.DaemonType;
import org.sagebionetworks.repo.model.daemon.RestoreSubmission;
import org.sagebionetworks.repo.model.search.Document;
import org.sagebionetworks.repo.model.status.StackStatus;
import org.sagebionetworks.schema.adapter.JSONObjectAdapterException;
import org.sagebionetworks.schema.adapter.org.json.EntityFactory;
import org.sagebionetworks.tool.migration.PrincipalRetriever012;
import org.sagebionetworks.tool.migration.dao.EntityData;

import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.PutObjectResult;
import com.amazonaws.services.s3.model.S3Object;

/**
 * This test will push data from a backup into synapse.
 * 
 * @author jmhill
 * 
 */
public class IT100BackupRestoration {

	public static final long TEST_TIME_OUT = 1000 * 60 * 4; // Currently 4 mins

	public static final String BACKUP_FILE_NAME = "Backup-staging-A-66004-4066545524488105200.zip";
	public static final String PRINCIPALS_BACKUP_FILE_NAME = "Backup-principals.zip";
	private static final String S3_DOMAIN = "https://s3.amazonaws.com/";
	private static String S3_WORKFLOW_BUCKET = StackConfiguration.getS3WorkflowBucket();
	private static final String S3_WORKFLOW_URL_PREFIX = S3_DOMAIN + S3_WORKFLOW_BUCKET + "/";
			
	
	private static SynapseAdministration synapse;
	private static AmazonS3Client s3Client;
	private static String bucket;
	
	private List<Entity> toDelete = null;

	@BeforeClass
	public static void beforeClass() throws Exception {
		// Use the synapse client to do some of the work for us.
		synapse = new SynapseAdministration();
		synapse.setAuthEndpoint(StackConfiguration
				.getAuthenticationServicePrivateEndpoint());
		synapse.setRepositoryEndpoint(StackConfiguration
				.getRepositoryServiceEndpoint());
		synapse.login(StackConfiguration.getIntegrationTestUserAdminName(),
				StackConfiguration.getIntegrationTestUserAdminPassword());

		System.out.println(StackConfiguration.getPortalEndpoint());
		String iamId = StackConfiguration.getIAMUserId();
		String iamKey = StackConfiguration.getIAMUserKey();
		if (iamId == null)
			throw new IllegalArgumentException("IAM id cannot be null");
		if (iamKey == null)
			throw new IllegalArgumentException("IAM key cannot be null");
		bucket = StackConfiguration.getSharedS3BackupBucket();
		if (bucket == null)
			throw new IllegalArgumentException("Bucket cannot be null null");
		AWSCredentials creds = new BasicAWSCredentials(iamId, iamKey);
		s3Client = new AmazonS3Client(creds);
	}
	
	@After
	public void after() throws Exception {
		if(synapse != null && toDelete != null){
			for(Entity e: toDelete){
				synapse.deleteEntity(e);
			}
		}
	}
	
	@Before
	public void before()throws Exception {
		synapse.login(StackConfiguration.getIntegrationTestUserAdminName(),
				StackConfiguration.getIntegrationTestUserAdminPassword());
		toDelete = new ArrayList<Entity>();
	}

	// This was used to create the backup used for the restoration step.
	@Ignore
	@Test
	public void createSnapshot() throws Exception {
		// Start the daemon
		BackupSubmission submission = new BackupSubmission();
		BackupRestoreStatus status = synapse.startBackupDaemon(submission, MigrationType.ENTITY);
		assertNotNull(status);
		assertNotNull(status.getStatus());
		assertFalse(DaemonStatus.FAILED == status.getStatus());
		assertEquals(DaemonType.BACKUP, status.getType());
		String backupId = status.getId();
		assertNotNull(backupId);
		String backupUri = "/daemon/" + backupId;
		long start = System.currentTimeMillis();
		// Wait for the backup to finish.
		while (true) {
			long now = System.currentTimeMillis();
			assertTrue("Timed out waiting for a backup to complete", now
					- start < TEST_TIME_OUT);
			status = synapse.getDaemonStatus(backupId);
			assertNotNull(status);
			assertNotNull(status.getStatus());
			// We are done if it failed.
			assertFalse(DaemonStatus.FAILED == status.getStatus());
			// Are we done?
			if (DaemonStatus.COMPLETED == status.getStatus()) {
				System.out.println("Backup Complete. Message: "
						+ status.getProgresssMessage());
				System.out.println("Backup File: "
						+ status.getBackupUrl());
				break;
			} else {
				long current = status.getProgresssCurrent();
				long total = status.getProgresssTotal();
				if (total <= 0) {
					total = 1000000;
				}
				String message = status.getProgresssMessage();
				double percent = ((double) current / (double) total) * 100.0;
				System.out.println("Backup progresss: " + percent
						+ " % Message: " + message);
			}
			// Wait.
			Thread.sleep(1000);
		}
	}
	
	// this will be deleted after the 012-0.13 migration
	@Ignore
	@Test
	public void createPrincipalsSnapshot012() throws Exception {
		String repoEndpoint = "https://repo-staging.sagebase.org/repo/v1";
		String authEndpoint = "https://auth-staging.sagebase.org/auth/v1";
		PrincipalRetriever012 pr = new PrincipalRetriever012(authEndpoint, repoEndpoint);
		String adminUser = "<EMAIL>";
		String adminPw = "<PASSWORD>";
		pr.login(adminUser, adminPw);
		Collection<EntityData> principalData = pr.getPrincipalData();
		// validate it
		Collection<PrincipalBackup> pbs = pr.getPrincipals();
		assertEquals(principalData.size(), pbs.size());
		for (PrincipalBackup pb : pbs) {
			UserGroup group = pb.getUserGroup();
			assertNotNull(group);
			UserProfile userProfile = pb.getUserProfile();
			assertNotNull(group.getIsIndividual());
			if (group.getIsIndividual()) assertNotNull(userProfile);
			if (!group.getIsIndividual()) assertNull(userProfile);
			assertNotNull(group.getId());
			assertNotNull(group.getName());
			if (userProfile!=null) assertEquals(group.getId(), userProfile.getOwnerId());
			if (userProfile!=null) assertNotNull(userProfile.getEtag());
		}
		File file = new File(PRINCIPALS_BACKUP_FILE_NAME);
		try {
{'fraction_non_alphanumeric': 0.1081303602058319, 'fraction_numerical': 0.011869639794168095, 'mean_word_length': 5.171041490262489, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 15, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '421770', 'n_tokens_mistral': 4907, 'n_tokens_neox': 4454, 'n_words': 945}