<commit_before>package fi.csc.chipster.archive;

import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;

import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.Download;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;

import fi.csc.chipster.auth.model.Role;
import fi.csc.chipster.filebroker.StorageBackup;
import fi.csc.chipster.rest.Config;
import fi.csc.chipster.rest.ProcessUtils;
import fi.csc.chipster.rest.hibernate.BackupRotation2;
import fi.csc.chipster.rest.hibernate.DbBackup;
import fi.csc.chipster.rest.hibernate.S3Util;

public class BackupArchive {
	
	public static final String BACKUP_INFO = "backup-info";
	public static final String ARCHIVE_INFO = "archive-info";
	
	private static final String CONF_BACKUP_MONTHLY_COUNT = "backup-monthly-count";
	private static final String CONF_BACKUP_DAILY_COUNT = "backup-daily-count";
	
	private enum BackupType {
		INCREMENTAL,
		FULL;
	}

	private Logger logger = LogManager.getLogger();
	
	private Config config;

	public BackupArchive() {	
				
		this.config = new Config();		
				
		archiveAndCleanUp();
	}
	
	private void archiveAndCleanUp() {
		
		for (String role : config.getDbBackupRoles()) {
			String backupPrefix = role + DbBackup.BACKUP_OBJECT_NAME_PART;
			archiveAndCleanUp(role, BackupType.FULL, backupPrefix);
		}
		
		archiveAndCleanUp(Role.FILE_BROKER, BackupType.INCREMENTAL, StorageBackup.FILE_BROKER_BACKUP_NAME_PREFIX);		
	}
	
	private void archiveAndCleanUp(String role, BackupType type, String backupPrefix) {
				
		int dailyCount = Math.max(3, Integer.parseInt(config.getString(CONF_BACKUP_DAILY_COUNT, role)));
		int monthlyCount = Integer.parseInt(config.getString(CONF_BACKUP_MONTHLY_COUNT, role));
		
		Path archiveRootPath = Paths.get("backup-archive");
		
		TransferManager transferManager = BackupUtils.getTransferManager(config, role);
		logger.info("find " + backupPrefix + " from S3");
		
		String bucket = BackupUtils.getBackupBucket(config, role);
		List<S3ObjectSummary> objects = S3Util.getObjects(transferManager, bucket);
		
		// archive all starting from the oldest
		List<String> backups = findBackups(objects, backupPrefix, BACKUP_INFO);
		List<String> archivedBackups = findBackups(objects, backupPrefix, ARCHIVE_INFO);
		
		// skip backups that are already archived
		backups.removeAll(archivedBackups);
		
		logger.info("found " + backups.size() + " unarchived backups");
		
		try {
			for (String backupName : backups) {					
				
				logger.info("archive backup " + backupName);
				
				try {
					archive(transferManager, backupPrefix, archiveRootPath, role, backupName, bucket, objects);				
					
					if (type == BackupType.FULL) {
						removeOldFullArchives(archiveRootPath, backupPrefix, dailyCount, monthlyCount);
					} else {
						removeOldIncrementalArchives(archiveRootPath, StorageBackup.FILE_BROKER_BACKUP_NAME_PREFIX, 60);
					}
					
				} catch (IOException | InterruptedException | CleanUpException e) {
					logger.error("archive backup error", e);				
				}
			}
			cleanUpS3(transferManager, backupPrefix, role, archivedBackups, bucket, objects);
		} catch (ArchiveException e) {
			// hopefully this is enough if the archiving takes longer than 24 hours to protect aagins multiple processes moving the files
			// at the same time
			logger.error("archive error, skip all " + backupPrefix, e);
		} finally {
			transferManager.shutdownNow();
		}
	}
	
	private void cleanUpS3(TransferManager transferManager, String backupNamePrefix, String role, List<String> archivedBackups, String bucket, List<S3ObjectSummary> objects) {
		
		logger.info("clean up archived S3 backups of " + backupNamePrefix);		
		
		// delete all but the latest
		
		Set<String> objectsToDelete = new HashSet<String>();
		
		if (archivedBackups.size() > 1) {
			
			for (String backupName : archivedBackups.subList(0, archivedBackups.size() - 1)) {				
				logger.info("delete backup " + backupName + " from S3");
				Set<String> objectsOfBackup = objects.stream()
				.map(obj -> obj.getKey())
				.filter(key -> key.startsWith(backupName + "/"))
				.collect(Collectors.toSet());
				
				objectsToDelete.addAll(objectsOfBackup);			
			}
		}
		
		for (int i = 0; i < 10 && !objectsToDelete.isEmpty(); i++) {
			logger.info("delete " + objectsToDelete.size() + " objects from S3, attempt " + i);
			objectsToDelete.stream()
			.forEach(key -> transferManager.getAmazonS3Client().deleteObject(bucket, key));
			
			Set<String> keys = transferManager.getAmazonS3Client().listObjects(bucket).getObjectSummaries().stream()
			.map(obj -> obj.getKey())
			.collect(Collectors.toSet());<commit_msg>Update the object list before the clean-up<commit_after>package fi.csc.chipster.archive;

import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;

import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.Download;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;

import fi.csc.chipster.auth.model.Role;
import fi.csc.chipster.filebroker.StorageBackup;
import fi.csc.chipster.rest.Config;
import fi.csc.chipster.rest.ProcessUtils;
import fi.csc.chipster.rest.hibernate.BackupRotation2;
import fi.csc.chipster.rest.hibernate.DbBackup;
import fi.csc.chipster.rest.hibernate.S3Util;

public class BackupArchive {
	
	public static final String BACKUP_INFO = "backup-info";
	public static final String ARCHIVE_INFO = "archive-info";
	
	private static final String CONF_BACKUP_MONTHLY_COUNT = "backup-monthly-count";
	private static final String CONF_BACKUP_DAILY_COUNT = "backup-daily-count";
	
	private enum BackupType {
		INCREMENTAL,
		FULL;
	}

	private Logger logger = LogManager.getLogger();
	
	private Config config;

	public BackupArchive() {	
				
		this.config = new Config();		
				
		archiveAndCleanUp();
	}
	
	private void archiveAndCleanUp() {
		
		for (String role : config.getDbBackupRoles()) {
			String backupPrefix = role + DbBackup.BACKUP_OBJECT_NAME_PART;
			archiveAndCleanUp(role, BackupType.FULL, backupPrefix);
		}
		
		archiveAndCleanUp(Role.FILE_BROKER, BackupType.INCREMENTAL, StorageBackup.FILE_BROKER_BACKUP_NAME_PREFIX);		
	}
	
	private void archiveAndCleanUp(String role, BackupType type, String backupPrefix) {
				
		int dailyCount = Math.max(3, Integer.parseInt(config.getString(CONF_BACKUP_DAILY_COUNT, role)));
		int monthlyCount = Integer.parseInt(config.getString(CONF_BACKUP_MONTHLY_COUNT, role));
		
		Path archiveRootPath = Paths.get("backup-archive");
		
		TransferManager transferManager = BackupUtils.getTransferManager(config, role);
		logger.info("find " + backupPrefix + " from S3");
		
		String bucket = BackupUtils.getBackupBucket(config, role);
		List<S3ObjectSummary> objects = S3Util.getObjects(transferManager, bucket);
		
		// archive all starting from the oldest
		List<String> backups = findBackups(objects, backupPrefix, BACKUP_INFO);
		List<String> archivedBackups = findBackups(objects, backupPrefix, ARCHIVE_INFO);
		
		// skip backups that are already archived
		backups.removeAll(archivedBackups);
		
		logger.info("found " + backups.size() + " unarchived backups");
		
		try {
			for (String backupName : backups) {					
				
				logger.info("archive backup " + backupName);
				
				try {
					archive(transferManager, backupPrefix, archiveRootPath, role, backupName, bucket, objects);				
					
					if (type == BackupType.FULL) {
						removeOldFullArchives(archiveRootPath, backupPrefix, dailyCount, monthlyCount);
					} else {
						removeOldIncrementalArchives(archiveRootPath, StorageBackup.FILE_BROKER_BACKUP_NAME_PREFIX, 60);
					}
					
				} catch (IOException | InterruptedException | CleanUpException e) {
					logger.error("archive backup error", e);				
				}
			}
			
			cleanUpS3(transferManager, backupPrefix, role, bucket);
		} catch (ArchiveException e) {
			// hopefully this is enough if the archiving takes longer than 24 hours to protect aagins multiple processes moving the files
			// at the same time
			logger.error("archive error, skip all " + backupPrefix, e);
		} finally {
			transferManager.shutdownNow();
		}
	}
	
	private void cleanUpS3(TransferManager transferManager, String backupNamePrefix, String role, String bucket) {
		
		logger.info("clean up archived S3 backups of " + backupNamePrefix);
		
		// get objects to find also those that we just archived
		List<S3ObjectSummary> objects = S3Util.getObjects(transferManager, bucket);
		List<String> archivedBackups = findBackups(objects, backupNamePrefix, ARCHIVE_INFO);
		
		// delete all but the latest
		
		Set<String> objectsToDelete = new HashSet<String>();
		
		if (archivedBackups.size() > 1) {
			
			for (String backupName : archivedBackups.subList(0, archivedBackups.size() - 1)) {				
				logger.info("delete backup " + backupName + " from S3");
				Set<String> objectsOfBackup = objects.stream()
				.map(obj -> obj.getKey())
				.filter(key -> key.startsWith(backupName + "/"))
				.collect(Collectors.toSet());
				
				objectsToDelete.addAll(objectsOfBackup);			
			}
		}
		
		for (int i = 0; i < 10 && !objectsToDelete.isEmpty(); i++) {
			logger.info("delete " + objectsToDelete.size() + " objects from S3, attempt " + i);
			objectsToDelete.stream()
			.forEach(key -> transferManager.getAmazonS3Client().deleteObject(bucket, key));
			
			Set<String> keys = transferManager.getAmazonS3Client().listObjects(bucket).getObjectSummaries().stream()
			.map(obj -> obj.getKey())
			.collect(Collectors.toSet());
{'fraction_non_alphanumeric': 0.10955991124260354, 'fraction_numerical': 0.005824704142011835, 'mean_word_length': 4.44662638469285, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6957560', 'n_tokens_mistral': 3816, 'n_tokens_neox': 3299, 'n_words': 710}