<commit_before>package org.ihtsdo.buildcloud.service;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import org.apache.log4j.MDC;
import org.ihtsdo.buildcloud.dao.BuildDAO;
import org.ihtsdo.buildcloud.dao.helper.BuildS3PathHelper;
import org.ihtsdo.buildcloud.entity.Build;
import org.ihtsdo.buildcloud.entity.ReleaseCenter;
import org.ihtsdo.buildcloud.manifest.FolderType;
import org.ihtsdo.buildcloud.manifest.ListingType;
import org.ihtsdo.buildcloud.service.build.RF2Constants;
import org.ihtsdo.buildcloud.service.file.ManifestXmlFileParser;
import org.ihtsdo.buildcloud.service.identifier.client.IdServiceRestClient;
import org.ihtsdo.buildcloud.service.identifier.client.SchemeIdType;
import org.ihtsdo.otf.dao.s3.S3Client;
import org.ihtsdo.otf.dao.s3.helper.FileHelper;
import org.ihtsdo.otf.dao.s3.helper.S3ClientHelper;
import org.ihtsdo.otf.rest.client.RestClientException;
import org.ihtsdo.otf.rest.exception.BadRequestException;
import org.ihtsdo.otf.rest.exception.BusinessServiceException;
import org.ihtsdo.otf.rest.exception.EntityAlreadyExistsException;
import org.ihtsdo.otf.rest.exception.ResourceNotFoundException;
import org.ihtsdo.otf.utils.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StreamUtils;

import javax.xml.bind.JAXBException;

@Service
@Transactional
public class PublishServiceImpl implements PublishService {

	private static final String PUBLISHED_BUILD = "published_build_backup";

	private static final String SEPARATOR = "/";

	private static final Logger LOGGER = LoggerFactory.getLogger(PublishServiceImpl.class);

	private final FileHelper buildFileHelper;

	private final FileHelper publishedFileHelper;

	private final String publishedBucketName;

	@Autowired
	private BuildS3PathHelper buildS3PathHelper;

	@Autowired
	private IdServiceRestClient idRestClient;

	@Autowired
	private ProductService productService;

	@Autowired
	private BuildDAO buildDao;
	
	private static final int BATCH_SIZE = 5000;
	
	private static final int MAX_FAILURE = 100;


	@Autowired
	public PublishServiceImpl(final String buildBucketName, final String publishedBucketName,
			final S3Client s3Client, final S3ClientHelper s3ClientHelper) {
		buildFileHelper = new FileHelper(buildBucketName, s3Client, s3ClientHelper);
		this.publishedBucketName = publishedBucketName;
		publishedFileHelper = new FileHelper(publishedBucketName, s3Client, s3ClientHelper);
	}

	private String getPublishDirPath(final ReleaseCenter releaseCenter) {
		return releaseCenter.getBusinessKey() + SEPARATOR;
	}

	private String getPublishFilePath(final ReleaseCenter releaseCenter, final String releaseFileName) {
		return getPublishDirPath(releaseCenter) + releaseFileName;
	}

	@Override
	public List<String> getPublishedPackages(final ReleaseCenter releaseCenter) {
		List<String> packages = new ArrayList<>();
		List<String> allFiles = publishedFileHelper.listFiles(getPublishDirPath(releaseCenter));
		for (String file : allFiles) {
			if (file.endsWith(RF2Constants.ZIP_FILE_EXTENSION)) {
				packages.add(file);
			}
		}
		return packages;
	}

	@Override
	public void publishBuild(final Build build, boolean publishComponentIds, String releaseCenterKey, String productKey) throws BusinessServiceException {
		MDC.put(BuildService.MDC_BUILD_KEY, build.getUniqueId());
		try {
			String pkgOutPutDir = buildS3PathHelper.getBuildOutputFilesPath(build).toString();
			List<String> filesFound = buildFileHelper.listFiles(pkgOutPutDir);
			String releaseFileName = null;
			String md5FileName = null;
			for (String fileName : filesFound) {
				if (releaseFileName == null && FileUtils.isZip(fileName)) {
					releaseFileName = fileName;
					//only one zip file per package
				}
				if (md5FileName == null && FileUtils.isMD5(fileName)) {
					//expected to be only one MD5 file.
					md5FileName = fileName;
				}
			}

			ReleaseCenter releaseCenter = build.getProduct().getReleaseCenter();
			if (releaseFileName == null) {
				LOGGER.error("No zip file found for build:{}", build.getUniqueId());
			} else {
				String fileLock = releaseFileName.intern();
				synchronized (fileLock) {
					//publish component ids
					if (publishComponentIds) {
						LOGGER.info("Start publishing component ids for product {}  with build id {} ", build.getProduct().getBusinessKey(), build.getId());
						String buildOutputDir = buildS3PathHelper.getBuildOutputFilesPath(build).toString();
						boolean isBetaRelease = build.getProduct().getBuildConfiguration().isBetaRelease();
						publishComponentIds(buildFileHelper, buildOutputDir, isBetaRelease, releaseFileName);
						LOGGER.info("End publishing component ids for product {}  with build id {} ", build.getProduct().getBusinessKey(), build.getId());
					}
					//Does a published file already exist for this product?
					if (exists(releaseCenter, releaseFileName)) {
						throw new EntityAlreadyExistsException(releaseFileName + " has already been published for Release Center " + releaseCenter.getName() + " (" + build.getCreationTime() + ")");
					}

					String outputFileFullPath = buildS3PathHelper.getBuildOutputFilePath(build, releaseFileName);
					String publishedFilePath = getPublishFilePath(releaseCenter, releaseFileName);
					buildFileHelper.copyFile(outputFileFullPath, publishedBucketName, publishedFilePath);
					LOGGER.info("Release file:{} is copied to the published bucket:{}", releaseFileName, publishedBucketName);
					publishExtractedVersionOfPackage(publishedFilePath, publishedFileHelper.getFileStream(publishedFilePath));
				}
				// copy MD5 file if available
				if (md5FileName != null) {
					String source = buildS3PathHelper.getBuildOutputFilePath(build, md5FileName);
					String target = getPublishFilePath(releaseCenter, md5FileName);
					buildFileHelper.copyFile(source, publishedBucketName, target);
					LOGGER.info("MD5 file:{} is copied to the published bucket:{}", md5FileName, publishedBucketName);
				}
				
				//copy build info to published bucket
				backupPublishedBuild(build,publishedBucketName);
				LOGGER.info("Build:{} is copied to the published bucket:{}", build.getProduct().getBusinessKey() + build.getId(), publishedBucketName);

				//mark this as previous published package for future builds
				markAsPreviousPublishedPackage(build, releaseCenterKey, productKey);
			}
		} catch (IOException e) {
			throw new BusinessServiceException("Failed to publish build " + build.getUniqueId(), e);
		} catch (BusinessServiceException | JAXBException e) {
			throw new BusinessServiceException("Failed to mark this package as previous published package for future builds {}", e);
		} finally {
			MDC.remove(BuildService.MDC_BUILD_KEY);
		}

	}

	private void backupPublishedBuild(Build build, String publishedBucketName) {
		String orginalBuildPath = buildS3PathHelper.getBuildPath(build).toString();
		List<String> buildFiles =  buildFileHelper.listFiles(orginalBuildPath);
		String buildBckUpPath = getPublishDirPath(build.getProduct().getReleaseCenter()) + PUBLISHED_BUILD + SEPARATOR <commit_msg>Update published package as the "previous published package" for all future SRS builds.<commit_after>package org.ihtsdo.buildcloud.service;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import org.apache.log4j.MDC;
import org.ihtsdo.buildcloud.dao.BuildDAO;
import org.ihtsdo.buildcloud.dao.helper.BuildS3PathHelper;
import org.ihtsdo.buildcloud.entity.Build;
import org.ihtsdo.buildcloud.entity.ReleaseCenter;
import org.ihtsdo.buildcloud.manifest.FolderType;
import org.ihtsdo.buildcloud.manifest.ListingType;
import org.ihtsdo.buildcloud.service.build.RF2Constants;
import org.ihtsdo.buildcloud.service.file.ManifestXmlFileParser;
import org.ihtsdo.buildcloud.service.identifier.client.IdServiceRestClient;
import org.ihtsdo.buildcloud.service.identifier.client.SchemeIdType;
import org.ihtsdo.otf.dao.s3.S3Client;
import org.ihtsdo.otf.dao.s3.helper.FileHelper;
import org.ihtsdo.otf.dao.s3.helper.S3ClientHelper;
import org.ihtsdo.otf.rest.client.RestClientException;
import org.ihtsdo.otf.rest.exception.BadRequestException;
import org.ihtsdo.otf.rest.exception.BusinessServiceException;
import org.ihtsdo.otf.rest.exception.EntityAlreadyExistsException;
import org.ihtsdo.otf.rest.exception.ResourceNotFoundException;
import org.ihtsdo.otf.utils.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StreamUtils;

import javax.xml.bind.JAXBException;

@Service
@Transactional
public class PublishServiceImpl implements PublishService {

	private static final String PUBLISHED_BUILD = "published_build_backup";

	private static final String SEPARATOR = "/";

	private static final Logger LOGGER = LoggerFactory.getLogger(PublishServiceImpl.class);

	private final FileHelper buildFileHelper;

	private final FileHelper publishedFileHelper;

	private final String publishedBucketName;

	@Autowired
	private BuildS3PathHelper buildS3PathHelper;

	@Autowired
	private IdServiceRestClient idRestClient;

	@Autowired
	private ProductService productService;

	@Autowired
	private BuildDAO buildDao;
	
	private static final int BATCH_SIZE = 5000;
	
	private static final int MAX_FAILURE = 100;


	@Autowired
	public PublishServiceImpl(final String buildBucketName, final String publishedBucketName,
			final S3Client s3Client, final S3ClientHelper s3ClientHelper) {
		buildFileHelper = new FileHelper(buildBucketName, s3Client, s3ClientHelper);
		this.publishedBucketName = publishedBucketName;
		publishedFileHelper = new FileHelper(publishedBucketName, s3Client, s3ClientHelper);
	}

	private String getPublishDirPath(final ReleaseCenter releaseCenter) {
		return releaseCenter.getBusinessKey() + SEPARATOR;
	}

	private String getPublishFilePath(final ReleaseCenter releaseCenter, final String releaseFileName) {
		return getPublishDirPath(releaseCenter) + releaseFileName;
	}

	@Override
	public List<String> getPublishedPackages(final ReleaseCenter releaseCenter) {
		List<String> packages = new ArrayList<>();
		List<String> allFiles = publishedFileHelper.listFiles(getPublishDirPath(releaseCenter));
		for (String file : allFiles) {
			if (file.endsWith(RF2Constants.ZIP_FILE_EXTENSION)) {
				packages.add(file);
			}
		}
		return packages;
	}

	@Override
	public void publishBuild(final Build build, boolean publishComponentIds, String releaseCenterKey, String productKey) throws BusinessServiceException {
		MDC.put(BuildService.MDC_BUILD_KEY, build.getUniqueId());
		try {
			String pkgOutPutDir = buildS3PathHelper.getBuildOutputFilesPath(build).toString();
			List<String> filesFound = buildFileHelper.listFiles(pkgOutPutDir);
			String releaseFileName = null;
			String md5FileName = null;
			for (String fileName : filesFound) {
				if (releaseFileName == null && FileUtils.isZip(fileName)) {
					releaseFileName = fileName;
					//only one zip file per package
				}
				if (md5FileName == null && FileUtils.isMD5(fileName)) {
					//expected to be only one MD5 file.
					md5FileName = fileName;
				}
			}

			ReleaseCenter releaseCenter = build.getProduct().getReleaseCenter();
			if (releaseFileName == null) {
				LOGGER.error("No zip file found for build:{}", build.getUniqueId());
			} else {
				String fileLock = releaseFileName.intern();
				synchronized (fileLock) {
					//publish component ids
					if (publishComponentIds) {
						LOGGER.info("Start publishing component ids for product {}  with build id {} ", build.getProduct().getBusinessKey(), build.getId());
						String buildOutputDir = buildS3PathHelper.getBuildOutputFilesPath(build).toString();
						boolean isBetaRelease = build.getProduct().getBuildConfiguration().isBetaRelease();
						publishComponentIds(buildFileHelper, buildOutputDir, isBetaRelease, releaseFileName);
						LOGGER.info("End publishing component ids for product {}  with build id {} ", build.getProduct().getBusinessKey(), build.getId());
					}
					//Does a published file already exist for this product?
					if (exists(releaseCenter, releaseFileName)) {
						throw new EntityAlreadyExistsException(releaseFileName + " has already been published for Release Center " + releaseCenter.getName() + " (" + build.getCreationTime() + ")");
					}

					String outputFileFullPath = buildS3PathHelper.getBuildOutputFilePath(build, releaseFileName);
					String publishedFilePath = getPublishFilePath(releaseCenter, releaseFileName);
					buildFileHelper.copyFile(outputFileFullPath, publishedBucketName, publishedFilePath);
					LOGGER.info("Release file:{} is copied to the published bucket:{}", releaseFileName, publishedBucketName);
					publishExtractedVersionOfPackage(publishedFilePath, publishedFileHelper.getFileStream(publishedFilePath));
				}
				// copy MD5 file if available
				if (md5FileName != null) {
					String source = buildS3PathHelper.getBuildOutputFilePath(build, md5FileName);
					String target = getPublishFilePath(releaseCenter, md5FileName);
					buildFileHelper.copyFile(source, publishedBucketName, target);
					LOGGER.info("MD5 file:{} is copied to the published bucket:{}", md5FileName, publishedBucketName);
				}
				
				//copy build info to published bucket
				backupPublishedBuild(build,publishedBucketName);
				LOGGER.info("Build:{} is copied to the published bucket:{}", build.getProduct().getBusinessKey() + build.getId(), publishedBucketName);

				//mark this as previous published package for future builds
				markAsPreviousPublishedPackage(build, releaseCenterKey, productKey);
			}
		} catch (IOException e) {
			throw new BusinessServiceException("Failed to publish build " + build.getUniqueId(), e);
		} catch (JAXBException e) {
			throw new BusinessServiceException("Failed to mark this package as previous published package for future builds {}", e);
		} finally {
			MDC.remove(BuildService.MDC_BUILD_KEY);
		}

	}

	private void backupPublishedBuild(Build build, String publishedBucketName) {
		String orginalBuildPath = buildS3PathHelper.getBuildPath(build).toString();
		List<String> buildFiles =  buildFileHelper.listFiles(orginalBuildPath);
		String buildBckUpPath = getPublishDirPath(build.getProduct().getReleaseCenter()) + PUBLISHED_BUILD + SEPARATOR 
{'fraction_non_alphanumeric': 0.09318255250403877, 'fraction_numerical': 0.005686591276252019, 'mean_word_length': 5.535472972972973, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6723657', 'n_tokens_mistral': 4856, 'n_tokens_neox': 4400, 'n_words': 993}