<commit_before>package org.ihtsdo.buildcloud.service.precondition;

import org.ihtsdo.buildcloud.dao.BuildDAO;
import org.ihtsdo.buildcloud.entity.Build;
import org.ihtsdo.buildcloud.manifest.ListingType;
import org.ihtsdo.buildcloud.service.exception.ResourceNotFoundException;
import org.ihtsdo.buildcloud.service.file.ManifestXmlFileParser;
import org.springframework.beans.factory.annotation.Autowired;

import javax.xml.bind.JAXBException;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import static org.ihtsdo.buildcloud.service.build.RF2Constants.*;

/**
 * To check all files specified in the manifest file can be derived from the input files.
 * If not the product process will be halted and the release manager should be alerted.
 * RF2 files:Full/snapshot/delta
 * der2_Refset_SimpleSnapshot_INT_20140831.txt
 * sct2_Concept_Snapshot_INT_20140131.txt
 * Documentation:
 * doc_Icd10MapTechnicalGuideExemplars_Current-en-US_INT_20140131.xlsx
 * doc_SnomedCTReleaseNotes_Current-en-US_INT_20140131.pdf
 * Readme files:
 * Readme_en_20140131.txt
 * Resources:
 * zres2_icRefset_OrderedTypeFull_INT_20110731.txt
 */
public class InputFilesExistenceCheck extends PreconditionCheck {

	private static final String ERROR_MSG = "The input files directory doesn't contain the following files required by the manifest.xml: ";
	@Autowired
	private BuildDAO buildDAO;

	@Override
	public void runCheck(Build build) {
		//check against the manifest file
		boolean isFailed = false;
		try (InputStream manifestInputSteam = buildDAO.getManifestStream(build)) {
			ManifestXmlFileParser parser = new ManifestXmlFileParser();
			ListingType listingType = parser.parse(manifestInputSteam);
			Set<String> filesExpected = new HashSet<>();
			boolean isFirstReadmeFound = false;
			for (String fileName : ManifestFileListingHelper.listAllFiles(listingType)) {
				//dealing with der2 and sct2 full/snapshot/delta files
				String[] splits = fileName.split(FILE_NAME_SEPARATOR);
				String fileNamePrefix = splits[0];
				if (DER2.equals(fileNamePrefix) || SCT2.equals(fileNamePrefix)) {
					String token3 = splits[2];
					String temp = fileName;
					if (token3.contains(FULL)) {
						temp = fileName.replace(FULL, DELTA);
					} else if (token3.contains(SNAPSHOT)) {
						temp = fileName.replace(SNAPSHOT, DELTA);
					}
					if (temp.contains(SCT2 + "_Relationship_")) {
						temp = temp.replace(SCT2 + "_Relationship_", SCT2 + "_StatedRelationship_");
					}
					filesExpected.add(temp.replace(fileNamePrefix, INPUT_FILE_PREFIX));
				} else if (!isFirstReadmeFound && README_FILENAME_PREFIX.equals(fileNamePrefix)) {
					//ignore the first one as it will be auto generated by release service.
					isFirstReadmeFound = true;
				} else {
					//add all static documentation and resources files.
					filesExpected.add(fileName);
				}

			}
			//get a list of input file names
			List<String> inputfilesList = buildDAO.listInputFileNames(build);
			//check expected against input files
			StringBuilder msgBuilder = new StringBuilder();
			int count = 0;
			for (String expectedFileName : filesExpected) {
				if (!inputfilesList.contains(expectedFileName)) {
					if (count++ > 0) {
						msgBuilder.append(",");
					}
					msgBuilder.append(expectedFileName);
				}
			}
			if (count > 0) {
				fatalError(ERROR_MSG + msgBuilder.toString());<commit_msg>SRS-467 Missing input files no longer considered 'Fatal'<commit_after>package org.ihtsdo.buildcloud.service.precondition;

import org.ihtsdo.buildcloud.dao.BuildDAO;
import org.ihtsdo.buildcloud.entity.Build;
import org.ihtsdo.buildcloud.manifest.ListingType;
import org.ihtsdo.buildcloud.service.exception.ResourceNotFoundException;
import org.ihtsdo.buildcloud.service.file.ManifestXmlFileParser;
import org.springframework.beans.factory.annotation.Autowired;

import javax.xml.bind.JAXBException;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import static org.ihtsdo.buildcloud.service.build.RF2Constants.*;

/**
 * To check all files specified in the manifest file can be derived from the input files.
 * If not the product process will be halted and the release manager should be alerted.
 * RF2 files:Full/snapshot/delta
 * der2_Refset_SimpleSnapshot_INT_20140831.txt
 * sct2_Concept_Snapshot_INT_20140131.txt
 * Documentation:
 * doc_Icd10MapTechnicalGuideExemplars_Current-en-US_INT_20140131.xlsx
 * doc_SnomedCTReleaseNotes_Current-en-US_INT_20140131.pdf
 * Readme files:
 * Readme_en_20140131.txt
 * Resources:
 * zres2_icRefset_OrderedTypeFull_INT_20110731.txt
 */
public class InputFilesExistenceCheck extends PreconditionCheck {

	private static final String ERROR_MSG = "The input files directory doesn't contain the following files required by the manifest.xml: ";
	@Autowired
	private BuildDAO buildDAO;

	@Override
	public void runCheck(Build build) {
		//check against the manifest file
		boolean isFailed = false;
		try (InputStream manifestInputSteam = buildDAO.getManifestStream(build)) {
			ManifestXmlFileParser parser = new ManifestXmlFileParser();
			ListingType listingType = parser.parse(manifestInputSteam);
			Set<String> filesExpected = new HashSet<>();
			boolean isFirstReadmeFound = false;
			for (String fileName : ManifestFileListingHelper.listAllFiles(listingType)) {
				//dealing with der2 and sct2 full/snapshot/delta files
				String[] splits = fileName.split(FILE_NAME_SEPARATOR);
				String fileNamePrefix = splits[0];
				if (DER2.equals(fileNamePrefix) || SCT2.equals(fileNamePrefix)) {
					String token3 = splits[2];
					String temp = fileName;
					if (token3.contains(FULL)) {
						temp = fileName.replace(FULL, DELTA);
					} else if (token3.contains(SNAPSHOT)) {
						temp = fileName.replace(SNAPSHOT, DELTA);
					}
					if (temp.contains(SCT2 + "_Relationship_")) {
						temp = temp.replace(SCT2 + "_Relationship_", SCT2 + "_StatedRelationship_");
					}
					filesExpected.add(temp.replace(fileNamePrefix, INPUT_FILE_PREFIX));
				} else if (!isFirstReadmeFound && README_FILENAME_PREFIX.equals(fileNamePrefix)) {
					//ignore the first one as it will be auto generated by release service.
					isFirstReadmeFound = true;
				} else {
					//add all static documentation and resources files.
					filesExpected.add(fileName);
				}

			}
			//get a list of input file names
			List<String> inputfilesList = buildDAO.listInputFileNames(build);
			//check expected against input files
			StringBuilder msgBuilder = new StringBuilder();
			int count = 0;
			for (String expectedFileName : filesExpected) {
				if (!inputfilesList.contains(expectedFileName)) {
					if (count++ > 0) {
						msgBuilder.append(",");
					}
					msgBuilder.append(expectedFileName);
				}
			}
			if (count > 0) {
				// Missing files is no longer considered FATAL
				fail(ERROR_MSG + msgBuilder.toString());
{'fraction_non_alphanumeric': 0.09460235429227677, 'fraction_numerical': 0.0205282802182027, 'mean_word_length': 4.472898664571877, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '988515', 'n_tokens_mistral': 2477, 'n_tokens_neox': 2128, 'n_words': 499}