<commit_before>package gov.nih.nci.cadsr.formloader.service.impl;

import gov.nih.nci.cadsr.formloader.domain.FormCollection;
import gov.nih.nci.cadsr.formloader.domain.FormDescriptor;
import gov.nih.nci.cadsr.formloader.service.XmlValidationService;
import gov.nih.nci.cadsr.formloader.service.common.FormLoaderHelper;
import gov.nih.nci.cadsr.formloader.service.common.FormLoaderServiceException;
import gov.nih.nci.cadsr.formloader.service.common.StaXParser;
import gov.nih.nci.cadsr.formloader.service.common.XmlValidationError;
import gov.nih.nci.cadsr.formloader.service.common.XmlValidationErrorHandler;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.List;

import javax.xml.XMLConstants;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import javax.xml.transform.Source;
import javax.xml.transform.stream.StreamSource;
import javax.xml.validation.SchemaFactory;

import org.apache.log4j.Logger;
import org.dom4j.DocumentException;
import org.dom4j.io.SAXReader;
import org.springframework.context.ResourceLoaderAware;
import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import org.springframework.stereotype.Service;
import org.xml.sax.SAXException;

@Service
public class XmlValidationServiceImpl implements XmlValidationService, ResourceLoaderAware {
	
	private static Logger logger = Logger.getLogger(XmlValidationServiceImpl.class.getName());
	
	protected ResourceLoader resourceLoader;
	
	//protected String XSD_PATH_NAME = "FormCartv22.xsd";
	protected String XSD_PATH_NAME = "FormLoaderv4.xsd";
	

	public XmlValidationServiceImpl() {}
	
	@Override
	public FormCollection validateXml(FormCollection collection) throws FormLoaderServiceException{
		
		if (collection == null) {
			throw new FormLoaderServiceException(FormLoaderServiceException.ERROR_COLLECTION_NULL,
					"Input collection object is null");
		}
		
		try {
			String xmlPathName = FormLoaderHelper.checkInputFile(collection.getXmlPathOnServer(), collection.getXmlFileName());

			XmlValidationError error = checkXmlWellFormedness(xmlPathName);
			if (error != null && error.getType() != XmlValidationError.XML_NO_ERROR) {
				collection.addMessage(error.getMessage());
				throw new FormLoaderServiceException(FormLoaderServiceException.ERROR_MALFORMED_XML,
						"Xml Malform Error", error);
			}

			logger.debug("XSD_PATH_NAME: " + XSD_PATH_NAME);
			Resource resource = this.resourceLoader.getResource(
					"classpath:gov/nih/nci/cadsr/formloader/service/impl/" + XSD_PATH_NAME);

			List<FormDescriptor> forms = null;

			List<XmlValidationError> errors = validateXml(xmlPathName, resource);

			//match errors with a form via line number in an error
			StaXParser parser = new StaXParser();
			forms = parser.parseFormHeaders(xmlPathName);		

			assignErrors(collection, forms, errors);

			collection.setForms(forms);
		} catch (FormLoaderServiceException e) {
			collection.addMessage(e.getMessage());
			throw e;
		}
		return collection;
	}
	
	/**
	 * Assign errors to forms based on line number where the error was detected
	 * @param forms
	 * @param errors
	 */
	protected void assignErrors(FormCollection collection, List<FormDescriptor> forms, List<XmlValidationError> errors) {
		
		if (errors != null && errors.size() > 0) {
			int currFormIdx = 0;
			for (XmlValidationError xmlError : errors) {
				int formSize = forms.size();
				if (formSize == 0)
					collection.addMessage(xmlError.toString());
				else {
					for (int i = 0; i < formSize; i++) {
						currFormIdx = assignErrorToForm(xmlError, forms, i);
						if (currFormIdx < 0)
							collection.addMessage(xmlError.toString());
					}
				}
			}
		}
		
		//Update xml validation status to all forms
		for (FormDescriptor form : forms) {
			if (form.getLoadStatus() != FormDescriptor.STATUS_XML_VALIDATION_FAILED)
				form.setLoadStatus(FormDescriptor.STATUS_XML_VALIDATED);	
		}
	}
	
	/**
	 * 
	 * @param xmlError
	 * @param forms
	 * @param startIdx 
	 * @return
	 */
	protected int assignErrorToForm(XmlValidationError xmlError, List<FormDescriptor> forms, int startIdx) {
		int total = forms.size();
		int errorLineNum = xmlError.getLineNumber();
		for (int i = startIdx; i < total; i++) {
			FormDescriptor form = forms.get(i);
			if (errorLineNum >= form.getXml_line_begin() && errorLineNum <= form.getXml_line_end()) {
				form.getErrors().add(xmlError);
				form.setLoadStatus(FormDescriptor.STATUS_XML_VALIDATION_FAILED);
				return i;
			}
		}
		
		//If we get here, no match is found. This could be errors outside of "form" elements
		logger.warn("Unable to find owner form for this error: " + xmlError.toString());
		return -1;
	}
	
	/**
	 * Check WellFormedness of an xml file
	 * @param xmlPathName
	 * @return If wellformed, XmlValidationError with XML_NO_ERROR. Otherwise, XmlValidationError with one of the error types
	 */
	public XmlValidationError checkXmlWellFormedness(String xmlPathName) 
		throws FormLoaderServiceException {
		
		XmlValidationError error;
	
		XmlValidationErrorHandler errorHandler = new XmlValidationErrorHandler();
		try {
            SAXReader reader = new SAXReader();
            reader.setValidation(false);
            reader.setErrorHandler(errorHandler);
            reader.read(xmlPathName);
        } catch (DocumentException e) {
        	//add message here
        	logger.error(e);
        	throw new FormLoaderServiceException(FormLoaderServiceException.ERROR_MALFORMED_XML,
					"Xml Malform Error: " + e.getMessage());
        }
		
		return (errorHandler.getXmlErrors().size() > 0) ? 
				errorHandler.getXmlErrors().get(0) : new XmlValidationError(XmlValidationError.XML_NO_ERROR, "Xml well formed", 0);
	}
	
	public List<XmlValidationError> validateXml(String xmlPathName, Resource resource) 
	throws FormLoaderServiceException {
		
		logger.debug("Start validating xml file: " + xmlPathName);
		XmlValidationErrorHandler handler = new XmlValidationErrorHandler();
		InputStream is = null;
		BufferedReader bufferredReader = null;
		try {
			is = resource.getInputStream();
			bufferredReader = new BufferedReader(new InputStreamReader(is));
			
            SAXParserFactory factory = SAXParserFactory.newInstance();
            SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
            factory.setSchema(schemaFactory.newSchema(new Source[] {new StreamSource(bufferredReader)}));

            SAXParser parser = factory.newSAXParser();

            SAXReader reader = new SAXReader(parser.getXMLReader());
            reader.setValidation(true);  //TODO: check on this
            reader.setErrorHandler(handler);<commit_msg>Made this protected:  List<XmlValidationError> validateXml(String xmlPathName, Resource resource) <commit_after>package gov.nih.nci.cadsr.formloader.service.impl;

import gov.nih.nci.cadsr.formloader.domain.FormCollection;
import gov.nih.nci.cadsr.formloader.domain.FormDescriptor;
import gov.nih.nci.cadsr.formloader.service.XmlValidationService;
import gov.nih.nci.cadsr.formloader.service.common.FormLoaderHelper;
import gov.nih.nci.cadsr.formloader.service.common.FormLoaderServiceException;
import gov.nih.nci.cadsr.formloader.service.common.StaXParser;
import gov.nih.nci.cadsr.formloader.service.common.XmlValidationError;
import gov.nih.nci.cadsr.formloader.service.common.XmlValidationErrorHandler;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.List;

import javax.xml.XMLConstants;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import javax.xml.transform.Source;
import javax.xml.transform.stream.StreamSource;
import javax.xml.validation.SchemaFactory;

import org.apache.log4j.Logger;
import org.dom4j.DocumentException;
import org.dom4j.io.SAXReader;
import org.springframework.context.ResourceLoaderAware;
import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import org.springframework.stereotype.Service;
import org.xml.sax.SAXException;

@Service
public class XmlValidationServiceImpl implements XmlValidationService, ResourceLoaderAware {
	
	private static Logger logger = Logger.getLogger(XmlValidationServiceImpl.class.getName());
	
	protected ResourceLoader resourceLoader;
	
	//protected String XSD_PATH_NAME = "FormCartv22.xsd";
	protected String XSD_PATH_NAME = "FormLoaderv4.xsd";
	

	public XmlValidationServiceImpl() {}
	
	@Override
	public FormCollection validateXml(FormCollection collection) throws FormLoaderServiceException{
		
		if (collection == null) {
			throw new FormLoaderServiceException(FormLoaderServiceException.ERROR_COLLECTION_NULL,
					"Input collection object is null");
		}
		
		try {
			String xmlPathName = FormLoaderHelper.checkInputFile(collection.getXmlPathOnServer(), collection.getXmlFileName());

			XmlValidationError error = checkXmlWellFormedness(xmlPathName);
			if (error != null && error.getType() != XmlValidationError.XML_NO_ERROR) {
				collection.addMessage(error.getMessage());
				throw new FormLoaderServiceException(FormLoaderServiceException.ERROR_MALFORMED_XML,
						"Xml Malform Error", error);
			}

			logger.debug("XSD_PATH_NAME: " + XSD_PATH_NAME);
			Resource resource = this.resourceLoader.getResource(
					"classpath:gov/nih/nci/cadsr/formloader/service/impl/" + XSD_PATH_NAME);

			List<FormDescriptor> forms = null;

			List<XmlValidationError> errors = validateXml(xmlPathName, resource);

			//match errors with a form via line number in an error
			StaXParser parser = new StaXParser();
			forms = parser.parseFormHeaders(xmlPathName);		

			assignErrors(collection, forms, errors);

			collection.setForms(forms);
		} catch (FormLoaderServiceException e) {
			collection.addMessage(e.getMessage());
			throw e;
		}
		return collection;
	}
	
	/**
	 * Assign errors to forms based on line number where the error was detected
	 * @param forms
	 * @param errors
	 */
	protected void assignErrors(FormCollection collection, List<FormDescriptor> forms, List<XmlValidationError> errors) {
		
		if (errors != null && errors.size() > 0) {
			int currFormIdx = 0;
			for (XmlValidationError xmlError : errors) {
				int formSize = forms.size();
				if (formSize == 0)
					collection.addMessage(xmlError.toString());
				else {
					for (int i = 0; i < formSize; i++) {
						currFormIdx = assignErrorToForm(xmlError, forms, i);
						if (currFormIdx < 0)
							collection.addMessage(xmlError.toString());
					}
				}
			}
		}
		
		//Update xml validation status to all forms
		for (FormDescriptor form : forms) {
			if (form.getLoadStatus() != FormDescriptor.STATUS_XML_VALIDATION_FAILED)
				form.setLoadStatus(FormDescriptor.STATUS_XML_VALIDATED);	
		}
	}
	
	/**
	 * 
	 * @param xmlError
	 * @param forms
	 * @param startIdx 
	 * @return
	 */
	protected int assignErrorToForm(XmlValidationError xmlError, List<FormDescriptor> forms, int startIdx) {
		int total = forms.size();
		int errorLineNum = xmlError.getLineNumber();
		for (int i = startIdx; i < total; i++) {
			FormDescriptor form = forms.get(i);
			if (errorLineNum >= form.getXml_line_begin() && errorLineNum <= form.getXml_line_end()) {
				form.getErrors().add(xmlError);
				form.setLoadStatus(FormDescriptor.STATUS_XML_VALIDATION_FAILED);
				return i;
			}
		}
		
		//If we get here, no match is found. This could be errors outside of "form" elements
		logger.warn("Unable to find owner form for this error: " + xmlError.toString());
		return -1;
	}
	
	/**
	 * Check WellFormedness of an xml file
	 * @param xmlPathName
	 * @return If wellformed, XmlValidationError with XML_NO_ERROR. Otherwise, XmlValidationError with one of the error types
	 */
	public XmlValidationError checkXmlWellFormedness(String xmlPathName) 
		throws FormLoaderServiceException {
		
		XmlValidationError error;
	
		XmlValidationErrorHandler errorHandler = new XmlValidationErrorHandler();
		try {
            SAXReader reader = new SAXReader();
            reader.setValidation(false);
            reader.setErrorHandler(errorHandler);
            reader.read(xmlPathName);
        } catch (DocumentException e) {
        	//add message here
        	logger.error(e);
        	throw new FormLoaderServiceException(FormLoaderServiceException.ERROR_MALFORMED_XML,
					"Xml Malform Error: " + e.getMessage());
        }
		
		return (errorHandler.getXmlErrors().size() > 0) ? 
				errorHandler.getXmlErrors().get(0) : new XmlValidationError(XmlValidationError.XML_NO_ERROR, "Xml well formed", 0);
	}
	
	protected List<XmlValidationError> validateXml(String xmlPathName, Resource resource) 
	throws FormLoaderServiceException {
		
		logger.debug("Start validating xml file: " + xmlPathName);
		XmlValidationErrorHandler handler = new XmlValidationErrorHandler();
		InputStream is = null;
		BufferedReader bufferredReader = null;
		try {
			is = resource.getInputStream();
			bufferredReader = new BufferedReader(new InputStreamReader(is));
			
            SAXParserFactory factory = SAXParserFactory.newInstance();
            SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
            factory.setSchema(schemaFactory.newSchema(new Source[] {new StreamSource(bufferredReader)}));

            SAXParser parser = factory.newSAXParser();

            SAXReader reader = new SAXReader(parser.getXMLReader());
            reader.setValidation(true);  //TODO: check on this
            reader.setErrorHandler(handler);
{'fraction_non_alphanumeric': 0.0990990990990991, 'fraction_numerical': 0.0023249055507120024, 'mean_word_length': 4.389584964761159, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 59}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5532801', 'n_tokens_mistral': 4447, 'n_tokens_neox': 3955, 'n_words': 863}