<commit_before>/*
 * Copyright 2006-2011 The MZmine 2 Development Team
 * 
 * This file is part of MZmine 2.
 * 
 * MZmine 2 is free software; you can redistribute it and/or modify it under the
 * terms of the GNU General Public License as published by the Free Software
 * Foundation; either version 2 of the License, or (at your option) any later
 * version.
 * 
 * MZmine 2 is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
 * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License along with
 * MZmine 2; if not, write to the Free Software Foundation, Inc., 51 Franklin
 * St, Fifth Floor, Boston, MA 02110-1301 USA
 */

package net.sf.mzmine.modules.rawdatamethods.rawdataimport.fileformats;

import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
import java.util.Date;
import java.util.LinkedList;
import java.util.logging.Logger;
import java.util.zip.DataFormatException;

import javax.xml.datatype.DatatypeFactory;
import javax.xml.datatype.Duration;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;

import net.sf.mzmine.data.DataPoint;
import net.sf.mzmine.data.RawDataFile;
import net.sf.mzmine.data.RawDataFileWriter;
import net.sf.mzmine.data.impl.SimpleDataPoint;
import net.sf.mzmine.data.impl.SimpleScan;
import net.sf.mzmine.taskcontrol.AbstractTask;
import net.sf.mzmine.taskcontrol.TaskStatus;
import net.sf.mzmine.util.CompressionUtils;
import net.sf.mzmine.util.ExceptionUtils;
import net.sf.mzmine.util.ScanUtils;

import org.jfree.xml.util.Base64;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;

/**
 * 
 */
public class MzXMLReadTask extends AbstractTask {

	private Logger logger = Logger.getLogger(this.getClass().getName());

	private File file;
	private RawDataFileWriter newMZmineFile;
	private RawDataFile finalRawDataFile;
	private int totalScans = 0, parsedScans;
	private int peaksCount = 0;
	private StringBuilder charBuffer;
	private boolean compressFlag = false;
	private DefaultHandler handler = new MzXMLHandler();

	// Retention time parser
	private DatatypeFactory dataTypeFactory;

	/*
	 * This variables are used to set the number of fragments that one single
	 * scan can have. The initial size of array is set to 10, but it depends of
	 * fragmentation level.
	 */
	private int parentTreeValue[] = new int[10];
	private int msLevelTree = 0;

	/*
	 * This stack stores the current scan and all his fragments until all the
	 * information is recover. The logic is FIFO at the moment of write into the
	 * RawDataFile
	 */
	private LinkedList<SimpleScan> parentStack;

	/*
	 * This variable hold the present scan or fragment, it is send to the stack
	 * when another scan/fragment appears as a parser.startElement
	 */
	private SimpleScan buildingScan;

	public MzXMLReadTask(File fileToOpen, RawDataFileWriter newMZmineFile) {
		// 256 kilo-chars buffer
		charBuffer = new StringBuilder(1 << 18);
		parentStack = new LinkedList<SimpleScan>();
		this.file = fileToOpen;
		this.newMZmineFile = newMZmineFile;
	}

	/**
	 * @see net.sf.mzmine.taskcontrol.Task#getFinishedPercentage()
	 */
	public double getFinishedPercentage() {
		return totalScans == 0 ? 0 : (double) parsedScans / totalScans;
	}

	/**
	 * @see java.lang.Runnable#run()
	 */
	public void run() {

		setStatus(TaskStatus.PROCESSING);
		logger.info("Started parsing file " + file);

		// Use the default (non-validating) parser
		SAXParserFactory factory = SAXParserFactory.newInstance();

		try {

			dataTypeFactory = DatatypeFactory.newInstance();

			SAXParser saxParser = factory.newSAXParser();
			saxParser.parse(file, handler);

			// Close file
			finalRawDataFile = newMZmineFile.finishWriting();

		} catch (Throwable e) {
			/* we may already have set the status to CANCELED */
			if (getStatus() == TaskStatus.PROCESSING) {
				setStatus(TaskStatus.ERROR);
				errorMessage = ExceptionUtils.exceptionToString(e);
			}
			return;
		}

		if (parsedScans == 0) {
			setStatus(TaskStatus.ERROR);
			errorMessage = "No scans found";
			return;
		}

		logger.info("Finished parsing " + file + ", parsed " + parsedScans
				+ " scans");
		setStatus(TaskStatus.FINISHED);

	}

	public String getTaskDescription() {
		return "Opening file" + file;
	}

	public Object[] getCreatedObjects() {
		return new Object[] { finalRawDataFile };
	}

	private class MzXMLHandler extends DefaultHandler {
		public void startElement(String namespaceURI, String lName, // local
																	// name
				String qName, // qualified name
				Attributes attrs) throws SAXException {

			if (isCanceled())
				throw new SAXException("Parsing Cancelled");

			// <msRun>
			if (qName.equals("msRun")) {
				String s = attrs.getValue("scanCount");
				if (s != null)
					totalScans = Integer.parseInt(s);
			}

			// <scan>
			if (qName.equalsIgnoreCase("scan")) {

				if (buildingScan != null) {
					parentStack.addFirst(buildingScan);
					buildingScan = null;
				}

				/*
				 * Only num, msLevel & peaksCount values are required according
				 * with mzxml standard, the others are optional
				 */
				int scanNumber = Integer.parseInt(attrs.getValue("num"));
				int msLevel = Integer.parseInt(attrs.getValue("msLevel"));
				peaksCount = Integer.parseInt(attrs.getValue("peaksCount"));

				// Parse retention time
				double retentionTime = 0;
				String retentionTimeStr = attrs.getValue("retentionTime");
				if (retentionTimeStr != null) {
					Date currentDate = new Date();
					Duration dur = dataTypeFactory
							.newDuration(retentionTimeStr);
					retentionTime = dur.getTimeInMillis(currentDate) / 1000d;
				} else {
					setStatus(TaskStatus.ERROR);
					errorMessage = "This file does not contain retentionTime for scans";
					throw new SAXException("Could not read retention time");
				}

				int parentScan = -1;

				if (msLevel > 9) {
					setStatus(TaskStatus.ERROR);
					errorMessage = "msLevel value bigger than 10";
					throw new SAXException(
							"The value of msLevel is bigger than 10");
				}

				if (msLevel > 1) {
					parentScan = parentTreeValue[msLevel - 1];
					for (SimpleScan p : parentStack) {
						if (p.getScanNumber() == parentScan) {
							p.addFragmentScan(scanNumber);
						}
					}
				}

				// Setting the level of fragment of scan and parent scan number
				msLevelTree++;
				parentTreeValue[msLevel] = scanNumber;

				buildingScan = new SimpleScan(null, scanNumber, msLevel,
						retentionTime, parentScan, 0, 0, null,
						new DataPoint[0], false);

			}

			// <peaks>
			if (qName.equalsIgnoreCase("peaks")) {
				// clean the current char buffer for the new element
				charBuffer.setLength(0);
				compressFlag = false;
				String compressionType = attrs.getValue("compressionType");
				if ((compressionType == null)
						|| (compressionType.equals("none")))
					compressFlag = false;
				else
					compressFlag = true;

			}

			// <precursorMz>
			if (qName.equalsIgnoreCase("precursorMz")) {
				// clean the current char buffer for the new element
				charBuffer.setLength(0);
				String precursorCharge = attrs.getValue("precursorCharge");
				if (precursorCharge != null)
					buildingScan.setPrecursorCharge(Integer
							.parseInt(precursorCharge));
			}

		}

		/**
		 * endElement()
		 */
		public void endElement(String namespaceURI, String sName, // simple name
				String qName // qualified name
		) throws SAXException {

			// </scan>
			if (qName.equalsIgnoreCase("scan")) {

				msLevelTree--;

				/*
				 * At this point we verify if the scan and his fragments are
				 * closed, so we include the present scan/fragment into the
				 * stack and start to take elements from them (FIFO) for the
				 * RawDataFile.
				 */

				if (msLevelTree == 0) {
					parentStack.addFirst(buildingScan);
					buildingScan = null;
					while (!parentStack.isEmpty()) {
						SimpleScan currentScan = parentStack.removeLast();
						try {
							newMZmineFile.addScan(currentScan);
						} catch (IOException e) {
							setStatus(TaskStatus.ERROR);
							errorMessage = "IO error: " + e;
							throw new SAXException("Parsing cancelled");
						}
						parsedScans++;
					}

					/*
					 * The scan with all his fragments is in the RawDataFile,
					 * now we clean the stack for the next scan and fragments.
					 */
					parentStack.clear();

				}

				return;
			}

			// <precursorMz>
			if (qName.equalsIgnoreCase("precursorMz")) {
				double precursorMz = Double.parseDouble(charBuffer.toString());
				buildingScan.setPrecursorMZ(precursorMz);
				return;
			}

			// <peaks>
			if (qName.equalsIgnoreCase("peaks")) {

				byte[] peakBytes = Base64.decode(charBuffer.toString()
						.toCharArray());

				if (compressFlag) {
					try {
						peakBytes = CompressionUtils.decompress(peakBytes);
					} catch (DataFormatException e) {
						setStatus(TaskStatus.ERROR);
						errorMessage = "Corrupt compressed peak: "
								+ e.toString();
						throw new SAXException("Parsing Cancelled");
					}
				}

				// make a data input stream
				DataInputStream peakStream = new DataInputStream(
						new ByteArrayInputStream(peakBytes));

				DataPoint completeDataPoints[] = new DataPoint[peaksCount];

				try {
					for (int i = 0; i < completeDataPoints.length; i++) {

						// Always respect this order pairOrder="m/z-int"
						double massOverCharge = (double) peakStream.readFloat();
						double intensity = (double) peakStream.readFloat();<commit_msg>Patch by <NAME> for 64 bit precision mzXML<commit_after>/*
 * Copyright 2006-2011 The MZmine 2 Development Team
 * 
 * This file is part of MZmine 2.
 * 
 * MZmine 2 is free software; you can redistribute it and/or modify it under the
 * terms of the GNU General Public License as published by the Free Software
 * Foundation; either version 2 of the License, or (at your option) any later
 * version.
 * 
 * MZmine 2 is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
 * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License along with
 * MZmine 2; if not, write to the Free Software Foundation, Inc., 51 Franklin
 * St, Fifth Floor, Boston, MA 02110-1301 USA
 */

package net.sf.mzmine.modules.rawdatamethods.rawdataimport.fileformats;

import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
import java.util.Date;
import java.util.LinkedList;
import java.util.logging.Logger;
import java.util.zip.DataFormatException;

import javax.xml.datatype.DatatypeFactory;
import javax.xml.datatype.Duration;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;

import net.sf.mzmine.data.DataPoint;
import net.sf.mzmine.data.RawDataFile;
import net.sf.mzmine.data.RawDataFileWriter;
import net.sf.mzmine.data.impl.SimpleDataPoint;
import net.sf.mzmine.data.impl.SimpleScan;
import net.sf.mzmine.taskcontrol.AbstractTask;
import net.sf.mzmine.taskcontrol.TaskStatus;
import net.sf.mzmine.util.CompressionUtils;
import net.sf.mzmine.util.ExceptionUtils;
import net.sf.mzmine.util.ScanUtils;

import org.jfree.xml.util.Base64;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;

/**
 * 
 */
public class MzXMLReadTask extends AbstractTask {

	private Logger logger = Logger.getLogger(this.getClass().getName());

	private File file;
	private RawDataFileWriter newMZmineFile;
	private RawDataFile finalRawDataFile;
	private int totalScans = 0, parsedScans;
	private int peaksCount = 0;
	private StringBuilder charBuffer;
	private boolean compressFlag = false;
	private DefaultHandler handler = new MzXMLHandler();
	private String precision;

	// Retention time parser
	private DatatypeFactory dataTypeFactory;

	/*
	 * This variables are used to set the number of fragments that one single
	 * scan can have. The initial size of array is set to 10, but it depends of
	 * fragmentation level.
	 */
	private int parentTreeValue[] = new int[10];
	private int msLevelTree = 0;

	/*
	 * This stack stores the current scan and all his fragments until all the
	 * information is recover. The logic is FIFO at the moment of write into the
	 * RawDataFile
	 */
	private LinkedList<SimpleScan> parentStack;

	/*
	 * This variable hold the present scan or fragment, it is send to the stack
	 * when another scan/fragment appears as a parser.startElement
	 */
	private SimpleScan buildingScan;

	public MzXMLReadTask(File fileToOpen, RawDataFileWriter newMZmineFile) {
		// 256 kilo-chars buffer
		charBuffer = new StringBuilder(1 << 18);
		parentStack = new LinkedList<SimpleScan>();
		this.file = fileToOpen;
		this.newMZmineFile = newMZmineFile;
	}

	/**
	 * @see net.sf.mzmine.taskcontrol.Task#getFinishedPercentage()
	 */
	public double getFinishedPercentage() {
		return totalScans == 0 ? 0 : (double) parsedScans / totalScans;
	}

	/**
	 * @see java.lang.Runnable#run()
	 */
	public void run() {

		setStatus(TaskStatus.PROCESSING);
		logger.info("Started parsing file " + file);

		// Use the default (non-validating) parser
		SAXParserFactory factory = SAXParserFactory.newInstance();

		try {

			dataTypeFactory = DatatypeFactory.newInstance();

			SAXParser saxParser = factory.newSAXParser();
			saxParser.parse(file, handler);

			// Close file
			finalRawDataFile = newMZmineFile.finishWriting();

		} catch (Throwable e) {
			/* we may already have set the status to CANCELED */
			if (getStatus() == TaskStatus.PROCESSING) {
				setStatus(TaskStatus.ERROR);
				errorMessage = ExceptionUtils.exceptionToString(e);
			}
			return;
		}

		if (parsedScans == 0) {
			setStatus(TaskStatus.ERROR);
			errorMessage = "No scans found";
			return;
		}

		logger.info("Finished parsing " + file + ", parsed " + parsedScans
				+ " scans");
		setStatus(TaskStatus.FINISHED);

	}

	public String getTaskDescription() {
		return "Opening file" + file;
	}

	public Object[] getCreatedObjects() {
		return new Object[] { finalRawDataFile };
	}

	private class MzXMLHandler extends DefaultHandler {
		public void startElement(String namespaceURI, String lName, // local
																	// name
				String qName, // qualified name
				Attributes attrs) throws SAXException {

			if (isCanceled())
				throw new SAXException("Parsing Cancelled");

			// <msRun>
			if (qName.equals("msRun")) {
				String s = attrs.getValue("scanCount");
				if (s != null)
					totalScans = Integer.parseInt(s);
			}

			// <scan>
			if (qName.equalsIgnoreCase("scan")) {

				if (buildingScan != null) {
					parentStack.addFirst(buildingScan);
					buildingScan = null;
				}

				/*
				 * Only num, msLevel & peaksCount values are required according
				 * with mzxml standard, the others are optional
				 */
				int scanNumber = Integer.parseInt(attrs.getValue("num"));
				int msLevel = Integer.parseInt(attrs.getValue("msLevel"));
				peaksCount = Integer.parseInt(attrs.getValue("peaksCount"));

				// Parse retention time
				double retentionTime = 0;
				String retentionTimeStr = attrs.getValue("retentionTime");
				if (retentionTimeStr != null) {
					Date currentDate = new Date();
					Duration dur = dataTypeFactory
							.newDuration(retentionTimeStr);
					retentionTime = dur.getTimeInMillis(currentDate) / 1000d;
				} else {
					setStatus(TaskStatus.ERROR);
					errorMessage = "This file does not contain retentionTime for scans";
					throw new SAXException("Could not read retention time");
				}

				int parentScan = -1;

				if (msLevel > 9) {
					setStatus(TaskStatus.ERROR);
					errorMessage = "msLevel value bigger than 10";
					throw new SAXException(
							"The value of msLevel is bigger than 10");
				}

				if (msLevel > 1) {
					parentScan = parentTreeValue[msLevel - 1];
					for (SimpleScan p : parentStack) {
						if (p.getScanNumber() == parentScan) {
							p.addFragmentScan(scanNumber);
						}
					}
				}

				// Setting the level of fragment of scan and parent scan number
				msLevelTree++;
				parentTreeValue[msLevel] = scanNumber;

				buildingScan = new SimpleScan(null, scanNumber, msLevel,
						retentionTime, parentScan, 0, 0, null,
						new DataPoint[0], false);

			}

			// <peaks>
			if (qName.equalsIgnoreCase("peaks")) {
				// clean the current char buffer for the new element
				charBuffer.setLength(0);
				compressFlag = false;
				String compressionType = attrs.getValue("compressionType");
				if ((compressionType == null)
						|| (compressionType.equals("none")))
					compressFlag = false;
				else
					compressFlag = true;
				precision = attrs.getValue("precision");

			}

			// <precursorMz>
			if (qName.equalsIgnoreCase("precursorMz")) {
				// clean the current char buffer for the new element
				charBuffer.setLength(0);
				String precursorCharge = attrs.getValue("precursorCharge");
				if (precursorCharge != null)
					buildingScan.setPrecursorCharge(Integer
							.parseInt(precursorCharge));
			}

		}

		/**
		 * endElement()
		 */
		public void endElement(String namespaceURI, String sName, // simple name
				String qName // qualified name
		) throws SAXException {

			// </scan>
			if (qName.equalsIgnoreCase("scan")) {

				msLevelTree--;

				/*
				 * At this point we verify if the scan and his fragments are
				 * closed, so we include the present scan/fragment into the
				 * stack and start to take elements from them (FIFO) for the
				 * RawDataFile.
				 */

				if (msLevelTree == 0) {
					parentStack.addFirst(buildingScan);
					buildingScan = null;
					while (!parentStack.isEmpty()) {
						SimpleScan currentScan = parentStack.removeLast();
						try {
							newMZmineFile.addScan(currentScan);
						} catch (IOException e) {
							setStatus(TaskStatus.ERROR);
							errorMessage = "IO error: " + e;
							throw new SAXException("Parsing cancelled");
						}
						parsedScans++;
					}

					/*
					 * The scan with all his fragments is in the RawDataFile,
					 * now we clean the stack for the next scan and fragments.
					 */
					parentStack.clear();

				}

				return;
			}

			// <precursorMz>
			if (qName.equalsIgnoreCase("precursorMz")) {
				double precursorMz = Double.parseDouble(charBuffer.toString());
				buildingScan.setPrecursorMZ(precursorMz);
				return;
			}

			// <peaks>
			if (qName.equalsIgnoreCase("peaks")) {

				byte[] peakBytes = Base64.decode(charBuffer.toString()
						.toCharArray());

				if (compressFlag) {
					try {
						peakBytes = CompressionUtils.decompress(peakBytes);
					} catch (DataFormatException e) {
						setStatus(TaskStatus.ERROR);
						errorMessage = "Corrupt compressed peak: "
								+ e.toString();
						throw new SAXException("Parsing Cancelled");
					}
				}

				// make a data input stream
				DataInputStream peakStream = new DataInputStream(
						new ByteArrayInputStream(peakBytes));

				DataPoint completeDataPoints[] = new DataPoint[peaksCount];

				try {
					for (int i = 0; i < completeDataPoints.length; i++) {

						// Always respect this order pairOrder="m/z-int"
						double massOverCharge;
						double intensity;
						if ("64".equals(precision)) {
							massOverCharge = peakStream.readDouble();
							intensity = peakStream.readDouble();
						} else {
							massOverCharge = (double) peakStream.readFloat();
							intensity = (double) peakStream.readFloat();
						}
{'fraction_non_alphanumeric': 0.10273207393865529, 'fraction_numerical': 0.006804793824903514, 'mean_word_length': 3.328131868131868, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1120403', 'n_tokens_mistral': 7231, 'n_tokens_neox': 5751, 'n_words': 1672}