<commit_before>/*****************************************************************************
 * Copyright (c) <NAME>, Monash Uni & UQ, All rights reserved.         *
 *                                                                           *
 * Redistribution and use in source and binary forms, with or without        *
 * modification, are permitted provided that the following conditions        *
 * are met:                                                                  * 
 *                                                                           *
 * 1. Redistributions of source code must retain the above copyright notice, *
 *    this list of conditions and the following disclaimer.                  *
 * 2. Redistributions in binary form must reproduce the above copyright      *
 *    notice, this list of conditions and the following disclaimer in the    *
 *    documentation and/or other materials provided with the distribution.   *
 * 3. Neither the names of the institutions nor the names of the contributors*
 *    may be used to endorse or promote products derived from this software  *
 *    without specific prior written permission.                             *
 *                                                                           *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS   *
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, *
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR    *
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR         *
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,     *
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,       *
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR        *
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF    *
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING      *
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS        *
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.              *
 ****************************************************************************/

/**************************     REVISION HISTORY    **************************
 * 21/07/2014 - <NAME>: Created                                        
 *  
 ****************************************************************************/

package japsa.seq.nanopore;

import java.io.File;
import java.io.IOException;
import java.net.Socket;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;

import japsa.seq.FastqSequence;
import japsa.seq.SequenceOutputStream;
import japsa.util.DoubleArray;
import japsa.util.IntArray;
import japsa.util.JapsaException;
import japsa.util.Logging;
import japsa.util.net.StreamClient;

/**
 * Read nanopore data (read sequence, events, alignment, models etc) from a raw
 * (fast5) format. 
 * @author minhduc
 *
 */
public class NanoporeReaderStream{
	public String prepareIO(){
		String msg = null;
		try{
			sos = SequenceOutputStream.makeOutputStream(output);
			if (streamServers != null && streamServers.trim().length() > 0){
				@SuppressWarnings("resource")
				StreamClient streamClient = new StreamClient(streamServers);
				ArrayList<Socket>  sockets = streamClient.getSockets();
				networkOS = new ArrayList<SequenceOutputStream>(sockets.size());				
				for (Socket socket:sockets)					
					networkOS.add(new SequenceOutputStream(socket.getOutputStream()));			
			}			
		}catch (Exception e){
			msg = e.getMessage();
		}finally{
			if (msg != null){
				sos = null;
				return msg;
			}
		}
		return msg;
	}


	public void close() throws IOException{
		Logging.info("npReader closing");
		sos.close();
		if (networkOS != null){
			for (SequenceOutputStream out:networkOS)
				out.close();
		}
		Logging.info("npReader closed");
		done = true;		
	}

	double tempLength = 0, compLength = 0, twoDLength = 0;
	int tempCount = 0, compCount = 0, twoDCount = 0;
	IntArray lengths = new IntArray();
	DoubleArray qual2D = new DoubleArray(), qualComp = new DoubleArray(), qualTemp = new DoubleArray();
	IntArray lengths2D = new IntArray(), lengthsComp = new IntArray(), lengthsTemp = new IntArray();

	int fileNumber = 0;
	int passNumber = 0, failNumber = 0;
	SequenceOutputStream sos;
	ArrayList<SequenceOutputStream> networkOS = null;
	public boolean stats, number;
	public String folder = null;
	public int minLength = 0;
	public boolean wait = true;
	public boolean realtime = true;
	public int interval = 1, age = 1000;
	public boolean doFail = false;
	public String output = "";
	public String streamServers = null;
	boolean doLow = true;
	public boolean getTime = false;
	boolean done = false;

	public String format = "fastq";
	public boolean ready = true;
	private static final byte MIN_QUAL = '!';//The minimum quality

	/**
	 * Compute average quality of a read
	 * @param fq
	 * @return
	 */
	public static double averageQuality(FastqSequence fq){
		if (fq.length() > 0){
			double sumQual  = 0;
			for (int p = 0; p < fq.length(); p++){
				sumQual += (fq.getQualByte(p) - MIN_QUAL);

			}
			return (sumQual/fq.length());
		}	
		else return 0;
	}

	public void print(FastqSequence fq) throws IOException{
		fq.print(sos);
		if (networkOS != null){
			for (SequenceOutputStream out:networkOS)
				fq.print(out);
		}
	}

	@SuppressWarnings("unused")
	private void flush() throws IOException{
		sos.flush();
		if (networkOS != null){
			for (SequenceOutputStream out:networkOS)
				out.flush();
		}
	}

	public boolean readFastq2(String fileName) throws JapsaException, IOException{
		//Logging.info("Open " + fileName);
		try{					
			NanoporeReader npReader = new NanoporeReader(fileName);
			npReader.readFastq(getTime);						
			npReader.close();

			//Get time & date
			String log = npReader.getLog();					
			if (log != null){
				String [] toks = log.split("\n");<commit_msg>fix print to fasta error in npReader<commit_after>/*****************************************************************************
 * Copyright (c) <NAME>, Monash Uni & UQ, All rights reserved.         *
 *                                                                           *
 * Redistribution and use in source and binary forms, with or without        *
 * modification, are permitted provided that the following conditions        *
 * are met:                                                                  * 
 *                                                                           *
 * 1. Redistributions of source code must retain the above copyright notice, *
 *    this list of conditions and the following disclaimer.                  *
 * 2. Redistributions in binary form must reproduce the above copyright      *
 *    notice, this list of conditions and the following disclaimer in the    *
 *    documentation and/or other materials provided with the distribution.   *
 * 3. Neither the names of the institutions nor the names of the contributors*
 *    may be used to endorse or promote products derived from this software  *
 *    without specific prior written permission.                             *
 *                                                                           *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS   *
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, *
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR    *
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR         *
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,     *
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,       *
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR        *
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF    *
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING      *
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS        *
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.              *
 ****************************************************************************/

/**************************     REVISION HISTORY    **************************
 * 21/07/2014 - <NAME>: Created                                        
 *  
 ****************************************************************************/

package japsa.seq.nanopore;

import java.io.File;
import java.io.IOException;
import java.net.Socket;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;

import japsa.seq.FastqSequence;
import japsa.seq.SequenceOutputStream;
import japsa.util.DoubleArray;
import japsa.util.IntArray;
import japsa.util.JapsaException;
import japsa.util.Logging;
import japsa.util.net.StreamClient;

/**
 * Read nanopore data (read sequence, events, alignment, models etc) from a raw
 * (fast5) format. 
 * @author minhduc
 *
 */
public class NanoporeReaderStream{
	public String prepareIO(){
		String msg = null;
		try{
			sos = SequenceOutputStream.makeOutputStream(output);
			if (streamServers != null && streamServers.trim().length() > 0){
				@SuppressWarnings("resource")
				StreamClient streamClient = new StreamClient(streamServers);
				ArrayList<Socket>  sockets = streamClient.getSockets();
				networkOS = new ArrayList<SequenceOutputStream>(sockets.size());				
				for (Socket socket:sockets)					
					networkOS.add(new SequenceOutputStream(socket.getOutputStream()));			
			}			
		}catch (Exception e){
			msg = e.getMessage();
		}finally{
			if (msg != null){
				sos = null;
				return msg;
			}
		}
		return msg;
	}


	public void close() throws IOException{
		Logging.info("npReader closing");
		sos.close();
		if (networkOS != null){
			for (SequenceOutputStream out:networkOS)
				out.close();
		}
		Logging.info("npReader closed");
		done = true;		
	}

	double tempLength = 0, compLength = 0, twoDLength = 0;
	int tempCount = 0, compCount = 0, twoDCount = 0;
	IntArray lengths = new IntArray();
	DoubleArray qual2D = new DoubleArray(), qualComp = new DoubleArray(), qualTemp = new DoubleArray();
	IntArray lengths2D = new IntArray(), lengthsComp = new IntArray(), lengthsTemp = new IntArray();

	int fileNumber = 0;
	int passNumber = 0, failNumber = 0;
	SequenceOutputStream sos;
	ArrayList<SequenceOutputStream> networkOS = null;
	public boolean stats, number;
	public String folder = null;
	public int minLength = 0;
	public boolean wait = true;
	public boolean realtime = true;
	public int interval = 1, age = 1000;
	public boolean doFail = false;
	public String output = "";
	public String streamServers = null;
	boolean doLow = true;
	public boolean getTime = false;
	boolean done = false;

	public String format = "fastq";
	public boolean ready = true;
	private static final byte MIN_QUAL = '!';//The minimum quality

	/**
	 * Compute average quality of a read
	 * @param fq
	 * @return
	 */
	public static double averageQuality(FastqSequence fq){
		if (fq.length() > 0){
			double sumQual  = 0;
			for (int p = 0; p < fq.length(); p++){
				sumQual += (fq.getQualByte(p) - MIN_QUAL);

			}
			return (sumQual/fq.length());
		}	
		else return 0;
	}

	public void print(FastqSequence fq) throws IOException{
		if (format.equals("fasta"))
			fq.writeFasta(sos);
		else
			fq.print(sos);

		if (networkOS != null){
			for (SequenceOutputStream out:networkOS)
				if (format.equals("fasta"))
					fq.writeFasta(out);
				else
					fq.print(out);			//fq.print(out);
		}
	}

	@SuppressWarnings("unused")
	private void flush() throws IOException{
		sos.flush();
		if (networkOS != null){
			for (SequenceOutputStream out:networkOS)
				out.flush();
		}
	}

	public boolean readFastq2(String fileName) throws JapsaException, IOException{
		//Logging.info("Open " + fileName);
		try{					
			NanoporeReader npReader = new NanoporeReader(fileName);
			npReader.readFastq(getTime);						
			npReader.close();

			//Get time & date
			String log = npReader.getLog();					
			if (log != null){
				String [] toks = log.split("\n");
{'fraction_non_alphanumeric': 0.1372888339513803, 'fraction_numerical': 0.0057684384013185, 'mean_word_length': 2.8478123018389345, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6813773', 'n_tokens_mistral': 3670, 'n_tokens_neox': 3175, 'n_words': 1106}