<commit_before>/*
 * The MIT License
 *
 * Copyright (c) 2009 The Broad Institute
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
package picard.sam;

import htsjdk.samtools.ReservedTagConstants;
import htsjdk.samtools.SAMFileHeader;
import htsjdk.samtools.SAMFileHeader.SortOrder;
import htsjdk.samtools.SAMFileWriter;
import htsjdk.samtools.SAMFileWriterFactory;
import htsjdk.samtools.SAMReadGroupRecord;
import htsjdk.samtools.SAMRecord;
import htsjdk.samtools.SAMUtils;
import htsjdk.samtools.fastq.FastqReader;
import htsjdk.samtools.fastq.FastqRecord;
import htsjdk.samtools.util.FastqQualityFormat;
import htsjdk.samtools.util.IOUtil;
import htsjdk.samtools.util.Iso8601Date;
import htsjdk.samtools.util.Log;
import htsjdk.samtools.util.ProgressLogger;
import htsjdk.samtools.util.QualityEncodingDetector;
import htsjdk.samtools.util.SolexaQualityConverter;
import htsjdk.samtools.util.StringUtil;
import picard.PicardException;
import picard.cmdline.CommandLineProgram;
import picard.cmdline.CommandLineProgramProperties;
import picard.cmdline.Option;
import picard.cmdline.StandardOptionDefinitions;
import picard.cmdline.programgroups.SamOrBam;

import java.io.File;
import java.util.ArrayList;
import java.util.List;

/**
 * Converts a fastq file to an unaligned BAM/SAM format.
 * See <a href="http://maq.sourceforge.net/fastq.shtml">MAQ FastQ specification</a> for details.
 * Three fastq versions are supported: FastqSanger, FastqSolexa and FastqIllumina.
 * Input files can be in GZip format (end in .gz).
 */
@CommandLineProgramProperties(
        usage = "Extracts read sequences and qualities from the input fastq file and writes them into the output file in unaligned BAM format."
                + " Input files can be in GZip format (end in .gz).\n",
        usageShort = "Converts a fastq file to an unaligned BAM or SAM file",
        programGroup = SamOrBam.class
)
public class FastqToSam extends CommandLineProgram {
    private static final Log LOG = Log.getInstance(FastqToSam.class);

    @Option(shortName="F1", doc="Input fastq file (optionally gzipped) for single end data, or first read in paired end data.")
    public File FASTQ;

    @Option(shortName="F2", doc="Input fastq file (optionally gzipped) for the second read of paired end data.", optional=true)
    public File FASTQ2;

    @Option(shortName="V", doc="A value describing how the quality values are encoded in the fastq.  Either Solexa for pre-pipeline 1.3 " +
            "style scores (solexa scaling + 66), Illumina for pipeline 1.3 and above (phred scaling + 64) or Standard for phred scaled " +
            "scores with a character shift of 33.  If this value is not specified, the quality format will be detected automatically.", optional = true)
    public FastqQualityFormat QUALITY_FORMAT;

    @Option(doc="Output SAM/BAM file. ", shortName=StandardOptionDefinitions.OUTPUT_SHORT_NAME) 
    public File OUTPUT ;

    @Option(shortName="RG", doc="Read group name")
    public String READ_GROUP_NAME = "A";

    @Option(shortName="SM", doc="Sample name to insert into the read group header")
    public String SAMPLE_NAME;

    @Option(shortName="LB", doc="The library name to place into the LB attribute in the read group header", optional=true)
    public String LIBRARY_NAME;

    @Option(shortName="PU", doc="The platform unit (often run_barcode.lane) to insert into the read group header", optional=true)
    public String PLATFORM_UNIT;

    @Option(shortName="PL", doc="The platform type (e.g. illumina, solid) to insert into the read group header", optional=true)
    public String PLATFORM;

    @Option(shortName="CN", doc="The sequencing center from which the data originated", optional=true)
    public String SEQUENCING_CENTER;

    @Option(shortName = "PI", doc = "Predicted median insert size, to insert into the read group header", optional = true)
    public Integer PREDICTED_INSERT_SIZE;

    @Option(doc="Comment(s) to include in the merged output file's header.", optional=true, shortName="CO")
    public List<String> COMMENT = new ArrayList<String>();

    @Option(shortName = "DS", doc = "Inserted into the read group header", optional = true)
    public String DESCRIPTION;

    @Option(shortName = "DT", doc = "Date the run was produced, to insert into the read group header", optional = true)
    public Iso8601Date RUN_DATE;

    @Option(shortName="SO", doc="The sort order for the output sam/bam file.")
    public SortOrder SORT_ORDER = SortOrder.queryname;

    @Option(doc="Minimum quality allowed in the input fastq.  An exception will be thrown if a quality is less than this value.")
    public int MIN_Q = 0;

    @Option(doc="Maximum quality allowed in the input fastq.  An exception will be thrown if a quality is greater than this value.")
    public int MAX_Q = SAMUtils.MAX_PHRED_SCORE;

    @Option(doc="If true and this is an unpaired fastq any occurance of '/1' will be removed from the end of a read name.")
    public Boolean STRIP_UNPAIRED_MATE_NUMBER = false;

    @Option(doc="Allow (and ignore) empty lines")
    public Boolean ALLOW_AND_IGNORE_EMPTY_LINES = false;

    private static final SolexaQualityConverter solexaQualityConverter = SolexaQualityConverter.getSingleton();

    /**
     * Looks at fastq input(s) and attempts to determine the proper quality format
     *
     * Closes the reader(s) by side effect
     *
     * @param reader1 The first fastq input
     * @param reader2 The second fastq input, if necessary. To not use this input, set it to null
     * @param expectedQuality If provided, will be used for sanity checking. If left null, autodetection will occur
     */
    public static FastqQualityFormat determineQualityFormat(final FastqReader reader1, final FastqReader reader2, final FastqQualityFormat expectedQuality) {
        final QualityEncodingDetector detector = new QualityEncodingDetector();

        if (reader2 == null) {
            detector.add(QualityEncodingDetector.DEFAULT_MAX_RECORDS_TO_ITERATE, reader1);
        } else {
            detector.add(QualityEncodingDetector.DEFAULT_MAX_RECORDS_TO_ITERATE, reader1, reader2);
            reader2.close();
        }

        reader1.close();

        final FastqQualityFormat qualityFormat =  detector.generateBestGuess(QualityEncodingDetector.FileContext.FASTQ, expectedQuality);
        if (detector.isDeterminationAmbiguous()) {
            LOG.warn("Making ambiguous determination about fastq's quality encoding; more than one format possible based on observed qualities.");
        }
        LOG.info(String.format("Auto-detected quality format as: %s.", qualityFormat));

        return qualityFormat;
    }

    /** Stock main method. */
    public static void main(final String[] argv) {
        System.exit(new FastqToSam().instanceMain(argv));
    }

    /* Simply invokes the right method for unpaired or paired data. */
    protected int doWork() {
        IOUtil.assertFileIsReadable(FASTQ);
        IOUtil.assertFileIsWritable(OUTPUT);

        FastqReader reader = fileToFastqReader(FASTQ);

        FastqReader reader2 = null;
        if (FASTQ2 != null) {
            IOUtil.assertFileIsReadable(FASTQ2);
            reader2 = fileToFastqReader(FASTQ2);
        }

        QUALITY_FORMAT = FastqToSam.determineQualityFormat(reader, reader2, QUALITY_FORMAT);

        final SAMFileHeader header = createSamFileHeader();
        final SAMFileWriter writer = new SAMFileWriterFactory().makeSAMOrBAMWriter(header, false, OUTPUT);

        reader = fileToFastqReader(FASTQ);
        if (FASTQ2 != null) {
            reader2 = fileToFastqReader(FASTQ2);
        }
        makeItSo(reader, reader2, writer);

        reader.close();

        if (reader2 != null) {
            reader2.close();
        }

        return 0;
    }

    /**
     * Handles the FastqToSam execution on the FastqReader(s).
     *
     * In some circumstances it might be useful to circumvent the command line based instantiation of this
     * class, however note that there is no handholding or guardrails to running in this manner.
     *
     * It is the caller's responsibility to close the reader(s)
     *
     * @param reader1 The FastqReader for the first fastq file
     * @param reader2 The second FastqReader if applicable. Pass in null if only using a single reader
     * @param writer The SAMFileWriter where the new SAM file is written
     *
     */
    public void makeItSo(final FastqReader reader1, final FastqReader reader2, final SAMFileWriter writer) {
        final int readCount = (FASTQ2 == null) ?  doUnpaired(reader1, writer) : doPaired(reader1, reader2, writer);
        LOG.info("Processed " + readCount + " fastq reads");
    }

    /** Creates a simple SAM file from a single fastq file. */
    protected int doUnpaired(final FastqReader freader, final SAMFileWriter writer) {
        int readCount = 0;
        final ProgressLogger progress = new ProgressLogger(LOG);
        for ( ; freader.hasNext()  ; readCount++) {
            final FastqRecord frec = freader.next();
            final SAMRecord srec = createSamRecord(writer.getFileHeader(), getReadName(frec.getReadHeader(), false) , frec, false) ;
            srec.setReadPairedFlag(false);
            writer.addAlignment(srec);
            progress.record(srec);
        }

        writer.close();
        return readCount;
    }

    /** More complicated method that takes two fastq files and builds pairing information in the SAM. */
    protected int doPaired(final FastqReader freader1, final FastqReader freader2, final SAMFileWriter writer) {
        int readCount = 0;
        final ProgressLogger progress = new ProgressLogger(LOG);
        for ( ; freader1.hasNext() && freader2.hasNext() ; readCount++) {<commit_msg>FASTQ2 should not be the CLP's parameter<commit_after>/*
 * The MIT License
 *
 * Copyright (c) 2009 The Broad Institute
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
package picard.sam;

import htsjdk.samtools.ReservedTagConstants;
import htsjdk.samtools.SAMFileHeader;
import htsjdk.samtools.SAMFileHeader.SortOrder;
import htsjdk.samtools.SAMFileWriter;
import htsjdk.samtools.SAMFileWriterFactory;
import htsjdk.samtools.SAMReadGroupRecord;
import htsjdk.samtools.SAMRecord;
import htsjdk.samtools.SAMUtils;
import htsjdk.samtools.fastq.FastqReader;
import htsjdk.samtools.fastq.FastqRecord;
import htsjdk.samtools.util.FastqQualityFormat;
import htsjdk.samtools.util.IOUtil;
import htsjdk.samtools.util.Iso8601Date;
import htsjdk.samtools.util.Log;
import htsjdk.samtools.util.ProgressLogger;
import htsjdk.samtools.util.QualityEncodingDetector;
import htsjdk.samtools.util.SolexaQualityConverter;
import htsjdk.samtools.util.StringUtil;
import picard.PicardException;
import picard.cmdline.CommandLineProgram;
import picard.cmdline.CommandLineProgramProperties;
import picard.cmdline.Option;
import picard.cmdline.StandardOptionDefinitions;
import picard.cmdline.programgroups.SamOrBam;

import java.io.File;
import java.util.ArrayList;
import java.util.List;

/**
 * Converts a fastq file to an unaligned BAM/SAM format.
 * See <a href="http://maq.sourceforge.net/fastq.shtml">MAQ FastQ specification</a> for details.
 * Three fastq versions are supported: FastqSanger, FastqSolexa and FastqIllumina.
 * Input files can be in GZip format (end in .gz).
 */
@CommandLineProgramProperties(
        usage = "Extracts read sequences and qualities from the input fastq file and writes them into the output file in unaligned BAM format."
                + " Input files can be in GZip format (end in .gz).\n",
        usageShort = "Converts a fastq file to an unaligned BAM or SAM file",
        programGroup = SamOrBam.class
)
public class FastqToSam extends CommandLineProgram {
    private static final Log LOG = Log.getInstance(FastqToSam.class);

    @Option(shortName="F1", doc="Input fastq file (optionally gzipped) for single end data, or first read in paired end data.")
    public File FASTQ;

    @Option(shortName="F2", doc="Input fastq file (optionally gzipped) for the second read of paired end data.", optional=true)
    public File FASTQ2;

    @Option(shortName="V", doc="A value describing how the quality values are encoded in the fastq.  Either Solexa for pre-pipeline 1.3 " +
            "style scores (solexa scaling + 66), Illumina for pipeline 1.3 and above (phred scaling + 64) or Standard for phred scaled " +
            "scores with a character shift of 33.  If this value is not specified, the quality format will be detected automatically.", optional = true)
    public FastqQualityFormat QUALITY_FORMAT;

    @Option(doc="Output SAM/BAM file. ", shortName=StandardOptionDefinitions.OUTPUT_SHORT_NAME) 
    public File OUTPUT ;

    @Option(shortName="RG", doc="Read group name")
    public String READ_GROUP_NAME = "A";

    @Option(shortName="SM", doc="Sample name to insert into the read group header")
    public String SAMPLE_NAME;

    @Option(shortName="LB", doc="The library name to place into the LB attribute in the read group header", optional=true)
    public String LIBRARY_NAME;

    @Option(shortName="PU", doc="The platform unit (often run_barcode.lane) to insert into the read group header", optional=true)
    public String PLATFORM_UNIT;

    @Option(shortName="PL", doc="The platform type (e.g. illumina, solid) to insert into the read group header", optional=true)
    public String PLATFORM;

    @Option(shortName="CN", doc="The sequencing center from which the data originated", optional=true)
    public String SEQUENCING_CENTER;

    @Option(shortName = "PI", doc = "Predicted median insert size, to insert into the read group header", optional = true)
    public Integer PREDICTED_INSERT_SIZE;

    @Option(doc="Comment(s) to include in the merged output file's header.", optional=true, shortName="CO")
    public List<String> COMMENT = new ArrayList<String>();

    @Option(shortName = "DS", doc = "Inserted into the read group header", optional = true)
    public String DESCRIPTION;

    @Option(shortName = "DT", doc = "Date the run was produced, to insert into the read group header", optional = true)
    public Iso8601Date RUN_DATE;

    @Option(shortName="SO", doc="The sort order for the output sam/bam file.")
    public SortOrder SORT_ORDER = SortOrder.queryname;

    @Option(doc="Minimum quality allowed in the input fastq.  An exception will be thrown if a quality is less than this value.")
    public int MIN_Q = 0;

    @Option(doc="Maximum quality allowed in the input fastq.  An exception will be thrown if a quality is greater than this value.")
    public int MAX_Q = SAMUtils.MAX_PHRED_SCORE;

    @Option(doc="If true and this is an unpaired fastq any occurance of '/1' will be removed from the end of a read name.")
    public Boolean STRIP_UNPAIRED_MATE_NUMBER = false;

    @Option(doc="Allow (and ignore) empty lines")
    public Boolean ALLOW_AND_IGNORE_EMPTY_LINES = false;

    private static final SolexaQualityConverter solexaQualityConverter = SolexaQualityConverter.getSingleton();

    /**
     * Looks at fastq input(s) and attempts to determine the proper quality format
     *
     * Closes the reader(s) by side effect
     *
     * @param reader1 The first fastq input
     * @param reader2 The second fastq input, if necessary. To not use this input, set it to null
     * @param expectedQuality If provided, will be used for sanity checking. If left null, autodetection will occur
     */
    public static FastqQualityFormat determineQualityFormat(final FastqReader reader1, final FastqReader reader2, final FastqQualityFormat expectedQuality) {
        final QualityEncodingDetector detector = new QualityEncodingDetector();

        if (reader2 == null) {
            detector.add(QualityEncodingDetector.DEFAULT_MAX_RECORDS_TO_ITERATE, reader1);
        } else {
            detector.add(QualityEncodingDetector.DEFAULT_MAX_RECORDS_TO_ITERATE, reader1, reader2);
            reader2.close();
        }

        reader1.close();

        final FastqQualityFormat qualityFormat =  detector.generateBestGuess(QualityEncodingDetector.FileContext.FASTQ, expectedQuality);
        if (detector.isDeterminationAmbiguous()) {
            LOG.warn("Making ambiguous determination about fastq's quality encoding; more than one format possible based on observed qualities.");
        }
        LOG.info(String.format("Auto-detected quality format as: %s.", qualityFormat));

        return qualityFormat;
    }

    /** Stock main method. */
    public static void main(final String[] argv) {
        System.exit(new FastqToSam().instanceMain(argv));
    }

    /* Simply invokes the right method for unpaired or paired data. */
    protected int doWork() {
        IOUtil.assertFileIsReadable(FASTQ);
        IOUtil.assertFileIsWritable(OUTPUT);

        FastqReader reader = fileToFastqReader(FASTQ);

        FastqReader reader2 = null;
        if (FASTQ2 != null) {
            IOUtil.assertFileIsReadable(FASTQ2);
            reader2 = fileToFastqReader(FASTQ2);
        }

        QUALITY_FORMAT = FastqToSam.determineQualityFormat(reader, reader2, QUALITY_FORMAT);

        final SAMFileHeader header = createSamFileHeader();
        final SAMFileWriter writer = new SAMFileWriterFactory().makeSAMOrBAMWriter(header, false, OUTPUT);

        reader = fileToFastqReader(FASTQ);
        if (FASTQ2 != null) {
            reader2 = fileToFastqReader(FASTQ2);
        }
        makeItSo(reader, reader2, writer);

        reader.close();

        if (reader2 != null) {
            reader2.close();
        }

        return 0;
    }

    /**
     * Handles the FastqToSam execution on the FastqReader(s).
     *
     * In some circumstances it might be useful to circumvent the command line based instantiation of this
     * class, however note that there is no handholding or guardrails to running in this manner.
     *
     * It is the caller's responsibility to close the reader(s)
     *
     * @param reader1 The FastqReader for the first fastq file
     * @param reader2 The second FastqReader if applicable. Pass in null if only using a single reader
     * @param writer The SAMFileWriter where the new SAM file is written
     *
     */
    public void makeItSo(final FastqReader reader1, final FastqReader reader2, final SAMFileWriter writer) {
        final int readCount = (reader2 == null) ?  doUnpaired(reader1, writer) : doPaired(reader1, reader2, writer);
        LOG.info("Processed " + readCount + " fastq reads");
    }

    /** Creates a simple SAM file from a single fastq file. */
    protected int doUnpaired(final FastqReader freader, final SAMFileWriter writer) {
        int readCount = 0;
        final ProgressLogger progress = new ProgressLogger(LOG);
        for ( ; freader.hasNext()  ; readCount++) {
            final FastqRecord frec = freader.next();
            final SAMRecord srec = createSamRecord(writer.getFileHeader(), getReadName(frec.getReadHeader(), false) , frec, false) ;
            srec.setReadPairedFlag(false);
            writer.addAlignment(srec);
            progress.record(srec);
        }

        writer.close();
        return readCount;
    }

    /** More complicated method that takes two fastq files and builds pairing information in the SAM. */
    protected int doPaired(final FastqReader freader1, final FastqReader freader2, final SAMFileWriter writer) {
        int readCount = 0;
        final ProgressLogger progress = new ProgressLogger(LOG);
        for ( ; freader1.hasNext() && freader2.hasNext() ; readCount++) {
{'fraction_non_alphanumeric': 0.08159015179563124, 'fraction_numerical': 0.005970011106997408, 'mean_word_length': 3.6222459893048127, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2538306', 'n_tokens_mistral': 6269, 'n_tokens_neox': 5951, 'n_words': 2138}