<commit_before> /*
 * The MIT License
 *
 * Copyright (c) 2012 The Broad Institute
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
package net.sf.picard.illumina.parser.readers;

import net.sf.picard.PicardException;
import net.sf.picard.util.UnsignedTypeUtil;
import net.sf.samtools.util.CloserUtil;

import java.io.*;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Iterator;
import java.util.zip.GZIPInputStream;

/**
 * BCL Files are base call and quality score binary files containing a (base,quality) pair for successive clusters.
 * The file is structured as followed:
 *  Bytes 1-4 : unsigned int numClusters
 *  Bytes 5-numClusters + 5 : 1 byte base/quality score
 *
 *  The base/quality scores are organized as follows (with one exception, SEE BELOW):
 *  The right 2 most bits (these are the LEAST significant bits) indicate the base, where
 *  A=00(0x00), C=01(0x01), G=10(0x02), and T=11(0x03)
 *
 *  The remaining bytes compose the quality score which is an unsigned int.
 *
 *  EXCEPTION: If a byte is entirely 0 (e.g. byteRead == 0) then it is a no call, the base
 *  becomes '.' and the Quality becomes 2, the default illumina masking value
 *
 *  (E.g. if we get a value in binary of 10001011 it gets transformed as follows:
 *
 *  Value read: 10001011(0x8B)
 *
 *  Quality     Base
 *
 *  100010      11
 *  00100010    0x03
 *  0x22        T
 *  34          T
 *
 *  So the output base/quality will be a (T/34)
 */
public class BclReader implements Iterator<BclReader.BclValue> {
    /** The size of the opening header (consisting solely of numClusters*/
    private static final int HEADER_SIZE = 4;
    
    private final BclQualityEvaluationStrategy bclQualityEvaluationStrategy;

    /** The number of clusters provided in this BCL */
    public final long numClusters;

    private final InputStream inputStream;
    private final String filePath;

    /* This strips off the leading bits in order to compare the read byte against the static values below */
    private final static byte BASE_MASK = 0x0003;
    private final static byte A_VAL = 0x00;
    private final static byte C_VAL = 0x01;
    private final static byte G_VAL = 0x02;
    private final static byte T_VAL = 0x03;

    /** The index to the next cluster that will be returned by this reader */
    private long nextCluster;

    public class BclValue {
        public final byte base;
        public final byte quality;

        public BclValue(final byte base, final byte quality) {
            this.base = base;
            this.quality = quality;
        }
    }

    public BclReader(final File file, final BclQualityEvaluationStrategy bclQualityEvaluationStrategy) {
        this.bclQualityEvaluationStrategy = bclQualityEvaluationStrategy;
        
        filePath = file.getAbsolutePath();
        final boolean isGzip = filePath.endsWith(".gz");

        // Open up a buffered stream to read from the file and optionally wrap it in a gzip stream
        // if necessary
        final BufferedInputStream bufferedInputStream;
        try {
             bufferedInputStream = new BufferedInputStream(new FileInputStream(file));
            inputStream = isGzip ? new GZIPInputStream(bufferedInputStream) : bufferedInputStream;
        } catch (FileNotFoundException fnfe) {
            throw new PicardException("File not found: (" + filePath + ")", fnfe);
        } catch (IOException ioe) {
            throw new PicardException("Error reading file: (" + filePath + ")", ioe);
        }

        // numClusters is used both for the file structure checks and in hasNext()
        numClusters = getNumClusters();

        if (file.length() == 0) {
            throw new PicardException("Zero length BCL file detected: " + filePath);
        }
        if (!isGzip) {
            // The file structure checks rely on the file size (as information is stored as individual bytes) but
            // we can't reliably know the number of uncompressed bytes in the file ahead of time for gzip files. Only
            // run the main check
            assertProperFileStructure(file);
        }

        nextCluster = 0;
    }<commit_msg>Set buffer size of BclReader to Defaults.BUFFER_SIZE, which defaults to 128K and can be controlled via -Dsamjdk.buffer_size=<buffer_size>.  Previous size was always 8192.<commit_after> /*
 * The MIT License
 *
 * Copyright (c) 2012 The Broad Institute
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
package net.sf.picard.illumina.parser.readers;

import net.sf.picard.PicardException;
import net.sf.picard.util.UnsignedTypeUtil;
import net.sf.samtools.Defaults;
import net.sf.samtools.util.CloserUtil;

import java.io.*;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Iterator;
import java.util.zip.GZIPInputStream;

/**
 * BCL Files are base call and quality score binary files containing a (base,quality) pair for successive clusters.
 * The file is structured as followed:
 *  Bytes 1-4 : unsigned int numClusters
 *  Bytes 5-numClusters + 5 : 1 byte base/quality score
 *
 *  The base/quality scores are organized as follows (with one exception, SEE BELOW):
 *  The right 2 most bits (these are the LEAST significant bits) indicate the base, where
 *  A=00(0x00), C=01(0x01), G=10(0x02), and T=11(0x03)
 *
 *  The remaining bytes compose the quality score which is an unsigned int.
 *
 *  EXCEPTION: If a byte is entirely 0 (e.g. byteRead == 0) then it is a no call, the base
 *  becomes '.' and the Quality becomes 2, the default illumina masking value
 *
 *  (E.g. if we get a value in binary of 10001011 it gets transformed as follows:
 *
 *  Value read: 10001011(0x8B)
 *
 *  Quality     Base
 *
 *  100010      11
 *  00100010    0x03
 *  0x22        T
 *  34          T
 *
 *  So the output base/quality will be a (T/34)
 */
public class BclReader implements Iterator<BclReader.BclValue> {
    /** The size of the opening header (consisting solely of numClusters*/
    private static final int HEADER_SIZE = 4;
    
    private final BclQualityEvaluationStrategy bclQualityEvaluationStrategy;

    /** The number of clusters provided in this BCL */
    public final long numClusters;

    private final InputStream inputStream;
    private final String filePath;

    /* This strips off the leading bits in order to compare the read byte against the static values below */
    private final static byte BASE_MASK = 0x0003;
    private final static byte A_VAL = 0x00;
    private final static byte C_VAL = 0x01;
    private final static byte G_VAL = 0x02;
    private final static byte T_VAL = 0x03;

    /** The index to the next cluster that will be returned by this reader */
    private long nextCluster;

    public class BclValue {
        public final byte base;
        public final byte quality;

        public BclValue(final byte base, final byte quality) {
            this.base = base;
            this.quality = quality;
        }
    }

    public BclReader(final File file, final BclQualityEvaluationStrategy bclQualityEvaluationStrategy) {
        this.bclQualityEvaluationStrategy = bclQualityEvaluationStrategy;
        
        filePath = file.getAbsolutePath();
        final boolean isGzip = filePath.endsWith(".gz");

        // Open up a buffered stream to read from the file and optionally wrap it in a gzip stream
        // if necessary
        final BufferedInputStream bufferedInputStream;
        try {
             bufferedInputStream = new BufferedInputStream(new FileInputStream(file), Defaults.BUFFER_SIZE);
            inputStream = isGzip ? new GZIPInputStream(bufferedInputStream) : bufferedInputStream;
        } catch (FileNotFoundException fnfe) {
            throw new PicardException("File not found: (" + filePath + ")", fnfe);
        } catch (IOException ioe) {
            throw new PicardException("Error reading file: (" + filePath + ")", ioe);
        }

        // numClusters is used both for the file structure checks and in hasNext()
        numClusters = getNumClusters();

        if (file.length() == 0) {
            throw new PicardException("Zero length BCL file detected: " + filePath);
        }
        if (!isGzip) {
            // The file structure checks rely on the file size (as information is stored as individual bytes) but
            // we can't reliably know the number of uncompressed bytes in the file ahead of time for gzip files. Only
            // run the main check
            assertProperFileStructure(file);
        }

        nextCluster = 0;
    }
{'fraction_non_alphanumeric': 0.07001133786848072, 'fraction_numerical': 0.01899092970521542, 'mean_word_length': 3.0447076805502484, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '136663', 'n_tokens_mistral': 3011, 'n_tokens_neox': 2748, 'n_words': 1230}