<commit_before>/*
 * Copyright (c) 2012, <NAME>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 *   list of conditions and the following disclaimer.
 *
 * * Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 *
 * * Neither the name of the copyright holder nor the names of its
 *   contributors may be used to endorse or promote products derived from
 *   this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package com.twelvemonkeys.imageio.plugins.tiff;

import com.twelvemonkeys.lang.Validate;

import java.io.EOFException;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;

/**
 * CCITT Modified Huffman RLE, Group 3 (T4) and Group 4 (T6) fax compression.
 *
 * @author <a href="mailto:<EMAIL>"><NAME></a>
 * @author <a href="https://github.com/Schmidor"><NAME></a>
 * @author last modified by $Author: haraldk$
 * @version $Id: CCITTFaxDecoderStream.java,v 1.0 23.05.12 15:55 haraldk Exp$
 */
final class CCITTFaxDecoderStream extends FilterInputStream {
    // See TIFF 6.0 Specification, Section 10: "Modified Huffman Compression", page 43.

    private final int columns;
    private final byte[] decodedRow;

    private final boolean optionG32D;
    // Leading zeros for aligning EOL
    private final boolean optionG3Fill;
    private final boolean optionUncompressed;
    private final boolean optionByteAligned;

    // Need to take fill order into account (?) (use flip table?)
    private final int fillOrder;
    private final int type;

    private int decodedLength;
    private int decodedPos;

    private int[] changesReferenceRow;
    private int[] changesCurrentRow;
    private int changesReferenceRowCount;
    private int changesCurrentRowCount;

    private int lastChangingElement = 0;

    /**
     * Creates a CCITTFaxDecoderStream.
     * This constructor may be used for CCITT streams embedded in PDF files,
     * which use EncodedByteAlign.
     *
     * @param stream the compressed CCITT stream.
     * @param columns the number of columns in the stream.
     * @param type the type of stream, must be one of {@code COMPRESSION_CCITT_MODIFIED_HUFFMAN_RLE},
     *             {@code COMPRESSION_CCITT_T4} or {@code COMPRESSION_CCITT_T6}.
     * @param fillOrder fillOrder, must be {@code FILL_LEFT_TO_RIGHT} or
     * {@code FILL_RIGHT_TO_LEFT}.
     * @param options CCITT T.4 or T.6 options.
     * @param byteAligned enable byte alignment used in PDF files (EncodedByteAlign).
     */
    public CCITTFaxDecoderStream(final InputStream stream, final int columns, final int type, final int fillOrder,
                                 final long options, final boolean byteAligned) {
        super(Validate.notNull(stream, "stream"));

        this.columns = Validate.isTrue(columns > 0, columns, "width must be greater than 0");
        this.type = Validate.isTrue(type == TIFFBaseline.COMPRESSION_CCITT_MODIFIED_HUFFMAN_RLE ||
                        type == TIFFExtension.COMPRESSION_CCITT_T4 ||
                        type == TIFFExtension.COMPRESSION_CCITT_T6,
                type, "Only CCITT Modified Huffman RLE compression (2), CCITT T4 (3) or CCITT T6 (4) supported: %s");
        this.fillOrder = Validate.isTrue(
                fillOrder == TIFFBaseline.FILL_LEFT_TO_RIGHT || fillOrder == TIFFExtension.FILL_RIGHT_TO_LEFT,
                fillOrder, "Expected fill order 1  or 2: %s"
        );

        // We know this is only used for b/w (1 bit)
        decodedRow = new byte[(columns + 7) / 8];
        changesReferenceRow = new int[columns + 2];
        changesCurrentRow = new int[columns + 2];

        switch (type) {
            case TIFFBaseline.COMPRESSION_CCITT_MODIFIED_HUFFMAN_RLE:
                optionByteAligned = byteAligned;
                optionG32D = false;
                optionG3Fill = false;
                optionUncompressed = false;
                break;
            case TIFFExtension.COMPRESSION_CCITT_T4:
                optionByteAligned = byteAligned;
                optionG32D = (options & TIFFExtension.GROUP3OPT_2DENCODING) != 0;
                optionG3Fill = (options & TIFFExtension.GROUP3OPT_FILLBITS) != 0;
                optionUncompressed = (options & TIFFExtension.GROUP3OPT_UNCOMPRESSED) != 0;
                break;
            case TIFFExtension.COMPRESSION_CCITT_T6:
                optionByteAligned = byteAligned;
                optionG32D = false;
                optionG3Fill = false;
                optionUncompressed = (options & TIFFExtension.GROUP4OPT_UNCOMPRESSED) != 0;
                break;
            default:
                // Guarded above
                throw new AssertionError();
        }

        Validate.isTrue(!optionUncompressed, optionUncompressed,
                "CCITT GROUP 3/4 OPTION UNCOMPRESSED is not supported");
    }

    /**
     * Creates a CCITTFaxDecoderStream.
     *
     * @param stream the compressed CCITT stream.
     * @param columns the number of columns in the stream.
     * @param type the type of stream, must be one of {@code COMPRESSION_CCITT_MODIFIED_HUFFMAN_RLE},
     *             {@code COMPRESSION_CCITT_T4} or {@code COMPRESSION_CCITT_T6}.
     * @param fillOrder fillOrder, must be {@code FILL_LEFT_TO_RIGHT} or
     * {@code FILL_RIGHT_TO_LEFT}.
     * @param options CCITT T.4 or T.6 options.
     */
    public CCITTFaxDecoderStream(final InputStream stream, final int columns, final int type, final int fillOrder,
                                 final long options) {
        this(stream, columns, type, fillOrder, options, type == TIFFBaseline.COMPRESSION_CCITT_MODIFIED_HUFFMAN_RLE);
    }

    static int findCompressionType(final int type, final InputStream in) throws IOException {
        // Discover possible incorrect type, revert to RLE
        if (type == TIFFExtension.COMPRESSION_CCITT_T4 && in.markSupported()) {
            byte[] streamData = new byte[32];

            try {
                in.mark(streamData.length);

                int offset = 0;
                while (offset < streamData.length) {
                    int read = in.read(streamData, offset, streamData.length - offset);
                    if (read <= 0) {
                        break;
                    }

                    offset += read;
                }
            }
            finally {
                in.reset();
            }

            if (streamData[0] != 0 || (streamData[1] >> 4 != 1 && streamData[1] != 1)) {
                // Leading EOL (0b000000000001) not found, search further and try RLE if not found
                int numBits = streamData.length * 8;
                short b = (short) (((streamData[0] << 8) + streamData[1]) >> 4);
                for (int i = 12; i < numBits; i++) {
                    b = (short) ((b << 1) + ((streamData[(i / 8)] >> (7 - (i % 8))) & 0x01));

                    if ((b & 0xFFF) == 1) {
                        return TIFFExtension.COMPRESSION_CCITT_T4;
                    }
                }

                return TIFFBaseline.COMPRESSION_CCITT_MODIFIED_HUFFMAN_RLE;
            }
        }

        return type;
    }

    private void fetch() throws IOException {
        if (decodedPos >= decodedLength) {
            decodedLength = 0;

            try {
                decodeRow();
            }
            catch (EOFException e) {
                // TODO: Rewrite to avoid throw/catch for normal flow...
                if (decodedLength != 0) {
                    throw e;
                }<commit_msg>#579 Deeper EOL search in the CCITT stream<commit_after>/*
 * Copyright (c) 2012, <NAME>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 *   list of conditions and the following disclaimer.
 *
 * * Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 *
 * * Neither the name of the copyright holder nor the names of its
 *   contributors may be used to endorse or promote products derived from
 *   this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package com.twelvemonkeys.imageio.plugins.tiff;

import com.twelvemonkeys.lang.Validate;

import java.io.EOFException;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;

/**
 * CCITT Modified Huffman RLE, Group 3 (T4) and Group 4 (T6) fax compression.
 *
 * @author <a href="mailto:<EMAIL>"><NAME></a>
 * @author <a href="https://github.com/Schmidor"><NAME></a>
 * @author last modified by $Author: haraldk$
 * @version $Id: CCITTFaxDecoderStream.java,v 1.0 23.05.12 15:55 haraldk Exp$
 */
final class CCITTFaxDecoderStream extends FilterInputStream {
    // See TIFF 6.0 Specification, Section 10: "Modified Huffman Compression", page 43.

    private final int columns;
    private final byte[] decodedRow;

    private final boolean optionG32D;
    // Leading zeros for aligning EOL
    private final boolean optionG3Fill;
    private final boolean optionUncompressed;
    private final boolean optionByteAligned;

    // Need to take fill order into account (?) (use flip table?)
    private final int fillOrder;
    private final int type;

    private int decodedLength;
    private int decodedPos;

    private int[] changesReferenceRow;
    private int[] changesCurrentRow;
    private int changesReferenceRowCount;
    private int changesCurrentRowCount;

    private int lastChangingElement = 0;

    /**
     * Creates a CCITTFaxDecoderStream.
     * This constructor may be used for CCITT streams embedded in PDF files,
     * which use EncodedByteAlign.
     *
     * @param stream the compressed CCITT stream.
     * @param columns the number of columns in the stream.
     * @param type the type of stream, must be one of {@code COMPRESSION_CCITT_MODIFIED_HUFFMAN_RLE},
     *             {@code COMPRESSION_CCITT_T4} or {@code COMPRESSION_CCITT_T6}.
     * @param fillOrder fillOrder, must be {@code FILL_LEFT_TO_RIGHT} or
     * {@code FILL_RIGHT_TO_LEFT}.
     * @param options CCITT T.4 or T.6 options.
     * @param byteAligned enable byte alignment used in PDF files (EncodedByteAlign).
     */
    public CCITTFaxDecoderStream(final InputStream stream, final int columns, final int type, final int fillOrder,
                                 final long options, final boolean byteAligned) {
        super(Validate.notNull(stream, "stream"));

        this.columns = Validate.isTrue(columns > 0, columns, "width must be greater than 0");
        this.type = Validate.isTrue(type == TIFFBaseline.COMPRESSION_CCITT_MODIFIED_HUFFMAN_RLE ||
                        type == TIFFExtension.COMPRESSION_CCITT_T4 ||
                        type == TIFFExtension.COMPRESSION_CCITT_T6,
                type, "Only CCITT Modified Huffman RLE compression (2), CCITT T4 (3) or CCITT T6 (4) supported: %s");
        this.fillOrder = Validate.isTrue(
                fillOrder == TIFFBaseline.FILL_LEFT_TO_RIGHT || fillOrder == TIFFExtension.FILL_RIGHT_TO_LEFT,
                fillOrder, "Expected fill order 1  or 2: %s"
        );

        // We know this is only used for b/w (1 bit)
        decodedRow = new byte[(columns + 7) / 8];
        changesReferenceRow = new int[columns + 2];
        changesCurrentRow = new int[columns + 2];

        switch (type) {
            case TIFFBaseline.COMPRESSION_CCITT_MODIFIED_HUFFMAN_RLE:
                optionByteAligned = byteAligned;
                optionG32D = false;
                optionG3Fill = false;
                optionUncompressed = false;
                break;
            case TIFFExtension.COMPRESSION_CCITT_T4:
                optionByteAligned = byteAligned;
                optionG32D = (options & TIFFExtension.GROUP3OPT_2DENCODING) != 0;
                optionG3Fill = (options & TIFFExtension.GROUP3OPT_FILLBITS) != 0;
                optionUncompressed = (options & TIFFExtension.GROUP3OPT_UNCOMPRESSED) != 0;
                break;
            case TIFFExtension.COMPRESSION_CCITT_T6:
                optionByteAligned = byteAligned;
                optionG32D = false;
                optionG3Fill = false;
                optionUncompressed = (options & TIFFExtension.GROUP4OPT_UNCOMPRESSED) != 0;
                break;
            default:
                // Guarded above
                throw new AssertionError();
        }

        Validate.isTrue(!optionUncompressed, optionUncompressed,
                "CCITT GROUP 3/4 OPTION UNCOMPRESSED is not supported");
    }

    /**
     * Creates a CCITTFaxDecoderStream.
     *
     * @param stream the compressed CCITT stream.
     * @param columns the number of columns in the stream.
     * @param type the type of stream, must be one of {@code COMPRESSION_CCITT_MODIFIED_HUFFMAN_RLE},
     *             {@code COMPRESSION_CCITT_T4} or {@code COMPRESSION_CCITT_T6}.
     * @param fillOrder fillOrder, must be {@code FILL_LEFT_TO_RIGHT} or
     * {@code FILL_RIGHT_TO_LEFT}.
     * @param options CCITT T.4 or T.6 options.
     */
    public CCITTFaxDecoderStream(final InputStream stream, final int columns, final int type, final int fillOrder,
                                 final long options) {
        this(stream, columns, type, fillOrder, options, type == TIFFBaseline.COMPRESSION_CCITT_MODIFIED_HUFFMAN_RLE);
    }

    static int findCompressionType(final int type, final InputStream in) throws IOException {
        // Discover possible incorrect type, revert to RLE
        if (type == TIFFExtension.COMPRESSION_CCITT_T4 && in.markSupported()) {
            int limit = 500;

            try {
                in.mark(limit);

                int first = in.read();
                int second = in.read();
                if (first == -1 || second == -1) {
                    // stream to short
                    return type;
                }
                else if (first == 0 && (((byte) second) >> 4 == 1 || ((byte) second) == 1)) {
                    // correct, starts with EOL or byte aligned EOL
                    return type;
                }
                short b = (short) (((((byte) first) << 8) + ((byte) second)) >> 4);
                int limitBits = limit * 8;
                int read = second;
                byte streamByte = (byte) read;
                for (int i = 12; i < limitBits; i++) {
                    if (i % 8 == 0) {
                        read = in.read();
                        if (read > -1) {
                            streamByte = (byte) read;
                        }
                        else {
                            // no EOL before stream end
                            return TIFFBaseline.COMPRESSION_CCITT_MODIFIED_HUFFMAN_RLE;
                        }
                    }

                    b = (short) ((b << 1) + ((streamByte >> (7 - (i % 8))) & 0x01));

                    if ((b & 0xFFF) == 1) {
                        // found EOL
                        return TIFFExtension.COMPRESSION_CCITT_T4;
                    }
                }
                // no EOL till limit
                return TIFFBaseline.COMPRESSION_CCITT_MODIFIED_HUFFMAN_RLE;
            }
            finally {
                in.reset();
            }
        }

        return type;
    }

    private void fetch() throws IOException {
        if (decodedPos >= decodedLength) {
            decodedLength = 0;

            try {
                decodeRow();
            }
            catch (EOFException e) {
                // TODO: Rewrite to avoid throw/catch for normal flow...
                if (decodedLength != 0) {
                    throw e;
                }
{'fraction_non_alphanumeric': 0.08088893864755933, 'fraction_numerical': 0.012763098969995522, 'mean_word_length': 2.1754354781372203, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 36, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6749095', 'n_tokens_mistral': 5241, 'n_tokens_neox': 4753, 'n_words': 1618}