<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.pdfbox.cos;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.Closeable;
import java.io.File;
import java.io.InputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.pdfbox.filter.DecodeResult;
import org.apache.pdfbox.filter.Filter;
import org.apache.pdfbox.filter.FilterFactory;
import org.apache.pdfbox.io.IOUtils;
import org.apache.pdfbox.io.RandomAccess;
import org.apache.pdfbox.io.RandomAccessBuffer;
import org.apache.pdfbox.io.RandomAccessFile;
import org.apache.pdfbox.io.RandomAccessFileInputStream;
import org.apache.pdfbox.io.RandomAccessFileOutputStream;
import org.apache.pdfbox.pdfparser.PDFStreamParser;

/**
 * This class represents a stream object in a PDF document.
 *
 * @author <NAME>
 */
public class COSStream extends COSDictionary implements Closeable
{
    /**
     * Log instance.
     */
    private static final Log LOG = LogFactory.getLog(COSStream.class);

    private static final int BUFFER_SIZE=16384;

    /**
     * internal buffer, either held in memory or within a scratch file.
     */
    private RandomAccess buffer;
    /**
     * The stream with all of the filters applied.
     */
    private RandomAccessFileOutputStream filteredStream;

    /**
     * The stream with no filters, this contains the useful data.
     */
    private RandomAccessFileOutputStream unFilteredStream;
    private DecodeResult decodeResult;

    /**
     * Constructor.  Creates a new stream with an empty dictionary.
     *
     */
    public COSStream( )
    {
        this(false, null);
    }

    /**
     * Constructor.
     *
     * @param dictionary The dictionary that is associated with this stream.
     * 
     */
    public COSStream( COSDictionary dictionary )
    {
        this(dictionary, false, null);
    }

    /**
     * Constructor.  Creates a new stream with an empty dictionary.
     * 
     * @param useScratchFiles enables the usage of a scratch file if set to true
     * @param scratchDirectory directory to be used to create the scratch file. If null java.io.temp is used instead.
     *     
     */
    public COSStream( boolean useScratchFiles, File scratchDirectory )
    {
        super();
        if (useScratchFiles)
        {
            createScratchFile(scratchDirectory);
        }
        if (buffer == null)
        {
            buffer = new RandomAccessBuffer();
        }
    }

    /**
     * Constructor.
     *
     * @param dictionary The dictionary that is associated with this stream.
     * @param useScratchFiles enables the usage of a scratch file if set to true
     * @param scratchDirectory directory to be used to create the scratch file. If null java.io.temp is used instead.
     * 
     */
    public COSStream( COSDictionary dictionary, boolean useScratchFiles, File scratchDirectory  )
    {
        super( dictionary );
        if (useScratchFiles)
        {
            createScratchFile(scratchDirectory);
        }
        if (buffer == null)
        {
            buffer = new RandomAccessBuffer();
        }
    }

    /**
     * Create a scratch file to be used as buffer to decrease memory foot print.
     * 
     * @param scratchDirectory directory to be used to create the scratch file. If null java.io.temp is used instead.
     * 
     */
    private void createScratchFile(File scratchDirectory)
    {
        try 
        {
            File scratchFile = File.createTempFile("PDFBox", null, scratchDirectory);
            // mark scratch file to deleted automatically after usage
            scratchFile.deleteOnExit();
            buffer = new RandomAccessFile(scratchFile, "rw");
        }
        catch (IOException exception)
        {
            LOG.error("Can't create temp file, using memory buffer instead", exception);
        }
    }

    /**
     * This will get all the tokens in the stream.
     *
     * @return All of the tokens in the stream.
     *
     * @throws IOException If there is an error parsing the stream.
     */
    public List<Object> getStreamTokens() throws IOException
    {
        PDFStreamParser parser = new PDFStreamParser( this );
        parser.parse();
        return parser.getTokens();
    }

    /**
     * This will get the stream with all of the filters applied.
     *
     * @return the bytes of the physical (encoded) stream
     *
     * @throws IOException when encoding/decoding causes an exception
     */
    public InputStream getFilteredStream() throws IOException
    {
        if( filteredStream == null )
        {
            doEncode();
        }
        long position = filteredStream.getPosition();
        long length = filteredStream.getLengthWritten();

        RandomAccessFileInputStream input =
            new RandomAccessFileInputStream( buffer, position, length );
        return new BufferedInputStream( input, BUFFER_SIZE );
    }

    /**
     * This will get the length of the encoded stream.
     * 
     * @return the length of the encoded stream as long
     *
     * @throws IOException 
     */
    public long getFilteredLength() throws IOException
    {
        if (filteredStream == null)
        {
            doEncode();
        }
        return filteredStream.getLength();
    }
    
    /**
     * This will set the expected length of the encoded stream. Call this method
     * if the previously set expected length is wrong, to avoid further trouble.
     * 
     * @param length the expected length of the encoded stream.
     */
    public void setFilteredLength(long length)
    {
        filteredStream.setExpectedLength(COSInteger.get(length));
    }

    /**
     * This will get the length of the data written in the encoded stream.
     *
     * @return the length of the data written in the encoded stream as long
     *
     * @throws IOException
     */
    public long getFilteredLengthWritten() throws IOException
    {
        if (filteredStream == null)
        {
            doEncode();
        }
        return filteredStream.getLengthWritten();
    }
    

    /**
     * This will get the logical content stream with none of the filters.
     *
     * @return the bytes of the logical (decoded) stream
     *
     * @throws IOException when encoding/decoding causes an exception
     */
    public InputStream getUnfilteredStream() throws IOException
    {
        InputStream retval;
        if( unFilteredStream == null )
        {
            doDecode();
        }

        //if unFilteredStream is still null then this stream has not been
        //created yet, so we should return null.
        if( unFilteredStream != null )
        {
            long position = unFilteredStream.getPosition();
            long length = unFilteredStream.getLengthWritten();
            RandomAccessFileInputStream input =
                new RandomAccessFileInputStream( buffer, position, length );
            retval = new BufferedInputStream( input, BUFFER_SIZE );
        }
        else
        {
            // We should check if the COSStream contains data, maybe it
            // has been created with a RandomAccessFile - which is not
            // necessary empty.
            // In this case, the creation was been done as an input, this should
            // be the unfiltered file, since no filter has been applied yet.
//            if ( (file != null) &&
//                    (file.length() > 0) )
//            {
//                retval = new RandomAccessFileInputStream( file,
//                                                          0,
//                                                          file.length() );
//            }
//            else
//            {
                //if there is no stream data then simply return an empty stream.
                retval = new ByteArrayInputStream( new byte[0] );
//            }
        }
        return retval;
    }

    /**
     * Returns the repaired stream parameters dictionary.
     *
     * @return the repaired stream parameters dictionary
     * @throws IOException when encoding/decoding causes an exception
     */
    public DecodeResult getDecodeResult() throws IOException
    {
        if (unFilteredStream == null)
        {
            doDecode();
        }

        if (unFilteredStream == null || decodeResult == null)
        {
            String filterInfo = "";
            COSBase filters = getFilters();
            if (filters != null)
            {
                filterInfo = " - filter: ";
                if (filters instanceof COSName)
                {
                    filterInfo += ((COSName) filters).getName();
                }
                else if (filters instanceof COSArray)
                {
                    COSArray filterArray = (COSArray) filters;
                    for (int i = 0; i < filterArray.size(); i++)
                    {
                        if (filterArray.size() > 1)
                        {
                            filterInfo += ", ";
                        }
                        filterInfo += ((COSName) filterArray.get(i)).getName();
                    }
                }
            }
            throw new IOException("Stream was not read" + filterInfo);
        }
        else
        {
            return decodeResult;
        }
    }

    @Override
    public Object accept(ICOSVisitor visitor) throws IOException
    {
        return visitor.visitFromStream(this);
    }

    /**
     * This will decode the physical byte stream applying all of the filters to the stream.
     *
     * @throws IOException If there is an error applying a filter to the stream.
     */
    private void doDecode() throws IOException
    {
// FIXME: We shouldn't keep the same reference?
        unFilteredStream = filteredStream;

        COSBase filters = getFilters();<commit_msg>PDFBOX-2535: mention subtype in COSStream IOException, as suggested by <NAME><commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.pdfbox.cos;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.Closeable;
import java.io.File;
import java.io.InputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.pdfbox.filter.DecodeResult;
import org.apache.pdfbox.filter.Filter;
import org.apache.pdfbox.filter.FilterFactory;
import org.apache.pdfbox.io.IOUtils;
import org.apache.pdfbox.io.RandomAccess;
import org.apache.pdfbox.io.RandomAccessBuffer;
import org.apache.pdfbox.io.RandomAccessFile;
import org.apache.pdfbox.io.RandomAccessFileInputStream;
import org.apache.pdfbox.io.RandomAccessFileOutputStream;
import org.apache.pdfbox.pdfparser.PDFStreamParser;

/**
 * This class represents a stream object in a PDF document.
 *
 * @author <NAME>
 */
public class COSStream extends COSDictionary implements Closeable
{
    /**
     * Log instance.
     */
    private static final Log LOG = LogFactory.getLog(COSStream.class);

    private static final int BUFFER_SIZE=16384;

    /**
     * internal buffer, either held in memory or within a scratch file.
     */
    private RandomAccess buffer;
    /**
     * The stream with all of the filters applied.
     */
    private RandomAccessFileOutputStream filteredStream;

    /**
     * The stream with no filters, this contains the useful data.
     */
    private RandomAccessFileOutputStream unFilteredStream;
    private DecodeResult decodeResult;

    /**
     * Constructor.  Creates a new stream with an empty dictionary.
     *
     */
    public COSStream( )
    {
        this(false, null);
    }

    /**
     * Constructor.
     *
     * @param dictionary The dictionary that is associated with this stream.
     * 
     */
    public COSStream( COSDictionary dictionary )
    {
        this(dictionary, false, null);
    }

    /**
     * Constructor.  Creates a new stream with an empty dictionary.
     * 
     * @param useScratchFiles enables the usage of a scratch file if set to true
     * @param scratchDirectory directory to be used to create the scratch file. If null java.io.temp is used instead.
     *     
     */
    public COSStream( boolean useScratchFiles, File scratchDirectory )
    {
        super();
        if (useScratchFiles)
        {
            createScratchFile(scratchDirectory);
        }
        if (buffer == null)
        {
            buffer = new RandomAccessBuffer();
        }
    }

    /**
     * Constructor.
     *
     * @param dictionary The dictionary that is associated with this stream.
     * @param useScratchFiles enables the usage of a scratch file if set to true
     * @param scratchDirectory directory to be used to create the scratch file. If null java.io.temp is used instead.
     * 
     */
    public COSStream( COSDictionary dictionary, boolean useScratchFiles, File scratchDirectory  )
    {
        super( dictionary );
        if (useScratchFiles)
        {
            createScratchFile(scratchDirectory);
        }
        if (buffer == null)
        {
            buffer = new RandomAccessBuffer();
        }
    }

    /**
     * Create a scratch file to be used as buffer to decrease memory foot print.
     * 
     * @param scratchDirectory directory to be used to create the scratch file. If null java.io.temp is used instead.
     * 
     */
    private void createScratchFile(File scratchDirectory)
    {
        try 
        {
            File scratchFile = File.createTempFile("PDFBox", null, scratchDirectory);
            // mark scratch file to deleted automatically after usage
            scratchFile.deleteOnExit();
            buffer = new RandomAccessFile(scratchFile, "rw");
        }
        catch (IOException exception)
        {
            LOG.error("Can't create temp file, using memory buffer instead", exception);
        }
    }

    /**
     * This will get all the tokens in the stream.
     *
     * @return All of the tokens in the stream.
     *
     * @throws IOException If there is an error parsing the stream.
     */
    public List<Object> getStreamTokens() throws IOException
    {
        PDFStreamParser parser = new PDFStreamParser( this );
        parser.parse();
        return parser.getTokens();
    }

    /**
     * This will get the stream with all of the filters applied.
     *
     * @return the bytes of the physical (encoded) stream
     *
     * @throws IOException when encoding/decoding causes an exception
     */
    public InputStream getFilteredStream() throws IOException
    {
        if( filteredStream == null )
        {
            doEncode();
        }
        long position = filteredStream.getPosition();
        long length = filteredStream.getLengthWritten();

        RandomAccessFileInputStream input =
            new RandomAccessFileInputStream( buffer, position, length );
        return new BufferedInputStream( input, BUFFER_SIZE );
    }

    /**
     * This will get the length of the encoded stream.
     * 
     * @return the length of the encoded stream as long
     *
     * @throws IOException 
     */
    public long getFilteredLength() throws IOException
    {
        if (filteredStream == null)
        {
            doEncode();
        }
        return filteredStream.getLength();
    }
    
    /**
     * This will set the expected length of the encoded stream. Call this method
     * if the previously set expected length is wrong, to avoid further trouble.
     * 
     * @param length the expected length of the encoded stream.
     */
    public void setFilteredLength(long length)
    {
        filteredStream.setExpectedLength(COSInteger.get(length));
    }

    /**
     * This will get the length of the data written in the encoded stream.
     *
     * @return the length of the data written in the encoded stream as long
     *
     * @throws IOException
     */
    public long getFilteredLengthWritten() throws IOException
    {
        if (filteredStream == null)
        {
            doEncode();
        }
        return filteredStream.getLengthWritten();
    }
    

    /**
     * This will get the logical content stream with none of the filters.
     *
     * @return the bytes of the logical (decoded) stream
     *
     * @throws IOException when encoding/decoding causes an exception
     */
    public InputStream getUnfilteredStream() throws IOException
    {
        InputStream retval;
        if( unFilteredStream == null )
        {
            doDecode();
        }

        //if unFilteredStream is still null then this stream has not been
        //created yet, so we should return null.
        if( unFilteredStream != null )
        {
            long position = unFilteredStream.getPosition();
            long length = unFilteredStream.getLengthWritten();
            RandomAccessFileInputStream input =
                new RandomAccessFileInputStream( buffer, position, length );
            retval = new BufferedInputStream( input, BUFFER_SIZE );
        }
        else
        {
            // We should check if the COSStream contains data, maybe it
            // has been created with a RandomAccessFile - which is not
            // necessary empty.
            // In this case, the creation was been done as an input, this should
            // be the unfiltered file, since no filter has been applied yet.
//            if ( (file != null) &&
//                    (file.length() > 0) )
//            {
//                retval = new RandomAccessFileInputStream( file,
//                                                          0,
//                                                          file.length() );
//            }
//            else
//            {
                //if there is no stream data then simply return an empty stream.
                retval = new ByteArrayInputStream( new byte[0] );
//            }
        }
        return retval;
    }

    /**
     * Returns the repaired stream parameters dictionary.
     *
     * @return the repaired stream parameters dictionary
     * @throws IOException when encoding/decoding causes an exception
     */
    public DecodeResult getDecodeResult() throws IOException
    {
        if (unFilteredStream == null)
        {
            doDecode();
        }

        if (unFilteredStream == null || decodeResult == null)
        {
            String filterInfo = "";
            COSBase filters = getFilters();
            if (filters != null)
            {
                filterInfo = " - filter: ";
                if (filters instanceof COSName)
                {
                    filterInfo += ((COSName) filters).getName();
                }
                else if (filters instanceof COSArray)
                {
                    COSArray filterArray = (COSArray) filters;
                    for (int i = 0; i < filterArray.size(); i++)
                    {
                        if (filterArray.size() > 1)
                        {
                            filterInfo += ", ";
                        }
                        filterInfo += ((COSName) filterArray.get(i)).getName();
                    }
                }
            }
            String subtype = getNameAsString(COSName.SUBTYPE);
            throw new IOException(subtype + " stream was not read" + filterInfo);
        }
        else
        {
            return decodeResult;
        }
    }

    @Override
    public Object accept(ICOSVisitor visitor) throws IOException
    {
        return visitor.visitFromStream(this);
    }

    /**
     * This will decode the physical byte stream applying all of the filters to the stream.
     *
     * @throws IOException If there is an error applying a filter to the stream.
     */
    private void doDecode() throws IOException
    {
// FIXME: We shouldn't keep the same reference?
        unFilteredStream = filteredStream;

        COSBase filters = getFilters();
{'fraction_non_alphanumeric': 0.07835820895522388, 'fraction_numerical': 0.0014741109268472453, 'mean_word_length': 1.9524003807969537, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4901394', 'n_tokens_mistral': 5420, 'n_tokens_neox': 5270, 'n_words': 1882}