<commit_before>//
// MessagePack for Java
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//
package org.msgpack.core;

import java.io.Closeable;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.math.BigInteger;
import java.nio.CharBuffer;
import java.nio.channels.ReadableByteChannel;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CoderResult;
import java.nio.charset.CodingErrorAction;

import org.msgpack.core.MessagePack.Code;
import org.msgpack.core.buffer.*;

import static org.msgpack.core.Preconditions.*;


/**
 * MessageUnpacker lets an application read message-packed values from a data stream.
 * The application needs to call {@link #getNextFormat()} followed by an appropriate unpackXXX method according to the the returned format type.
 *
 * <pre>
 * <code>
 *     MessageUnpacker unpacker = new MessageUnpacker(...);
 *     while(unpacker.hasNext()) {
 *         MessageFormat f = unpacker.getNextFormat();
 *         switch(f) {
 *             case MessageFormat.POSFIXINT =>
 *             case MessageFormat.INT8 =>
 *             case MessageFormat.UINT8 => {
 *                int v = unpacker.unpackInt();
 *                break;
 *             }
 *             case MessageFormat.STRING => {
 *                String v = unpacker.unpackString();
 *                break;
 *             }
 *             // ...
  *       }
 *     }
 *
 * </code>
 * </pre>
 */
public class MessageUnpacker implements Closeable {

    private final MessagePack.Config config;

    private final MessageBufferInput in;

    /**
     * Points to the current buffer to read
     */
    private MessageBuffer buffer;
    /**
     * Cursor position in the current buffer
     */
    private int position;

    /**
     * For preserving the next buffer to use
     */
    private MessageBuffer secondaryBuffer = null;

    /**
     * Extra buffer for string data at the buffer boundary. Using 17-byte buffer (for FIXEXT16) is sufficient.
     */
    private final MessageBuffer extraBuffer = MessageBuffer.wrap(new byte[24]);

    /**
     * True if no more data is available from the MessageBufferInput
     */
    private boolean reachedEOF = false;

    /**
     * For decoding String in unpackString.
     */
    private CharsetDecoder decoder;

    /**
     * Buffer for decoding strings
     */
    private CharBuffer decodeBuffer;

    /**
     * Create an MessageUnpacker that reads data from the given byte array.
     *
     * @param arr
     */
    public MessageUnpacker(byte[] arr) {
        this(new ArrayBufferInput(arr));
    }

    /**
     * Create an MessageUnpacker that reads data from the given InputStream.
     * @param in
     */
    public MessageUnpacker(InputStream in) {
        this(new InputStreamBufferInput(in));
    }

    /**
     * Create an MessageUnpacker that reads data from the given ReadableByteChannel.
     * @param in
     */
    public MessageUnpacker(ReadableByteChannel in) {
        this(new ChannelBufferInput(in));
    }

    /**
     * Create an MessageUnpacker that reads data from the given MessageBufferInput
     *
     * @param in
     */
    public MessageUnpacker(MessageBufferInput in) {
        this(in, MessagePack.DEFAULT_CONFIG);
    }


    /**
     * Create an MessageUnpacker
     * @param in
     * @param config configuration
     */
    public MessageUnpacker(MessageBufferInput in, MessagePack.Config config) {
        // Root constructor. All of the constructors must call this constructor.
        this.in = checkNotNull(in, "MessageBufferInput");
        this.config = checkNotNull(config, "Config");
    }

    private void prepareDecoder() {
        if(decoder == null) {
            decodeBuffer = CharBuffer.allocate(config.STRING_DECODER_BUFFER_SIZE);
            decoder = MessagePack.UTF8.newDecoder()
                    .onMalformedInput(config.MALFORMED_INPUT_ACTION)
                    .onUnmappableCharacter(config.UNMAPPABLE_CHARACTER_ACTION);
        }
    }


    /**
     * Relocate the cursor position so that it points to the real buffer
     *
     * @return true if it succeeds to move the cursor, or false if there is no more buffer to read
     * @throws IOException when failed to retrieve next buffer
     */
    private boolean ensureBuffer() throws IOException {
        if(buffer == null)
            buffer = takeNextBuffer();

        while(buffer != null && position >= buffer.size()) {
            // Fetch the next buffer
            position -= buffer.size();
            buffer = takeNextBuffer();
        }
        return buffer != null;
    }

    private MessageBuffer takeNextBuffer() throws IOException {
        if(reachedEOF)
            return null;

        MessageBuffer nextBuffer = null;
        if(secondaryBuffer == null) {
            nextBuffer = in.next();
        }
        else {
            nextBuffer = secondaryBuffer;
            secondaryBuffer = null;
        }

        if(nextBuffer == null) {
            reachedEOF = true;
        }
        return nextBuffer;
    }


    /**
     * Ensure that the buffer has the data of at least the specified size.
     *
     * @param byteSizeToRead the data size to be read
     * @return if the buffer can have the data of the specified size returns true, or if the input source reached an EOF, it returns false.
     * @throws IOException
     */
    private boolean ensure(int byteSizeToRead) throws IOException {
        if(!ensureBuffer())
            return false;

        // The buffer contains the data
        if(position + byteSizeToRead <= buffer.size()) {
            // OK
            return true;
        }

        // When the data is at the boundary
        /*
             |---(byte size to read) ----|
             -- current buffer --|
             |--- extra buffer (slice) --|----|
                                 |-------|---------- secondary buffer (slice) ----------------|

             */

        // If the byte size to read fits within the extra buffer, use the extraBuffer
        MessageBuffer newBuffer = byteSizeToRead <= extraBuffer.size() ? extraBuffer : MessageBuffer.newBuffer(byteSizeToRead);

        // Copy the remaining buffer contents to the new buffer
        int firstHalfSize = buffer.size() - position;
        if(firstHalfSize > 0)
            buffer.copyTo(position, newBuffer, 0, firstHalfSize);

        // Read the last half contents from the next buffers
        int cursor = firstHalfSize;
        while(cursor < byteSizeToRead) {
            secondaryBuffer = takeNextBuffer();
            if(secondaryBuffer == null)
                return false; // No more buffer to read

            // Copy the contents from the secondary buffer to the new buffer
            int copyLen = Math.min(byteSizeToRead - cursor, secondaryBuffer.size());
            secondaryBuffer.copyTo(0, newBuffer, cursor, copyLen);

            // Truncate the copied part from the secondaryBuffer
            secondaryBuffer = copyLen == secondaryBuffer.size() ? null : secondaryBuffer.slice(copyLen, secondaryBuffer.size()-copyLen);
            cursor += copyLen;
        }

        // Replace the current buffer with the new buffer
        buffer = byteSizeToRead == newBuffer.size() ? newBuffer : newBuffer.slice(0, byteSizeToRead);
        position = 0;

        return true;
    }

    /**
     * Returns true true if this unpacker has more elements.
     * When this returns true, subsequent call to {@link #getNextFormat()} returns an
     * MessageFormat instance. If false, {@link #getNextFormat()} will throw an EOFException.
     *
     * @return true if this unpacker has more elements to read
     */
    public boolean hasNext() throws IOException {
        return ensure(1);
    }

    /**
     * Returns the next MessageFormat type. This method should be called after {@link #hasNext()} returns true.
     * If {@link #hasNext()} returns false, calling this method throws {@link java.io.EOFException}.
     *
     * This method does not proceed the internal cursor.
     * @return the next MessageFormat
     * @throws IOException when failed to read the input data.
     * @throws EOFException when the end of file reached, i.e. {@link #hasNext()} == false.
     */
    public MessageFormat getNextFormat() throws IOException {
        byte b = lookAhead();
        return MessageFormat.valueOf(b);
    }

    /**
     * Look-ahead a byte value at the current cursor position.
     * This method does not proceed the cursor.
     *
     * @return
     * @throws IOException
     */
    private byte lookAhead() throws IOException {
        if(ensure(1))
            return buffer.getByte(position);
        else
            throw new EOFException();
    }


    /**
     * Get the head byte value and proceeds the cursor by 1
     */
    private byte consume() throws IOException {
        byte b = lookAhead();
        position += 1;
        return b;
    }

    /**
     * Proceeds the cursor by the specified byte length
     */
    private void consume(int numBytes) throws IOException {
        assert(numBytes >= 0);
        // If position + numBytes becomes negative, it indicates an overflow from Integer.MAX_VALUE.
        if(position + numBytes < 0)
            ensureBuffer();
        position += numBytes;
     }

    /**
     * Read a byte value at the cursor and proceed the cursor.
     *
     * @return
     * @throws IOException
     */
    private byte readByte() throws IOException {
        if(!ensure(1)) {
            throw new MessageFormatException("insufficient data length for reading byte value");
        }
        byte b = buffer.getByte(position);
        consume(1);
        return b;
    }

    private short readShort() throws IOException {
        if(!ensure(2)) {
            throw new MessageFormatException("insufficient data length for reading short value");
        }
        short s = buffer.getShort(position);
        consume(2);
        return s;
    }

    private int readInt() throws IOException {
        if(!ensure(4)) {
            throw new MessageFormatException("insufficient data length for reading int value");
        }
        int i = buffer.getInt(position);
        consume(4);
        return i;
    }

    private float readFloat() throws IOException {
        if(!ensure(4)) {
            throw new MessageFormatException("insufficient data length for reading float value");
        }
        float f = buffer.getFloat(position);
        consume(4);
        return f;
    }

    private long readLong() throws IOException {
        if(!ensure(8)) {
            throw new MessageFormatException("insufficient data length for reading long value");
        }
        long l = buffer.getLong(position);
        consume(8);
        return l;
    }

    private double readDouble() throws IOException {
        if(!ensure(8)) {
            throw new MessageFormatException("insufficient data length for reading double value");
        }
        double d = buffer.getDouble(position);
        consume(8);
        return d;
    }


    /**
     * Skip reading the specified number of bytes. Use this method only if you know skipping data is safe.
     * For simply skip the next value, use {@link #skipValue()}.<commit_msg>fix comment<commit_after>//
// MessagePack for Java
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//
package org.msgpack.core;

import java.io.Closeable;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.math.BigInteger;
import java.nio.CharBuffer;
import java.nio.channels.ReadableByteChannel;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CoderResult;
import java.nio.charset.CodingErrorAction;

import org.msgpack.core.MessagePack.Code;
import org.msgpack.core.buffer.*;

import static org.msgpack.core.Preconditions.*;


/**
 * MessageUnpacker lets an application read message-packed values from a data stream.
 * The application needs to call {@link #getNextFormat()} followed by an appropriate unpackXXX method according to the the returned format type.
 *
 * <pre>
 * <code>
 *     MessageUnpacker unpacker = new MessageUnpacker(...);
 *     while(unpacker.hasNext()) {
 *         MessageFormat f = unpacker.getNextFormat();
 *         switch(f) {
 *             case MessageFormat.POSFIXINT =>
 *             case MessageFormat.INT8 =>
 *             case MessageFormat.UINT8 => {
 *                int v = unpacker.unpackInt();
 *                break;
 *             }
 *             case MessageFormat.STRING => {
 *                String v = unpacker.unpackString();
 *                break;
 *             }
 *             // ...
  *       }
 *     }
 *
 * </code>
 * </pre>
 */
public class MessageUnpacker implements Closeable {

    private final MessagePack.Config config;

    private final MessageBufferInput in;

    /**
     * Points to the current buffer to read
     */
    private MessageBuffer buffer;
    /**
     * Cursor position in the current buffer
     */
    private int position;

    /**
     * For preserving the next buffer to use
     */
    private MessageBuffer secondaryBuffer = null;

    /**
     * Extra buffer for string data at the buffer boundary. Using 17-byte buffer (for FIXEXT16) is sufficient.
     */
    private final MessageBuffer extraBuffer = MessageBuffer.wrap(new byte[24]);

    /**
     * True if no more data is available from the MessageBufferInput
     */
    private boolean reachedEOF = false;

    /**
     * For decoding String in unpackString.
     */
    private CharsetDecoder decoder;

    /**
     * Buffer for decoding strings
     */
    private CharBuffer decodeBuffer;

    /**
     * Create an MessageUnpacker that reads data from the given byte array.
     *
     * @param arr
     */
    public MessageUnpacker(byte[] arr) {
        this(new ArrayBufferInput(arr));
    }

    /**
     * Create an MessageUnpacker that reads data from the given InputStream.
     * @param in
     */
    public MessageUnpacker(InputStream in) {
        this(new InputStreamBufferInput(in));
    }

    /**
     * Create an MessageUnpacker that reads data from the given ReadableByteChannel.
     * @param in
     */
    public MessageUnpacker(ReadableByteChannel in) {
        this(new ChannelBufferInput(in));
    }

    /**
     * Create an MessageUnpacker that reads data from the given MessageBufferInput
     *
     * @param in
     */
    public MessageUnpacker(MessageBufferInput in) {
        this(in, MessagePack.DEFAULT_CONFIG);
    }


    /**
     * Create an MessageUnpacker
     * @param in
     * @param config configuration
     */
    public MessageUnpacker(MessageBufferInput in, MessagePack.Config config) {
        // Root constructor. All of the constructors must call this constructor.
        this.in = checkNotNull(in, "MessageBufferInput");
        this.config = checkNotNull(config, "Config");
    }

    private void prepareDecoder() {
        if(decoder == null) {
            decodeBuffer = CharBuffer.allocate(config.STRING_DECODER_BUFFER_SIZE);
            decoder = MessagePack.UTF8.newDecoder()
                    .onMalformedInput(config.MALFORMED_INPUT_ACTION)
                    .onUnmappableCharacter(config.UNMAPPABLE_CHARACTER_ACTION);
        }
    }


    /**
     * Relocate the cursor position so that it points to the real buffer
     *
     * @return true if it succeeds to move the cursor, or false if there is no more buffer to read
     * @throws IOException when failed to retrieve next buffer
     */
    private boolean ensureBuffer() throws IOException {
        if(buffer == null)
            buffer = takeNextBuffer();

        while(buffer != null && position >= buffer.size()) {
            // Fetch the next buffer
            position -= buffer.size();
            buffer = takeNextBuffer();
        }
        return buffer != null;
    }

    private MessageBuffer takeNextBuffer() throws IOException {
        if(reachedEOF)
            return null;

        MessageBuffer nextBuffer = null;
        if(secondaryBuffer == null) {
            nextBuffer = in.next();
        }
        else {
            nextBuffer = secondaryBuffer;
            secondaryBuffer = null;
        }

        if(nextBuffer == null) {
            reachedEOF = true;
        }
        return nextBuffer;
    }


    /**
     * Ensure that the buffer has the data of at least the specified size.
     *
     * @param byteSizeToRead the data size to be read
     * @return if the buffer can have the data of the specified size returns true, or if the input source reached an EOF, it returns false.
     * @throws IOException
     */
    private boolean ensure(int byteSizeToRead) throws IOException {
        if(!ensureBuffer())
            return false;

        // The buffer contains the data
        if(position + byteSizeToRead <= buffer.size()) {
            // OK
            return true;
        }

        // When the data is at the boundary
        /*
             |---(byte size to read) ----|
             -- current buffer --|
             |--- extra buffer (slice) --|----|
                                 |-------|---------- secondary buffer (slice) ----------------|

             */

        // If the byte size to read fits within the extra buffer, use the extraBuffer
        MessageBuffer newBuffer = byteSizeToRead <= extraBuffer.size() ? extraBuffer : MessageBuffer.newBuffer(byteSizeToRead);

        // Copy the remaining buffer contents to the new buffer
        int firstHalfSize = buffer.size() - position;
        if(firstHalfSize > 0)
            buffer.copyTo(position, newBuffer, 0, firstHalfSize);

        // Read the last half contents from the next buffers
        int cursor = firstHalfSize;
        while(cursor < byteSizeToRead) {
            secondaryBuffer = takeNextBuffer();
            if(secondaryBuffer == null)
                return false; // No more buffer to read

            // Copy the contents from the secondary buffer to the new buffer
            int copyLen = Math.min(byteSizeToRead - cursor, secondaryBuffer.size());
            secondaryBuffer.copyTo(0, newBuffer, cursor, copyLen);

            // Truncate the copied part from the secondaryBuffer
            secondaryBuffer = copyLen == secondaryBuffer.size() ? null : secondaryBuffer.slice(copyLen, secondaryBuffer.size()-copyLen);
            cursor += copyLen;
        }

        // Replace the current buffer with the new buffer
        buffer = byteSizeToRead == newBuffer.size() ? newBuffer : newBuffer.slice(0, byteSizeToRead);
        position = 0;

        return true;
    }

    /**
     * Returns true true if this unpacker has more elements.
     * When this returns true, subsequent call to {@link #getNextFormat()} returns an
     * MessageFormat instance. If false, {@link #getNextFormat()} will throw an EOFException.
     *
     * @return true if this unpacker has more elements to read
     */
    public boolean hasNext() throws IOException {
        return ensure(1);
    }

    /**
     * Returns the next MessageFormat type. This method should be called after {@link #hasNext()} returns true.
     * If {@link #hasNext()} returns false, calling this method throws {@link java.io.EOFException}.
     *
     * This method does not proceed the internal cursor.
     * @return the next MessageFormat
     * @throws IOException when failed to read the input data.
     * @throws EOFException when the end of file reached, i.e. {@link #hasNext()} == false.
     */
    public MessageFormat getNextFormat() throws IOException {
        byte b = lookAhead();
        return MessageFormat.valueOf(b);
    }

    /**
     * Look-ahead a byte value at the current cursor position.
     * This method does not proceed the cursor.
     *
     * @return
     * @throws IOException
     */
    private byte lookAhead() throws IOException {
        if(ensure(1))
            return buffer.getByte(position);
        else
            throw new EOFException();
    }


    /**
     * Get the head byte value and proceeds the cursor by 1
     */
    private byte consume() throws IOException {
        byte b = lookAhead();
        position += 1;
        return b;
    }

    /**
     * Proceeds the cursor by the specified byte length
     */
    private void consume(int numBytes) throws IOException {
        assert(numBytes >= 0);
        // If position + numBytes becomes negative, it indicates an overflow from Integer.MAX_VALUE.
        if(position + numBytes < 0)
            ensureBuffer();
        position += numBytes;
     }

    /**
     * Read a byte value at the cursor and proceed the cursor.
     *
     * @return
     * @throws IOException
     */
    private byte readByte() throws IOException {
        if(!ensure(1)) {
            throw new MessageFormatException("insufficient data length for reading byte value");
        }
        byte b = buffer.getByte(position);
        consume(1);
        return b;
    }

    private short readShort() throws IOException {
        if(!ensure(2)) {
            throw new MessageFormatException("insufficient data length for reading short value");
        }
        short s = buffer.getShort(position);
        consume(2);
        return s;
    }

    private int readInt() throws IOException {
        if(!ensure(4)) {
            throw new MessageFormatException("insufficient data length for reading int value");
        }
        int i = buffer.getInt(position);
        consume(4);
        return i;
    }

    private float readFloat() throws IOException {
        if(!ensure(4)) {
            throw new MessageFormatException("insufficient data length for reading float value");
        }
        float f = buffer.getFloat(position);
        consume(4);
        return f;
    }

    private long readLong() throws IOException {
        if(!ensure(8)) {
            throw new MessageFormatException("insufficient data length for reading long value");
        }
        long l = buffer.getLong(position);
        consume(8);
        return l;
    }

    private double readDouble() throws IOException {
        if(!ensure(8)) {
            throw new MessageFormatException("insufficient data length for reading double value");
        }
        double d = buffer.getDouble(position);
        consume(8);
        return d;
    }


    /**
     * Skip reading the specified number of bytes. Use this method only if you know skipping data is safe.
     * For simply skipping the next value, use {@link #skipValue()}.
{'fraction_non_alphanumeric': 0.09252399393837346, 'fraction_numerical': 0.0030308132682269743, 'mean_word_length': 2.1424603174603174, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3402133', 'n_tokens_mistral': 6129, 'n_tokens_neox': 5981, 'n_words': 2082}