<commit_before>/*
 * Copyright @ 2015 Atlassian Pty Ltd
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.jitsi.impl.neomedia.codec.video.vp8;

import org.jitsi.impl.neomedia.codec.*;
import org.jitsi.service.neomedia.codec.*;
import org.jitsi.util.*;
import javax.media.*;
import javax.media.format.*;
import java.util.*;
import java.util.concurrent.*;

/**
 * A depacketizer from VP8.
 * See {@link "http://tools.ietf.org/html/draft-ietf-payload-vp8-11"}
 *
 * @author <NAME>
 * @author <NAME>
 */
public class DePacketizer
    extends AbstractCodec2
{
    /**
     * The <tt>Logger</tt> used by the <tt>DePacketizer</tt> class and its
     * instances for logging output.
     */
    private static final Logger logger = Logger.getLogger(DePacketizer.class);

    /**
     * Whether trace logging is enabled.
     */
    private static final boolean TRACE = logger.isTraceEnabled();

    /**
     * A <tt>Comparator</tt> implementation for RTP sequence numbers.
     * Compares <tt>a</tt> and <tt>b</tt>, taking into account the wrap at 2^16.
     *
     * IMPORTANT: This is a valid <tt>Comparator</tt> implementation only if
     * used for subsets of [0, 2^16) which don't span more than 2^15 elements.
     *
     * E.g. it works for: [0, 2^15-1] and ([50000, 2^16) u [0, 10000])
     * Doesn't work for: [0, 2^15] and ([0, 2^15-1] u {2^16-1}) and [0, 2^16)
     */
    private static final Comparator<? super Long> seqNumComparator
            = new Comparator<Long>() {
        @Override
        public int compare(Long a, Long b)
        {
            if (a.equals(b))
                return 0;
            else if (a > b)
            {
                if (a - b < 32768)
                    return 1;
                else
                    return -1;
            }
            else //a < b
            {
                if (b - a < 32768)
                    return -1;
                else
                    return 1;
            }
        }
    };

    /**
     * Stores the RTP payloads (VP8 payload descriptor stripped) from RTP packets
     * belonging to a single VP8 compressed frame.
     */
    private SortedMap<Long, Container> data
            = new TreeMap<Long, Container>(seqNumComparator);

    /**
     * Stores unused <tt>Container</tt>'s.
     */
    private Queue<Container> free = new ArrayBlockingQueue<Container>(100);

    /**
     * Stores the first (earliest) sequence number stored in <tt>data</tt>, or
     * -1 if <tt>data</tt> is empty.
     */
    private long firstSeq = -1;

    /**
     * Stores the last (latest) sequence number stored in <tt>data</tt>, or -1
     * if <tt>data</tt> is empty.
     */
    private long lastSeq = -1;

    /**
     * Stores the value of the <tt>PictureID</tt> field for the VP8 compressed
     * frame, parts of which are currently stored in <tt>data</tt>, or -1 if
     * the <tt>PictureID</tt> field is not in use or <tt>data</tt> is empty.
     */
    private int pictureId = -1;

    /**
     * Stores the RTP timestamp of the packets stored in <tt>data</tt>, or -1
     * if they don't have a timestamp set.
     */
    private long timestamp = -1;

    /**
     * Whether we have stored any packets in <tt>data</tt>. Equivalent to
     * <tt>data.isEmpty()</tt>.
     */
    private boolean empty = true;

    /**
     * Whether we have stored in <tt>data</tt> the last RTP packet of the VP8
     * compressed frame, parts of which are currently stored in <tt>data</tt>.
     */
    private boolean haveEnd = false;

    /**
     * Whether we have stored in <tt>data</tt> the first RTP packet of the VP8
     * compressed frame, parts of which are currently stored in <tt>data</tt>.
     */
    private boolean haveStart = false;

    /**
     * Stores the sum of the lengths of the data stored in <tt>data</tt>, that
     * is the total length of the VP8 compressed frame to be constructed.
     */
    private int frameLength = 0;

    /**
     * The sequence number of the last RTP packet, which was included in the
     * output.
     */
    private long lastSentSeq = -1;

    /**
     * Initializes a new <tt>JNIEncoder</tt> instance.
     */
    public DePacketizer()
    {
        super("VP8 RTP DePacketizer",
                VideoFormat.class,
                new VideoFormat[]{ new VideoFormat(Constants.VP8) });
        inputFormats = new VideoFormat[] { new VideoFormat(Constants.VP8_RTP) };
    }

    /**
     * {@inheritDoc}
     */
    @Override
    protected void doClose()
    {
    }

    /**
     * {@inheritDoc}
     */
    @Override
    protected void doOpen() throws ResourceUnavailableException
    {
        if(logger.isInfoEnabled())
            logger.info("Opened VP8 depacketizer");
    }

    /**
     * Re-initializes the fields which store information about the currently
     * held data. Empties <tt>data</tt>.
     */
    private void reinit()
    {
        firstSeq = lastSeq = timestamp = -1;
        pictureId = -1;
        empty = true;
        haveEnd = haveStart = false;
        frameLength = 0;

        Iterator<Map.Entry<Long,Container>> it = data.entrySet().iterator();
        Map.Entry<Long, Container> e;
        while (it.hasNext())
        {
            e = it.next();
            free.offer(e.getValue());
            it.remove();
        }
    }

    /**
     * Checks whether the currently held VP8 compressed frame is complete (e.g
     * all its packets are stored in <tt>data</tt>).
     * @return <tt>true</tt> if the currently help VP8 compressed frame is
     * complete, <tt>false</tt> otherwise.
     */
    private boolean frameComplete()
    {
        return haveStart && haveEnd && !haveMissing();
    }

    /**
     * Checks whether there are packets with sequence numbers between
     * <tt>firstSeq</tt> and <tt>lastSeq</tt> which are *not* stored in
     * <tt>data</tt>.
     * @return <tt>true</tt> if there are packets with sequence numbers between
     * <tt>firstSeq</tt> and <tt>lastSeq</tt> which are *not* stored in
     * <tt>data</tt>.
     */
    private boolean haveMissing()
    {
        Set<Long> seqs = data.keySet();
        long s = firstSeq;
        while (s != lastSeq)
        {
            if (!seqs.contains(s))
                return true;
            s = (s+1) % (1<<16);
        }
        return false;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    protected int doProcess(Buffer inBuffer, Buffer outBuffer)
    {
        byte[] inData = (byte[])inBuffer.getData();
        int inOffset = inBuffer.getOffset();

        if (!VP8PayloadDescriptor.isValid(inData, inOffset))
        {
            logger.warn("Invalid RTP/VP8 packet discarded.");
            outBuffer.setDiscard(true);
            return BUFFER_PROCESSED_FAILED; //XXX: FAILED or OK?
        }

        long inSeq = inBuffer.getSequenceNumber();
        long inRtpTimestamp = inBuffer.getRtpTimeStamp();
        int inPictureId = VP8PayloadDescriptor.getPictureId(inData, inOffset);
        boolean inMarker = (inBuffer.getFlags() & Buffer.FLAG_RTP_MARKER) != 0;
        boolean inIsStartOfFrame
                = VP8PayloadDescriptor.isStartOfFrame(inData, inOffset);
        int inLength = inBuffer.getLength();
        int inPdSize = VP8PayloadDescriptor.getSize(inData, inOffset);
        int inPayloadLength = inLength - inPdSize;

        if (empty
                && lastSentSeq != -1
                && seqNumComparator.compare(inSeq, lastSentSeq) != 1)
        {
            if (logger.isInfoEnabled())
                logger.info("Discarding old packet (while empty) " + inSeq);
            outBuffer.setDiscard(true);
            return BUFFER_PROCESSED_OK;
        }

        if (!empty)
        {
            // if the incoming packet has a different PictureID or timestamp
            // than those of the current frame, then it belongs to a different
            // frame.
            if ( (inPictureId != -1 && pictureId != -1
                    && inPictureId != pictureId)
                 | (timestamp != -1 && inRtpTimestamp != -1
                    && inRtpTimestamp != timestamp) )
            {
                if (seqNumComparator
                        .compare(inSeq, firstSeq) != 1) //inSeq <= firstSeq
                {
                    // the packet belongs to a previous frame. discard it
                    if (logger.isInfoEnabled())
                        logger.info("Discarding old packet " + inSeq);
                    outBuffer.setDiscard(true);
                    return BUFFER_PROCESSED_OK;
                }
                else //inSeq > firstSeq (and also presumably isSeq > lastSeq)
                {
                    // the packet belongs to a subsequent frame (to the one
                    // currently being held). Drop the current frame.

                    if (logger.isInfoEnabled())
                        logger.info("Discarding saved packets on arrival of" +
                                " a packet for a subsequent frame: " + inSeq);

                    // TODO: this would be the place to complain about the
                    // not-well-received PictureID by sending a RTCP SLI or NACK.
                    reinit();
                }
            }
        }

        // a whole frame in a single packet. avoid the extra copy to
        // this.data and output it immediately.
        if (empty && inMarker && inIsStartOfFrame)
        {
            byte[] outData
                    = validateByteArraySize(outBuffer, inPayloadLength, false);
            System.arraycopy(
                    inData,
                    inOffset + inPdSize,
                    outData,
                    0,
                    inPayloadLength);
            outBuffer.setOffset(0);
            outBuffer.setLength(inPayloadLength);
            outBuffer.setRtpTimeStamp(inBuffer.getRtpTimeStamp());

            if (TRACE)
                logger.trace("Out PictureID=" + inPictureId);

            lastSentSeq = inSeq;

            return BUFFER_PROCESSED_OK;
        }

        // add to this.data
        Container container = free.poll();
        if (container == null)
            container = new Container();
        if (container.buf == null || container.buf.length < inPayloadLength)
            container.buf = new byte[inPayloadLength];

        if (data.get(inSeq) != null)
        {
            if (logger.isInfoEnabled())
                logger.info("(Probable) duplicate packet detected, discarding "
                                    + inSeq);
            outBuffer.setDiscard(true);
            return BUFFER_PROCESSED_OK;
        }

        System.arraycopy(
                inData,
                inOffset + inPdSize,
                container.buf,
                0,
                inPayloadLength);
        container.len = inPayloadLength;
        data.put(inSeq, container);

        // update fields
        frameLength += inPayloadLength;
        if (firstSeq == -1
                || (seqNumComparator.compare(firstSeq, inSeq) == 1))
            firstSeq = inSeq;
        if (lastSeq == -1
                || (seqNumComparator.compare(inSeq, lastSeq) == 1))
            lastSeq = inSeq;

        if (empty)
        {
            // the first received packet for the current frame was just added
            empty = false;
            timestamp = inRtpTimestamp;
            pictureId = inPictureId;
        }

        if (inMarker)
            haveEnd = true;
        if (inIsStartOfFrame)
            haveStart = true;

        // check if we have a full frame
        if (frameComplete())
        {
            byte[] outData
                    = validateByteArraySize(outBuffer, frameLength, false);
            int ptr = 0;
            Container b;
            for (Map.Entry<Long, Container> entry : data.entrySet())
            {
                b = entry.getValue();
                System.arraycopy(
                        b.buf,
                        0,
                        outData,
                        ptr,
                        b.len);
                ptr += b.len;
            }

            outBuffer.setOffset(0);
            outBuffer.setLength(frameLength);
            outBuffer.setRtpTimeStamp(inBuffer.getRtpTimeStamp());

            if (TRACE)
                logger.trace("Out PictureID=" + inPictureId);
            lastSentSeq = lastSeq;

            // prepare for the next frame
            reinit();

            return BUFFER_PROCESSED_OK;
        }
        else
        {
            // frame not complete yet
            outBuffer.setDiscard(true);
            return OUTPUT_BUFFER_NOT_FILLED;
        }
    }

    /**
     */
    public static boolean isKeyFrame(byte[] buff, int off, int len)
    {
        if (buff == null || buff.length == 0 || buff.length < off + len)
        {
            return false;
        }

        // Check if this is the start of a VP8 partition in the payload
        // descriptor.
        if (!DePacketizer.VP8PayloadDescriptor.isValid(buff, off))
        {
            return false;
        }

        if (!DePacketizer.VP8PayloadDescriptor.isStartOfFrame(buff, off))
        {
            return false;
        }

        int szVP8PayloadDescriptor = DePacketizer
            .VP8PayloadDescriptor.getSize(buff, off);

        return DePacketizer.VP8PayloadHeader.isKeyFrame(
                buff, off + szVP8PayloadDescriptor);
    }

    /**<commit_msg>Fixes DePacketizer.isKeyFrame method arguments check.<commit_after>/*
 * Copyright @ 2015 Atlassian Pty Ltd
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.jitsi.impl.neomedia.codec.video.vp8;

import org.jitsi.impl.neomedia.codec.*;
import org.jitsi.service.neomedia.codec.*;
import org.jitsi.util.*;
import javax.media.*;
import javax.media.format.*;
import java.util.*;
import java.util.concurrent.*;

/**
 * A depacketizer from VP8.
 * See {@link "http://tools.ietf.org/html/draft-ietf-payload-vp8-11"}
 *
 * @author <NAME>
 * @author <NAME>
 */
public class DePacketizer
    extends AbstractCodec2
{
    /**
     * The <tt>Logger</tt> used by the <tt>DePacketizer</tt> class and its
     * instances for logging output.
     */
    private static final Logger logger = Logger.getLogger(DePacketizer.class);

    /**
     * Whether trace logging is enabled.
     */
    private static final boolean TRACE = logger.isTraceEnabled();

    /**
     * A <tt>Comparator</tt> implementation for RTP sequence numbers.
     * Compares <tt>a</tt> and <tt>b</tt>, taking into account the wrap at 2^16.
     *
     * IMPORTANT: This is a valid <tt>Comparator</tt> implementation only if
     * used for subsets of [0, 2^16) which don't span more than 2^15 elements.
     *
     * E.g. it works for: [0, 2^15-1] and ([50000, 2^16) u [0, 10000])
     * Doesn't work for: [0, 2^15] and ([0, 2^15-1] u {2^16-1}) and [0, 2^16)
     */
    private static final Comparator<? super Long> seqNumComparator
            = new Comparator<Long>() {
        @Override
        public int compare(Long a, Long b)
        {
            if (a.equals(b))
                return 0;
            else if (a > b)
            {
                if (a - b < 32768)
                    return 1;
                else
                    return -1;
            }
            else //a < b
            {
                if (b - a < 32768)
                    return -1;
                else
                    return 1;
            }
        }
    };

    /**
     * Stores the RTP payloads (VP8 payload descriptor stripped) from RTP packets
     * belonging to a single VP8 compressed frame.
     */
    private SortedMap<Long, Container> data
            = new TreeMap<Long, Container>(seqNumComparator);

    /**
     * Stores unused <tt>Container</tt>'s.
     */
    private Queue<Container> free = new ArrayBlockingQueue<Container>(100);

    /**
     * Stores the first (earliest) sequence number stored in <tt>data</tt>, or
     * -1 if <tt>data</tt> is empty.
     */
    private long firstSeq = -1;

    /**
     * Stores the last (latest) sequence number stored in <tt>data</tt>, or -1
     * if <tt>data</tt> is empty.
     */
    private long lastSeq = -1;

    /**
     * Stores the value of the <tt>PictureID</tt> field for the VP8 compressed
     * frame, parts of which are currently stored in <tt>data</tt>, or -1 if
     * the <tt>PictureID</tt> field is not in use or <tt>data</tt> is empty.
     */
    private int pictureId = -1;

    /**
     * Stores the RTP timestamp of the packets stored in <tt>data</tt>, or -1
     * if they don't have a timestamp set.
     */
    private long timestamp = -1;

    /**
     * Whether we have stored any packets in <tt>data</tt>. Equivalent to
     * <tt>data.isEmpty()</tt>.
     */
    private boolean empty = true;

    /**
     * Whether we have stored in <tt>data</tt> the last RTP packet of the VP8
     * compressed frame, parts of which are currently stored in <tt>data</tt>.
     */
    private boolean haveEnd = false;

    /**
     * Whether we have stored in <tt>data</tt> the first RTP packet of the VP8
     * compressed frame, parts of which are currently stored in <tt>data</tt>.
     */
    private boolean haveStart = false;

    /**
     * Stores the sum of the lengths of the data stored in <tt>data</tt>, that
     * is the total length of the VP8 compressed frame to be constructed.
     */
    private int frameLength = 0;

    /**
     * The sequence number of the last RTP packet, which was included in the
     * output.
     */
    private long lastSentSeq = -1;

    /**
     * Initializes a new <tt>JNIEncoder</tt> instance.
     */
    public DePacketizer()
    {
        super("VP8 RTP DePacketizer",
                VideoFormat.class,
                new VideoFormat[]{ new VideoFormat(Constants.VP8) });
        inputFormats = new VideoFormat[] { new VideoFormat(Constants.VP8_RTP) };
    }

    /**
     * {@inheritDoc}
     */
    @Override
    protected void doClose()
    {
    }

    /**
     * {@inheritDoc}
     */
    @Override
    protected void doOpen() throws ResourceUnavailableException
    {
        if(logger.isInfoEnabled())
            logger.info("Opened VP8 depacketizer");
    }

    /**
     * Re-initializes the fields which store information about the currently
     * held data. Empties <tt>data</tt>.
     */
    private void reinit()
    {
        firstSeq = lastSeq = timestamp = -1;
        pictureId = -1;
        empty = true;
        haveEnd = haveStart = false;
        frameLength = 0;

        Iterator<Map.Entry<Long,Container>> it = data.entrySet().iterator();
        Map.Entry<Long, Container> e;
        while (it.hasNext())
        {
            e = it.next();
            free.offer(e.getValue());
            it.remove();
        }
    }

    /**
     * Checks whether the currently held VP8 compressed frame is complete (e.g
     * all its packets are stored in <tt>data</tt>).
     * @return <tt>true</tt> if the currently help VP8 compressed frame is
     * complete, <tt>false</tt> otherwise.
     */
    private boolean frameComplete()
    {
        return haveStart && haveEnd && !haveMissing();
    }

    /**
     * Checks whether there are packets with sequence numbers between
     * <tt>firstSeq</tt> and <tt>lastSeq</tt> which are *not* stored in
     * <tt>data</tt>.
     * @return <tt>true</tt> if there are packets with sequence numbers between
     * <tt>firstSeq</tt> and <tt>lastSeq</tt> which are *not* stored in
     * <tt>data</tt>.
     */
    private boolean haveMissing()
    {
        Set<Long> seqs = data.keySet();
        long s = firstSeq;
        while (s != lastSeq)
        {
            if (!seqs.contains(s))
                return true;
            s = (s+1) % (1<<16);
        }
        return false;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    protected int doProcess(Buffer inBuffer, Buffer outBuffer)
    {
        byte[] inData = (byte[])inBuffer.getData();
        int inOffset = inBuffer.getOffset();

        if (!VP8PayloadDescriptor.isValid(inData, inOffset))
        {
            logger.warn("Invalid RTP/VP8 packet discarded.");
            outBuffer.setDiscard(true);
            return BUFFER_PROCESSED_FAILED; //XXX: FAILED or OK?
        }

        long inSeq = inBuffer.getSequenceNumber();
        long inRtpTimestamp = inBuffer.getRtpTimeStamp();
        int inPictureId = VP8PayloadDescriptor.getPictureId(inData, inOffset);
        boolean inMarker = (inBuffer.getFlags() & Buffer.FLAG_RTP_MARKER) != 0;
        boolean inIsStartOfFrame
                = VP8PayloadDescriptor.isStartOfFrame(inData, inOffset);
        int inLength = inBuffer.getLength();
        int inPdSize = VP8PayloadDescriptor.getSize(inData, inOffset);
        int inPayloadLength = inLength - inPdSize;

        if (empty
                && lastSentSeq != -1
                && seqNumComparator.compare(inSeq, lastSentSeq) != 1)
        {
            if (logger.isInfoEnabled())
                logger.info("Discarding old packet (while empty) " + inSeq);
            outBuffer.setDiscard(true);
            return BUFFER_PROCESSED_OK;
        }

        if (!empty)
        {
            // if the incoming packet has a different PictureID or timestamp
            // than those of the current frame, then it belongs to a different
            // frame.
            if ( (inPictureId != -1 && pictureId != -1
                    && inPictureId != pictureId)
                 | (timestamp != -1 && inRtpTimestamp != -1
                    && inRtpTimestamp != timestamp) )
            {
                if (seqNumComparator
                        .compare(inSeq, firstSeq) != 1) //inSeq <= firstSeq
                {
                    // the packet belongs to a previous frame. discard it
                    if (logger.isInfoEnabled())
                        logger.info("Discarding old packet " + inSeq);
                    outBuffer.setDiscard(true);
                    return BUFFER_PROCESSED_OK;
                }
                else //inSeq > firstSeq (and also presumably isSeq > lastSeq)
                {
                    // the packet belongs to a subsequent frame (to the one
                    // currently being held). Drop the current frame.

                    if (logger.isInfoEnabled())
                        logger.info("Discarding saved packets on arrival of" +
                                " a packet for a subsequent frame: " + inSeq);

                    // TODO: this would be the place to complain about the
                    // not-well-received PictureID by sending a RTCP SLI or NACK.
                    reinit();
                }
            }
        }

        // a whole frame in a single packet. avoid the extra copy to
        // this.data and output it immediately.
        if (empty && inMarker && inIsStartOfFrame)
        {
            byte[] outData
                    = validateByteArraySize(outBuffer, inPayloadLength, false);
            System.arraycopy(
                    inData,
                    inOffset + inPdSize,
                    outData,
                    0,
                    inPayloadLength);
            outBuffer.setOffset(0);
            outBuffer.setLength(inPayloadLength);
            outBuffer.setRtpTimeStamp(inBuffer.getRtpTimeStamp());

            if (TRACE)
                logger.trace("Out PictureID=" + inPictureId);

            lastSentSeq = inSeq;

            return BUFFER_PROCESSED_OK;
        }

        // add to this.data
        Container container = free.poll();
        if (container == null)
            container = new Container();
        if (container.buf == null || container.buf.length < inPayloadLength)
            container.buf = new byte[inPayloadLength];

        if (data.get(inSeq) != null)
        {
            if (logger.isInfoEnabled())
                logger.info("(Probable) duplicate packet detected, discarding "
                                    + inSeq);
            outBuffer.setDiscard(true);
            return BUFFER_PROCESSED_OK;
        }

        System.arraycopy(
                inData,
                inOffset + inPdSize,
                container.buf,
                0,
                inPayloadLength);
        container.len = inPayloadLength;
        data.put(inSeq, container);

        // update fields
        frameLength += inPayloadLength;
        if (firstSeq == -1
                || (seqNumComparator.compare(firstSeq, inSeq) == 1))
            firstSeq = inSeq;
        if (lastSeq == -1
                || (seqNumComparator.compare(inSeq, lastSeq) == 1))
            lastSeq = inSeq;

        if (empty)
        {
            // the first received packet for the current frame was just added
            empty = false;
            timestamp = inRtpTimestamp;
            pictureId = inPictureId;
        }

        if (inMarker)
            haveEnd = true;
        if (inIsStartOfFrame)
            haveStart = true;

        // check if we have a full frame
        if (frameComplete())
        {
            byte[] outData
                    = validateByteArraySize(outBuffer, frameLength, false);
            int ptr = 0;
            Container b;
            for (Map.Entry<Long, Container> entry : data.entrySet())
            {
                b = entry.getValue();
                System.arraycopy(
                        b.buf,
                        0,
                        outData,
                        ptr,
                        b.len);
                ptr += b.len;
            }

            outBuffer.setOffset(0);
            outBuffer.setLength(frameLength);
            outBuffer.setRtpTimeStamp(inBuffer.getRtpTimeStamp());

            if (TRACE)
                logger.trace("Out PictureID=" + inPictureId);
            lastSentSeq = lastSeq;

            // prepare for the next frame
            reinit();

            return BUFFER_PROCESSED_OK;
        }
        else
        {
            // frame not complete yet
            outBuffer.setDiscard(true);
            return OUTPUT_BUFFER_NOT_FILLED;
        }
    }

    /**
     */
    public static boolean isKeyFrame(byte[] buff, int off, int len)
    {
        if (buff == null || len < 1 || buff.length < off + len)
        {
            return false;
        }

        // Check if this is the start of a VP8 partition in the payload
        // descriptor.
        if (!DePacketizer.VP8PayloadDescriptor.isValid(buff, off))
        {
            return false;
        }

        if (!DePacketizer.VP8PayloadDescriptor.isStartOfFrame(buff, off))
        {
            return false;
        }

        int szVP8PayloadDescriptor = DePacketizer
            .VP8PayloadDescriptor.getSize(buff, off);

        return DePacketizer.VP8PayloadHeader.isKeyFrame(
                buff, off + szVP8PayloadDescriptor);
    }

    /**
{'fraction_non_alphanumeric': 0.10347673208711564, 'fraction_numerical': 0.00990276631624269, 'mean_word_length': 1.6529602132114982, 'pattern_counts': {'":': 0, '<': 186, '<?xml version=': 0, '>': 175, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '676848', 'n_tokens_mistral': 7940, 'n_tokens_neox': 7527, 'n_words': 2256}