<commit_before>/*
 * MIT License
 *
 * Copyright (c) 2017 <NAME>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "tsdemux.h"
#include "string.h"
#include <stdio.h>

uint16_t parse_u16(const uint8_t *bytes)
{
    uint16_t val = *((uint16_t*)bytes);
#ifdef __BIG_ENDIAN__
    return val;
#else
    return ((val >> 8) & 0x00FF) | ((val << 8) & 0xFF00);
#endif
}

uint64_t parse_u32(const uint8_t *bytes)
{
    uint32_t val = *((uint32_t*)bytes);
#ifdef __BIG_ENDIAN__
    return val;
#else
    return ((val >> 24) & 0x000000FF) |
           ((val >>  8) & 0x0000FF00) |
           ((val <<  8) & 0x00FF0000) |
           ((val << 24) & 0xFF000000) ;
#endif
}

uint64_t parse_u64(const uint8_t *bytes)
{
    uint64_t val = *((uint64_t*)bytes);
#ifdef __BIG_ENDIAN__
    return val;
#else
    return ((val >> 56) & 0x00000000000000FFL) |
           ((val >> 40) & 0x000000000000FF00L) |
           ((val >> 24) & 0x0000000000FF0000L) |
           ((val >>  8) & 0x00000000FF000000L) |
           ((val <<  8) & 0x000000FF00000000L) |
           ((val << 24) & 0x0000FF0000000000L) |
           ((val << 40) & 0x00FF000000000000L) |
           ((val << 56) & 0xFF00000000000000L) ;
#endif
}

const char* tsd_get_version(void)
{
    return TSD_VERSION;
}

TSDCode tsd_context_init(TSDemuxContext *ctx)
{
    if(ctx == NULL) return TSD_INVALID_CONTEXT;

    memset(ctx, 0, sizeof(TSDemuxContext));

    ctx->malloc = malloc;
    ctx->realloc = realloc;
    ctx->calloc = calloc;
    ctx->free = free;

    // initialize the user defined event callback
    ctx->event_cb = (tsd_on_event) NULL;

    return TSD_OK;
}

TSDCode tsd_context_destroy(TSDemuxContext *ctx)
{
    if(ctx == NULL) return TSD_INVALID_CONTEXT;

    int i=0;
    int size = ctx->registered_pids_length;
    // destroyregistered pid list
    for(i=0; i<size; ++i) {
        tsd_data_context_destroy(ctx, ctx->registered_pids_data[i]);
        ctx->free(ctx->registered_pids_data[i]);
    }

    // destroy data context buffer pool
    size = ctx->buffers.length;
    for(i=0; i<size; ++i) {
        tsd_data_context_destroy(ctx, &ctx->buffers.pool[i]);
    }
    if(size) {
        ctx->free(ctx->buffers.pool);
    }

    // destroy PAT data
    if(ctx->pat.valid && ctx->pat.value.length > 0) {
        ctx->free(ctx->pat.value.pid);
        ctx->free(ctx->pat.value.program_number);
    }

    // clear everything
    memset(ctx, 0, sizeof(TSDemuxContext));

    return TSD_OK;
}

TSDCode tsd_set_event_callback(TSDemuxContext *ctx, tsd_on_event callback)
{
    if(ctx == NULL)     return TSD_INVALID_CONTEXT;
    // Note that we allow the callback to be NULL
    ctx->event_cb = callback;
    return TSD_OK;
}

TSDCode tsd_parse_packet_header(TSDemuxContext *ctx,
                                const uint8_t *data,
                                size_t size,
                                TSDPacket *hdr)
{
    if(ctx == NULL)                 return TSD_INVALID_CONTEXT;
    if(data == NULL)                return TSD_INVALID_DATA;
    if(size < TSD_TSPACKET_SIZE)    return TSD_INVALID_DATA_SIZE;
    if(hdr == NULL)                 return TSD_INVALID_ARGUMENT;

    const uint8_t *ptr = data;
    const uint8_t *end = &ptr[size];
    // check the sync byte
    hdr->sync_byte = *ptr;
    if(hdr->sync_byte != TSD_SYNC_BYTE)  return TSD_INVALID_SYNC_BYTE;
    ptr++;

    hdr->flags = (*ptr) >> 5;
    hdr->pid = parse_u16(ptr) & 0x1FFF;
    ptr+=2;

    hdr->transport_scrambling_control = (TSDScramblingControl)(((*ptr) >> 6) & 0x03);
    hdr->adaptation_field_control = (TSDAdaptionFieldControl)(((*ptr) >> 4) & 0x03);
    hdr->continuity_counter = (*ptr) & 0x0F;
    ptr++;

    hdr->data_bytes = NULL;
    hdr->data_bytes_length = 0;

    // parse the adaptation field if it exists
    if(hdr->adaptation_field_control == TSD_AFC_ADAP_FIELD_AND_PAYLOAD ||
       hdr->adaptation_field_control == TSD_AFC_ADAP_FIELD_ONLY) {

        TSDCode res = tsd_parse_adaptation_field(ctx, ptr, size-4,
                      &hdr->adaptation_field);

        if(res != TSD_OK) return res;

        if(end < &ptr[hdr->adaptation_field.adaptation_field_length]) {
            return TSD_INVALID_DATA_SIZE;
        };
        ptr = &ptr[hdr->adaptation_field.adaptation_field_length + 1];
    } else {
        memset(&hdr->adaptation_field, 0, sizeof(TSDAdaptationField));
    }

    // is there a payload in this packet?
    if(hdr->adaptation_field_control == TSD_AFC_NO_FIELD_PRESENT ||
       hdr->adaptation_field_control == TSD_AFC_ADAP_FIELD_AND_PAYLOAD) {

        // handle error packets
        if(!(hdr->flags & TSD_PF_TRAN_ERR_INDICATOR) &&
           (hdr->pid != TSD_PID_NULL_PACKETS)) {

            size_t len = TSD_TSPACKET_SIZE - ((size_t)ptr - (size_t)data);
            if(len <= TSD_TSPACKET_SIZE - 4) {
                // note that there may or may not be a pointer_field in the
                // data, we can't tell whether there is yet.
                hdr->data_bytes = ptr;
                hdr->data_bytes_length = len;
            }
        }
    }

    return TSD_OK;
}

TSDCode tsd_parse_adaptation_field(TSDemuxContext *ctx,
                                   const uint8_t *data,
                                   size_t size,
                                   TSDAdaptationField *adap)
{
    if(ctx == NULL)                 return TSD_INVALID_CONTEXT;
    if(data == NULL)                return TSD_INVALID_DATA;
    if(size == 0)                   return TSD_INVALID_DATA_SIZE;
    if(adap == NULL)                return TSD_INVALID_ARGUMENT;

    const uint8_t *ptr = data;
    const uint8_t *end = &ptr[size];

    if(ptr+1 > end) return TSD_INVALID_DATA_SIZE;

    adap->adaptation_field_length = *ptr;
    if(adap->adaptation_field_length > TSD_TSPACKET_SIZE - 5) {
        return TSD_PARSE_ERROR;
    }

    if(adap->adaptation_field_length > 0) {
        ptr++;
        if(ptr+1 > end) return TSD_INVALID_DATA_SIZE;
        adap->flags = *ptr;
        ptr++;

        if(adap->flags & TSD_AF_PCR_FLAG) {
            if(ptr+6 > end) return TSD_INVALID_DATA_SIZE;
            uint64_t val = parse_u64(ptr);
            adap->program_clock_ref_base = (val >> 31) & 0x1FFFFFFFFL;
            // ignore the 6 reserved bytes
            adap->program_clock_ref_ext = (uint16_t)((val >> 16) & 0x1FFL);
            ptr+=6;
        }

        if(adap->flags & TSD_AF_OPCR_FLAG) {
            if(ptr+6 > end) return TSD_INVALID_DATA_SIZE;
            uint64_t val = parse_u64(ptr);
            adap->orig_program_clock_ref_base = (val >> 31) & 0x1FFFFFFFFL;
            // ignore the 6 reserved bytes
            adap->orig_program_clock_ref_ext = (uint16_t)((val >> 16) & 0x1FFL);
            ptr+=6;
        }

        if(adap->flags & TSD_AF_SPLICING_POINT_FLAG) {
            if(ptr+1 > end) return TSD_INVALID_DATA_SIZE;
            adap->splice_countdown = *ptr;
            ptr++;
        }

        if(adap->flags & TSD_AF_TRAN_PRIVATE_DATA_FLAG) {
            if(ptr+1 > end) return TSD_INVALID_DATA_SIZE;
            adap->transport_private_data_length = *ptr;
            ptr++;

            if(ptr + adap->transport_private_data_length > end) {
                return TSD_INVALID_DATA_SIZE;
            }
            adap->private_data_bytes = ptr;
            ptr += adap->transport_private_data_length;
        }

        if(adap->flags & TSD_AF_ADAP_FIELD_EXT_FLAG) {
            if(ptr+1 > end) return TSD_INVALID_DATA_SIZE;
            adap->adap_field_ext.length = *ptr;
            ptr++;
            if(ptr+1 > end) return TSD_INVALID_DATA_SIZE;
            adap->adap_field_ext.flags = ((*ptr) >> 5) & 0x07;
            ptr++;

            if(adap->adap_field_ext.flags & TSD_AFEF_LTW_FLAG) {
                if(ptr+2 > end) return TSD_INVALID_DATA_SIZE;
                adap->adap_field_ext.ltw_valid_flag =
                    (*ptr) >> 7 & 0x01;
                uint16_t offset = parse_u16(ptr);
                adap->adap_field_ext.ltw_offset = offset & 0x7FFF;
                ptr += 2;
            }

            if(adap->adap_field_ext.flags & TSD_AFEF_PIECEWISE_RATE_FLAG) {
                if(ptr+3 > end) return TSD_INVALID_DATA_SIZE;
                uint32_t rate = parse_u32(ptr) >> 8;
                adap->adap_field_ext.piecewise_rate = rate & 0x3FFFFF;
                ptr += 3;
            }

            if(adap->adap_field_ext.flags & TSD_AFEF_SEAMLESS_SPLCE_FLAG) {
                if(ptr+5 > end) return TSD_INVALID_DATA_SIZE;
                adap->adap_field_ext.splice_type = ((*ptr) >> 4) & 0x0F;
                uint64_t au1 = (uint64_t) (((*ptr) >> 1) & 0x07);
                uint64_t au2 = (uint64_t) ((parse_u16(ptr+1) >> 1) & 0x7FFF);
                uint64_t au3 = (uint64_t) ((parse_u16(ptr+3) >> 1) & 0x7FFF);
                adap->adap_field_ext.dts_next_au = (au1 << 30) | (au2 << 15) | au3;
            }
        }
    }

    return TSD_OK;
}

TSDCode get_data_context(TSDemuxContext *ctx,
                         uint8_t table_id,
                         uint16_t table_ext,
                         uint8_t version,
                         TSDDataContext **context)
{
    if(ctx == NULL)         return TSD_INVALID_CONTEXT;
    if(context == NULL)     return TSD_INVALID_ARGUMENT;

    // create a 32-bit Id for this table
    uint32_t id = (((uint32_t)table_id) << 24) |
                  (((uint32_t)table_ext) << 8) |
                  ((uint32_t)version);

    // go through the DataContexts already Available and see if we have a match
    size_t i=0;
    for(; i<ctx->buffers.length; ++i) {
        if(ctx->buffers.pool[i].id == id) {
            // found a matching TSDDataContext
            *context = &ctx->buffers.pool[i];
            return TSD_OK;
        }
    }

    // we'll need to find the active buffer if one is set
    int active_idx = -1;
    if(ctx->buffers.active) {
        for(i=0; i<ctx->buffers.length; ++i) {
            if(ctx->buffers.pool[i].id == ctx->buffers.active->id) {
                active_idx = (int)i;
                break;
            }
        }
    }

    // we didn't find a TSDDataContext, create a new one and add it to the Pool
    TSDDataContext *dataCtx = NULL;
    size_t len = ctx->buffers.length + 1;
    if(len == 1) {
        dataCtx = (TSDDataContext*) ctx->malloc(sizeof(TSDDataContext));
    } else {
        size_t size = sizeof(TSDDataContext) * len;
        dataCtx = (TSDDataContext*) ctx->realloc(ctx->buffers.pool, size);
    }

    if(!dataCtx) {
        return TSD_OUT_OF_MEMORY;
    }
    ctx->buffers.pool = dataCtx;
    ctx->buffers.length = len;
    // reassign the active buffer if it was previously set
    if(active_idx >= 0) {
        ctx->buffers.active = &ctx->buffers.pool[active_idx];
    }

    // initialize the TSDDataContext
    TSDCode res = tsd_data_context_init(ctx, &(dataCtx[len-1]));
    *context = dataCtx;
    return res;
}

TSDCode tsd_parse_table(TSDemuxContext *ctx,
                        TSDPacket *pkt,
                        TSDTable *table)
{
    if(ctx == NULL)                 return TSD_INVALID_CONTEXT;
    if(pkt == NULL)                 return TSD_INVALID_ARGUMENT;
    if(table == NULL)               return TSD_INVALID_ARGUMENT;

    TSDCode res;

    // payload data error handling.
    if(pkt->data_bytes == NULL || pkt->data_bytes_length == 0) {
        return TSD_INCOMPLETE_TABLE;
    }

    // keep hold of the data location.
    const uint8_t *ptr = pkt->data_bytes;
    size_t ptr_len = pkt->data_bytes_length;

    if(pkt->flags & TSD_PF_PAYLOAD_UNIT_START_IND) {
        // there is a new table section somewhere in this packet.
        // parse the pointer_field.
        size_t pointer_field = *ptr;
        if(pointer_field >= pkt->data_bytes_length - 6) {
            return TSD_INVALID_POINTER_FIELD;
        }
        ptr = &ptr[pointer_field + 1];
        ptr_len -= (pointer_field + 1);
        // parse some of the table info so that we can find the TSDDataContext
        // assoicated with this table.
        uint8_t table_id = *ptr;
        uint16_t table_ext = 0;
        uint8_t version = 0;
        if(ptr[1] & 0x80) {
            table_ext = parse_u16(&ptr[3]);
            version = (ptr[5] & 0x3E) >> 1;
        }

        // set the TSDDataContext for this table as active.
        res = get_data_context(ctx,
                               table_id,
                               table_ext,
                               version,
                               &(ctx->buffers.active));
        if(res != TSD_OK) {
            return res;
        }

    } else {
        // if we don't already have an active buffer this must be old table data
        // that we aren't prepared to parse
        if(ctx->buffers.active == NULL) {
            return TSD_INCOMPLETE_TABLE;
        }
    }

    TSDDataContext *dataCtx = ctx->buffers.active;
    // write the data into our buffer
    res = tsd_data_context_write(ctx, dataCtx, ptr, ptr_len);
    if(res != TSD_OK) {
        return res;
    }

    // has the table completed yet? We'll need enough data to see the section
    // length to figure that out.
    ptr = dataCtx->buffer;
    int section_count = 0;

    while(ptr < dataCtx->write) {
        uint16_t section_len = parse_u16(ptr+1);
        section_len &= 0x0FFF;
        ptr += section_len + 3;
        section_count++;

        if((ptr+1) < dataCtx->write && (*(ptr+1) == 0xFF) || (0x00 == *(ptr+1))) {
            // create and parse the sections.
            table->length = section_count;
            table->sections = (TSDTableSection*) ctx->calloc(section_count,
                              sizeof(TSDTableSection));
<commit_msg>fixed an issue within PAT table parsing data verification<commit_after>/*
 * MIT License
 *
 * Copyright (c) 2017 <NAME>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "tsdemux.h"
#include "string.h"
#include <stdio.h>

uint16_t parse_u16(const uint8_t *bytes)
{
    uint16_t val = *((uint16_t*)bytes);
#ifdef __BIG_ENDIAN__
    return val;
#else
    return ((val >> 8) & 0x00FF) | ((val << 8) & 0xFF00);
#endif
}

uint64_t parse_u32(const uint8_t *bytes)
{
    uint32_t val = *((uint32_t*)bytes);
#ifdef __BIG_ENDIAN__
    return val;
#else
    return ((val >> 24) & 0x000000FF) |
           ((val >>  8) & 0x0000FF00) |
           ((val <<  8) & 0x00FF0000) |
           ((val << 24) & 0xFF000000) ;
#endif
}

uint64_t parse_u64(const uint8_t *bytes)
{
    uint64_t val = *((uint64_t*)bytes);
#ifdef __BIG_ENDIAN__
    return val;
#else
    return ((val >> 56) & 0x00000000000000FFL) |
           ((val >> 40) & 0x000000000000FF00L) |
           ((val >> 24) & 0x0000000000FF0000L) |
           ((val >>  8) & 0x00000000FF000000L) |
           ((val <<  8) & 0x000000FF00000000L) |
           ((val << 24) & 0x0000FF0000000000L) |
           ((val << 40) & 0x00FF000000000000L) |
           ((val << 56) & 0xFF00000000000000L) ;
#endif
}

const char* tsd_get_version(void)
{
    return TSD_VERSION;
}

TSDCode tsd_context_init(TSDemuxContext *ctx)
{
    if(ctx == NULL) return TSD_INVALID_CONTEXT;

    memset(ctx, 0, sizeof(TSDemuxContext));

    ctx->malloc = malloc;
    ctx->realloc = realloc;
    ctx->calloc = calloc;
    ctx->free = free;

    // initialize the user defined event callback
    ctx->event_cb = (tsd_on_event) NULL;

    return TSD_OK;
}

TSDCode tsd_context_destroy(TSDemuxContext *ctx)
{
    if(ctx == NULL) return TSD_INVALID_CONTEXT;

    int i=0;
    int size = ctx->registered_pids_length;
    // destroyregistered pid list
    for(i=0; i<size; ++i) {
        tsd_data_context_destroy(ctx, ctx->registered_pids_data[i]);
        ctx->free(ctx->registered_pids_data[i]);
    }

    // destroy data context buffer pool
    size = ctx->buffers.length;
    for(i=0; i<size; ++i) {
        tsd_data_context_destroy(ctx, &ctx->buffers.pool[i]);
    }
    if(size) {
        ctx->free(ctx->buffers.pool);
    }

    // destroy PAT data
    if(ctx->pat.valid && ctx->pat.value.length > 0) {
        ctx->free(ctx->pat.value.pid);
        ctx->free(ctx->pat.value.program_number);
    }

    // clear everything
    memset(ctx, 0, sizeof(TSDemuxContext));

    return TSD_OK;
}

TSDCode tsd_set_event_callback(TSDemuxContext *ctx, tsd_on_event callback)
{
    if(ctx == NULL)     return TSD_INVALID_CONTEXT;
    // Note that we allow the callback to be NULL
    ctx->event_cb = callback;
    return TSD_OK;
}

TSDCode tsd_parse_packet_header(TSDemuxContext *ctx,
                                const uint8_t *data,
                                size_t size,
                                TSDPacket *hdr)
{
    if(ctx == NULL)                 return TSD_INVALID_CONTEXT;
    if(data == NULL)                return TSD_INVALID_DATA;
    if(size < TSD_TSPACKET_SIZE)    return TSD_INVALID_DATA_SIZE;
    if(hdr == NULL)                 return TSD_INVALID_ARGUMENT;

    const uint8_t *ptr = data;
    const uint8_t *end = &ptr[size];
    // check the sync byte
    hdr->sync_byte = *ptr;
    if(hdr->sync_byte != TSD_SYNC_BYTE)  return TSD_INVALID_SYNC_BYTE;
    ptr++;

    hdr->flags = (*ptr) >> 5;
    hdr->pid = parse_u16(ptr) & 0x1FFF;
    ptr+=2;

    hdr->transport_scrambling_control = (TSDScramblingControl)(((*ptr) >> 6) & 0x03);
    hdr->adaptation_field_control = (TSDAdaptionFieldControl)(((*ptr) >> 4) & 0x03);
    hdr->continuity_counter = (*ptr) & 0x0F;
    ptr++;

    hdr->data_bytes = NULL;
    hdr->data_bytes_length = 0;

    // parse the adaptation field if it exists
    if(hdr->adaptation_field_control == TSD_AFC_ADAP_FIELD_AND_PAYLOAD ||
       hdr->adaptation_field_control == TSD_AFC_ADAP_FIELD_ONLY) {

        TSDCode res = tsd_parse_adaptation_field(ctx, ptr, size-4,
                      &hdr->adaptation_field);

        if(res != TSD_OK) return res;

        if(end < &ptr[hdr->adaptation_field.adaptation_field_length]) {
            return TSD_INVALID_DATA_SIZE;
        };
        ptr = &ptr[hdr->adaptation_field.adaptation_field_length + 1];
    } else {
        memset(&hdr->adaptation_field, 0, sizeof(TSDAdaptationField));
    }

    // is there a payload in this packet?
    if(hdr->adaptation_field_control == TSD_AFC_NO_FIELD_PRESENT ||
       hdr->adaptation_field_control == TSD_AFC_ADAP_FIELD_AND_PAYLOAD) {

        // handle error packets
        if(!(hdr->flags & TSD_PF_TRAN_ERR_INDICATOR) &&
           (hdr->pid != TSD_PID_NULL_PACKETS)) {

            size_t len = TSD_TSPACKET_SIZE - ((size_t)ptr - (size_t)data);
            if(len <= TSD_TSPACKET_SIZE - 4) {
                // note that there may or may not be a pointer_field in the
                // data, we can't tell whether there is yet.
                hdr->data_bytes = ptr;
                hdr->data_bytes_length = len;
            }
        }
    }

    return TSD_OK;
}

TSDCode tsd_parse_adaptation_field(TSDemuxContext *ctx,
                                   const uint8_t *data,
                                   size_t size,
                                   TSDAdaptationField *adap)
{
    if(ctx == NULL)                 return TSD_INVALID_CONTEXT;
    if(data == NULL)                return TSD_INVALID_DATA;
    if(size == 0)                   return TSD_INVALID_DATA_SIZE;
    if(adap == NULL)                return TSD_INVALID_ARGUMENT;

    const uint8_t *ptr = data;
    const uint8_t *end = &ptr[size];

    if(ptr+1 > end) return TSD_INVALID_DATA_SIZE;

    adap->adaptation_field_length = *ptr;
    if(adap->adaptation_field_length > TSD_TSPACKET_SIZE - 5) {
        return TSD_PARSE_ERROR;
    }

    if(adap->adaptation_field_length > 0) {
        ptr++;
        if(ptr+1 > end) return TSD_INVALID_DATA_SIZE;
        adap->flags = *ptr;
        ptr++;

        if(adap->flags & TSD_AF_PCR_FLAG) {
            if(ptr+6 > end) return TSD_INVALID_DATA_SIZE;
            uint64_t val = parse_u64(ptr);
            adap->program_clock_ref_base = (val >> 31) & 0x1FFFFFFFFL;
            // ignore the 6 reserved bytes
            adap->program_clock_ref_ext = (uint16_t)((val >> 16) & 0x1FFL);
            ptr+=6;
        }

        if(adap->flags & TSD_AF_OPCR_FLAG) {
            if(ptr+6 > end) return TSD_INVALID_DATA_SIZE;
            uint64_t val = parse_u64(ptr);
            adap->orig_program_clock_ref_base = (val >> 31) & 0x1FFFFFFFFL;
            // ignore the 6 reserved bytes
            adap->orig_program_clock_ref_ext = (uint16_t)((val >> 16) & 0x1FFL);
            ptr+=6;
        }

        if(adap->flags & TSD_AF_SPLICING_POINT_FLAG) {
            if(ptr+1 > end) return TSD_INVALID_DATA_SIZE;
            adap->splice_countdown = *ptr;
            ptr++;
        }

        if(adap->flags & TSD_AF_TRAN_PRIVATE_DATA_FLAG) {
            if(ptr+1 > end) return TSD_INVALID_DATA_SIZE;
            adap->transport_private_data_length = *ptr;
            ptr++;

            if(ptr + adap->transport_private_data_length > end) {
                return TSD_INVALID_DATA_SIZE;
            }
            adap->private_data_bytes = ptr;
            ptr += adap->transport_private_data_length;
        }

        if(adap->flags & TSD_AF_ADAP_FIELD_EXT_FLAG) {
            if(ptr+1 > end) return TSD_INVALID_DATA_SIZE;
            adap->adap_field_ext.length = *ptr;
            ptr++;
            if(ptr+1 > end) return TSD_INVALID_DATA_SIZE;
            adap->adap_field_ext.flags = ((*ptr) >> 5) & 0x07;
            ptr++;

            if(adap->adap_field_ext.flags & TSD_AFEF_LTW_FLAG) {
                if(ptr+2 > end) return TSD_INVALID_DATA_SIZE;
                adap->adap_field_ext.ltw_valid_flag =
                    (*ptr) >> 7 & 0x01;
                uint16_t offset = parse_u16(ptr);
                adap->adap_field_ext.ltw_offset = offset & 0x7FFF;
                ptr += 2;
            }

            if(adap->adap_field_ext.flags & TSD_AFEF_PIECEWISE_RATE_FLAG) {
                if(ptr+3 > end) return TSD_INVALID_DATA_SIZE;
                uint32_t rate = parse_u32(ptr) >> 8;
                adap->adap_field_ext.piecewise_rate = rate & 0x3FFFFF;
                ptr += 3;
            }

            if(adap->adap_field_ext.flags & TSD_AFEF_SEAMLESS_SPLCE_FLAG) {
                if(ptr+5 > end) return TSD_INVALID_DATA_SIZE;
                adap->adap_field_ext.splice_type = ((*ptr) >> 4) & 0x0F;
                uint64_t au1 = (uint64_t) (((*ptr) >> 1) & 0x07);
                uint64_t au2 = (uint64_t) ((parse_u16(ptr+1) >> 1) & 0x7FFF);
                uint64_t au3 = (uint64_t) ((parse_u16(ptr+3) >> 1) & 0x7FFF);
                adap->adap_field_ext.dts_next_au = (au1 << 30) | (au2 << 15) | au3;
            }
        }
    }

    return TSD_OK;
}

TSDCode get_data_context(TSDemuxContext *ctx,
                         uint8_t table_id,
                         uint16_t table_ext,
                         uint8_t version,
                         TSDDataContext **context)
{
    if(ctx == NULL)         return TSD_INVALID_CONTEXT;
    if(context == NULL)     return TSD_INVALID_ARGUMENT;

    // create a 32-bit Id for this table
    uint32_t id = (((uint32_t)table_id) << 24) |
                  (((uint32_t)table_ext) << 8) |
                  ((uint32_t)version);

    // go through the DataContexts already Available and see if we have a match
    size_t i=0;
    for(; i<ctx->buffers.length; ++i) {
        if(ctx->buffers.pool[i].id == id) {
            // found a matching TSDDataContext
            *context = &ctx->buffers.pool[i];
            return TSD_OK;
        }
    }

    // we'll need to find the active buffer if one is set
    int active_idx = -1;
    if(ctx->buffers.active) {
        for(i=0; i<ctx->buffers.length; ++i) {
            if(ctx->buffers.pool[i].id == ctx->buffers.active->id) {
                active_idx = (int)i;
                break;
            }
        }
    }

    // we didn't find a TSDDataContext, create a new one and add it to the Pool
    TSDDataContext *dataCtx = NULL;
    size_t len = ctx->buffers.length + 1;
    if(len == 1) {
        dataCtx = (TSDDataContext*) ctx->malloc(sizeof(TSDDataContext));
    } else {
        size_t size = sizeof(TSDDataContext) * len;
        dataCtx = (TSDDataContext*) ctx->realloc(ctx->buffers.pool, size);
    }

    if(!dataCtx) {
        return TSD_OUT_OF_MEMORY;
    }
    ctx->buffers.pool = dataCtx;
    ctx->buffers.length = len;
    // reassign the active buffer if it was previously set
    if(active_idx >= 0) {
        ctx->buffers.active = &ctx->buffers.pool[active_idx];
    }

    // initialize the TSDDataContext
    TSDCode res = tsd_data_context_init(ctx, &(dataCtx[len-1]));
    *context = dataCtx;
    return res;
}

TSDCode tsd_parse_table(TSDemuxContext *ctx,
                        TSDPacket *pkt,
                        TSDTable *table)
{
    if(ctx == NULL)                 return TSD_INVALID_CONTEXT;
    if(pkt == NULL)                 return TSD_INVALID_ARGUMENT;
    if(table == NULL)               return TSD_INVALID_ARGUMENT;

    TSDCode res;

    // payload data error handling.
    if(pkt->data_bytes == NULL || pkt->data_bytes_length == 0) {
        return TSD_INCOMPLETE_TABLE;
    }

    // keep hold of the data location.
    const uint8_t *ptr = pkt->data_bytes;
    size_t ptr_len = pkt->data_bytes_length;

    if(pkt->flags & TSD_PF_PAYLOAD_UNIT_START_IND) {
        // there is a new table section somewhere in this packet.
        // parse the pointer_field.
        size_t pointer_field = *ptr;
        if(pointer_field >= pkt->data_bytes_length - 6) {
            return TSD_INVALID_POINTER_FIELD;
        }
        ptr = &ptr[pointer_field + 1];
        ptr_len -= (pointer_field + 1);
        // parse some of the table info so that we can find the TSDDataContext
        // assoicated with this table.
        uint8_t table_id = *ptr;
        uint16_t table_ext = 0;
        uint8_t version = 0;
        if(ptr[1] & 0x80) {
            table_ext = parse_u16(&ptr[3]);
            version = (ptr[5] & 0x3E) >> 1;
        }

        // set the TSDDataContext for this table as active.
        res = get_data_context(ctx,
                               table_id,
                               table_ext,
                               version,
                               &(ctx->buffers.active));
        if(res != TSD_OK) {
            return res;
        }

    } else {
        // if we don't already have an active buffer this must be old table data
        // that we aren't prepared to parse
        if(ctx->buffers.active == NULL) {
            return TSD_INCOMPLETE_TABLE;
        }
    }

    TSDDataContext *dataCtx = ctx->buffers.active;
    // write the data into our buffer
    res = tsd_data_context_write(ctx, dataCtx, ptr, ptr_len);
    if(res != TSD_OK) {
        return res;
    }

    // has the table completed yet? We'll need enough data to see the section
    // length to figure that out.
    ptr = dataCtx->buffer;
    int section_count = 0;

    while(ptr < dataCtx->write) {
        uint16_t section_len = parse_u16(ptr+1);
        section_len &= 0x0FFF;
        ptr += section_len + 3;
        section_count++;

        if((ptr <= dataCtx->write) && ((*(ptr+1) == 0xFF) || (0x00 == *(ptr+1)))) {
            // create and parse the sections.
            table->length = section_count;
            table->sections = (TSDTableSection*) ctx->calloc(section_count,
                              sizeof(TSDTableSection));

{'fraction_non_alphanumeric': 0.11050287307469994, 'fraction_numerical': 0.027982727550916325, 'mean_word_length': 2.0814038763750653, 'pattern_counts': {'":': 0, '<': 69, '<?xml version=': 0, '>': 329, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 20, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7339855', 'n_tokens_mistral': 10483, 'n_tokens_neox': 9676, 'n_words': 2212}