<commit_before>#include <stddef.h>
#include <string.h>
#include <stdlib.h>
#include <libavutil/log.h>

#include "config.h"

#include "ts_muxer.h"
#include "libcseg.h"


// TS payload type
#define TS_MUXER_PAYLOAD_PAT          0
#define TS_MUXER_PAYLOAD_PMT          1
#define TS_MUXER_PAYLOAD_EMPTY_PES    2
#define TS_MUXER_PAYLOAD_H264_PES     10
#define TS_MUXER_PAYLOAD_AAC_PES      20

#define TS_MUXER_TX_PACKET_SIZE  188


#define TS_PTS_MAX_DELAY    63000  /*1/90K, 0.7sec*/

#define MIN(a,b) (((a)<(b))?(a):(b))
#define MAX(a,b) (((a)>(b))?(a):(b))

#define H264_NAL_TYPE_AUD   9


typedef struct {
    uint8_t                len;
    uint16_t             pid;
    uint8_t                payload_unit_start_indicator;
    uint8_t                continuity_count;

    int64_t             pcr;  // 90kHz time
    uint8_t                write_pcr;
    uint8_t                random_access_indicator;
    uint8_t                es_priority_indicator;

    uint8_t                header_stuffing_size;
    uint8_t                end_stuffing_size;

    uint8_t       buf[TS_MUXER_TX_PACKET_SIZE];

} ts_muxer_ts_packet_t;

// TS packet IEC 13818-1:2000 2.4.4.3
typedef struct {
    uint8_t                start;
    uint16_t              pid;
    size_t                size;
    size_t                remain;
    uint8_t                program_cnt;
    uint16_t             pmt_pid;
} ts_muxer_ts_pat_t;

typedef struct {
    uint8_t                start;
    uint16_t              pid;
    size_t                size;
    size_t                remain;

}ts_muxer_ts_pmt_t;

typedef struct {
    uint16_t  pid;
    uint8_t   header_ready;      // determine TS packet payload_unit_start_indicator
    uint8_t   is_IDR;
    uint8_t   start;

    int64_t  dts;   // decoding time stamp in 90kHz unit
    int64_t  pts;   // presentation time stamp in 90kHz unit

    uint8_t   header_data[256];     // for header, PPS SPS etc
    size_t    header_len;
    uint8_t*  payload;
    size_t    payload_len;

    size_t    filled;       // number of bytes already filled in TS packets

} ts_muxer_h264_pes_t;

typedef struct {
    uint16_t            pid;
    uint8_t             start;

    int64_t             pts;   // presentation time stamp in 90kHz unit

    uint8_t                header_data[256];     // PES header + ADTS header
    size_t                header_len;
    uint8_t*  payload;
    size_t    payload_len;

    size_t                filled;       // number of bytes already filled in TS packets

} ts_muxer_aac_pes_t;

// 1 byte stream type used in PMT table
#define TS_MUXER_STREAM_TYPE_H264 0x1B
#define TS_MUXER_STREAM_TYPE_AAC  0x0F
#define TS_MUXER_STREAM_TYPE_MP3  0x03

typedef struct {
    int          stream_index;  // stream index of av_context_t.streams
    uint16_t             pid;   // PID of corresponding TS packet for this ES
    // 0 indicates this ES not exists
    uint8_t               stream_type;
    uint8_t               done;  // indicate whole stream is encoded in TS stream
    uint8_t               continuity_count; // for TS packet header
    ts_muxer_h264_pes_t   pes;    // current PES, used to compose TS packet

} ts_muxer_h264_stream_t;


typedef struct {
    int          stream_index;  // stream index of av_context_t.streams
    uint16_t             pid;   // PID of corresponding TS packet for this ES
    // 0 indicates this ES not exists
    av_stream_codec_t     stream_codec;
    uint8_t               stream_type;

    uint32_t            frame_count;              // count of AAC audio ADTS frame in one PES
    // each frame consists of one raw data block, which is 1024 samples (audio sample)
    uint8_t               done;

    uint8_t               object_type_indication;   // ObjectTypeIndication in DecoderConfigDescriptor
    uint8_t               aac_audio_object_type;    // audioObjectType from AudioSpecificConfig
    // used in ADTS fixed header
    uint8_t               aac_sampling_frequency_index;  // samplingFrequencyIndex from AudioSpecificConfig
    // used in ADTS fixed header
    uint8_t               aac_channel_config;       // channelConfiguration from AudioSpecificConfig

    uint8_t               continuity_count; // for TS packet header

    ts_muxer_aac_pes_t pes;    // current PES, used to compose TS packet

} ts_muxer_aac_stream_t;

typedef struct {
    uint16_t             pmt_pid;       // can be choose from 0x0020-0x1FFA
    uint16_t             pcr_pid;       // pid of stream which contain pcr
    ts_muxer_h264_stream_t  video_stream;
    ts_muxer_aac_stream_t   audio_stream;
} ts_muxer_program_t;


struct _ts_muxer {
    av_stream_t* av_streams;
    int av_stream_count;
    void* avio_context;
    avio_write_func avio_write;
    ts_muxer_program_t program;
    ts_muxer_ts_packet_t ts_packet;
    uint8_t pat_continuity_count;
    uint8_t pmt_continuity_count;
};

#define ts_muxer_set_32value(p, n)                                             \
    ((uint8_t *) (p))[0] = (uint8_t) ((n) >> 24);                               \
    ((uint8_t *) (p))[1] = (uint8_t) ((n) >> 16);                               \
    ((uint8_t *) (p))[2] = (uint8_t) ((n) >> 8);                                \
    ((uint8_t *) (p))[3] = (uint8_t)  (n)


uint32_t ts_muxer_crc32_table[256];
uint8_t ts_muxer_crc32_table_inited = 0;

void ts_muxer_init_crc32_table()
{
    if (ts_muxer_crc32_table_inited) {
        return;
    }

    uint32_t i, j, k;
    for( i = 0; i < 256; i++ )
    {
        k = 0;
        for( j = (i << 24) | 0x800000; j != 0x80000000; j <<= 1 )
            k = (k << 1) ^ (((k ^ j) & 0x80000000) ? 0x04c11db7 : 0);

        ts_muxer_crc32_table[i] = k;
    }

    ts_muxer_crc32_table_inited = 1;
}

uint32_t ts_muxer_calc_crc32(uint8_t *data, uint32_t nLen)
{
    uint32_t     i;
    uint32_t     i_crc = 0xffffffff;

    for( i = 0; i < nLen; i++ ) {
        i_crc = (i_crc << 8) ^
                ts_muxer_crc32_table[((i_crc >> 24) ^ data[i]) & 0xff];
    }
    return i_crc;
}


const u_int32_t ts_muxer_aac_sample_frequencies[] = {
        96000,
        88200,
        64000,
        48000,
        44100,
        32000,
        24000,
        22050,
        16000,
        12000,
        11025,
        8000,
        7350,
        0           // mark the end
};

/* TS packet IEC 13818-1:2000 2.4.3
 *
 */
int ts_muxer_prepare_ts_packet_info(ts_muxer_ts_packet_t *packet, uint8_t payload_type, void *payload,
                                    uint16_t pcr_pid,
                                    uint8_t continuity_count)
{
    ts_muxer_ts_pat_t *pat = NULL;
    ts_muxer_ts_pmt_t *pmt = NULL;
    ts_muxer_h264_pes_t *pes = NULL;
    ts_muxer_aac_pes_t  *aac_pes = NULL;
    size_t           payload_remain;
    uint8_t           is_adaptation_filed=0;

    if (!packet || !payload) {
        return -1;
    }

    memset(packet, 0, sizeof(ts_muxer_ts_packet_t));

    packet->continuity_count = continuity_count;
    packet->len = 4;  // 1 byte sync, 2 bytes indicator and PID, 1 byte control and continuity count

    if (TS_MUXER_PAYLOAD_PAT == payload_type) {
        pat = (ts_muxer_ts_pat_t*)payload;
        packet->pid = 0;
        packet->payload_unit_start_indicator = pat->start;
        if ((size_t)(188-packet->len) > pat->remain)
            packet->end_stuffing_size = 188-packet->len-pat->remain;
    } else if (TS_MUXER_PAYLOAD_PMT == payload_type) {
        pmt = (ts_muxer_ts_pmt_t*)payload;
        packet->pid = pmt->pid;
        packet->payload_unit_start_indicator = pmt->start;
        if ((size_t)(188-packet->len) > pmt->remain)
            packet->end_stuffing_size = 188-packet->len-pmt->remain;
    } else if (TS_MUXER_PAYLOAD_H264_PES == payload_type) {
        pes = (ts_muxer_h264_pes_t*)payload;
        packet->pid = pes->pid;
        packet->payload_unit_start_indicator = pes->start;
        if (pes->is_IDR) {
            packet->es_priority_indicator = 1;
        }
        if (pes->start && pes->is_IDR) {
            packet->random_access_indicator = 1;
        }
        // for access unit start
        if (pes->start && packet->pid == pcr_pid) {
            // prepare PCR
            if(pes->dts != NOPTS_VALUE){
                packet->pcr = pes->dts - TS_PTS_MAX_DELAY; /* 63000 delay*/                
            }else{
                packet->pcr = pes->pts - TS_PTS_MAX_DELAY; /* 63000 delay*/
            }
            
            packet->write_pcr = 1;
            is_adaptation_filed = 1;
            packet->len += 8;
        }
        payload_remain = pes->header_len + pes->payload_len - pes->filled;
        if (payload_remain < 0) {
            return -1;
        }
        if (payload_remain < (size_t)188 - packet->len) {
            if (188 - packet->len - payload_remain >= 3) {
                packet->header_stuffing_size = 188 - packet->len - payload_remain;
                if (!is_adaptation_filed) {
                    packet->header_stuffing_size -= 2; // need to add adaptation field header
                    is_adaptation_filed = 1;
                }
                packet->len = 188 - payload_remain;
            } else {
                // fill only 1 byte stuff here, and move the rest of the remain to next packet
                if (!is_adaptation_filed) {
                    packet->len += 3; // 2 byte adaption field header + 1 byte stuffing;
                    is_adaptation_filed = 1;
                    packet->header_stuffing_size = 1;
                } else {
                    packet->len += 3;
                    packet->header_stuffing_size = 3;
                }
            }
        }
    } else if (TS_MUXER_PAYLOAD_AAC_PES == payload_type) {
        aac_pes = (ts_muxer_aac_pes_t*)payload;
        packet->pid = aac_pes->pid;
        packet->payload_unit_start_indicator = aac_pes->start;
        payload_remain = aac_pes->header_len + aac_pes->payload_len - aac_pes->filled;
        if (aac_pes->start && packet->pid == pcr_pid) {
            // prepare PCR
            packet->pcr = aac_pes->pts - TS_PTS_MAX_DELAY; /* 63000 delay*/
            packet->write_pcr = 1;
            is_adaptation_filed = 1;
            packet->len += 8;
        }
        if (payload_remain < 0) {
            return -1;
        }
        if (payload_remain < (size_t)188 - packet->len) {
            if (188 - packet->len - payload_remain >= 3) {
                packet->header_stuffing_size = 188 - packet->len - payload_remain;
                if (!is_adaptation_filed) {
                    packet->header_stuffing_size -= 2; // need to add adaptation field header
                    is_adaptation_filed = 1;
                }
                packet->len = 188 - payload_remain;
            } else {
                // fill only 1 byte stuff here, and move the rest of the remain to next packet
                if (!is_adaptation_filed) {
                    packet->len += 3; // 2 byte adaption field header + 1 byte stuffing;
                    is_adaptation_filed = 1;
                    packet->header_stuffing_size = 1;
                } else {
                    packet->len += 3;
                    packet->header_stuffing_size = 3;
                }
            }
        }
    } else if (TS_MUXER_PAYLOAD_EMPTY_PES == payload_type) {
        packet->pid = *(uint16_t*)payload;
        packet->payload_unit_start_indicator = 0;
        packet->len += 2;  // 2 byte adaption field header
        packet->header_stuffing_size = 188 - packet->len;
        packet->len = 188;
    } else {
        return -1;
    }

    return 0;
}



/* TS packet IEC 13818-1:2000 2.4.3
 *
 */
uint8_t* ts_muxer_enc_packet_header(ts_muxer_ts_packet_t *packet, uint8_t *buf)
{
    uint32_t    len, stuff_len;

    // buf should always be large enough for the packet content
    if (!packet || !buf) {
        return NULL;
    }

    *buf++ = 0x47;  // 1 byte sync
    // 2 bytes indicators and PID
    *buf++ = packet->pid >> 8 | (packet->payload_unit_start_indicator ? 0x40 : 0);
    *buf++ = packet->pid;
    // continuity count and adaptation field control
    if (packet->write_pcr || packet->header_stuffing_size) {
        *buf++ = packet->continuity_count | 0x30;
        len = 1+(packet->write_pcr?6:0)+packet->header_stuffing_size;    // adaptation_field_length = flag + PCR + stuffing
        *buf++ = len;
        *buf++ = 0 | (packet->random_access_indicator?0x60:0) | (packet->write_pcr?0x10:0); // 1 byte indicators
        if (packet->write_pcr) {
            // write 6bytes PCR
            *buf++ = packet->pcr >> 25;
            *buf++ = packet->pcr >> 17;
            *buf++ = packet->pcr >> 9;
            *buf++ = packet->pcr >> 1;
            *buf++ = (packet->pcr & 1) | 0x7E;
            *buf++ = 0;
        }
        // stuff
        stuff_len = packet->header_stuffing_size;
        while (stuff_len > 0) {
            *buf++ = 0xFF;
            stuff_len--;
        }
    } else {
        *buf++ = packet->continuity_count | 0x10;
    }

    return buf;
}
<commit_msg>support priority indicator<commit_after>#include <stddef.h>
#include <string.h>
#include <stdlib.h>
#include <libavutil/log.h>

#include "config.h"

#include "ts_muxer.h"
#include "libcseg.h"


// TS payload type
#define TS_MUXER_PAYLOAD_PAT          0
#define TS_MUXER_PAYLOAD_PMT          1
#define TS_MUXER_PAYLOAD_EMPTY_PES    2
#define TS_MUXER_PAYLOAD_H264_PES     10
#define TS_MUXER_PAYLOAD_AAC_PES      20

#define TS_MUXER_TX_PACKET_SIZE  188


#define TS_PTS_MAX_DELAY    63000  /*1/90K, 0.7sec*/

#define MIN(a,b) (((a)<(b))?(a):(b))
#define MAX(a,b) (((a)>(b))?(a):(b))

#define H264_NAL_TYPE_AUD   9


typedef struct {
    uint8_t                len;
    uint16_t             pid;
    uint8_t                payload_unit_start_indicator;
    uint8_t                continuity_count;

    int64_t             pcr;  // 90kHz time
    uint8_t                write_pcr;
    uint8_t                random_access_indicator;
    uint8_t                es_priority_indicator;

    uint8_t                header_stuffing_size;
    uint8_t                end_stuffing_size;

    uint8_t       buf[TS_MUXER_TX_PACKET_SIZE];

} ts_muxer_ts_packet_t;

// TS packet IEC 13818-1:2000 2.4.4.3
typedef struct {
    uint8_t                start;
    uint16_t              pid;
    size_t                size;
    size_t                remain;
    uint8_t                program_cnt;
    uint16_t             pmt_pid;
} ts_muxer_ts_pat_t;

typedef struct {
    uint8_t                start;
    uint16_t              pid;
    size_t                size;
    size_t                remain;

}ts_muxer_ts_pmt_t;

typedef struct {
    uint16_t  pid;
    uint8_t   header_ready;      // determine TS packet payload_unit_start_indicator
    uint8_t   is_IDR;
    uint8_t   start;

    int64_t  dts;   // decoding time stamp in 90kHz unit
    int64_t  pts;   // presentation time stamp in 90kHz unit

    uint8_t   header_data[256];     // for header, PPS SPS etc
    size_t    header_len;
    uint8_t*  payload;
    size_t    payload_len;

    size_t    filled;       // number of bytes already filled in TS packets

} ts_muxer_h264_pes_t;

typedef struct {
    uint16_t            pid;
    uint8_t             start;

    int64_t             pts;   // presentation time stamp in 90kHz unit

    uint8_t                header_data[256];     // PES header + ADTS header
    size_t                header_len;
    uint8_t*  payload;
    size_t    payload_len;

    size_t                filled;       // number of bytes already filled in TS packets

} ts_muxer_aac_pes_t;

// 1 byte stream type used in PMT table
#define TS_MUXER_STREAM_TYPE_H264 0x1B
#define TS_MUXER_STREAM_TYPE_AAC  0x0F
#define TS_MUXER_STREAM_TYPE_MP3  0x03

typedef struct {
    int          stream_index;  // stream index of av_context_t.streams
    uint16_t             pid;   // PID of corresponding TS packet for this ES
    // 0 indicates this ES not exists
    uint8_t               stream_type;
    uint8_t               done;  // indicate whole stream is encoded in TS stream
    uint8_t               continuity_count; // for TS packet header
    ts_muxer_h264_pes_t   pes;    // current PES, used to compose TS packet

} ts_muxer_h264_stream_t;


typedef struct {
    int          stream_index;  // stream index of av_context_t.streams
    uint16_t             pid;   // PID of corresponding TS packet for this ES
    // 0 indicates this ES not exists
    av_stream_codec_t     stream_codec;
    uint8_t               stream_type;

    uint32_t            frame_count;              // count of AAC audio ADTS frame in one PES
    // each frame consists of one raw data block, which is 1024 samples (audio sample)
    uint8_t               done;

    uint8_t               object_type_indication;   // ObjectTypeIndication in DecoderConfigDescriptor
    uint8_t               aac_audio_object_type;    // audioObjectType from AudioSpecificConfig
    // used in ADTS fixed header
    uint8_t               aac_sampling_frequency_index;  // samplingFrequencyIndex from AudioSpecificConfig
    // used in ADTS fixed header
    uint8_t               aac_channel_config;       // channelConfiguration from AudioSpecificConfig

    uint8_t               continuity_count; // for TS packet header

    ts_muxer_aac_pes_t pes;    // current PES, used to compose TS packet

} ts_muxer_aac_stream_t;

typedef struct {
    uint16_t             pmt_pid;       // can be choose from 0x0020-0x1FFA
    uint16_t             pcr_pid;       // pid of stream which contain pcr
    ts_muxer_h264_stream_t  video_stream;
    ts_muxer_aac_stream_t   audio_stream;
} ts_muxer_program_t;


struct _ts_muxer {
    av_stream_t* av_streams;
    int av_stream_count;
    void* avio_context;
    avio_write_func avio_write;
    ts_muxer_program_t program;
    ts_muxer_ts_packet_t ts_packet;
    uint8_t pat_continuity_count;
    uint8_t pmt_continuity_count;
};

#define ts_muxer_set_32value(p, n)                                             \
    ((uint8_t *) (p))[0] = (uint8_t) ((n) >> 24);                               \
    ((uint8_t *) (p))[1] = (uint8_t) ((n) >> 16);                               \
    ((uint8_t *) (p))[2] = (uint8_t) ((n) >> 8);                                \
    ((uint8_t *) (p))[3] = (uint8_t)  (n)


uint32_t ts_muxer_crc32_table[256];
uint8_t ts_muxer_crc32_table_inited = 0;

void ts_muxer_init_crc32_table()
{
    if (ts_muxer_crc32_table_inited) {
        return;
    }

    uint32_t i, j, k;
    for( i = 0; i < 256; i++ )
    {
        k = 0;
        for( j = (i << 24) | 0x800000; j != 0x80000000; j <<= 1 )
            k = (k << 1) ^ (((k ^ j) & 0x80000000) ? 0x04c11db7 : 0);

        ts_muxer_crc32_table[i] = k;
    }

    ts_muxer_crc32_table_inited = 1;
}

uint32_t ts_muxer_calc_crc32(uint8_t *data, uint32_t nLen)
{
    uint32_t     i;
    uint32_t     i_crc = 0xffffffff;

    for( i = 0; i < nLen; i++ ) {
        i_crc = (i_crc << 8) ^
                ts_muxer_crc32_table[((i_crc >> 24) ^ data[i]) & 0xff];
    }
    return i_crc;
}


const u_int32_t ts_muxer_aac_sample_frequencies[] = {
        96000,
        88200,
        64000,
        48000,
        44100,
        32000,
        24000,
        22050,
        16000,
        12000,
        11025,
        8000,
        7350,
        0           // mark the end
};

/* TS packet IEC 13818-1:2000 2.4.3
 *
 */
int ts_muxer_prepare_ts_packet_info(ts_muxer_ts_packet_t *packet, uint8_t payload_type, void *payload,
                                    uint16_t pcr_pid,
                                    uint8_t continuity_count)
{
    ts_muxer_ts_pat_t *pat = NULL;
    ts_muxer_ts_pmt_t *pmt = NULL;
    ts_muxer_h264_pes_t *pes = NULL;
    ts_muxer_aac_pes_t  *aac_pes = NULL;
    size_t           payload_remain;
    uint8_t           is_adaptation_filed=0;

    if (!packet || !payload) {
        return -1;
    }

    memset(packet, 0, sizeof(ts_muxer_ts_packet_t));

    packet->continuity_count = continuity_count;
    packet->len = 4;  // 1 byte sync, 2 bytes indicator and PID, 1 byte control and continuity count

    if (TS_MUXER_PAYLOAD_PAT == payload_type) {
        pat = (ts_muxer_ts_pat_t*)payload;
        packet->pid = 0;
        packet->payload_unit_start_indicator = pat->start;
        if ((size_t)(188-packet->len) > pat->remain)
            packet->end_stuffing_size = 188-packet->len-pat->remain;
    } else if (TS_MUXER_PAYLOAD_PMT == payload_type) {
        pmt = (ts_muxer_ts_pmt_t*)payload;
        packet->pid = pmt->pid;
        packet->payload_unit_start_indicator = pmt->start;
        if ((size_t)(188-packet->len) > pmt->remain)
            packet->end_stuffing_size = 188-packet->len-pmt->remain;
    } else if (TS_MUXER_PAYLOAD_H264_PES == payload_type) {
        pes = (ts_muxer_h264_pes_t*)payload;
        packet->pid = pes->pid;
        packet->payload_unit_start_indicator = pes->start;
        if (pes->is_IDR) {
            packet->es_priority_indicator = 1;
        }
        // for access unit start and pcr 
        if (pes->start && packet->pid == pcr_pid) {
            if(pes->is_IDR){
                packet->random_access_indicator = 1;
            }
            // prepare PCR
            if(pes->dts != NOPTS_VALUE){
                packet->pcr = pes->dts - TS_PTS_MAX_DELAY; /* 63000 delay*/                
            }else{
                packet->pcr = pes->pts - TS_PTS_MAX_DELAY; /* 63000 delay*/
            }
            
            packet->write_pcr = 1;
            is_adaptation_filed = 1;
            packet->len += 8;
        }
        payload_remain = pes->header_len + pes->payload_len - pes->filled;
        if (payload_remain < 0) {
            return -1;
        }
        if (payload_remain < (size_t)188 - packet->len) {
            if (188 - packet->len - payload_remain >= 3) {
                packet->header_stuffing_size = 188 - packet->len - payload_remain;
                if (!is_adaptation_filed) {
                    packet->header_stuffing_size -= 2; // need to add adaptation field header
                    is_adaptation_filed = 1;
                }
                packet->len = 188 - payload_remain;
            } else {
                // fill only 1 byte stuff here, and move the rest of the remain to next packet
                if (!is_adaptation_filed) {
                    packet->len += 3; // 2 byte adaption field header + 1 byte stuffing;
                    is_adaptation_filed = 1;
                    packet->header_stuffing_size = 1;
                } else {
                    packet->len += 3;
                    packet->header_stuffing_size = 3;
                }
            }
        }
    } else if (TS_MUXER_PAYLOAD_AAC_PES == payload_type) {
        aac_pes = (ts_muxer_aac_pes_t*)payload;
        packet->pid = aac_pes->pid;
        packet->payload_unit_start_indicator = aac_pes->start;
        payload_remain = aac_pes->header_len + aac_pes->payload_len - aac_pes->filled;
        if (aac_pes->start && packet->pid == pcr_pid) {
            packet->random_access_indicator = 1;
            // prepare PCR
            packet->pcr = aac_pes->pts - TS_PTS_MAX_DELAY; /* 63000 delay*/
            packet->write_pcr = 1;
            is_adaptation_filed = 1;
            packet->len += 8;
        }
        if (payload_remain < 0) {
            return -1;
        }
        if (payload_remain < (size_t)188 - packet->len) {
            if (188 - packet->len - payload_remain >= 3) {
                packet->header_stuffing_size = 188 - packet->len - payload_remain;
                if (!is_adaptation_filed) {
                    packet->header_stuffing_size -= 2; // need to add adaptation field header
                    is_adaptation_filed = 1;
                }
                packet->len = 188 - payload_remain;
            } else {
                // fill only 1 byte stuff here, and move the rest of the remain to next packet
                if (!is_adaptation_filed) {
                    packet->len += 3; // 2 byte adaption field header + 1 byte stuffing;
                    is_adaptation_filed = 1;
                    packet->header_stuffing_size = 1;
                } else {
                    packet->len += 3;
                    packet->header_stuffing_size = 3;
                }
            }
        }
    } else if (TS_MUXER_PAYLOAD_EMPTY_PES == payload_type) {
        packet->pid = *(uint16_t*)payload;
        packet->payload_unit_start_indicator = 0;
        packet->len += 2;  // 2 byte adaption field header
        packet->header_stuffing_size = 188 - packet->len;
        packet->len = 188;
    } else {
        return -1;
    }

    return 0;
}



/* TS packet IEC 13818-1:2000 2.4.3
 *
 */
uint8_t* ts_muxer_enc_packet_header(ts_muxer_ts_packet_t *packet, uint8_t *buf)
{
    uint32_t    len, stuff_len;

    // buf should always be large enough for the packet content
    if (!packet || !buf) {
        return NULL;
    }

    *buf++ = 0x47;  // 1 byte sync
    // 2 bytes indicators and PID
    *buf++ = packet->pid >> 8 | (packet->payload_unit_start_indicator ? 0x40 : 0);
    *buf++ = packet->pid;
    // continuity count and adaptation field control
    if (packet->write_pcr || packet->header_stuffing_size) {
        *buf++ = packet->continuity_count | 0x30;
        len = 1+(packet->write_pcr?6:0)+packet->header_stuffing_size;    // adaptation_field_length = flag + PCR + stuffing
        *buf++ = len;
        *buf++ = 0 | (packet->random_access_indicator?0x40:0) | (packet->es_priority_indicator?0x20:0) |
                (packet->write_pcr?0x10:0); // 1 byte indicators
        if (packet->write_pcr) {
            // write 6bytes PCR
            *buf++ = packet->pcr >> 25;
            *buf++ = packet->pcr >> 17;
            *buf++ = packet->pcr >> 9;
            *buf++ = packet->pcr >> 1;
            *buf++ = (packet->pcr & 1) | 0x7E;
            *buf++ = 0;
        }
        // stuff
        stuff_len = packet->header_stuffing_size;
        while (stuff_len > 0) {
            *buf++ = 0xFF;
            stuff_len--;
        }
    } else {
        *buf++ = packet->continuity_count | 0x10;
    }

    return buf;
}

{'fraction_non_alphanumeric': 0.09622006274389777, 'fraction_numerical': 0.03944448695386028, 'mean_word_length': 1.7508945485161018, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 254, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 6, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2312397', 'n_tokens_mistral': 9750, 'n_tokens_neox': 8710, 'n_words': 1937}