<commit_before>/*
 * Copyright © 2010 Mozilla Foundation
 *
 * This program is made available under an ISC-style license.  See the
 * accompanying file LICENSE for details.
 */
#include <assert.h>
#include <stdlib.h>
#include <string.h>

#include "halloc.h"
#include "nestegg/nestegg.h"

/* EBML Elements */
#define ID_EBML                 0x1a45dfa3
#define ID_EBML_VERSION         0x4286
#define ID_EBML_READ_VERSION    0x42f7
#define ID_EBML_MAX_ID_LENGTH   0x42f2
#define ID_EBML_MAX_SIZE_LENGTH 0x42f3
#define ID_DOCTYPE              0x4282
#define ID_DOCTYPE_VERSION      0x4287
#define ID_DOCTYPE_READ_VERSION 0x4285

/* Global Elements */
#define ID_VOID                 0xec
#define ID_CRC32                0xbf

/* WebM Elements */
#define ID_SEGMENT              0x18538067

/* Seek Head Elements */
#define ID_SEEK_HEAD            0x114d9b74
#define ID_SEEK                 0x4dbb
#define ID_SEEK_ID              0x53ab
#define ID_SEEK_POSITION        0x53ac

/* Info Elements */
#define ID_INFO                 0x1549a966
#define ID_TIMECODE_SCALE       0x2ad7b1
#define ID_DURATION             0x4489

/* Cluster Elements */
#define ID_CLUSTER              0x1f43b675
#define ID_TIMECODE             0xe7
#define ID_BLOCK_GROUP          0xa0
#define ID_SIMPLE_BLOCK         0xa3

/* BlockGroup Elements */
#define ID_BLOCK                0xa1
#define ID_BLOCK_ADDITIONS      0x75a1
#define ID_BLOCK_DURATION       0x9b
#define ID_REFERENCE_BLOCK      0xfb
#define ID_DISCARD_PADDING      0x75a2

/* BlockAdditions Elements */
#define ID_BLOCK_MORE           0xa6

/* BlockMore Elements */
#define ID_BLOCK_ADD_ID         0xee
#define ID_BLOCK_ADDITIONAL     0xa5

/* Tracks Elements */
#define ID_TRACKS               0x1654ae6b
#define ID_TRACK_ENTRY          0xae
#define ID_TRACK_NUMBER         0xd7
#define ID_TRACK_UID            0x73c5
#define ID_TRACK_TYPE           0x83
#define ID_FLAG_ENABLED         0xb9
#define ID_FLAG_DEFAULT         0x88
#define ID_FLAG_LACING          0x9c
#define ID_TRACK_TIMECODE_SCALE 0x23314f
#define ID_LANGUAGE             0x22b59c
#define ID_CODEC_ID             0x86
#define ID_CODEC_PRIVATE        0x63a2
#define ID_CODEC_DELAY          0x56aa
#define ID_SEEK_PREROLL         0x56bb
#define ID_DEFAULT_DURATION     0x23e383

/* Video Elements */
#define ID_VIDEO                0xe0
#define ID_STEREO_MODE          0x53b8
#define ID_ALPHA_MODE           0x53c0
#define ID_PIXEL_WIDTH          0xb0
#define ID_PIXEL_HEIGHT         0xba
#define ID_PIXEL_CROP_BOTTOM    0x54aa
#define ID_PIXEL_CROP_TOP       0x54bb
#define ID_PIXEL_CROP_LEFT      0x54cc
#define ID_PIXEL_CROP_RIGHT     0x54dd
#define ID_DISPLAY_WIDTH        0x54b0
#define ID_DISPLAY_HEIGHT       0x54ba

/* Audio Elements */
#define ID_AUDIO                0xe1
#define ID_SAMPLING_FREQUENCY   0xb5
#define ID_CHANNELS             0x9f
#define ID_BIT_DEPTH            0x6264

/* Cues Elements */
#define ID_CUES                 0x1c53bb6b
#define ID_CUE_POINT            0xbb
#define ID_CUE_TIME             0xb3
#define ID_CUE_TRACK_POSITIONS  0xb7
#define ID_CUE_TRACK            0xf7
#define ID_CUE_CLUSTER_POSITION 0xf1
#define ID_CUE_BLOCK_NUMBER     0x5378

/* EBML Types */
enum ebml_type_enum {
  TYPE_UNKNOWN,
  TYPE_MASTER,
  TYPE_UINT,
  TYPE_FLOAT,
  TYPE_INT,
  TYPE_STRING,
  TYPE_BINARY
};

#define LIMIT_STRING            (1 << 20)
#define LIMIT_BINARY            (1 << 24)
#define LIMIT_BLOCK             (1 << 30)
#define LIMIT_FRAME             (1 << 28)

/* Field Flags */
#define DESC_FLAG_NONE          0
#define DESC_FLAG_MULTI         (1 << 0)
#define DESC_FLAG_SUSPEND       (1 << 1)
#define DESC_FLAG_OFFSET        (1 << 2)

/* Block Header Flags */
#define BLOCK_FLAGS_LACING      6

/* Lacing Constants */
#define LACING_NONE             0
#define LACING_XIPH             1
#define LACING_FIXED            2
#define LACING_EBML             3

/* Track Types */
#define TRACK_TYPE_VIDEO        1
#define TRACK_TYPE_AUDIO        2

/* Track IDs */
#define TRACK_ID_VP8            "V_VP8"
#define TRACK_ID_VP9            "V_VP9"
#define TRACK_ID_VORBIS         "A_VORBIS"
#define TRACK_ID_OPUS           "A_OPUS"

enum vint_mask {
  MASK_NONE,
  MASK_FIRST_BIT
};

struct ebml_binary {
  unsigned char * data;
  size_t length;
};

struct ebml_list_node {
  struct ebml_list_node * next;
  uint64_t id;
  void * data;
};

struct ebml_list {
  struct ebml_list_node * head;
  struct ebml_list_node * tail;
};

struct ebml_type {
  union ebml_value {
    uint64_t u;
    double f;
    int64_t i;
    char * s;
    struct ebml_binary b;
  } v;
  enum ebml_type_enum type;
  int read;
};

/* EBML Definitions */
struct ebml {
  struct ebml_type ebml_version;
  struct ebml_type ebml_read_version;
  struct ebml_type ebml_max_id_length;
  struct ebml_type ebml_max_size_length;
  struct ebml_type doctype;
  struct ebml_type doctype_version;
  struct ebml_type doctype_read_version;
};

/* Matroksa Definitions */
struct seek {
  struct ebml_type id;
  struct ebml_type position;
};

struct seek_head {
  struct ebml_list seek;
};

struct info {
  struct ebml_type timecode_scale;
  struct ebml_type duration;
};

struct block_more {
  struct ebml_type block_add_id;
  struct ebml_type block_additional;
};

struct block_additions {
  struct ebml_list block_more;
};

struct block_group {
  struct ebml_type block_additions;
  struct ebml_type duration;
  struct ebml_type reference_block;
  struct ebml_type discard_padding;
};

struct cluster {
  struct ebml_type timecode;
  struct ebml_list block_group;
};

struct video {
  struct ebml_type stereo_mode;
  struct ebml_type alpha_mode;
  struct ebml_type pixel_width;
  struct ebml_type pixel_height;
  struct ebml_type pixel_crop_bottom;
  struct ebml_type pixel_crop_top;
  struct ebml_type pixel_crop_left;
  struct ebml_type pixel_crop_right;
  struct ebml_type display_width;
  struct ebml_type display_height;
};

struct audio {
  struct ebml_type sampling_frequency;
  struct ebml_type channels;
  struct ebml_type bit_depth;
};

struct track_entry {
  struct ebml_type number;
  struct ebml_type uid;
  struct ebml_type type;
  struct ebml_type flag_enabled;
  struct ebml_type flag_default;
  struct ebml_type flag_lacing;
  struct ebml_type track_timecode_scale;
  struct ebml_type language;
  struct ebml_type codec_id;
  struct ebml_type codec_private;
  struct ebml_type codec_delay;
  struct ebml_type seek_preroll;
  struct ebml_type default_duration;
  struct video video;
  struct audio audio;
};

struct tracks {
  struct ebml_list track_entry;
};

struct cue_track_positions {
  struct ebml_type track;
  struct ebml_type cluster_position;
  struct ebml_type block_number;
};

struct cue_point {
  struct ebml_type time;
  struct ebml_list cue_track_positions;
};

struct cues {
  struct ebml_list cue_point;
};

struct segment {
  struct ebml_list seek_head;
  struct info info;
  struct ebml_list cluster;
  struct tracks tracks;
  struct cues cues;
};

/* Misc. */
struct pool_ctx {
  char dummy;
};

struct list_node {
  struct list_node * previous;
  struct ebml_element_desc * node;
  unsigned char * data;
};

struct saved_state {
  int64_t stream_offset;
  struct list_node * ancestor;
  uint64_t last_id;
  uint64_t last_size;
  int last_valid;
};

struct frame {
  unsigned char * data;
  size_t length;
  struct frame * next;
};

struct block_additional {
  unsigned int id;
  unsigned char * data;
  size_t length;
  struct block_additional * next;
};

#define NE_IO_BUFSZ 16384

struct nestegg_io_buf {
  nestegg_io io;
  uint8_t buffer[NE_IO_BUFSZ];
  size_t bufsz;
  int32_t offset;
};

/* Public (opaque) Structures */
struct nestegg {
  struct nestegg_io_buf * io;
  nestegg_log log;
  struct pool_ctx * alloc_pool;
  uint64_t last_id;
  uint64_t last_size;
  int last_valid;
  struct list_node * ancestor;
  struct ebml ebml;
  struct segment segment;
  int64_t segment_offset;
  unsigned int track_count;
};

struct nestegg_packet {
  uint64_t track;
  uint64_t timecode;
  uint64_t duration;
  struct frame * frame;
  struct block_additional * block_additional;
  int64_t discard_padding;
};

/* Element Descriptor */
struct ebml_element_desc {
  char const * name;
  uint64_t id;
  enum ebml_type_enum type;
  size_t offset;
  unsigned int flags;
  struct ebml_element_desc * children;
  size_t size;
  size_t data_offset;
};

#define E_FIELD(ID, TYPE, STRUCT, FIELD) \
  { #ID, ID, TYPE, offsetof(STRUCT, FIELD), DESC_FLAG_NONE, NULL, 0, 0 }
#define E_MASTER(ID, TYPE, STRUCT, FIELD) \
  { #ID, ID, TYPE, offsetof(STRUCT, FIELD), DESC_FLAG_MULTI, ne_ ## FIELD ## _elements, \
      sizeof(struct FIELD), 0 }
#define E_SINGLE_MASTER_O(ID, TYPE, STRUCT, FIELD) \
  { #ID, ID, TYPE, offsetof(STRUCT, FIELD), DESC_FLAG_OFFSET, ne_ ## FIELD ## _elements, 0, \
      offsetof(STRUCT, FIELD ## _offset) }
#define E_SINGLE_MASTER(ID, TYPE, STRUCT, FIELD) \
  { #ID, ID, TYPE, offsetof(STRUCT, FIELD), DESC_FLAG_NONE, ne_ ## FIELD ## _elements, 0, 0 }
#define E_SUSPEND(ID, TYPE) \
  { #ID, ID, TYPE, 0, DESC_FLAG_SUSPEND, NULL, 0, 0 }
#define E_LAST \
  { NULL, 0, 0, 0, DESC_FLAG_NONE, NULL, 0, 0 }

/* EBML Element Lists */
static struct ebml_element_desc ne_ebml_elements[] = {
  E_FIELD(ID_EBML_VERSION, TYPE_UINT, struct ebml, ebml_version),
  E_FIELD(ID_EBML_READ_VERSION, TYPE_UINT, struct ebml, ebml_read_version),
  E_FIELD(ID_EBML_MAX_ID_LENGTH, TYPE_UINT, struct ebml, ebml_max_id_length),
  E_FIELD(ID_EBML_MAX_SIZE_LENGTH, TYPE_UINT, struct ebml, ebml_max_size_length),
  E_FIELD(ID_DOCTYPE, TYPE_STRING, struct ebml, doctype),
  E_FIELD(ID_DOCTYPE_VERSION, TYPE_UINT, struct ebml, doctype_version),
  E_FIELD(ID_DOCTYPE_READ_VERSION, TYPE_UINT, struct ebml, doctype_read_version),
  E_LAST
};

/* WebM Element Lists */
static struct ebml_element_desc ne_seek_elements[] = {
  E_FIELD(ID_SEEK_ID, TYPE_BINARY, struct seek, id),
  E_FIELD(ID_SEEK_POSITION, TYPE_UINT, struct seek, position),
  E_LAST
};

static struct ebml_element_desc ne_seek_head_elements[] = {
  E_MASTER(ID_SEEK, TYPE_MASTER, struct seek_head, seek),
  E_LAST
};

static struct ebml_element_desc ne_info_elements[] = {
  E_FIELD(ID_TIMECODE_SCALE, TYPE_UINT, struct info, timecode_scale),
  E_FIELD(ID_DURATION, TYPE_FLOAT, struct info, duration),
  E_LAST
};

static struct ebml_element_desc ne_block_more_elements[] = {
  E_FIELD(ID_BLOCK_ADD_ID, TYPE_UINT, struct block_more, block_add_id),
  E_FIELD(ID_BLOCK_ADDITIONAL, TYPE_BINARY, struct block_more, block_additional),
  E_LAST
};

static struct ebml_element_desc ne_block_additions_elements[] = {
  E_MASTER(ID_BLOCK_MORE, TYPE_MASTER, struct block_additions, block_more),
  E_LAST
};

static struct ebml_element_desc ne_block_group_elements[] = {
  E_SUSPEND(ID_BLOCK, TYPE_BINARY),
  E_FIELD(ID_BLOCK_DURATION, TYPE_UINT, struct block_group, duration),
  E_FIELD(ID_REFERENCE_BLOCK, TYPE_INT, struct block_group, reference_block),
  E_FIELD(ID_DISCARD_PADDING, TYPE_INT, struct block_group, discard_padding),
  E_SINGLE_MASTER(ID_BLOCK_ADDITIONS, TYPE_MASTER, struct block_group, block_additions),
  E_LAST
};

static struct ebml_element_desc ne_cluster_elements[] = {
  E_FIELD(ID_TIMECODE, TYPE_UINT, struct cluster, timecode),
  E_MASTER(ID_BLOCK_GROUP, TYPE_MASTER, struct cluster, block_group),
  E_SUSPEND(ID_SIMPLE_BLOCK, TYPE_BINARY),
  E_LAST
};

static struct ebml_element_desc ne_video_elements[] = {
  E_FIELD(ID_STEREO_MODE, TYPE_UINT, struct video, stereo_mode),
  E_FIELD(ID_ALPHA_MODE, TYPE_UINT, struct video, alpha_mode),
  E_FIELD(ID_PIXEL_WIDTH, TYPE_UINT, struct video, pixel_width),
  E_FIELD(ID_PIXEL_HEIGHT, TYPE_UINT, struct video, pixel_height),
  E_FIELD(ID_PIXEL_CROP_BOTTOM, TYPE_UINT, struct video, pixel_crop_bottom),
  E_FIELD(ID_PIXEL_CROP_TOP, TYPE_UINT, struct video, pixel_crop_top),
  E_FIELD(ID_PIXEL_CROP_LEFT, TYPE_UINT, struct video, pixel_crop_left),
  E_FIELD(ID_PIXEL_CROP_RIGHT, TYPE_UINT, struct video, pixel_crop_right),
  E_FIELD(ID_DISPLAY_WIDTH, TYPE_UINT, struct video, display_width),
  E_FIELD(ID_DISPLAY_HEIGHT, TYPE_UINT, struct video, display_height),
  E_LAST
};

static struct ebml_element_desc ne_audio_elements[] = {
  E_FIELD(ID_SAMPLING_FREQUENCY, TYPE_FLOAT, struct audio, sampling_frequency),
  E_FIELD(ID_CHANNELS, TYPE_UINT, struct audio, channels),
  E_FIELD(ID_BIT_DEPTH, TYPE_UINT, struct audio, bit_depth),
  E_LAST
};

static struct ebml_element_desc ne_track_entry_elements[] = {
  E_FIELD(ID_TRACK_NUMBER, TYPE_UINT, struct track_entry, number),
  E_FIELD(ID_TRACK_UID, TYPE_UINT, struct track_entry, uid),
  E_FIELD(ID_TRACK_TYPE, TYPE_UINT, struct track_entry, type),
  E_FIELD(ID_FLAG_ENABLED, TYPE_UINT, struct track_entry, flag_enabled),
  E_FIELD(ID_FLAG_DEFAULT, TYPE_UINT, struct track_entry, flag_default),
  E_FIELD(ID_FLAG_LACING, TYPE_UINT, struct track_entry, flag_lacing),
  E_FIELD(ID_TRACK_TIMECODE_SCALE, TYPE_FLOAT, struct track_entry, track_timecode_scale),
  E_FIELD(ID_LANGUAGE, TYPE_STRING, struct track_entry, language),
  E_FIELD(ID_CODEC_ID, TYPE_STRING, struct track_entry, codec_id),
  E_FIELD(ID_CODEC_PRIVATE, TYPE_BINARY, struct track_entry, codec_private),
  E_FIELD(ID_CODEC_DELAY, TYPE_UINT, struct track_entry, codec_delay),
  E_FIELD(ID_SEEK_PREROLL, TYPE_UINT, struct track_entry, seek_preroll),
  E_FIELD(ID_DEFAULT_DURATION, TYPE_UINT, struct track_entry, default_duration),
  E_SINGLE_MASTER(ID_VIDEO, TYPE_MASTER, struct track_entry, video),
  E_SINGLE_MASTER(ID_AUDIO, TYPE_MASTER, struct track_entry, audio),
  E_LAST
};

static struct ebml_element_desc ne_tracks_elements[] = {
  E_MASTER(ID_TRACK_ENTRY, TYPE_MASTER, struct tracks, track_entry),
  E_LAST
};

static struct ebml_element_desc ne_cue_track_positions_elements[] = {
  E_FIELD(ID_CUE_TRACK, TYPE_UINT, struct cue_track_positions, track),
  E_FIELD(ID_CUE_CLUSTER_POSITION, TYPE_UINT, struct cue_track_positions, cluster_position),
  E_FIELD(ID_CUE_BLOCK_NUMBER, TYPE_UINT, struct cue_track_positions, block_number),
  E_LAST
};

static struct ebml_element_desc ne_cue_point_elements[] = {
  E_FIELD(ID_CUE_TIME, TYPE_UINT, struct cue_point, time),
  E_MASTER(ID_CUE_TRACK_POSITIONS, TYPE_MASTER, struct cue_point, cue_track_positions),
  E_LAST
};

static struct ebml_element_desc ne_cues_elements[] = {
  E_MASTER(ID_CUE_POINT, TYPE_MASTER, struct cues, cue_point),
  E_LAST
};

static struct ebml_element_desc ne_segment_elements[] = {
  E_MASTER(ID_SEEK_HEAD, TYPE_MASTER, struct segment, seek_head),
  E_SINGLE_MASTER(ID_INFO, TYPE_MASTER, struct segment, info),
  E_MASTER(ID_CLUSTER, TYPE_MASTER, struct segment, cluster),
  E_SINGLE_MASTER(ID_TRACKS, TYPE_MASTER, struct segment, tracks),
  E_SINGLE_MASTER(ID_CUES, TYPE_MASTER, struct segment, cues),
  E_LAST
};

static struct ebml_element_desc ne_top_level_elements[] = {
  E_SINGLE_MASTER(ID_EBML, TYPE_MASTER, nestegg, ebml),
  E_SINGLE_MASTER_O(ID_SEGMENT, TYPE_MASTER, nestegg, segment),
  E_LAST
};

#undef E_FIELD
#undef E_MASTER
#undef E_SINGLE_MASTER_O
#undef E_SINGLE_MASTER
#undef E_SUSPEND
#undef E_LAST

static struct pool_ctx *
ne_pool_init(void)
{
  return h_malloc(sizeof(struct pool_ctx));
}

static void
ne_pool_destroy(struct pool_ctx * pool)
{
  h_free(pool);
}

static void *
ne_pool_alloc(size_t size, struct pool_ctx * pool)
{
  void * p;

  p = h_malloc(size);
  if (!p)
    return NULL;
  hattach(p, pool);
  memset(p, 0, size);
  return p;
}

static void *
ne_alloc(size_t size)
{
  return calloc(1, size);
}

static int
ne_io_read(struct nestegg_io_buf * io, void * buffer, size_t length)
{
  int64_t off;
  int r;
  size_t avail;

  assert(io->offset == -1 || (io->offset >= 0 && (uint32_t) io->offset < io->bufsz));

  /* Too big to buffer, invalidate buffer and read through */
  if (length > io->bufsz) {
    if (io->offset != -1) {
      r = io->io.seek(-(io->bufsz - io->offset), NESTEGG_SEEK_CUR, io->io.userdata);
      if (r != 0) {
        return -1;
      }
    }
    io->offset = -1;
    return io->io.read(buffer, length, io->io.userdata);
  }

  /* Buffer invalid */
  if (io->offset == -1) {
    off = io->io.tell(io->io.userdata);
    if (off == -1) {
      return -1;
    }
    /* Refill buffer */
    r = io->io.read(io->buffer, io->bufsz, io->io.userdata);
    if (r != 1) {
      /* Read truncated due to being within io->bufsz of EOS, reset read
         position and switch to read through mode */
      io->offset = -1;
      io->bufsz = 0;
      if (r == 0) {
        r = io->io.seek(off, NESTEGG_SEEK_SET, io->io.userdata);
      }
      if (r == 0) {
        return io->io.read(buffer, length, io->io.userdata);
      }
      return -1;
    }
    if (r == 1) {
      io->offset = 0;
    }
  }

  /* Service request with what we have */
  avail = length;
  if (io->bufsz - io->offset < length) {
    avail = io->bufsz - io->offset;
  }
  memcpy(buffer, io->buffer + io->offset, avail);
  io->offset += avail;

  if ((uint32_t) io->offset == io->bufsz) {
    io->offset = -1;
  }

  /* Still more to read, invalidate buffer and read more */
  if (length - avail > 0) {
    return ne_io_read(io, (char *) buffer + avail, length - avail);
  }

  return 1;
}

static int
ne_io_seek(struct nestegg_io_buf * io, int64_t offset, int whence)
{
  /* Invalidate buffer */
  io->offset = -1;

  return io->io.seek(offset, whence, io->io.userdata);
}

static int64_t
ne_io_tell(struct nestegg_io_buf * io)
{
  int64_t off;

  off = io->io.tell(io->io.userdata);
  if (off == -1) {
    return -1;
  }
  if (io->offset == -1) {
    return off;
  }
  assert(off >= (int64_t) io->bufsz - io->offset);
  return off - io->bufsz + (uint32_t) io->offset;
}

static int
ne_io_read_skip(struct nestegg_io_buf * io, size_t length)
{
  size_t get;
  unsigned char buf[8192];
  int r = 1;

  while (length > 0) {<commit_msg>Fix win32 build: C99 types issue.<commit_after>/*
 * Copyright © 2010 Mozilla Foundation
 *
 * This program is made available under an ISC-style license.  See the
 * accompanying file LICENSE for details.
 */
#include <assert.h>
#include <stdlib.h>
#include <string.h>

#include "halloc.h"
#include "nestegg/nestegg.h"

/* EBML Elements */
#define ID_EBML                 0x1a45dfa3
#define ID_EBML_VERSION         0x4286
#define ID_EBML_READ_VERSION    0x42f7
#define ID_EBML_MAX_ID_LENGTH   0x42f2
#define ID_EBML_MAX_SIZE_LENGTH 0x42f3
#define ID_DOCTYPE              0x4282
#define ID_DOCTYPE_VERSION      0x4287
#define ID_DOCTYPE_READ_VERSION 0x4285

/* Global Elements */
#define ID_VOID                 0xec
#define ID_CRC32                0xbf

/* WebM Elements */
#define ID_SEGMENT              0x18538067

/* Seek Head Elements */
#define ID_SEEK_HEAD            0x114d9b74
#define ID_SEEK                 0x4dbb
#define ID_SEEK_ID              0x53ab
#define ID_SEEK_POSITION        0x53ac

/* Info Elements */
#define ID_INFO                 0x1549a966
#define ID_TIMECODE_SCALE       0x2ad7b1
#define ID_DURATION             0x4489

/* Cluster Elements */
#define ID_CLUSTER              0x1f43b675
#define ID_TIMECODE             0xe7
#define ID_BLOCK_GROUP          0xa0
#define ID_SIMPLE_BLOCK         0xa3

/* BlockGroup Elements */
#define ID_BLOCK                0xa1
#define ID_BLOCK_ADDITIONS      0x75a1
#define ID_BLOCK_DURATION       0x9b
#define ID_REFERENCE_BLOCK      0xfb
#define ID_DISCARD_PADDING      0x75a2

/* BlockAdditions Elements */
#define ID_BLOCK_MORE           0xa6

/* BlockMore Elements */
#define ID_BLOCK_ADD_ID         0xee
#define ID_BLOCK_ADDITIONAL     0xa5

/* Tracks Elements */
#define ID_TRACKS               0x1654ae6b
#define ID_TRACK_ENTRY          0xae
#define ID_TRACK_NUMBER         0xd7
#define ID_TRACK_UID            0x73c5
#define ID_TRACK_TYPE           0x83
#define ID_FLAG_ENABLED         0xb9
#define ID_FLAG_DEFAULT         0x88
#define ID_FLAG_LACING          0x9c
#define ID_TRACK_TIMECODE_SCALE 0x23314f
#define ID_LANGUAGE             0x22b59c
#define ID_CODEC_ID             0x86
#define ID_CODEC_PRIVATE        0x63a2
#define ID_CODEC_DELAY          0x56aa
#define ID_SEEK_PREROLL         0x56bb
#define ID_DEFAULT_DURATION     0x23e383

/* Video Elements */
#define ID_VIDEO                0xe0
#define ID_STEREO_MODE          0x53b8
#define ID_ALPHA_MODE           0x53c0
#define ID_PIXEL_WIDTH          0xb0
#define ID_PIXEL_HEIGHT         0xba
#define ID_PIXEL_CROP_BOTTOM    0x54aa
#define ID_PIXEL_CROP_TOP       0x54bb
#define ID_PIXEL_CROP_LEFT      0x54cc
#define ID_PIXEL_CROP_RIGHT     0x54dd
#define ID_DISPLAY_WIDTH        0x54b0
#define ID_DISPLAY_HEIGHT       0x54ba

/* Audio Elements */
#define ID_AUDIO                0xe1
#define ID_SAMPLING_FREQUENCY   0xb5
#define ID_CHANNELS             0x9f
#define ID_BIT_DEPTH            0x6264

/* Cues Elements */
#define ID_CUES                 0x1c53bb6b
#define ID_CUE_POINT            0xbb
#define ID_CUE_TIME             0xb3
#define ID_CUE_TRACK_POSITIONS  0xb7
#define ID_CUE_TRACK            0xf7
#define ID_CUE_CLUSTER_POSITION 0xf1
#define ID_CUE_BLOCK_NUMBER     0x5378

/* EBML Types */
enum ebml_type_enum {
  TYPE_UNKNOWN,
  TYPE_MASTER,
  TYPE_UINT,
  TYPE_FLOAT,
  TYPE_INT,
  TYPE_STRING,
  TYPE_BINARY
};

#define LIMIT_STRING            (1 << 20)
#define LIMIT_BINARY            (1 << 24)
#define LIMIT_BLOCK             (1 << 30)
#define LIMIT_FRAME             (1 << 28)

/* Field Flags */
#define DESC_FLAG_NONE          0
#define DESC_FLAG_MULTI         (1 << 0)
#define DESC_FLAG_SUSPEND       (1 << 1)
#define DESC_FLAG_OFFSET        (1 << 2)

/* Block Header Flags */
#define BLOCK_FLAGS_LACING      6

/* Lacing Constants */
#define LACING_NONE             0
#define LACING_XIPH             1
#define LACING_FIXED            2
#define LACING_EBML             3

/* Track Types */
#define TRACK_TYPE_VIDEO        1
#define TRACK_TYPE_AUDIO        2

/* Track IDs */
#define TRACK_ID_VP8            "V_VP8"
#define TRACK_ID_VP9            "V_VP9"
#define TRACK_ID_VORBIS         "A_VORBIS"
#define TRACK_ID_OPUS           "A_OPUS"

enum vint_mask {
  MASK_NONE,
  MASK_FIRST_BIT
};

struct ebml_binary {
  unsigned char * data;
  size_t length;
};

struct ebml_list_node {
  struct ebml_list_node * next;
  uint64_t id;
  void * data;
};

struct ebml_list {
  struct ebml_list_node * head;
  struct ebml_list_node * tail;
};

struct ebml_type {
  union ebml_value {
    uint64_t u;
    double f;
    int64_t i;
    char * s;
    struct ebml_binary b;
  } v;
  enum ebml_type_enum type;
  int read;
};

/* EBML Definitions */
struct ebml {
  struct ebml_type ebml_version;
  struct ebml_type ebml_read_version;
  struct ebml_type ebml_max_id_length;
  struct ebml_type ebml_max_size_length;
  struct ebml_type doctype;
  struct ebml_type doctype_version;
  struct ebml_type doctype_read_version;
};

/* Matroksa Definitions */
struct seek {
  struct ebml_type id;
  struct ebml_type position;
};

struct seek_head {
  struct ebml_list seek;
};

struct info {
  struct ebml_type timecode_scale;
  struct ebml_type duration;
};

struct block_more {
  struct ebml_type block_add_id;
  struct ebml_type block_additional;
};

struct block_additions {
  struct ebml_list block_more;
};

struct block_group {
  struct ebml_type block_additions;
  struct ebml_type duration;
  struct ebml_type reference_block;
  struct ebml_type discard_padding;
};

struct cluster {
  struct ebml_type timecode;
  struct ebml_list block_group;
};

struct video {
  struct ebml_type stereo_mode;
  struct ebml_type alpha_mode;
  struct ebml_type pixel_width;
  struct ebml_type pixel_height;
  struct ebml_type pixel_crop_bottom;
  struct ebml_type pixel_crop_top;
  struct ebml_type pixel_crop_left;
  struct ebml_type pixel_crop_right;
  struct ebml_type display_width;
  struct ebml_type display_height;
};

struct audio {
  struct ebml_type sampling_frequency;
  struct ebml_type channels;
  struct ebml_type bit_depth;
};

struct track_entry {
  struct ebml_type number;
  struct ebml_type uid;
  struct ebml_type type;
  struct ebml_type flag_enabled;
  struct ebml_type flag_default;
  struct ebml_type flag_lacing;
  struct ebml_type track_timecode_scale;
  struct ebml_type language;
  struct ebml_type codec_id;
  struct ebml_type codec_private;
  struct ebml_type codec_delay;
  struct ebml_type seek_preroll;
  struct ebml_type default_duration;
  struct video video;
  struct audio audio;
};

struct tracks {
  struct ebml_list track_entry;
};

struct cue_track_positions {
  struct ebml_type track;
  struct ebml_type cluster_position;
  struct ebml_type block_number;
};

struct cue_point {
  struct ebml_type time;
  struct ebml_list cue_track_positions;
};

struct cues {
  struct ebml_list cue_point;
};

struct segment {
  struct ebml_list seek_head;
  struct info info;
  struct ebml_list cluster;
  struct tracks tracks;
  struct cues cues;
};

/* Misc. */
struct pool_ctx {
  char dummy;
};

struct list_node {
  struct list_node * previous;
  struct ebml_element_desc * node;
  unsigned char * data;
};

struct saved_state {
  int64_t stream_offset;
  struct list_node * ancestor;
  uint64_t last_id;
  uint64_t last_size;
  int last_valid;
};

struct frame {
  unsigned char * data;
  size_t length;
  struct frame * next;
};

struct block_additional {
  unsigned int id;
  unsigned char * data;
  size_t length;
  struct block_additional * next;
};

#define NE_IO_BUFSZ 16384

struct nestegg_io_buf {
  nestegg_io io;
  unsigned char buffer[NE_IO_BUFSZ];
  size_t bufsz;
  int offset;
};

/* Public (opaque) Structures */
struct nestegg {
  struct nestegg_io_buf * io;
  nestegg_log log;
  struct pool_ctx * alloc_pool;
  uint64_t last_id;
  uint64_t last_size;
  int last_valid;
  struct list_node * ancestor;
  struct ebml ebml;
  struct segment segment;
  int64_t segment_offset;
  unsigned int track_count;
};

struct nestegg_packet {
  uint64_t track;
  uint64_t timecode;
  uint64_t duration;
  struct frame * frame;
  struct block_additional * block_additional;
  int64_t discard_padding;
};

/* Element Descriptor */
struct ebml_element_desc {
  char const * name;
  uint64_t id;
  enum ebml_type_enum type;
  size_t offset;
  unsigned int flags;
  struct ebml_element_desc * children;
  size_t size;
  size_t data_offset;
};

#define E_FIELD(ID, TYPE, STRUCT, FIELD) \
  { #ID, ID, TYPE, offsetof(STRUCT, FIELD), DESC_FLAG_NONE, NULL, 0, 0 }
#define E_MASTER(ID, TYPE, STRUCT, FIELD) \
  { #ID, ID, TYPE, offsetof(STRUCT, FIELD), DESC_FLAG_MULTI, ne_ ## FIELD ## _elements, \
      sizeof(struct FIELD), 0 }
#define E_SINGLE_MASTER_O(ID, TYPE, STRUCT, FIELD) \
  { #ID, ID, TYPE, offsetof(STRUCT, FIELD), DESC_FLAG_OFFSET, ne_ ## FIELD ## _elements, 0, \
      offsetof(STRUCT, FIELD ## _offset) }
#define E_SINGLE_MASTER(ID, TYPE, STRUCT, FIELD) \
  { #ID, ID, TYPE, offsetof(STRUCT, FIELD), DESC_FLAG_NONE, ne_ ## FIELD ## _elements, 0, 0 }
#define E_SUSPEND(ID, TYPE) \
  { #ID, ID, TYPE, 0, DESC_FLAG_SUSPEND, NULL, 0, 0 }
#define E_LAST \
  { NULL, 0, 0, 0, DESC_FLAG_NONE, NULL, 0, 0 }

/* EBML Element Lists */
static struct ebml_element_desc ne_ebml_elements[] = {
  E_FIELD(ID_EBML_VERSION, TYPE_UINT, struct ebml, ebml_version),
  E_FIELD(ID_EBML_READ_VERSION, TYPE_UINT, struct ebml, ebml_read_version),
  E_FIELD(ID_EBML_MAX_ID_LENGTH, TYPE_UINT, struct ebml, ebml_max_id_length),
  E_FIELD(ID_EBML_MAX_SIZE_LENGTH, TYPE_UINT, struct ebml, ebml_max_size_length),
  E_FIELD(ID_DOCTYPE, TYPE_STRING, struct ebml, doctype),
  E_FIELD(ID_DOCTYPE_VERSION, TYPE_UINT, struct ebml, doctype_version),
  E_FIELD(ID_DOCTYPE_READ_VERSION, TYPE_UINT, struct ebml, doctype_read_version),
  E_LAST
};

/* WebM Element Lists */
static struct ebml_element_desc ne_seek_elements[] = {
  E_FIELD(ID_SEEK_ID, TYPE_BINARY, struct seek, id),
  E_FIELD(ID_SEEK_POSITION, TYPE_UINT, struct seek, position),
  E_LAST
};

static struct ebml_element_desc ne_seek_head_elements[] = {
  E_MASTER(ID_SEEK, TYPE_MASTER, struct seek_head, seek),
  E_LAST
};

static struct ebml_element_desc ne_info_elements[] = {
  E_FIELD(ID_TIMECODE_SCALE, TYPE_UINT, struct info, timecode_scale),
  E_FIELD(ID_DURATION, TYPE_FLOAT, struct info, duration),
  E_LAST
};

static struct ebml_element_desc ne_block_more_elements[] = {
  E_FIELD(ID_BLOCK_ADD_ID, TYPE_UINT, struct block_more, block_add_id),
  E_FIELD(ID_BLOCK_ADDITIONAL, TYPE_BINARY, struct block_more, block_additional),
  E_LAST
};

static struct ebml_element_desc ne_block_additions_elements[] = {
  E_MASTER(ID_BLOCK_MORE, TYPE_MASTER, struct block_additions, block_more),
  E_LAST
};

static struct ebml_element_desc ne_block_group_elements[] = {
  E_SUSPEND(ID_BLOCK, TYPE_BINARY),
  E_FIELD(ID_BLOCK_DURATION, TYPE_UINT, struct block_group, duration),
  E_FIELD(ID_REFERENCE_BLOCK, TYPE_INT, struct block_group, reference_block),
  E_FIELD(ID_DISCARD_PADDING, TYPE_INT, struct block_group, discard_padding),
  E_SINGLE_MASTER(ID_BLOCK_ADDITIONS, TYPE_MASTER, struct block_group, block_additions),
  E_LAST
};

static struct ebml_element_desc ne_cluster_elements[] = {
  E_FIELD(ID_TIMECODE, TYPE_UINT, struct cluster, timecode),
  E_MASTER(ID_BLOCK_GROUP, TYPE_MASTER, struct cluster, block_group),
  E_SUSPEND(ID_SIMPLE_BLOCK, TYPE_BINARY),
  E_LAST
};

static struct ebml_element_desc ne_video_elements[] = {
  E_FIELD(ID_STEREO_MODE, TYPE_UINT, struct video, stereo_mode),
  E_FIELD(ID_ALPHA_MODE, TYPE_UINT, struct video, alpha_mode),
  E_FIELD(ID_PIXEL_WIDTH, TYPE_UINT, struct video, pixel_width),
  E_FIELD(ID_PIXEL_HEIGHT, TYPE_UINT, struct video, pixel_height),
  E_FIELD(ID_PIXEL_CROP_BOTTOM, TYPE_UINT, struct video, pixel_crop_bottom),
  E_FIELD(ID_PIXEL_CROP_TOP, TYPE_UINT, struct video, pixel_crop_top),
  E_FIELD(ID_PIXEL_CROP_LEFT, TYPE_UINT, struct video, pixel_crop_left),
  E_FIELD(ID_PIXEL_CROP_RIGHT, TYPE_UINT, struct video, pixel_crop_right),
  E_FIELD(ID_DISPLAY_WIDTH, TYPE_UINT, struct video, display_width),
  E_FIELD(ID_DISPLAY_HEIGHT, TYPE_UINT, struct video, display_height),
  E_LAST
};

static struct ebml_element_desc ne_audio_elements[] = {
  E_FIELD(ID_SAMPLING_FREQUENCY, TYPE_FLOAT, struct audio, sampling_frequency),
  E_FIELD(ID_CHANNELS, TYPE_UINT, struct audio, channels),
  E_FIELD(ID_BIT_DEPTH, TYPE_UINT, struct audio, bit_depth),
  E_LAST
};

static struct ebml_element_desc ne_track_entry_elements[] = {
  E_FIELD(ID_TRACK_NUMBER, TYPE_UINT, struct track_entry, number),
  E_FIELD(ID_TRACK_UID, TYPE_UINT, struct track_entry, uid),
  E_FIELD(ID_TRACK_TYPE, TYPE_UINT, struct track_entry, type),
  E_FIELD(ID_FLAG_ENABLED, TYPE_UINT, struct track_entry, flag_enabled),
  E_FIELD(ID_FLAG_DEFAULT, TYPE_UINT, struct track_entry, flag_default),
  E_FIELD(ID_FLAG_LACING, TYPE_UINT, struct track_entry, flag_lacing),
  E_FIELD(ID_TRACK_TIMECODE_SCALE, TYPE_FLOAT, struct track_entry, track_timecode_scale),
  E_FIELD(ID_LANGUAGE, TYPE_STRING, struct track_entry, language),
  E_FIELD(ID_CODEC_ID, TYPE_STRING, struct track_entry, codec_id),
  E_FIELD(ID_CODEC_PRIVATE, TYPE_BINARY, struct track_entry, codec_private),
  E_FIELD(ID_CODEC_DELAY, TYPE_UINT, struct track_entry, codec_delay),
  E_FIELD(ID_SEEK_PREROLL, TYPE_UINT, struct track_entry, seek_preroll),
  E_FIELD(ID_DEFAULT_DURATION, TYPE_UINT, struct track_entry, default_duration),
  E_SINGLE_MASTER(ID_VIDEO, TYPE_MASTER, struct track_entry, video),
  E_SINGLE_MASTER(ID_AUDIO, TYPE_MASTER, struct track_entry, audio),
  E_LAST
};

static struct ebml_element_desc ne_tracks_elements[] = {
  E_MASTER(ID_TRACK_ENTRY, TYPE_MASTER, struct tracks, track_entry),
  E_LAST
};

static struct ebml_element_desc ne_cue_track_positions_elements[] = {
  E_FIELD(ID_CUE_TRACK, TYPE_UINT, struct cue_track_positions, track),
  E_FIELD(ID_CUE_CLUSTER_POSITION, TYPE_UINT, struct cue_track_positions, cluster_position),
  E_FIELD(ID_CUE_BLOCK_NUMBER, TYPE_UINT, struct cue_track_positions, block_number),
  E_LAST
};

static struct ebml_element_desc ne_cue_point_elements[] = {
  E_FIELD(ID_CUE_TIME, TYPE_UINT, struct cue_point, time),
  E_MASTER(ID_CUE_TRACK_POSITIONS, TYPE_MASTER, struct cue_point, cue_track_positions),
  E_LAST
};

static struct ebml_element_desc ne_cues_elements[] = {
  E_MASTER(ID_CUE_POINT, TYPE_MASTER, struct cues, cue_point),
  E_LAST
};

static struct ebml_element_desc ne_segment_elements[] = {
  E_MASTER(ID_SEEK_HEAD, TYPE_MASTER, struct segment, seek_head),
  E_SINGLE_MASTER(ID_INFO, TYPE_MASTER, struct segment, info),
  E_MASTER(ID_CLUSTER, TYPE_MASTER, struct segment, cluster),
  E_SINGLE_MASTER(ID_TRACKS, TYPE_MASTER, struct segment, tracks),
  E_SINGLE_MASTER(ID_CUES, TYPE_MASTER, struct segment, cues),
  E_LAST
};

static struct ebml_element_desc ne_top_level_elements[] = {
  E_SINGLE_MASTER(ID_EBML, TYPE_MASTER, nestegg, ebml),
  E_SINGLE_MASTER_O(ID_SEGMENT, TYPE_MASTER, nestegg, segment),
  E_LAST
};

#undef E_FIELD
#undef E_MASTER
#undef E_SINGLE_MASTER_O
#undef E_SINGLE_MASTER
#undef E_SUSPEND
#undef E_LAST

static struct pool_ctx *
ne_pool_init(void)
{
  return h_malloc(sizeof(struct pool_ctx));
}

static void
ne_pool_destroy(struct pool_ctx * pool)
{
  h_free(pool);
}

static void *
ne_pool_alloc(size_t size, struct pool_ctx * pool)
{
  void * p;

  p = h_malloc(size);
  if (!p)
    return NULL;
  hattach(p, pool);
  memset(p, 0, size);
  return p;
}

static void *
ne_alloc(size_t size)
{
  return calloc(1, size);
}

static int
ne_io_read(struct nestegg_io_buf * io, void * buffer, size_t length)
{
  int64_t off;
  int r;
  size_t avail;

  assert(io->offset == -1 || (io->offset >= 0 && (unsigned int) io->offset < io->bufsz));

  /* Too big to buffer, invalidate buffer and read through */
  if (length > io->bufsz) {
    if (io->offset != -1) {
      r = io->io.seek(-(io->bufsz - io->offset), NESTEGG_SEEK_CUR, io->io.userdata);
      if (r != 0) {
        return -1;
      }
    }
    io->offset = -1;
    return io->io.read(buffer, length, io->io.userdata);
  }

  /* Buffer invalid */
  if (io->offset == -1) {
    off = io->io.tell(io->io.userdata);
    if (off == -1) {
      return -1;
    }
    /* Refill buffer */
    r = io->io.read(io->buffer, io->bufsz, io->io.userdata);
    if (r != 1) {
      /* Read truncated due to being within io->bufsz of EOS, reset read
         position and switch to read through mode */
      io->offset = -1;
      io->bufsz = 0;
      if (r == 0) {
        r = io->io.seek(off, NESTEGG_SEEK_SET, io->io.userdata);
      }
      if (r == 0) {
        return io->io.read(buffer, length, io->io.userdata);
      }
      return -1;
    }
    if (r == 1) {
      io->offset = 0;
    }
  }

  /* Service request with what we have */
  avail = length;
  if (io->bufsz - io->offset < length) {
    avail = io->bufsz - io->offset;
  }
  memcpy(buffer, io->buffer + io->offset, avail);
  io->offset += avail;

  if ((unsigned int) io->offset == io->bufsz) {
    io->offset = -1;
  }

  /* Still more to read, invalidate buffer and read more */
  if (length - avail > 0) {
    return ne_io_read(io, (char *) buffer + avail, length - avail);
  }

  return 1;
}

static int
ne_io_seek(struct nestegg_io_buf * io, int64_t offset, int whence)
{
  /* Invalidate buffer */
  io->offset = -1;

  return io->io.seek(offset, whence, io->io.userdata);
}

static int64_t
ne_io_tell(struct nestegg_io_buf * io)
{
  int64_t off;

  off = io->io.tell(io->io.userdata);
  if (off == -1) {
    return -1;
  }
  if (io->offset == -1) {
    return off;
  }
  assert(off >= (int64_t) io->bufsz - io->offset);
  return off - io->bufsz + (unsigned int) io->offset;
}

static int
ne_io_read_skip(struct nestegg_io_buf * io, size_t length)
{
  size_t get;
  unsigned char buf[8192];
  int r = 1;

  while (length > 0) {
{'fraction_non_alphanumeric': 0.08915012092918612, 'fraction_numerical': 0.020107992575510433, 'mean_word_length': 3.8784469748936754, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 115, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1917055', 'n_tokens_mistral': 13832, 'n_tokens_neox': 13349, 'n_words': 3067}