<commit_before>/*
 *  Copyright (c) 2009 <NAME>
 *
 *  This is free software; see gpl-3.0.txt
 */

#include <libavformat/avformat.h>

#include <chunk.h>

#include "../input.h"
#define STATIC_BUFF_SIZE 1000 * 1024

struct input_stream {
  AVFormatContext *s;
  int audio_stream;
  int video_stream;
};

struct input_stream *input_stream_open(const char *fname, int *period)
{
  struct input_stream *desc;
  int i, res;

  avcodec_register_all();
  av_register_all();

  desc = malloc(sizeof(struct input_stream));
  if (desc == NULL) {
    return NULL;
  }
  res = av_open_input_file(&desc->s, fname, NULL, 0, NULL);
  if (res < 0) {
    fprintf(stderr, "Error opening %s: %d\n", fname, res);

    return NULL;
  }

  res = av_find_stream_info(desc->s);
  if (res < 0) {
    fprintf(stderr, "Cannot find codec parameters for %s\n", fname);

    return NULL;
  }
  desc->video_stream = -1;
  desc->audio_stream = -1;
  for (i = 0; i < desc->s->nb_streams; i++) {
    if (desc->video_stream == -1 && desc->s->streams[i]->codec->codec_type == CODEC_TYPE_VIDEO) {
      desc->video_stream = i;
      fprintf(stderr, "Video Frame Rate = %d/%d --- Period: %lld\n",
              desc->s->streams[i]->r_frame_rate.num,
              desc->s->streams[i]->r_frame_rate.den,
              av_rescale(1000000, desc->s->streams[i]->r_frame_rate.den, desc->s->streams[i]->r_frame_rate.num));
      *period = av_rescale(1000000, desc->s->streams[i]->r_frame_rate.den, desc->s->streams[i]->r_frame_rate.num);
    }
    if (desc->audio_stream == -1 && desc->s->streams[i]->codec->codec_type == CODEC_TYPE_AUDIO) {
      desc->audio_stream = i;
    }
  }

  dump_format(desc->s, 0, fname, 0);

  return desc;
}

void input_stream_close(struct input_stream *s)
{
    av_close_input_file(s->s);
    free(s);
}

int input_get_1(struct input_stream *s, struct chunk *c)
{
    static AVPacket pkt;
    static int inited;
    AVStream *st;
    int res;
    static uint8_t static_buff[STATIC_BUFF_SIZE];
    static int cid;
    uint8_t *p;

    p = static_buff;
    if (inited == 0) {
        inited = 1;
        res = av_read_frame(s->s, &pkt);
        if (res < 0) {
            fprintf(stderr, "First read failed: %d!!!\n", res);

            return 0;
        }
        if ((pkt.flags & PKT_FLAG_KEY) == 0) {
            fprintf(stderr, "First frame is not key frame!!!\n");

            return 0;
        }
    }
    c->timestamp = pkt.dts;
    memcpy(p, pkt.data, pkt.size);
    p += pkt.size;
    while (1) {
        res = av_read_frame(s->s, &pkt);
        if (res >= 0) {
            st = s->s->streams[pkt.stream_index];
            if (pkt.flags & PKT_FLAG_KEY) {
                c->size = p - static_buff;
                c->data = malloc(c->size);
                if (c->data == NULL) {
                  return 0;
                }
                memcpy(c->data, static_buff, c->size);
                c->attributes_size = 0;
                c->attributes = NULL;
                c->id = cid++; 
                return 1;
            }
            memcpy(p, pkt.data, pkt.size);
            p += pkt.size;
        } else {
            if (p - static_buff > 0) {
                c->size = p - static_buff;
                c->data = malloc(c->size);
                if (c->data == NULL) {
                  return 0;
                }
                memcpy(c->data, static_buff, c->size);
                c->attributes_size = 0;
                c->attributes = NULL;
                c->id = cid++; 
                return 1;
            }
            return 0;
        }
    }

    return 0;
}

uint8_t *chunkise(struct input_stream *s, int id, int *size, uint64_t *ts)
{<commit_msg>Correct include files<commit_after>/*
 *  Copyright (c) 2009 <NAME>
 *
 *  This is free software; see gpl-3.0.txt
 */

#include <libavformat/avformat.h>

#include "../input-stream.h"
#define STATIC_BUFF_SIZE 1000 * 1024

struct input_stream {
  AVFormatContext *s;
  int audio_stream;
  int video_stream;
};

struct input_stream *input_stream_open(const char *fname, int *period)
{
  struct input_stream *desc;
  int i, res;

  avcodec_register_all();
  av_register_all();

  desc = malloc(sizeof(struct input_stream));
  if (desc == NULL) {
    return NULL;
  }
  res = av_open_input_file(&desc->s, fname, NULL, 0, NULL);
  if (res < 0) {
    fprintf(stderr, "Error opening %s: %d\n", fname, res);

    return NULL;
  }

  res = av_find_stream_info(desc->s);
  if (res < 0) {
    fprintf(stderr, "Cannot find codec parameters for %s\n", fname);

    return NULL;
  }
  desc->video_stream = -1;
  desc->audio_stream = -1;
  for (i = 0; i < desc->s->nb_streams; i++) {
    if (desc->video_stream == -1 && desc->s->streams[i]->codec->codec_type == CODEC_TYPE_VIDEO) {
      desc->video_stream = i;
      fprintf(stderr, "Video Frame Rate = %d/%d --- Period: %lld\n",
              desc->s->streams[i]->r_frame_rate.num,
              desc->s->streams[i]->r_frame_rate.den,
              av_rescale(1000000, desc->s->streams[i]->r_frame_rate.den, desc->s->streams[i]->r_frame_rate.num));
      *period = av_rescale(1000000, desc->s->streams[i]->r_frame_rate.den, desc->s->streams[i]->r_frame_rate.num);
    }
    if (desc->audio_stream == -1 && desc->s->streams[i]->codec->codec_type == CODEC_TYPE_AUDIO) {
      desc->audio_stream = i;
    }
  }

  dump_format(desc->s, 0, fname, 0);

  return desc;
}

void input_stream_close(struct input_stream *s)
{
    av_close_input_file(s->s);
    free(s);
}

#if 0
int input_get_1(struct input_stream *s, struct chunk *c)
{
    static AVPacket pkt;
    static int inited;
    AVStream *st;
    int res;
    static uint8_t static_buff[STATIC_BUFF_SIZE];
    static int cid;
    uint8_t *p;

    p = static_buff;
    if (inited == 0) {
        inited = 1;
        res = av_read_frame(s->s, &pkt);
        if (res < 0) {
            fprintf(stderr, "First read failed: %d!!!\n", res);

            return 0;
        }
        if ((pkt.flags & PKT_FLAG_KEY) == 0) {
            fprintf(stderr, "First frame is not key frame!!!\n");

            return 0;
        }
    }
    c->timestamp = pkt.dts;
    memcpy(p, pkt.data, pkt.size);
    p += pkt.size;
    while (1) {
        res = av_read_frame(s->s, &pkt);
        if (res >= 0) {
            st = s->s->streams[pkt.stream_index];
            if (pkt.flags & PKT_FLAG_KEY) {
                c->size = p - static_buff;
                c->data = malloc(c->size);
                if (c->data == NULL) {
                  return 0;
                }
                memcpy(c->data, static_buff, c->size);
                c->attributes_size = 0;
                c->attributes = NULL;
                c->id = cid++; 
                return 1;
            }
            memcpy(p, pkt.data, pkt.size);
            p += pkt.size;
        } else {
            if (p - static_buff > 0) {
                c->size = p - static_buff;
                c->data = malloc(c->size);
                if (c->data == NULL) {
                  return 0;
                }
                memcpy(c->data, static_buff, c->size);
                c->attributes_size = 0;
                c->attributes = NULL;
                c->id = cid++; 
                return 1;
            }
            return 0;
        }
    }

    return 0;
}
#endif

uint8_t *chunkise(struct input_stream *s, int id, int *size, uint64_t *ts)
{
{'fraction_non_alphanumeric': 0.15111534276387378, 'fraction_numerical': 0.016730141458106636, 'mean_word_length': 1.8778864970645792, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 134, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3368145', 'n_tokens_mistral': 2713, 'n_tokens_neox': 2582, 'n_words': 569}