<commit_before>/* Copyright 2016 <NAME>, NVIDIA Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "scanner/ingest.h"

#include "scanner/util/common.h"
#include "scanner/util/util.h"

#include "storehouse/storage_backend.h"

#include <glog/logging.h>

#include <mpi.h>

// For video
extern "C" {
#include "libavcodec/avcodec.h"
#include "libavfilter/avfilter.h"
#include "libavformat/avformat.h"
#include "libavformat/avio.h"
#include "libavutil/error.h"
#include "libavutil/opt.h"
#include "libavutil/pixdesc.h"
#include "libswscale/swscale.h"
}

#include <cassert>
#include <fstream>

using storehouse::StoreResult;
using storehouse::WriteFile;
using storehouse::RandomReadFile;
using storehouse::exit_on_error;

namespace scanner {

namespace {

const std::string BAD_VIDEOS_FILE_PATH = "bad_videos.txt";

struct BufferData {
  u8* ptr;
  size_t size;  // size left in the buffer

  u8* orig_ptr;
  size_t initial_size;
};

// For custom AVIOContext that loads from memory
i32 read_packet(void* opaque, u8* buf, i32 buf_size) {
  BufferData* bd = (BufferData*)opaque;
  buf_size = std::min(static_cast<size_t>(buf_size), bd->size);
  /* copy internal buffer data to buf */
  memcpy(buf, bd->ptr, buf_size);
  bd->ptr += buf_size;
  bd->size -= buf_size;
  return buf_size;
}

i64 seek(void* opaque, i64 offset, i32 whence) {
  BufferData* bd = (BufferData*)opaque;
  {
    switch (whence) {
      case SEEK_SET:
        bd->ptr = bd->orig_ptr + offset;
        bd->size = bd->initial_size - offset;
        break;
      case SEEK_CUR:
        bd->ptr += offset;
        bd->size -= offset;
        break;
      case SEEK_END:
        bd->ptr = bd->orig_ptr + bd->initial_size;
        bd->size = 0;
        break;
      case AVSEEK_SIZE:
        return bd->initial_size;
        break;
    }
    return bd->initial_size - bd->size;
  }
}

struct CodecState {
  AVPacket av_packet;
  AVFrame* picture;
  AVFormatContext* format_context;
  AVIOContext* io_context;
  AVCodec* in_codec;
  AVCodecContext* in_cc;
  AVCodecParameters* in_cc_params;
  i32 video_stream_index;
  AVBitStreamFilterContext* annexb;
};

CodecState setup_video_codec(BufferData* buffer) {
  printf("Setting up video codec\n");
  CodecState state;
  av_init_packet(&state.av_packet);
  state.picture = av_frame_alloc();
  state.format_context = avformat_alloc_context();

  size_t avio_context_buffer_size = 4096;
  u8* avio_context_buffer =
      static_cast<u8*>(av_malloc(avio_context_buffer_size));
  state.io_context =
      avio_alloc_context(avio_context_buffer, avio_context_buffer_size, 0,
                         buffer, &read_packet, NULL, &seek);
  state.format_context->pb = state.io_context;

  // Read file header
  printf("Opening input file to read format\n");
  if (avformat_open_input(&state.format_context, NULL, NULL, NULL) < 0) {
    fprintf(stderr, "open input failed\n");
    assert(false);
  }
  // Some formats don't have a header
  if (avformat_find_stream_info(state.format_context, NULL) < 0) {
    fprintf(stderr, "find stream info failed\n");
    assert(false);
  }

  av_dump_format(state.format_context, 0, NULL, 0);

  // Find the best video stream in our input video
  state.video_stream_index = av_find_best_stream(
      state.format_context, AVMEDIA_TYPE_VIDEO, -1 /* auto select */,
      -1 /* no related stream */, &state.in_codec, 0 /* flags */);
  if (state.video_stream_index < 0) {
    fprintf(stderr, "could not find best stream\n");
    assert(false);
  }

  AVStream const* const in_stream =
      state.format_context->streams[state.video_stream_index];

  state.in_codec = avcodec_find_decoder(AV_CODEC_ID_H264);
  if (state.in_codec == NULL) {
    fprintf(stderr, "could not find h264 decoder\n");
    exit(EXIT_FAILURE);
  }

  state.in_cc = avcodec_alloc_context3(state.in_codec);
  state.in_cc_params = avcodec_parameters_alloc();
  if (avcodec_parameters_from_context(state.in_cc_params, in_stream->codec) < 0)
  {
    fprintf(stderr, "could not copy codec params from input stream\n");
    exit(EXIT_FAILURE);
  }
  if (avcodec_parameters_to_context(state.in_cc, state.in_cc_params) < 0) {
    fprintf(stderr, "could not copy codec params to in cc\n");
    assert(false);
  }

  if (avcodec_open2(state.in_cc, state.in_codec, NULL) < 0) {
    fprintf(stderr, "could not open codec\n");
    assert(false);
  }

  state.annexb = av_bitstream_filter_init("h264_mp4toannexb");

  return state;
}

void cleanup_video_codec(CodecState state) {
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 53, 0)
  avcodec_free_context(&state.in_cc);
#else
  avcodec_close(state.in_cc);
  av_freep(&state.in_cc);
#endif
  avformat_close_input(&state.format_context);
  if (state.io_context) {<commit_msg>AVCodecParameters only in avcodec 3.0<commit_after>/* Copyright 2016 <NAME>, NVIDIA Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "scanner/ingest.h"

#include "scanner/util/common.h"
#include "scanner/util/util.h"

#include "storehouse/storage_backend.h"

#include <glog/logging.h>

#include <mpi.h>

// For video
extern "C" {
#include "libavcodec/avcodec.h"
#include "libavfilter/avfilter.h"
#include "libavformat/avformat.h"
#include "libavformat/avio.h"
#include "libavutil/error.h"
#include "libavutil/opt.h"
#include "libavutil/pixdesc.h"
#include "libswscale/swscale.h"
}

#include <cassert>
#include <fstream>

using storehouse::StoreResult;
using storehouse::WriteFile;
using storehouse::RandomReadFile;
using storehouse::exit_on_error;

namespace scanner {

namespace {

const std::string BAD_VIDEOS_FILE_PATH = "bad_videos.txt";

struct BufferData {
  u8* ptr;
  size_t size;  // size left in the buffer

  u8* orig_ptr;
  size_t initial_size;
};

// For custom AVIOContext that loads from memory
i32 read_packet(void* opaque, u8* buf, i32 buf_size) {
  BufferData* bd = (BufferData*)opaque;
  buf_size = std::min(static_cast<size_t>(buf_size), bd->size);
  /* copy internal buffer data to buf */
  memcpy(buf, bd->ptr, buf_size);
  bd->ptr += buf_size;
  bd->size -= buf_size;
  return buf_size;
}

i64 seek(void* opaque, i64 offset, i32 whence) {
  BufferData* bd = (BufferData*)opaque;
  {
    switch (whence) {
      case SEEK_SET:
        bd->ptr = bd->orig_ptr + offset;
        bd->size = bd->initial_size - offset;
        break;
      case SEEK_CUR:
        bd->ptr += offset;
        bd->size -= offset;
        break;
      case SEEK_END:
        bd->ptr = bd->orig_ptr + bd->initial_size;
        bd->size = 0;
        break;
      case AVSEEK_SIZE:
        return bd->initial_size;
        break;
    }
    return bd->initial_size - bd->size;
  }
}

struct CodecState {
  AVPacket av_packet;
  AVFrame* picture;
  AVFormatContext* format_context;
  AVIOContext* io_context;
  AVCodec* in_codec;
  AVCodecContext* in_cc;
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 34, 0)
  AVCodecParameters* in_cc_params;
#endif
  i32 video_stream_index;
  AVBitStreamFilterContext* annexb;
};

CodecState setup_video_codec(BufferData* buffer) {
  printf("Setting up video codec\n");
  CodecState state;
  av_init_packet(&state.av_packet);
  state.picture = av_frame_alloc();
  state.format_context = avformat_alloc_context();

  size_t avio_context_buffer_size = 4096;
  u8* avio_context_buffer =
      static_cast<u8*>(av_malloc(avio_context_buffer_size));
  state.io_context =
      avio_alloc_context(avio_context_buffer, avio_context_buffer_size, 0,
                         buffer, &read_packet, NULL, &seek);
  state.format_context->pb = state.io_context;

  // Read file header
  printf("Opening input file to read format\n");
  if (avformat_open_input(&state.format_context, NULL, NULL, NULL) < 0) {
    fprintf(stderr, "open input failed\n");
    assert(false);
  }
  // Some formats don't have a header
  if (avformat_find_stream_info(state.format_context, NULL) < 0) {
    fprintf(stderr, "find stream info failed\n");
    assert(false);
  }

  av_dump_format(state.format_context, 0, NULL, 0);

  // Find the best video stream in our input video
  state.video_stream_index = av_find_best_stream(
      state.format_context, AVMEDIA_TYPE_VIDEO, -1 /* auto select */,
      -1 /* no related stream */, &state.in_codec, 0 /* flags */);
  if (state.video_stream_index < 0) {
    fprintf(stderr, "could not find best stream\n");
    assert(false);
  }

  AVStream const* const in_stream =
      state.format_context->streams[state.video_stream_index];

  state.in_codec = avcodec_find_decoder(AV_CODEC_ID_H264);
  if (state.in_codec == NULL) {
    fprintf(stderr, "could not find h264 decoder\n");
    exit(EXIT_FAILURE);
  }

  state.in_cc = avcodec_alloc_context3(state.in_codec);
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 34, 0)
  state.in_cc_params = avcodec_parameters_alloc();
  if (avcodec_parameters_from_context(state.in_cc_params, in_stream->codec) <
      0) {
    fprintf(stderr, "could not copy codec params from input stream\n");
    exit(EXIT_FAILURE);
  }
  if (avcodec_parameters_to_context(state.in_cc, state.in_cc_params) < 0) {
    fprintf(stderr, "could not copy codec params to in cc\n");
    assert(false);
  }
#else
  if (avcodec_copy_context(state.in_cc, in_stream->codec) < 0) {
    fprintf(stderr, "could not copy codec params to in cc\n");
    assert(false);
  }
#endif

  if (avcodec_open2(state.in_cc, state.in_codec, NULL) < 0) {
    fprintf(stderr, "could not open codec\n");
    assert(false);
  }

  state.annexb = av_bitstream_filter_init("h264_mp4toannexb");

  return state;
}

void cleanup_video_codec(CodecState state) {
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 53, 0)
  avcodec_free_context(&state.in_cc);
#else
  avcodec_close(state.in_cc);
  av_freep(&state.in_cc);
#endif
  avformat_close_input(&state.format_context);
  if (state.io_context) {
{'fraction_non_alphanumeric': 0.11703210287723194, 'fraction_numerical': 0.012119530021278564, 'mean_word_length': 3.933820173436787, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7615518', 'n_tokens_mistral': 3881, 'n_tokens_neox': 3762, 'n_words': 895}