<commit_before>/*
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#include "./aomenc.h"
#include "./aom_config.h"

#include <assert.h>
#include <limits.h>
#include <math.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#if CONFIG_LIBYUV
#include "third_party/libyuv/include/libyuv/scale.h"
#endif

#include "aom/aom_encoder.h"
#if CONFIG_DECODERS
#include "aom/aom_decoder.h"
#endif

#include "./args.h"
#include "./ivfenc.h"
#include "./tools_common.h"

#if CONFIG_AV1_ENCODER
#include "aom/aomcx.h"
#endif
#if CONFIG_AV1_DECODER
#include "aom/aomdx.h"
#endif

#include "aom/aom_integer.h"
#include "aom_ports/mem_ops.h"
#include "aom_ports/aom_timer.h"
#include "./rate_hist.h"
#include "./aomstats.h"
#include "./warnings.h"
#if CONFIG_WEBM_IO
#include "./webmenc.h"
#endif
#include "./y4minput.h"

/* Swallow warnings about unused results of fread/fwrite */
static size_t wrap_fread(void *ptr, size_t size, size_t nmemb, FILE *stream) {
  return fread(ptr, size, nmemb, stream);
}
#define fread wrap_fread

static size_t wrap_fwrite(const void *ptr, size_t size, size_t nmemb,
                          FILE *stream) {
  return fwrite(ptr, size, nmemb, stream);
}
#define fwrite wrap_fwrite

static const char *exec_name;

static void warn_or_exit_on_errorv(aom_codec_ctx_t *ctx, int fatal,
                                   const char *s, va_list ap) {
  if (ctx->err) {
    const char *detail = aom_codec_error_detail(ctx);

    vfprintf(stderr, s, ap);
    fprintf(stderr, ": %s\n", aom_codec_error(ctx));

    if (detail) fprintf(stderr, "    %s\n", detail);

    if (fatal) exit(EXIT_FAILURE);
  }
}

static void ctx_exit_on_error(aom_codec_ctx_t *ctx, const char *s, ...) {
  va_list ap;

  va_start(ap, s);
  warn_or_exit_on_errorv(ctx, 1, s, ap);
  va_end(ap);
}

static void warn_or_exit_on_error(aom_codec_ctx_t *ctx, int fatal,
                                  const char *s, ...) {
  va_list ap;

  va_start(ap, s);
  warn_or_exit_on_errorv(ctx, fatal, s, ap);
  va_end(ap);
}

static int read_frame(struct AvxInputContext *input_ctx, aom_image_t *img) {
  FILE *f = input_ctx->file;
  y4m_input *y4m = &input_ctx->y4m;
  int shortread = 0;

  if (input_ctx->file_type == FILE_TYPE_Y4M) {
    if (y4m_input_fetch_frame(y4m, f, img) < 1) return 0;
  } else {
    shortread = read_yuv_frame(input_ctx, img);
  }

  return !shortread;
}

static int file_is_y4m(const char detect[4]) {
  if (memcmp(detect, "YUV4", 4) == 0) {
    return 1;
  }
  return 0;
}

static int fourcc_is_ivf(const char detect[4]) {
  if (memcmp(detect, "DKIF", 4) == 0) {
    return 1;
  }
  return 0;
}

static const arg_def_t debugmode =
    ARG_DEF("D", "debug", 0, "Debug mode (makes output deterministic)");
static const arg_def_t outputfile =
    ARG_DEF("o", "output", 1, "Output filename");
static const arg_def_t use_yv12 =
    ARG_DEF(NULL, "yv12", 0, "Input file is YV12 ");
static const arg_def_t use_i420 =
    ARG_DEF(NULL, "i420", 0, "Input file is I420 (default)");
static const arg_def_t use_i422 =
    ARG_DEF(NULL, "i422", 0, "Input file is I422");
static const arg_def_t use_i444 =
    ARG_DEF(NULL, "i444", 0, "Input file is I444");
static const arg_def_t use_i440 =
    ARG_DEF(NULL, "i440", 0, "Input file is I440");
static const arg_def_t codecarg = ARG_DEF(NULL, "codec", 1, "Codec to use");
static const arg_def_t passes =
    ARG_DEF("p", "passes", 1, "Number of passes (1/2)");
static const arg_def_t pass_arg =
    ARG_DEF(NULL, "pass", 1, "Pass to execute (1/2)");
static const arg_def_t fpf_name =
    ARG_DEF(NULL, "fpf", 1, "First pass statistics file name");
#if CONFIG_FP_MB_STATS
static const arg_def_t fpmbf_name =
    ARG_DEF(NULL, "fpmbf", 1, "First pass block statistics file name");
#endif
static const arg_def_t limit =
    ARG_DEF(NULL, "limit", 1, "Stop encoding after n input frames");
static const arg_def_t skip =
    ARG_DEF(NULL, "skip", 1, "Skip the first n input frames");
static const arg_def_t deadline =
    ARG_DEF("d", "deadline", 1, "Deadline per frame (usec)");
static const arg_def_t best_dl =
    ARG_DEF(NULL, "best", 0, "Use Best Quality Deadline");
static const arg_def_t good_dl =
    ARG_DEF(NULL, "good", 0, "Use Good Quality Deadline");
static const arg_def_t rt_dl =
    ARG_DEF(NULL, "rt", 0, "Use Realtime Quality Deadline");
static const arg_def_t quietarg =
    ARG_DEF("q", "quiet", 0, "Do not print encode progress");
static const arg_def_t verbosearg =
    ARG_DEF("v", "verbose", 0, "Show encoder parameters");
static const arg_def_t psnrarg =
    ARG_DEF(NULL, "psnr", 0, "Show PSNR in status line");

static const struct arg_enum_list test_decode_enum[] = {
  { "off", TEST_DECODE_OFF },
  { "fatal", TEST_DECODE_FATAL },
  { "warn", TEST_DECODE_WARN },
  { NULL, 0 }
};
static const arg_def_t recontest = ARG_DEF_ENUM(
    NULL, "test-decode", 1, "Test encode/decode mismatch", test_decode_enum);
static const arg_def_t framerate =
    ARG_DEF(NULL, "fps", 1, "Stream frame rate (rate/scale)");
static const arg_def_t use_webm =
    ARG_DEF(NULL, "webm", 0, "Output WebM (default when WebM IO is enabled)");
static const arg_def_t use_ivf = ARG_DEF(NULL, "ivf", 0, "Output IVF");
static const arg_def_t out_part =
    ARG_DEF("P", "output-partitions", 0,
            "Makes encoder output partitions. Requires IVF output!");
static const arg_def_t q_hist_n =
    ARG_DEF(NULL, "q-hist", 1, "Show quantizer histogram (n-buckets)");
static const arg_def_t rate_hist_n =
    ARG_DEF(NULL, "rate-hist", 1, "Show rate histogram (n-buckets)");
static const arg_def_t disable_warnings =
    ARG_DEF(NULL, "disable-warnings", 0,
            "Disable warnings about potentially incorrect encode settings.");
static const arg_def_t disable_warning_prompt =
    ARG_DEF("y", "disable-warning-prompt", 0,
            "Display warnings, but do not prompt user to continue.");

#if CONFIG_AOM_HIGHBITDEPTH
static const arg_def_t test16bitinternalarg = ARG_DEF(
    NULL, "test-16bit-internal", 0, "Force use of 16 bit internal buffer");
#endif

static const arg_def_t *main_args[] = { &debugmode,
                                        &outputfile,
                                        &codecarg,
                                        &passes,
                                        &pass_arg,
                                        &fpf_name,
                                        &limit,
                                        &skip,
                                        &deadline,
                                        &best_dl,
                                        &good_dl,
                                        &rt_dl,
                                        &quietarg,
                                        &verbosearg,
                                        &psnrarg,
                                        &use_webm,
                                        &use_ivf,
                                        &out_part,
                                        &q_hist_n,
                                        &rate_hist_n,
                                        &disable_warnings,
                                        &disable_warning_prompt,
                                        &recontest,
                                        NULL };

static const arg_def_t usage =
    ARG_DEF("u", "usage", 1, "Usage profile number to use");
static const arg_def_t threads =
    ARG_DEF("t", "threads", 1, "Max number of threads to use");
static const arg_def_t profile =
    ARG_DEF(NULL, "profile", 1, "Bitstream profile number to use");
static const arg_def_t width = ARG_DEF("w", "width", 1, "Frame width");
static const arg_def_t height = ARG_DEF("h", "height", 1, "Frame height");
#if CONFIG_WEBM_IO
static const struct arg_enum_list stereo_mode_enum[] = {
  { "mono", STEREO_FORMAT_MONO },
  { "left-right", STEREO_FORMAT_LEFT_RIGHT },
  { "bottom-top", STEREO_FORMAT_BOTTOM_TOP },
  { "top-bottom", STEREO_FORMAT_TOP_BOTTOM },
  { "right-left", STEREO_FORMAT_RIGHT_LEFT },
  { NULL, 0 }
};
static const arg_def_t stereo_mode = ARG_DEF_ENUM(
    NULL, "stereo-mode", 1, "Stereo 3D video format", stereo_mode_enum);
#endif
static const arg_def_t timebase = ARG_DEF(
    NULL, "timebase", 1, "Output timestamp precision (fractional seconds)");
static const arg_def_t error_resilient =
    ARG_DEF(NULL, "error-resilient", 1, "Enable error resiliency features");
static const arg_def_t lag_in_frames =
    ARG_DEF(NULL, "lag-in-frames", 1, "Max number of frames to lag");

static const arg_def_t *global_args[] = { &use_yv12,
                                          &use_i420,
                                          &use_i422,
                                          &use_i444,
                                          &use_i440,
                                          &usage,
                                          &threads,
                                          &profile,
                                          &width,
                                          &height,
#if CONFIG_WEBM_IO
                                          &stereo_mode,
#endif
                                          &timebase,
                                          &framerate,
                                          &error_resilient,
#if CONFIG_AOM_HIGHBITDEPTH
                                          &test16bitinternalarg,
#endif
                                          &lag_in_frames,
                                          NULL };

static const arg_def_t dropframe_thresh =
    ARG_DEF(NULL, "drop-frame", 1, "Temporal resampling threshold (buf %)");
static const arg_def_t resize_allowed =
    ARG_DEF(NULL, "resize-allowed", 1, "Spatial resampling enabled (bool)");
static const arg_def_t resize_width =
    ARG_DEF(NULL, "resize-width", 1, "Width of encoded frame");
static const arg_def_t resize_height =
    ARG_DEF(NULL, "resize-height", 1, "Height of encoded frame");
static const arg_def_t resize_up_thresh =
    ARG_DEF(NULL, "resize-up", 1, "Upscale threshold (buf %)");
static const arg_def_t resize_down_thresh =
    ARG_DEF(NULL, "resize-down", 1, "Downscale threshold (buf %)");
static const struct arg_enum_list end_usage_enum[] = { { "vbr", AOM_VBR },
                                                       { "cbr", AOM_CBR },
                                                       { "cq", AOM_CQ },
                                                       { "q", AOM_Q },
                                                       { NULL, 0 } };
static const arg_def_t end_usage =
    ARG_DEF_ENUM(NULL, "end-usage", 1, "Rate control mode", end_usage_enum);
static const arg_def_t target_bitrate =
    ARG_DEF(NULL, "target-bitrate", 1, "Bitrate (kbps)");
static const arg_def_t min_quantizer =
    ARG_DEF(NULL, "min-q", 1, "Minimum (best) quantizer");
static const arg_def_t max_quantizer =
    ARG_DEF(NULL, "max-q", 1, "Maximum (worst) quantizer");
static const arg_def_t undershoot_pct =
    ARG_DEF(NULL, "undershoot-pct", 1, "Datarate undershoot (min) target (%)");
static const arg_def_t overshoot_pct =
    ARG_DEF(NULL, "overshoot-pct", 1, "Datarate overshoot (max) target (%)");
static const arg_def_t buf_sz =
    ARG_DEF(NULL, "buf-sz", 1, "Client buffer size (ms)");
static const arg_def_t buf_initial_sz =
    ARG_DEF(NULL, "buf-initial-sz", 1, "Client initial buffer size (ms)");
static const arg_def_t buf_optimal_sz =
    ARG_DEF(NULL, "buf-optimal-sz", 1, "Client optimal buffer size (ms)");
static const arg_def_t *rc_args[] = {
  &dropframe_thresh, &resize_allowed,     &resize_width,   &resize_height,
  &resize_up_thresh, &resize_down_thresh, &end_usage,      &target_bitrate,
  &min_quantizer,    &max_quantizer,      &undershoot_pct, &overshoot_pct,
  &buf_sz,           &buf_initial_sz,     &buf_optimal_sz, NULL
};

static const arg_def_t bias_pct =
    ARG_DEF(NULL, "bias-pct", 1, "CBR/VBR bias (0=CBR, 100=VBR)");
static const arg_def_t minsection_pct =
    ARG_DEF(NULL, "minsection-pct", 1, "GOP min bitrate (% of target)");
static const arg_def_t maxsection_pct =
    ARG_DEF(NULL, "maxsection-pct", 1, "GOP max bitrate (% of target)");
static const arg_def_t *rc_twopass_args[] = { &bias_pct, &minsection_pct,
                                              &maxsection_pct, NULL };

static const arg_def_t kf_min_dist =
    ARG_DEF(NULL, "kf-min-dist", 1, "Minimum keyframe interval (frames)");
static const arg_def_t kf_max_dist =
    ARG_DEF(NULL, "kf-max-dist", 1, "Maximum keyframe interval (frames)");
static const arg_def_t kf_disabled =
    ARG_DEF(NULL, "disable-kf", 0, "Disable keyframe placement");
static const arg_def_t *kf_args[] = { &kf_min_dist, &kf_max_dist, &kf_disabled,
                                      NULL };

static const arg_def_t noise_sens =
    ARG_DEF(NULL, "noise-sensitivity", 1, "Noise sensitivity (frames to blur)");
static const arg_def_t sharpness =
    ARG_DEF(NULL, "sharpness", 1, "Loop filter sharpness (0..7)");
static const arg_def_t static_thresh =
    ARG_DEF(NULL, "static-thresh", 1, "Motion detection threshold");
static const arg_def_t auto_altref =
    ARG_DEF(NULL, "auto-alt-ref", 1, "Enable automatic alt reference frames");
static const arg_def_t arnr_maxframes =
    ARG_DEF(NULL, "arnr-maxframes", 1, "AltRef max frames (0..15)");
static const arg_def_t arnr_strength =
    ARG_DEF(NULL, "arnr-strength", 1, "AltRef filter strength (0..6)");
static const arg_def_t arnr_type = ARG_DEF(NULL, "arnr-type", 1, "AltRef type");
static const struct arg_enum_list tuning_enum[] = {
  { "psnr", AOM_TUNE_PSNR }, { "ssim", AOM_TUNE_SSIM }, { NULL, 0 }
};
static const arg_def_t tune_ssim =
    ARG_DEF_ENUM(NULL, "tune", 1, "Material to favor", tuning_enum);
static const arg_def_t cq_level =
    ARG_DEF(NULL, "cq-level", 1, "Constant/Constrained Quality level");
static const arg_def_t max_intra_rate_pct =
    ARG_DEF(NULL, "max-intra-rate", 1, "Max I-frame bitrate (pct)");

#if CONFIG_AV1_ENCODER
static const arg_def_t cpu_used_av1 =
    ARG_DEF(NULL, "cpu-used", 1, "CPU Used (-8..8)");
static const arg_def_t tile_cols =
    ARG_DEF(NULL, "tile-columns", 1, "Number of tile columns to use, log2");
static const arg_def_t tile_rows =
    ARG_DEF(NULL, "tile-rows", 1, "Number of tile rows to use, log2");
static const arg_def_t lossless =
    ARG_DEF(NULL, "lossless", 1, "Lossless mode (0: false (default), 1: true)");
#if CONFIG_AOM_QM
static const arg_def_t enable_qm =
    ARG_DEF(NULL, "enable_qm", 1,
            "Enable quantisation matrices (0: false (default), 1: true)");
static const arg_def_t qm_min = ARG_DEF(
    NULL, "qm_min", 1, "Min quant matrix flatness (0..15), default is 8");
static const arg_def_t qm_max = ARG_DEF(
    NULL, "qm_max", 1, "Max quant matrix flatness (0..15), default is 16");
#endif
static const arg_def_t frame_parallel_decoding = ARG_DEF(
    NULL, "frame-parallel", 0, "Enable frame parallel decodability features");
static const arg_def_t aq_mode = ARG_DEF(
    NULL, "aq-mode", 1,
    "Adaptive quantization mode (0: off (default), 1: variance 2: complexity, "
    "3: cyclic refresh, 4: equator360)");
static const arg_def_t frame_periodic_boost =
    ARG_DEF(NULL, "frame-boost", 1,
            "Enable frame periodic boost (0: off (default), 1: on)");
static const arg_def_t gf_cbr_boost_pct = ARG_DEF(
    NULL, "gf-cbr-boost", 1, "Boost for Golden Frame in CBR mode (pct)");<commit_msg>Undo accidental change to command line parameter count.<commit_after>/*
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#include "./aomenc.h"
#include "./aom_config.h"

#include <assert.h>
#include <limits.h>
#include <math.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#if CONFIG_LIBYUV
#include "third_party/libyuv/include/libyuv/scale.h"
#endif

#include "aom/aom_encoder.h"
#if CONFIG_DECODERS
#include "aom/aom_decoder.h"
#endif

#include "./args.h"
#include "./ivfenc.h"
#include "./tools_common.h"

#if CONFIG_AV1_ENCODER
#include "aom/aomcx.h"
#endif
#if CONFIG_AV1_DECODER
#include "aom/aomdx.h"
#endif

#include "aom/aom_integer.h"
#include "aom_ports/mem_ops.h"
#include "aom_ports/aom_timer.h"
#include "./rate_hist.h"
#include "./aomstats.h"
#include "./warnings.h"
#if CONFIG_WEBM_IO
#include "./webmenc.h"
#endif
#include "./y4minput.h"

/* Swallow warnings about unused results of fread/fwrite */
static size_t wrap_fread(void *ptr, size_t size, size_t nmemb, FILE *stream) {
  return fread(ptr, size, nmemb, stream);
}
#define fread wrap_fread

static size_t wrap_fwrite(const void *ptr, size_t size, size_t nmemb,
                          FILE *stream) {
  return fwrite(ptr, size, nmemb, stream);
}
#define fwrite wrap_fwrite

static const char *exec_name;

static void warn_or_exit_on_errorv(aom_codec_ctx_t *ctx, int fatal,
                                   const char *s, va_list ap) {
  if (ctx->err) {
    const char *detail = aom_codec_error_detail(ctx);

    vfprintf(stderr, s, ap);
    fprintf(stderr, ": %s\n", aom_codec_error(ctx));

    if (detail) fprintf(stderr, "    %s\n", detail);

    if (fatal) exit(EXIT_FAILURE);
  }
}

static void ctx_exit_on_error(aom_codec_ctx_t *ctx, const char *s, ...) {
  va_list ap;

  va_start(ap, s);
  warn_or_exit_on_errorv(ctx, 1, s, ap);
  va_end(ap);
}

static void warn_or_exit_on_error(aom_codec_ctx_t *ctx, int fatal,
                                  const char *s, ...) {
  va_list ap;

  va_start(ap, s);
  warn_or_exit_on_errorv(ctx, fatal, s, ap);
  va_end(ap);
}

static int read_frame(struct AvxInputContext *input_ctx, aom_image_t *img) {
  FILE *f = input_ctx->file;
  y4m_input *y4m = &input_ctx->y4m;
  int shortread = 0;

  if (input_ctx->file_type == FILE_TYPE_Y4M) {
    if (y4m_input_fetch_frame(y4m, f, img) < 1) return 0;
  } else {
    shortread = read_yuv_frame(input_ctx, img);
  }

  return !shortread;
}

static int file_is_y4m(const char detect[4]) {
  if (memcmp(detect, "YUV4", 4) == 0) {
    return 1;
  }
  return 0;
}

static int fourcc_is_ivf(const char detect[4]) {
  if (memcmp(detect, "DKIF", 4) == 0) {
    return 1;
  }
  return 0;
}

static const arg_def_t debugmode =
    ARG_DEF("D", "debug", 0, "Debug mode (makes output deterministic)");
static const arg_def_t outputfile =
    ARG_DEF("o", "output", 1, "Output filename");
static const arg_def_t use_yv12 =
    ARG_DEF(NULL, "yv12", 0, "Input file is YV12 ");
static const arg_def_t use_i420 =
    ARG_DEF(NULL, "i420", 0, "Input file is I420 (default)");
static const arg_def_t use_i422 =
    ARG_DEF(NULL, "i422", 0, "Input file is I422");
static const arg_def_t use_i444 =
    ARG_DEF(NULL, "i444", 0, "Input file is I444");
static const arg_def_t use_i440 =
    ARG_DEF(NULL, "i440", 0, "Input file is I440");
static const arg_def_t codecarg = ARG_DEF(NULL, "codec", 1, "Codec to use");
static const arg_def_t passes =
    ARG_DEF("p", "passes", 1, "Number of passes (1/2)");
static const arg_def_t pass_arg =
    ARG_DEF(NULL, "pass", 1, "Pass to execute (1/2)");
static const arg_def_t fpf_name =
    ARG_DEF(NULL, "fpf", 1, "First pass statistics file name");
#if CONFIG_FP_MB_STATS
static const arg_def_t fpmbf_name =
    ARG_DEF(NULL, "fpmbf", 1, "First pass block statistics file name");
#endif
static const arg_def_t limit =
    ARG_DEF(NULL, "limit", 1, "Stop encoding after n input frames");
static const arg_def_t skip =
    ARG_DEF(NULL, "skip", 1, "Skip the first n input frames");
static const arg_def_t deadline =
    ARG_DEF("d", "deadline", 1, "Deadline per frame (usec)");
static const arg_def_t best_dl =
    ARG_DEF(NULL, "best", 0, "Use Best Quality Deadline");
static const arg_def_t good_dl =
    ARG_DEF(NULL, "good", 0, "Use Good Quality Deadline");
static const arg_def_t rt_dl =
    ARG_DEF(NULL, "rt", 0, "Use Realtime Quality Deadline");
static const arg_def_t quietarg =
    ARG_DEF("q", "quiet", 0, "Do not print encode progress");
static const arg_def_t verbosearg =
    ARG_DEF("v", "verbose", 0, "Show encoder parameters");
static const arg_def_t psnrarg =
    ARG_DEF(NULL, "psnr", 0, "Show PSNR in status line");

static const struct arg_enum_list test_decode_enum[] = {
  { "off", TEST_DECODE_OFF },
  { "fatal", TEST_DECODE_FATAL },
  { "warn", TEST_DECODE_WARN },
  { NULL, 0 }
};
static const arg_def_t recontest = ARG_DEF_ENUM(
    NULL, "test-decode", 1, "Test encode/decode mismatch", test_decode_enum);
static const arg_def_t framerate =
    ARG_DEF(NULL, "fps", 1, "Stream frame rate (rate/scale)");
static const arg_def_t use_webm =
    ARG_DEF(NULL, "webm", 0, "Output WebM (default when WebM IO is enabled)");
static const arg_def_t use_ivf = ARG_DEF(NULL, "ivf", 0, "Output IVF");
static const arg_def_t out_part =
    ARG_DEF("P", "output-partitions", 0,
            "Makes encoder output partitions. Requires IVF output!");
static const arg_def_t q_hist_n =
    ARG_DEF(NULL, "q-hist", 1, "Show quantizer histogram (n-buckets)");
static const arg_def_t rate_hist_n =
    ARG_DEF(NULL, "rate-hist", 1, "Show rate histogram (n-buckets)");
static const arg_def_t disable_warnings =
    ARG_DEF(NULL, "disable-warnings", 0,
            "Disable warnings about potentially incorrect encode settings.");
static const arg_def_t disable_warning_prompt =
    ARG_DEF("y", "disable-warning-prompt", 0,
            "Display warnings, but do not prompt user to continue.");

#if CONFIG_AOM_HIGHBITDEPTH
static const arg_def_t test16bitinternalarg = ARG_DEF(
    NULL, "test-16bit-internal", 0, "Force use of 16 bit internal buffer");
#endif

static const arg_def_t *main_args[] = { &debugmode,
                                        &outputfile,
                                        &codecarg,
                                        &passes,
                                        &pass_arg,
                                        &fpf_name,
                                        &limit,
                                        &skip,
                                        &deadline,
                                        &best_dl,
                                        &good_dl,
                                        &rt_dl,
                                        &quietarg,
                                        &verbosearg,
                                        &psnrarg,
                                        &use_webm,
                                        &use_ivf,
                                        &out_part,
                                        &q_hist_n,
                                        &rate_hist_n,
                                        &disable_warnings,
                                        &disable_warning_prompt,
                                        &recontest,
                                        NULL };

static const arg_def_t usage =
    ARG_DEF("u", "usage", 1, "Usage profile number to use");
static const arg_def_t threads =
    ARG_DEF("t", "threads", 1, "Max number of threads to use");
static const arg_def_t profile =
    ARG_DEF(NULL, "profile", 1, "Bitstream profile number to use");
static const arg_def_t width = ARG_DEF("w", "width", 1, "Frame width");
static const arg_def_t height = ARG_DEF("h", "height", 1, "Frame height");
#if CONFIG_WEBM_IO
static const struct arg_enum_list stereo_mode_enum[] = {
  { "mono", STEREO_FORMAT_MONO },
  { "left-right", STEREO_FORMAT_LEFT_RIGHT },
  { "bottom-top", STEREO_FORMAT_BOTTOM_TOP },
  { "top-bottom", STEREO_FORMAT_TOP_BOTTOM },
  { "right-left", STEREO_FORMAT_RIGHT_LEFT },
  { NULL, 0 }
};
static const arg_def_t stereo_mode = ARG_DEF_ENUM(
    NULL, "stereo-mode", 1, "Stereo 3D video format", stereo_mode_enum);
#endif
static const arg_def_t timebase = ARG_DEF(
    NULL, "timebase", 1, "Output timestamp precision (fractional seconds)");
static const arg_def_t error_resilient =
    ARG_DEF(NULL, "error-resilient", 1, "Enable error resiliency features");
static const arg_def_t lag_in_frames =
    ARG_DEF(NULL, "lag-in-frames", 1, "Max number of frames to lag");

static const arg_def_t *global_args[] = { &use_yv12,
                                          &use_i420,
                                          &use_i422,
                                          &use_i444,
                                          &use_i440,
                                          &usage,
                                          &threads,
                                          &profile,
                                          &width,
                                          &height,
#if CONFIG_WEBM_IO
                                          &stereo_mode,
#endif
                                          &timebase,
                                          &framerate,
                                          &error_resilient,
#if CONFIG_AOM_HIGHBITDEPTH
                                          &test16bitinternalarg,
#endif
                                          &lag_in_frames,
                                          NULL };

static const arg_def_t dropframe_thresh =
    ARG_DEF(NULL, "drop-frame", 1, "Temporal resampling threshold (buf %)");
static const arg_def_t resize_allowed =
    ARG_DEF(NULL, "resize-allowed", 1, "Spatial resampling enabled (bool)");
static const arg_def_t resize_width =
    ARG_DEF(NULL, "resize-width", 1, "Width of encoded frame");
static const arg_def_t resize_height =
    ARG_DEF(NULL, "resize-height", 1, "Height of encoded frame");
static const arg_def_t resize_up_thresh =
    ARG_DEF(NULL, "resize-up", 1, "Upscale threshold (buf %)");
static const arg_def_t resize_down_thresh =
    ARG_DEF(NULL, "resize-down", 1, "Downscale threshold (buf %)");
static const struct arg_enum_list end_usage_enum[] = { { "vbr", AOM_VBR },
                                                       { "cbr", AOM_CBR },
                                                       { "cq", AOM_CQ },
                                                       { "q", AOM_Q },
                                                       { NULL, 0 } };
static const arg_def_t end_usage =
    ARG_DEF_ENUM(NULL, "end-usage", 1, "Rate control mode", end_usage_enum);
static const arg_def_t target_bitrate =
    ARG_DEF(NULL, "target-bitrate", 1, "Bitrate (kbps)");
static const arg_def_t min_quantizer =
    ARG_DEF(NULL, "min-q", 1, "Minimum (best) quantizer");
static const arg_def_t max_quantizer =
    ARG_DEF(NULL, "max-q", 1, "Maximum (worst) quantizer");
static const arg_def_t undershoot_pct =
    ARG_DEF(NULL, "undershoot-pct", 1, "Datarate undershoot (min) target (%)");
static const arg_def_t overshoot_pct =
    ARG_DEF(NULL, "overshoot-pct", 1, "Datarate overshoot (max) target (%)");
static const arg_def_t buf_sz =
    ARG_DEF(NULL, "buf-sz", 1, "Client buffer size (ms)");
static const arg_def_t buf_initial_sz =
    ARG_DEF(NULL, "buf-initial-sz", 1, "Client initial buffer size (ms)");
static const arg_def_t buf_optimal_sz =
    ARG_DEF(NULL, "buf-optimal-sz", 1, "Client optimal buffer size (ms)");
static const arg_def_t *rc_args[] = {
  &dropframe_thresh, &resize_allowed,     &resize_width,   &resize_height,
  &resize_up_thresh, &resize_down_thresh, &end_usage,      &target_bitrate,
  &min_quantizer,    &max_quantizer,      &undershoot_pct, &overshoot_pct,
  &buf_sz,           &buf_initial_sz,     &buf_optimal_sz, NULL
};

static const arg_def_t bias_pct =
    ARG_DEF(NULL, "bias-pct", 1, "CBR/VBR bias (0=CBR, 100=VBR)");
static const arg_def_t minsection_pct =
    ARG_DEF(NULL, "minsection-pct", 1, "GOP min bitrate (% of target)");
static const arg_def_t maxsection_pct =
    ARG_DEF(NULL, "maxsection-pct", 1, "GOP max bitrate (% of target)");
static const arg_def_t *rc_twopass_args[] = { &bias_pct, &minsection_pct,
                                              &maxsection_pct, NULL };

static const arg_def_t kf_min_dist =
    ARG_DEF(NULL, "kf-min-dist", 1, "Minimum keyframe interval (frames)");
static const arg_def_t kf_max_dist =
    ARG_DEF(NULL, "kf-max-dist", 1, "Maximum keyframe interval (frames)");
static const arg_def_t kf_disabled =
    ARG_DEF(NULL, "disable-kf", 0, "Disable keyframe placement");
static const arg_def_t *kf_args[] = { &kf_min_dist, &kf_max_dist, &kf_disabled,
                                      NULL };

static const arg_def_t noise_sens =
    ARG_DEF(NULL, "noise-sensitivity", 1, "Noise sensitivity (frames to blur)");
static const arg_def_t sharpness =
    ARG_DEF(NULL, "sharpness", 1, "Loop filter sharpness (0..7)");
static const arg_def_t static_thresh =
    ARG_DEF(NULL, "static-thresh", 1, "Motion detection threshold");
static const arg_def_t auto_altref =
    ARG_DEF(NULL, "auto-alt-ref", 1, "Enable automatic alt reference frames");
static const arg_def_t arnr_maxframes =
    ARG_DEF(NULL, "arnr-maxframes", 1, "AltRef max frames (0..15)");
static const arg_def_t arnr_strength =
    ARG_DEF(NULL, "arnr-strength", 1, "AltRef filter strength (0..6)");
static const arg_def_t arnr_type = ARG_DEF(NULL, "arnr-type", 1, "AltRef type");
static const struct arg_enum_list tuning_enum[] = {
  { "psnr", AOM_TUNE_PSNR }, { "ssim", AOM_TUNE_SSIM }, { NULL, 0 }
};
static const arg_def_t tune_ssim =
    ARG_DEF_ENUM(NULL, "tune", 1, "Material to favor", tuning_enum);
static const arg_def_t cq_level =
    ARG_DEF(NULL, "cq-level", 1, "Constant/Constrained Quality level");
static const arg_def_t max_intra_rate_pct =
    ARG_DEF(NULL, "max-intra-rate", 1, "Max I-frame bitrate (pct)");

#if CONFIG_AV1_ENCODER
static const arg_def_t cpu_used_av1 =
    ARG_DEF(NULL, "cpu-used", 1, "CPU Used (-8..8)");
static const arg_def_t tile_cols =
    ARG_DEF(NULL, "tile-columns", 1, "Number of tile columns to use, log2");
static const arg_def_t tile_rows =
    ARG_DEF(NULL, "tile-rows", 1, "Number of tile rows to use, log2");
static const arg_def_t lossless =
    ARG_DEF(NULL, "lossless", 1, "Lossless mode (0: false (default), 1: true)");
#if CONFIG_AOM_QM
static const arg_def_t enable_qm =
    ARG_DEF(NULL, "enable_qm", 1,
            "Enable quantisation matrices (0: false (default), 1: true)");
static const arg_def_t qm_min = ARG_DEF(
    NULL, "qm_min", 1, "Min quant matrix flatness (0..15), default is 8");
static const arg_def_t qm_max = ARG_DEF(
    NULL, "qm_max", 1, "Max quant matrix flatness (0..15), default is 16");
#endif
static const arg_def_t frame_parallel_decoding =
    ARG_DEF(NULL, "frame-parallel", 1,
            "Enable frame parallel decodability features "
            "(0: false (default), 1: true)");
static const arg_def_t aq_mode = ARG_DEF(
    NULL, "aq-mode", 1,
    "Adaptive quantization mode (0: off (default), 1: variance 2: complexity, "
    "3: cyclic refresh, 4: equator360)");
static const arg_def_t frame_periodic_boost =
    ARG_DEF(NULL, "frame-boost", 1,
            "Enable frame periodic boost (0: off (default), 1: on)");
static const arg_def_t gf_cbr_boost_pct = ARG_DEF(
    NULL, "gf-cbr-boost", 1, "Boost for Golden Frame in CBR mode (pct)");
{'fraction_non_alphanumeric': 0.11554661469793173, 'fraction_numerical': 0.014521908593700887, 'mean_word_length': 2.605507706255666, 'pattern_counts': {'":': 2, '<': 19, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 136, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5775956', 'n_tokens_mistral': 11014, 'n_tokens_neox': 10441, 'n_words': 2922}