<commit_before>#include <stdint.h>
#include <pthread.h>
#include <float.h> /* for DBL_MAX */

#include <libavcodec/avfft.h>
#include <libavformat/avformat.h>
#include <libavfilter/avfilter.h>
#include <libavfilter/buffersink.h>
#include <libavfilter/buffersrc.h>
#include <libavutil/avassert.h>
#include <libavutil/avstring.h>
#include <libavutil/opt.h>
#include <libavutil/timestamp.h>

#include "sfxmp.h"

struct Frame {
    AVFrame *frame;         // a frame the user might want
    double ts;              // rescaled timestamp
};

#define AUDIO_NBITS      10
#define AUDIO_NBSAMPLES  (1<<(AUDIO_NBITS))
#define AUDIO_NBCHANNELS 2

struct sfxmp_ctx {
    /* configurable options */
    char *filename;                         // input filename
    int avselect;                           // select audio or video
    double visible_time;                    // see public header
    double start_time;                      // see public header
    double skip;                            // see public header
    double trim_duration;                   // see public header
    int max_nb_frames;                      // maximum number of frames in the queue
    double dist_time_seek_trigger;          // distance time triggering a seek

    /* misc general fields */
    enum AVMediaType media_type;            // AVMEDIA_TYPE_{VIDEO,AUDIO} according to avselect
    const char *media_type_string;          // "audio" or "video" according to avselect

    /* main vs demuxer/decoder thread negotiation */
    pthread_t dec_thread;                   // decoding thread
    pthread_mutex_t queue_lock;             // for any op related to the queue
    pthread_cond_t queue_reduce;            // notify a reducing queue (MUST be ONLY signaled from main thread)
    pthread_cond_t queue_grow;              // notify a growing queue (MUST be ONLY signaled from decoding thread)

    /* fields that MUST be protected by queue_lock */
    int queue_terminated;                   // 0 if queue is not going to be filled anymore (because thread is dead), 1 otherwise
    struct Frame *frames;                   // queue of the decoded (and filtered) frames
    int nb_frames;                          // total number of frames in the queue
    double request_seek;                    // field used by the main thread to request a seek to the decoding thread
    int can_seek_again;                     // field used to avoid seeking again until the requested time is reached

    /* fields specific to main thread */
    struct Frame non_visible;               // frame to display for the "non visible" time zone
    struct sfxmp_frame rframe;              // user returned frame container
    double last_pushed_frame_ts;            // ts value of the latest pushed frame (it acts as a UID)

    /* fields specific to decoding thread */
    AVFrame *decoded_frame;                 // decoded frame
    AVFrame *filtered_frame;                // filtered version of decoded_frame
    AVFrame *audio_texture_frame;           // wave/fft texture in case of audio
    AVFormatContext *fmt_ctx;               // demuxing context
    AVCodecContext  *dec_ctx;               // decoder context
    AVStream *stream;                       // selected stream
    int stream_idx;                         // selected stream index
    AVFilterGraph *filter_graph;            // libavfilter graph
    char *filter_graph_str;                 // libavfilter graph string
    AVFilterContext *buffersink_ctx;        // sink of the graph (from where we pull)
    AVFilterContext *buffersrc_ctx;         // source of the graph (where we push)
    float *window_func_lut;                 // audio window function lookup table
    RDFTContext *rdft;                      // real discrete fourier transform context
    FFTSample *rdft_data[AUDIO_NBCHANNELS]; // real discrete fourier transform data for each channel
};

#define ENABLE_DBG 0

#define DBG_SFXMP(mod, ...) do { printf("[sfxmp:"mod"] " __VA_ARGS__); fflush(stdout); } while (0)
#if ENABLE_DBG
# define DBG(mod, ...) DBG_SFXMP(mod, __VA_ARGS__)
#else
# define DBG(mod, ...) do { if (0) DBG_SFXMP(mod, __VA_ARGS__); } while (0)
#endif

/**
 * Allocate a small frame to be displayed before visible_time
 */
static AVFrame *get_invisible_frame(enum AVMediaType media_type)
{
    AVFrame *frame = av_frame_alloc();
    if (!frame)
        return NULL;

    frame->format = AV_PIX_FMT_RGB32;

    if (media_type == AVMEDIA_TYPE_VIDEO) {
        frame->width  = 2;
        frame->height = 2;
    } else {
        frame->width  = AUDIO_NBSAMPLES/2;      // samples are float (32 bits), pix fmt is rgb32 (32 bits as well)
        /* height:
         *   AUDIO_NBCHANNELS (waves lines)
         * + AUDIO_NBCHANNELS (fft lines of width AUDIO_NBCHANNELS/2, or 1<<(AUDIO_NBITS-1))
         * + AUDIO_NBITS-1 AUDIO_NBCHANNELS (fft lines downscaled) */
        frame->height = (1 + AUDIO_NBITS) * AUDIO_NBCHANNELS;
    }

    if (av_frame_get_buffer(frame, 16) < 0) {
        av_frame_free(&frame);
        return NULL;
    }

#define SET_COLOR(x, y, color) *(uint32_t *)&frame->data[0][y*frame->linesize[0] + x*4] = color

    if (media_type == AVMEDIA_TYPE_VIDEO) {
#if ENABLE_DBG
    /* In debug more, we make them colored for visual debug */
    SET_COLOR(0, 0, 0xff0000ff);
    SET_COLOR(1, 1, 0x00ff00ff);
    SET_COLOR(0, 1, 0x0000ffff);
    SET_COLOR(1, 0, 0xffffffff);
#else
    /* ...but in production we want it to be black */
    SET_COLOR(0, 0, 0x00000000);
    SET_COLOR(1, 1, 0x00000000);
    SET_COLOR(0, 1, 0x00000000);
    SET_COLOR(1, 0, 0x00000000);
#endif
    } else {
        memset(frame->data[0], 0, frame->height * frame->linesize[0]);
    }
    return frame;
}

static void free_context(struct sfxmp_ctx *s)
{
    int i;

    if (!s)
        return;

    if (s->frames) {
        for (i = 0; i < s->max_nb_frames; i++)
            av_frame_free(&s->frames[i].frame);
        av_freep(&s->frames);
    }

    av_frame_free(&s->non_visible.frame);
    av_freep(&s->filename);
    av_freep(&s->filter_graph_str);
    av_freep(&s);
}

static char *get_filter_graph_str(enum AVMediaType media_type, const char *filters)
{
    char buf[512];

    if (media_type == AVMEDIA_TYPE_VIDEO)
        snprintf(buf, sizeof(buf), "format=rgb32");
    else
        snprintf(buf, sizeof(buf), "aformat=sample_fmts=fltp:channel_layouts=stereo, asetnsamples=%d", AUDIO_NBSAMPLES);

    return filters ? av_asprintf("%s,%s", filters, buf)
                   : av_strdup(buf);
}

struct sfxmp_ctx *sfxmp_create(const char *filename,
                               int avselect,
                               double visible_time,
                               double start_time,
                               double skip,
                               double trim_duration,
                               double dist_time_seek_trigger,
                               double max_nb_frames,
                               const char *filters)
{
    int i;
    struct sfxmp_ctx *s;

    av_register_all();
    avfilter_register_all();
    av_log_set_level(AV_LOG_ERROR);

    s = av_mallocz(sizeof(*s));
    if (!s)
        return NULL;

    s->filename               = av_strdup(filename);
    s->avselect               = avselect;
    s->visible_time           = visible_time;
    s->start_time             = start_time;
    s->skip                   = skip;
    s->trim_duration          = trim_duration;
    s->dist_time_seek_trigger = dist_time_seek_trigger < 0 ? 3 : dist_time_seek_trigger;
    s->max_nb_frames          = max_nb_frames < 0 ? 5 : max_nb_frames;

    switch (s->avselect) {
    case SFXMP_SELECT_VIDEO: s->media_type = AVMEDIA_TYPE_VIDEO; break;
    case SFXMP_SELECT_AUDIO: s->media_type = AVMEDIA_TYPE_AUDIO; break;
    default:
        fprintf(stderr, "unknown avselect value %d\n", s->avselect);
        goto fail;
    }

    s->media_type_string = av_get_media_type_string(s->media_type);
    av_assert0(s->media_type_string);

    s->filter_graph_str = get_filter_graph_str(s->media_type, filters);

    if (s->max_nb_frames < 2) {
        fprintf(stderr, "max_nb_frames < 2 is not supported\n");
        goto fail;
    }

    if (!s->filename || !s->filter_graph_str)
        goto fail;

    s->frames = av_malloc_array(s->max_nb_frames, sizeof(*s->frames));
    if (!s->frames)
        goto fail;
    for (i = 0; i < s->max_nb_frames; i++) {
        s->frames[i].frame = av_frame_alloc();
        if (!s->frames[i].frame)
            goto fail;
    }

    s->last_pushed_frame_ts = DBL_MIN;

    s->non_visible.ts    = -1;
    s->non_visible.frame = get_invisible_frame(s->media_type);
    if (!s->non_visible.frame)
        goto fail;

    s->queue_terminated = 1;

    pthread_mutex_init(&s->queue_lock, NULL);
    pthread_cond_init(&s->queue_reduce, NULL);
    pthread_cond_init(&s->queue_grow, NULL);

    DBG("init", "filename:%s avselect:%d visible_time:%f start_time:%f skip:%f trim_duration:%f (%p)\n",
        filename, avselect, visible_time, start_time, skip, trim_duration, s);

    return s;

fail:
    free_context(s);
    return NULL;
}

void sfxmp_free(struct sfxmp_ctx **ss)
{
    struct sfxmp_ctx *s = *ss;

    DBG("free", "calling sfxmp_free() (%p)\n", s);

    if (!s)
        return;

    if (!s->queue_terminated) {
        DBG("free", "queue is not terminated yet\n");
        pthread_mutex_lock(&s->queue_lock);
        s->queue_terminated = 1; // notify decoding thread must die
        pthread_mutex_unlock(&s->queue_lock);
        pthread_cond_signal(&s->queue_reduce); // wake up decoding thread

        pthread_join(s->dec_thread, NULL);
    }
    pthread_cond_destroy(&s->queue_reduce);
    pthread_cond_destroy(&s->queue_grow);
    pthread_mutex_destroy(&s->queue_lock);
    free_context(s);
}

static int decode_packet(struct sfxmp_ctx *s, AVPacket *pkt,
                         AVFrame *frame, int *got_frame)
{
    int decoded = pkt->size;

    *got_frame = 0;
    if (pkt->stream_index == s->stream_idx) {
        int ret;

        if (s->media_type == AVMEDIA_TYPE_VIDEO)
            ret = avcodec_decode_video2(s->dec_ctx, frame, got_frame, pkt);
        else
            ret = avcodec_decode_audio4(s->dec_ctx, frame, got_frame, pkt);

        if (ret < 0) {
            fprintf(stderr, "Error decoding %s frame\n", s->media_type_string);
            return ret;
        }
        decoded = FFMIN(ret, pkt->size);
    }
    return decoded;
}

static int open_ifile(struct sfxmp_ctx *s, const char *infile)
{
    int ret = 0;
    AVCodec *dec;
    AVDictionary *opts = NULL;
    AVInputFormat *ifmt = NULL;

    ret = avformat_open_input(&s->fmt_ctx, infile, ifmt, &opts);
    if (ret < 0) {
        fprintf(stderr, "Unable to open input file '%s'\n", infile);
        return ret;
    }

    ret = avformat_find_stream_info(s->fmt_ctx, NULL);
    if (ret < 0) {
        fprintf(stderr, "Unable to find input stream information\n");
        return ret;
    }

    ret = av_find_best_stream(s->fmt_ctx, s->media_type, -1, -1, &dec, 0);
    if (ret < 0) {
        fprintf(stderr, "Unable to find a %s stream in the input file\n", s->media_type_string);
        return ret;
    }
    s->stream_idx = ret;
    s->stream = s->fmt_ctx->streams[s->stream_idx];
    s->dec_ctx = s->stream->codec;
    av_opt_set_int(s->dec_ctx, "refcounted_frames", 1, 0);

    ret = avcodec_open2(s->dec_ctx, dec, NULL);
    if (ret < 0) {
        fprintf(stderr, "Unable to open input %s decoder\n", s->media_type_string);
        return ret;
    }

    /* If trim_duration is not specified, we try to probe from the stream or
     * format (presentation) duration */
    if (s->trim_duration < 0) {
        int64_t duration = s->fmt_ctx->duration;
        double scale = av_q2d(AV_TIME_BASE_Q);

        if (duration == AV_NOPTS_VALUE) {
            duration = s->stream->duration;
            if (s->stream->time_base.den)
                scale = av_q2d(s->stream->time_base);
            else
                duration = AV_NOPTS_VALUE;
        }
        if (duration == AV_NOPTS_VALUE) {
            fprintf(stderr, "trim_duration is not set and can't estimate the duration, aborting\n");
            return AVERROR_INVALIDDATA;
        }
        s->trim_duration = duration * scale;
        DBG("decoder", "set trim duration to %f\n", s->trim_duration);
    }

    return 0;
}

/**
 * Map the timeline time to the media time
 */
static double get_media_time(const struct sfxmp_ctx *s, double t)
{
    return s->skip + av_clipd(t - s->start_time, 0, s->trim_duration);
}

/**
 * Request a seek to the demuxer
 */
static int seek_to(struct sfxmp_ctx *s, double t)
{
    const double vt = get_media_time(s, t);
    const int64_t ts = vt * AV_TIME_BASE;

    if (ENABLE_DBG)
        DBG("decoder", "Seek in media at %f (t=%f)\n", vt, t);
    else
        printf("Seek in media at %f (t=%f)\n", vt, t);
    return avformat_seek_file(s->fmt_ctx, -1, INT64_MIN, ts, ts, 0);
}

/**
 * Setup the libavfilter filtergraph for user filter but also to have a way to
 * request a pixel format we want, and let libavfilter insert the necessary
 * scaling filter (typically, an automatic conversion from yuv420p to rgb32).
 */
static int setup_filtergraph(struct sfxmp_ctx *s)
{
    int ret = 0;
    char args[512];
    const AVRational time_base = s->stream->time_base;
    const AVRational framerate = av_guess_frame_rate(s->fmt_ctx, s->stream, NULL);
    AVFilter *buffersrc  = avfilter_get_by_name(s->media_type == AVMEDIA_TYPE_VIDEO ? "buffer" : "abuffer");
    AVFilter *buffersink = avfilter_get_by_name(s->media_type == AVMEDIA_TYPE_VIDEO ? "buffersink" : "abuffersink");
    AVFilterInOut *outputs = avfilter_inout_alloc();
    AVFilterInOut *inputs  = avfilter_inout_alloc();

    avfilter_graph_free(&s->filter_graph);
    s->filter_graph = avfilter_graph_alloc();

    if (!inputs || !outputs || !s->filter_graph) {
        ret = AVERROR(ENOMEM);
        goto end;
    }

    inputs->name  = av_strdup("out");
    outputs->name = av_strdup("in");
    if (!inputs->name || !outputs->name) {
        ret = AVERROR(ENOMEM);
        goto end;
    }

    /* create buffer filter source (where we push the frame) */
    if (s->media_type == AVMEDIA_TYPE_VIDEO) {
        snprintf(args, sizeof(args),
                 "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d:sws_param=flags=bicubic",
                 s->dec_ctx->width, s->dec_ctx->height, s->dec_ctx->pix_fmt,
                 time_base.num, time_base.den,
                 s->dec_ctx->sample_aspect_ratio.num, s->dec_ctx->sample_aspect_ratio.den);
    } else {
        snprintf(args, sizeof(args), "time_base=%d/%d:sample_rate=%d:sample_fmt=%s",
                 1, s->dec_ctx->sample_rate, s->dec_ctx->sample_rate,
                 av_get_sample_fmt_name(s->dec_ctx->sample_fmt));
        if (s->dec_ctx->channel_layout)
            av_strlcatf(args, sizeof(args), ":channel_layout=0x%"PRIx64, s->dec_ctx->channel_layout);
        else
            av_strlcatf(args, sizeof(args), ":channels=%d", s->dec_ctx->channels);
    }

    if (framerate.num && framerate.den)
        av_strlcatf(args, sizeof(args), ":frame_rate=%d/%d", framerate.num, framerate.den);

    ret = avfilter_graph_create_filter(&s->buffersrc_ctx, buffersrc,
                                       outputs->name, args, NULL, s->filter_graph);
    if (ret < 0) {
        fprintf(stderr, "Unable to create buffer filter source\n");
        goto end;
    }

    /* create buffer filter sink (where we pull the frame) */
    ret = avfilter_graph_create_filter(&s->buffersink_ctx, buffersink,
                                       inputs->name, NULL, NULL, s->filter_graph);
    if (ret < 0) {
        fprintf(stderr, "Unable to create buffer filter sink\n");
        goto end;
    }

    /* create our filter graph */
    inputs->filter_ctx  = s->buffersink_ctx;
    outputs->filter_ctx = s->buffersrc_ctx;
    ret = avfilter_graph_parse_ptr(s->filter_graph, s->filter_graph_str,
                                   &inputs, &outputs, NULL);
    if (ret < 0)
        goto end;

    ret = avfilter_graph_config(s->filter_graph, NULL);
    if (ret < 0)
        goto end;

end:
    avfilter_inout_free(&inputs);
    avfilter_inout_free(&outputs);
    return ret;
}

/**
 * Convert an audio frame (PCM data) to a textured video frame with waves and
 * FFT lines
 */
static void audio_frame_to_sound_texture(struct sfxmp_ctx *s, AVFrame *dst_video,
                                         const AVFrame *audio_src)
{
    int i, j, ch;
    const int nb_samples = audio_src->nb_samples;
    const int width = nb_samples / 2;
    const float scale = 1.f / sqrt(AUDIO_NBSAMPLES/2 + 1);

    memset(dst_video->data[0], 0, dst_video->height * dst_video->linesize[0]);

    /* Copy waves */
    for (ch = 0; ch < AUDIO_NBCHANNELS; ch++) {
        const int lz = dst_video->linesize[0];
        float *samples_dst = (float *)(dst_video->data[0] + ch * lz);
        const float *samples_src = (const float *)audio_src->extended_data[ch];

        for (i = 0; i < width; i++)
            samples_dst[i] = (samples_src[width/2 + i] + 1.f) / 2.f;
    }

    /* Fourier transform */
    for (ch = 0; ch < AUDIO_NBCHANNELS; ch++) {
        const int lz = dst_video->linesize[0];
        float *fft_dst = (float *)(dst_video->data[0] + (AUDIO_NBCHANNELS + ch) * lz);
        const float *samples_src = (const float *)audio_src->extended_data[ch];
        float *bins = s->rdft_data[ch];

        /* Apply window function to input samples */
        for (i = 0; i < nb_samples; i++)
            bins[i] = samples_src[i] * s->window_func_lut[i];

        /* Run transform */
        av_rdft_calc(s->rdft, bins);

        /* Get magnitude of frequency bins and copy result into texture */
#define MAGNITUDE(re, im) sqrtf(((re)*(re) + (im)*(im)) * scale)
        for (i = 0; i < width - 1; i++)
            fft_dst[i] = MAGNITUDE(bins[2*(i + 1)], bins[2*(i + 1) + 1]);
        fft_dst[width - 1] = MAGNITUDE(bins[1], 0);
    }

    /* Downscaled versions of the FFT */
    for (i = 0; i < AUDIO_NBITS-1; i++) {
        for (ch = 0; ch < AUDIO_NBCHANNELS; ch++) {
            const int lz = dst_video->linesize[0];
            const int source_line = (i + 1)*AUDIO_NBCHANNELS + ch;
            float *fft_src = (float *)(dst_video->data[0] +  source_line                     * lz);
            float *fft_dst = (float *)(dst_video->data[0] + (source_line + AUDIO_NBCHANNELS) * lz);

            const int source_step = 1 << i;
            const int nb_identical_values = source_step << 1;
            const int nb_dest_pixels = width / nb_identical_values;

            DBG("audio2tex", "line %2d->%2d: %3d different pixels (copied %3dx) as destination, step source: %d\n",
                source_line, source_line + AUDIO_NBCHANNELS, nb_dest_pixels, nb_identical_values, source_step);

            for (j = 0; j < nb_dest_pixels; j++) {
                int x;
                const float avg = (fft_src[ j*2      * source_step] +
                                   fft_src[(j*2 + 1) * source_step]) / 2.f;

                for (x = 0; x < nb_identical_values; x++)
                    fft_dst[j*nb_identical_values + x] = avg;
            }
        }<commit_msg>sfxmp: clarify FFT code<commit_after>#include <stdint.h>
#include <pthread.h>
#include <float.h> /* for DBL_MAX */

#include <libavcodec/avfft.h>
#include <libavformat/avformat.h>
#include <libavfilter/avfilter.h>
#include <libavfilter/buffersink.h>
#include <libavfilter/buffersrc.h>
#include <libavutil/avassert.h>
#include <libavutil/avstring.h>
#include <libavutil/opt.h>
#include <libavutil/timestamp.h>

#include "sfxmp.h"

struct Frame {
    AVFrame *frame;         // a frame the user might want
    double ts;              // rescaled timestamp
};

#define AUDIO_NBITS      10
#define AUDIO_NBSAMPLES  (1<<(AUDIO_NBITS))
#define AUDIO_NBCHANNELS 2

struct sfxmp_ctx {
    /* configurable options */
    char *filename;                         // input filename
    int avselect;                           // select audio or video
    double visible_time;                    // see public header
    double start_time;                      // see public header
    double skip;                            // see public header
    double trim_duration;                   // see public header
    int max_nb_frames;                      // maximum number of frames in the queue
    double dist_time_seek_trigger;          // distance time triggering a seek

    /* misc general fields */
    enum AVMediaType media_type;            // AVMEDIA_TYPE_{VIDEO,AUDIO} according to avselect
    const char *media_type_string;          // "audio" or "video" according to avselect

    /* main vs demuxer/decoder thread negotiation */
    pthread_t dec_thread;                   // decoding thread
    pthread_mutex_t queue_lock;             // for any op related to the queue
    pthread_cond_t queue_reduce;            // notify a reducing queue (MUST be ONLY signaled from main thread)
    pthread_cond_t queue_grow;              // notify a growing queue (MUST be ONLY signaled from decoding thread)

    /* fields that MUST be protected by queue_lock */
    int queue_terminated;                   // 0 if queue is not going to be filled anymore (because thread is dead), 1 otherwise
    struct Frame *frames;                   // queue of the decoded (and filtered) frames
    int nb_frames;                          // total number of frames in the queue
    double request_seek;                    // field used by the main thread to request a seek to the decoding thread
    int can_seek_again;                     // field used to avoid seeking again until the requested time is reached

    /* fields specific to main thread */
    struct Frame non_visible;               // frame to display for the "non visible" time zone
    struct sfxmp_frame rframe;              // user returned frame container
    double last_pushed_frame_ts;            // ts value of the latest pushed frame (it acts as a UID)

    /* fields specific to decoding thread */
    AVFrame *decoded_frame;                 // decoded frame
    AVFrame *filtered_frame;                // filtered version of decoded_frame
    AVFrame *audio_texture_frame;           // wave/fft texture in case of audio
    AVFormatContext *fmt_ctx;               // demuxing context
    AVCodecContext  *dec_ctx;               // decoder context
    AVStream *stream;                       // selected stream
    int stream_idx;                         // selected stream index
    AVFilterGraph *filter_graph;            // libavfilter graph
    char *filter_graph_str;                 // libavfilter graph string
    AVFilterContext *buffersink_ctx;        // sink of the graph (from where we pull)
    AVFilterContext *buffersrc_ctx;         // source of the graph (where we push)
    float *window_func_lut;                 // audio window function lookup table
    RDFTContext *rdft;                      // real discrete fourier transform context
    FFTSample *rdft_data[AUDIO_NBCHANNELS]; // real discrete fourier transform data for each channel
};

#define ENABLE_DBG 0

#define DBG_SFXMP(mod, ...) do { printf("[sfxmp:"mod"] " __VA_ARGS__); fflush(stdout); } while (0)
#if ENABLE_DBG
# define DBG(mod, ...) DBG_SFXMP(mod, __VA_ARGS__)
#else
# define DBG(mod, ...) do { if (0) DBG_SFXMP(mod, __VA_ARGS__); } while (0)
#endif

/**
 * Allocate a small frame to be displayed before visible_time
 */
static AVFrame *get_invisible_frame(enum AVMediaType media_type)
{
    AVFrame *frame = av_frame_alloc();
    if (!frame)
        return NULL;

    frame->format = AV_PIX_FMT_RGB32;

    if (media_type == AVMEDIA_TYPE_VIDEO) {
        frame->width  = 2;
        frame->height = 2;
    } else {
        frame->width  = AUDIO_NBSAMPLES/2;      // samples are float (32 bits), pix fmt is rgb32 (32 bits as well)
        /* height:
         *   AUDIO_NBCHANNELS (waves lines)
         * + AUDIO_NBCHANNELS (fft lines of width AUDIO_NBCHANNELS/2, or 1<<(AUDIO_NBITS-1))
         * + AUDIO_NBITS-1 AUDIO_NBCHANNELS (fft lines downscaled) */
        frame->height = (1 + AUDIO_NBITS) * AUDIO_NBCHANNELS;
    }

    if (av_frame_get_buffer(frame, 16) < 0) {
        av_frame_free(&frame);
        return NULL;
    }

#define SET_COLOR(x, y, color) *(uint32_t *)&frame->data[0][y*frame->linesize[0] + x*4] = color

    if (media_type == AVMEDIA_TYPE_VIDEO) {
#if ENABLE_DBG
    /* In debug more, we make them colored for visual debug */
    SET_COLOR(0, 0, 0xff0000ff);
    SET_COLOR(1, 1, 0x00ff00ff);
    SET_COLOR(0, 1, 0x0000ffff);
    SET_COLOR(1, 0, 0xffffffff);
#else
    /* ...but in production we want it to be black */
    SET_COLOR(0, 0, 0x00000000);
    SET_COLOR(1, 1, 0x00000000);
    SET_COLOR(0, 1, 0x00000000);
    SET_COLOR(1, 0, 0x00000000);
#endif
    } else {
        memset(frame->data[0], 0, frame->height * frame->linesize[0]);
    }
    return frame;
}

static void free_context(struct sfxmp_ctx *s)
{
    int i;

    if (!s)
        return;

    if (s->frames) {
        for (i = 0; i < s->max_nb_frames; i++)
            av_frame_free(&s->frames[i].frame);
        av_freep(&s->frames);
    }

    av_frame_free(&s->non_visible.frame);
    av_freep(&s->filename);
    av_freep(&s->filter_graph_str);
    av_freep(&s);
}

static char *get_filter_graph_str(enum AVMediaType media_type, const char *filters)
{
    char buf[512];

    if (media_type == AVMEDIA_TYPE_VIDEO)
        snprintf(buf, sizeof(buf), "format=rgb32");
    else
        snprintf(buf, sizeof(buf), "aformat=sample_fmts=fltp:channel_layouts=stereo, asetnsamples=%d", AUDIO_NBSAMPLES);

    return filters ? av_asprintf("%s,%s", filters, buf)
                   : av_strdup(buf);
}

struct sfxmp_ctx *sfxmp_create(const char *filename,
                               int avselect,
                               double visible_time,
                               double start_time,
                               double skip,
                               double trim_duration,
                               double dist_time_seek_trigger,
                               double max_nb_frames,
                               const char *filters)
{
    int i;
    struct sfxmp_ctx *s;

    av_register_all();
    avfilter_register_all();
    av_log_set_level(AV_LOG_ERROR);

    s = av_mallocz(sizeof(*s));
    if (!s)
        return NULL;

    s->filename               = av_strdup(filename);
    s->avselect               = avselect;
    s->visible_time           = visible_time;
    s->start_time             = start_time;
    s->skip                   = skip;
    s->trim_duration          = trim_duration;
    s->dist_time_seek_trigger = dist_time_seek_trigger < 0 ? 3 : dist_time_seek_trigger;
    s->max_nb_frames          = max_nb_frames < 0 ? 5 : max_nb_frames;

    switch (s->avselect) {
    case SFXMP_SELECT_VIDEO: s->media_type = AVMEDIA_TYPE_VIDEO; break;
    case SFXMP_SELECT_AUDIO: s->media_type = AVMEDIA_TYPE_AUDIO; break;
    default:
        fprintf(stderr, "unknown avselect value %d\n", s->avselect);
        goto fail;
    }

    s->media_type_string = av_get_media_type_string(s->media_type);
    av_assert0(s->media_type_string);

    s->filter_graph_str = get_filter_graph_str(s->media_type, filters);

    if (s->max_nb_frames < 2) {
        fprintf(stderr, "max_nb_frames < 2 is not supported\n");
        goto fail;
    }

    if (!s->filename || !s->filter_graph_str)
        goto fail;

    s->frames = av_malloc_array(s->max_nb_frames, sizeof(*s->frames));
    if (!s->frames)
        goto fail;
    for (i = 0; i < s->max_nb_frames; i++) {
        s->frames[i].frame = av_frame_alloc();
        if (!s->frames[i].frame)
            goto fail;
    }

    s->last_pushed_frame_ts = DBL_MIN;

    s->non_visible.ts    = -1;
    s->non_visible.frame = get_invisible_frame(s->media_type);
    if (!s->non_visible.frame)
        goto fail;

    s->queue_terminated = 1;

    pthread_mutex_init(&s->queue_lock, NULL);
    pthread_cond_init(&s->queue_reduce, NULL);
    pthread_cond_init(&s->queue_grow, NULL);

    DBG("init", "filename:%s avselect:%d visible_time:%f start_time:%f skip:%f trim_duration:%f (%p)\n",
        filename, avselect, visible_time, start_time, skip, trim_duration, s);

    return s;

fail:
    free_context(s);
    return NULL;
}

void sfxmp_free(struct sfxmp_ctx **ss)
{
    struct sfxmp_ctx *s = *ss;

    DBG("free", "calling sfxmp_free() (%p)\n", s);

    if (!s)
        return;

    if (!s->queue_terminated) {
        DBG("free", "queue is not terminated yet\n");
        pthread_mutex_lock(&s->queue_lock);
        s->queue_terminated = 1; // notify decoding thread must die
        pthread_mutex_unlock(&s->queue_lock);
        pthread_cond_signal(&s->queue_reduce); // wake up decoding thread

        pthread_join(s->dec_thread, NULL);
    }
    pthread_cond_destroy(&s->queue_reduce);
    pthread_cond_destroy(&s->queue_grow);
    pthread_mutex_destroy(&s->queue_lock);
    free_context(s);
}

static int decode_packet(struct sfxmp_ctx *s, AVPacket *pkt,
                         AVFrame *frame, int *got_frame)
{
    int decoded = pkt->size;

    *got_frame = 0;
    if (pkt->stream_index == s->stream_idx) {
        int ret;

        if (s->media_type == AVMEDIA_TYPE_VIDEO)
            ret = avcodec_decode_video2(s->dec_ctx, frame, got_frame, pkt);
        else
            ret = avcodec_decode_audio4(s->dec_ctx, frame, got_frame, pkt);

        if (ret < 0) {
            fprintf(stderr, "Error decoding %s frame\n", s->media_type_string);
            return ret;
        }
        decoded = FFMIN(ret, pkt->size);
    }
    return decoded;
}

static int open_ifile(struct sfxmp_ctx *s, const char *infile)
{
    int ret = 0;
    AVCodec *dec;
    AVDictionary *opts = NULL;
    AVInputFormat *ifmt = NULL;

    ret = avformat_open_input(&s->fmt_ctx, infile, ifmt, &opts);
    if (ret < 0) {
        fprintf(stderr, "Unable to open input file '%s'\n", infile);
        return ret;
    }

    ret = avformat_find_stream_info(s->fmt_ctx, NULL);
    if (ret < 0) {
        fprintf(stderr, "Unable to find input stream information\n");
        return ret;
    }

    ret = av_find_best_stream(s->fmt_ctx, s->media_type, -1, -1, &dec, 0);
    if (ret < 0) {
        fprintf(stderr, "Unable to find a %s stream in the input file\n", s->media_type_string);
        return ret;
    }
    s->stream_idx = ret;
    s->stream = s->fmt_ctx->streams[s->stream_idx];
    s->dec_ctx = s->stream->codec;
    av_opt_set_int(s->dec_ctx, "refcounted_frames", 1, 0);

    ret = avcodec_open2(s->dec_ctx, dec, NULL);
    if (ret < 0) {
        fprintf(stderr, "Unable to open input %s decoder\n", s->media_type_string);
        return ret;
    }

    /* If trim_duration is not specified, we try to probe from the stream or
     * format (presentation) duration */
    if (s->trim_duration < 0) {
        int64_t duration = s->fmt_ctx->duration;
        double scale = av_q2d(AV_TIME_BASE_Q);

        if (duration == AV_NOPTS_VALUE) {
            duration = s->stream->duration;
            if (s->stream->time_base.den)
                scale = av_q2d(s->stream->time_base);
            else
                duration = AV_NOPTS_VALUE;
        }
        if (duration == AV_NOPTS_VALUE) {
            fprintf(stderr, "trim_duration is not set and can't estimate the duration, aborting\n");
            return AVERROR_INVALIDDATA;
        }
        s->trim_duration = duration * scale;
        DBG("decoder", "set trim duration to %f\n", s->trim_duration);
    }

    return 0;
}

/**
 * Map the timeline time to the media time
 */
static double get_media_time(const struct sfxmp_ctx *s, double t)
{
    return s->skip + av_clipd(t - s->start_time, 0, s->trim_duration);
}

/**
 * Request a seek to the demuxer
 */
static int seek_to(struct sfxmp_ctx *s, double t)
{
    const double vt = get_media_time(s, t);
    const int64_t ts = vt * AV_TIME_BASE;

    if (ENABLE_DBG)
        DBG("decoder", "Seek in media at %f (t=%f)\n", vt, t);
    else
        printf("Seek in media at %f (t=%f)\n", vt, t);
    return avformat_seek_file(s->fmt_ctx, -1, INT64_MIN, ts, ts, 0);
}

/**
 * Setup the libavfilter filtergraph for user filter but also to have a way to
 * request a pixel format we want, and let libavfilter insert the necessary
 * scaling filter (typically, an automatic conversion from yuv420p to rgb32).
 */
static int setup_filtergraph(struct sfxmp_ctx *s)
{
    int ret = 0;
    char args[512];
    const AVRational time_base = s->stream->time_base;
    const AVRational framerate = av_guess_frame_rate(s->fmt_ctx, s->stream, NULL);
    AVFilter *buffersrc  = avfilter_get_by_name(s->media_type == AVMEDIA_TYPE_VIDEO ? "buffer" : "abuffer");
    AVFilter *buffersink = avfilter_get_by_name(s->media_type == AVMEDIA_TYPE_VIDEO ? "buffersink" : "abuffersink");
    AVFilterInOut *outputs = avfilter_inout_alloc();
    AVFilterInOut *inputs  = avfilter_inout_alloc();

    avfilter_graph_free(&s->filter_graph);
    s->filter_graph = avfilter_graph_alloc();

    if (!inputs || !outputs || !s->filter_graph) {
        ret = AVERROR(ENOMEM);
        goto end;
    }

    inputs->name  = av_strdup("out");
    outputs->name = av_strdup("in");
    if (!inputs->name || !outputs->name) {
        ret = AVERROR(ENOMEM);
        goto end;
    }

    /* create buffer filter source (where we push the frame) */
    if (s->media_type == AVMEDIA_TYPE_VIDEO) {
        snprintf(args, sizeof(args),
                 "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d:sws_param=flags=bicubic",
                 s->dec_ctx->width, s->dec_ctx->height, s->dec_ctx->pix_fmt,
                 time_base.num, time_base.den,
                 s->dec_ctx->sample_aspect_ratio.num, s->dec_ctx->sample_aspect_ratio.den);
    } else {
        snprintf(args, sizeof(args), "time_base=%d/%d:sample_rate=%d:sample_fmt=%s",
                 1, s->dec_ctx->sample_rate, s->dec_ctx->sample_rate,
                 av_get_sample_fmt_name(s->dec_ctx->sample_fmt));
        if (s->dec_ctx->channel_layout)
            av_strlcatf(args, sizeof(args), ":channel_layout=0x%"PRIx64, s->dec_ctx->channel_layout);
        else
            av_strlcatf(args, sizeof(args), ":channels=%d", s->dec_ctx->channels);
    }

    if (framerate.num && framerate.den)
        av_strlcatf(args, sizeof(args), ":frame_rate=%d/%d", framerate.num, framerate.den);

    ret = avfilter_graph_create_filter(&s->buffersrc_ctx, buffersrc,
                                       outputs->name, args, NULL, s->filter_graph);
    if (ret < 0) {
        fprintf(stderr, "Unable to create buffer filter source\n");
        goto end;
    }

    /* create buffer filter sink (where we pull the frame) */
    ret = avfilter_graph_create_filter(&s->buffersink_ctx, buffersink,
                                       inputs->name, NULL, NULL, s->filter_graph);
    if (ret < 0) {
        fprintf(stderr, "Unable to create buffer filter sink\n");
        goto end;
    }

    /* create our filter graph */
    inputs->filter_ctx  = s->buffersink_ctx;
    outputs->filter_ctx = s->buffersrc_ctx;
    ret = avfilter_graph_parse_ptr(s->filter_graph, s->filter_graph_str,
                                   &inputs, &outputs, NULL);
    if (ret < 0)
        goto end;

    ret = avfilter_graph_config(s->filter_graph, NULL);
    if (ret < 0)
        goto end;

end:
    avfilter_inout_free(&inputs);
    avfilter_inout_free(&outputs);
    return ret;
}

/**
 * Convert an audio frame (PCM data) to a textured video frame with waves and
 * FFT lines
 */
static void audio_frame_to_sound_texture(struct sfxmp_ctx *s, AVFrame *dst_video,
                                         const AVFrame *audio_src)
{
    int i, j, ch;
    const int nb_samples = audio_src->nb_samples;
    const int width = nb_samples / 2;
    const float scale = 1.f / sqrt(AUDIO_NBSAMPLES/2 + 1);

    memset(dst_video->data[0], 0, dst_video->height * dst_video->linesize[0]);

    /* Copy waves */
    for (ch = 0; ch < AUDIO_NBCHANNELS; ch++) {
        const int lz = dst_video->linesize[0];
        float *samples_dst = (float *)(dst_video->data[0] + ch * lz);
        const float *samples_src = (const float *)audio_src->extended_data[ch];

        for (i = 0; i < width; i++)
            samples_dst[i] = (samples_src[width/2 + i] + 1.f) / 2.f;
    }

    /* Fourier transform */
    for (ch = 0; ch < AUDIO_NBCHANNELS; ch++) {
        const int lz = dst_video->linesize[0];
        float *fft_dst = (float *)(dst_video->data[0] + (AUDIO_NBCHANNELS + ch) * lz);
        const float *samples_src = (const float *)audio_src->extended_data[ch];
        float *bins = s->rdft_data[ch];

        /* Apply window function to input samples */
        for (i = 0; i < nb_samples; i++)
            bins[i] = samples_src[i] * s->window_func_lut[i];

        /* Run transform.
         *
         * After av_rdft_calc(), the bins is an array of successive real and
         * imaginary floats, except for the first two bins which are
         * respectively the real corresponding to the lower frequency and the
         * real for the higher frequency.
         *
         * The imaginary parts for these two frequencies are always 0 so they
         * are assumed as such. This trick allowed an in-place processing for
         * the N samples into N+1 complex.
         */
        av_rdft_calc(s->rdft, bins);

        /* Get magnitude of frequency bins and copy result into texture
         *
         * Note: since we only have space for N samples in the texture, we skip
         * the first complex (lower frequency one).
         */
#define MAGNITUDE(re, im) sqrtf(((re)*(re) + (im)*(im)) * scale)
        for (i = 1; i < width - 1; i++)
            fft_dst[i] = MAGNITUDE(bins[2*i], bins[2*i + 1]);

        /* Last complex (higher frequency one) is one of the the special case
         * mentioned above */
        fft_dst[width - 1] = MAGNITUDE(bins[1], 0);
    }

    /* Downscaled versions of the FFT */
    for (i = 0; i < AUDIO_NBITS-1; i++) {
        for (ch = 0; ch < AUDIO_NBCHANNELS; ch++) {
            const int lz = dst_video->linesize[0];
            const int source_line = (i + 1)*AUDIO_NBCHANNELS + ch;
            float *fft_src = (float *)(dst_video->data[0] +  source_line                     * lz);
            float *fft_dst = (float *)(dst_video->data[0] + (source_line + AUDIO_NBCHANNELS) * lz);

            const int source_step = 1 << i;
            const int nb_identical_values = source_step << 1;
            const int nb_dest_pixels = width / nb_identical_values;

            DBG("audio2tex", "line %2d->%2d: %3d different pixels (copied %3dx) as destination, step source: %d\n",
                source_line, source_line + AUDIO_NBCHANNELS, nb_dest_pixels, nb_identical_values, source_step);

            for (j = 0; j < nb_dest_pixels; j++) {
                int x;
                const float avg = (fft_src[ j*2      * source_step] +
                                   fft_src[(j*2 + 1) * source_step]) / 2.f;

                for (x = 0; x < nb_identical_values; x++)
                    fft_dst[j*nb_identical_values + x] = avg;
            }
        }
{'fraction_non_alphanumeric': 0.11786733238231098, 'fraction_numerical': 0.010953739555736702, 'mean_word_length': 2.3105920053972, 'pattern_counts': {'":': 6, '<': 95, '<?xml version=': 0, '>': 361, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 10, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2412842', 'n_tokens_mistral': 13268, 'n_tokens_neox': 12758, 'n_words': 3240}