<commit_before>/**
 * Copyright 2013-2014, <NAME> <<EMAIL>>
 *                2014, <NAME> <<EMAIL>>
 *
 * This file is part of Musly, a program for high performance music
 * similarity computation: http://www.musly.org/.
 *
 * This Source Code Form is subject to the terms of the Mozilla
 * Public License v. 2.0. If a copy of the MPL was not distributed
 * with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

#define __STDC_CONSTANT_MACROS
#define __STDC_FORMAT_MACROS

#include <inttypes.h>
#include <stdint.h>
#include <vector>
#include <algorithm>
extern "C" {
    #include <libavcodec/avcodec.h>
    #include <libavformat/avformat.h>
    #include <libavutil/channel_layout.h>
}

#include "minilog.h"
#include "resampler.h"
#include "libav_0_8.h"

namespace musly {
namespace decoders {

MUSLY_DECODER_REGIMPL(libav_0_8, 0);

libav_0_8::libav_0_8()
{
    av_register_all();
    avcodec_register_all();
}

int
libav_0_8::samples_tofloat(
        void* const out,
        const void* const in,
        const int out_stride,
        const int in_stride,
        const AVSampleFormat in_fmt,
        int len)
{
    const int is = in_stride;
    const int os = out_stride;
    const uint8_t* pi = (uint8_t*)in;
    uint8_t* po = (uint8_t*)out;
    uint8_t* end = po + os*len;

#define CONVFLOAT(ifmt, expr)\
if (in_fmt == ifmt) {\
    do{\
        *(float*)po = expr; pi += is; po += os;\
    } while(po < end);\
}
    CONVFLOAT(AV_SAMPLE_FMT_U8, (*(const uint8_t*)pi - 0x80)*(1.0 / (1<<7)))
    else CONVFLOAT(AV_SAMPLE_FMT_S16, *(const int16_t*)pi*(1.0 / (1<<15)))
    else CONVFLOAT(AV_SAMPLE_FMT_S32, *(const int32_t*)pi*(1.0 / (1U<<31)))
    else CONVFLOAT(AV_SAMPLE_FMT_FLT, *(const float*)pi)
    else CONVFLOAT(AV_SAMPLE_FMT_DBL, *(const double*)pi)
    else return -1;

    return 0;
}

std::vector<float>
libav_0_8::decodeto_22050hz_mono_float(
        const std::string& file,
        int max_seconds)
{
    MINILOG(logTRACE) << "Decoding: " << file << " started.";

    int target_rate = 22050;

    // silence libav
    av_log_set_level(0);


    // guess input format
    AVFormatContext* fmtx = NULL;
    int avret = avformat_open_input(&fmtx, file.c_str(), NULL, NULL);
    if (avret < 0) {
        MINILOG(logERROR) << "Could not open file, or detect file format";
        return std::vector<float>(0);
    }

    // retrieve stream information
    if (avformat_find_stream_info(fmtx, NULL) < 0) {
        MINILOG(logERROR) << "Could not find stream info";

        avformat_close_input(&fmtx);
        return std::vector<float>(0);
    }

    // if there are multiple audio streams, find the best one..
    int audio_stream_idx =
            av_find_best_stream(fmtx, AVMEDIA_TYPE_AUDIO, -1, -1, NULL, 0);
    if (audio_stream_idx < 0) {
        MINILOG(logERROR) << "Could not find audio stream in input file";

        avformat_close_input(&fmtx);
        return std::vector<float>(0);
    }
    AVStream *st = fmtx->streams[audio_stream_idx];

    // find a decoder for the stream
    AVCodecContext *decx = st->codec;
    AVCodec *dec = avcodec_find_decoder(decx->codec_id);
    if (!dec) {
        MINILOG(logERROR) << "Could not find codec.";

        avformat_close_input(&fmtx);
        return std::vector<float>(0);
    }

    // open the decoder
    // (kindly ask for stereo downmix and floats, but not all decoders care)
    decx->request_channel_layout = AV_CH_LAYOUT_STEREO_DOWNMIX;
    decx->request_sample_fmt = AV_SAMPLE_FMT_FLT;
    avret = avcodec_open2(decx, dec, NULL);
    if (avret < 0) {
        MINILOG(logERROR) << "Could not open codec.";

        avformat_close_input(&fmtx);
        return std::vector<float>(0);
    }

    // Currently only mono and stereo files are supported.
    if ((decx->channels != 1) && (decx->channels != 2)) {
        MINILOG(logWARNING) << "Unsupported number of channels: "
                << decx->channels;

        avformat_close_input(&fmtx);
        return std::vector<float>(0);
    }

    // allocate a frame
    AVFrame* frame = avcodec_alloc_frame();
    if (!frame) {
        MINILOG(logWARNING) << "Could not allocate frame";

        avformat_close_input(&fmtx);
        return std::vector<float>(0);
    }

    // allocate and initialize a packet
    AVPacket pkt;
    av_init_packet(&pkt);
    pkt.data = NULL;
    pkt.size = 0;
    int got_frame = 0;

    // configuration
    const int input_stride = av_get_bytes_per_sample(decx->sample_fmt);
    const int num_planes = av_sample_fmt_is_planar(decx->sample_fmt) ? decx->channels : 1;
    const int output_stride = sizeof(float) * num_planes;

    // read packets
    float* buffer = NULL;
    int buffersize = 0;
    std::vector<float> decoded_pcm;
    int subsequent_errors = 0;
    int subsequent_errors_max = 20;
    while ((av_read_frame(fmtx, &pkt) >= 0) &&
            ((max_seconds == 0) ||
                    ((int)decoded_pcm.size() < (max_seconds*decx->sample_rate))))
   {
        // use only audio frames
        if (pkt.stream_index == audio_stream_idx) {
            uint8_t* data = pkt.data;
            int size = pkt.size;<commit_msg>Fix memory leak in libav decoder<commit_after>/**
 * Copyright 2013-2014, <NAME> <<EMAIL>>
 *                2014, <NAME> <<EMAIL>>
 *
 * This file is part of Musly, a program for high performance music
 * similarity computation: http://www.musly.org/.
 *
 * This Source Code Form is subject to the terms of the Mozilla
 * Public License v. 2.0. If a copy of the MPL was not distributed
 * with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

#define __STDC_CONSTANT_MACROS
#define __STDC_FORMAT_MACROS

#include <inttypes.h>
#include <stdint.h>
#include <vector>
#include <algorithm>
extern "C" {
    #include <libavcodec/avcodec.h>
    #include <libavformat/avformat.h>
    #include <libavutil/channel_layout.h>
}

#include "minilog.h"
#include "resampler.h"
#include "libav_0_8.h"

namespace musly {
namespace decoders {

MUSLY_DECODER_REGIMPL(libav_0_8, 0);

libav_0_8::libav_0_8()
{
    av_register_all();
    avcodec_register_all();
}

int
libav_0_8::samples_tofloat(
        void* const out,
        const void* const in,
        const int out_stride,
        const int in_stride,
        const AVSampleFormat in_fmt,
        int len)
{
    const int is = in_stride;
    const int os = out_stride;
    const uint8_t* pi = (uint8_t*)in;
    uint8_t* po = (uint8_t*)out;
    uint8_t* end = po + os*len;

#define CONVFLOAT(ifmt, expr)\
if (in_fmt == ifmt) {\
    do{\
        *(float*)po = expr; pi += is; po += os;\
    } while(po < end);\
}
    CONVFLOAT(AV_SAMPLE_FMT_U8, (*(const uint8_t*)pi - 0x80)*(1.0 / (1<<7)))
    else CONVFLOAT(AV_SAMPLE_FMT_S16, *(const int16_t*)pi*(1.0 / (1<<15)))
    else CONVFLOAT(AV_SAMPLE_FMT_S32, *(const int32_t*)pi*(1.0 / (1U<<31)))
    else CONVFLOAT(AV_SAMPLE_FMT_FLT, *(const float*)pi)
    else CONVFLOAT(AV_SAMPLE_FMT_DBL, *(const double*)pi)
    else return -1;

    return 0;
}

std::vector<float>
libav_0_8::decodeto_22050hz_mono_float(
        const std::string& file,
        int max_seconds)
{
    MINILOG(logTRACE) << "Decoding: " << file << " started.";

    int target_rate = 22050;

    // silence libav
    av_log_set_level(0);


    // guess input format
    AVFormatContext* fmtx = NULL;
    int avret = avformat_open_input(&fmtx, file.c_str(), NULL, NULL);
    if (avret < 0) {
        MINILOG(logERROR) << "Could not open file, or detect file format";
        return std::vector<float>(0);
    }

    // retrieve stream information
    if (avformat_find_stream_info(fmtx, NULL) < 0) {
        MINILOG(logERROR) << "Could not find stream info";

        avformat_close_input(&fmtx);
        return std::vector<float>(0);
    }

    // if there are multiple audio streams, find the best one..
    int audio_stream_idx =
            av_find_best_stream(fmtx, AVMEDIA_TYPE_AUDIO, -1, -1, NULL, 0);
    if (audio_stream_idx < 0) {
        MINILOG(logERROR) << "Could not find audio stream in input file";

        avformat_close_input(&fmtx);
        return std::vector<float>(0);
    }
    AVStream *st = fmtx->streams[audio_stream_idx];

    // find a decoder for the stream
    AVCodecContext *decx = st->codec;
    AVCodec *dec = avcodec_find_decoder(decx->codec_id);
    if (!dec) {
        MINILOG(logERROR) << "Could not find codec.";

        avformat_close_input(&fmtx);
        return std::vector<float>(0);
    }

    // open the decoder
    // (kindly ask for stereo downmix and floats, but not all decoders care)
    decx->request_channel_layout = AV_CH_LAYOUT_STEREO_DOWNMIX;
    decx->request_sample_fmt = AV_SAMPLE_FMT_FLT;
    avret = avcodec_open2(decx, dec, NULL);
    if (avret < 0) {
        MINILOG(logERROR) << "Could not open codec.";

        avformat_close_input(&fmtx);
        return std::vector<float>(0);
    }

    // Currently only mono and stereo files are supported.
    if ((decx->channels != 1) && (decx->channels != 2)) {
        MINILOG(logWARNING) << "Unsupported number of channels: "
                << decx->channels;

        avformat_close_input(&fmtx);
        return std::vector<float>(0);
    }

    // allocate a frame
    AVFrame* frame = avcodec_alloc_frame();
    if (!frame) {
        MINILOG(logWARNING) << "Could not allocate frame";

        avformat_close_input(&fmtx);
        return std::vector<float>(0);
    }

    // allocate and initialize a packet
    AVPacket pkt;
    av_init_packet(&pkt);
    pkt.data = NULL;
    pkt.size = 0;
    int got_frame = 0;

    // configuration
    const int input_stride = av_get_bytes_per_sample(decx->sample_fmt);
    const int num_planes = av_sample_fmt_is_planar(decx->sample_fmt) ? decx->channels : 1;
    const int output_stride = sizeof(float) * num_planes;

    // read packets
    float* buffer = NULL;
    int buffersize = 0;
    std::vector<float> decoded_pcm;
    int subsequent_errors = 0;
    int subsequent_errors_max = 20;
    while (((max_seconds == 0) || ((int)decoded_pcm.size() < max_seconds*decx->sample_rate))
            && (av_read_frame(fmtx, &pkt) >= 0))
    {
        // use only audio frames
        if (pkt.stream_index == audio_stream_idx) {
            uint8_t* data = pkt.data;
            int size = pkt.size;
{'fraction_non_alphanumeric': 0.12866640594446618, 'fraction_numerical': 0.019749706687524442, 'mean_word_length': 2.8849221420432967, 'pattern_counts': {'":': 0, '<': 115, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '795142', 'n_tokens_mistral': 3710, 'n_tokens_neox': 3574, 'n_words': 866}