<commit_before>/*****************************************************************************
 * video.c: transcoding stream output module (video)
 *****************************************************************************
 * Copyright (C) 2003-2009 VLC authors and VideoLAN
 * $Id$
 *
 * Authors: <NAME> <<EMAIL>>
 *          <NAME> <<EMAIL>>
 *          <NAME> <jpsaman #_at_# m2x dot nl>
 *          <NAME> <dionoea at videolan dot org>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

/*****************************************************************************
 * Preamble
 *****************************************************************************/

#include "transcode.h"

#include <math.h>
#include <vlc_meta.h>
#include <vlc_spu.h>
#include <vlc_modules.h>

#define ENC_FRAMERATE (25 * 1000)
#define ENC_FRAMERATE_BASE 1000

struct decoder_owner_sys_t
{
    sout_stream_sys_t *p_sys;
};

static int video_update_format_decoder( decoder_t *p_dec )
{
    p_dec->fmt_out.video.i_chroma = p_dec->fmt_out.i_codec;
    return 0;
}

static picture_t *video_new_buffer_decoder( decoder_t *p_dec )
{
    return picture_NewFromFormat( &p_dec->fmt_out.video );
}

static picture_t *video_new_buffer_encoder( encoder_t *p_enc )
{
    p_enc->fmt_in.video.i_chroma = p_enc->fmt_in.i_codec;
    return picture_NewFromFormat( &p_enc->fmt_in.video );
}

static picture_t *transcode_video_filter_buffer_new( filter_t *p_filter )
{
    p_filter->fmt_out.video.i_chroma = p_filter->fmt_out.i_codec;
    return picture_NewFromFormat( &p_filter->fmt_out.video );
}

static void* EncoderThread( void *obj )
{
    sout_stream_sys_t *p_sys = (sout_stream_sys_t*)obj;
    sout_stream_id_sys_t *id = p_sys->id_video;
    picture_t *p_pic = NULL;
    int canc = vlc_savecancel ();
    block_t *p_block = NULL;

    vlc_mutex_lock( &p_sys->lock_out );

    for( ;; )
    {
        while( !p_sys->b_abort &&
               (p_pic = picture_fifo_Pop( p_sys->pp_pics )) == NULL )
            vlc_cond_wait( &p_sys->cond, &p_sys->lock_out );

        if( p_pic )
        {
            /* release lock while encoding */
            vlc_mutex_unlock( &p_sys->lock_out );
            p_block = id->p_encoder->pf_encode_video( id->p_encoder, p_pic );
            picture_Release( p_pic );
            vlc_mutex_lock( &p_sys->lock_out );

            block_ChainAppend( &p_sys->p_buffers, p_block );
        }

        if( p_sys->b_abort )
            break;
    }

    /*Encode what we have in the buffer on closing*/
    while( (p_pic = picture_fifo_Pop( p_sys->pp_pics )) != NULL )
    {
        p_block = id->p_encoder->pf_encode_video( id->p_encoder, p_pic );
        picture_Release( p_pic );
        block_ChainAppend( &p_sys->p_buffers, p_block );
    }

    /*Now flush encoder*/
    do {
        p_block = id->p_encoder->pf_encode_video(id->p_encoder, NULL );
        block_ChainAppend( &p_sys->p_buffers, p_block );
    } while( p_block );

    vlc_mutex_unlock( &p_sys->lock_out );

    vlc_restorecancel (canc);

    return NULL;
}

int transcode_video_new( sout_stream_t *p_stream, sout_stream_id_sys_t *id )
{
    sout_stream_sys_t *p_sys = p_stream->p_sys;

    /* Open decoder
     * Initialization of decoder structures
     */
    id->p_decoder->fmt_out = id->p_decoder->fmt_in;
    id->p_decoder->fmt_out.i_extra = 0;
    id->p_decoder->fmt_out.p_extra = NULL;
    id->p_decoder->fmt_out.psz_language = NULL;
    id->p_decoder->pf_decode_video = NULL;
    id->p_decoder->pf_get_cc = NULL;
    id->p_decoder->pf_vout_format_update = video_update_format_decoder;
    id->p_decoder->pf_vout_buffer_new = video_new_buffer_decoder;
    id->p_decoder->p_owner = malloc( sizeof(decoder_owner_sys_t) );
    if( !id->p_decoder->p_owner )
        return VLC_EGENERIC;

    id->p_decoder->p_owner->p_sys = p_sys;
    /* id->p_decoder->p_cfg = p_sys->p_video_cfg; */

    id->p_decoder->p_module =
        module_need( id->p_decoder, "decoder", "$codec", false );

    if( !id->p_decoder->p_module )
    {
        msg_Err( p_stream, "cannot find video decoder" );
        free( id->p_decoder->p_owner );
        return VLC_EGENERIC;
    }

    /*
     * Open encoder.
     * Because some info about the decoded input will only be available
     * once the first frame is decoded, we actually only test the availability
     * of the encoder here.
     */

    /* Initialization of encoder format structures */
    es_format_Init( &id->p_encoder->fmt_in, id->p_decoder->fmt_in.i_cat,
                    id->p_decoder->fmt_out.i_codec );
    id->p_encoder->fmt_in.video.i_chroma = id->p_decoder->fmt_out.i_codec;

    /* The dimensions will be set properly later on.
     * Just put sensible values so we can test an encoder is available. */
    id->p_encoder->fmt_in.video.i_width =
        id->p_encoder->fmt_out.video.i_width
          ? id->p_encoder->fmt_out.video.i_width
          : id->p_decoder->fmt_in.video.i_width
            ? id->p_decoder->fmt_in.video.i_width : 16;
    id->p_encoder->fmt_in.video.i_height =
        id->p_encoder->fmt_out.video.i_height
          ? id->p_encoder->fmt_out.video.i_height
          : id->p_decoder->fmt_in.video.i_height
            ? id->p_decoder->fmt_in.video.i_height : 16;
    id->p_encoder->fmt_in.video.i_visible_width =
        id->p_encoder->fmt_out.video.i_visible_width
          ? id->p_encoder->fmt_out.video.i_visible_width
          : id->p_decoder->fmt_in.video.i_visible_width
            ? id->p_decoder->fmt_in.video.i_visible_width : id->p_encoder->fmt_in.video.i_width;
    id->p_encoder->fmt_in.video.i_visible_height =
        id->p_encoder->fmt_out.video.i_visible_height
          ? id->p_encoder->fmt_out.video.i_visible_height
          : id->p_decoder->fmt_in.video.i_visible_height
            ? id->p_decoder->fmt_in.video.i_visible_height : id->p_encoder->fmt_in.video.i_height;

    id->p_encoder->i_threads = p_sys->i_threads;
    id->p_encoder->p_cfg = p_sys->p_video_cfg;

    id->p_encoder->p_module =
        module_need( id->p_encoder, "encoder", p_sys->psz_venc, true );
    if( !id->p_encoder->p_module )
    {
        msg_Err( p_stream, "cannot find video encoder (module:%s fourcc:%4.4s). Take a look few lines earlier to see possible reason.",
                 p_sys->psz_venc ? p_sys->psz_venc : "any",
                 (char *)&p_sys->i_vcodec );
        module_unneed( id->p_decoder, id->p_decoder->p_module );
        id->p_decoder->p_module = 0;
        free( id->p_decoder->p_owner );
        return VLC_EGENERIC;
    }

    /* Close the encoder.
     * We'll open it only when we have the first frame. */
    module_unneed( id->p_encoder, id->p_encoder->p_module );
    if( id->p_encoder->fmt_out.p_extra )
    {
        free( id->p_encoder->fmt_out.p_extra );
        id->p_encoder->fmt_out.p_extra = NULL;
        id->p_encoder->fmt_out.i_extra = 0;
    }
    id->p_encoder->p_module = NULL;

    if( p_sys->i_threads <= 0 )
        return VLC_SUCCESS;

    int i_priority = p_sys->b_high_priority ? VLC_THREAD_PRIORITY_OUTPUT :
                       VLC_THREAD_PRIORITY_VIDEO;
    p_sys->id_video = id;
    p_sys->pp_pics = picture_fifo_New();
    if( p_sys->pp_pics == NULL )
    {
        msg_Err( p_stream, "cannot create picture fifo" );
        module_unneed( id->p_decoder, id->p_decoder->p_module );
        id->p_decoder->p_module = NULL;
        free( id->p_decoder->p_owner );
        return VLC_ENOMEM;
    }
    vlc_mutex_init( &p_sys->lock_out );
    vlc_cond_init( &p_sys->cond );
    p_sys->p_buffers = NULL;
    p_sys->b_abort = false;
    if( vlc_clone( &p_sys->thread, EncoderThread, p_sys, i_priority ) )
    {
        msg_Err( p_stream, "cannot spawn encoder thread" );
        vlc_mutex_destroy( &p_sys->lock_out );
        vlc_cond_destroy( &p_sys->cond );
        picture_fifo_Delete( p_sys->pp_pics );
        module_unneed( id->p_decoder, id->p_decoder->p_module );
        id->p_decoder->p_module = NULL;
        free( id->p_decoder->p_owner );
        return VLC_EGENERIC;
    }
    return VLC_SUCCESS;
}

static void transcode_video_filter_init( sout_stream_t *p_stream,
                                         sout_stream_id_sys_t *id )
{
    filter_owner_t owner = {
        .sys = p_stream->p_sys,
        .video = {
            .buffer_new = transcode_video_filter_buffer_new,
        },
    };
    es_format_t *p_fmt_out = &id->p_decoder->fmt_out;

    id->p_encoder->fmt_in.video.i_chroma = id->p_encoder->fmt_in.i_codec;
    id->p_f_chain = filter_chain_NewVideo( p_stream, false, &owner );
    filter_chain_Reset( id->p_f_chain, p_fmt_out, p_fmt_out );

    /* Deinterlace */
    if( p_stream->p_sys->b_deinterlace )
    {
        filter_chain_AppendFilter( id->p_f_chain,
                                   p_stream->p_sys->psz_deinterlace,
                                   p_stream->p_sys->p_deinterlace_cfg,
                                   &id->p_decoder->fmt_out,
                                   &id->p_decoder->fmt_out );

        p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
    }
    if( p_stream->p_sys->b_master_sync )
    {
        filter_chain_AppendFilter( id->p_f_chain,
                                   "fps",
                                   NULL,
                                   p_fmt_out,
                                   &id->p_encoder->fmt_in );

        p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
    }

    /* Check that we have visible_width/height*/
    if( !p_fmt_out->video.i_visible_height )
        p_fmt_out->video.i_visible_height = p_fmt_out->video.i_height;
    if( !p_fmt_out->video.i_visible_width )
        p_fmt_out->video.i_visible_width = p_fmt_out->video.i_width;

    if( p_stream->p_sys->psz_vf2 )
    {
        id->p_uf_chain = filter_chain_NewVideo( p_stream, true, &owner );
        filter_chain_Reset( id->p_uf_chain, p_fmt_out,
                            &id->p_encoder->fmt_in );
        if( p_fmt_out->video.i_chroma != id->p_encoder->fmt_in.video.i_chroma )
        {
            filter_chain_AppendFilter( id->p_uf_chain,
                                   NULL, NULL,
                                   p_fmt_out,
                                   &id->p_encoder->fmt_in );
        }
        filter_chain_AppendFromString( id->p_uf_chain, p_stream->p_sys->psz_vf2 );
        p_fmt_out = filter_chain_GetFmtOut( id->p_uf_chain );
        es_format_Copy( &id->p_encoder->fmt_in, p_fmt_out );
        id->p_encoder->fmt_out.video.i_width =
            id->p_encoder->fmt_in.video.i_width;
        id->p_encoder->fmt_out.video.i_height =
            id->p_encoder->fmt_in.video.i_height;
        id->p_encoder->fmt_out.video.i_sar_num =
            id->p_encoder->fmt_in.video.i_sar_num;
        id->p_encoder->fmt_out.video.i_sar_den =
            id->p_encoder->fmt_in.video.i_sar_den;
    }

    /* Keep colorspace etc info along */
    id->p_encoder->fmt_in.video.space     = id->p_decoder->fmt_out.video.space;
    id->p_encoder->fmt_in.video.transfer  = id->p_decoder->fmt_out.video.transfer;
    id->p_encoder->fmt_in.video.primaries = id->p_decoder->fmt_out.video.primaries;
    id->p_encoder->fmt_in.video.b_color_range_full = id->p_decoder->fmt_out.video.b_color_range_full;
}

/* Take care of the scaling and chroma conversions. */
static void conversion_video_filter_append( sout_stream_id_sys_t *id )
{
    const es_format_t *p_fmt_out = &id->p_decoder->fmt_out;
    if( id->p_f_chain )
        p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );

    if( id->p_uf_chain )
        p_fmt_out = filter_chain_GetFmtOut( id->p_uf_chain );

    if( ( p_fmt_out->video.i_chroma != id->p_encoder->fmt_in.video.i_chroma ) ||
        ( p_fmt_out->video.i_width != id->p_encoder->fmt_in.video.i_width ) ||
        ( p_fmt_out->video.i_height != id->p_encoder->fmt_in.video.i_height ) )
    {
        filter_chain_AppendFilter( id->p_uf_chain ? id->p_uf_chain : id->p_f_chain,
                                   NULL, NULL,
                                   p_fmt_out,
                                   &id->p_encoder->fmt_in );
    }
}

static void transcode_video_encoder_init( sout_stream_t *p_stream,
                                          sout_stream_id_sys_t *id )
{
    sout_stream_sys_t *p_sys = p_stream->p_sys;

    const es_format_t *p_fmt_out = &id->p_decoder->fmt_out;
    if( id->p_f_chain ) {
        p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
    }
    if( id->p_uf_chain ) {
        p_fmt_out = filter_chain_GetFmtOut( id->p_uf_chain );
    }

    /* Calculate scaling
     * width/height of source */
    int i_src_visible_width = p_fmt_out->video.i_visible_width;
    int i_src_visible_height = p_fmt_out->video.i_visible_height;

    if (i_src_visible_width == 0)
        i_src_visible_width = p_fmt_out->video.i_width;
    if (i_src_visible_height == 0)
        i_src_visible_height = p_fmt_out->video.i_height;


    /* with/height scaling */
    float f_scale_width = 1;
    float f_scale_height = 1;

    /* aspect ratio */
    float f_aspect = (double)p_fmt_out->video.i_sar_num *
                     p_fmt_out->video.i_width /
                     p_fmt_out->video.i_sar_den /
                     p_fmt_out->video.i_height;

    msg_Dbg( p_stream, "decoder aspect is %f:1", (double) f_aspect );

    /* Change f_aspect from source frame to source pixel */
    f_aspect = f_aspect * i_src_visible_height / i_src_visible_width;
    msg_Dbg( p_stream, "source pixel aspect is %f:1", (double) f_aspect );

    /* Calculate scaling factor for specified parameters */
    if( id->p_encoder->fmt_out.video.i_visible_width <= 0 &&
        id->p_encoder->fmt_out.video.i_visible_height <= 0 && p_sys->f_scale )
    {
        /* Global scaling. Make sure width will remain a factor of 16 */
        float f_real_scale;
        int  i_new_height;
        int i_new_width = i_src_visible_width * p_sys->f_scale;

        if( i_new_width % 16 <= 7 && i_new_width >= 16 )
            i_new_width -= i_new_width % 16;
        else
            i_new_width += 16 - i_new_width % 16;

        f_real_scale = (float)( i_new_width ) / (float) i_src_visible_width;

        i_new_height = __MAX( 16, i_src_visible_height * (float)f_real_scale );

        f_scale_width = f_real_scale;
        f_scale_height = (float) i_new_height / (float) i_src_visible_height;
    }
    else if( id->p_encoder->fmt_out.video.i_visible_width > 0 &&
             id->p_encoder->fmt_out.video.i_visible_height <= 0 )
    {
        /* Only width specified */
        f_scale_width = (float)id->p_encoder->fmt_out.video.i_visible_width/i_src_visible_width;
        f_scale_height = f_scale_width;
    }
    else if( id->p_encoder->fmt_out.video.i_visible_width <= 0 &&
             id->p_encoder->fmt_out.video.i_visible_height > 0 )
    {
         /* Only height specified */
         f_scale_height = (float)id->p_encoder->fmt_out.video.i_visible_height/i_src_visible_height;
         f_scale_width = f_scale_height;
     }
     else if( id->p_encoder->fmt_out.video.i_visible_width > 0 &&
              id->p_encoder->fmt_out.video.i_visible_height > 0 )
     {
         /* Width and height specified */
         f_scale_width = (float)id->p_encoder->fmt_out.video.i_visible_width/i_src_visible_width;
         f_scale_height = (float)id->p_encoder->fmt_out.video.i_visible_height/i_src_visible_height;
     }

     /* check maxwidth and maxheight */
     if( p_sys->i_maxwidth && f_scale_width > (float)p_sys->i_maxwidth /
                                                     i_src_visible_width )
     {
         f_scale_width = (float)p_sys->i_maxwidth / i_src_visible_width;
     }

     if( p_sys->i_maxheight && f_scale_height > (float)p_sys->i_maxheight /
                                                       i_src_visible_height )
     {
         f_scale_height = (float)p_sys->i_maxheight / i_src_visible_height;
     }


     /* Change aspect ratio from source pixel to scaled pixel */
     f_aspect = f_aspect * f_scale_height / f_scale_width;
     msg_Dbg( p_stream, "scaled pixel aspect is %f:1", (double) f_aspect );

     /* f_scale_width and f_scale_height are now final */
     /* Calculate width, height from scaling
      * Make sure its multiple of 2
      */
     /* width/height of output stream */
     int i_dst_visible_width =  2 * lroundf(f_scale_width*i_src_visible_width/2);
     int i_dst_visible_height = 2 * lroundf(f_scale_height*i_src_visible_height/2);
     int i_dst_width =  2 * lroundf(f_scale_width*p_fmt_out->video.i_width/2);
     int i_dst_height = 2 * lroundf(f_scale_height*p_fmt_out->video.i_height/2);

     /* Change aspect ratio from scaled pixel to output frame */
     f_aspect = f_aspect * i_dst_visible_width / i_dst_visible_height;

     /* Store calculated values */
     id->p_encoder->fmt_out.video.i_width = i_dst_width;
     id->p_encoder->fmt_out.video.i_visible_width = i_dst_visible_width;
     id->p_encoder->fmt_out.video.i_height = i_dst_height;
     id->p_encoder->fmt_out.video.i_visible_height = i_dst_visible_height;

     id->p_encoder->fmt_in.video.i_width = i_dst_width;
     id->p_encoder->fmt_in.video.i_visible_width = i_dst_visible_width;
     id->p_encoder->fmt_in.video.i_height = i_dst_height;
     id->p_encoder->fmt_in.video.i_visible_height = i_dst_visible_height;

     msg_Dbg( p_stream, "source %ix%i, destination %ix%i",
         i_src_visible_width, i_src_visible_height,
         i_dst_visible_width, i_dst_visible_height
     );

    /* Handle frame rate conversion */
    if( !id->p_encoder->fmt_out.video.i_frame_rate ||
        !id->p_encoder->fmt_out.video.i_frame_rate_base )
    {
        if( p_fmt_out->video.i_frame_rate &&
            p_fmt_out->video.i_frame_rate_base )
        {
            id->p_encoder->fmt_out.video.i_frame_rate =
                p_fmt_out->video.i_frame_rate;
            id->p_encoder->fmt_out.video.i_frame_rate_base =
                p_fmt_out->video.i_frame_rate_base;
        }
        else
        {
            /* Pick a sensible default value */
            id->p_encoder->fmt_out.video.i_frame_rate = ENC_FRAMERATE;
            id->p_encoder->fmt_out.video.i_frame_rate_base = ENC_FRAMERATE_BASE;
        }
    }

    id->p_encoder->fmt_in.video.orientation =
        id->p_encoder->fmt_out.video.orientation =
        id->p_decoder->fmt_in.video.orientation;

    id->p_encoder->fmt_in.video.i_frame_rate =
        id->p_encoder->fmt_out.video.i_frame_rate;
    id->p_encoder->fmt_in.video.i_frame_rate_base =
        id->p_encoder->fmt_out.video.i_frame_rate_base;

    vlc_ureduce( &id->p_encoder->fmt_in.video.i_frame_rate,
        &id->p_encoder->fmt_in.video.i_frame_rate_base,
        id->p_encoder->fmt_in.video.i_frame_rate,
        id->p_encoder->fmt_in.video.i_frame_rate_base,
        0 );
     msg_Dbg( p_stream, "source fps %u/%u, destination %u/%u",
        id->p_decoder->fmt_out.video.i_frame_rate,
        id->p_decoder->fmt_out.video.i_frame_rate_base,
        id->p_encoder->fmt_in.video.i_frame_rate,
        id->p_encoder->fmt_in.video.i_frame_rate_base );


    /* Check whether a particular aspect ratio was requested */
    if( id->p_encoder->fmt_out.video.i_sar_num <= 0 ||
        id->p_encoder->fmt_out.video.i_sar_den <= 0 )
    {
        vlc_ureduce( &id->p_encoder->fmt_out.video.i_sar_num,
                     &id->p_encoder->fmt_out.video.i_sar_den,
                     (uint64_t)p_fmt_out->video.i_sar_num * i_src_visible_width  * i_dst_visible_height,
                     (uint64_t)p_fmt_out->video.i_sar_den * i_src_visible_height * i_dst_visible_width,
                     0 );
    }
    else
    {
        vlc_ureduce( &id->p_encoder->fmt_out.video.i_sar_num,
                     &id->p_encoder->fmt_out.video.i_sar_den,
                     id->p_encoder->fmt_out.video.i_sar_num,
                     id->p_encoder->fmt_out.video.i_sar_den,
                     0 );
    }

    id->p_encoder->fmt_in.video.i_sar_num =
        id->p_encoder->fmt_out.video.i_sar_num;
    id->p_encoder->fmt_in.video.i_sar_den =
        id->p_encoder->fmt_out.video.i_sar_den;

    msg_Dbg( p_stream, "encoder aspect is %i:%i",
             id->p_encoder->fmt_out.video.i_sar_num * id->p_encoder->fmt_out.video.i_width,
             id->p_encoder->fmt_out.video.i_sar_den * id->p_encoder->fmt_out.video.i_height );

}

static int transcode_video_encoder_open( sout_stream_t *p_stream,
                                         sout_stream_id_sys_t *id )
{
    sout_stream_sys_t *p_sys = p_stream->p_sys;


    msg_Dbg( p_stream, "destination (after video filters) %ix%i",
             id->p_encoder->fmt_in.video.i_width,
             id->p_encoder->fmt_in.video.i_height );

    id->p_encoder->p_module =
        module_need( id->p_encoder, "encoder", p_sys->psz_venc, true );
    if( !id->p_encoder->p_module )
    {
        msg_Err( p_stream, "cannot find video encoder (module:%s fourcc:%4.4s)",
                 p_sys->psz_venc ? p_sys->psz_venc : "any",
                 (char *)&p_sys->i_vcodec );
        return VLC_EGENERIC;
    }

    id->p_encoder->fmt_in.video.i_chroma = id->p_encoder->fmt_in.i_codec;<commit_msg>transcode: refactor video init function<commit_after>/*****************************************************************************
 * video.c: transcoding stream output module (video)
 *****************************************************************************
 * Copyright (C) 2003-2009 VLC authors and VideoLAN
 * $Id$
 *
 * Authors: <NAME> <<EMAIL>>
 *          <NAME> <<EMAIL>>
 *          <NAME> <jpsaman #_at_# m2x dot nl>
 *          <NAME> <dionoea at videolan dot org>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

/*****************************************************************************
 * Preamble
 *****************************************************************************/

#include "transcode.h"

#include <math.h>
#include <vlc_meta.h>
#include <vlc_spu.h>
#include <vlc_modules.h>

#define ENC_FRAMERATE (25 * 1000)
#define ENC_FRAMERATE_BASE 1000

struct decoder_owner_sys_t
{
    sout_stream_sys_t *p_sys;
};

static int video_update_format_decoder( decoder_t *p_dec )
{
    p_dec->fmt_out.video.i_chroma = p_dec->fmt_out.i_codec;
    return 0;
}

static picture_t *video_new_buffer_decoder( decoder_t *p_dec )
{
    return picture_NewFromFormat( &p_dec->fmt_out.video );
}

static picture_t *video_new_buffer_encoder( encoder_t *p_enc )
{
    p_enc->fmt_in.video.i_chroma = p_enc->fmt_in.i_codec;
    return picture_NewFromFormat( &p_enc->fmt_in.video );
}

static picture_t *transcode_video_filter_buffer_new( filter_t *p_filter )
{
    p_filter->fmt_out.video.i_chroma = p_filter->fmt_out.i_codec;
    return picture_NewFromFormat( &p_filter->fmt_out.video );
}

static void* EncoderThread( void *obj )
{
    sout_stream_sys_t *p_sys = (sout_stream_sys_t*)obj;
    sout_stream_id_sys_t *id = p_sys->id_video;
    picture_t *p_pic = NULL;
    int canc = vlc_savecancel ();
    block_t *p_block = NULL;

    vlc_mutex_lock( &p_sys->lock_out );

    for( ;; )
    {
        while( !p_sys->b_abort &&
               (p_pic = picture_fifo_Pop( p_sys->pp_pics )) == NULL )
            vlc_cond_wait( &p_sys->cond, &p_sys->lock_out );

        if( p_pic )
        {
            /* release lock while encoding */
            vlc_mutex_unlock( &p_sys->lock_out );
            p_block = id->p_encoder->pf_encode_video( id->p_encoder, p_pic );
            picture_Release( p_pic );
            vlc_mutex_lock( &p_sys->lock_out );

            block_ChainAppend( &p_sys->p_buffers, p_block );
        }

        if( p_sys->b_abort )
            break;
    }

    /*Encode what we have in the buffer on closing*/
    while( (p_pic = picture_fifo_Pop( p_sys->pp_pics )) != NULL )
    {
        p_block = id->p_encoder->pf_encode_video( id->p_encoder, p_pic );
        picture_Release( p_pic );
        block_ChainAppend( &p_sys->p_buffers, p_block );
    }

    /*Now flush encoder*/
    do {
        p_block = id->p_encoder->pf_encode_video(id->p_encoder, NULL );
        block_ChainAppend( &p_sys->p_buffers, p_block );
    } while( p_block );

    vlc_mutex_unlock( &p_sys->lock_out );

    vlc_restorecancel (canc);

    return NULL;
}

int transcode_video_new( sout_stream_t *p_stream, sout_stream_id_sys_t *id )
{
    sout_stream_sys_t *p_sys = p_stream->p_sys;

    /* Open decoder
     * Initialization of decoder structures
     */
    id->p_decoder->fmt_out = id->p_decoder->fmt_in;
    id->p_decoder->fmt_out.i_extra = 0;
    id->p_decoder->fmt_out.p_extra = NULL;
    id->p_decoder->fmt_out.psz_language = NULL;
    id->p_decoder->pf_decode_video = NULL;
    id->p_decoder->pf_get_cc = NULL;
    id->p_decoder->pf_vout_format_update = video_update_format_decoder;
    id->p_decoder->pf_vout_buffer_new = video_new_buffer_decoder;
    id->p_decoder->p_owner = malloc( sizeof(decoder_owner_sys_t) );
    if( !id->p_decoder->p_owner )
        return VLC_EGENERIC;

    id->p_decoder->p_owner->p_sys = p_sys;
    /* id->p_decoder->p_cfg = p_sys->p_video_cfg; */

    id->p_decoder->p_module =
        module_need( id->p_decoder, "decoder", "$codec", false );

    if( !id->p_decoder->p_module )
    {
        msg_Err( p_stream, "cannot find video decoder" );
        free( id->p_decoder->p_owner );
        return VLC_EGENERIC;
    }

    /*
     * Open encoder.
     * Because some info about the decoded input will only be available
     * once the first frame is decoded, we actually only test the availability
     * of the encoder here.
     */

    /* Initialization of encoder format structures */
    es_format_Init( &id->p_encoder->fmt_in, id->p_decoder->fmt_in.i_cat,
                    id->p_decoder->fmt_out.i_codec );
    id->p_encoder->fmt_in.video.i_chroma = id->p_decoder->fmt_out.i_codec;

    /* The dimensions will be set properly later on.
     * Just put sensible values so we can test an encoder is available. */
    id->p_encoder->fmt_in.video.i_width =
        id->p_encoder->fmt_out.video.i_width
          ? id->p_encoder->fmt_out.video.i_width
          : id->p_decoder->fmt_in.video.i_width
            ? id->p_decoder->fmt_in.video.i_width : 16;
    id->p_encoder->fmt_in.video.i_height =
        id->p_encoder->fmt_out.video.i_height
          ? id->p_encoder->fmt_out.video.i_height
          : id->p_decoder->fmt_in.video.i_height
            ? id->p_decoder->fmt_in.video.i_height : 16;
    id->p_encoder->fmt_in.video.i_visible_width =
        id->p_encoder->fmt_out.video.i_visible_width
          ? id->p_encoder->fmt_out.video.i_visible_width
          : id->p_decoder->fmt_in.video.i_visible_width
            ? id->p_decoder->fmt_in.video.i_visible_width : id->p_encoder->fmt_in.video.i_width;
    id->p_encoder->fmt_in.video.i_visible_height =
        id->p_encoder->fmt_out.video.i_visible_height
          ? id->p_encoder->fmt_out.video.i_visible_height
          : id->p_decoder->fmt_in.video.i_visible_height
            ? id->p_decoder->fmt_in.video.i_visible_height : id->p_encoder->fmt_in.video.i_height;

    id->p_encoder->i_threads = p_sys->i_threads;
    id->p_encoder->p_cfg = p_sys->p_video_cfg;

    id->p_encoder->p_module =
        module_need( id->p_encoder, "encoder", p_sys->psz_venc, true );
    if( !id->p_encoder->p_module )
    {
        msg_Err( p_stream, "cannot find video encoder (module:%s fourcc:%4.4s). Take a look few lines earlier to see possible reason.",
                 p_sys->psz_venc ? p_sys->psz_venc : "any",
                 (char *)&p_sys->i_vcodec );
        module_unneed( id->p_decoder, id->p_decoder->p_module );
        id->p_decoder->p_module = 0;
        free( id->p_decoder->p_owner );
        return VLC_EGENERIC;
    }

    /* Close the encoder.
     * We'll open it only when we have the first frame. */
    module_unneed( id->p_encoder, id->p_encoder->p_module );
    if( id->p_encoder->fmt_out.p_extra )
    {
        free( id->p_encoder->fmt_out.p_extra );
        id->p_encoder->fmt_out.p_extra = NULL;
        id->p_encoder->fmt_out.i_extra = 0;
    }
    id->p_encoder->p_module = NULL;

    if( p_sys->i_threads <= 0 )
        return VLC_SUCCESS;

    int i_priority = p_sys->b_high_priority ? VLC_THREAD_PRIORITY_OUTPUT :
                       VLC_THREAD_PRIORITY_VIDEO;
    p_sys->id_video = id;
    p_sys->pp_pics = picture_fifo_New();
    if( p_sys->pp_pics == NULL )
    {
        msg_Err( p_stream, "cannot create picture fifo" );
        module_unneed( id->p_decoder, id->p_decoder->p_module );
        id->p_decoder->p_module = NULL;
        free( id->p_decoder->p_owner );
        return VLC_ENOMEM;
    }
    vlc_mutex_init( &p_sys->lock_out );
    vlc_cond_init( &p_sys->cond );
    p_sys->p_buffers = NULL;
    p_sys->b_abort = false;
    if( vlc_clone( &p_sys->thread, EncoderThread, p_sys, i_priority ) )
    {
        msg_Err( p_stream, "cannot spawn encoder thread" );
        vlc_mutex_destroy( &p_sys->lock_out );
        vlc_cond_destroy( &p_sys->cond );
        picture_fifo_Delete( p_sys->pp_pics );
        module_unneed( id->p_decoder, id->p_decoder->p_module );
        id->p_decoder->p_module = NULL;
        free( id->p_decoder->p_owner );
        return VLC_EGENERIC;
    }
    return VLC_SUCCESS;
}

static void transcode_video_filter_init( sout_stream_t *p_stream,
                                         sout_stream_id_sys_t *id )
{
    filter_owner_t owner = {
        .sys = p_stream->p_sys,
        .video = {
            .buffer_new = transcode_video_filter_buffer_new,
        },
    };
    es_format_t *p_fmt_out = &id->p_decoder->fmt_out;

    id->p_encoder->fmt_in.video.i_chroma = id->p_encoder->fmt_in.i_codec;
    id->p_f_chain = filter_chain_NewVideo( p_stream, false, &owner );
    filter_chain_Reset( id->p_f_chain, p_fmt_out, p_fmt_out );

    /* Deinterlace */
    if( p_stream->p_sys->b_deinterlace )
    {
        filter_chain_AppendFilter( id->p_f_chain,
                                   p_stream->p_sys->psz_deinterlace,
                                   p_stream->p_sys->p_deinterlace_cfg,
                                   &id->p_decoder->fmt_out,
                                   &id->p_decoder->fmt_out );

        p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
    }
    if( p_stream->p_sys->b_master_sync )
    {
        filter_chain_AppendFilter( id->p_f_chain,
                                   "fps",
                                   NULL,
                                   p_fmt_out,
                                   &id->p_encoder->fmt_in );

        p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
    }

    /* Check that we have visible_width/height*/
    if( !p_fmt_out->video.i_visible_height )
        p_fmt_out->video.i_visible_height = p_fmt_out->video.i_height;
    if( !p_fmt_out->video.i_visible_width )
        p_fmt_out->video.i_visible_width = p_fmt_out->video.i_width;

    if( p_stream->p_sys->psz_vf2 )
    {
        id->p_uf_chain = filter_chain_NewVideo( p_stream, true, &owner );
        filter_chain_Reset( id->p_uf_chain, p_fmt_out,
                            &id->p_encoder->fmt_in );
        if( p_fmt_out->video.i_chroma != id->p_encoder->fmt_in.video.i_chroma )
        {
            filter_chain_AppendFilter( id->p_uf_chain,
                                   NULL, NULL,
                                   p_fmt_out,
                                   &id->p_encoder->fmt_in );
        }
        filter_chain_AppendFromString( id->p_uf_chain, p_stream->p_sys->psz_vf2 );
        p_fmt_out = filter_chain_GetFmtOut( id->p_uf_chain );
        es_format_Copy( &id->p_encoder->fmt_in, p_fmt_out );
        id->p_encoder->fmt_out.video.i_width =
            id->p_encoder->fmt_in.video.i_width;
        id->p_encoder->fmt_out.video.i_height =
            id->p_encoder->fmt_in.video.i_height;
        id->p_encoder->fmt_out.video.i_sar_num =
            id->p_encoder->fmt_in.video.i_sar_num;
        id->p_encoder->fmt_out.video.i_sar_den =
            id->p_encoder->fmt_in.video.i_sar_den;
    }

    /* Keep colorspace etc info along */
    id->p_encoder->fmt_in.video.space     = id->p_decoder->fmt_out.video.space;
    id->p_encoder->fmt_in.video.transfer  = id->p_decoder->fmt_out.video.transfer;
    id->p_encoder->fmt_in.video.primaries = id->p_decoder->fmt_out.video.primaries;
    id->p_encoder->fmt_in.video.b_color_range_full = id->p_decoder->fmt_out.video.b_color_range_full;
}

/* Take care of the scaling and chroma conversions. */
static void conversion_video_filter_append( sout_stream_id_sys_t *id )
{
    const es_format_t *p_fmt_out = &id->p_decoder->fmt_out;
    if( id->p_f_chain )
        p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );

    if( id->p_uf_chain )
        p_fmt_out = filter_chain_GetFmtOut( id->p_uf_chain );

    if( ( p_fmt_out->video.i_chroma != id->p_encoder->fmt_in.video.i_chroma ) ||
        ( p_fmt_out->video.i_width != id->p_encoder->fmt_in.video.i_width ) ||
        ( p_fmt_out->video.i_height != id->p_encoder->fmt_in.video.i_height ) )
    {
        filter_chain_AppendFilter( id->p_uf_chain ? id->p_uf_chain : id->p_f_chain,
                                   NULL, NULL,
                                   p_fmt_out,
                                   &id->p_encoder->fmt_in );
    }
}

static void transcode_video_framerate_init( sout_stream_t *p_stream,
                                            sout_stream_id_sys_t *id,
                                            const es_format_t *p_fmt_out )
{
    /* Handle frame rate conversion */
    if( !id->p_encoder->fmt_out.video.i_frame_rate ||
        !id->p_encoder->fmt_out.video.i_frame_rate_base )
    {
        if( p_fmt_out->video.i_frame_rate &&
            p_fmt_out->video.i_frame_rate_base )
        {
            id->p_encoder->fmt_out.video.i_frame_rate =
                p_fmt_out->video.i_frame_rate;
            id->p_encoder->fmt_out.video.i_frame_rate_base =
                p_fmt_out->video.i_frame_rate_base;
        }
        else
        {
            /* Pick a sensible default value */
            id->p_encoder->fmt_out.video.i_frame_rate = ENC_FRAMERATE;
            id->p_encoder->fmt_out.video.i_frame_rate_base = ENC_FRAMERATE_BASE;
        }
    }

    id->p_encoder->fmt_in.video.i_frame_rate =
        id->p_encoder->fmt_out.video.i_frame_rate;
    id->p_encoder->fmt_in.video.i_frame_rate_base =
        id->p_encoder->fmt_out.video.i_frame_rate_base;

    vlc_ureduce( &id->p_encoder->fmt_in.video.i_frame_rate,
        &id->p_encoder->fmt_in.video.i_frame_rate_base,
        id->p_encoder->fmt_in.video.i_frame_rate,
        id->p_encoder->fmt_in.video.i_frame_rate_base,
        0 );
     msg_Dbg( p_stream, "source fps %u/%u, destination %u/%u",
        id->p_decoder->fmt_out.video.i_frame_rate,
        id->p_decoder->fmt_out.video.i_frame_rate_base,
        id->p_encoder->fmt_in.video.i_frame_rate,
        id->p_encoder->fmt_in.video.i_frame_rate_base );

}

static void transcode_video_size_init( sout_stream_t *p_stream,
                                     sout_stream_id_sys_t *id,
                                     const es_format_t *p_fmt_out )
{
    sout_stream_sys_t *p_sys = p_stream->p_sys;

    /* Calculate scaling
     * width/height of source */
    int i_src_visible_width = p_fmt_out->video.i_visible_width;
    int i_src_visible_height = p_fmt_out->video.i_visible_height;

    if (i_src_visible_width == 0)
        i_src_visible_width = p_fmt_out->video.i_width;
    if (i_src_visible_height == 0)
        i_src_visible_height = p_fmt_out->video.i_height;


    /* with/height scaling */
    float f_scale_width = 1;
    float f_scale_height = 1;

    /* aspect ratio */
    float f_aspect = (double)p_fmt_out->video.i_sar_num *
                     p_fmt_out->video.i_width /
                     p_fmt_out->video.i_sar_den /
                     p_fmt_out->video.i_height;

    msg_Dbg( p_stream, "decoder aspect is %f:1", (double) f_aspect );

    /* Change f_aspect from source frame to source pixel */
    f_aspect = f_aspect * i_src_visible_height / i_src_visible_width;
    msg_Dbg( p_stream, "source pixel aspect is %f:1", (double) f_aspect );

    /* Calculate scaling factor for specified parameters */
    if( id->p_encoder->fmt_out.video.i_visible_width <= 0 &&
        id->p_encoder->fmt_out.video.i_visible_height <= 0 && p_sys->f_scale )
    {
        /* Global scaling. Make sure width will remain a factor of 16 */
        float f_real_scale;
        int  i_new_height;
        int i_new_width = i_src_visible_width * p_sys->f_scale;

        if( i_new_width % 16 <= 7 && i_new_width >= 16 )
            i_new_width -= i_new_width % 16;
        else
            i_new_width += 16 - i_new_width % 16;

        f_real_scale = (float)( i_new_width ) / (float) i_src_visible_width;

        i_new_height = __MAX( 16, i_src_visible_height * (float)f_real_scale );

        f_scale_width = f_real_scale;
        f_scale_height = (float) i_new_height / (float) i_src_visible_height;
    }
    else if( id->p_encoder->fmt_out.video.i_visible_width > 0 &&
             id->p_encoder->fmt_out.video.i_visible_height <= 0 )
    {
        /* Only width specified */
        f_scale_width = (float)id->p_encoder->fmt_out.video.i_visible_width/i_src_visible_width;
        f_scale_height = f_scale_width;
    }
    else if( id->p_encoder->fmt_out.video.i_visible_width <= 0 &&
             id->p_encoder->fmt_out.video.i_visible_height > 0 )
    {
         /* Only height specified */
         f_scale_height = (float)id->p_encoder->fmt_out.video.i_visible_height/i_src_visible_height;
         f_scale_width = f_scale_height;
     }
     else if( id->p_encoder->fmt_out.video.i_visible_width > 0 &&
              id->p_encoder->fmt_out.video.i_visible_height > 0 )
     {
         /* Width and height specified */
         f_scale_width = (float)id->p_encoder->fmt_out.video.i_visible_width/i_src_visible_width;
         f_scale_height = (float)id->p_encoder->fmt_out.video.i_visible_height/i_src_visible_height;
     }

     /* check maxwidth and maxheight */
     if( p_sys->i_maxwidth && f_scale_width > (float)p_sys->i_maxwidth /
                                                     i_src_visible_width )
     {
         f_scale_width = (float)p_sys->i_maxwidth / i_src_visible_width;
     }

     if( p_sys->i_maxheight && f_scale_height > (float)p_sys->i_maxheight /
                                                       i_src_visible_height )
     {
         f_scale_height = (float)p_sys->i_maxheight / i_src_visible_height;
     }


     /* Change aspect ratio from source pixel to scaled pixel */
     f_aspect = f_aspect * f_scale_height / f_scale_width;
     msg_Dbg( p_stream, "scaled pixel aspect is %f:1", (double) f_aspect );

     /* f_scale_width and f_scale_height are now final */
     /* Calculate width, height from scaling
      * Make sure its multiple of 2
      */
     /* width/height of output stream */
     int i_dst_visible_width =  2 * lroundf(f_scale_width*i_src_visible_width/2);
     int i_dst_visible_height = 2 * lroundf(f_scale_height*i_src_visible_height/2);
     int i_dst_width =  2 * lroundf(f_scale_width*p_fmt_out->video.i_width/2);
     int i_dst_height = 2 * lroundf(f_scale_height*p_fmt_out->video.i_height/2);

     /* Change aspect ratio from scaled pixel to output frame */
     f_aspect = f_aspect * i_dst_visible_width / i_dst_visible_height;

     /* Store calculated values */
     id->p_encoder->fmt_out.video.i_width = i_dst_width;
     id->p_encoder->fmt_out.video.i_visible_width = i_dst_visible_width;
     id->p_encoder->fmt_out.video.i_height = i_dst_height;
     id->p_encoder->fmt_out.video.i_visible_height = i_dst_visible_height;

     id->p_encoder->fmt_in.video.i_width = i_dst_width;
     id->p_encoder->fmt_in.video.i_visible_width = i_dst_visible_width;
     id->p_encoder->fmt_in.video.i_height = i_dst_height;
     id->p_encoder->fmt_in.video.i_visible_height = i_dst_visible_height;

     msg_Dbg( p_stream, "source %ix%i, destination %ix%i",
         i_src_visible_width, i_src_visible_height,
         i_dst_visible_width, i_dst_visible_height
     );
};

static void transcode_video_sar_init( sout_stream_t *p_stream,
                                     sout_stream_id_sys_t *id,
                                     const es_format_t *p_fmt_out )
{
    int i_src_visible_width = p_fmt_out->video.i_visible_width;
    int i_src_visible_height = p_fmt_out->video.i_visible_height;

    if (i_src_visible_width == 0)
        i_src_visible_width = p_fmt_out->video.i_width;
    if (i_src_visible_height == 0)
        i_src_visible_height = p_fmt_out->video.i_height;

    /* Check whether a particular aspect ratio was requested */
    if( id->p_encoder->fmt_out.video.i_sar_num <= 0 ||
        id->p_encoder->fmt_out.video.i_sar_den <= 0 )
    {
        vlc_ureduce( &id->p_encoder->fmt_out.video.i_sar_num,
                     &id->p_encoder->fmt_out.video.i_sar_den,
                     (uint64_t)p_fmt_out->video.i_sar_num * id->p_encoder->fmt_out.video.i_width * p_fmt_out->video.i_height,
                     (uint64_t)p_fmt_out->video.i_sar_den * id->p_encoder->fmt_out.video.i_height * p_fmt_out->video.i_width,
                     0 );
    }
    else
    {
        vlc_ureduce( &id->p_encoder->fmt_out.video.i_sar_num,
                     &id->p_encoder->fmt_out.video.i_sar_den,
                     id->p_encoder->fmt_out.video.i_sar_num,
                     id->p_encoder->fmt_out.video.i_sar_den,
                     0 );
    }

    id->p_encoder->fmt_in.video.i_sar_num =
        id->p_encoder->fmt_out.video.i_sar_num;
    id->p_encoder->fmt_in.video.i_sar_den =
        id->p_encoder->fmt_out.video.i_sar_den;

    msg_Dbg( p_stream, "encoder aspect is %i:%i",
             id->p_encoder->fmt_out.video.i_sar_num * id->p_encoder->fmt_out.video.i_width,
             id->p_encoder->fmt_out.video.i_sar_den * id->p_encoder->fmt_out.video.i_height );

}

static void transcode_video_encoder_init( sout_stream_t *p_stream,
                                          sout_stream_id_sys_t *id )
{
    const es_format_t *p_fmt_out = &id->p_decoder->fmt_out;
    if( id->p_f_chain ) {
        p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
    }
    if( id->p_uf_chain ) {
        p_fmt_out = filter_chain_GetFmtOut( id->p_uf_chain );
    }

    id->p_encoder->fmt_in.video.orientation =
        id->p_encoder->fmt_out.video.orientation =
        id->p_decoder->fmt_in.video.orientation;

    transcode_video_framerate_init( p_stream, id, p_fmt_out );

    transcode_video_size_init( p_stream, id, p_fmt_out );
    transcode_video_sar_init( p_stream, id, p_fmt_out );

}

static int transcode_video_encoder_open( sout_stream_t *p_stream,
                                         sout_stream_id_sys_t *id )
{
    sout_stream_sys_t *p_sys = p_stream->p_sys;


    msg_Dbg( p_stream, "destination (after video filters) %ix%i",
             id->p_encoder->fmt_in.video.i_width,
             id->p_encoder->fmt_in.video.i_height );

    id->p_encoder->p_module =
        module_need( id->p_encoder, "encoder", p_sys->psz_venc, true );
    if( !id->p_encoder->p_module )
    {
        msg_Err( p_stream, "cannot find video encoder (module:%s fourcc:%4.4s)",
                 p_sys->psz_venc ? p_sys->psz_venc : "any",
                 (char *)&p_sys->i_vcodec );
        return VLC_EGENERIC;
    }

    id->p_encoder->fmt_in.video.i_chroma = id->p_encoder->fmt_in.i_codec;
{'fraction_non_alphanumeric': 0.13135696957557144, 'fraction_numerical': 0.004273218935701409, 'mean_word_length': 2.5160810705062993, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 955, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 56, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5307902', 'n_tokens_mistral': 16334, 'n_tokens_neox': 15748, 'n_words': 2451}