<commit_before>/*
 *			GPAC - Multimedia Framework C SDK
 *
 *			Authors: <NAME>
 *			Copyright (c) Telecom ParisTech 2010-
 *					All rights reserved
 *
 *  This file is part of GPAC / 3GPP/MPEG Media Presentation Description input module
 *
 *  GPAC is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU Lesser General Public License as published by
 *  the Free Software Foundation; either version 2, or (at your option)
 *  any later version.
 *
 *  GPAC is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; see the file COPYING.  If not, write to
 *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 */

#include <gpac/modules/service.h>
#include <gpac/internal/terminal_dev.h>
#include "mpd.h"
#include <gpac/thread.h>
#include <gpac/network.h>
#include <gpac/crypt.h>
#include "m3u8.h"

/*!
 * All the possible Mime-types for MPD files
 */
static const char * MPD_MIME_TYPES[] = { "video/vnd.3gpp.mpd", "audio/vnd.3gpp.mpd", NULL };

/*!
 * All the possible Mime-types for M3U8 files
 */
static const char * M3U8_MIME_TYPES[] = { "video/x-mpegurl", "audio/x-mpegurl", "application/x-mpegurl", "application/vnd.apple.mpegurl", NULL};


typedef struct
{
    char *cache;
    char *url;
} segment_cache_entry;

typedef struct __mpd_module {
    /* GPAC Service object (i.e. how this module is seen by the terminal)*/
    GF_ClientService *service;
    /* URL to which this service is connected
       Used to detect when audio service connection request is made on the same URL as video */
    char *url;
    Bool is_service_connected;
    /* number of time the service has been connected */
    u32 nb_service_connections;
    u32 nb_connected_channels;
    u32 nb_playing_or_paused_channels;

    /* Download of the MPD */
    volatile Bool in_download;

    GF_DownloadSession *mpd_dnload;
    char * urlToDeleteNext;
    volatile u32 max_cached, nb_cached;
    u32 option_max_cached;
    segment_cache_entry *cached;

    /* MPD and active informations */
    GF_MPD *mpd;
    u32 active_period_index;
    u32 active_rep_index;
    u32 download_segment_index;

    /* playback status */
    Double playback_speed;
    Double playback_start_range, playback_end_range;

    /* For Segment downloads */
    GF_DownloadSession *seg_dnload;
    const char *seg_local_url;
    GF_Thread *dl_thread;
    GF_Mutex *dl_mutex;

    /* 0 not started, 1 download in progress */
    Bool dl_status;
    Bool dl_stop_request;

    /* Service really managing the segments */
    GF_InputService *seg_ifce;

    u32 reload_count;
    u32 last_update_time;

    u32 nb_segs_done;
    Bool auto_switch;
    u8 lastMPDSignature[20];
    Bool segment_must_be_streamed;
} GF_MPD_In;

static void dumpStatus( GF_MPD_In *mpdin) {
    u32 i;
    for (i = 0 ; i < mpdin->nb_cached; i++) {
        printf("\t[%u] - %s\t:\t%s\n", i, mpdin->cached[i].cache, mpdin->cached[i].url);
    }
    printf("\t***********\n\n");
}


GF_Err m3u8_to_mpd(GF_MPD_In *mpdin, const char *m3u8_file, const char *url)
{
    GF_Err e;
    u32 i, count;
    Double update_interval;
    VariantPlaylist * pl = NULL;
    Program *prog;
    PlaylistElement *pe;
    FILE *fmpd;
    Bool is_end;

    e = parse_root_playlist(m3u8_file, &pl, url);
    if (e) {
        GF_LOG(GF_LOG_WARNING, GF_LOG_CONTAINER, ("[M3U8] Failed to parse root playlist '%s', error = %s\n", m3u8_file, gf_error_to_string(e)));
        return e;
    }
    gf_delete_file((char *)m3u8_file);
    is_end = !pl->playlistNeedsRefresh;
    i=0;
    assert( pl );
    assert( pl->programs );
    while ((prog = gf_list_enum(pl->programs, &i))) {
        u32 j=0;
        while (NULL != (pe = gf_list_enum(prog->bitrates, &j))) {
            if (pe->url && strstr(pe->url, ".m3u8")) {
                char *suburl = gf_url_concatenate(url, pe->url);
                if (!strcmp(url, suburl)) {
                    gf_free(suburl);
                    GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] Not downloading, programs are identical for %s...\n", pe->url));
                    continue;
                }
                {
                    GF_DownloadSession *sess = gf_term_download_new(mpdin->service, suburl, GF_NETIO_SESSION_NOT_THREADED, NULL, NULL);
                    if (!sess) {
                        gf_free(suburl);
                        break;
                    }
                    e = gf_dm_sess_process(sess);
                    if (e==GF_OK) {
                        e = parse_sub_playlist(gf_dm_sess_get_cache_name(sess), &pl, suburl, prog, pe);
                    }
                    gf_term_download_del(sess);
                    gf_free(suburl);
                }
            }
        }
    }

    update_interval = 0;
    prog = gf_list_get(pl->programs, 0);
    assert( prog );
    pe = gf_list_last(prog->bitrates);
    assert( pe );
    /*update interval is set to the duration of the last media file with rules defined in http live streaming RFC section 6.3.4*/
    switch (mpdin->reload_count) {
    case 0:
        update_interval = pe->durationInfo;
        break;
    case 1:
        update_interval = pe->durationInfo/2;
        break;
    case 2:
        update_interval = 3*(pe->durationInfo/2);
        break;
    default:
        update_interval = 3*(pe->durationInfo);
        break;
    }
    if (is_end || (pe->elementType == TYPE_PLAYLIST) && pe->element.playlist.is_ended) {
        update_interval = 0;
        GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] NO NEED to refresh playlist !\n"));
    } else {
        GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] Playlist will be refresh every %g seconds, len=%d\n", update_interval, pe->durationInfo));
    }

    fmpd = fopen(m3u8_file, "wt");
    fprintf(fmpd, "<MPD type=\"Live\" xmlns=\"urn:3GPP:ns:PSS:AdaptiveHTTPStreamingMPD:2009\" ");
    if (update_interval) fprintf(fmpd, "minimumUpdatePeriodMPD=\"PT%02.2gS\"", update_interval);
    fprintf(fmpd, ">\n");

    fprintf(fmpd, " <ProgramInformation moreInformationURL=\"http://gpac.sourceforge.net\">\n");
    {
        char * title = pe->title;
        if (!title || strlen(title) < 2)
            title = pe->url;
        fprintf(fmpd, "  <Title>%s</Title>\n", title );
    }
    fprintf(fmpd, "  <Copyright>Generated from URL %s</Copyright>\n", url );
    fprintf(fmpd, "  <Source>Generated by GPAC %s from %s</Source>\n", GPAC_FULL_VERSION, url);

    fprintf(fmpd, " </ProgramInformation>\n");
    fprintf(fmpd, " <Period start=\"PT0S\">\n");

    count = gf_list_count(pl->programs);
    for (i=0; i<count; i++) {
        u32 j, count2;
        Program *prog = gf_list_get(pl->programs, i);
        count2 = gf_list_count(prog->bitrates);
        for (j = 0; j<count2; j++) {
            PlaylistElement *pe = gf_list_get(prog->bitrates, j);
            if (pe->elementType == TYPE_PLAYLIST) {
                u32 k, count3;
                fprintf(fmpd, "  <Representation mimeType=\"video/mp2t\">\n");
                fprintf(fmpd, "   <SegmentInfo duration=\"PT%dS\" baseURL=\"%s\">\n", pe->durationInfo, url);
                count3 = gf_list_count(pe->element.playlist.elements);
                update_interval = (count3 - 1) * pe->durationInfo * 1000;
                for (k=0; k<count3; k++) {
                    PlaylistElement *elt = gf_list_get(pe->element.playlist.elements, k);
                    fprintf(fmpd, "    <Url sourceURL=\"%s\"/>\n", elt->url);
                }
                fprintf(fmpd, "   </SegmentInfo>\n");
                fprintf(fmpd, "  </Representation>\n");
            } else if (pe->elementType == TYPE_STREAM) {
                fprintf(stdout, "NOT SUPPORTED: M3U8 Stream\n");
            }
        }
    }
    fprintf(fmpd, " </Period>\n");
    fprintf(fmpd, "</MPD>");
    fclose(fmpd);
    variant_playlist_del(pl);
    return GF_OK;
}

static Bool MPD_CheckRootType(const char *local_url)
{
    if (local_url) {
        char *rtype = gf_xml_get_root_type(local_url, NULL);
        if (rtype) {
            Bool handled = 0;
            if (!strcmp(rtype, "MPD")) {
                handled = 1;
            }
            gf_free(rtype);
            return handled;
        }
    }
    return 0;
}

/**
 * NET IO for MPD, we don't need this anymore since mime-type can be given by session
 */
void MPD_NetIO_Segment(void *cbk, GF_NETIO_Parameter *param)
{
    GF_Err e;
    GF_MPD_In *mpdin = (GF_MPD_In*) cbk;

    /*handle service message*/
    gf_term_download_update_stats(mpdin->seg_dnload);
    e = param->error;
    if (param->msg_type == GF_NETIO_PARSE_REPLY){
	if (! gf_dm_sess_can_be_cached_on_disk(mpdin->seg_dnload)){
	    GF_LOG(GF_LOG_INFO, GF_LOG_MODULE,
		   ("[MPD_IN] Segment %s cannot be cached on disk, will use direct streaming\n", gf_dm_sess_get_resource_name(mpdin->seg_dnload)));
	    mpdin->segment_must_be_streamed = 1;
	    gf_dm_sess_abort(mpdin->seg_dnload);
	} else {
	    mpdin->segment_must_be_streamed = 0;
	}
    }
}

/*!
 * Returns true if given mime type is a MPD file
 * \param mime the mime-type to check
 * \return true if mime-type if MPD-OK
 */
static Bool MPD_is_MPD_mime(const char * mime) {
    u32 i;
    if (!mime)
        return 0;
    i = 0;
    while (MPD_MIME_TYPES[i]) {
        if ( !stricmp(mime, MPD_MIME_TYPES[i++]))
            return 1;
    }
    return 0;
}

/*!
 * Returns true if mime type is an M3U8 mime-type
 * \param mime The mime-type to check
 * \return true if mime-type is OK for M3U8
 */
static Bool MPD_isM3U8_mime(const char * mime) {
    u32 i;
    if (!mime)
        return 0;
    i = 0;
    while (M3U8_MIME_TYPES[i]) {
        if ( !stricmp(mime, M3U8_MIME_TYPES[i++]))
            return 1;
    }
    return 0;
}

static Bool MPD_segments_are_equals(GF_MPD_SegmentInfo * info1, GF_MPD_SegmentInfo * info2) {
    assert( info1 );
    assert( info2 );
    return info1->byterange_end == info2->byterange_end &&
           info1->byterange_start == info2->byterange_start &&
           info1->use_byterange == info2->use_byterange &&
           (info1->url == info2->url || !strcmp(info1->url, info2->url));
}

static GF_Err MPD_UpdatePlaylist(GF_MPD_In *mpdin)
{
    GF_Err e;
    u32 i, j, rep_idx;
    Bool seg_found = 0;
    GF_DOMParser *mpd_parser;
    GF_MPD *new_mpd;
    GF_MPD_Period *period, *new_period;
    GF_MPD_Representation *rep, *new_rep;
    GF_List *segs;
    const char *local_url, * url;
    Bool is_m3u8 = 0;
    u32 oldUpdateTime = mpdin->mpd->min_update_time;
    /*reset update time - if any error occurs, we will no longer attempt to update the MPD*/
    mpdin->mpd->min_update_time = 0;

    if (!mpdin->mpd_dnload) {
        GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: missing downloader\n"));
        return GF_BAD_PARAM;
    }

    local_url = gf_dm_sess_get_cache_name(mpdin->mpd_dnload);
    if (!local_url) {
        GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: wrong cache file %s\n", local_url));
        return GF_IO_ERR;
    }
    gf_delete_file(local_url);
    gf_dm_sess_reset(mpdin->mpd_dnload);
    url = gf_dm_sess_get_resource_name(mpdin->mpd_dnload);
    GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] Updating Playlist %s...\n", url));
    e = gf_dm_sess_process(mpdin->mpd_dnload);
    if (e!=GF_OK) {
        GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: download problem %s for MPD file\n", gf_error_to_string(e)));
        return gf_dm_sess_last_error(mpdin->mpd_dnload);
    } else {
        GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] Playlist %s updated with success\n", url));
    }
    {
        const char * mime = gf_dm_sess_mime_type(mpdin->mpd_dnload);
        /* Some servers, for instance http://tv.freebox.fr, serve m3u8 as text/plain */
        if (MPD_isM3U8_mime(mime) || strstr(url, ".m3u8")) {
            m3u8_to_mpd(mpdin, local_url, url );
        } else if (!MPD_is_MPD_mime(mime)) {
            GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] mime '%s' should be m3u8 or mpd\n", mime));
            gf_term_on_connect(mpdin->service, NULL, GF_BAD_PARAM);
            return GF_BAD_PARAM;
        }
    }

    if (!MPD_CheckRootType(local_url)) {
        GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: MPD file type is not correct %s\n", local_url));
        return GF_NON_COMPLIANT_BITSTREAM;
    }
    {
        u8 signature[sizeof(mpdin->lastMPDSignature)];
        if (gf_sha1_file( local_url, signature)) {
            GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] : cannot SHA1 file %s\n", local_url));
        } else {
            if (! memcmp( signature, mpdin->lastMPDSignature, sizeof(mpdin->lastMPDSignature))) {
                GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] MPD file did not change\n"));
                mpdin->reload_count++;
                mpdin->mpd->min_update_time = oldUpdateTime;
            } else {
                mpdin->reload_count = 0;
                memccpy(mpdin->lastMPDSignature, signature, sizeof(char), sizeof(mpdin->lastMPDSignature));

                /* It means we have to reparse the file ... */
                /* parse the MPD */
                mpd_parser = gf_xml_dom_new();
                e = gf_xml_dom_parse(mpd_parser, local_url, NULL, NULL);
                if (e != GF_OK) {
                    gf_xml_dom_del(mpd_parser);
                    GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: error in XML parsing %s\n", gf_error_to_string(e)));
                    return GF_NON_COMPLIANT_BITSTREAM;
                }
                new_mpd = gf_mpd_new();
                e = gf_mpd_init_from_dom(gf_xml_dom_get_root(mpd_parser), new_mpd);
                gf_xml_dom_del(mpd_parser);
                if (e) {
                    GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: error in MPD creation %s\n", gf_error_to_string(e)));
                    gf_mpd_del(new_mpd);
                    return GF_NON_COMPLIANT_BITSTREAM;
                }

                /*TODO - check periods are the same*/
                period = gf_list_get(mpdin->mpd->periods, mpdin->active_period_index);
                new_period = gf_list_get(new_mpd->periods, mpdin->active_period_index);
                if (!new_period) {
                    GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: missing period\n"));
                    gf_mpd_del(new_mpd);
                    return GF_NON_COMPLIANT_BITSTREAM;
                }

                if (gf_list_count(period->representations) != gf_list_count(new_period->representations)) {
                    GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: missing representation\n"));
                    gf_mpd_del(new_mpd);
                    return GF_NON_COMPLIANT_BITSTREAM;
                }

                rep = gf_list_get(period->representations, mpdin->active_rep_index);
                GF_MPD_SegmentInfo *info1 = gf_list_get(rep->segments, mpdin->download_segment_index - 1);

                for (rep_idx = 0; rep_idx<gf_list_count(period->representations); rep_idx++) {
                    rep = gf_list_get(period->representations, rep_idx);
                    new_rep = gf_list_get(new_period->representations, rep_idx);
                    if (!new_rep) {
                        GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: missing representation in period\n"));
                        gf_mpd_del(new_mpd);
                        return GF_NON_COMPLIANT_BITSTREAM;
                    }
                    /*merge init segment*/
                    if (new_rep->init_url) {
                        seg_found = 0;

                        if (!strcmp(new_rep->init_url, rep->init_url)) {
                            seg_found = 1;
                        } else {
                            for (j=0; j<gf_list_count(rep->segments); j++) {
                                GF_MPD_SegmentInfo *seg = gf_list_get(rep->segments, j);
                                if (!strcmp(new_rep->init_url, seg->url)) {
                                    seg_found = 1;
                                    break;
                                }
                            }
                        }
                        /*remove from new list and push to old one*/
                        if (!seg_found) {
                            GF_MPD_SegmentInfo *new_seg;
                            GF_SAFEALLOC(new_seg, GF_MPD_SegmentInfo);
                            new_seg->url = gf_strdup(new_rep->init_url);
                            new_seg->use_byterange = new_rep->init_use_range;
                            new_seg->byterange_start = new_rep->init_byterange_start;
                            new_seg->byterange_end = new_rep->init_byterange_end;
                            gf_list_add(rep->segments, new_seg);
                            GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] Representation #%d: Adding new segment from initialization segment %s\n", rep_idx+1, new_seg->url));
                        }
                    }

                    /*merge segment list*/
                    for (i=0; i<gf_list_count(new_rep->segments); i++) {
                        GF_MPD_SegmentInfo *new_seg = gf_list_get(new_rep->segments, i);
                        assert( new_seg );
                        assert( new_seg->url);
                        seg_found = 0;
                        for (j=0; j<gf_list_count(rep->segments); j++) {
                            GF_MPD_SegmentInfo *seg = gf_list_get(rep->segments, j);
                            assert( seg );
                            assert( seg->url);
                            if (!strcmp(new_seg->url, seg->url)) {
                                seg_found = 1;
                                break;
                            }
                        }
                        /*remove from new list and push to old one*/
                        if (!seg_found) {
                            gf_list_rem(new_rep->segments, i);
                            gf_list_add(rep->segments, new_seg);
                            GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] Representation #%d: Adding new segment %s\n", rep_idx+1, new_seg->url));
                            i--;
                        }
                    }
                    /*swap lists*/
                    {
                        GF_MPD_SegmentInfo *info2;
                        segs = new_rep->segments;
                        new_rep->segments = rep->segments;
                        rep->segments = segs;
                        info2 = gf_list_get(rep->segments, mpdin->download_segment_index);
                    }
                }

                /*swap representations - we don't need to update download_segment_index as it still points to the right entry in the merged list*/
                gf_mpd_del(mpdin->mpd);
                mpdin->mpd = new_mpd;
                mpdin->last_update_time = gf_sys_clock();


            }
        }
    }

    return GF_OK;
}

/*!
 * Download a file with possible retry if GF_IP_CONNECTION_FAILURE|GF_IP_NETWORK_FAILURE
 * (I discovered that with my WIFI connection, I had many issues with BFM-TV downloads)
 * Similar to gf_term_download_new() and gf_dm_sess_process().
 * Parameters are identical to the ones of gf_term_download_new.
 * \see gf_term_download_new()
 */
GF_Err MPD_downloadWithRetry( GF_ClientService * service, GF_DownloadSession ** sess, const char *url, gf_dm_user_io user_io,  void *usr_cbk)
{
    GF_Err e;
    if (*sess) {
        gf_term_download_del(*sess);
        *sess = NULL;
    }
    GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] Downloading %s...\n", url));
    *sess = gf_term_download_new(service, url, GF_NETIO_SESSION_NOT_THREADED, user_io, usr_cbk);
    if (!(*sess))
        return GF_OUT_OF_MEM;
    e = gf_dm_sess_process(*sess);
    switch (e) {
    case GF_IP_CONNECTION_FAILURE:
    case GF_IP_NETWORK_FAILURE:
    {
        gf_term_download_del(*sess);
        GF_LOG(GF_LOG_WARNING, GF_LOG_MODULE,
               ("[MPD_IN] failed to download, retrying once with %s...\n", url));
        *sess = gf_term_download_new(service, url, GF_NETIO_SESSION_NOT_THREADED, user_io, usr_cbk);
        if (!(*sess))
            return GF_OUT_OF_MEM;
        e = gf_dm_sess_process(*sess);
        if (e != GF_OK) {
            GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE,
                   ("[MPD_IN] two consecutive failures, aborting the download %s.\n", url));
        }
        return e;
    }
    case GF_OK:
        GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] OK, Download %s complete\n", url));
        return e;
    default:
        GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] FAILED to download %s = %s...\n", url, gf_error_to_string(e)));
        return e;
    }
}


static GF_Err MPD_DownloadInitSegment(GF_MPD_In *mpdin, GF_MPD_Period *period)
{
    GF_Err e;
    char *base_init_url;
    char * url_to_dl;
    GF_MPD_Representation *rep;
    /* This variable is 0 if there is a initURL, the index of first segment downloaded otherwise */
    u32 firstSegment = 0;
    if (!mpdin || !period)
        return GF_BAD_PARAM;
    rep = gf_list_get(period->representations, mpdin->active_rep_index);
    if (!rep) {
        GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("Unable to find any representation, aborting.\n"));
        return GF_IO_ERR;
    }
    if (!rep->init_url) {
        GF_MPD_SegmentInfo * seg = gf_list_get(rep->segments, 0);
        /* No init URL provided, we have to download the first segment then */
        if (!seg->url)
            return GF_BAD_PARAM;
	firstSegment = 1;
        url_to_dl = seg->url;
    } else {
        url_to_dl = rep->init_url;
    }
    if (rep->default_base_url) {
        base_init_url = gf_url_concatenate(rep->default_base_url, url_to_dl);
    } else {
        base_init_url = gf_strdup(url_to_dl);
    }
    e = MPD_downloadWithRetry(mpdin->service, &(mpdin->seg_dnload), base_init_url, MPD_NetIO_Segment, mpdin);
    if (e == GF_URL_ERROR && !base_init_url) { /* We have a 404 and started with segments */
        /* It is possible that the first segment has been deleted while we made the first request...
         * so we try with the next segment on some M3U8 servers */
        GF_MPD_SegmentInfo * seg = gf_list_get(rep->segments, 1);
        gf_free(base_init_url);
        if (!seg || !seg->url) {
            return e;
        }
        if (rep->default_base_url) {
            base_init_url = gf_url_concatenate(rep->default_base_url, seg->url);
        } else {
            base_init_url = gf_strdup(seg->url);
        }
        GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("Download of first segment failed... retrying with second one : %s\n", base_init_url));
	firstSegment = 2;
        e = MPD_downloadWithRetry(mpdin->service, &(mpdin->seg_dnload), base_init_url, MPD_NetIO_Segment, mpdin);
    } /* end of 404 */

    if (e!= GF_OK && !mpdin->segment_must_be_streamed) {
        gf_free(base_init_url);
        return e;
    } else {
        GF_MPD_Representation *rep = gf_list_get(period->representations, mpdin->active_rep_index);

        {
            u32 count = gf_list_count(rep->segments) + 1;
            if (count < mpdin->max_cached) {
                if (count < 1) {
                    GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] 0 representations, aborting\n"));
                    gf_free(base_init_url);
                    return GF_BAD_PARAM;
                }
                GF_LOG(GF_LOG_INFO, GF_LOG_MODULE, ("[MPD_IN] Resizing to %u max_cached elements instead of %u.\n", count, mpdin->max_cached));
                /* OK, we have a problem, it may ends download */
                mpdin->max_cached = count;
            }
        }

        e = gf_dm_sess_process(mpdin->seg_dnload);
        {
            /* Mime-Type check */
            const char * mime = gf_dm_sess_mime_type(mpdin->seg_dnload);
            if (!mime || (stricmp(mime, rep->mime) && stricmp("video/mpeg", mime))) {
                GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Mime '%s' is not correct for '%s', it should be '%s'\n", mime, base_init_url, rep->mime));
                gf_free(base_init_url);
                base_init_url = NULL;
                return GF_BAD_PARAM;
            }
        }
	if (mpdin->segment_must_be_streamed ){
	    mpdin->seg_local_url = gf_dm_sess_get_resource_name(mpdin->seg_dnload);
	    e = GF_OK;
	} else {
	  mpdin->seg_local_url = rep->init_use_range ? gf_cache_get_cache_filename_range(mpdin->seg_dnload, rep->init_byterange_start, rep->init_byterange_end )  : gf_dm_sess_get_cache_name(mpdin->seg_dnload);
	}
        if ((e!=GF_OK) || !mpdin->seg_local_url) {
            GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error with initialization segment: download result:%s, cache file:%s\n", gf_error_to_string(e), mpdin->seg_local_url));
            gf_free(base_init_url);
            return GF_BAD_PARAM;
        } else {
            gf_mx_p(mpdin->dl_mutex);
            assert(!mpdin->nb_cached);
            mpdin->cached[0].cache = gf_strdup(mpdin->seg_local_url);
            mpdin->cached[0].url = gf_strdup(gf_dm_sess_get_resource_name(mpdin->seg_dnload));
            mpdin->nb_cached = 1;
	    mpdin->download_segment_index = firstSegment;
            GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] Adding initialization segment %s to cache: %s\n", mpdin->seg_local_url, mpdin->cached[0].url ));
            gf_mx_v(mpdin->dl_mutex);
	    gf_free(base_init_url);
#ifndef DONT_USE_TERMINAL_MODULE_API
            GF_NetworkCommand com;
            com.base.command_type = GF_NET_SERVICE_INFO;
            gf_term_on_command(mpdin->service, &com, GF_OK);
#endif
            return GF_OK;
        }
    }
}

static u32 download_segments(void *par)
{
    GF_Err e;
    GF_MPD_In *mpdin = (GF_MPD_In*) par;
    GF_MPD_Period *period;
    GF_MPD_Representation *rep;
    GF_MPD_SegmentInfo *seg;
    char *new_base_seg_url;<commit_msg>Fixed compilation<commit_after>/*
 *			GPAC - Multimedia Framework C SDK
 *
 *			Authors: <NAME>
 *			Copyright (c) Telecom ParisTech 2010-
 *					All rights reserved
 *
 *  This file is part of GPAC / 3GPP/MPEG Media Presentation Description input module
 *
 *  GPAC is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU Lesser General Public License as published by
 *  the Free Software Foundation; either version 2, or (at your option)
 *  any later version.
 *
 *  GPAC is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; see the file COPYING.  If not, write to
 *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 */

#include <gpac/modules/service.h>
#include <gpac/internal/terminal_dev.h>
#include "mpd.h"
#include <gpac/thread.h>
#include <gpac/network.h>
#include <gpac/crypt.h>
#include "m3u8.h"

/*!
 * All the possible Mime-types for MPD files
 */
static const char * MPD_MIME_TYPES[] = { "video/vnd.3gpp.mpd", "audio/vnd.3gpp.mpd", NULL };

/*!
 * All the possible Mime-types for M3U8 files
 */
static const char * M3U8_MIME_TYPES[] = { "video/x-mpegurl", "audio/x-mpegurl", "application/x-mpegurl", "application/vnd.apple.mpegurl", NULL};


typedef struct
{
    char *cache;
    char *url;
} segment_cache_entry;

typedef struct __mpd_module {
    /* GPAC Service object (i.e. how this module is seen by the terminal)*/
    GF_ClientService *service;
    /* URL to which this service is connected
       Used to detect when audio service connection request is made on the same URL as video */
    char *url;
    Bool is_service_connected;
    /* number of time the service has been connected */
    u32 nb_service_connections;
    u32 nb_connected_channels;
    u32 nb_playing_or_paused_channels;

    /* Download of the MPD */
    volatile Bool in_download;

    GF_DownloadSession *mpd_dnload;
    char * urlToDeleteNext;
    volatile u32 max_cached, nb_cached;
    u32 option_max_cached;
    segment_cache_entry *cached;

    /* MPD and active informations */
    GF_MPD *mpd;
    u32 active_period_index;
    u32 active_rep_index;
    u32 download_segment_index;

    /* playback status */
    Double playback_speed;
    Double playback_start_range, playback_end_range;

    /* For Segment downloads */
    GF_DownloadSession *seg_dnload;
    const char *seg_local_url;
    GF_Thread *dl_thread;
    GF_Mutex *dl_mutex;

    /* 0 not started, 1 download in progress */
    Bool dl_status;
    Bool dl_stop_request;

    /* Service really managing the segments */
    GF_InputService *seg_ifce;

    u32 reload_count;
    u32 last_update_time;

    u32 nb_segs_done;
    Bool auto_switch;
    u8 lastMPDSignature[20];
    Bool segment_must_be_streamed;
} GF_MPD_In;

static void dumpStatus( GF_MPD_In *mpdin) {
    u32 i;
    for (i = 0 ; i < mpdin->nb_cached; i++) {
        printf("\t[%u] - %s\t:\t%s\n", i, mpdin->cached[i].cache, mpdin->cached[i].url);
    }
    printf("\t***********\n\n");
}


GF_Err m3u8_to_mpd(GF_MPD_In *mpdin, const char *m3u8_file, const char *url)
{
    GF_Err e;
    u32 i, count;
    Double update_interval;
    VariantPlaylist * pl = NULL;
    Program *prog;
    PlaylistElement *pe;
    FILE *fmpd;
    Bool is_end;

    e = parse_root_playlist(m3u8_file, &pl, url);
    if (e) {
        GF_LOG(GF_LOG_WARNING, GF_LOG_CONTAINER, ("[M3U8] Failed to parse root playlist '%s', error = %s\n", m3u8_file, gf_error_to_string(e)));
        return e;
    }
    gf_delete_file((char *)m3u8_file);
    is_end = !pl->playlistNeedsRefresh;
    i=0;
    assert( pl );
    assert( pl->programs );
    while ((prog = gf_list_enum(pl->programs, &i))) {
        u32 j=0;
        while (NULL != (pe = gf_list_enum(prog->bitrates, &j))) {
            if (pe->url && strstr(pe->url, ".m3u8")) {
                char *suburl = gf_url_concatenate(url, pe->url);
                if (!strcmp(url, suburl)) {
                    gf_free(suburl);
                    GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] Not downloading, programs are identical for %s...\n", pe->url));
                    continue;
                }
                {
                    GF_DownloadSession *sess = gf_term_download_new(mpdin->service, suburl, GF_NETIO_SESSION_NOT_THREADED, NULL, NULL);
                    if (!sess) {
                        gf_free(suburl);
                        break;
                    }
                    e = gf_dm_sess_process(sess);
                    if (e==GF_OK) {
                        e = parse_sub_playlist(gf_dm_sess_get_cache_name(sess), &pl, suburl, prog, pe);
                    }
                    gf_term_download_del(sess);
                    gf_free(suburl);
                }
            }
        }
    }

    update_interval = 0;
    prog = gf_list_get(pl->programs, 0);
    assert( prog );
    pe = gf_list_last(prog->bitrates);
    assert( pe );
    /*update interval is set to the duration of the last media file with rules defined in http live streaming RFC section 6.3.4*/
    switch (mpdin->reload_count) {
    case 0:
        update_interval = pe->durationInfo;
        break;
    case 1:
        update_interval = pe->durationInfo/2;
        break;
    case 2:
        update_interval = 3*(pe->durationInfo/2);
        break;
    default:
        update_interval = 3*(pe->durationInfo);
        break;
    }
    if (is_end || (pe->elementType == TYPE_PLAYLIST) && pe->element.playlist.is_ended) {
        update_interval = 0;
        GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] NO NEED to refresh playlist !\n"));
    } else {
        GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] Playlist will be refresh every %g seconds, len=%d\n", update_interval, pe->durationInfo));
    }

    fmpd = fopen(m3u8_file, "wt");
    fprintf(fmpd, "<MPD type=\"Live\" xmlns=\"urn:3GPP:ns:PSS:AdaptiveHTTPStreamingMPD:2009\" ");
    if (update_interval) fprintf(fmpd, "minimumUpdatePeriodMPD=\"PT%02.2gS\"", update_interval);
    fprintf(fmpd, ">\n");

    fprintf(fmpd, " <ProgramInformation moreInformationURL=\"http://gpac.sourceforge.net\">\n");
    {
        char * title = pe->title;
        if (!title || strlen(title) < 2)
            title = pe->url;
        fprintf(fmpd, "  <Title>%s</Title>\n", title );
    }
    fprintf(fmpd, "  <Copyright>Generated from URL %s</Copyright>\n", url );
    fprintf(fmpd, "  <Source>Generated by GPAC %s from %s</Source>\n", GPAC_FULL_VERSION, url);

    fprintf(fmpd, " </ProgramInformation>\n");
    fprintf(fmpd, " <Period start=\"PT0S\">\n");

    count = gf_list_count(pl->programs);
    for (i=0; i<count; i++) {
        u32 j, count2;
        Program *prog = gf_list_get(pl->programs, i);
        count2 = gf_list_count(prog->bitrates);
        for (j = 0; j<count2; j++) {
            PlaylistElement *pe = gf_list_get(prog->bitrates, j);
            if (pe->elementType == TYPE_PLAYLIST) {
                u32 k, count3;
                fprintf(fmpd, "  <Representation mimeType=\"video/mp2t\">\n");
                fprintf(fmpd, "   <SegmentInfo duration=\"PT%dS\" baseURL=\"%s\">\n", pe->durationInfo, url);
                count3 = gf_list_count(pe->element.playlist.elements);
                update_interval = (count3 - 1) * pe->durationInfo * 1000;
                for (k=0; k<count3; k++) {
                    PlaylistElement *elt = gf_list_get(pe->element.playlist.elements, k);
                    fprintf(fmpd, "    <Url sourceURL=\"%s\"/>\n", elt->url);
                }
                fprintf(fmpd, "   </SegmentInfo>\n");
                fprintf(fmpd, "  </Representation>\n");
            } else if (pe->elementType == TYPE_STREAM) {
                fprintf(stdout, "NOT SUPPORTED: M3U8 Stream\n");
            }
        }
    }
    fprintf(fmpd, " </Period>\n");
    fprintf(fmpd, "</MPD>");
    fclose(fmpd);
    variant_playlist_del(pl);
    return GF_OK;
}

static Bool MPD_CheckRootType(const char *local_url)
{
    if (local_url) {
        char *rtype = gf_xml_get_root_type(local_url, NULL);
        if (rtype) {
            Bool handled = 0;
            if (!strcmp(rtype, "MPD")) {
                handled = 1;
            }
            gf_free(rtype);
            return handled;
        }
    }
    return 0;
}

/**
 * NET IO for MPD, we don't need this anymore since mime-type can be given by session
 */
void MPD_NetIO_Segment(void *cbk, GF_NETIO_Parameter *param)
{
    GF_Err e;
    GF_MPD_In *mpdin = (GF_MPD_In*) cbk;

    /*handle service message*/
    gf_term_download_update_stats(mpdin->seg_dnload);
    e = param->error;
    if (param->msg_type == GF_NETIO_PARSE_REPLY){
	if (! gf_dm_sess_can_be_cached_on_disk(mpdin->seg_dnload)){
	    GF_LOG(GF_LOG_INFO, GF_LOG_MODULE,
		   ("[MPD_IN] Segment %s cannot be cached on disk, will use direct streaming\n", gf_dm_sess_get_resource_name(mpdin->seg_dnload)));
	    mpdin->segment_must_be_streamed = 1;
	    gf_dm_sess_abort(mpdin->seg_dnload);
	} else {
	    mpdin->segment_must_be_streamed = 0;
	}
    }
}

/*!
 * Returns true if given mime type is a MPD file
 * \param mime the mime-type to check
 * \return true if mime-type if MPD-OK
 */
static Bool MPD_is_MPD_mime(const char * mime) {
    u32 i;
    if (!mime)
        return 0;
    i = 0;
    while (MPD_MIME_TYPES[i]) {
        if ( !stricmp(mime, MPD_MIME_TYPES[i++]))
            return 1;
    }
    return 0;
}

/*!
 * Returns true if mime type is an M3U8 mime-type
 * \param mime The mime-type to check
 * \return true if mime-type is OK for M3U8
 */
static Bool MPD_isM3U8_mime(const char * mime) {
    u32 i;
    if (!mime)
        return 0;
    i = 0;
    while (M3U8_MIME_TYPES[i]) {
        if ( !stricmp(mime, M3U8_MIME_TYPES[i++]))
            return 1;
    }
    return 0;
}

static Bool MPD_segments_are_equals(GF_MPD_SegmentInfo * info1, GF_MPD_SegmentInfo * info2) {
    assert( info1 );
    assert( info2 );
    return info1->byterange_end == info2->byterange_end &&
           info1->byterange_start == info2->byterange_start &&
           info1->use_byterange == info2->use_byterange &&
           (info1->url == info2->url || !strcmp(info1->url, info2->url));
}

static GF_Err MPD_UpdatePlaylist(GF_MPD_In *mpdin)
{
    GF_Err e;
    u32 i, j, rep_idx;
    Bool seg_found = 0;
    GF_DOMParser *mpd_parser;
    GF_MPD *new_mpd;
    GF_MPD_Period *period, *new_period;
    GF_MPD_Representation *rep, *new_rep;
    GF_List *segs;
    const char *local_url, * url;
    Bool is_m3u8 = 0;
    u32 oldUpdateTime = mpdin->mpd->min_update_time;
    /*reset update time - if any error occurs, we will no longer attempt to update the MPD*/
    mpdin->mpd->min_update_time = 0;

    if (!mpdin->mpd_dnload) {
        GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: missing downloader\n"));
        return GF_BAD_PARAM;
    }

    local_url = gf_dm_sess_get_cache_name(mpdin->mpd_dnload);
    if (!local_url) {
        GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: wrong cache file %s\n", local_url));
        return GF_IO_ERR;
    }
    gf_delete_file(local_url);
    gf_dm_sess_reset(mpdin->mpd_dnload);
    url = gf_dm_sess_get_resource_name(mpdin->mpd_dnload);
    GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] Updating Playlist %s...\n", url));
    e = gf_dm_sess_process(mpdin->mpd_dnload);
    if (e!=GF_OK) {
        GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: download problem %s for MPD file\n", gf_error_to_string(e)));
        return gf_dm_sess_last_error(mpdin->mpd_dnload);
    } else {
        GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] Playlist %s updated with success\n", url));
    }
    {
        const char * mime = gf_dm_sess_mime_type(mpdin->mpd_dnload);
        /* Some servers, for instance http://tv.freebox.fr, serve m3u8 as text/plain */
        if (MPD_isM3U8_mime(mime) || strstr(url, ".m3u8")) {
            m3u8_to_mpd(mpdin, local_url, url );
        } else if (!MPD_is_MPD_mime(mime)) {
            GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] mime '%s' should be m3u8 or mpd\n", mime));
            gf_term_on_connect(mpdin->service, NULL, GF_BAD_PARAM);
            return GF_BAD_PARAM;
        }
    }

    if (!MPD_CheckRootType(local_url)) {
        GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: MPD file type is not correct %s\n", local_url));
        return GF_NON_COMPLIANT_BITSTREAM;
    }
    {
        u8 signature[sizeof(mpdin->lastMPDSignature)];
        if (gf_sha1_file( local_url, signature)) {
            GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] : cannot SHA1 file %s\n", local_url));
        } else {
            if (! memcmp( signature, mpdin->lastMPDSignature, sizeof(mpdin->lastMPDSignature))) {
                GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] MPD file did not change\n"));
                mpdin->reload_count++;
                mpdin->mpd->min_update_time = oldUpdateTime;
            } else {
				GF_MPD_SegmentInfo *info1;
                mpdin->reload_count = 0;
                memccpy(mpdin->lastMPDSignature, signature, sizeof(char), sizeof(mpdin->lastMPDSignature));

                /* It means we have to reparse the file ... */
                /* parse the MPD */
                mpd_parser = gf_xml_dom_new();
                e = gf_xml_dom_parse(mpd_parser, local_url, NULL, NULL);
                if (e != GF_OK) {
                    gf_xml_dom_del(mpd_parser);
                    GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: error in XML parsing %s\n", gf_error_to_string(e)));
                    return GF_NON_COMPLIANT_BITSTREAM;
                }
                new_mpd = gf_mpd_new();
                e = gf_mpd_init_from_dom(gf_xml_dom_get_root(mpd_parser), new_mpd);
                gf_xml_dom_del(mpd_parser);
                if (e) {
                    GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: error in MPD creation %s\n", gf_error_to_string(e)));
                    gf_mpd_del(new_mpd);
                    return GF_NON_COMPLIANT_BITSTREAM;
                }

                /*TODO - check periods are the same*/
                period = gf_list_get(mpdin->mpd->periods, mpdin->active_period_index);
                new_period = gf_list_get(new_mpd->periods, mpdin->active_period_index);
                if (!new_period) {
                    GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: missing period\n"));
                    gf_mpd_del(new_mpd);
                    return GF_NON_COMPLIANT_BITSTREAM;
                }

                if (gf_list_count(period->representations) != gf_list_count(new_period->representations)) {
                    GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: missing representation\n"));
                    gf_mpd_del(new_mpd);
                    return GF_NON_COMPLIANT_BITSTREAM;
                }

                rep = gf_list_get(period->representations, mpdin->active_rep_index);
                info1 = gf_list_get(rep->segments, mpdin->download_segment_index - 1);

                for (rep_idx = 0; rep_idx<gf_list_count(period->representations); rep_idx++) {
                    rep = gf_list_get(period->representations, rep_idx);
                    new_rep = gf_list_get(new_period->representations, rep_idx);
                    if (!new_rep) {
                        GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error - cannot update playlist: missing representation in period\n"));
                        gf_mpd_del(new_mpd);
                        return GF_NON_COMPLIANT_BITSTREAM;
                    }
                    /*merge init segment*/
                    if (new_rep->init_url) {
                        seg_found = 0;

                        if (!strcmp(new_rep->init_url, rep->init_url)) {
                            seg_found = 1;
                        } else {
                            for (j=0; j<gf_list_count(rep->segments); j++) {
                                GF_MPD_SegmentInfo *seg = gf_list_get(rep->segments, j);
                                if (!strcmp(new_rep->init_url, seg->url)) {
                                    seg_found = 1;
                                    break;
                                }
                            }
                        }
                        /*remove from new list and push to old one*/
                        if (!seg_found) {
                            GF_MPD_SegmentInfo *new_seg;
                            GF_SAFEALLOC(new_seg, GF_MPD_SegmentInfo);
                            new_seg->url = gf_strdup(new_rep->init_url);
                            new_seg->use_byterange = new_rep->init_use_range;
                            new_seg->byterange_start = new_rep->init_byterange_start;
                            new_seg->byterange_end = new_rep->init_byterange_end;
                            gf_list_add(rep->segments, new_seg);
                            GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] Representation #%d: Adding new segment from initialization segment %s\n", rep_idx+1, new_seg->url));
                        }
                    }

                    /*merge segment list*/
                    for (i=0; i<gf_list_count(new_rep->segments); i++) {
                        GF_MPD_SegmentInfo *new_seg = gf_list_get(new_rep->segments, i);
                        assert( new_seg );
                        assert( new_seg->url);
                        seg_found = 0;
                        for (j=0; j<gf_list_count(rep->segments); j++) {
                            GF_MPD_SegmentInfo *seg = gf_list_get(rep->segments, j);
                            assert( seg );
                            assert( seg->url);
                            if (!strcmp(new_seg->url, seg->url)) {
                                seg_found = 1;
                                break;
                            }
                        }
                        /*remove from new list and push to old one*/
                        if (!seg_found) {
                            gf_list_rem(new_rep->segments, i);
                            gf_list_add(rep->segments, new_seg);
                            GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] Representation #%d: Adding new segment %s\n", rep_idx+1, new_seg->url));
                            i--;
                        }
                    }
                    /*swap lists*/
                    {
                        GF_MPD_SegmentInfo *info2;
                        segs = new_rep->segments;
                        new_rep->segments = rep->segments;
                        rep->segments = segs;
                        info2 = gf_list_get(rep->segments, mpdin->download_segment_index);
                    }
                }

                /*swap representations - we don't need to update download_segment_index as it still points to the right entry in the merged list*/
                gf_mpd_del(mpdin->mpd);
                mpdin->mpd = new_mpd;
                mpdin->last_update_time = gf_sys_clock();


            }
        }
    }

    return GF_OK;
}

/*!
 * Download a file with possible retry if GF_IP_CONNECTION_FAILURE|GF_IP_NETWORK_FAILURE
 * (I discovered that with my WIFI connection, I had many issues with BFM-TV downloads)
 * Similar to gf_term_download_new() and gf_dm_sess_process().
 * Parameters are identical to the ones of gf_term_download_new.
 * \see gf_term_download_new()
 */
GF_Err MPD_downloadWithRetry( GF_ClientService * service, GF_DownloadSession ** sess, const char *url, gf_dm_user_io user_io,  void *usr_cbk)
{
    GF_Err e;
    if (*sess) {
        gf_term_download_del(*sess);
        *sess = NULL;
    }
    GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] Downloading %s...\n", url));
    *sess = gf_term_download_new(service, url, GF_NETIO_SESSION_NOT_THREADED, user_io, usr_cbk);
    if (!(*sess))
        return GF_OUT_OF_MEM;
    e = gf_dm_sess_process(*sess);
    switch (e) {
    case GF_IP_CONNECTION_FAILURE:
    case GF_IP_NETWORK_FAILURE:
    {
        gf_term_download_del(*sess);
        GF_LOG(GF_LOG_WARNING, GF_LOG_MODULE,
               ("[MPD_IN] failed to download, retrying once with %s...\n", url));
        *sess = gf_term_download_new(service, url, GF_NETIO_SESSION_NOT_THREADED, user_io, usr_cbk);
        if (!(*sess))
            return GF_OUT_OF_MEM;
        e = gf_dm_sess_process(*sess);
        if (e != GF_OK) {
            GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE,
                   ("[MPD_IN] two consecutive failures, aborting the download %s.\n", url));
        }
        return e;
    }
    case GF_OK:
        GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] OK, Download %s complete\n", url));
        return e;
    default:
        GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] FAILED to download %s = %s...\n", url, gf_error_to_string(e)));
        return e;
    }
}


static GF_Err MPD_DownloadInitSegment(GF_MPD_In *mpdin, GF_MPD_Period *period)
{
    GF_Err e;
    char *base_init_url;
    char * url_to_dl;
    GF_MPD_Representation *rep;
    /* This variable is 0 if there is a initURL, the index of first segment downloaded otherwise */
    u32 firstSegment = 0;
    if (!mpdin || !period)
        return GF_BAD_PARAM;
    rep = gf_list_get(period->representations, mpdin->active_rep_index);
    if (!rep) {
        GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("Unable to find any representation, aborting.\n"));
        return GF_IO_ERR;
    }
    if (!rep->init_url) {
        GF_MPD_SegmentInfo * seg = gf_list_get(rep->segments, 0);
        /* No init URL provided, we have to download the first segment then */
        if (!seg->url)
            return GF_BAD_PARAM;
	firstSegment = 1;
        url_to_dl = seg->url;
    } else {
        url_to_dl = rep->init_url;
    }
    if (rep->default_base_url) {
        base_init_url = gf_url_concatenate(rep->default_base_url, url_to_dl);
    } else {
        base_init_url = gf_strdup(url_to_dl);
    }
    e = MPD_downloadWithRetry(mpdin->service, &(mpdin->seg_dnload), base_init_url, MPD_NetIO_Segment, mpdin);
    if (e == GF_URL_ERROR && !base_init_url) { /* We have a 404 and started with segments */
        /* It is possible that the first segment has been deleted while we made the first request...
         * so we try with the next segment on some M3U8 servers */
        GF_MPD_SegmentInfo * seg = gf_list_get(rep->segments, 1);
        gf_free(base_init_url);
        if (!seg || !seg->url) {
            return e;
        }
        if (rep->default_base_url) {
            base_init_url = gf_url_concatenate(rep->default_base_url, seg->url);
        } else {
            base_init_url = gf_strdup(seg->url);
        }
        GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("Download of first segment failed... retrying with second one : %s\n", base_init_url));
	firstSegment = 2;
        e = MPD_downloadWithRetry(mpdin->service, &(mpdin->seg_dnload), base_init_url, MPD_NetIO_Segment, mpdin);
    } /* end of 404 */

    if (e!= GF_OK && !mpdin->segment_must_be_streamed) {
        gf_free(base_init_url);
        return e;
    } else {
		const char * mime;
        GF_MPD_Representation *rep = gf_list_get(period->representations, mpdin->active_rep_index);
        u32 count = gf_list_count(rep->segments) + 1;
        if (count < mpdin->max_cached) {
            if (count < 1) {
                GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] 0 representations, aborting\n"));
                gf_free(base_init_url);
                return GF_BAD_PARAM;
            }
            GF_LOG(GF_LOG_INFO, GF_LOG_MODULE, ("[MPD_IN] Resizing to %u max_cached elements instead of %u.\n", count, mpdin->max_cached));
            /* OK, we have a problem, it may ends download */
            mpdin->max_cached = count;
        }
        e = gf_dm_sess_process(mpdin->seg_dnload);
        /* Mime-Type check */
        mime = gf_dm_sess_mime_type(mpdin->seg_dnload);
        if (!mime || (stricmp(mime, rep->mime) && stricmp("video/mpeg", mime))) {
            GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Mime '%s' is not correct for '%s', it should be '%s'\n", mime, base_init_url, rep->mime));
            gf_free(base_init_url);
            base_init_url = NULL;
            return GF_BAD_PARAM;
        }
		if (mpdin->segment_must_be_streamed ){
			mpdin->seg_local_url = gf_dm_sess_get_resource_name(mpdin->seg_dnload);
			e = GF_OK;
		} else {
		  mpdin->seg_local_url = rep->init_use_range ? gf_cache_get_cache_filename_range(mpdin->seg_dnload, rep->init_byterange_start, rep->init_byterange_end )  : gf_dm_sess_get_cache_name(mpdin->seg_dnload);
		}

		if ((e!=GF_OK) || !mpdin->seg_local_url) {
            GF_LOG(GF_LOG_ERROR, GF_LOG_MODULE, ("[MPD_IN] Error with initialization segment: download result:%s, cache file:%s\n", gf_error_to_string(e), mpdin->seg_local_url));
            gf_free(base_init_url);
            return GF_BAD_PARAM;
        } else {
#ifndef DONT_USE_TERMINAL_MODULE_API
            GF_NetworkCommand com;
#endif
			gf_mx_p(mpdin->dl_mutex);
            assert(!mpdin->nb_cached);
            mpdin->cached[0].cache = gf_strdup(mpdin->seg_local_url);
            mpdin->cached[0].url = gf_strdup(gf_dm_sess_get_resource_name(mpdin->seg_dnload));
            mpdin->nb_cached = 1;
			mpdin->download_segment_index = firstSegment;
            GF_LOG(GF_LOG_DEBUG, GF_LOG_MODULE, ("[MPD_IN] Adding initialization segment %s to cache: %s\n", mpdin->seg_local_url, mpdin->cached[0].url ));
            gf_mx_v(mpdin->dl_mutex);
			gf_free(base_init_url);
#ifndef DONT_USE_TERMINAL_MODULE_API
            com.base.command_type = GF_NET_SERVICE_INFO;
            gf_term_on_command(mpdin->service, &com, GF_OK);
#endif
            return GF_OK;
        }
    }
}

static u32 download_segments(void *par)
{
    GF_Err e;
    GF_MPD_In *mpdin = (GF_MPD_In*) par;
    GF_MPD_Period *period;
    GF_MPD_Representation *rep;
    GF_MPD_SegmentInfo *seg;
    char *new_base_seg_url;
{'fraction_non_alphanumeric': 0.1138380608170757, 'fraction_numerical': 0.00853407005128011, 'mean_word_length': 2.15453948546529, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 405, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 92, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4404561', 'n_tokens_mistral': 18681, 'n_tokens_neox': 17715, 'n_words': 3823}