<commit_before>/*
This function generates a manifest (.ism) from the MP4/M4A files in the asset. It makes this file primary.
This manifest is needed to stream MP4 file(s) with Azure Media Services.

Caution : such assets are not guaranteed to work with Dynamic Packaging.

Note : this function makes  guesses to determine the files for the video tracks and audio tracks.
These guesses can be wrong. Please check the SMIL generated data for your scenario and your source assets.

Input:
{
    "assetId" : "nb:cid:UUID:88432c30-cb4a-4496-88c2-b2a05ce9033b", // Mandatory, Id of the asset
    "fileName" : "manifest.ism", // Optional. file name of the manifest to create
}

Output:
{
    "fileName" : "manifest.ism" // The name of the manifest file created
    "manifestContent" : "" // the SMIL data created as an asset file    
}
*/

#r "Newtonsoft.Json"
#r "Microsoft.WindowsAzure.Storage"
#r "System.Xml.Linq"
#load "../Shared/mediaServicesHelpers.csx"
#load "../Shared/copyBlobHelpers.csx"

using System;
using System.Net;
using System.Net.Http;
using Newtonsoft.Json;
using Microsoft.WindowsAzure.MediaServices.Client;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.IO;
using System.Web;
using Microsoft.Azure;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using Microsoft.WindowsAzure.Storage.Auth;
using Microsoft.Azure.WebJobs;
using System.Xml.Linq;
using Microsoft.IdentityModel.Clients.ActiveDirectory;

// Read values from the App.config file.
static string _storageAccountName = Environment.GetEnvironmentVariable("MediaServicesStorageAccountName");
static string _storageAccountKey = Environment.GetEnvironmentVariable("MediaServicesStorageAccountKey");

static readonly string _AADTenantDomain = Environment.GetEnvironmentVariable("AMSAADTenantDomain");
static readonly string _RESTAPIEndpoint = Environment.GetEnvironmentVariable("AMSRESTAPIEndpoint");

static readonly string _mediaservicesClientId = Environment.GetEnvironmentVariable("AMSClientId");
static readonly string _mediaservicesClientSecret = Environment.GetEnvironmentVariable("AMSClientSecret");

// Field for service context.
private static CloudMediaContext _context = null;
private static CloudStorageAccount _destinationStorageAccount = null;


public static async Task<object> Run(HttpRequestMessage req, TraceWriter log, Microsoft.Azure.WebJobs.ExecutionContext execContext)
{
    log.Info($"Webhook was triggered!");

    // Init variables

    string jsonContent = await req.Content.ReadAsStringAsync();
    dynamic data = JsonConvert.DeserializeObject(jsonContent);

    string fileName = null;
    var manifestInfo = new ManifestGenerated();

    log.Info(jsonContent);

    if (data.assetId == null)
    {
        return req.CreateResponse(HttpStatusCode.BadRequest, new
        {
            error = "Please pass asset ID in the input object (assetId)"
        });
    }

    log.Info($"Using Azure Media Service Rest API Endpoint : {_RESTAPIEndpoint}");

    try
    {
        fileName = (string)data.fileName;

        AzureAdTokenCredentials tokenCredentials = new AzureAdTokenCredentials(_AADTenantDomain,
                              new AzureAdClientSymmetricKey(_mediaservicesClientId, _mediaservicesClientSecret),
                              AzureEnvironments.AzureCloudEnvironment);

        AzureAdTokenProvider tokenProvider = new AzureAdTokenProvider(tokenCredentials);

        _context = new CloudMediaContext(new Uri(_RESTAPIEndpoint), tokenProvider);

        // Get the asset
        string assetid = data.assetId;
        var destAsset = _context.Assets.Where(a => a.Id == assetid).FirstOrDefault();

        if (destAsset == null)
        {
            log.Info($"Asset not found {assetid}");
            return req.CreateResponse(HttpStatusCode.BadRequest, new
            {
                error = "Asset not found"
            });
        }

        log.Info($"creation of file {fileName}");

        // Manifest generate
        manifestInfo = LoadAndUpdateManifestTemplate(destAsset, execContext);

        // if not file name passed, then we use the one generated based on mp4 files names
        if (fileName == null)
        {
            fileName = manifestInfo.FileName;
        }

        var filetocreate = destAsset.AssetFiles.Create(fileName);

        using (Stream s = GenerateStreamFromString(manifestInfo.Content))
        {
            filetocreate.Upload(s);
        }

        log.Info("Manifest file created.");

        // let's make the manifest the primary file of the asset
        SetFileAsPrimary(destAsset, fileName);
        log.Info("Manifest file set as primary.");
    }
    catch (Exception ex)
    {
        log.Info($"Exception {ex}");
        return req.CreateResponse(HttpStatusCode.InternalServerError, new
        {
            Error = ex.ToString()
        });
    }

    log.Info($"");
    return req.CreateResponse(HttpStatusCode.OK, new
    {
        fileName = fileName,
        manifestContent = manifestInfo.Content
    });
}

public static Stream GenerateStreamFromString(string s)
{
    MemoryStream stream = new MemoryStream();
    StreamWriter writer = new StreamWriter(stream);
    writer.Write(s);
    writer.Flush();
    stream.Position = 0;
    return stream;
}
<commit_msg>Add option to test streaming endpoint manifest returns<commit_after>/*
This function generates a manifest (.ism) from the MP4/M4A files in the asset. It makes this file primary.
This manifest is needed to stream MP4 file(s) with Azure Media Services.

Caution : such assets are not guaranteed to work with Dynamic Packaging.

Note : this function makes  guesses to determine the files for the video tracks and audio tracks.
These guesses can be wrong. Please check the SMIL generated data for your scenario and your source assets.

Input:
{
    "assetId" : "nb:cid:UUID:88432c30-cb4a-4496-88c2-b2a05ce9033b", // Mandatory, Id of the asset
    "fileName" : "manifest.ism", // Optional. file name of the manifest to create
    "checkStreamingEndpointResponse" : true // Optional. If true, then the asset is published temporarly and the function checks that the streaming endpoint returns a valid client manifest. It's a good way to know if the asset looks streamable (GOP aligned, etc)
}

Output:
{
    "fileName" : "manifest.ism" // The name of the manifest file created
    "manifestContent" : "" // the SMIL data created as an asset file 
    "checkStreamingEndpointResponseSuccess" : true //if check is successful 
}
*/

#r "Newtonsoft.Json"
#r "Microsoft.WindowsAzure.Storage"
#r "System.Xml.Linq"
#load "../Shared/mediaServicesHelpers.csx"
#load "../Shared/copyBlobHelpers.csx"

using System;
using System.Net;
using System.Net.Http;
using Newtonsoft.Json;
using Microsoft.WindowsAzure.MediaServices.Client;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.IO;
using System.Web;
using Microsoft.Azure;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using Microsoft.WindowsAzure.Storage.Auth;
using Microsoft.Azure.WebJobs;
using System.Xml.Linq;
using Microsoft.IdentityModel.Clients.ActiveDirectory;

// Read values from the App.config file.
static string _storageAccountName = Environment.GetEnvironmentVariable("MediaServicesStorageAccountName");
static string _storageAccountKey = Environment.GetEnvironmentVariable("MediaServicesStorageAccountKey");

static readonly string _AADTenantDomain = Environment.GetEnvironmentVariable("AMSAADTenantDomain");
static readonly string _RESTAPIEndpoint = Environment.GetEnvironmentVariable("AMSRESTAPIEndpoint");

static readonly string _mediaservicesClientId = Environment.GetEnvironmentVariable("AMSClientId");
static readonly string _mediaservicesClientSecret = Environment.GetEnvironmentVariable("AMSClientSecret");

// Field for service context.
private static CloudMediaContext _context = null;
private static CloudStorageAccount _destinationStorageAccount = null;


public static async Task<object> Run(HttpRequestMessage req, TraceWriter log, Microsoft.Azure.WebJobs.ExecutionContext execContext)
{
    log.Info($"Webhook was triggered!");

    // Init variables

    string jsonContent = await req.Content.ReadAsStringAsync();
    dynamic data = JsonConvert.DeserializeObject(jsonContent);

    string fileName = null;
    var manifestInfo = new ManifestGenerated();

    log.Info(jsonContent);

    if (data.assetId == null)
    {
        return req.CreateResponse(HttpStatusCode.BadRequest, new
        {
            error = "Please pass asset ID in the input object (assetId)"
        });
    }

    log.Info($"Using Azure Media Service Rest API Endpoint : {_RESTAPIEndpoint}");

    bool checkStreamingEndpointResponse = false;
    bool checkStreamingEndpointResponseSuccess = true;

    try
    {
        fileName = (string)data.fileName;

        AzureAdTokenCredentials tokenCredentials = new AzureAdTokenCredentials(_AADTenantDomain,
                              new AzureAdClientSymmetricKey(_mediaservicesClientId, _mediaservicesClientSecret),
                              AzureEnvironments.AzureCloudEnvironment);

        AzureAdTokenProvider tokenProvider = new AzureAdTokenProvider(tokenCredentials);

        _context = new CloudMediaContext(new Uri(_RESTAPIEndpoint), tokenProvider);

        // Get the asset
        string assetid = data.assetId;
        var destAsset = _context.Assets.Where(a => a.Id == assetid).FirstOrDefault();

        if (destAsset == null)
        {
            log.Info($"Asset not found {assetid}");
            return req.CreateResponse(HttpStatusCode.BadRequest, new
            {
                error = "Asset not found"
            });
        }

        log.Info($"creation of file {fileName}");

        // Manifest generate
        manifestInfo = LoadAndUpdateManifestTemplate(destAsset, execContext);

        // if not file name passed, then we use the one generated based on mp4 files names
        if (fileName == null)
        {
            fileName = manifestInfo.FileName;
        }

        var filetocreate = destAsset.AssetFiles.Create(fileName);

        using (Stream s = GenerateStreamFromString(manifestInfo.Content))
        {
            filetocreate.Upload(s);
        }

        log.Info("Manifest file created.");

        // let's make the manifest the primary file of the asset
        SetFileAsPrimary(destAsset, fileName);
        log.Info("Manifest file set as primary.");



        if (data.checkStreamingEndpointResponse != null && (bool)data.checkStreamingEndpointResponse)
        {
            checkStreamingEndpointResponse = true;
            // testing streaming
            // publish with a streaming locator (1 hour)
            IAccessPolicy readPolicy = _context.AccessPolicies.Create("readPolicy", TimeSpan.FromHours(1), AccessPermissions.Read);
            ILocator outputLocator = _context.Locators.CreateLocator(LocatorType.OnDemandOrigin, destAsset, readPolicy);
            var publishurlsmooth = GetValidOnDemandURI(destAsset);

            try
            {
                WebRequest request = WebRequest.Create(publishurlsmooth.ToString());
                WebResponse response = request.GetResponse();
                response.Close();
            }

            catch (Exception ex)
            {
                checkStreamingEndpointResponseSuccess = false;
            }
            outputLocator.Delete();
            readPolicy.Delete();
        }

    }
    catch (Exception ex)
    {
        log.Info($"Exception {ex}");
        return req.CreateResponse(HttpStatusCode.InternalServerError, new
        {
            Error = ex.ToString()
        });
    }

    log.Info($"");

    if (checkStreamingEndpointResponse)
    {
        return req.CreateResponse(HttpStatusCode.OK, new
        {
            fileName = fileName,
            manifestContent = manifestInfo.Content,
            checkStreamingEndpointResponseSuccess = checkStreamingEndpointResponseSuccess
        });
    }
    else
    {
        return req.CreateResponse(HttpStatusCode.OK, new
        {
            fileName = fileName,
            manifestContent = manifestInfo.Content
        });
    }
}

public static Stream GenerateStreamFromString(string s)
{
    MemoryStream stream = new MemoryStream();
    StreamWriter writer = new StreamWriter(stream);
    writer.Write(s);
    writer.Flush();
    stream.Position = 0;
    return stream;
}

{'fraction_non_alphanumeric': 0.08644822649912517, 'fraction_numerical': 0.004453634483855575, 'mean_word_length': 3.1694297082228116, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6125264', 'n_tokens_mistral': 3280, 'n_tokens_neox': 3230, 'n_words': 918}