<commit_before>﻿using System;
using System.Configuration;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Common;
using Microsoft.Azure.WebJobs;
using Microsoft.WindowsAzure.MediaServices.Client;

namespace VideoConverter
{
    public class Functions
    {
        private static readonly Lazy<MediaServicesCredentials> MediaServicesCredentials =
            new Lazy<MediaServicesCredentials>(() =>
            {
                MediaServicesCredentials credentials = new MediaServicesCredentials(
                    ConfigurationManager.AppSettings["MediaServiceAccountName"],
                    ConfigurationManager.AppSettings["MediaServiceAccountKey"]);

                credentials.RefreshToken();
                return credentials;
            });

        public static async void ProcessQueueMessage(
            [QueueTrigger("videoqueue")] VideoInformation videoInfo,
            [Blob("videos/{Name}", FileAccess.Read)] Stream input)
        {
            CloudMediaContext context = new CloudMediaContext(MediaServicesCredentials.Value);

            string mediaId = await UploadVideo(context, videoInfo.Name, videoInfo.Uri.ToString());

            IJob job = await CreateMediaEncodeJob(context, mediaId);
            await job.GetExecutionProgressTask(CancellationToken.None);
        }

        private static async Task<IJob> CreateMediaEncodeJob(CloudMediaContext context, string assetId)
        {
            string encodingPreset = "H264 Broadband 720p";
            IAsset assetToEncode = context.Assets.FirstOrDefault(a => a.Id == assetId);
            if (assetToEncode == null)
            {
                throw new ArgumentException("Could not find assetId: " + assetId);
            }

            IJob job = context.Jobs.Create("Encoding " + assetToEncode.Name + " to " + encodingPreset);

            IMediaProcessor latestWameMediaProcessor =
                (from p in context.MediaProcessors where p.Name == "Azure Media Encoder" select p).ToList()
                    .OrderBy(wame => new Version(wame.Version))
                    .LastOrDefault();
            ITask encodeTask = job.Tasks.AddNew("Encoding", latestWameMediaProcessor, encodingPreset, TaskOptions.None);
            encodeTask.InputAssets.Add(assetToEncode);
            encodeTask.OutputAssets.AddNew(assetToEncode.Name + " as " + encodingPreset, AssetCreationOptions.None);

            job.StateChanged += new EventHandler<JobStateChangedEventArgs>((sender, jsc) => Console.WriteLine(
                $"{((IJob) sender).Name}\n  State: {jsc.CurrentState}\n  Time: {DateTime.UtcNow.ToString(@"yyyy_M_d_hhmmss")}\n\n"));
            return await job.SubmitAsync();
        }

        public static async Task<string> UploadVideo(CloudMediaContext context, string name, string address)
        {
            IAsset uploadAsset =
                await
                    context.Assets.CreateAsync(Path.GetFileNameWithoutExtension(name), AssetCreationOptions.None,
                        CancellationToken.None);
            IAssetFile assetFile = await uploadAsset.AssetFiles.CreateAsync(name, CancellationToken.None);

            IAccessPolicy accessPolicy = context.AccessPolicies.Create(uploadAsset.Name, TimeSpan.FromMinutes(5),
                AccessPermissions.Write);
            ILocator locator = context.Locators.CreateSasLocator(uploadAsset, accessPolicy);
            BlobTransferClient client = new BlobTransferClient()
            {
                NumberOfConcurrentTransfers = 5,
                ParallelTransferThreadCount = 5
            };
            await assetFile.UploadAsync(address, client, locator, CancellationToken.None);

            locator.Delete();
            accessPolicy.Delete();

            return uploadAsset.Id;
        }
    }
}<commit_msg>Medai processing job<commit_after>﻿using System;
using System.Configuration;
using System.IO;
using System.Linq;
using System.Security.AccessControl;
using System.Threading;
using System.Threading.Tasks;
using Common;
using Microsoft.Azure.WebJobs;
using Microsoft.WindowsAzure.MediaServices.Client;

namespace VideoConverter
{
    public class Functions
    {
        private static readonly Lazy<MediaServicesCredentials> MediaServicesCredentials =
            new Lazy<MediaServicesCredentials>(() =>
            {
                MediaServicesCredentials credentials = new MediaServicesCredentials(
                    ConfigurationManager.AppSettings["MediaServiceAccountName"],
                    ConfigurationManager.AppSettings["MediaServiceAccountKey"]);

                credentials.RefreshToken();
                return credentials;
            });

        public static async void ProcessQueueMessage(
            [QueueTrigger("videoqueue")] VideoInformation videoInfo,
            [Blob("asset-9320445d-1500-80c4-b779-f1e5791e1257/{Name}", FileAccess.Read)] Stream input)
        {
            CloudMediaContext context = new CloudMediaContext(MediaServicesCredentials.Value);

            string file = CopyFileLocaly(input, videoInfo.Name);

            //string mediaId = await UploadVideo(context, videoInfo.Name, videoInfo.Uri.ToString());
            IAsset mediaAsset = await UploadVideo(context, Path.GetFileName(file), file);

            IJob job = await CreateMediaEncodeJob(context, mediaAsset);
            await job.GetExecutionProgressTask(CancellationToken.None);
        }

        private static string CopyFileLocaly(Stream input, string name)
        {
            //string filaPath = Path.GetTempFileName();
            string filaPath = Path.Combine(Path.GetTempPath(), name);
            //input.CopyTo(File.Open(filaPath, FileMode.Append, FileAccess.Write, FileShare.Read));
            input.CopyTo(File.Open(filaPath, FileMode.Create, FileAccess.Write, FileShare.Read));
            return filaPath;
        }

        private static async Task<IJob> CreateMediaEncodeJob(CloudMediaContext context, IAsset assetToEncode)
        {
            //string encodingPreset = "H264 Broadband 720p";
            string encodingPreset = "H264 Smooth Streaming 720p";

            IJob job = context.Jobs.Create("Encoding " + assetToEncode.Name + " to " + encodingPreset);

            string[] y = context.MediaProcessors.Where(mp => mp.Name.Equals("Azure Media Encoder")).ToArray().Select(mp=>mp.Version).ToArray();

            IMediaProcessor latestWameMediaProcessor =
                context.MediaProcessors.Where(mp => mp.Name.Equals("Azure Media Encoder"))
                    .ToArray()
                    .OrderByDescending(
                        mp =>
                            new Version(int.Parse(mp.Version.Split('.', ',')[0]),
                                int.Parse(mp.Version.Split('.', ',')[1])))
                    .First();

            ITask encodeTask = job.Tasks.AddNew("Encoding", latestWameMediaProcessor, encodingPreset, TaskOptions.None);
            encodeTask.InputAssets.Add(assetToEncode);
            encodeTask.OutputAssets.AddNew(assetToEncode.Name + " as " + encodingPreset, AssetCreationOptions.None);

            job.StateChanged += new EventHandler<JobStateChangedEventArgs>((sender, jsc) => Console.WriteLine(
                $"{((IJob) sender).Name}\n  State: {jsc.CurrentState}\n  Time: {DateTime.UtcNow.ToString(@"yyyy_M_d_hhmmss")}\n\n"));
            return await job.SubmitAsync();
        }

        public static async Task<IAsset> UploadVideo(CloudMediaContext context, string name, string address)
        {
            IAsset uploadAsset =
                await
                    context.Assets.CreateAsync(Path.GetFileNameWithoutExtension(name), AssetCreationOptions.None,
                        CancellationToken.None);
            IAssetFile assetFile = await uploadAsset.AssetFiles.CreateAsync(name, CancellationToken.None);

            IAccessPolicy accessPolicy = context.AccessPolicies.Create(uploadAsset.Name, TimeSpan.FromMinutes(5),
                AccessPermissions.Write);
            ILocator locator = context.Locators.CreateSasLocator(uploadAsset, accessPolicy);
            BlobTransferClient client = new BlobTransferClient()
            {
                NumberOfConcurrentTransfers = 5,
                ParallelTransferThreadCount = 5
            };
            await assetFile.UploadAsync(address, client, locator, CancellationToken.None);

            locator.Delete();
            accessPolicy.Delete();

            return uploadAsset;
        }
    }
}
{'fraction_non_alphanumeric': 0.09305604511808248, 'fraction_numerical': 0.006109740336035718, 'mean_word_length': 2.4913863822805578, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6328365', 'n_tokens_mistral': 2151, 'n_tokens_neox': 2094, 'n_words': 415}