<commit_before>﻿using ImageProcessingLibrary;
using Newtonsoft.Json;
using ServiceHelpers;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices.WindowsRuntime;
using System.Threading.Tasks;
using Windows.Storage;
using Windows.Storage.Pickers;
using Windows.UI.Xaml;
using Windows.UI.Xaml.Controls;
using Windows.UI.Xaml.Media;
using Windows.UI.Xaml.Navigation;

// The Blank Page item template is documented at http://go.microsoft.com/fwlink/?LinkId=402352&clcid=0x409

namespace TestApp
{
    /// <summary>
    /// An empty page that can be used on its own or navigated to within a Frame.
    /// </summary>
    public sealed partial class MainPage : Page
    {
        public List<ImageInsightsViewModel> AllResults { get; set; } = new List<ImageInsightsViewModel>();
        public ObservableCollection<ImageInsightsViewModel> FilteredResults { get; set; } = new ObservableCollection<ImageInsightsViewModel>();
        public ObservableCollection<TagFilterViewModel> TagFilters { get; set; } = new ObservableCollection<TagFilterViewModel>();
        public ObservableCollection<FaceFilterViewModel> FaceFilters { get; set; } = new ObservableCollection<FaceFilterViewModel>();
        public ObservableCollection<EmotionFilterViewModel> EmotionFilters { get; set; } = new ObservableCollection<EmotionFilterViewModel>();

        public MainPage()
        {
            this.InitializeComponent();
        }

        protected override async void OnNavigatedTo(NavigationEventArgs e)
        {
            // callbacks for core library
            FaceServiceHelper.Throttled = () => Util.ShowToastNotification("The Face API is throttling your requests. Consider upgrading to a Premium Key.");
            EmotionServiceHelper.Throttled = () => Util.ShowToastNotification("The Emotion API is throttling your requests. Consider upgrading to a Premium Key.");
            VisionServiceHelper.Throttled = () => Util.ShowToastNotification("The Vision API is throttling your requests. Consider upgrading to a Premium Key.");
            ErrorTrackingHelper.TrackException = (exception, message) => { Debug.WriteLine("ImageProcessingLibrary exception: {0}", message); }; 
            ErrorTrackingHelper.GenericApiCallExceptionHandler = Util.GenericApiCallExceptionHandler;

            // Enter API Keys here
            FaceServiceHelper.ApiKey = "";
            EmotionServiceHelper.ApiKey = "";
            VisionServiceHelper.ApiKey = "";

            await FaceListManager.ResetFaceLists();

            base.OnNavigatedTo(e);
        }

        private async void ProcessImagesClicked(object sender, RoutedEventArgs e)
        {
            try
            {
                FolderPicker folderPicker = new FolderPicker();
                folderPicker.SuggestedStartLocation = PickerLocationId.PicturesLibrary;
                folderPicker.FileTypeFilter.Add(".jpeg");
                folderPicker.FileTypeFilter.Add(".bmp");
                StorageFolder folder = await folderPicker.PickSingleFolderAsync();

                if (folder != null)
                {
                    await ProcessImagesAsync(folder);
                }
            }
            catch (Exception ex)
            {
                await ErrorTrackingHelper.GenericApiCallExceptionHandler(ex, "Error picking the target folder.");
            }
        }

        private async Task ProcessImagesAsync(StorageFolder rootFolder)
        {
            this.progressRing.IsActive = true;

            this.AllResults.Clear();
            this.FilteredResults.Clear();
            this.TagFilters.Clear();

            List<ImageInsights> insightsList = new List<ImageInsights>();

            // see if we have pre-computed results and if so load it from the json file
            try
            {
                StorageFile insightsResultFile = (await rootFolder.TryGetItemAsync("ImageInsights.json")) as StorageFile;
                if (insightsResultFile != null)
                {
                    using (StreamReader reader = new StreamReader(await insightsResultFile.OpenStreamForReadAsync()))
                    {
                        insightsList = JsonConvert.DeserializeObject<List<ImageInsights>>(await reader.ReadToEndAsync());
                        foreach (var insights in insightsList)
                        {
                            await AddImageInsightsToViewModel(rootFolder, insights);
                        }
                    }
                }
            }
            catch
            {
                // We will just compute everything again in case of errors
            }

            if (!insightsList.Any())
            {
                // enumerate through the first 50 files and extract the insights 
                foreach (var item in (await rootFolder.GetFilesAsync()).Take(50))
                {
                    ImageInsights insights = await ImageProcessor.ProcessImageAsync(item.OpenStreamForReadAsync, item.Name);
                    insightsList.Add(insights);
                    await AddImageInsightsToViewModel(rootFolder, insights);
                }

                // save to json
                StorageFile jsonFile = await rootFolder.CreateFileAsync("ImageInsights.json", CreationCollisionOption.ReplaceExisting);
                using (StreamWriter writer = new StreamWriter(await jsonFile.OpenStreamForWriteAsync()))
                {
                    string jsonStr = JsonConvert.SerializeObject(insightsList, Formatting.Indented);
                    await writer.WriteAsync(jsonStr);
                }
            }

            var sortedTags = this.TagFilters.OrderBy(t => t.Tag).ToArray();
            this.TagFilters.Clear();
            this.TagFilters.AddRange(sortedTags);

            var sortedEmotions = this.EmotionFilters.OrderBy(t => t.Emotion).ToArray();
            this.EmotionFilters.Clear();
            this.EmotionFilters.AddRange(sortedEmotions);

            this.progressRing.IsActive = false;
        }

        private async Task AddImageInsightsToViewModel(StorageFolder rootFolder, ImageInsights insights)
        {
            ImageInsightsViewModel insightsViewModel = new ImageInsightsViewModel(insights, await (await rootFolder.GetFileAsync(insights.ImageId)).OpenStreamForReadAsync());

            this.AllResults.Add(insightsViewModel);
            this.FilteredResults.Add(insightsViewModel);

            foreach (var tag in insights.VisionInsights.Tags)
            {
                if (!this.TagFilters.Any(t => t.Tag == tag))<commit_msg>Sikipping images with errors during processing loop, plus couple of bug fixes<commit_after>﻿using ImageProcessingLibrary;
using Newtonsoft.Json;
using ServiceHelpers;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices.WindowsRuntime;
using System.Threading.Tasks;
using Windows.Storage;
using Windows.Storage.Pickers;
using Windows.UI.Popups;
using Windows.UI.Xaml;
using Windows.UI.Xaml.Controls;
using Windows.UI.Xaml.Media;
using Windows.UI.Xaml.Navigation;

// The Blank Page item template is documented at http://go.microsoft.com/fwlink/?LinkId=402352&clcid=0x409

namespace TestApp
{
    /// <summary>
    /// An empty page that can be used on its own or navigated to within a Frame.
    /// </summary>
    public sealed partial class MainPage : Page
    {
        public List<ImageInsightsViewModel> AllResults { get; set; } = new List<ImageInsightsViewModel>();
        public ObservableCollection<ImageInsightsViewModel> FilteredResults { get; set; } = new ObservableCollection<ImageInsightsViewModel>();
        public ObservableCollection<TagFilterViewModel> TagFilters { get; set; } = new ObservableCollection<TagFilterViewModel>();
        public ObservableCollection<FaceFilterViewModel> FaceFilters { get; set; } = new ObservableCollection<FaceFilterViewModel>();
        public ObservableCollection<EmotionFilterViewModel> EmotionFilters { get; set; } = new ObservableCollection<EmotionFilterViewModel>();

        public MainPage()
        {
            this.InitializeComponent();
        }

        protected override void OnNavigatedTo(NavigationEventArgs e)
        {
            // callbacks for core library
            FaceServiceHelper.Throttled = () => Util.ShowToastNotification("The Face API is throttling your requests. Consider upgrading to a Premium Key.");
            EmotionServiceHelper.Throttled = () => Util.ShowToastNotification("The Emotion API is throttling your requests. Consider upgrading to a Premium Key.");
            VisionServiceHelper.Throttled = () => Util.ShowToastNotification("The Vision API is throttling your requests. Consider upgrading to a Premium Key.");
            ErrorTrackingHelper.TrackException = (exception, message) => { Debug.WriteLine("ImageProcessingLibrary exception: {0}", message); }; 
            ErrorTrackingHelper.GenericApiCallExceptionHandler = Util.GenericApiCallExceptionHandler;

            // Enter API Keys here
            FaceServiceHelper.ApiKey = "";
            EmotionServiceHelper.ApiKey = "";
            VisionServiceHelper.ApiKey = "";

            base.OnNavigatedTo(e);
        }

        private async void ProcessImagesClicked(object sender, RoutedEventArgs e)
        {
            try
            {
                FolderPicker folderPicker = new FolderPicker();
                folderPicker.SuggestedStartLocation = PickerLocationId.PicturesLibrary;
                folderPicker.FileTypeFilter.Add(".jpeg");
                folderPicker.FileTypeFilter.Add(".jpg");
                folderPicker.FileTypeFilter.Add(".png");
                folderPicker.FileTypeFilter.Add(".gif");
                folderPicker.FileTypeFilter.Add(".bmp");
                StorageFolder folder = await folderPicker.PickSingleFolderAsync();

                if (folder != null)
                {
                    await ProcessImagesAsync(folder);
                }
            }
            catch (Exception ex)
            {
                await ErrorTrackingHelper.GenericApiCallExceptionHandler(ex, "Error picking the target folder.");
            }
        }

        private async Task ProcessImagesAsync(StorageFolder rootFolder)
        {
            this.progressRing.IsActive = true;

            this.AllResults.Clear();
            this.FilteredResults.Clear();
            this.TagFilters.Clear();
            this.EmotionFilters.Clear();
            this.FaceFilters.Clear();

            List<ImageInsights> insightsList = new List<ImageInsights>();

            // see if we have pre-computed results and if so load it from the json file
            try
            {
                StorageFile insightsResultFile = (await rootFolder.TryGetItemAsync("ImageInsights.json")) as StorageFile;
                if (insightsResultFile != null)
                {
                    using (StreamReader reader = new StreamReader(await insightsResultFile.OpenStreamForReadAsync()))
                    {
                        insightsList = JsonConvert.DeserializeObject<List<ImageInsights>>(await reader.ReadToEndAsync());
                        foreach (var insights in insightsList)
                        {
                            await AddImageInsightsToViewModel(rootFolder, insights);
                        }
                    }
                }
            }
            catch
            {
                // We will just compute everything again in case of errors
            }

            bool foundErrors = false;

            if (!insightsList.Any())
            {
                // start with fresh face lists
                await FaceListManager.ResetFaceLists();

                // enumerate through the first 50 files and extract the insights 
                foreach (var item in (await rootFolder.GetFilesAsync()).Take(50))
                {
                    try
                    {
                        ImageInsights insights = await ImageProcessor.ProcessImageAsync(item.OpenStreamForReadAsync, item.Name);
                        insightsList.Add(insights);
                        await AddImageInsightsToViewModel(rootFolder, insights);
                    }
                    catch (Exception)
                    {
                        // Ignore the error with this image but continue so we try process as many as we can.
                        foundErrors = true;
                    }
                }

                // save to json
                StorageFile jsonFile = await rootFolder.CreateFileAsync("ImageInsights.json", CreationCollisionOption.ReplaceExisting);
                using (StreamWriter writer = new StreamWriter(await jsonFile.OpenStreamForWriteAsync()))
                {
                    string jsonStr = JsonConvert.SerializeObject(insightsList, Formatting.Indented);
                    await writer.WriteAsync(jsonStr);
                }
            }

            var sortedTags = this.TagFilters.OrderBy(t => t.Tag).ToArray();
            this.TagFilters.Clear();
            this.TagFilters.AddRange(sortedTags);

            var sortedEmotions = this.EmotionFilters.OrderBy(t => t.Emotion).ToArray();
            this.EmotionFilters.Clear();
            this.EmotionFilters.AddRange(sortedEmotions);

            this.progressRing.IsActive = false;

            if (foundErrors)
            {
                await new MessageDialog("Error processing some of the photos. We skipped the errors and did what we could.").ShowAsync();
            }
        }

        private async Task AddImageInsightsToViewModel(StorageFolder rootFolder, ImageInsights insights)
        {
            ImageInsightsViewModel insightsViewModel = new ImageInsightsViewModel(insights, await (await rootFolder.GetFileAsync(insights.ImageId)).OpenStreamForReadAsync());

            this.AllResults.Add(insightsViewModel);
            this.FilteredResults.Add(insightsViewModel);

            foreach (var tag in insights.VisionInsights.Tags)
            {
                if (!this.TagFilters.Any(t => t.Tag == tag))
{'fraction_non_alphanumeric': 0.08179973409838359, 'fraction_numerical': 0.002099223287383668, 'mean_word_length': 2.1480176211453745, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2752082', 'n_tokens_mistral': 3462, 'n_tokens_neox': 3280, 'n_words': 797}