<commit_before>﻿// This is free and unencumbered software released into the public domain.
// Anyone is free to copy, modify, publish, use, compile, sell, or
// distribute this software, either in source code form or as a compiled
// binary, for any purpose, commercial or non-commercial, and by any
// means.

using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices.WindowsRuntime;
using System.Threading.Tasks;
using Windows.Foundation;
using Windows.Foundation.Collections;
using Windows.Security.Cryptography.Core;
using Windows.Storage;
using Windows.Storage.AccessCache;
using Windows.Storage.Pickers;
using Windows.Storage.Streams;
using Windows.UI;
using Windows.UI.Popups;
using Windows.UI.ViewManagement;
using Windows.UI.Xaml;
using Windows.UI.Xaml.Controls;
using Windows.UI.Xaml.Controls.Primitives;
using Windows.UI.Xaml.Data;
using Windows.UI.Xaml.Input;
using Windows.UI.Xaml.Media;
using Windows.UI.Xaml.Navigation;
using Windows.UI.Xaml.Shapes;

// The Blank Page item template is documented at http://go.microsoft.com/fwlink/?LinkId=402352&clcid=0x409

namespace goroutines_filescanner
{
    /// <summary>
    /// An empty page that can be used on its own or navigated to within a Frame.
    /// </summary>
    public sealed partial class MainPage : Page
    {
        readonly Channel<string> m_directoryChanged = new Channel<string>();
        ObservableCollection<FileInfo> m_observableCollection = new ObservableCollection<FileInfo>();

        public static Color BackgroundColor = Color.FromArgb(255, 45, 187, 40);

        public MainPage()
        {
            this.InitializeComponent();
            textBox.Text = "";
            listView.ItemsSource = m_observableCollection;

            canvas.SizeChanged += (s, e) => { var _ = DrawTreeMap(); };

            var titleBar = ApplicationView.GetForCurrentView().TitleBar;
            titleBar.BackgroundColor = titleBar.ButtonBackgroundColor = titleBar.InactiveBackgroundColor = titleBar.ButtonInactiveBackgroundColor = BackgroundColor;
            relativePanel.Background = new SolidColorBrush(BackgroundColor);

            StartListeningOnChannels();
        }

        void StartListeningOnChannels()
        {
            var _ = m_directoryChanged
                .Select(openDirectory)
                .Where(f => f.HasValue)
                .Select(o => o.Value)
                .ForEach(async storageFolder => {
                    using (var scanStarts = new Channel<string>())
                    using (var scanCompletes = new Channel<FileInfo>()) {

                        Go.Run(ScanDirectoryAsync, storageFolder, true, scanStarts, scanCompletes);

                        m_observableCollection.Clear();
                        while (true) {
                            var rv = await scanStarts.ReceiveEx();
                            if (!rv.IsValid)
                                break;  // channel closed

                            var fi = new FileInfo { Name = rv.Value };

                            var insertedIndex = m_observableCollection.Count;
                            m_observableCollection.Add(fi);

                            var rv2 = await scanCompletes.ReceiveEx();

                            if (!rv2.IsValid)
                                break; // channel closed
                            fi.CopyFrom(rv2.Value);

                            await DrawTreeMap();
                        }
                    }
                });
        }

        private async Task ScanDirectoryAsync(StorageFolder storageFolder, bool doSha1, Channel<string> starts, Channel<FileInfo> completes)
        {
            using (starts)
            using (completes) {
                foreach (var entry in await storageFolder.GetItemsAsync()) {
                    var folder = entry as StorageFolder; // folder
                    if (folder != null) {
                        await starts.Send(folder.Name);

                        var innerStarts = new Channel<string>();
                        var innerCompletes = new Channel<FileInfo>();
                        Go.Run(ScanDirectoryAsync, folder, false, innerStarts, innerCompletes);

                        var totalSize = await innerStarts
                            .Zip(innerCompletes, (s, f) => f.Size)
                            .Sum(x => x).ConfigureAwait(false);

                        await completes.Send(new FileInfo { Name = folder.Name, Size = totalSize, IsLoaded = true });
                    }
<commit_msg>back to regular loop, it looks cleaner<commit_after>﻿// This is free and unencumbered software released into the public domain.
// Anyone is free to copy, modify, publish, use, compile, sell, or
// distribute this software, either in source code form or as a compiled
// binary, for any purpose, commercial or non-commercial, and by any
// means.

using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices.WindowsRuntime;
using System.Threading.Tasks;
using Windows.Foundation;
using Windows.Foundation.Collections;
using Windows.Security.Cryptography.Core;
using Windows.Storage;
using Windows.Storage.AccessCache;
using Windows.Storage.Pickers;
using Windows.Storage.Streams;
using Windows.UI;
using Windows.UI.Popups;
using Windows.UI.ViewManagement;
using Windows.UI.Xaml;
using Windows.UI.Xaml.Controls;
using Windows.UI.Xaml.Controls.Primitives;
using Windows.UI.Xaml.Data;
using Windows.UI.Xaml.Input;
using Windows.UI.Xaml.Media;
using Windows.UI.Xaml.Navigation;
using Windows.UI.Xaml.Shapes;

// The Blank Page item template is documented at http://go.microsoft.com/fwlink/?LinkId=402352&clcid=0x409

namespace goroutines_filescanner
{
    /// <summary>
    /// An empty page that can be used on its own or navigated to within a Frame.
    /// </summary>
    public sealed partial class MainPage : Page
    {
        readonly Channel<string> m_directoryChanged = new Channel<string>();
        ObservableCollection<FileInfo> m_observableCollection = new ObservableCollection<FileInfo>();

        public static Color BackgroundColor = Color.FromArgb(255, 45, 187, 40);

        public MainPage()
        {
            this.InitializeComponent();
            textBox.Text = "";
            listView.ItemsSource = m_observableCollection;

            canvas.SizeChanged += (s, e) => { var _ = DrawTreeMap(); };

            var titleBar = ApplicationView.GetForCurrentView().TitleBar;
            titleBar.BackgroundColor = titleBar.ButtonBackgroundColor = titleBar.InactiveBackgroundColor = titleBar.ButtonInactiveBackgroundColor = BackgroundColor;
            relativePanel.Background = new SolidColorBrush(BackgroundColor);

            StartListeningOnChannels();
        }

        async void StartListeningOnChannels()
        {
            while (m_directoryChanged.IsOpen) {
                var directoryName = await m_directoryChanged.Receive();
                StorageFolder storageFolder;
                try {
                    storageFolder = await StorageFolder.GetFolderFromPathAsync(directoryName);
                }
                catch (Exception) {
                    continue; // invalid folder
                }

                using (var scanStarts = new Channel<string>())
                using (var scanCompletes = new Channel<FileInfo>()) {

                    Go.Run(ScanDirectoryAsync, storageFolder, true, scanStarts, scanCompletes);

                    m_observableCollection.Clear();
                    while (true) {
                        var rv = await scanStarts.ReceiveEx();
                        if (!rv.IsValid)
                            break;  // channel closed

                        var fi = new FileInfo { Name = rv.Value };

                        var insertedIndex = m_observableCollection.Count;
                        m_observableCollection.Add(fi);

                        var rv2 = await scanCompletes.ReceiveEx();
                        if (!rv2.IsValid)
                            break; // channel closed
                        fi.CopyFrom(rv2.Value);

                        await DrawTreeMap();
                    }
                }
            }
        }

        private async Task ScanDirectoryAsync(StorageFolder storageFolder, bool doSha1, Channel<string> starts, Channel<FileInfo> completes)
        {
            using (starts)
            using (completes) {
                foreach (var entry in await storageFolder.GetItemsAsync()) {
                    var folder = entry as StorageFolder; // folder
                    if (folder != null) {
                        await starts.Send(folder.Name);

                        var innerStarts = new Channel<string>();
                        var innerCompletes = new Channel<FileInfo>();
                        Go.Run(ScanDirectoryAsync, folder, false, innerStarts, innerCompletes);

                        var totalSize = await innerStarts
                            .Zip(innerCompletes, (s, f) => f.Size)
                            .Sum(x => x).ConfigureAwait(false);

                        await completes.Send(new FileInfo { Name = folder.Name, Size = totalSize, IsLoaded = true });
                    }

{'fraction_non_alphanumeric': 0.08684733703044889, 'fraction_numerical': 0.005022496599351261, 'mean_word_length': 1.980355472404116, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4772677', 'n_tokens_mistral': 2441, 'n_tokens_neox': 2302, 'n_words': 617}