<commit_before>﻿#region Usings

using System;
using System.ComponentModel.Composition;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using ModInstallation.Annotations;
using ModInstallation.Exceptions;
using ModInstallation.Implementations.Util;
using ModInstallation.Interfaces;
using ModInstallation.Interfaces.Mods;
using ModInstallation.Interfaces.Net;
using Splat;

#endregion

namespace ModInstallation.Implementations
{
    [Export(typeof(IFileDownloader))]
    public class DefaultFileDownloader : IFileDownloader, IEnableLogger
    {
        private const long BufferSize = 64L * 1024L;

        private const int DefaultMaxConcurrentDownloads = 1;

        private SemaphoreSlim _downloadSemaphore;

        private readonly IWebClient _webclient;

        [ImportingConstructor]
        public DefaultFileDownloader(IWebClient webclient)
        {
            _webclient = webclient;
            MaxConcurrentDownloads = DefaultMaxConcurrentDownloads;
        }

        [NotNull]
        private SemaphoreSlim DownloadSemaphore
        {
            get { return _downloadSemaphore ?? (_downloadSemaphore = new SemaphoreSlim(MaxConcurrentDownloads)); }
        }

        [NotNull]
        private async Task<string> DownloadFile([NotNull] IProgress<IDownloadProgress> progressReporter,
            CancellationToken cancellationToken,
            [NotNull] Uri uri)
        {
            var outputFilePath = GetFileOutputPath(Path.GetFileName(uri.LocalPath));
            try
            {
                cancellationToken.ThrowIfCancellationRequested();

                var stopwatch = new Stopwatch();
                stopwatch.Start();

                long[] lastReport = {0L};
                await _webclient.DownloadAsync(uri,
                    outputFilePath,
                    progress =>
                    {
                        var elapsed = stopwatch.Elapsed.TotalSeconds;

                        // Only update 10 times per second
                        if (elapsed < 0.1)
                        {
                            return;
                        }

                        stopwatch.Restart();

                        progressReporter.Report(DefaultDownloadProgress.Downloading(uri,
                            progress.Current,
                            progress.Total,
                            (progress.Current - lastReport[0]) / elapsed));

                        lastReport[0] = progress.Current;
                    },
                    cancellationToken);

                cancellationToken.ThrowIfCancellationRequested();

                var directoryName = Path.GetDirectoryName(outputFilePath);
                if (directoryName != null)
                {
                    Directory.CreateDirectory(directoryName);
                }
            }
            catch (OperationCanceledException)
            {
                if (File.Exists(outputFilePath))<commit_msg>Improve handling of the download speed<commit_after>﻿#region Usings

using System;
using System.ComponentModel.Composition;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using ModInstallation.Annotations;
using ModInstallation.Exceptions;
using ModInstallation.Implementations.Util;
using ModInstallation.Interfaces;
using ModInstallation.Interfaces.Mods;
using ModInstallation.Interfaces.Net;
using Splat;

#endregion

namespace ModInstallation.Implementations
{
    [Export(typeof(IFileDownloader))]
    public class DefaultFileDownloader : IFileDownloader, IEnableLogger
    {
        private const long BufferSize = 64L * 1024L;

        private const int DefaultMaxConcurrentDownloads = 1;

        private SemaphoreSlim _downloadSemaphore;

        private readonly IWebClient _webclient;

        [ImportingConstructor]
        public DefaultFileDownloader(IWebClient webclient)
        {
            _webclient = webclient;
            MaxConcurrentDownloads = DefaultMaxConcurrentDownloads;
        }

        [NotNull]
        private SemaphoreSlim DownloadSemaphore
        {
            get { return _downloadSemaphore ?? (_downloadSemaphore = new SemaphoreSlim(MaxConcurrentDownloads)); }
        }

        [NotNull]
        private async Task<string> DownloadFile([NotNull] IProgress<IDownloadProgress> progressReporter,
            CancellationToken cancellationToken,
            [NotNull] Uri uri)
        {
            var outputFilePath = GetFileOutputPath(Path.GetFileName(uri.LocalPath));
            try
            {
                cancellationToken.ThrowIfCancellationRequested();

                var stopwatch = new Stopwatch();
                stopwatch.Start();

                long[] lastReport = {-1L};
                await _webclient.DownloadAsync(uri,
                    outputFilePath,
                    progress =>
                    {
                        var elapsed = stopwatch.Elapsed.TotalSeconds;

                        // Only update 10 times per second
                        if (elapsed < 0.1)
                        {
                            return;
                        }

                        stopwatch.Restart();

                        if (lastReport[0] < 0)
                        {
                            // First report
                            progressReporter.Report(DefaultDownloadProgress.Downloading(uri, progress.Current, progress.Total, 0.0));
                        }
                        else
                        {
                            progressReporter.Report(DefaultDownloadProgress.Downloading(uri,
                                progress.Current,
                                progress.Total,
                                (progress.Current - lastReport[0]) / elapsed));
                        }

                        lastReport[0] = progress.Current;
                    },
                    cancellationToken);

                cancellationToken.ThrowIfCancellationRequested();

                var directoryName = Path.GetDirectoryName(outputFilePath);
                if (directoryName != null)
                {
                    Directory.CreateDirectory(directoryName);
                }
            }
            catch (OperationCanceledException)
            {
                if (File.Exists(outputFilePath))
{'fraction_non_alphanumeric': 0.0664281269446173, 'fraction_numerical': 0.004978220286247666, 'mean_word_length': 1.6326781326781328, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 37, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6000139', 'n_tokens_mistral': 1531, 'n_tokens_neox': 1446, 'n_words': 292}