<commit_before>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using Newtonsoft.Json.Linq;
using PatchKit.Unity.Patcher.Cancellation;
using PatchKit.Unity.Patcher.Statistics;
using Debug = UnityEngine.Debug;

namespace PatchKit.Unity.Patcher.Net
{
    internal class TorrentDownloader
    {
        private readonly string _streamingAssetsPath;

        public TorrentDownloader(string streamingAssetsPath)
        {
            _streamingAssetsPath = streamingAssetsPath;
        }

        public void DownloadFile(string torrentPath, string destinationPath,
            CustomProgressReporter<DownloadProgress> progressReporter, CancellationToken cancellationToken)
        {
            using (var torrentClient = new TorrentClient(_streamingAssetsPath))
            {
                string destinationDirectoryPath = destinationPath + "_dir";

                var addTorrentResult =
                    torrentClient.ExecuteCommand(string.Format("add-torrent {0} {1}", torrentPath,
                        destinationDirectoryPath));

                if (addTorrentResult.Value<string>("status") != "ok")
                {
                    throw new Exception("Wrong torrent-client status - " + addTorrentResult.Value<string>("status"));
                }

                bool downloaded = false;

                Stopwatch stopwatch = new Stopwatch();
                stopwatch.Start();

                double lastProgress = 0.0;

                const int maxSpeedListCount = 30;

                List<double> lastSpeed = new List<double>();

                while (!downloaded)
                {
                    var statusResult = torrentClient.ExecuteCommand("status");

                    Debug.Log("Torrent status:\n" + statusResult);

                    if (statusResult.Value<string>("status") != "ok")
                    {
                        throw new Exception("Wrong torrent-client status - " + statusResult.Value<string>("status"));
                    }

                    if (statusResult["data"].Value<int>("count") < 1)
                    {
                        throw new Exception("Couldn't check status of torrent-client.");
                    }

                    var torrentStatus = statusResult["data"].Value<JArray>("torrents")[0];

                    if (torrentStatus.Value<string>("error") != string.Empty)
                    {
                        throw new Exception("torrent-client error: " + torrentStatus.Value<string>("error"));
                    }

                    if (torrentStatus.Value<bool>("is_seeding"))
                    {
                        downloaded = true;
                    }

                    double progress = torrentStatus.Value<double>("progress");
                    long totalBytes = torrentStatus.Value<long>("total_wanted");
                    long bytes = (long)(progress * totalBytes);
                    float speed = CalculateDownloadSpeed(progress, lastProgress, totalBytes,
                        stopwatch.ElapsedMilliseconds);

                    if (lastSpeed.Count > 0 && lastSpeed.Last() == 0.0)
                    {
                        lastSpeed.Clear();
                    }

                    lastSpeed.Add(speed);
                    while (lastSpeed.Count > maxSpeedListCount)
                    {
                        lastSpeed.RemoveAt(0);
                    }

                    stopwatch.Reset();
                    stopwatch.Start();
                    lastProgress = progress;

                    progressReporter.Progress = new DownloadProgress
                    {
                        DownloadedBytes = bytes,
                        TotalBytes = totalBytes, 
                        KilobytesPerSecond = lastSpeed.Sum() / lastSpeed.Count,
                        Progress = progress
                    };

                    Thread.Sleep(1000);
                }

                var dirInfo = new DirectoryInfo(destinationDirectoryPath);

                if (File.Exists(destinationPath))
                {
                    File.Delete(destinationPath);
                }

                File.Move(dirInfo.GetFiles()[0].FullName, destinationPath);

                Directory.Delete(destinationDirectoryPath, true);
            }
        }

        private static float CalculateDownloadSpeed(double progress, double lastProgress, long totalBytes, long elapsedMilliseconds)
        {
            if (elapsedMilliseconds == 0)
            {
                return 0.0f;
            }

            double elapsedSeconds = elapsedMilliseconds / 1000.0f;

            double progressDelta = progress - lastProgress;

            double bytes = progressDelta * totalBytes;

            return (float)(bytes / 1024.0 / elapsedSeconds);
        }
    }
}<commit_msg>Fix torrent downloader speed calculation (based on probes from previous 10 seconds)<commit_after>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using Newtonsoft.Json.Linq;
using PatchKit.Unity.Patcher.Cancellation;
using PatchKit.Unity.Patcher.Statistics;
using Debug = UnityEngine.Debug;

namespace PatchKit.Unity.Patcher.Net
{
    internal class TorrentDownloader
    {
        private readonly string _streamingAssetsPath;

        private struct DownloadSpeed
        {
            public long Bytes;

            public long Time;

            public DateTime AddTime;
        }

        public TorrentDownloader(string streamingAssetsPath)
        {
            _streamingAssetsPath = streamingAssetsPath;
        }

        public void DownloadFile(string torrentPath, string destinationPath,
            CustomProgressReporter<DownloadProgress> progressReporter, CancellationToken cancellationToken)
        {
            using (var torrentClient = new TorrentClient(_streamingAssetsPath))
            {
                string destinationDirectoryPath = destinationPath + "_dir";

                var addTorrentResult =
                    torrentClient.ExecuteCommand(string.Format("add-torrent {0} {1}", torrentPath,
                        destinationDirectoryPath));

                if (addTorrentResult.Value<string>("status") != "ok")
                {
                    throw new Exception("Wrong torrent-client status - " + addTorrentResult.Value<string>("status"));
                }

                bool downloaded = false;

                Stopwatch stopwatch = new Stopwatch();
                stopwatch.Start();

                const int maxSpeedListCount = 30;

                List<DownloadSpeed> downloadSpeedList = new List<DownloadSpeed>();

                long lastBytes = 0;

                while (!downloaded)
                {
                    var statusResult = torrentClient.ExecuteCommand("status");

                    Debug.Log("Torrent status:\n" + statusResult);

                    if (statusResult.Value<string>("status") != "ok")
                    {
                        throw new Exception("Wrong torrent-client status - " + statusResult.Value<string>("status"));
                    }

                    if (statusResult["data"].Value<int>("count") < 1)
                    {
                        throw new Exception("Couldn't check status of torrent-client.");
                    }

                    var torrentStatus = statusResult["data"].Value<JArray>("torrents")[0];

                    if (torrentStatus.Value<string>("error") != string.Empty)
                    {
                        throw new Exception("torrent-client error: " + torrentStatus.Value<string>("error"));
                    }

                    if (torrentStatus.Value<bool>("is_seeding"))
                    {
                        downloaded = true;
                    }

                    double progress = torrentStatus.Value<double>("progress");
                    long totalBytes = torrentStatus.Value<long>("total_wanted");
                    long bytes = (long)(progress * totalBytes);

                    downloadSpeedList.Add(new DownloadSpeed
                    {
                        Bytes = bytes - lastBytes,
                        Time = stopwatch.ElapsedMilliseconds,
                        AddTime = DateTime.Now
                    });

                    lastBytes = bytes;

                    downloadSpeedList.RemoveAll(s => (DateTime.Now - s.AddTime).Seconds > 10);

                    while (downloadSpeedList.Count > maxSpeedListCount)
                    {
                        downloadSpeedList.RemoveAt(0);
                    }

                    float speed = CalculateDownloadSpeed(downloadSpeedList.Sum(s => s.Bytes),
                        downloadSpeedList.Sum(s => s.Time));

                    stopwatch.Reset();
                    stopwatch.Start();

                    progressReporter.Progress = new DownloadProgress
                    {
                        DownloadedBytes = bytes,
                        TotalBytes = totalBytes, 
                        KilobytesPerSecond = speed,
                        Progress = progress
                    };

                    Thread.Sleep(1000);
                }

                var dirInfo = new DirectoryInfo(destinationDirectoryPath);

                if (File.Exists(destinationPath))
                {
                    File.Delete(destinationPath);
                }

                File.Move(dirInfo.GetFiles()[0].FullName, destinationPath);

                Directory.Delete(destinationDirectoryPath, true);
            }
        }

        private static float CalculateDownloadSpeed(long bytes, long time)
        {
            if (bytes == 0)
            {
                return 0.0f;
            }

            double elapsedSeconds = time / 1000.0f;

            return (float)(bytes / 1024.0 / elapsedSeconds);
        }
    }
}
{'fraction_non_alphanumeric': 0.0859796285200719, 'fraction_numerical': 0.005991611743559017, 'mean_word_length': 1.4486552567237163, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6071296', 'n_tokens_mistral': 2413, 'n_tokens_neox': 2141, 'n_words': 452}