<commit_before>﻿using RelhaxModpack.Database;
using RelhaxModpack.UI;
using RelhaxModpack.Utilities;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;

namespace RelhaxModpack.Common
{
    public class DownloadManager : IDisposable
    {
        public const int BYTE_CHUNKS = 4096;

        public string Hash { get; protected set; } = string.Empty;

        public int RetryCount { get; set; } = 3;

        public string UrlBase { get; set; }

        public string DownloadLocationBase { get; set; }

        public IProgress<RelhaxDownloadProgress> Progress { get; set; }

        public CancellationToken CancellationToken { get; set; }

        private WebClient webClient;
        private Stream stream;
        private MD5 md5Hash;
        private FileStream filestream;
        private RelhaxDownloadProgress downloadProgress;

        //RelhaxDownloadProgress params and exit codes TODO

        public DownloadManager()
        {
            webClient = new WebClient();
            md5Hash = MD5.Create();
            downloadProgress = new RelhaxDownloadProgress();
        }

        public async Task DownloadPackagesAsync(List<DatabasePackage> packagesToDownload)
        {
            if (string.IsNullOrEmpty(UrlBase))
            {
                throw new BadMemeException("UrlBase is empty/null");
            }

            if (string.IsNullOrEmpty(DownloadLocationBase))
            {
                throw new BadMemeException("DownloadLocationBase is empty/null");
            }

            if (Progress == null)
            {
                Logging.Warning("Progress is null, no progress will be reported for this download operation");
            }

            if (CancellationToken == null)
            {
                Logging.Warning("CancellationToken is null, no cancellations will be acknowledged for this download operation");
            }

            foreach (DatabasePackage package in packagesToDownload)
            {
                Logging.Debug(Utilities.Enums.LogOptions.ClassName, "Download of package {0} from formed URL {1}", package.PackageName, UrlBase + package.ZipFile);
                await DownloadPackageAsync(package);
            }
        }

        public async Task DownloadPackageAsync(DatabasePackage package)
        {
            string downloadUrl = UrlBase + package.ZipFile;
            string downloadLocation = Path.Combine(DownloadLocationBase, package.ZipFile);
            Logging.Debug("Delete {0} if exists", downloadLocation);
            if (File.Exists(downloadLocation))
                FileUtils.FileDelete(downloadLocation);

            ThrowIfCancellationRequested();
            Progress.Report(downloadProgress);

            //open the stream to the file to download. A fail here means that the file might not exist
            bool retry = true;
            while (retry)
            {
                for(int failCount = 1; failCount <= RetryCount; failCount++)
                {
                    try
                    {
                        using (stream = webClient.OpenRead(downloadUrl))
                        using (filestream = new FileStream(downloadLocation, FileMode.Create, FileAccess.Write))
                        {
                            byte[] buffer = new byte[BYTE_CHUNKS];
                            ThrowIfCancellationRequested();
                            Progress.Report(downloadProgress);

                            while (true)
                            {
                                int readBytes = stream.Read(buffer, 0, BYTE_CHUNKS);
                                if (readBytes < BYTE_CHUNKS)
                                {
                                    StringBuilder sBuilder = new StringBuilder();
                                    md5Hash.TransformFinalBlock(buffer, 0, readBytes);
                                    filestream.Write(buffer, 0, readBytes);

                                    for (int i = 0; i < md5Hash.Hash.Length; i++)
                                    {
                                        sBuilder.Append(md5Hash.Hash[i].ToString("x2"));
                                    }
                                    Hash = sBuilder.ToString();

                                    if ((!Hash.Equals("f") && (!Hash.Equals(package.CRC))))
                                    {
                                        //download failed, the hash doesn't match. try again
                                        if (failCount == 3)
                                        {
                                            //this is the third time the file download has failed. something is wrong here.
                                            throw new BadMemeException(string.Format("The file download hash failed to match. Downloaded = {0}, Database = {1}", Hash, package.CRC));
                                        }
                                    }
                                    else
                                    {
                                        if (Hash.Equals("f"))
                                        {
                                            Logging.Info("The hash was calculated to be {0}, but package reports {1} to mean it's a beta package that can change. Skipping saving to database.", Hash, package.CRC);
                                        }
                                        else
                                        {
                                            //save to local database cache file
                                            //TODO
                                        }
                                        ThrowIfCancellationRequested();
                                        Progress.Report(downloadProgress);

                                        filestream.Close();
                                        md5Hash.Clear();
                                        stream.Close();
                                        package.DownloadFlag = false;

                                        //set the failCount for loop to jump out and the userRetry to stop asking
                                        failCount = 4;<commit_msg>finish draft (test) implementation of Md5 hash database class into DownloadManager<commit_after>﻿using RelhaxModpack.Database;
using RelhaxModpack.UI;
using RelhaxModpack.Utilities;
using RelhaxModpack.Utilities.Enums;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;

namespace RelhaxModpack.Common
{
    public class DownloadManager : IDisposable
    {
        public const int BYTE_CHUNKS = 4096;

        public string Hash { get; protected set; } = string.Empty;

        public int RetryCount { get; set; } = 3;

        public string UrlBase { get; set; }

        public string DownloadLocationBase { get; set; }

        public IProgress<RelhaxDownloadProgress> Progress { get; set; }

        public CancellationToken CancellationToken { get; set; }

        private WebClient webClient;
        private Stream stream;
        private MD5 md5Hash;
        private FileStream filestream;
        private RelhaxDownloadProgress downloadProgress;
        private Md5DatabaseManager databaseManager;

        //RelhaxDownloadProgress params and exit codes TODO

        public DownloadManager()
        {
            webClient = new WebClient();
            md5Hash = MD5.Create();
            downloadProgress = new RelhaxDownloadProgress();
            databaseManager = new Md5DatabaseManager();
        }

        public async Task DownloadPackagesAsync(List<DatabasePackage> packagesToDownload)
        {
            if (string.IsNullOrEmpty(UrlBase))
            {
                throw new BadMemeException("UrlBase is empty/null");
            }

            if (string.IsNullOrEmpty(DownloadLocationBase))
            {
                throw new BadMemeException("DownloadLocationBase is empty/null");
            }

            if (Progress == null)
            {
                Logging.Warning("Progress is null, no progress will be reported for this download operation");
            }

            if (CancellationToken == null)
            {
                Logging.Warning("CancellationToken is null, no cancellations will be acknowledged for this download operation");
            }

            //load md5 database manager before downloading packages
            if (!databaseManager.DatabaseLoaded)
                databaseManager.LoadMd5Database(ApplicationConstants.MD5HashDatabaseXmlFile);

            for (int i = 0; i < packagesToDownload.Count; i++)
            {
                DatabasePackage package = packagesToDownload[i];
                Logging.Info(LogOptions.ClassName, "Download {0} of {1}, package {2} start", i + 1, packagesToDownload.Count, package.PackageName);
                Logging.Debug(LogOptions.ClassName, "Download of package {0} from formed URL {1}", package.PackageName, UrlBase + package.ZipFile);
                await DownloadPackageAsync(package);
                Logging.Info(LogOptions.ClassName, "Download {0} of {1}, package {2} finish", i + 1, packagesToDownload.Count, package.PackageName);
            }
        }

        public async Task DownloadPackageAsync(DatabasePackage package)
        {
            if (!databaseManager.DatabaseLoaded)
                databaseManager.LoadMd5Database(ApplicationConstants.MD5HashDatabaseXmlFile);

            string downloadUrl = UrlBase + package.ZipFile;
            string downloadLocation = Path.Combine(DownloadLocationBase, package.ZipFile);

            Logging.Debug("Delete {0} if exists", downloadLocation);
            if (File.Exists(downloadLocation))
                FileUtils.FileDelete(downloadLocation);

            ThrowIfCancellationRequested();
            Progress.Report(downloadProgress);

            //open the stream to the file to download. A fail here means that the file might not exist
            bool retry = true;
            while (retry)
            {
                for(int failCount = 1; failCount <= RetryCount; failCount++)
                {
                    try
                    {
                        Logging.Debug(LogOptions.ClassName, "Opening download stream and file write stream for {0}", package.ZipFile);
                        using (stream = webClient.OpenRead(downloadUrl))
                        using (filestream = new FileStream(downloadLocation, FileMode.Create, FileAccess.Write))
                        {
                            byte[] buffer = new byte[BYTE_CHUNKS];
                            ThrowIfCancellationRequested();
                            Progress.Report(downloadProgress);

                            while (true)
                            {
                                int readBytes = stream.Read(buffer, 0, BYTE_CHUNKS);
                                if (readBytes < BYTE_CHUNKS)
                                {
                                    StringBuilder sBuilder = new StringBuilder();
                                    md5Hash.TransformFinalBlock(buffer, 0, readBytes);
                                    filestream.Write(buffer, 0, readBytes);

                                    for (int i = 0; i < md5Hash.Hash.Length; i++)
                                    {
                                        sBuilder.Append(md5Hash.Hash[i].ToString("x2"));
                                    }
                                    Hash = sBuilder.ToString();
                                    Logging.Info(LogOptions.ClassName, "Hash for package {0} calculated to be {1}", package.PackageName, Hash);

                                    if ((!package.CRC.Equals("f") && (!Hash.Equals(package.CRC))))
                                    {
                                        //download failed, the hash doesn't match. try again
                                        if (failCount == 3)
                                        {
                                            //this is the third time the file download has failed. something is wrong here.
                                            throw new BadMemeException(string.Format("The file download hash failed to match. Downloaded = {0}, Database = {1}", Hash, package.CRC));
                                        }
                                    }
                                    else
                                    {
                                        ThrowIfCancellationRequested();
                                        Progress.Report(downloadProgress);

                                        filestream.Close();
                                        md5Hash.Clear();
                                        stream.Close();
                                        package.DownloadFlag = false;

                                        //save to local database cache file, even if the crc is "f". The loading in the selection list will skip getting the hash if the crc is f
                                        databaseManager.UpdateFileEntry(package.ZipFile, File.GetLastWriteTime(downloadLocation), Hash);
                                        databaseManager.SaveMd5Database(ApplicationConstants.MD5HashDatabaseXmlFile);

                                        //set the failCount for loop to jump out and the userRetry to stop asking
                                        failCount = 4;
{'fraction_non_alphanumeric': 0.07050307636626856, 'fraction_numerical': 0.005501266739051755, 'mean_word_length': 1.3057409879839785, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 166, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '367892', 'n_tokens_mistral': 2951, 'n_tokens_neox': 2824, 'n_words': 817}