<commit_before>﻿/*
 * Copyright (c) 2013-2015, SteamDB. All rights reserved.
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using SteamKit2;

namespace SteamDatabaseBackend
{
    static class FileDownloader
    {
        private static readonly Dictionary<uint, List<string>> ImportantDepots = new Dictionary<uint, List<string>>
        {
            // Team Fortress 2
            {
                232252,
                new List<string>
                {
                    //"tf/bin/client.dylib",
                    "tf/bin/server.dylib"
                }
            },
            {
                441,
                new List<string>
                {
                    "tf/steam.inf",
                    "tf/resource/tf_english.txt"
                }
            },
            // Dota 2
            {
                574,
                new List<string>
                {
                    "bin/engine.dylib",
                    //"dota/bin/client.dylib",
                    "dota/bin/server.dylib"
                }
            },
            {
                571,
                new List<string>
                {
                    "dota/steam.inf",
                    "dota/resource/dota_english.txt"
                }
            },
            // Dota 2 Workshop
            {
                313250,
                new List<string>
                {
                    "dota_ugc/game/dota/bin/win64/server.dll",
                    "dota_ugc/game/bin/win64/engine2.dll"
                }
            },
            // Counter-Strike: Global Offensive
            {
                733,
                new List<string>
                {
                    "bin/engine.dylib",
                    //"csgo/bin/client.dylib",
                    "csgo/bin/server.dylib",
                    "csgo/bin/matchmaking.dylib"
                }
            },
            {
                731,
                new List<string>
                {
                    "csgo/steam.inf",
                    "csgo/resource/csgo_english.txt"
                }
            },
        };

        private static CDNClient CDNClient;

        public static void SetCDNClient(CDNClient cdnClient)
        {
            CDNClient = cdnClient;
        }

        public static bool IsImportantDepot(uint depotID)
        {
            return ImportantDepots.ContainsKey(depotID);
        }

        public static void DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
        {
            var files = depotManifest.Files.Where(x => ImportantDepots[job.DepotID].Contains(x.FileName.Replace('\\', '/'))).ToList();
            var filesUpdated = false;

            Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count(), job.DepotID);

            foreach (var file in files)
            {
                string directory = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "files", job.DepotID.ToString());
                string downloadPath = Path.Combine(directory, string.Concat("staged_", Path.GetFileName(file.FileName)));

                if (!Directory.Exists(directory))
                {
                    Directory.CreateDirectory(directory);
                }
                else if(File.Exists(downloadPath))
                {
                    Log.WriteDebug("FileDownloader", "Removing stale {0}", file.FileName);

                    File.Delete(downloadPath);
                }

                Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", Path.GetFileName(file.FileName), file.TotalSize, file.Chunks.Count);

                uint count = 0;

                using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate))
                {
                    fs.SetLength((long)file.TotalSize);<commit_msg>Remove matchmaking.dylib; print full filename<commit_after>﻿/*
 * Copyright (c) 2013-2015, SteamDB. All rights reserved.
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using SteamKit2;

namespace SteamDatabaseBackend
{
    static class FileDownloader
    {
        private static readonly Dictionary<uint, List<string>> ImportantDepots = new Dictionary<uint, List<string>>
        {
            // Team Fortress 2
            {
                232252,
                new List<string>
                {
                    //"tf/bin/client.dylib",
                    "tf/bin/server.dylib"
                }
            },
            {
                441,
                new List<string>
                {
                    "tf/steam.inf",
                    "tf/resource/tf_english.txt"
                }
            },
            // Dota 2
            {
                574,
                new List<string>
                {
                    "bin/engine.dylib",
                    //"dota/bin/client.dylib",
                    "dota/bin/server.dylib"
                }
            },
            {
                571,
                new List<string>
                {
                    "dota/steam.inf",
                    "dota/resource/dota_english.txt"
                }
            },
            // Dota 2 Workshop
            {
                313250,
                new List<string>
                {
                    "dota_ugc/game/dota/bin/win64/server.dll",
                    "dota_ugc/game/bin/win64/engine2.dll"
                }
            },
            // Counter-Strike: Global Offensive
            {
                733,
                new List<string>
                {
                    "bin/engine.dylib",
                    //"csgo/bin/client.dylib",
                    "csgo/bin/server.dylib"
                }
            },
            {
                731,
                new List<string>
                {
                    "csgo/steam.inf",
                    "csgo/resource/csgo_english.txt"
                }
            },
        };

        private static CDNClient CDNClient;

        public static void SetCDNClient(CDNClient cdnClient)
        {
            CDNClient = cdnClient;
        }

        public static bool IsImportantDepot(uint depotID)
        {
            return ImportantDepots.ContainsKey(depotID);
        }

        public static void DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
        {
            var files = depotManifest.Files.Where(x => ImportantDepots[job.DepotID].Contains(x.FileName.Replace('\\', '/'))).ToList();
            var filesUpdated = false;

            Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count(), job.DepotID);

            foreach (var file in files)
            {
                string directory = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "files", job.DepotID.ToString());
                string downloadPath = Path.Combine(directory, string.Concat("staged_", Path.GetFileName(file.FileName)));

                if (!Directory.Exists(directory))
                {
                    Directory.CreateDirectory(directory);
                }
                else if(File.Exists(downloadPath))
                {
                    Log.WriteDebug("FileDownloader", "Removing stale {0}", file.FileName);

                    File.Delete(downloadPath);
                }

                Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, file.Chunks.Count);

                uint count = 0;

                using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate))
                {
                    fs.SetLength((long)file.TotalSize);
{'fraction_non_alphanumeric': 0.098875, 'fraction_numerical': 0.01275, 'mean_word_length': 1.177142857142857, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7529476', 'n_tokens_mistral': 2222, 'n_tokens_neox': 2030, 'n_words': 382}