<commit_before>﻿/*
 * Copyright (c) 2013-2015, SteamDB. All rights reserved.
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Security.Cryptography;
using System.Threading.Tasks;
using SteamKit2;

namespace SteamDatabaseBackend
{
    static class FileDownloader
    {
        private const string FILES_DIRECTORY = "files";

        private static readonly Dictionary<uint, List<string>> ImportantDepots = new Dictionary<uint, List<string>>
        {
            // Team Fortress 2
            {
                232252,
                new List<string>
                {
                    "tf/bin/client.dylib",
                    "tf/bin/server.dylib"
                }
            },
            {
                441,
                new List<string>
                {
                    "tf/steam.inf",
                    "tf/resource/tf_english.txt",
                    "tf/tf2_misc_dir.vpk",
                    "tf/tf2_sound_misc_dir.vpk",
                    "tf/tf2_sound_vo_english_dir.vpk",
                    "tf/tf2_textures_dir.vpk"
                }
            },
            // Dota 2
            {
                574,
                new List<string>
                {
                    "bin/engine.dylib",
                    "dota/bin/client.dylib",
                    "dota/bin/server.dylib"
                }
            },
            {
                571,
                new List<string>
                {
                    "dota/steam.inf",
                    "dota/resource/dota_english.txt",
                    "dota/pak01_dir.vpk"
                }
            },
            // Dota 2 Workshop
            {
                313250,
                new List<string>
                {
                    "dota_ugc/game/bin/win64/engine2.dll",
                    "dota_ugc/game/bin/win64/vphysics2.dll",
                    "dota_ugc/game/dota/bin/win64/client.dll",
                    "dota_ugc/game/dota/bin/win64/server.dll"
                }
            },
            // Dota 2 Test
            {
                205794,
                new List<string>
                {
                    "bin/engine.dylib",
                    "dota/bin/client.dylib",
                    "dota/bin/server.dylib"
                }
            },
            {
                205791,
                new List<string>
                {
                    "dota/steam.inf",
                    "dota/resource/dota_english.txt",
                    "dota/pak01_dir.vpk"
                }
            },
            // Counter-Strike: Global Offensive
            {
                733,
                new List<string>
                {
                    "bin/engine.dylib",
                    "csgo/bin/client.dylib",
                    "csgo/bin/server.dylib"
                }
            },
            {
                731,
                new List<string>
                {
                    "csgo/steam.inf",
                    "csgo/resource/csgo_english.txt",
                    "csgo/pak01_dir.vpk"
                }<commit_msg>Download item schemas<commit_after>﻿/*
 * Copyright (c) 2013-2015, SteamDB. All rights reserved.
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Security.Cryptography;
using System.Threading.Tasks;
using SteamKit2;

namespace SteamDatabaseBackend
{
    static class FileDownloader
    {
        private const string FILES_DIRECTORY = "files";

        private static readonly Dictionary<uint, List<string>> ImportantDepots = new Dictionary<uint, List<string>>
        {
            // Team Fortress 2
            {
                232252,
                new List<string>
                {
                    "tf/bin/client.dylib",
                    "tf/bin/server.dylib"
                }
            },
            {
                441,
                new List<string>
                {
                    "tf/steam.inf",
                    "tf/scripts/items/items_game.txt",
                    "tf/resource/tf_english.txt",
                    "tf/tf2_misc_dir.vpk",
                    "tf/tf2_sound_misc_dir.vpk",
                    "tf/tf2_sound_vo_english_dir.vpk",
                    "tf/tf2_textures_dir.vpk"
                }
            },
            // Dota 2
            {
                574,
                new List<string>
                {
                    "bin/engine.dylib",
                    "dota/bin/client.dylib",
                    "dota/bin/server.dylib"
                }
            },
            {
                571,
                new List<string>
                {
                    "dota/steam.inf",
                    "dota/resource/items_english.txt",
                    "dota/resource/dota_english.txt",
                    "dota/pak01_dir.vpk"
                }
            },
            // Dota 2 Workshop
            {
                313250,
                new List<string>
                {
                    "dota_ugc/game/bin/win64/engine2.dll",
                    "dota_ugc/game/bin/win64/vphysics2.dll",
                    "dota_ugc/game/dota/bin/win64/client.dll",
                    "dota_ugc/game/dota/bin/win64/server.dll"
                }
            },
            // Dota 2 Test
            {
                205794,
                new List<string>
                {
                    "bin/engine.dylib",
                    "dota/bin/client.dylib",
                    "dota/bin/server.dylib"
                }
            },
            {
                205791,
                new List<string>
                {
                    "dota/steam.inf",
                    "dota/resource/items_english.txt",
                    "dota/resource/dota_english.txt",
                    "dota/pak01_dir.vpk"
                }
            },
            // Counter-Strike: Global Offensive
            {
                733,
                new List<string>
                {
                    "bin/engine.dylib",
                    "csgo/bin/client.dylib",
                    "csgo/bin/server.dylib"
                }
            },
            {
                731,
                new List<string>
                {
                    "csgo/steam.inf",
                    "csgo/scripts/items/items_game.txt",
                    "csgo/resource/csgo_english.txt",
                    "csgo/pak01_dir.vpk"
                }
{'fraction_non_alphanumeric': 0.09355692850838482, 'fraction_numerical': 0.02118270079435128, 'mean_word_length': 0.8622295261572172, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '747093', 'n_tokens_mistral': 2037, 'n_tokens_neox': 1819, 'n_words': 276}