<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Text;
// Launchpad files are all LZMA compressed. I've included it as binary for easier building of this project.
// SevenZip (lzma.dll) is compiled from v9.20 of the LZMA SDK http://www.7-zip.org/sdk.html
using SevenZip;

namespace Everquest
{
    public class LaunchpadPatcher
    {
        /// <param name="server">null for the live servers. "-test" for test server.</param>
        public static void DownloadManifest(string server, string saveToPath)
        {
            string url = String.Format("http://manifest.patch.station.sony.com/patch/sha/manifest/eq/eq-en{0}/live/eq-en{0}.sha.soe", server);
            DownloadFile(url, saveToPath);
        }

        public static void DownloadFile(string url, string saveToPath)
        {
            Console.Error.WriteLine("=> " + url);

            using (WebClient web = new WebClient())
            {
                web.Headers["User-Agent"] = "Quicksilver Player/1.0.3.183";
                web.Proxy = null;
                byte[] data = web.DownloadData(url);
                Stream inStream = new MemoryStream(data);
                using (FileStream outStream = new FileStream(saveToPath, FileMode.Create))
                {
                    // Launchpad files are LZMA compressed
                    Decompress(inStream, outStream);
                    Console.Error.WriteLine("   {2} [{0} bytes] {1}", outStream.Length, web.ResponseHeaders["Last-Modified"], saveToPath);
                    Console.Error.WriteLine();
                }

                DateTime lastMod;
                if (DateTime.TryParse(web.ResponseHeaders["Last-Modified"], out lastMod))
                    File.SetLastWriteTimeUtc(saveToPath, lastMod);
            }
        }

        private static void Decompress(Stream inStream, Stream outStream)
        {
            byte[] properties = new byte[5];
            if (inStream.Read(properties, 0, 5) != 5)
                throw new Exception("Input .lzma is too short");

            SevenZip.Compression.LZMA.Decoder decoder = new SevenZip.Compression.LZMA.Decoder();
            decoder.SetDecoderProperties(properties);
            long outSize = 0;
            for (int i = 0; i < 8; i++)
            {
                int v = inStream.ReadByte();
                outSize |= ((long)(byte)v) << (8 * i);
            }
            long compressedSize = inStream.Length - inStream.Position;<commit_msg>-patcher should parse LastModified timestamp in invariant culture<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Text;
// Launchpad files are all LZMA compressed. I've included it as binary for easier building of this project.
// SevenZip (lzma.dll) is compiled from v9.20 of the LZMA SDK http://www.7-zip.org/sdk.html
using SevenZip;
using System.Globalization;

namespace Everquest
{
    public class LaunchpadPatcher
    {
        /// <param name="server">null for the live servers. "-test" for test server.</param>
        public static void DownloadManifest(string server, string saveToPath)
        {
            string url = String.Format("http://manifest.patch.station.sony.com/patch/sha/manifest/eq/eq-en{0}/live/eq-en{0}.sha.soe", server);
            DownloadFile(url, saveToPath);
        }

        public static void DownloadFile(string url, string saveToPath)
        {
            Console.Error.WriteLine("=> " + url);

            using (WebClient web = new WebClient())
            {
                web.Headers["User-Agent"] = "Quicksilver Player/1.0.3.183";
                web.Proxy = null;
                byte[] data = web.DownloadData(url);
                Stream inStream = new MemoryStream(data);
                using (FileStream outStream = new FileStream(saveToPath, FileMode.Create))
                {
                    // Launchpad files are LZMA compressed
                    Decompress(inStream, outStream);
                    Console.Error.WriteLine("   {2} [{0:#,#} bytes] {1}", outStream.Length, web.ResponseHeaders["Last-Modified"], saveToPath);
                    Console.Error.WriteLine();
                }

                // this timestamp may be different than what was reported in the manifest
                DateTime lastMod;
                if (DateTime.TryParse(web.ResponseHeaders["Last-Modified"], CultureInfo.InvariantCulture, DateTimeStyles.None, out lastMod))
                    File.SetLastWriteTimeUtc(saveToPath, lastMod);
            }
        }

        private static void Decompress(Stream inStream, Stream outStream)
        {
            byte[] properties = new byte[5];
            if (inStream.Read(properties, 0, 5) != 5)
                throw new Exception("Input .lzma is too short");

            SevenZip.Compression.LZMA.Decoder decoder = new SevenZip.Compression.LZMA.Decoder();
            decoder.SetDecoderProperties(properties);
            long outSize = 0;
            for (int i = 0; i < 8; i++)
            {
                int v = inStream.ReadByte();
                outSize |= ((long)(byte)v) << (8 * i);
            }
            long compressedSize = inStream.Length - inStream.Position;
{'fraction_non_alphanumeric': 0.10861708198592353, 'fraction_numerical': 0.008750237778200495, 'mean_word_length': 2.0028555111364934, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2908538', 'n_tokens_mistral': 1466, 'n_tokens_neox': 1366, 'n_words': 368}