<commit_before>﻿using System.IO;
using System.IO.Compression;

namespace Hybrasyl
{
    public static class ZlibCompression
    {
        public static void Compress(Stream originalStream, Stream compressedStream)
        {
            compressedStream.Write(new byte[] { 0x78, 0x9C }, 0, 2);
            using (var compressionStream = new DeflateStream(compressedStream, CompressionMode.Compress, true))
            {
                originalStream.Seek(0, SeekOrigin.Begin);
                originalStream.CopyTo(compressionStream);
            }
        }

        public static void Decompress(Stream originalStream, Stream decompressedStream)
        {
            originalStream.Seek(2, SeekOrigin.Begin);
            using (var decompressionStream = new DeflateStream(originalStream, CompressionMode.Decompress, true))
            {
                decompressionStream.CopyTo(decompressedStream);
            }
        }
    }<commit_msg>Fix incomplete zlib compression (#102)<commit_after>﻿using System.IO;
using System.IO.Compression;

namespace Hybrasyl
{
    public static class ZlibCompression
    {
        public static void Compress(Stream originalStream, Stream compressedStream)
        {
            uint checksum = 0;

            using (var checksumStream = new MemoryStream())
            {
                var position = originalStream.Position;
                originalStream.CopyTo(checksumStream);
                originalStream.Seek(position, SeekOrigin.Begin);
                var buffer = checksumStream.ToArray();
                checksum = Adler32.ComputeHash(buffer);
            }

            compressedStream.Write(new byte[] { 0x78, 0x9C }, 0, 2);
            using (var compressionStream = new DeflateStream(compressedStream, CompressionMode.Compress, true))
            {
                originalStream.Seek(0, SeekOrigin.Begin);
                originalStream.CopyTo(compressionStream);
            }

            compressedStream.Write(new byte[]
            {
                    (byte)(checksum >> 24),
                    (byte)(checksum >> 16),
                    (byte)(checksum >> 8),
                    (byte)checksum
            }, 0, 4);
        }

        public static void Decompress(Stream originalStream, Stream decompressedStream)
        {
            originalStream.Seek(2, SeekOrigin.Begin);
            using (var decompressionStream = new DeflateStream(originalStream, CompressionMode.Decompress, true))
            {
                decompressionStream.CopyTo(decompressedStream);
            }
            originalStream.Seek(4, SeekOrigin.Current);
        }
    }

    public static class Adler32
    {
        public static uint ComputeHash(byte[] buffer) =>
            ComputeHash(buffer, 0, buffer.Length);

        public static uint ComputeHash(byte[] buffer, int offset, int count)
        {
            uint checksum = 1;

            int n;
            uint s1 = checksum & 0xFFFF;
            uint s2 = checksum >> 16;

            while (count > 0)
            {
                n = (3800 > count) ? count : 3800;
                count -= n;

                while (--n >= 0)
                {
                    s1 = s1 + (uint)(buffer[offset++] & 0xFF);
                    s2 = s2 + s1;
                }

                s1 %= 65521;
                s2 %= 65521;
            }

            checksum = (s2 << 16) | s1;
            return checksum;
        }
    }
{'fraction_non_alphanumeric': 0.08829509148997967, 'fraction_numerical': 0.02120243973279117, 'mean_word_length': 1.6431312356101304, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6576798', 'n_tokens_mistral': 958, 'n_tokens_neox': 871, 'n_words': 200}