<commit_before>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;

namespace Snappy.Sharp
{
    public class SnappyDecompressor
    {
        private const int MAX_INCREMENT_COPY_OVERFLOW = 10;
        private const int bitMask = 0x80;

        public int[] ReadUncompressedLength(byte[] data, int offset)
        {
            int sum = 0, currentShift = 0;
            while ((data[offset] & bitMask) != 0)
            {
                sum = UpdateSum(data, offset, currentShift, sum);
                offset++;
                currentShift += 7;
            }
            sum = UpdateSum(data, offset, currentShift, sum);
            offset++;
            return new[] { sum, offset };
        }

        private static int UpdateSum(byte[] data, int offset, int currentShift, int sum)
        {
            int nextValue = data[offset] & (bitMask - 1);
            nextValue <<= currentShift;
            sum += nextValue;
            return sum;
        }

        public byte[] Decompress(byte[] compressed, int compressedOffset, int compressedSize)
        {
            var sizeHeader = ReadUncompressedLength(compressed, 0);
            var data = new byte[sizeHeader[0]];

            Decompress(compressed, compressedOffset + sizeHeader[1], compressedSize - sizeHeader[1], data, 0, data.Length);

            return data;
        }
        public int Decompress(byte[] input, int inputOffset, int inputSize, byte[] output, int outputOffset, int outputLimit)
        {
            int ipLimit = inputOffset + inputSize;
            int opIndex = outputOffset;
            int ipIndex = inputOffset;

            while (ipIndex < ipLimit - 5)
            {
                byte opCode = input[ipIndex++];
                ushort entry = opLookupTable[opCode];
                byte trailerBytes = (byte) (entry >> 11);

                uint trailer = (BitConverter.ToUInt32(input, ipIndex) & wordmask[trailerBytes]);

                // advance the ipIndex past the op codes
                ipIndex += entry >> 11;
                int length = entry & 0xff;

                if ((opCode & 0x3) == Snappy.LITERAL)
                {
                    int literalLength = (int) (length + trailer);
                    CopyLiteral(input, ipIndex, output, opIndex, literalLength);
                    ipIndex += literalLength;
                    opIndex += literalLength;
                }<commit_msg>Fix decompression with nonzero compressedOffsets<commit_after>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;

namespace Snappy.Sharp
{
    public class SnappyDecompressor
    {
        private const int MAX_INCREMENT_COPY_OVERFLOW = 10;
        private const int bitMask = 0x80;

        public int[] ReadUncompressedLength(byte[] data, int offset)
        {
            int sum = 0, currentShift = 0;
            while ((data[offset] & bitMask) != 0)
            {
                sum = UpdateSum(data, offset, currentShift, sum);
                offset++;
                currentShift += 7;
            }
            sum = UpdateSum(data, offset, currentShift, sum);
            offset++;
            return new[] { sum, offset };
        }

        private static int UpdateSum(byte[] data, int offset, int currentShift, int sum)
        {
            int nextValue = data[offset] & (bitMask - 1);
            nextValue <<= currentShift;
            sum += nextValue;
            return sum;
        }

        public byte[] Decompress(byte[] compressed, int compressedOffset, int compressedSize)
        {
            var sizeHeader = ReadUncompressedLength(compressed, compressedOffset);
            var data = new byte[sizeHeader[0]];

            Decompress(compressed, sizeHeader[1], compressedSize - sizeHeader[1], data, 0, data.Length);

            return data;
        }
        public int Decompress(byte[] input, int inputOffset, int inputSize, byte[] output, int outputOffset, int outputLimit)
        {
            int ipLimit = inputOffset + inputSize;
            int opIndex = outputOffset;
            int ipIndex = inputOffset;

            while (ipIndex < ipLimit - 5)
            {
                byte opCode = input[ipIndex++];
                ushort entry = opLookupTable[opCode];
                byte trailerBytes = (byte) (entry >> 11);

                uint trailer = (BitConverter.ToUInt32(input, ipIndex) & wordmask[trailerBytes]);

                // advance the ipIndex past the op codes
                ipIndex += entry >> 11;
                int length = entry & 0xff;

                if ((opCode & 0x3) == Snappy.LITERAL)
                {
                    int literalLength = (int) (length + trailer);
                    CopyLiteral(input, ipIndex, output, opIndex, literalLength);
                    ipIndex += literalLength;
                    opIndex += literalLength;
                }
{'fraction_non_alphanumeric': 0.09197021533507747, 'fraction_numerical': 0.00986113906218555, 'mean_word_length': 1.750415052573326, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1060292', 'n_tokens_mistral': 1339, 'n_tokens_neox': 1228, 'n_words': 367}