<commit_before>﻿//
// Copyright (c) 2008-2009, <NAME>
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//

using System;
using System.IO;

namespace DiscUtils.Wim
{
    /// <summary>
    /// Provides access to a (compressed) resource within the WIM file.
    /// </summary>
    /// <remarks>Stream access must be strictly sequential.</remarks>
    internal class FileResourceStream : Stream
    {
        private const int ChunkSize = 32768;
        private const int E8DecodeFileSize = 12000000;

        private Stream _baseStream;
        private ShortResourceHeader _header;
        private bool _lzxCompression;

        private long[] _chunkOffsets;
        private long[] _chunkLength;
        private long _offsetDelta;

        private int _currentChunk;
        private Stream _currentChunkStream;

        private long _position;

        public FileResourceStream(Stream baseStream, ShortResourceHeader header, bool lzxCompression)
        {
            _baseStream = baseStream;
            _header = header;
            _lzxCompression = lzxCompression;

            if (baseStream.Length > uint.MaxValue)
            {
                throw new NotImplementedException("Large files >4GB");
            }

            int numChunks = (int)Utilities.Ceil(header.OriginalSize, ChunkSize);

            _chunkOffsets = new long[numChunks];
            _chunkLength = new long[numChunks];
            for (int i = 1; i < numChunks; ++i)
            {
                _chunkOffsets[i] = Utilities.ToUInt32LittleEndian(Utilities.ReadFully(_baseStream, 4), 0);
                _chunkLength[i - 1] = _chunkOffsets[i] - _chunkOffsets[i - 1];
            }
            _chunkLength[numChunks - 1] = (_baseStream.Length - _baseStream.Position) - _chunkOffsets[numChunks - 1];
            _offsetDelta = _baseStream.Position;

            _currentChunk = -1;
        }

        public override bool CanRead
        {
            get { return true; }
        }

        public override bool CanSeek
        {
            get { return false; }
        }

        public override bool CanWrite
        {
            get { return false; }
        }

        public override void Flush()
        {
        }

        public override long Length
        {
            get { return _header.OriginalSize; }
        }

        public override long Position
        {
            get
            {
                return _position;
            }
            set
            {
                throw new NotSupportedException();
            }
        }

        public override int Read(byte[] buffer, int offset, int count)
        {
            if (_position >= Length)
            {
                return 0;
            }

            int totalRead = 0;
            while (totalRead < count)
            {
                int chunk = (int)(_position / ChunkSize);
                int chunkOffset = (int)(_position % ChunkSize);
                int numToRead = Math.Min(count - totalRead, ChunkSize - chunkOffset);<commit_msg>Wim: Fix large reads on FileResourceStream.<commit_after>﻿//
// Copyright (c) 2008-2009, <NAME>
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//

using System;
using System.IO;

namespace DiscUtils.Wim
{
    /// <summary>
    /// Provides access to a (compressed) resource within the WIM file.
    /// </summary>
    /// <remarks>Stream access must be strictly sequential.</remarks>
    internal class FileResourceStream : Stream
    {
        private const int ChunkSize = 32768;
        private const int E8DecodeFileSize = 12000000;

        private Stream _baseStream;
        private ShortResourceHeader _header;
        private bool _lzxCompression;

        private long[] _chunkOffsets;
        private long[] _chunkLength;
        private long _offsetDelta;

        private int _currentChunk;
        private Stream _currentChunkStream;

        private long _position;

        public FileResourceStream(Stream baseStream, ShortResourceHeader header, bool lzxCompression)
        {
            _baseStream = baseStream;
            _header = header;
            _lzxCompression = lzxCompression;

            if (baseStream.Length > uint.MaxValue)
            {
                throw new NotImplementedException("Large files >4GB");
            }

            int numChunks = (int)Utilities.Ceil(header.OriginalSize, ChunkSize);

            _chunkOffsets = new long[numChunks];
            _chunkLength = new long[numChunks];
            for (int i = 1; i < numChunks; ++i)
            {
                _chunkOffsets[i] = Utilities.ToUInt32LittleEndian(Utilities.ReadFully(_baseStream, 4), 0);
                _chunkLength[i - 1] = _chunkOffsets[i] - _chunkOffsets[i - 1];
            }
            _chunkLength[numChunks - 1] = (_baseStream.Length - _baseStream.Position) - _chunkOffsets[numChunks - 1];
            _offsetDelta = _baseStream.Position;

            _currentChunk = -1;
        }

        public override bool CanRead
        {
            get { return true; }
        }

        public override bool CanSeek
        {
            get { return false; }
        }

        public override bool CanWrite
        {
            get { return false; }
        }

        public override void Flush()
        {
        }

        public override long Length
        {
            get { return _header.OriginalSize; }
        }

        public override long Position
        {
            get
            {
                return _position;
            }
            set
            {
                throw new NotSupportedException();
            }
        }

        public override int Read(byte[] buffer, int offset, int count)
        {
            if (_position >= Length)
            {
                return 0;
            }

            int maxToRead = (int)Math.Min(Length - _position, count);

            int totalRead = 0;
            while (totalRead < maxToRead)
            {
                int chunk = (int)(_position / ChunkSize);
                int chunkOffset = (int)(_position % ChunkSize);
                int numToRead = Math.Min(maxToRead - totalRead, ChunkSize - chunkOffset);
{'fraction_non_alphanumeric': 0.07622880839126722, 'fraction_numerical': 0.00853762653982193, 'mean_word_length': 1.9464606539705354, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7032850', 'n_tokens_mistral': 2170, 'n_tokens_neox': 2030, 'n_words': 741}