<commit_before>﻿using System;
using System.IO;
using System.Diagnostics;
using System.Collections.Generic;
using System.Linq;

namespace Pdelvo.Minecraft.Network
{
    public class FullyReadStream : Stream
    {
        private readonly byte[] _readAheadBuffer;
        private Stream _sourceStream;
        private long _pos; // pseudo-position
        private int _readAheadLength;
        private int _readAheadOffset;
        private List<byte> _buffer = new List<byte>();

        public FullyReadStream(Stream sourceStream)
        {
            _sourceStream = sourceStream;
            _readAheadBuffer = new byte[1024*1024];
        }

        public Stream BaseStream
        {
            get { return _sourceStream; }
            set { _sourceStream = value; }
        }

        public override bool CanRead
        {
            get { return true; }
        }

        public override bool CanSeek
        {
            get { return false; }
        }

        public override bool CanWrite
        {
            get { return false; }
        }

        public override long Length
        {
            get { return _pos; }
        }

        public override long Position
        {
            get { return _pos; }
            set { throw new InvalidOperationException(); }
        }

        public override void Flush()
        {
            _sourceStream.Write(_buffer.ToArray(), 0, _buffer.Count);
            _sourceStream.Flush();
            _buffer.Clear();
        }

        //[DebuggerStepThrough]
        public override int Read(byte[] buffer, int offset, int count)
        {
            int bytesRead = 0;
            int bytesRequired = count - bytesRead;
            while (bytesRead < count)
            {
                int readAheadAvailableBytes = _readAheadLength - _readAheadOffset;

                if (readAheadAvailableBytes > 0)
                {
                    int toCopy = Math.Min(readAheadAvailableBytes, bytesRequired);
                    //Array.Copy(readAheadBuffer, readAheadOffset, buffer, offset + bytesRead, toCopy);
                    Buffer.BlockCopy(_readAheadBuffer, _readAheadOffset, buffer, offset + bytesRead, toCopy);
                    bytesRead += toCopy;
                    _readAheadOffset += toCopy;
                }
                else
                {
                    ReadData(bytesRequired);
                }
            }
            _pos += bytesRead;
            return bytesRead;
        }

        private void ReadData(int maxCount)
        {
            _readAheadOffset = 0;
            _readAheadLength = _sourceStream.Read(_readAheadBuffer, 0, maxCount);
            if (_readAheadLength == 0)
                throw new EndOfStreamException();<commit_msg>Fixed FullyReadStream<commit_after>﻿using System;
using System.IO;
using System.Diagnostics;
using System.Collections.Generic;
using System.Linq;

namespace Pdelvo.Minecraft.Network
{
    public class FullyReadStream : Stream
    {
        private readonly byte[] _readAheadBuffer;
        private Stream _sourceStream;
        private long _pos; // pseudo-position
        private int _readAheadLength;
        private int _readAheadOffset;
        private List<byte> _buffer = new List<byte>();

        public FullyReadStream(Stream sourceStream)
        {
            _sourceStream = sourceStream;
            _readAheadBuffer = new byte[1024*1024];
        }

        public Stream BaseStream
        {
            get { return _sourceStream; }
            set { _sourceStream = value; }
        }

        public override bool CanRead
        {
            get { return true; }
        }

        public override bool CanSeek
        {
            get { return false; }
        }

        public override bool CanWrite
        {
            get { return false; }
        }

        public override long Length
        {
            get { return _pos; }
        }

        public override long Position
        {
            get { return _pos; }
            set { throw new InvalidOperationException(); }
        }

        public override void Flush()
        {
            _sourceStream.Write(_buffer.ToArray(), 0, _buffer.Count);
            _sourceStream.Flush();
            _buffer.Clear();
        }

        //[DebuggerStepThrough]
        public override int Read(byte[] buffer, int offset, int count)
        {
            int bytesRead = 0;
            while (bytesRead < count)
            {
                int readAheadAvailableBytes = _readAheadLength - _readAheadOffset;

                if (readAheadAvailableBytes > 0)
                {
                    int bytesRequired = count - bytesRead;
                    int toCopy = Math.Min(readAheadAvailableBytes, bytesRequired);
                    //Array.Copy(readAheadBuffer, readAheadOffset, buffer, offset + bytesRead, toCopy);
                    Buffer.BlockCopy(_readAheadBuffer, _readAheadOffset, buffer, offset + bytesRead, toCopy);
                    bytesRead += toCopy;
                    _readAheadOffset += toCopy;
                }
                else
                {
                    ReadData(count);
                }
            }
            _pos += bytesRead;
            return bytesRead;
        }

        private void ReadData(int maxCount)
        {
            _readAheadOffset = 0;
            _readAheadLength = _sourceStream.Read(_readAheadBuffer, 0, maxCount);
            if (_readAheadLength == 0)
                throw new EndOfStreamException();
{'fraction_non_alphanumeric': 0.07217990569459558, 'fraction_numerical': 0.005077983315197679, 'mean_word_length': 1.511384335154827, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7048125', 'n_tokens_mistral': 1419, 'n_tokens_neox': 1347, 'n_words': 344}