<commit_before>﻿#region Licence
/*
    Babbacombe.Webserver
    https://github.com/trevorprinn/Babbacombe.Webserver
    Copyright © 2014 Babbacombe Computers Ltd.

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2.1 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
    USA
 */
#endregion
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Babbacombe.Webserver {

    /// <summary>
    /// Used for reading a stream that contains a mixture of text and binary.
    /// Contains methods to read text (like a StreamReader) but can then switch
    /// to transferring via a buffer until a text delimiter is reached.
    /// </summary>
    internal class TextBinaryReader : IDisposable {
        private BufferedStream _stream;

        /// <summary>
        /// True when the end of the stream has been reached. When reading binary there
        /// could still be buffered (pushed back) data to be sent.
        /// </summary>
        public bool EndOfStream { get; private set; }
        private Queue<int> _pushBackBuffer = new Queue<int>();

        public TextBinaryReader(Stream stream) {
            if (!stream.CanRead) throw new ArgumentException("TextBinaryReader must be able to read the stream");
            _stream = new BufferedStream(stream);
        }

        public TextBinaryReader(BufferedStream stream) {
            if (!stream.CanRead) throw new ArgumentException("TextBinaryReader must be able to read the stream");
            _stream = stream;
        }

        /// <summary>
        /// Reads and returns a line from the stream, not including the line delimiter.
        /// </summary>
        /// <returns>null at the end of the stream.</returns>
        public string ReadLine() {
            if (EndOfStream) return null;
            var buf = new StringBuilder();
            int ch = _stream.ReadByte();
            while (ch >= 0 && ch != '\n') {
                buf.Append((char)ch);
                ch = _stream.ReadByte();
            }
            if (ch < 0) EndOfStream = true;
            if (EndOfStream && buf.Length == 0) return null;
            while (buf.Length > 0 && buf[buf.Length - 1] == '\r') buf.Length--;
            return buf.ToString();
        }

        /// <summary>
        /// Copies up to count bytes from the stream into the buffer. Stops when
        /// the delimiter is reached (does not return it).
        /// </summary>
        /// <param name="buffer"></param>
        /// <param name="offset"></param>
        /// <param name="count"></param>
        /// <param name="delimiter">
        /// The delimiter at which the read will stop (not including an initial \n or \r\n).
        /// </param>
        /// <param name="delimiterReached">
        /// True if the position in the stream has reached (and passed) the delimiter.
        /// </param>
        /// <returns>
        /// 0 if the end of the stream has been reached.
        /// </returns>
        public int ReadBinary(byte[] buffer, int offset, int count, string delimiter, out bool delimiterReached) {
            if (string.IsNullOrEmpty(delimiter)) throw new ArgumentException("Delimiter cannot be null or empty");
            if (delimiter.Contains('\r') || delimiter.Contains('\n')) throw new ArgumentException(@"Delimiter cannot contain \r or \n");

            delimiterReached = false;
            if (EndOfStream && !_pushBackBuffer.Any()) return 0;
            
            List<int> delimiterBuffer = new List<int>();
            int delimCount = 0; // Count of how many delimiter characters have been read and matched (not inc \r\n)
            int bytesRead = 0;
            while (bytesRead < count && !delimiterReached) {
                int ch = getNextByte();
                if (ch < 0) {
                    EndOfStream = true;
                    if (delimiterBuffer.Any()) {
                        // Push back any delimiter data that has been held aside.
                        pushback(delimiterBuffer);
                        delimCount = 0;
                        continue;
                    }
                }
                if (!_pushBackBuffer.Any() && (ch == '\n' || ch == '\r')) {
                    if (delimCount == 0) {
                        // Possibly at or near start of delimiter
                        delimiterBuffer.Add(ch);
                    } else {
                        // End of line reached without matching the delimiter
                        pushback(delimiterBuffer, ch);
                        delimCount = 0;<commit_msg>Fixed new line missing at end of uploaded text file.<commit_after>﻿#region Licence
/*
    Babbacombe.Webserver
    https://github.com/trevorprinn/Babbacombe.Webserver
    Copyright © 2014 Babbacombe Computers Ltd.

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2.1 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
    USA
 */
#endregion
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Babbacombe.Webserver {

    /// <summary>
    /// Used for reading a stream that contains a mixture of text and binary.
    /// Contains methods to read text (like a StreamReader) but can then switch
    /// to transferring via a buffer until a text delimiter is reached.
    /// </summary>
    internal class TextBinaryReader : IDisposable {
        private BufferedStream _stream;

        /// <summary>
        /// True when the end of the stream has been reached. When reading binary there
        /// could still be buffered (pushed back) data to be sent.
        /// </summary>
        public bool EndOfStream { get; private set; }
        private Queue<int> _pushBackBuffer = new Queue<int>();

        public TextBinaryReader(Stream stream) {
            if (!stream.CanRead) throw new ArgumentException("TextBinaryReader must be able to read the stream");
            _stream = new BufferedStream(stream);
        }

        public TextBinaryReader(BufferedStream stream) {
            if (!stream.CanRead) throw new ArgumentException("TextBinaryReader must be able to read the stream");
            _stream = stream;
        }

        /// <summary>
        /// Reads and returns a line from the stream, not including the line delimiter.
        /// </summary>
        /// <returns>null at the end of the stream.</returns>
        public string ReadLine() {
            if (EndOfStream) return null;
            var buf = new StringBuilder();
            int ch = _stream.ReadByte();
            while (ch >= 0 && ch != '\n') {
                buf.Append((char)ch);
                ch = _stream.ReadByte();
            }
            if (ch < 0) EndOfStream = true;
            if (EndOfStream && buf.Length == 0) return null;
            while (buf.Length > 0 && buf[buf.Length - 1] == '\r') buf.Length--;
            return buf.ToString();
        }

        /// <summary>
        /// Copies up to count bytes from the stream into the buffer. Stops when
        /// the delimiter is reached (does not return it).
        /// </summary>
        /// <param name="buffer"></param>
        /// <param name="offset"></param>
        /// <param name="count"></param>
        /// <param name="delimiter">
        /// The delimiter at which the read will stop (not including an initial \n or \r\n).
        /// </param>
        /// <param name="delimiterReached">
        /// True if the position in the stream has reached (and passed) the delimiter.
        /// </param>
        /// <returns>
        /// 0 if the end of the stream has been reached.
        /// </returns>
        public int ReadBinary(byte[] buffer, int offset, int count, string delimiter, out bool delimiterReached) {
            if (string.IsNullOrEmpty(delimiter)) throw new ArgumentException("Delimiter cannot be null or empty");
            if (delimiter.Contains('\r') || delimiter.Contains('\n')) throw new ArgumentException(@"Delimiter cannot contain \r or \n");

            delimiterReached = false;
            if (EndOfStream && !_pushBackBuffer.Any()) return 0;
            
            List<int> delimiterBuffer = new List<int>();
            int delimCount = 0; // Count of how many delimiter characters have been read and matched (not inc \r\n)
            int bytesRead = 0;
            while (bytesRead < count && !delimiterReached) {
                int ch = getNextByte();
                if (ch < 0) {
                    EndOfStream = true;
                    if (delimiterBuffer.Any()) {
                        // Push back any delimiter data that has been held aside.
                        pushback(delimiterBuffer);
                        delimCount = 0;
                        continue;
                    }
                }
                if (!_pushBackBuffer.Any() && (ch == '\n' || ch == '\r')) {
                    if (delimiterBuffer.Contains('\n')) {
                        // Found 2 new lines in a row
                        pushback(delimiterBuffer, ch);
                        delimCount = 0;
                    } else if (delimCount == 0) {
                        // Possibly at or near start of delimiter
                        delimiterBuffer.Add(ch);
                    } else {
                        // End of line reached without matching the delimiter
                        pushback(delimiterBuffer, ch);
                        delimCount = 0;
{'fraction_non_alphanumeric': 0.0967562041081885, 'fraction_numerical': 0.005762617343619296, 'mean_word_length': 1.8353096179183135, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 59, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4527051', 'n_tokens_mistral': 2733, 'n_tokens_neox': 2624, 'n_words': 1066}