<commit_before>﻿using System;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;

namespace InvertedTomato.Feather {
    public sealed class Payload {
        public int Length { get { return (int)Inner.Length; } }
        public int Position { get { return (int)Inner.Position; } }
        public int Remaining { get { return (int)(Inner.Length - Inner.Position); } }
        public byte Opcode { get; }

        private MemoryStream Inner;

        /// <summary>
        /// Initialize payload in read-only mode with a byte-array equivalent.
        /// </summary>
        public Payload(byte[] payload) {
            if (null == payload) {
                throw new ArgumentNullException("payload");
            }
            if (payload.Length < 1) {
                throw new ArgumentException("Must be at least one byte long.", "payload");
            }

            // Setup inner store
            Inner = new MemoryStream(payload, false);
            Inner.Position = 1;

            // Note opcode separately
            Opcode = payload[0];
        }
        
        // TODO: fix unit tests

        public Payload(byte opcode) {
            // Setup inner store
            Inner = new MemoryStream();
            Inner.Write(opcode);

            // Note opcode separately
            Opcode = opcode;
        }

        public void Rewind() {
            Inner.Position = 1;
        }


        public byte[] ReadByteArray() {
            var valueLength = Inner.ReadUInt16();
            var rawValue = Inner.Read(valueLength);
            return rawValue;
        }
        public Payload Append(byte[] value) {
            if (null == value) {
                throw new ArgumentNullException("value");
            }

            var rawLength = (ushort)value.Length;
            Inner.Write(rawLength);
            Inner.Write(value);
            return this; // Allow chaining
        }
        public byte[] ReadNullableByteArray() {
            var hasValue = Inner.ReadByte() > 0;
            if (!hasValue) {
                return null;
            }

            return ReadByteArray();
        }
        public Payload AppendNullable(byte[] value) {
            if (null == value) {
                Inner.Write(0x00);
                return this; // Allow chaining
            }

            Inner.Write(0x01);
            return Append(value); // Allow chaining
        }
        public byte[] ReadByteArrayFixedLength(int count) {
            return Inner.Read(count);
        }
        public Payload AppendFixedLength(byte[] value) {
            if (null == value) {
                throw new ArgumentNullException("value");
            }

            Inner.Write(value);
            return this; // Allow chaining
        }

        public DateTime ReadDateTime() {
            var rawValue = Inner.ReadUInt64();
            return DateUtility.FromUnixTimestamp(rawValue);
        }
        public Payload Append(DateTime value) {
            var rawValue = value.ToUnixTimeAsUInt64();
            Inner.Write(rawValue);
            return this; // Allow chaining
        }
        public DateTime? ReadNullableDateTime() {
            var hasValue = Inner.ReadByte() > 0;
            if (!hasValue) {
                return null;
            }

            return ReadDateTime();
        }
        public Payload AppendNullable(DateTime? value) {
            if (!value.HasValue) {
                Inner.Write(0x00);
                return this; // Allow chaining
            }

            Inner.Write(0x01);
            return Append(value.Value); // Allow chaining
        }

        public TimeSpan ReadTimeSpan() {
            var rawValue = Inner.ReadUInt32();
            return TimeSpan.FromSeconds(rawValue);
        }
        public Payload Append(TimeSpan value) {
            var rawValue = (int)value.TotalSeconds;
            Inner.Write(rawValue);
            return this; // Allow chaining
        }
        public TimeSpan? ReadNullableTimeSpan() {
            var hasValue = Inner.ReadByte() > 0;
            if (!hasValue) {
                return null;
            }

            return ReadTimeSpan();
        }
        public Payload AppendNullable(TimeSpan? value) {
            if (!value.HasValue) {
                Inner.Write(0x00);
                return this; // Allow chaining
            }

            Inner.Write(0x01);
            return Append(value.Value); // Allow chaining
        }

        public string ReadString() {
            var valueLength = Inner.ReadUInt16();
            var rawValue = Inner.Read(valueLength);
            return Encoding.UTF8.GetString(rawValue);
        }
        public Payload Append(string value) {
            if (null == value) {
                throw new ArgumentNullException("Value cannot be null. If this was intentional use WriteNullable instead.", "value");
            }

            var rawValue = Encoding.UTF8.GetBytes(value);
            Inner.Write((ushort)rawValue.Length);
            Inner.Write(rawValue);
            return this; // Allow chaining
        }
        public string ReadNullableString() {
            var hasValue = Inner.ReadUInt8() > 0;
            if (!hasValue) {
                return null;
            }
            return ReadString();<commit_msg>Added 65KB limit checking.<commit_after>﻿using System;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;

namespace InvertedTomato.Feather {
    public sealed class Payload {
        public int Length { get { return (int)Inner.Length; } }
        public int Position { get { return (int)Inner.Position; } }
        public int Remaining { get { return (int)(Inner.Length - Inner.Position); } }
        public byte Opcode { get; }

        private MemoryStream Inner;

        /// <summary>
        /// Initialize payload in read-only mode with a byte-array equivalent.
        /// </summary>
        public Payload(byte[] payload) {
            if (null == payload) {
                throw new ArgumentNullException("payload");
            }
            if (payload.Length < 1) {
                throw new ArgumentException("Must be at least one byte long.", "payload");
            }

            // Setup inner store
            Inner = new MemoryStream(payload, false);
            Inner.Position = 1;

            // Note opcode separately
            Opcode = payload[0];
        }
        
        // TODO: fix unit tests

        public Payload(byte opcode) {
            // Setup inner store
            Inner = new MemoryStream();
            Inner.Write(opcode);

            // Note opcode separately
            Opcode = opcode;
        }

        public void Rewind() {
            Inner.Position = 1;
        }


        public byte[] ReadByteArray() {
            var valueLength = Inner.ReadUInt16();
            var rawValue = Inner.Read(valueLength);
            return rawValue;
        }
        public Payload Append(byte[] value) {
            if (null == value) {
                throw new ArgumentNullException("value");
            }
            if (value.Length > ushort.MaxValue) {
                throw new InternalBufferOverflowException("Must be less than 65KB.");
            }

            var rawLength = (ushort)value.Length;
            Inner.Write(rawLength);
            Inner.Write(value);
            return this; // Allow chaining
        }
        public byte[] ReadNullableByteArray() {
            var hasValue = Inner.ReadByte() > 0;
            if (!hasValue) {
                return null;
            }

            return ReadByteArray();
        }
        public Payload AppendNullable(byte[] value) {
            if (null == value) {
                Inner.Write(0x00);
                return this; // Allow chaining
            }
            if (value.Length > ushort.MaxValue) {
                throw new InternalBufferOverflowException("Must be less than 65KB.");
            }

            Inner.Write(0x01);
            return Append(value); // Allow chaining
        }
        public byte[] ReadByteArrayFixedLength(int count) {
            return Inner.Read(count);
        }
        public Payload AppendFixedLength(byte[] value) {
            if (null == value) {
                throw new ArgumentNullException("value");
            }

            Inner.Write(value);
            return this; // Allow chaining
        }

        public DateTime ReadDateTime() {
            var rawValue = Inner.ReadUInt64();
            return DateUtility.FromUnixTimestamp(rawValue);
        }
        public Payload Append(DateTime value) {
            var rawValue = value.ToUnixTimeAsUInt64();
            Inner.Write(rawValue);
            return this; // Allow chaining
        }
        public DateTime? ReadNullableDateTime() {
            var hasValue = Inner.ReadByte() > 0;
            if (!hasValue) {
                return null;
            }

            return ReadDateTime();
        }
        public Payload AppendNullable(DateTime? value) {
            if (!value.HasValue) {
                Inner.Write(0x00);
                return this; // Allow chaining
            }

            Inner.Write(0x01);
            return Append(value.Value); // Allow chaining
        }

        public TimeSpan ReadTimeSpan() {
            var rawValue = Inner.ReadUInt32();
            return TimeSpan.FromSeconds(rawValue);
        }
        public Payload Append(TimeSpan value) {
            var rawValue = (int)value.TotalSeconds;
            Inner.Write(rawValue);
            return this; // Allow chaining
        }
        public TimeSpan? ReadNullableTimeSpan() {
            var hasValue = Inner.ReadByte() > 0;
            if (!hasValue) {
                return null;
            }

            return ReadTimeSpan();
        }
        public Payload AppendNullable(TimeSpan? value) {
            if (!value.HasValue) {
                Inner.Write(0x00);
                return this; // Allow chaining
            }

            Inner.Write(0x01);
            return Append(value.Value); // Allow chaining
        }

        public string ReadString() {
            var valueLength = Inner.ReadUInt16();
            var rawValue = Inner.Read(valueLength);
            return Encoding.UTF8.GetString(rawValue);
        }
        public Payload Append(string value) {
            if (null == value) {
                throw new ArgumentNullException("Value cannot be null. If this was intentional use WriteNullable instead.", "value");
            }

            var rawValue = Encoding.UTF8.GetBytes(value);
            if (rawValue.Length > ushort.MaxValue) {
                throw new InternalBufferOverflowException("Must be less than 65KB when encoded in UTF8.");
            }
            Inner.Write((ushort)rawValue.Length);
            Inner.Write(rawValue);
            return this; // Allow chaining
        }
        public string ReadNullableString() {
            var hasValue = Inner.ReadUInt8() > 0;
            if (!hasValue) {
                return null;
            }
            return ReadString();
{'fraction_non_alphanumeric': 0.09446693657219973, 'fraction_numerical': 0.007827260458839407, 'mean_word_length': 1.4884710096261473, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1763781', 'n_tokens_mistral': 2870, 'n_tokens_neox': 2709, 'n_words': 730}