<commit_before>﻿/*
 * Copyright 2015 ArenaNet, LLC.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this 
 * file except in compliance with the License. You may obtain a copy of the License at
 *
 * 	 http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under 
 * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 
 * ANY KIND, either express or implied. See the License for the specific language governing 
 * permissions and limitations under the License.
 */
using System;
using System.Collections.Generic;
using System.Net;
using System.IO;
using System.Text;
using Ionic.Zlib;

namespace ArenaNet.SockNet.Protocols.Gds
{
    /// <summary>
    /// Represents a Gds frame.
    /// 
    /// ===========================================================================
    /// GDS (Generic Data Stream) Application Protocol
    /// ---------------------------------------------------------------------------
    /// 
    /// +--------------------+
    /// | Frame Definition   |
    /// +--------------------+
    /// | Headers (Optional) |
    /// +--------------------+
    /// | Body (Optional)    |
    /// +--------------------+
    /// 
    /// The only part in the protocol that is required is the frame definition
    /// which is 32 bits. 
    /// 
    /// Stream
    /// ------
    /// All frames belong to a stream. Streams get created when a frame targeted
    /// towards a stream is sent across the wire. A connected system can send
    /// a "Close" frame to close a stream with a particular ID, at which point
    /// the Stream ID can be reused.
    /// 
    /// Fragmentation
    /// -------------
    /// Fragmentation is implemented by sending a 0 for the "Is Complete" field
    /// in the frame definition. The way that the body and headers behave is
    /// different while fragmented. Headers will be evaluated per fragment (which
    /// means that the headers have to be in a valid format per fragment). I.e. if
    /// fragment 1 sets the header "foo" to "bar" and fragment 2 sets "foo" to
    /// "bar2", then the value of foo will become "bar2". The body content is
    /// usually sliced between fragments - which means that the content doesn't
    /// have to be valid until the message is complete. The last fragmented
    /// frame of a message has the send a 1 for the "Is Complete" field.
    ///  
    /// Compression
    /// -----------
    /// Compression is applied to the headers using DEFLATE (or gzip) if the
    /// "Is Compressed" flag is set to 1 on the Headers block. The compression is
    /// applied to the entire Headers block (excluding the Flag and total Length) 
    /// field.
    /// 
    /// ===========================================================================
    /// Frame Definition
    /// ---------------------------------------------------------------------------
    /// 
    /// +-+-----+----+----------------------------+
    /// |C|Res  |Type| Stream Id (24)             |
    /// +-+-----+----+----------------------------+
    ///  
    /// 1) Is Complete (1-bit)
    /// Whether the message in the current stream is complete or chunked.
    ///  
    /// 2) Reserved (3-bit)
    /// Reserved for future flags.
    /// 
    /// 3) Type (4-bit)
    /// Valid types are:
    /// 0000 - RESERVED
    /// 0001 - Headers Only (No Body)
    /// 0010 - Body Only (No Headers)
    /// 0011 - Header and Body (Everything)
    /// 0100 - RESERVED
    /// 0101 - RESERVED
    /// 0110 - RESERVED
    /// 0111 - RESERVED
    /// 1000 - Ping (No Headers or Body)
    /// 1001 - Pong (No Headers or Body)
    /// 1010 - RESERVED
    /// 1011 - RESERVED
    /// 1100 - RESERVED
    /// 1101 - RESERVED
    /// 1110 - RESERVED
    /// 1111 - Close (No Headers or Body)
    /// 
    /// 4) Stream ID (24-bit)
    /// The stream identifier. Unique per connection.
    /// 
    /// ===========================================================================
    /// Headers
    /// ---------------------------------------------------------------------------
    /// 
    /// +-+---------------+
    /// |X| Count (15)    |
    /// +-+---------------+
    /// 
    /// +----------------+----------------+
    /// | Name Length    | Value Length   |
    /// +----------------+----------------+
    /// | Name                          ...
    /// +----------------------------------
    /// | Value                         ...
    /// +----------------------------------
    /// 
    /// 1) Is Compressed (1-bit)
    /// Whether the headers are compressed using DEFLATE.
    /// 
    /// 2) Count (15-bit)
    /// The number of the headers. The value is an unsigned short integer.
    ///
    /// 3) Name Length (16-bit)
    /// The length of the name field. The value is an unsigned short integer.
    ///
    /// 4) Value Length (16-bit)
    /// The length of the value field. The value is an unsigned short integer.
    ///
    /// 5) Name
    /// The name of the header. This is a case insensitive UTF-8 encoded string.
    ///
    /// 6) Value
    /// The value of the header.
    ///
    /// ===========================================================================
    /// Body
    /// ---------------------------------------------------------------------------
    ///
    /// +--------------------------------+
    /// | Body Length (32)               |
    /// +--------------------------------+
    /// | Body                         ...
    /// +---------------------------------
    ///
    /// 1) Body Length (32-bit)
    /// The size of the body. The value is an unsigned integer.
    ///
    /// 2) Body
    /// The body.
    ///
    /// ===========================================================================
    /// 
    /// </summary>
    public class GdsFrame
    {
        // Encoding
        private static readonly UTF8Encoding HeaderEncoding = new UTF8Encoding(false);

        // Masks and Shifts
        private static uint StreamIdMask = Convert.ToUInt32("00000000111111111111111111111111", 2);
        private static int StreamIdShift = 0;

        private static uint TypeMask = Convert.ToUInt32("00001111000000000000000000000000", 2);
        private static int TypeShift = 24;

        private static uint IsCompleteMask = Convert.ToUInt32("10000000000000000000000000000000", 2);
        private static int IsCompleteShift = 31;

        private static uint HeadersIsCompressedMask = Convert.ToUInt32("1000000000000000", 2);
        private static int HeadersIsCompressedShift = 15;

        private static uint HeadersLengthMask = Convert.ToUInt32("0111111111111111", 2);
        private static int HeadersLengthShift = 0;
        
        /// <summary><commit_msg>Fixing constants.<commit_after>﻿/*
 * Copyright 2015 ArenaNet, LLC.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this 
 * file except in compliance with the License. You may obtain a copy of the License at
 *
 * 	 http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under 
 * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 
 * ANY KIND, either express or implied. See the License for the specific language governing 
 * permissions and limitations under the License.
 */
using System;
using System.Collections.Generic;
using System.Net;
using System.IO;
using System.Text;
using Ionic.Zlib;

namespace ArenaNet.SockNet.Protocols.Gds
{
    /// <summary>
    /// Represents a Gds frame.
    /// 
    /// ===========================================================================
    /// GDS (Generic Data Stream) Application Protocol
    /// ---------------------------------------------------------------------------
    /// 
    /// +--------------------+
    /// | Frame Definition   |
    /// +--------------------+
    /// | Headers (Optional) |
    /// +--------------------+
    /// | Body (Optional)    |
    /// +--------------------+
    /// 
    /// The only part in the protocol that is required is the frame definition
    /// which is 32 bits. 
    /// 
    /// Stream
    /// ------
    /// All frames belong to a stream. Streams get created when a frame targeted
    /// towards a stream is sent across the wire. A connected system can send
    /// a "Close" frame to close a stream with a particular ID, at which point
    /// the Stream ID can be reused.
    /// 
    /// Fragmentation
    /// -------------
    /// Fragmentation is implemented by sending a 0 for the "Is Complete" field
    /// in the frame definition. The way that the body and headers behave is
    /// different while fragmented. Headers will be evaluated per fragment (which
    /// means that the headers have to be in a valid format per fragment). I.e. if
    /// fragment 1 sets the header "foo" to "bar" and fragment 2 sets "foo" to
    /// "bar2", then the value of foo will become "bar2". The body content is
    /// usually sliced between fragments - which means that the content doesn't
    /// have to be valid until the message is complete. The last fragmented
    /// frame of a message has the send a 1 for the "Is Complete" field.
    ///  
    /// Compression
    /// -----------
    /// Compression is applied to the headers using DEFLATE (or gzip) if the
    /// "Is Compressed" flag is set to 1 on the Headers block. The compression is
    /// applied to the entire Headers block (excluding the Flag and total Length) 
    /// field.
    /// 
    /// ===========================================================================
    /// Frame Definition
    /// ---------------------------------------------------------------------------
    /// 
    /// +-+-----+----+----------------------------+
    /// |C|Res  |Type| Stream Id (24)             |
    /// +-+-----+----+----------------------------+
    ///  
    /// 1) Is Complete (1-bit)
    /// Whether the message in the current stream is complete or chunked.
    ///  
    /// 2) Reserved (3-bit)
    /// Reserved for future flags.
    /// 
    /// 3) Type (4-bit)
    /// Valid types are:
    /// 0000 - RESERVED
    /// 0001 - Headers Only (No Body)
    /// 0010 - Body Only (No Headers)
    /// 0011 - Header and Body (Everything)
    /// 0100 - RESERVED
    /// 0101 - RESERVED
    /// 0110 - RESERVED
    /// 0111 - RESERVED
    /// 1000 - Ping (No Headers or Body)
    /// 1001 - Pong (No Headers or Body)
    /// 1010 - RESERVED
    /// 1011 - RESERVED
    /// 1100 - RESERVED
    /// 1101 - RESERVED
    /// 1110 - RESERVED
    /// 1111 - Close (No Headers or Body)
    /// 
    /// 4) Stream ID (24-bit)
    /// The stream identifier. Unique per connection.
    /// 
    /// ===========================================================================
    /// Headers
    /// ---------------------------------------------------------------------------
    /// 
    /// +-+---------------+
    /// |X| Count (15)    |
    /// +-+---------------+
    /// 
    /// +----------------+----------------+
    /// | Name Length    | Value Length   |
    /// +----------------+----------------+
    /// | Name                          ...
    /// +----------------------------------
    /// | Value                         ...
    /// +----------------------------------
    /// 
    /// 1) Is Compressed (1-bit)
    /// Whether the headers are compressed using DEFLATE.
    /// 
    /// 2) Count (15-bit)
    /// The number of the headers. The value is an unsigned short integer.
    ///
    /// 3) Name Length (16-bit)
    /// The length of the name field. The value is an unsigned short integer.
    ///
    /// 4) Value Length (16-bit)
    /// The length of the value field. The value is an unsigned short integer.
    ///
    /// 5) Name
    /// The name of the header. This is a case insensitive UTF-8 encoded string.
    ///
    /// 6) Value
    /// The value of the header.
    ///
    /// ===========================================================================
    /// Body
    /// ---------------------------------------------------------------------------
    ///
    /// +--------------------------------+
    /// | Body Length (32)               |
    /// +--------------------------------+
    /// | Body                         ...
    /// +---------------------------------
    ///
    /// 1) Body Length (32-bit)
    /// The size of the body. The value is an unsigned integer.
    ///
    /// 2) Body
    /// The body.
    ///
    /// ===========================================================================
    /// 
    /// </summary>
    public class GdsFrame
    {
        // Encoding
        private static readonly UTF8Encoding HeaderEncoding = new UTF8Encoding(false);

        // Masks and Shifts
        private static readonly uint StreamIdMask = Convert.ToUInt32("00000000111111111111111111111111", 2);
        private static readonly int StreamIdShift = 0;

        private static readonly uint TypeMask = Convert.ToUInt32("00001111000000000000000000000000", 2);
        private static readonly int TypeShift = 24;

        private static readonly uint IsCompleteMask = Convert.ToUInt32("10000000000000000000000000000000", 2);
        private static readonly int IsCompleteShift = 31;

        private static readonly uint HeadersIsCompressedMask = Convert.ToUInt32("1000000000000000", 2);
        private static readonly int HeadersIsCompressedShift = 15;

        private static readonly uint HeadersLengthMask = Convert.ToUInt32("0111111111111111", 2);
        private static readonly int HeadersLengthShift = 0;
        
        /// <summary>
{'fraction_non_alphanumeric': 0.27156619471609983, 'fraction_numerical': 0.03897241278645453, 'mean_word_length': 1.3476100736679801, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 12, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7298945', 'n_tokens_mistral': 3776, 'n_tokens_neox': 3215, 'n_words': 1244}