<commit_before>﻿/*
 *      Copyright (c) 2014-2015, <NAME>
 *      All Rights reserved.
 *      
 *      Released under BSD 3-Clause licence. See terms in
 *      /LICENCE, or online: http://opensource.org/licenses/BSD-3-Clause
 */
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;

using McSherry.Zener.Core;
using McSherry.Zener.Core.Coding;

namespace McSherry.Zener.Net.Serialisation
{
    /// <summary>
    /// A class for serialising an HttpResponse class to
    /// an HTTP/1.1 (RFC 7230) response.
    /// </summary>
    public class Rfc7230Serialiser
        : HttpSerialiser, IDisposable
    {        
        /// <summary>
        /// A class providing a set of constants for common
        /// HTTP header names.
        /// </summary>
        protected internal static class Headers
        {
            /// <summary>
            /// The header that will contain the encodings that the
            /// client may find acceptable.
            /// </summary>
            public const string AcceptEncoding = "Accept-Encoding";
            /// <summary>
            /// The header that indicates to the client which encoding has
            /// been applied to the response body.
            /// </summary>
            public const string ContentEncoding = "Content-Encoding";
            /// <summary>
            /// The header that indicates how the response has been encoded
            /// for transfer.
            /// </summary>
            public const string TransferEncoding = "Transfer-Encoding";

            /// <summary>
            /// The header providing the length, in bytes, of the response
            /// body.
            /// </summary>
            public const string ContentLength = "Content-Length";
            /// <summary>
            /// The header which tells the client what type of content the
            /// server is responding with.
            /// </summary>
            public const string ContentType = "Content-Type";
            /// <summary>
            /// The header that provides information about the content,
            /// such as its file-name.
            /// </summary>
            public const string ContentDisposition = "Content-Disposition";

            /// <summary>
            /// The header used by the server to provide date and time the
            /// response was generated on to the client.
            /// </summary>
            public const string Date = "Date";
            /// <summary>
            /// The header used by the server to provide the client with
            /// information about the server software in use.
            /// </summary>
            public const string Server = "Server";
            /// <summary>
            /// The header field used to send cookies.
            /// </summary>
            public const string SetCookie = "Set-Cookie";
            /// <summary>
            /// The header used by the client to send any cookies
            /// it has stored.
            /// </summary>
            public const string Cookie = "Cookie";

            /// <summary>
            /// The header used by client and server to indicate how
            /// the connection should be handled after the response is sent.
            /// </summary>
            public const string Connection = "Connection";
        }

        /// <summary>
        /// The format string used for the response line.
        /// </summary>
        private const string ResponseLineFormat = "HTTP/1.1 {0} {1}\r\n";
        /// <summary>
        /// The format string used for HTTP/1.1 headers.
        /// </summary>
        protected const string HeaderFormat       = "{0}: {1}\r\n";
        /// <summary>
        /// The encoder used to apply chunked encoding to data.
        /// </summary>
        protected static readonly IEncoder Chunker = ChunkedEncoder.Create();
        /// <summary>
        /// The bytes used as a newline in HTTP/1.1 headers.
        /// </summary>
        protected static readonly byte[] HttpNewline;

        static Rfc7230Serialiser()
        {
            HttpNewline = Encoding.ASCII.GetBytes("\r\n");
        }

        /// <summary>
        /// Whether we are able to enable HTTP compression.
        /// </summary>
        protected bool _canCompress;
        /// <summary>
        /// Whether we are going to use HTTP compression.
        /// </summary>
        protected bool _useCompression;
        /// <summary>
        /// Whether output buffering is enabled.
        /// </summary>
        protected bool _buffer;
        /// <summary>
        /// Whether we've started writing the
        /// response body to the network.
        /// </summary>
        protected bool _bodyWritten;

        /// <summary>
        /// Evaluates the capabilities of the client and
        /// sets any private fields appropriately.
        /// </summary>
        /// <param name="request">
        /// The client's request.
        /// </param>
        /// <exception cref="System.ArgumentNullException">
        /// Thrown when the provided HttpRequest is null.
        /// </exception>
        protected internal virtual void EvaluateClient(HttpRequest request)
        {
            if (request == null)
            {
                throw new ArgumentNullException(
                    "The provided HttpRequest must not be null."
                    );
            }

            // Attempt to retrieve the client's 'Accept-Encoding' header. We
            // can use this to determine which headers are considered acceptable.
            var acceptEncoding = request.Headers[Headers.AcceptEncoding].FirstOrDefault();
            // We only want to do something if the client has sent an 'Accept-Encoding'
            // header. If it hasn't, no changes need be made.
            if (acceptEncoding != default(HttpHeader))
            {
                // Attempt to parse the 'Accept-Encoding' header as an ordered
                // HTTP header. This will allow us to filter out any encodings
                // the client has explicitly forbade.
                OrderedCsvHttpHeader accEncOcsv;
                try
                {
                    accEncOcsv = new OrderedCsvHttpHeader(
                                    header: acceptEncoding,
                        removeUnacceptable: true
                        );
                }
                catch (ArgumentException)
                {
                    // We can't parse the HTTP header, so we'll consider
                    // the client as not accepting any "special" encodings.
                    //
                    // Thanks to our other constructor being called before
                    // the body of this constructor, all values that would
                    // otherwise be set are defaulted to the "no encoding"
                    // values.
                    return;
                }

                // If we get here, we managed to parse at least one item
                // from the HTTP header. We now need to try to select an
                // encoder based on the items.
                //
                // By iterating through the items, we should maintain order
                // the order of preference specified by the client.
                foreach (string encName in accEncOcsv.Items)
                {
                    // Attempt to retrieve an encoder based on the name.
                    // If the value we retrieve isn't null, then we've
                    // found an encoding that both we and the client support.
                    if ((_compressor = Encoders.Get(encName)) != null)
                    {
                        // We've got an encoder, so we can now allow
                        // compression to be enabled.
                        _canCompress = true;
                        // We'll use "enabled" as a sane default for
                        // compression. There aren't a great deal of
                        // situations where you wouldn't want the
                        // output to be compressed.
                        _useCompression = true;
                        // We've found our compressor, so break out of
                        // the loop. We don't need to check for any other
                        // supported encodings.
                        break;
                    }
                }
            }
        }

        /// <summary>
        /// The stream we'll buffer output to.
        /// </summary>
        protected MemoryStream _outputBuffer;
        /// <summary>
        /// The encoder we'll be using when compressing
        /// our response.
        /// </summary>
        protected IEncoder _compressor;
        /// <summary>
        /// The backing field we're using for our Connection
        /// property.
        /// </summary>
        protected HttpConnection _connection;

        /// <summary>
        /// Creates a new Rfc7230Serialiser.
        /// </summary>
        /// <param name="response">
        /// The HttpResponse to be serialised.
        /// </param>
        /// <param name="output">
        /// The stream that should underlie the serialiser,
        /// and to which any serialised data should be written.
        /// </param>
        /// <remarks>
        /// Using this constructor will result in HTTP
        /// compression being disabled, and it will not
        /// be possible to enable compression.
        /// </remarks>
        /// <exception cref="System.ArgumentNullException">
        /// Thrown when one of the provided parameters is null.
        /// </exception>
        /// <exception cref="System.ArgumentException">
        /// Thrown when the provided response stream does not
        /// support writing.
        /// </exception>
        public Rfc7230Serialiser(HttpResponse response, Stream output)
            : base(response, output)
        {
            // We haven't been provided with an HttpRequest, so it
            // isn't possible for us to determine whether the client
            // supports compression.
            _canCompress = false;
            _useCompression = false;
            _compressor = null;
            // We're disabling output buffering by default, as it doing
            // so will generally provide better performance. Output buffering
            // being disabled means we'll be sending using chunked encoding,
            // which means we can send a little at a time and the client can
            // start displaying earlier.
            _buffer = false;
            // As buffering is disabled by default, we want to null our output
            // buffer.
            _outputBuffer = null;
        }
        /// <summary>
        /// Creates a new Rfc7230Serialiser.
        /// </summary><commit_msg>implement proper HttpSerialiser#Connection support in Rfc7230Serialiser<commit_after>﻿/*
 *      Copyright (c) 2014-2015, <NAME>
 *      All Rights reserved.
 *      
 *      Released under BSD 3-Clause licence. See terms in
 *      /LICENCE, or online: http://opensource.org/licenses/BSD-3-Clause
 */
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;

using McSherry.Zener.Core;
using McSherry.Zener.Core.Coding;

namespace McSherry.Zener.Net.Serialisation
{
    /// <summary>
    /// A class for serialising an HttpResponse class to
    /// an HTTP/1.1 (RFC 7230) response.
    /// </summary>
    public class Rfc7230Serialiser
        : HttpSerialiser, IDisposable
    {        
        /// <summary>
        /// A class providing a set of constants for common
        /// HTTP header names.
        /// </summary>
        protected internal static class Headers
        {
            /// <summary>
            /// The header that will contain the encodings that the
            /// client may find acceptable.
            /// </summary>
            public const string AcceptEncoding = "Accept-Encoding";
            /// <summary>
            /// The header that indicates to the client which encoding has
            /// been applied to the response body.
            /// </summary>
            public const string ContentEncoding = "Content-Encoding";
            /// <summary>
            /// The header that indicates how the response has been encoded
            /// for transfer.
            /// </summary>
            public const string TransferEncoding = "Transfer-Encoding";

            /// <summary>
            /// The header providing the length, in bytes, of the response
            /// body.
            /// </summary>
            public const string ContentLength = "Content-Length";
            /// <summary>
            /// The header which tells the client what type of content the
            /// server is responding with.
            /// </summary>
            public const string ContentType = "Content-Type";
            /// <summary>
            /// The header that provides information about the content,
            /// such as its file-name.
            /// </summary>
            public const string ContentDisposition = "Content-Disposition";

            /// <summary>
            /// The header used by the server to provide date and time the
            /// response was generated on to the client.
            /// </summary>
            public const string Date = "Date";
            /// <summary>
            /// The header used by the server to provide the client with
            /// information about the server software in use.
            /// </summary>
            public const string Server = "Server";
            /// <summary>
            /// The header field used to send cookies.
            /// </summary>
            public const string SetCookie = "Set-Cookie";
            /// <summary>
            /// The header used by the client to send any cookies
            /// it has stored.
            /// </summary>
            public const string Cookie = "Cookie";

            /// <summary>
            /// The header used by client and server to indicate how
            /// the connection should be handled after the response is sent.
            /// </summary>
            public const string Connection = "Connection";
        }

        /// <summary>
        /// The format string used for the response line.
        /// </summary>
        private const string ResponseLineFormat = "HTTP/1.1 {0} {1}\r\n";
        /// <summary>
        /// The format string used for HTTP/1.1 headers.
        /// </summary>
        protected const string HeaderFormat       = "{0}: {1}\r\n";
        /// <summary>
        /// The encoder used to apply chunked encoding to data.
        /// </summary>
        protected static readonly IEncoder Chunker = ChunkedEncoder.Create();
        /// <summary>
        /// The bytes used as a newline in HTTP/1.1 headers.
        /// </summary>
        protected static readonly byte[] HttpNewline;

        static Rfc7230Serialiser()
        {
            HttpNewline = Encoding.ASCII.GetBytes("\r\n");
        }

        /// <summary>
        /// Whether we are able to enable HTTP compression.
        /// </summary>
        protected bool _canCompress;
        /// <summary>
        /// Whether we are going to use HTTP compression.
        /// </summary>
        protected bool _useCompression;
        /// <summary>
        /// Whether output buffering is enabled.
        /// </summary>
        protected bool _buffer;
        /// <summary>
        /// Whether we've started writing the
        /// response body to the network.
        /// </summary>
        protected bool _bodyWritten;

        /// <summary>
        /// Evaluates the capabilities of the client and
        /// sets any private fields appropriately.
        /// </summary>
        /// <param name="request">
        /// The client's request.
        /// </param>
        /// <exception cref="System.ArgumentNullException">
        /// Thrown when the provided HttpRequest is null.
        /// </exception>
        protected internal virtual void EvaluateClient(HttpRequest request)
        {
            if (request == null)
            {
                throw new ArgumentNullException(
                    "The provided HttpRequest must not be null."
                    );
            }

            // Attempt to retrieve the client's 'Accept-Encoding' header. We
            // can use this to determine which headers are considered acceptable.
            var acceptEncoding = request.Headers[Headers.AcceptEncoding].FirstOrDefault();
            // We only want to do something if the client has sent an 'Accept-Encoding'
            // header. If it hasn't, no changes need be made.
            if (acceptEncoding != default(HttpHeader))
            {
                // Attempt to parse the 'Accept-Encoding' header as an ordered
                // HTTP header. This will allow us to filter out any encodings
                // the client has explicitly forbade.
                OrderedCsvHttpHeader accEncOcsv;
                try
                {
                    accEncOcsv = new OrderedCsvHttpHeader(
                                    header: acceptEncoding,
                        removeUnacceptable: true
                        );
                }
                catch (ArgumentException)
                {
                    // We can't parse the HTTP header, so we'll consider
                    // the client as not accepting any "special" encodings.
                    //
                    // Thanks to our other constructor being called before
                    // the body of this constructor, all values that would
                    // otherwise be set are defaulted to the "no encoding"
                    // values.
                    return;
                }

                // If we get here, we managed to parse at least one item
                // from the HTTP header. We now need to try to select an
                // encoder based on the items.
                //
                // By iterating through the items, we should maintain order
                // the order of preference specified by the client.
                foreach (string encName in accEncOcsv.Items)
                {
                    // Attempt to retrieve an encoder based on the name.
                    // If the value we retrieve isn't null, then we've
                    // found an encoding that both we and the client support.
                    if ((_compressor = Encoders.Get(encName)) != null)
                    {
                        // We've got an encoder, so we can now allow
                        // compression to be enabled.
                        _canCompress = true;
                        // We'll use "enabled" as a sane default for
                        // compression. There aren't a great deal of
                        // situations where you wouldn't want the
                        // output to be compressed.
                        _useCompression = true;
                        // We've found our compressor, so break out of
                        // the loop. We don't need to check for any other
                        // supported encodings.
                        break;
                    }
                }
            }

            // We need to determine whether the client will accept persistent
            // connections, or whether it wants to close the connection after
            // we send the response.
            //
            // The client indicates this via its 'Connection' header.
            var connHdr = request.Headers[Headers.Connection].LastOrDefault();
            // If the client hasn't sent an HTTP header, we don't have to make
            // any changes because we have defaulted to closing the connection.
            if (connHdr != default(HttpHeader))
            {
                // If the value of the 'Connection' header sent to us by the
                // client equals 'keep-alive', we'll set the value of our
                // property Connection to KeepAlive instead of the default Close.
                if (connHdr.Value.Equals(
                    value:          HttpConnection.KeepAlive.GetValue(),
                    comparisonType: StringComparison.OrdinalIgnoreCase
                    ))
                {
                    this.Connection = HttpConnection.KeepAlive;
                }

                // If we haven't made any changes above, we don't need to do
                // anything special as we already default to 'close'.
            }
        }

        /// <summary>
        /// The stream we'll buffer output to.
        /// </summary>
        protected MemoryStream _outputBuffer;
        /// <summary>
        /// The encoder we'll be using when compressing
        /// our response.
        /// </summary>
        protected IEncoder _compressor;
        /// <summary>
        /// The backing field we're using for our Connection
        /// property.
        /// </summary>
        protected HttpConnection _connection;

        /// <summary>
        /// Creates a new Rfc7230Serialiser.
        /// </summary>
        /// <param name="response">
        /// The HttpResponse to be serialised.
        /// </param>
        /// <param name="output">
        /// The stream that should underlie the serialiser,
        /// and to which any serialised data should be written.
        /// </param>
        /// <remarks>
        /// Using this constructor will result in HTTP
        /// compression being disabled, and it will not
        /// be possible to enable compression.
        /// </remarks>
        /// <exception cref="System.ArgumentNullException">
        /// Thrown when one of the provided parameters is null.
        /// </exception>
        /// <exception cref="System.ArgumentException">
        /// Thrown when the provided response stream does not
        /// support writing.
        /// </exception>
        public Rfc7230Serialiser(HttpResponse response, Stream output)
            : base(response, output)
        {
            // We haven't been provided with an HttpRequest, so it
            // isn't possible for us to determine whether the client
            // supports compression.
            _canCompress = false;
            _useCompression = false;
            _compressor = null;
            // As we haven't been provided with an HttpRequest, we don't
            // know whether the client deems persistent connections acceptable,
            // so we default to closing them.
            //
            // This default value has the added benefit of improving HTTP/1.0
            // client compatibility, as 1.0 clients may not support persistent
            // connections, and so may not send a 'Connection' header.
            _connection = HttpConnection.Close;
            // We're disabling output buffering by default, as it doing
            // so will generally provide better performance. Output buffering
            // being disabled means we'll be sending using chunked encoding,
            // which means we can send a little at a time and the client can
            // start displaying earlier.
            _buffer = false;
            // As buffering is disabled by default, we want to null our output
            // buffer.
            _outputBuffer = null;
        }
        /// <summary>
        /// Creates a new Rfc7230Serialiser.
        /// </summary>
{'fraction_non_alphanumeric': 0.09070824659047458, 'fraction_numerical': 0.004248629816884055, 'mean_word_length': 1.3337299226650803, 'pattern_counts': {'":': 0, '<': 141, '<?xml version=': 0, '>': 141, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '114822', 'n_tokens_mistral': 5395, 'n_tokens_neox': 5119, 'n_words': 2113}