<commit_before>﻿namespace AngleSharp.Network.Default
{
    using AngleSharp.Extensions;
    using System;
    using System.Collections.Generic;
    using System.IO;
    using System.Linq;
    using System.Net;
    using System.Reflection;
    using System.Threading;
    using System.Threading.Tasks;

    /// <summary>
    /// The default (ready-to-use) HTTP requester.
    /// </summary>
    public sealed class HttpRequester : IRequester
    {
        #region Constants

        private const Int32 BufferSize = 4096;

        private static readonly String _version = typeof(HttpRequester).GetTypeInfo().Assembly.GetCustomAttribute<AssemblyFileVersionAttribute>().Version;
        private static readonly String _agentName = "AngleSharp/" + _version;
        private static readonly Dictionary<String, PropertyInfo> _propCache = new Dictionary<String, PropertyInfo>();
        private static readonly List<String> _restricted = new List<String>();

        #endregion

        #region Fields

        private TimeSpan _timeOut;
        private readonly Dictionary<String, String> _headers;

        #endregion

        #region ctor

        /// <summary>
        /// Constructs a default HTTP requester with the information presented
        /// in the info object.
        /// </summary>
        /// <param name="userAgent">The user-agent name to use, if any.</param>
        public HttpRequester(String userAgent = null)
        {
            _timeOut = new TimeSpan(0, 0, 0, 45);
            _headers = new Dictionary<String, String>();
            _headers.Add("User-Agent", userAgent ?? _agentName);
        }

        #endregion

        #region Properties

        /// <summary>
        /// Gets the used headers.
        /// </summary>
        public IDictionary<String, String> Headers
        {
            get { return _headers; }
        }

        /// <summary>
        /// Gets or sets the timeout value.
        /// </summary>
        public TimeSpan Timeout
        {
            get { return _timeOut; }
            set { _timeOut = value; }
        }

        #endregion

        #region Methods

        /// <summary>
        /// Checks if the given protocol is supported.
        /// </summary>
        /// <param name="protocol">
        /// The protocol to check for, e.g. http.
        /// </param>
        /// <returns>
        /// True if the protocol is supported, otherwise false.
        /// </returns>
        public Boolean SupportsProtocol(String protocol)
        {
            return protocol.IsOneOf(ProtocolNames.Http, ProtocolNames.Https);
        }

        /// <summary>
        /// Performs an asynchronous http request that can be cancelled.
        /// </summary>
        /// <param name="request">The options to consider.</param>
        /// <param name="cancellationToken">
        /// The token for cancelling the task.
        /// </param>
        /// <returns>
        /// The task that will eventually give the response data.
        /// </returns>
        public Task<IResponse> RequestAsync(IRequest request, CancellationToken cancellationToken)
        {
            var cache = new RequestState(request, _headers);
            return cache.RequestAsync(cancellationToken);
        }

        #endregion

        #region Transport

        private sealed class RequestState
        {
            readonly CookieContainer _cookies;
            readonly HttpWebRequest _http;
            readonly IRequest _request;
            readonly Byte[] _buffer;

            public RequestState(IRequest request, IDictionary<String, String> headers)
            {
                var cookieHeader = request.Headers.GetOrDefault(HeaderNames.Cookie, String.Empty);
                _cookies = new CookieContainer();
                _request = request;
                _http = WebRequest.Create(request.Address) as HttpWebRequest;
                _http.CookieContainer = _cookies;
                _http.Method = request.Method.ToString().ToUpperInvariant();
                _buffer = new Byte[BufferSize];

                foreach (var header in headers)
                {
                    AddHeader(header.Key, header.Value);
                }

                foreach (var header in request.Headers)
                {
                    AddHeader(header.Key, header.Value);
                }

                _cookies.SetCookies(_http.RequestUri, cookieHeader);
            }

            public async Task<IResponse> RequestAsync(CancellationToken cancellationToken)
            {
                cancellationToken.Register(_http.Abort);

                if (_request.Method == HttpMethod.Post || _request.Method == HttpMethod.Put)
                {
                    var target = await Task.Factory.FromAsync<Stream>(_http.BeginGetRequestStream, _http.EndGetRequestStream, null).ConfigureAwait(false);
                    SendRequest(target);
                }

                var response = default(WebResponse);
<commit_msg>Private fields and timeout registration<commit_after>﻿namespace AngleSharp.Network.Default
{
    using AngleSharp.Extensions;
    using System;
    using System.Collections.Generic;
    using System.IO;
    using System.Linq;
    using System.Net;
    using System.Reflection;
    using System.Threading;
    using System.Threading.Tasks;

    /// <summary>
    /// The default (ready-to-use) HTTP requester.
    /// </summary>
    public sealed class HttpRequester : IRequester
    {
        #region Constants

        private const Int32 BufferSize = 4096;

        private static readonly String _version = typeof(HttpRequester).GetTypeInfo().Assembly.GetCustomAttribute<AssemblyFileVersionAttribute>().Version;
        private static readonly String _agentName = "AngleSharp/" + _version;
        private static readonly Dictionary<String, PropertyInfo> _propCache = new Dictionary<String, PropertyInfo>();
        private static readonly List<String> _restricted = new List<String>();

        #endregion

        #region Fields

        private TimeSpan _timeOut;
        private readonly Dictionary<String, String> _headers;

        #endregion

        #region ctor

        /// <summary>
        /// Constructs a default HTTP requester with the information presented
        /// in the info object.
        /// </summary>
        /// <param name="userAgent">The user-agent name to use, if any.</param>
        public HttpRequester(String userAgent = null)
        {
            _timeOut = new TimeSpan(0, 0, 0, 45);
            _headers = new Dictionary<String, String>();
            _headers.Add("User-Agent", userAgent ?? _agentName);
        }

        #endregion

        #region Properties

        /// <summary>
        /// Gets the used headers.
        /// </summary>
        public IDictionary<String, String> Headers
        {
            get { return _headers; }
        }

        /// <summary>
        /// Gets or sets the timeout value.
        /// </summary>
        public TimeSpan Timeout
        {
            get { return _timeOut; }
            set { _timeOut = value; }
        }

        #endregion

        #region Methods

        /// <summary>
        /// Checks if the given protocol is supported.
        /// </summary>
        /// <param name="protocol">
        /// The protocol to check for, e.g. http.
        /// </param>
        /// <returns>
        /// True if the protocol is supported, otherwise false.
        /// </returns>
        public Boolean SupportsProtocol(String protocol)
        {
            return protocol.IsOneOf(ProtocolNames.Http, ProtocolNames.Https);
        }

        /// <summary>
        /// Performs an asynchronous http request that can be cancelled.
        /// </summary>
        /// <param name="request">The options to consider.</param>
        /// <param name="cancellationToken">
        /// The token for cancelling the task.
        /// </param>
        /// <returns>
        /// The task that will eventually give the response data.
        /// </returns>
        public async Task<IResponse> RequestAsync(IRequest request, CancellationToken cancellationToken)
        {
            var cts = new CancellationTokenSource(_timeOut);
            var cache = new RequestState(request, _headers);

            using (cancellationToken.Register(cts.Cancel))
            {
                return await cache.RequestAsync(cts.Token).ConfigureAwait(false);
            }
        }

        #endregion

        #region Transport

        private sealed class RequestState
        {
            private readonly CookieContainer _cookies;
            private readonly HttpWebRequest _http;
            private readonly IRequest _request;
            private readonly Byte[] _buffer;

            public RequestState(IRequest request, IDictionary<String, String> headers)
            {
                var cookieHeader = request.Headers.GetOrDefault(HeaderNames.Cookie, String.Empty);
                _cookies = new CookieContainer();
                _request = request;
                _http = WebRequest.CreateHttp(request.Address);
                _http.CookieContainer = _cookies;
                _http.Method = request.Method.ToString().ToUpperInvariant();
                _buffer = new Byte[BufferSize];

                foreach (var header in headers)
                {
                    AddHeader(header.Key, header.Value);
                }

                foreach (var header in request.Headers)
                {
                    AddHeader(header.Key, header.Value);
                }

                _cookies.SetCookies(_http.RequestUri, cookieHeader);
            }

            public async Task<IResponse> RequestAsync(CancellationToken cancellationToken)
            {
                cancellationToken.Register(_http.Abort);
                
                if (_request.Method == HttpMethod.Post || _request.Method == HttpMethod.Put)
                {
                    var target = await Task.Factory.FromAsync<Stream>(_http.BeginGetRequestStream, _http.EndGetRequestStream, null).ConfigureAwait(false);
                    SendRequest(target);
                }

                var response = default(WebResponse);

{'fraction_non_alphanumeric': 0.09498577175939554, 'fraction_numerical': 0.0021587675399862625, 'mean_word_length': 1.7703180212014133, 'pattern_counts': {'":': 0, '<': 75, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4503280', 'n_tokens_mistral': 2513, 'n_tokens_neox': 2419, 'n_words': 664}