<commit_before>﻿namespace AngleSharp.Network
{
    using System;
    using System.Collections.Generic;
    using System.Linq;
    using System.Net;
    using System.Reflection;
    using System.Threading;
    using System.Threading.Tasks;

    /// <summary>
    /// The default (ready-to-use) HTTP requester.
    /// </summary>
    sealed class DefaultRequester : IRequester
    {
        #region Constants

        const Int32 BufferSize = 4096;

        static readonly Dictionary<String, PropertyInfo> _propCache;

        #endregion

        #region Fields

        TimeSpan _timeOut;
        readonly Dictionary<String, String> _headers;

        #endregion

        #region ctor

        static DefaultRequester()
        {
            _propCache = new Dictionary<String, PropertyInfo>();
        }

        /// <summary>
        /// Constructs a default HTTP requester with the information
        /// presented in the info object.
        /// </summary>
        /// <param name="info">The information to use.</param>
        public DefaultRequester(IInfo info)
        {
            _timeOut = new TimeSpan(0, 0, 0, 45);
            _headers = new Dictionary<String, String>();
            _headers.Add("User-Agent", info.Agent);
        }

        #endregion

        #region Properties

        /// <summary>
        /// Gets the default headers.
        /// </summary>
        public Dictionary<String, String> Headers
        {
            get { return _headers; }
        }

        /// <summary>
        /// Gets or sets the timeout value.
        /// </summary>
        public TimeSpan Timeout
        {
            get { return _timeOut; }
            set { _timeOut = value; }
        }

        #endregion

        #region Methods

        /// <summary>
        /// Checks if the given protocol is supported.
        /// </summary>
        /// <param name="protocol">The protocol to check for, e.g. http.</param>
        /// <returns>True if the protocol is supported, otherwise false.</returns>
        public Boolean SupportsProtocol(String protocol)
        {
            return protocol == KnownProtocols.Http || protocol == KnownProtocols.Https;
        }

        /// <summary>
        /// Performs a blocking http request with the given options.
        /// </summary>
        /// <param name="request">The options to consider.</param>
        /// <returns>The response data.</returns>
        public IResponse Request(IRequest request)
        {
            var cache = new RequestState(request, _headers);
            return cache.Request();
        }

        /// <summary>
        /// Performs an asynchronous http request with the given options.
        /// </summary>
        /// <param name="request">The options to consider.</param>
        /// <returns>The task that will eventually give the response data.</returns>
        public Task<IResponse> RequestAsync(IRequest request)
        {
            return RequestAsync(request, CancellationToken.None);
        }

        /// <summary>
        /// Performs an asynchronous http request that can be cancelled.
        /// </summary>
        /// <param name="request">The options to consider.</param>
        /// <param name="cancellationToken">The token for cancelling the task.</param>
        /// <returns>The task that will eventually give the response data.</returns>
        public Task<IResponse> RequestAsync(IRequest request, CancellationToken cancellationToken)
        {
            var cache = new RequestState(request, _headers);
            return cache.RequestAsync(cancellationToken);
        }

        #endregion

        #region Transport

        sealed class RequestState
        {
            TaskCompletionSource<Boolean> _completed;
            HttpWebResponse _response;

            readonly HttpWebRequest _http;
            readonly IRequest _request;
            readonly Byte[] _buffer;

            public RequestState(IRequest request, IDictionary<String, String> headers)
            {
                _request = request;
                _http = WebRequest.Create(request.Address) as HttpWebRequest;
                _http.Method = request.Method.ToString();
                _buffer = new Byte[BufferSize];
                _completed = new TaskCompletionSource<Boolean>();

                foreach (var header in headers)
                    AddHeader(header.Key, header.Value);

                foreach (var header in request.Headers)
                    AddHeader(header.Key, header.Value);
            }

            public IResponse Request()
            {
                if (_request.Method == HttpMethod.Post || _request.Method == HttpMethod.Put)
                {
                    _http.BeginGetRequestStream(SendRequest, _request);
                    _completed.Task.Wait();
                    _completed = new TaskCompletionSource<Boolean>();
                }

                _http.BeginGetResponse(ReceiveResponse, null);
                _completed.Task.Wait();
                return GetResponse();
            }

            public async Task<IResponse> RequestAsync(CancellationToken cancellationToken)
            {
                if (_request.Method == HttpMethod.Post || _request.Method == HttpMethod.Put)
                {
                    _http.BeginGetRequestStream(SendRequest, _request);

                    if (cancellationToken.IsCancellationRequested)
                        return null;

                    await _completed.Task;
                    _completed = new TaskCompletionSource<Boolean>();
                }

                if (cancellationToken.IsCancellationRequested)
                    return null;<commit_msg>Removed obsolete implementation<commit_after>﻿namespace AngleSharp.Network
{
    using System;
    using System.Collections.Generic;
    using System.Linq;
    using System.Net;
    using System.Reflection;
    using System.Threading;
    using System.Threading.Tasks;

    /// <summary>
    /// The default (ready-to-use) HTTP requester.
    /// </summary>
    sealed class DefaultRequester : IRequester
    {
        #region Constants

        const Int32 BufferSize = 4096;

        static readonly Dictionary<String, PropertyInfo> _propCache;

        #endregion

        #region Fields

        TimeSpan _timeOut;
        readonly Dictionary<String, String> _headers;

        #endregion

        #region ctor

        static DefaultRequester()
        {
            _propCache = new Dictionary<String, PropertyInfo>();
        }

        /// <summary>
        /// Constructs a default HTTP requester with the information
        /// presented in the info object.
        /// </summary>
        /// <param name="info">The information to use.</param>
        public DefaultRequester(IInfo info)
        {
            _timeOut = new TimeSpan(0, 0, 0, 45);
            _headers = new Dictionary<String, String>();
            _headers.Add("User-Agent", info.Agent);
        }

        #endregion

        #region Properties

        /// <summary>
        /// Gets the default headers.
        /// </summary>
        public Dictionary<String, String> Headers
        {
            get { return _headers; }
        }

        /// <summary>
        /// Gets or sets the timeout value.
        /// </summary>
        public TimeSpan Timeout
        {
            get { return _timeOut; }
            set { _timeOut = value; }
        }

        #endregion

        #region Methods

        /// <summary>
        /// Checks if the given protocol is supported.
        /// </summary>
        /// <param name="protocol">The protocol to check for, e.g. http.</param>
        /// <returns>True if the protocol is supported, otherwise false.</returns>
        public Boolean SupportsProtocol(String protocol)
        {
            return protocol == KnownProtocols.Http || protocol == KnownProtocols.Https;
        }

        /// <summary>
        /// Performs an asynchronous http request with the given options.
        /// </summary>
        /// <param name="request">The options to consider.</param>
        /// <returns>The task that will eventually give the response data.</returns>
        public Task<IResponse> RequestAsync(IRequest request)
        {
            return RequestAsync(request, CancellationToken.None);
        }

        /// <summary>
        /// Performs an asynchronous http request that can be cancelled.
        /// </summary>
        /// <param name="request">The options to consider.</param>
        /// <param name="cancellationToken">The token for cancelling the task.</param>
        /// <returns>The task that will eventually give the response data.</returns>
        public Task<IResponse> RequestAsync(IRequest request, CancellationToken cancellationToken)
        {
            var cache = new RequestState(request, _headers);
            return cache.RequestAsync(cancellationToken);
        }

        #endregion

        #region Transport

        sealed class RequestState
        {
            TaskCompletionSource<Boolean> _completed;
            HttpWebResponse _response;

            readonly HttpWebRequest _http;
            readonly IRequest _request;
            readonly Byte[] _buffer;

            public RequestState(IRequest request, IDictionary<String, String> headers)
            {
                _request = request;
                _http = WebRequest.Create(request.Address) as HttpWebRequest;
                _http.Method = request.Method.ToString();
                _buffer = new Byte[BufferSize];
                _completed = new TaskCompletionSource<Boolean>();

                foreach (var header in headers)
                    AddHeader(header.Key, header.Value);

                foreach (var header in request.Headers)
                    AddHeader(header.Key, header.Value);
            }

            public async Task<IResponse> RequestAsync(CancellationToken cancellationToken)
            {
                if (_request.Method == HttpMethod.Post || _request.Method == HttpMethod.Put)
                {
                    _http.BeginGetRequestStream(SendRequest, _request);

                    if (cancellationToken.IsCancellationRequested)
                        return null;

                    await _completed.Task;
                    _completed = new TaskCompletionSource<Boolean>();
                }

                if (cancellationToken.IsCancellationRequested)
                    return null;
{'fraction_non_alphanumeric': 0.09390838672659463, 'fraction_numerical': 0.0021038538777852158, 'mean_word_length': 1.707222366036759, 'pattern_counts': {'":': 0, '<': 94, '<?xml version=': 0, '>': 94, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7049687', 'n_tokens_mistral': 2535, 'n_tokens_neox': 2434, 'n_words': 682}