<commit_before>﻿using System;
using System.IO;
using System.Net.Http;
using System.Net;
using System.Net.Http.Headers;
using System.Net.Mime;
using System.Text;
using System.Threading.Tasks;
using NLog;
using Swensen.Utils;
using System.Threading;

namespace Swensen.Ior.Core {
    /// <summary>
    /// A light wrapper around our underlying http client.
    /// </summary>
    public class IorClient {
        private static readonly Logger log = LogManager.GetCurrentClassLogger();

        private readonly string defaultRequestContentType;
        private readonly string proxyServer;
        private readonly bool includeUtf8Bom;
        private readonly bool followRedirects;

        public IorClient(string defaultRequestContentType, string proxyServer, bool includeUtf8Bom, bool followRedirects) {
            this.defaultRequestContentType = defaultRequestContentType;
            this.proxyServer = proxyServer;
            this.includeUtf8Bom = includeUtf8Bom;
            this.followRedirects = followRedirects;
        }

        /// <summary>
        /// Execute a request for the given requestModel asynchronously, executing the given callback upon completions
        /// other than cancellations (gui updates for cancellation should be done at point of cancellation).
        /// </summary>
        /// <param name="requestModel"></param>
        /// <param name="callback"></param>
        /// <returns></returns>
        public CancellationTokenSource ExecuteAsync(RequestModel requestModel, Action<ResponseModel> callback) {
            //todo: add using statements for disposible objects

            var handler = new HttpClientHandler {
                AllowAutoRedirect = followRedirects
            };

            if (!proxyServer.IsBlank()) {
                handler.Proxy = new WebProxy(proxyServer, false); //make second arg a config option.
                handler.UseProxy = true;
            }

            var client = new HttpClient(handler);
            var request = new HttpRequestMessage {
                RequestUri = requestModel.Url, 
                Method = requestModel.Method
            };

            foreach (var header in requestModel.RequestHeaders)
                request.Headers.Add(header.Key, header.Value);

            if (requestModel.Method != HttpMethod.Get) {
                //default content-type: http://mattryall.net/blog/2008/03/default-content-type
                string textCt;
                requestModel.ContentHeaders.TryGetValue("Content-Type", out textCt);
                textCt = textCt.IsBlank() ? defaultRequestContentType : textCt; //then try settings supplied
                textCt = textCt.IsBlank() ? "application/octet-stream" : textCt; // then try w3 spec default

                //get encoding
                var ct = new ContentType(textCt);
                //write content w/ BOM  if needed
                var contentBytes = GetEncodedBytes(requestModel.Body, ct.CharSet, includeUtf8Bom);
                var content = new ByteArrayContent(contentBytes);
                
                foreach (var header in requestModel.ContentHeaders) {
                    content.Headers.Remove(header.Key); //remove defaults
                    if (String.Equals(header.Key, "content-type", StringComparison.OrdinalIgnoreCase)) //treat special w/ defaults, etc.
                        content.Headers.Add(header.Key, textCt);
                    else
                        content.Headers.Add(header.Key, header.Value);
                }

                request.Content = content;
            }

            var start = DateTime.Now;

            var ctokenSource = new CancellationTokenSource();<commit_msg>Issue 42: send cookies according to system.net.http requirements<commit_after>﻿using System;
using System.IO;
using System.Net.Http;
using System.Net;
using System.Net.Http.Headers;
using System.Net.Mime;
using System.Text;
using System.Threading.Tasks;
using NLog;
using Swensen.Utils;
using System.Threading;

namespace Swensen.Ior.Core {
    /// <summary>
    /// A light wrapper around our underlying http client.
    /// </summary>
    public class IorClient {
        private static readonly Logger log = LogManager.GetCurrentClassLogger();

        private readonly string defaultRequestContentType;
        private readonly string proxyServer;
        private readonly bool includeUtf8Bom;
        private readonly bool followRedirects;

        public IorClient(string defaultRequestContentType, string proxyServer, bool includeUtf8Bom, bool followRedirects) {
            this.defaultRequestContentType = defaultRequestContentType;
            this.proxyServer = proxyServer;
            this.includeUtf8Bom = includeUtf8Bom;
            this.followRedirects = followRedirects;
        }

        /// <summary>
        /// Execute a request for the given requestModel asynchronously, executing the given callback upon completions
        /// other than cancellations (gui updates for cancellation should be done at point of cancellation).
        /// </summary>
        /// <param name="requestModel"></param>
        /// <param name="callback"></param>
        /// <returns></returns>
        public CancellationTokenSource ExecuteAsync(RequestModel requestModel, Action<ResponseModel> callback) {
            //todo: add using statements for disposible objects
            var hasCookies = requestModel.RequestHeaders.ContainsKey("cookie");

            var handler = new HttpClientHandler {
                AllowAutoRedirect = followRedirects,
                UseCookies = hasCookies
            };

            if (!proxyServer.IsBlank()) {
                handler.Proxy = new WebProxy(proxyServer, false); //make second arg a config option.
                handler.UseProxy = true;
            }

            var client = new HttpClient(handler);
            var request = new HttpRequestMessage {
                RequestUri = requestModel.Url, 
                Method = requestModel.Method
            };

            foreach (var header in requestModel.RequestHeaders)
                request.Headers.Add(header.Key, header.Value);

            if (hasCookies) {
                foreach (var ch in requestModel.RequestHeaders) {
                    handler.CookieContainer.Add(requestModel.Url, new Cookie(ch.Key, ch.Value));
                }
            }

            if (requestModel.Method != HttpMethod.Get) {
                //default content-type: http://mattryall.net/blog/2008/03/default-content-type
                string textCt;
                requestModel.ContentHeaders.TryGetValue("Content-Type", out textCt);
                textCt = textCt.IsBlank() ? defaultRequestContentType : textCt; //then try settings supplied
                textCt = textCt.IsBlank() ? "application/octet-stream" : textCt; // then try w3 spec default

                //get encoding
                var ct = new ContentType(textCt);
                //write content w/ BOM  if needed
                var contentBytes = GetEncodedBytes(requestModel.Body, ct.CharSet, includeUtf8Bom);
                var content = new ByteArrayContent(contentBytes);
                
                foreach (var header in requestModel.ContentHeaders) {
                    content.Headers.Remove(header.Key); //remove defaults
                    if (String.Equals(header.Key, "content-type", StringComparison.OrdinalIgnoreCase)) //treat special w/ defaults, etc.
                        content.Headers.Add(header.Key, textCt);
                    else
                        content.Headers.Add(header.Key, header.Value);
                }

                request.Content = content;
            }

            var start = DateTime.Now;

            var ctokenSource = new CancellationTokenSource();
{'fraction_non_alphanumeric': 0.08521560574948665, 'fraction_numerical': 0.0033367556468172485, 'mean_word_length': 2.047712162690653, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7110069', 'n_tokens_mistral': 1940, 'n_tokens_neox': 1828, 'n_words': 508}