<commit_before>﻿// -----------------------------------------------------------------------
// <copyright file="HttpClient.cs" company="">
// TODO: Update copyright text.
// </copyright>
// -----------------------------------------------------------------------

namespace ContinuousSeo.Core.Net
{
    using System;
    using System.Collections.Generic;
    using System.Text;
    using System.IO;
    using System.Collections.Specialized;
    using System.Net;

    /// <summary>
    /// Used to send either a Get or Post request, filling the request body as appropriate
    /// and returning the payload if the output stream is not null.
    /// </summary>
    public class HttpClient : IHttpClient
    {

        public NameValueCollection Get(Stream output, string url)
        {
            return Post(output, url, String.Empty);
        }


        public NameValueCollection Post(Stream output, string url, string data)
        {
            if (string.IsNullOrEmpty(url))
            {
                throw new ArgumentNullException("url");
            }

            bool headersOnly = (output == null);

            NameValueCollection result = new NameValueCollection();
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);

            if (!string.IsNullOrEmpty(data))
            {
                SetRequestStream(request, data);
            }

            // Get the response from the server
            using (WebResponse response = request.GetResponse())
            {
                if (response != null && response.Headers != null)
                {
                    // Get the headers
                    result.Add(response.Headers);

                    if (!headersOnly)
                    {
                        using (Stream responseStream = response.GetResponseStream())
                        {
                            responseStream.CopyTo(output);
                        }
                    }
                }
            }

            return result;
        }

        public Stream GetResponseStream(string url)
        {
            if (string.IsNullOrEmpty(url))
            {
                throw new ArgumentNullException("url");
            }

            NameValueCollection result = new NameValueCollection();
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);

            using (WebResponse response = request.GetResponse())
            {
                return response.GetResponseStream();
            }
        }

        public string GetResponseText(string url)
        {
            //if (string.IsNullOrEmpty(url))
            //{
            //    throw new ArgumentNullException("url");
            //}

            //NameValueCollection result = new NameValueCollection();
            //HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);

            //using (WebResponse response = request.GetResponse())
            //{
            //    return response.GetResponseStream();
            //}
            string result = string.Empty;
            using (var document = GetResponseStream(url))
            {
                using (var reader = new StreamReader(document))
                {
                    result = reader.ReadToEnd();
                }
            }
            return result;
        }


        private void SetRequestStream(HttpWebRequest request, string data)
        {
            request.Method = "POST";
            request.ContentType = "application/x-www-form-urlencoded";
            byte[] dataBytes = Encoding.UTF8.GetBytes(data);
            request.ContentLength = dataBytes.Length;

            // Load the request data
            using (Stream requestStream = request.GetRequestStream())
            {
                requestStream.Write(dataBytes, 0, dataBytes.Length);
            }<commit_msg>Fixed GetResponseText() method because it was not returning anything.<commit_after>﻿// -----------------------------------------------------------------------
// <copyright file="HttpClient.cs" company="">
// TODO: Update copyright text.
// </copyright>
// -----------------------------------------------------------------------

namespace ContinuousSeo.Core.Net
{
    using System;
    using System.Collections.Generic;
    using System.Text;
    using System.IO;
    using System.Collections.Specialized;
    using System.Net;

    /// <summary>
    /// Used to send either a Get or Post request, filling the request body as appropriate
    /// and returning the payload if the output stream is not null.
    /// </summary>
    public class HttpClient : IHttpClient
    {

        public NameValueCollection Get(Stream output, string url)
        {
            return Post(output, url, String.Empty);
        }


        public NameValueCollection Post(Stream output, string url, string data)
        {
            if (string.IsNullOrEmpty(url))
            {
                throw new ArgumentNullException("url");
            }

            bool headersOnly = (output == null);

            NameValueCollection result = new NameValueCollection();
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);

            if (!string.IsNullOrEmpty(data))
            {
                SetRequestStream(request, data);
            }

            // Get the response from the server
            using (WebResponse response = request.GetResponse())
            {
                if (response != null && response.Headers != null)
                {
                    // Get the headers
                    result.Add(response.Headers);

                    if (!headersOnly)
                    {
                        using (Stream responseStream = response.GetResponseStream())
                        {
                            responseStream.CopyTo(output);
                        }
                    }
                }
            }

            return result;
        }

        public Stream GetResponseStream(string url)
        {
            if (string.IsNullOrEmpty(url))
            {
                throw new ArgumentNullException("url");
            }

            NameValueCollection result = new NameValueCollection();
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);

            using (WebResponse response = request.GetResponse())
            {
                return response.GetResponseStream();
            }
        }

        public string GetResponseText(string url)
        {
            string result = string.Empty;
            using (var client = new WebClient())
            {
                // Gets the encoding from the HTTP header information automatically
                // and falls back to byte order mark if header not supplied.
                result = client.DownloadString(url);
            }
            return result;

            //string result = string.Empty;
            //using (var document = new MemoryStream())
            //{
            //    Get(document, url);
            //    document.Position = 0;
            //    using (var reader = new StreamReader(document))
            //    {
            //        result = reader.ReadToEnd();
            //    }
            //}
            //return result;
        }


        private void SetRequestStream(HttpWebRequest request, string data)
        {
            request.Method = "POST";
            request.ContentType = "application/x-www-form-urlencoded";
            byte[] dataBytes = Encoding.UTF8.GetBytes(data);
            request.ContentLength = dataBytes.Length;

            // Load the request data
            using (Stream requestStream = request.GetRequestStream())
            {
                requestStream.Write(dataBytes, 0, dataBytes.Length);
            }
{'fraction_non_alphanumeric': 0.12250128139415685, 'fraction_numerical': 0.0006406970784213224, 'mean_word_length': 1.3263785394932937, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4225836', 'n_tokens_mistral': 1748, 'n_tokens_neox': 1670, 'n_words': 441}