<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using Newtonsoft.Json;

namespace Radosgw.AdminAPI
{
    public class RadosGWAdminConnection
    {
        private string accessKey { get; set; }
        private string secretKey { get; set; }
        private string adminPrefix { get; set; }
        private Uri host { get; set; }

        private HashSet<string> subresourcesS3 = new HashSet<string>{ "acl",
            "lifecycle",
            "location",
            "logging",
            "notification",
            "partNumber",
            "policy",
            "requestPayment",
            "torrent",
            "uploadId",
            "uploads",
            "versionId",
            "versioning",
            "versions",
            "website",
        };

        private string CanonicalizeResource(string resource)
        {
            if (resource.Contains("?"))
            {
                var ret = new StringBuilder();
                var ioqm = resource.IndexOf('?');
                ret.Append(resource.Substring(0, ioqm));
                var subresources = resource.Substring(ioqm + 1).Split('&');
                var first = true;
                foreach (var item in subresources)
                {
                    var key = item.Split('=')[0];
                    if (subresourcesS3.Contains(key))
                    {
                        if (first)
                        {
                            first = false;
                            ret.Append("?");
                        }
                        else {
                            ret.Append("&");
                        }
                        ret.Append(WebUtility.UrlEncode(item));
                    }
                }
                return ret.ToString();
            }
            else {
                return resource;
            }
        }

        public string SendRequest(string httpVerb, string path, Dictionary<string, string> requestParameters = null, TimeSpan? timeout = null)
        {
            string HttpVerb = httpVerb;
            if (!path.StartsWith(this.adminPrefix))
                path = adminPrefix + path;
            string ContentMD5 = "";
            string ContentType = "";

            var queryparams = "";
            if (requestParameters != null && requestParameters.Count > 0)
            {
                if (path.Contains("?"))
                    queryparams = "&";
                else
                    queryparams = "?";
                queryparams += string.Join("&", requestParameters.Select(kvp =>
                    string.Format("{0}={1}", kvp.Key, Uri.EscapeDataString(kvp.Value))));
                queryparams += "&format=json";
            }
            else {
                queryparams = "?format=json";
            }


            string httpDate = DateTime.UtcNow.ToString("ddd, dd MMM yyyy HH:mm:ss +0000\n");
            var CPath = CanonicalizeResource(path + queryparams);

            string canonicalString = HttpVerb + "\n" +
                ContentMD5 + "\n" +
                ContentType + "\n" +
                "\n" +
                "x-amz-date:" + httpDate +
                CPath;

            // now encode the canonical string
            Encoding ae = new UTF8Encoding();
            // create a hashing object
            HMACSHA1 signature = new HMACSHA1();
            // secretId is the hash key
            signature.Key = ae.GetBytes(this.secretKey);
            byte[] bytes = ae.GetBytes(canonicalString);
            byte[] moreBytes = signature.ComputeHash(bytes);
            // convert the hash byte array into a base64 encoding
            string encodedCanonical = Convert.ToBase64String(moreBytes);


            var u = new Uri(host, path + queryparams);

            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(u);
            request.Headers.Add("x-amz-date", httpDate);
            request.Headers.Add("Authorization", "AWS " + this.accessKey + ":" + encodedCanonical);
            request.Method = HttpVerb;
            if (timeout != null) // default timeout if none specified, no timeout: TimeSpan(-1)
                request.Timeout = timeout.Value.Ticks < 0 ? -1 : (int)timeout.Value.TotalMilliseconds;

            // Get the response
            //HttpWebResponse response = null;
            try
            {
                var response = (HttpWebResponse)request.GetResponse();
                string responseString = "";

                using (Stream stream = response.GetResponseStream())
                {
                    StreamReader reader = new StreamReader(stream, Encoding.UTF8);
                    responseString = reader.ReadToEnd();
                }
                response.Close();
                return responseString;

            }
            catch (WebException ex)
            {
                if (ex.Status == WebExceptionStatus.Timeout)
                    throw new TimeoutException();
                string responseString = "";
                var response = ex.Response as HttpWebResponse;
                using (Stream stream = ex.Response.GetResponseStream())
                {
                    StreamReader reader = new StreamReader(stream, Encoding.UTF8);
                    responseString = reader.ReadToEnd();
                }
<commit_msg>Fix NullReferenceException when connection fails<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using Newtonsoft.Json;

namespace Radosgw.AdminAPI
{
    public class RadosGWAdminConnection
    {
        private string accessKey { get; set; }
        private string secretKey { get; set; }
        private string adminPrefix { get; set; }
        private Uri host { get; set; }

        private HashSet<string> subresourcesS3 = new HashSet<string>{ "acl",
            "lifecycle",
            "location",
            "logging",
            "notification",
            "partNumber",
            "policy",
            "requestPayment",
            "torrent",
            "uploadId",
            "uploads",
            "versionId",
            "versioning",
            "versions",
            "website",
        };

        private string CanonicalizeResource(string resource)
        {
            if (resource.Contains("?"))
            {
                var ret = new StringBuilder();
                var ioqm = resource.IndexOf('?');
                ret.Append(resource.Substring(0, ioqm));
                var subresources = resource.Substring(ioqm + 1).Split('&');
                var first = true;
                foreach (var item in subresources)
                {
                    var key = item.Split('=')[0];
                    if (subresourcesS3.Contains(key))
                    {
                        if (first)
                        {
                            first = false;
                            ret.Append("?");
                        }
                        else {
                            ret.Append("&");
                        }
                        ret.Append(WebUtility.UrlEncode(item));
                    }
                }
                return ret.ToString();
            }
            else {
                return resource;
            }
        }

        public string SendRequest(string httpVerb, string path, Dictionary<string, string> requestParameters = null, TimeSpan? timeout = null)
        {
            string HttpVerb = httpVerb;
            if (!path.StartsWith(this.adminPrefix))
                path = adminPrefix + path;
            string ContentMD5 = "";
            string ContentType = "";

            var queryparams = "";
            if (requestParameters != null && requestParameters.Count > 0)
            {
                if (path.Contains("?"))
                    queryparams = "&";
                else
                    queryparams = "?";
                queryparams += string.Join("&", requestParameters.Select(kvp =>
                    string.Format("{0}={1}", kvp.Key, Uri.EscapeDataString(kvp.Value))));
                queryparams += "&format=json";
            }
            else {
                queryparams = "?format=json";
            }


            string httpDate = DateTime.UtcNow.ToString("ddd, dd MMM yyyy HH:mm:ss +0000\n");
            var CPath = CanonicalizeResource(path + queryparams);

            string canonicalString = HttpVerb + "\n" +
                ContentMD5 + "\n" +
                ContentType + "\n" +
                "\n" +
                "x-amz-date:" + httpDate +
                CPath;

            // now encode the canonical string
            Encoding ae = new UTF8Encoding();
            // create a hashing object
            HMACSHA1 signature = new HMACSHA1();
            // secretId is the hash key
            signature.Key = ae.GetBytes(this.secretKey);
            byte[] bytes = ae.GetBytes(canonicalString);
            byte[] moreBytes = signature.ComputeHash(bytes);
            // convert the hash byte array into a base64 encoding
            string encodedCanonical = Convert.ToBase64String(moreBytes);


            var u = new Uri(host, path + queryparams);

            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(u);
            request.Headers.Add("x-amz-date", httpDate);
            request.Headers.Add("Authorization", "AWS " + this.accessKey + ":" + encodedCanonical);
            request.Method = HttpVerb;
            if (timeout != null) // default timeout if none specified, no timeout: TimeSpan(-1)
                request.Timeout = timeout.Value.Ticks < 0 ? -1 : (int)timeout.Value.TotalMilliseconds;

            // Get the response
            try
            {
                var response = (HttpWebResponse)request.GetResponse();
                string responseString = "";

                using (Stream stream = response.GetResponseStream())
                {
                    StreamReader reader = new StreamReader(stream, Encoding.UTF8);
                    responseString = reader.ReadToEnd();
                }
                response.Close();
                return responseString;

            }
            catch (WebException ex)
            {
                if (ex.Status == WebExceptionStatus.ConnectFailure)
                    throw;
                if (ex.Status == WebExceptionStatus.Timeout)
                    throw new TimeoutException();
                string responseString = "";
                var response = ex.Response as HttpWebResponse;
                using (Stream stream = ex.Response.GetResponseStream())
                {
                    StreamReader reader = new StreamReader(stream, Encoding.UTF8);
                    responseString = reader.ReadToEnd();
                }

{'fraction_non_alphanumeric': 0.09986394557823129, 'fraction_numerical': 0.00471655328798186, 'mean_word_length': 1.3620394173093402, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2354878', 'n_tokens_mistral': 2740, 'n_tokens_neox': 2573, 'n_words': 621}