<commit_before>﻿#if !DOTNET5_4
using Discord.API;
using RestSharp;
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;

namespace Discord.Net.Rest
{
    internal sealed class RestSharpEngine : IRestEngine
	{
		private readonly DiscordConfig _config;
		private readonly RestSharp.RestClient _client;

        public RestSharpEngine(DiscordConfig config)
		{
			_config = config;
			_client = new RestSharp.RestClient(Endpoints.BaseApi)
			{
				PreAuthenticate = false,
				ReadWriteTimeout = _config.RestTimeout,
				UserAgent = config.UserAgent
			};
			/*if (_config.ProxyUrl != null)
				_client.Proxy = new WebProxy(_config.ProxyUrl, true, new string[0], _config.ProxyCredentials);
			else*/
			_client.Proxy = null;
            _client.RemoveDefaultParameter("Accept");
            _client.AddDefaultHeader("accept", "*/*");
			_client.AddDefaultHeader("accept-encoding", "gzip,deflate");
        }

		public void SetToken(string token)
		{
			_client.RemoveDefaultParameter("authorization");
			if (token != null)
				_client.AddDefaultHeader("authorization", token);
		}

		public Task<string> Send(string method, string path, string json, CancellationToken cancelToken)
		{
			var request = new RestRequest(path, GetMethod(method));
			request.AddParameter("application/json", json, ParameterType.RequestBody);
			return Send(request, cancelToken);
		}
		public Task<string> SendFile(string method, string path, string filename, Stream stream, CancellationToken cancelToken)
		{
			var request = new RestRequest(path, Method.POST);
			request.AddHeader("content-length", (stream.Length - stream.Position).ToString());

			byte[] bytes = new byte[stream.Length - stream.Position];
			stream.Read(bytes, 0, bytes.Length);
			request.AddFileBytes("file", bytes, filename);
			//request.AddFile("file", x => stream.CopyTo(x), filename); (Broken in latest ver)

			return Send(request, cancelToken);
		}
		private async Task<string> Send(RestRequest request, CancellationToken cancelToken)
		{
			int retryCount = 0;
			while (true)
			{
				var response = await _client.ExecuteTaskAsync(request, cancelToken).ConfigureAwait(false);
				int statusCode = (int)response.StatusCode;
				if (statusCode == 0) //Internal Error
				{
					if (response.ErrorException.HResult == -2146233079 && retryCount++ < 5) //The request was aborted: Could not create SSL/TLS secure channel.
						continue; //Seems to work if we immediately retry
					throw response.ErrorException;
				}
				if (statusCode < 200 || statusCode >= 300) //2xx = Success
					throw new HttpException(response.StatusCode);
				return response.Content;
			}
		}

		private Method GetMethod(string method)
		{
			switch (method)
			{
				case "DELETE": return Method.DELETE;
				case "GET": return Method.GET;
				case "PATCH": return Method.PATCH;
				case "POST": return Method.POST;
				case "PUT": return Method.PUT;
				default: throw new InvalidOperationException($"Unknown HttpMethod: {method}");
			}
		}
	}
}
#endif<commit_msg>Added rate limit support<commit_after>﻿#if !DOTNET5_4
using Discord.API;
using RestSharp;
using System;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;

namespace Discord.Net.Rest
{
    internal sealed class RestSharpEngine : IRestEngine
	{
		private readonly DiscordConfig _config;
		private readonly RestSharp.RestClient _client;

        public RestSharpEngine(DiscordConfig config)
		{
			_config = config;
			_client = new RestSharp.RestClient(Endpoints.BaseApi)
			{
				PreAuthenticate = false,
				ReadWriteTimeout = _config.RestTimeout,
				UserAgent = config.UserAgent
			};
			/*if (_config.ProxyUrl != null)
				_client.Proxy = new WebProxy(_config.ProxyUrl, true, new string[0], _config.ProxyCredentials);
			else*/
			_client.Proxy = null;
            _client.RemoveDefaultParameter("Accept");
            _client.AddDefaultHeader("accept", "*/*");
			_client.AddDefaultHeader("accept-encoding", "gzip,deflate");
        }

		public void SetToken(string token)
		{
			_client.RemoveDefaultParameter("authorization");
			if (token != null)
				_client.AddDefaultHeader("authorization", token);
		}

		public Task<string> Send(string method, string path, string json, CancellationToken cancelToken)
		{
			var request = new RestRequest(path, GetMethod(method));
			request.AddParameter("application/json", json, ParameterType.RequestBody);
			return Send(request, cancelToken);
		}
		public Task<string> SendFile(string method, string path, string filename, Stream stream, CancellationToken cancelToken)
		{
			var request = new RestRequest(path, Method.POST);
			request.AddHeader("content-length", (stream.Length - stream.Position).ToString());

			byte[] bytes = new byte[stream.Length - stream.Position];
			stream.Read(bytes, 0, bytes.Length);
			request.AddFileBytes("file", bytes, filename);
			//request.AddFile("file", x => stream.CopyTo(x), filename); (Broken in latest ver)

			return Send(request, cancelToken);
		}
		private async Task<string> Send(RestRequest request, CancellationToken cancelToken)
		{
			int retryCount = 0;
			while (true)
			{
				var response = await _client.ExecuteTaskAsync(request, cancelToken).ConfigureAwait(false);
				int statusCode = (int)response.StatusCode;
                if (statusCode == 0) //Internal Error
                {
                    //The request was aborted: Could not create SSL/TLS secure channel.
                    if (response.ErrorException.HResult == -2146233079 && retryCount++ < 5)
                        continue; //Retrying seems to fix this somehow?
                    throw response.ErrorException;
                }
                else if (statusCode == 429) //Rate limit
                {
                    var retryAfter = response.Headers
                        .FirstOrDefault(x => x.Name.Equals("Retry-After", StringComparison.OrdinalIgnoreCase));
                    int milliseconds;
                    if (retryAfter != null)
                    {
                        await Task.Delay((int)retryAfter.Value);
                        continue;
                    }
                    throw new HttpException(response.StatusCode);
                }
				else if (statusCode < 200 || statusCode >= 300) //2xx = Success
					throw new HttpException(response.StatusCode);
                else
				    return response.Content;
			}
		}

		private Method GetMethod(string method)
		{
			switch (method)
			{
				case "DELETE": return Method.DELETE;
				case "GET": return Method.GET;
				case "PATCH": return Method.PATCH;
				case "POST": return Method.POST;
				case "PUT": return Method.PUT;
				default: throw new InvalidOperationException($"Unknown HttpMethod: {method}");
			}
		}
	}
}
#endif
{'fraction_non_alphanumeric': 0.10906139315230225, 'fraction_numerical': 0.0075265643447461625, 'mean_word_length': 3.1373626373626373, 'pattern_counts': {'":': 10, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7310118', 'n_tokens_mistral': 2201, 'n_tokens_neox': 1888, 'n_words': 484}