<commit_before>﻿using System;
using System.Collections.Specialized;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Mime;
using System.Text;
using System.Threading.Tasks;
using System.Web;
using ServiceStack.Text;

namespace FitBot.Services
{
    public class WebRequestService : IWebRequestService
    {
        private const string RootUrl = "https://www.fitocracy.com/";

        public WebRequestService()
        {
            Cookies = new CookieContainer();
        }

        public CookieContainer Cookies { get; set; }

        public async Task<Stream> Get(string endpoint, object args, string expectedContentType)
        {
            var url = RootUrl + endpoint + "/";
            if (args != null)
            {
                url += "?" + FormatArgs(args);
            }
            var request = (HttpWebRequest) WebRequest.Create(url);
            request.CookieContainer = Cookies;
            request.UserAgent = "FitBot";
            var response = await request.GetResponseAsync();
            if (expectedContentType != null &&
                expectedContentType != response.ContentType &&
                new ContentType(expectedContentType).MediaType != new ContentType(response.ContentType).MediaType)
            {
                throw new Exception("TODO: unexpected content type");
            }
            return response.GetResponseStream();
        }

        public async Task Post(string endpoint, object data, NameValueCollection headers)
        {
            var url = RootUrl + endpoint + "/";
            var request = (HttpWebRequest) WebRequest.Create(url);
            request.CookieContainer = Cookies;
            request.Method = "POST";
            request.Referer = url;
            request.UserAgent = "FitBot";
            if (data != null)
            {
                using (var stream = await request.GetRequestStreamAsync())
                {
                    var bytes = Encoding.UTF8.GetBytes(FormatArgs(data));
                    await stream.WriteAsync(bytes, 0, bytes.Length);
                }
            }
            using (var response = await request.GetResponseAsync())
            {
                if (response.ContentType != "application/json")
                {
                    throw new Exception("TODO: unexpected content type");
                }

                JsonObject json;
                using (var stream = response.GetResponseStream())
                {
                    json = JsonSerializer.DeserializeFromStream<JsonObject>(stream);
                }

                if (json != null && json["success"] != "true" && json["result"] != "true")
                {
                    throw new Exception("TODO: " + json["error"]);
                }

                if (headers != null)
                {
                    foreach (string key in response.Headers)
                    {
                        headers[key] = response.Headers[key];
                    }
                }
            }
        }<commit_msg>Include failure reason in failed POST request exceptions<commit_after>﻿using System;
using System.Collections.Specialized;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Mime;
using System.Text;
using System.Threading.Tasks;
using System.Web;
using ServiceStack.Text;

namespace FitBot.Services
{
    public class WebRequestService : IWebRequestService
    {
        private const string RootUrl = "https://www.fitocracy.com/";

        public WebRequestService()
        {
            Cookies = new CookieContainer();
        }

        public CookieContainer Cookies { get; set; }

        public async Task<Stream> Get(string endpoint, object args, string expectedContentType)
        {
            var url = RootUrl + endpoint + "/";
            if (args != null)
            {
                url += "?" + FormatArgs(args);
            }
            var request = (HttpWebRequest) WebRequest.Create(url);
            request.CookieContainer = Cookies;
            request.UserAgent = "FitBot";
            var response = await request.GetResponseAsync();
            if (expectedContentType != null &&
                expectedContentType != response.ContentType &&
                new ContentType(expectedContentType).MediaType != new ContentType(response.ContentType).MediaType)
            {
                throw new Exception("TODO: unexpected content type");
            }
            return response.GetResponseStream();
        }

        public async Task Post(string endpoint, object data, NameValueCollection headers)
        {
            var url = RootUrl + endpoint + "/";
            var request = (HttpWebRequest) WebRequest.Create(url);
            request.CookieContainer = Cookies;
            request.Method = "POST";
            request.Referer = url;
            request.UserAgent = "FitBot";
            if (data != null)
            {
                using (var stream = await request.GetRequestStreamAsync())
                {
                    var bytes = Encoding.UTF8.GetBytes(FormatArgs(data));
                    await stream.WriteAsync(bytes, 0, bytes.Length);
                }
            }
            using (var response = await request.GetResponseAsync())
            {
                if (response.ContentType != "application/json")
                {
                    throw new Exception("TODO: unexpected content type");
                }

                JsonObject json;
                using (var stream = response.GetResponseStream())
                {
                    json = JsonSerializer.DeserializeFromStream<JsonObject>(stream);
                }

                if (json != null && json["success"] != "true" && json["result"] != "true")
                {
                    throw new Exception("TODO: " + json["error"] + json["reason"]);
                }

                if (headers != null)
                {
                    foreach (string key in response.Headers)
                    {
                        headers[key] = response.Headers[key];
                    }
                }
            }
        }
{'fraction_non_alphanumeric': 0.09632878492527615, 'fraction_numerical': 0.000649772579597141, 'mean_word_length': 1.5568936877076411, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3382003', 'n_tokens_mistral': 1482, 'n_tokens_neox': 1393, 'n_words': 350}