<commit_before>﻿/* **************************************************************
 * Name:      BotSuite.NET
 * Purpose:   Framework for creating bots
 * Homepage:  http://www.wieschoo.com
 * Copyright: (c) 2013 wieschoo & enWare
 * License:   http://www.wieschoo.com/botsuite/license/
 * *************************************************************/

using System;
using System.Drawing;
using System.IO;
using System.Net;
using System.Text;

namespace BotSuite.Net
{
	/// <summary>
	/// HttpClient class for making bots for Browsergames
	/// </summary>
	public class HttpClient
	{
		CookieContainer _Cookies = new CookieContainer();
		String _UserAgent = String.Empty;

		private Boolean _AutoReferer = true;
		/// <summary>
		/// true, if the HttpClient should adjust the Referer at every change of the URL
		/// </summary>
		public Boolean AutoReferer
		{
			get
			{
				return this._AutoReferer;
			}
			set
			{
				this._AutoReferer = value;
			}
		}

		private String _Referer = null;
		/// <summary>
		/// the current Referer
		/// </summary>
		public String Referer
		{
			get
			{
				return this._Referer;
			}
			set
			{
				this._Referer = value;
			}
		}

		/// <summary>
		/// constructor for the HttpClient class
		/// </summary>
		/// <param name="useragent">a useragent string</param>
		/// <param name="initialReferer">the Referer the first request will be sent from, i.e. www.google.com</param>
		public HttpClient(String useragent, String initialReferer = null)
		{
			this._UserAgent = useragent;
			this._Referer = initialReferer;
		}

		/// <summary>
		/// sends a HTTP POST request to a given URL with given POST data and a optional referer
		/// </summary>
		/// <example>
		/// <code>
		/// var hpdc = new HttpPostDataCollection();
		/// hpdc.Add(new HttpPostData("sender", "wieschoo");
		/// hpdc.Add(new HttpPostData("message", "hallo welt");
		/// var hc = new HttpClient("some-user-agent", "www.google.de");
		/// var html = hc.POST("http://www.codebot.de", hpdc);
		/// </code>
		/// </example>
		/// <param name="url">the URL to send the post request to</param>
		/// <param name="postdata">the POST data</param>
		/// <param name="referer">the referer to send the request from</param>
		/// <returns>the HTML sourcecode</returns>
		public String POST(String url, HttpPostDataCollection postdata, String referer = null)
		{
			if(referer != null)
				this._Referer = referer;

			String src = null;
			try
			{
				HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(url);
				req.CookieContainer = _Cookies;
				req.Method = "POST";
				req.UserAgent = _UserAgent;
				req.ContentType = "application/x-www-form-urlencoded";
				if(this._Referer != null)
					req.Referer = this._Referer;
				byte[] data = Encoding.Default.GetBytes(postdata.ToString());
				req.ContentLength = data.Length;
				using(System.IO.Stream s = req.GetRequestStream())
				{
					s.Write(data, 0, data.Length);
				}
				using(HttpWebResponse resp = (HttpWebResponse)req.GetResponse())
				{
					using(System.IO.Stream s = resp.GetResponseStream())
					{
						using(System.IO.StreamReader sr = new System.IO.StreamReader(s))
						{
							src = sr.ReadToEnd();
							if(this._AutoReferer)
								this._Referer = resp.ResponseUri.AbsolutePath;
						}
					}
				}
			}
			catch(Exception ex)
			{
				throw new RequestException("POST request to " + url + " failed.", ex);
			}
			return src;
		}

		/// <summary>
		/// sends a HTTP GET request to a given URL with a optional referer
		/// </summary>
		/// <example>
		/// <code>
		/// var hc = new HttpClient("some-user-agent", "www.google.de");
		/// var html = hc.GET("http://www.codebot.de");
		/// </code>
		/// </example>
		/// <param name="url">the URL to send the request to</param>
		/// <param name="referer">the referer to send the request from</param>
		/// <returns></returns>
		public String GET(String url, String referer = null)
		{
			if(referer != null)
				this._Referer = referer;

			String src = null;
			try
			{
				HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(url);
				req.CookieContainer = _Cookies;
				req.Method = "GET";
				req.UserAgent = _UserAgent;
				if(this._Referer != null)
					req.Referer = this._Referer;
				using(HttpWebResponse resp = (HttpWebResponse)req.GetResponse())
				{
					using(System.IO.Stream s = resp.GetResponseStream())
					{
						using(System.IO.StreamReader sr = new System.IO.StreamReader(s, Encoding.Default))
						{
							src = sr.ReadToEnd();
							if(this._AutoReferer)
								this._Referer = resp.ResponseUri.AbsoluteUri;
						}
					}
				}
			}
			catch(Exception ex)
			{
				throw new RequestException("GET request to " + url + " failed.", ex);
			}
			return src;
		}

		/// <summary>
		/// tries to convert a sourcecode-string (i.e. from a POST or GET request) into an image (works only if the response was an image)
		/// </summary>
		/// <example>
		/// <code><commit_msg>- fixed HTML content encoding in HttpClient.cs<commit_after>﻿/* **************************************************************
 * Name:      BotSuite.NET
 * Purpose:   Framework for creating bots
 * Homepage:  http://www.wieschoo.com
 * Copyright: (c) 2013 wieschoo & enWare
 * License:   http://www.wieschoo.com/botsuite/license/
 * *************************************************************/

using System;
using System.Drawing;
using System.IO;
using System.Net;
using System.Text;

namespace BotSuite.Net
{
	/// <summary>
	/// HttpClient class for making bots for Browsergames
	/// </summary>
	public class HttpClient
	{
		CookieContainer _Cookies = new CookieContainer();
		String _UserAgent = String.Empty;

		private Boolean _AutoReferer = true;
		/// <summary>
		/// true, if the HttpClient should adjust the Referer at every change of the URL
		/// </summary>
		public Boolean AutoReferer
		{
			get
			{
				return this._AutoReferer;
			}
			set
			{
				this._AutoReferer = value;
			}
		}

		private String _Referer = null;
		/// <summary>
		/// the current Referer
		/// </summary>
		public String Referer
		{
			get
			{
				return this._Referer;
			}
			set
			{
				this._Referer = value;
			}
		}

		/// <summary>
		/// constructor for the HttpClient class
		/// </summary>
		/// <param name="useragent">a useragent string</param>
		/// <param name="initialReferer">the Referer the first request will be sent from, i.e. www.google.com</param>
		public HttpClient(String useragent, String initialReferer = null)
		{
			this._UserAgent = useragent;
			this._Referer = initialReferer;
		}

		/// <summary>
		/// sends a HTTP POST request to a given URL with given POST data and a optional referer
		/// </summary>
		/// <example>
		/// <code>
		/// var hpdc = new HttpPostDataCollection();
		/// hpdc.Add(new HttpPostData("sender", "wieschoo");
		/// hpdc.Add(new HttpPostData("message", "hallo welt");
		/// var hc = new HttpClient("some-user-agent", "www.google.de");
		/// var html = hc.POST("http://www.codebot.de", hpdc);
		/// </code>
		/// </example>
		/// <param name="url">the URL to send the post request to</param>
		/// <param name="postdata">the POST data</param>
		/// <param name="referer">the referer to send the request from</param>
		/// <returns>the HTML sourcecode</returns>
		public String POST(String url, HttpPostDataCollection postdata, String referer = null)
		{
			if(referer != null)
				this._Referer = referer;

			String src = null;
			try
			{
				HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(url);
				req.CookieContainer = _Cookies;
				req.Method = "POST";
				req.UserAgent = _UserAgent;
				req.ContentType = "application/x-www-form-urlencoded";
				if(this._Referer != null)
					req.Referer = this._Referer;
				byte[] data = Encoding.Default.GetBytes(postdata.ToString());
				req.ContentLength = data.Length;
				using(System.IO.Stream s = req.GetRequestStream())
				{
					s.Write(data, 0, data.Length);
				}
				using(HttpWebResponse resp = (HttpWebResponse)req.GetResponse())
				{
					using(System.IO.Stream s = resp.GetResponseStream())
					{
						var enc = (!String.IsNullOrEmpty(resp.CharacterSet)) ? Encoding.GetEncoding(resp.CharacterSet) : Encoding.Default;
						using(System.IO.StreamReader sr = new System.IO.StreamReader(s, enc))
						{
							src = sr.ReadToEnd();
							if(this._AutoReferer)
								this._Referer = resp.ResponseUri.AbsolutePath;
						}
					}
				}
			}
			catch(Exception ex)
			{
				throw new RequestException("POST request to " + url + " failed.", ex);
			}
			return src;
		}

		/// <summary>
		/// sends a HTTP GET request to a given URL with a optional referer
		/// </summary>
		/// <example>
		/// <code>
		/// var hc = new HttpClient("some-user-agent", "www.google.de");
		/// var html = hc.GET("http://www.codebot.de");
		/// </code>
		/// </example>
		/// <param name="url">the URL to send the request to</param>
		/// <param name="referer">the referer to send the request from</param>
		/// <returns></returns>
		public String GET(String url, String referer = null)
		{
			if(referer != null)
				this._Referer = referer;

			String src = null;
			try
			{
				HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(url);
				req.CookieContainer = _Cookies;
				req.Method = "GET";
				req.UserAgent = _UserAgent;
				if(this._Referer != null)
					req.Referer = this._Referer;
				using(HttpWebResponse resp = (HttpWebResponse)req.GetResponse())
				{
					using(System.IO.Stream s = resp.GetResponseStream())
					{
						var enc = (!String.IsNullOrEmpty(resp.CharacterSet)) ? Encoding.GetEncoding(resp.CharacterSet) : Encoding.Default;
						using(System.IO.StreamReader sr = new System.IO.StreamReader(s, enc))
						{
							src = sr.ReadToEnd();
							if(this._AutoReferer)
								this._Referer = resp.ResponseUri.AbsoluteUri;
						}
					}
				}
			}
			catch(Exception ex)
			{
				throw new RequestException("GET request to " + url + " failed.", ex);
			}
			return src;
		}

		/// <summary>
		/// tries to convert a sourcecode-string (i.e. from a POST or GET request) into an image (works only if the response was an image)
		/// </summary>
		/// <example>
		/// <code>
{'fraction_non_alphanumeric': 0.16500935315545928, 'fraction_numerical': 0.0009845426799251747, 'mean_word_length': 2.7888847444983216, 'pattern_counts': {'":': 0, '<': 87, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 16, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7570235', 'n_tokens_mistral': 3974, 'n_tokens_neox': 3278, 'n_words': 800}