<commit_before>﻿using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Web;
using System.Xml.Linq;
using SimpleBrowser.Parser;
using SimpleBrowser.Query;
using SimpleBrowser.Elements;
using SimpleBrowser.Network;

namespace SimpleBrowser
{
	public class Browser
	{
		private WebProxy _proxy;
		private int _timeoutMilliseconds = 30000;
		private NameValueCollection _includeFormValues;
		private XDocument _doc;
		private HttpRequestLog _lastRequestLog;
		private List<LogItem> _logs = new List<LogItem>();
		private IWebRequestFactory _reqFactory;

		static Browser()
		{
			if(ServicePointManager.Expect100Continue)
				ServicePointManager.Expect100Continue = false;
			ServicePointManager.ServerCertificateValidationCallback = delegate { return true; };
		}

		public Browser(IWebRequestFactory requestFactory = null, string name = null)
		{
			UserAgent = "SimpleBrowser (http://github.com/axefrog/SimpleBrowser)";
			RetainLogs = true;
            Cookies = new CookieContainer();
			if (requestFactory == null) requestFactory = new DefaultRequestFactory();
			_reqFactory = requestFactory;
			WindowHandle = name;
			Browser.Register(this);
		}


		private static List<Browser> _allWindows = new List<Browser>();
		public static IEnumerable<Browser> Windows
		{
			get
			{
				return _allWindows;
			}
		}
		public IEnumerable<Browser> Frames
		{
			get
			{
				var dummy = this.XDocument; // Parsing the HTML into XDocument will trigger the loading of frames
				return Browser.Windows.Where(b => b.ParentWindow == this);
			}
		}
		public static void ClearWindows()
		{
			_allWindows.Clear();
		}

		public void Close()
		{
			_history = null;
			_allWindows.Remove(this);
		}
		private void CheckDisposed()
		{
			if (_history == null)
			{
				throw new ObjectDisposedException("This browser has been closed. You cannot access the content or history after closing.");
			}
		}

		public static Browser GetWindowByName(string name)
		{
			return Windows.FirstOrDefault(b => b.WindowHandle == name);
		}
		private static void Register(Browser browser)
		{
			_allWindows.Add(browser);
			if (browser.WindowHandle == null)
			{
				browser.WindowHandle = Guid.NewGuid().ToString().Substring(0, 8);
			}
		}
		public string WindowHandle { get; private set; }
		private Browser ParentWindow { get; set; }
		internal Browser CreateChildBrowser(string name = null)
		{
			Browser child = new Browser(this._reqFactory, name);
			child.ParentWindow = this;
			return child;
		}
		internal void RemoveChildBrowsers()
		{
			_allWindows.RemoveAll((b) => b.ParentWindow == this);
		}

		public string UserAgent { get; set; }
        public bool UseGZip {get; set;}
		public bool RetainLogs { get; set; }
        public CookieContainer Cookies { get; set; }

		public Uri Url { get { return CurrentState.Url; } }
		public string CurrentHtml { get { return CurrentState.Html; } }
        /// <summary>
        /// Obtain the XDocument of the current state to allow XDocument Queries.
        /// </summary>
        public XDocument Document { get  { return XDocument; } }
		public string ResponseText { get { return CurrentState.Html /*TODO What is the difference here?*/; } }
		public string Text { get { return XDocument.Root.Value; } }
		public string ContentType { get { return CurrentState.ContentType; } }

		/// <summary>
		/// Returns a dictionary reflecting the current navigation history. The keys are integers reflecting the position in the history,
		/// where the current page equals 0, the history has negative numbers and the future (pages you can navigate forward to) have positive
		/// numbers.
		/// </summary>
		public IDictionary<int,Uri> NavigationHistory
		{ 
			get
			{
				CheckDisposed();
				return _history.Select((s, i) => new { Index = i, State = s })
					.ToDictionary((i) => i.Index - _historyPosition, (i) => i.State.Url);
			}
		}



		/// <summary>
		/// Returns the current HTML document parsed and converted to a valid XDocument object. Note that the
		/// originating HTML does not have to be valid XML; the parser will use a variety of methods to convert any
		/// invalid markup to valid XML.
		/// </summary>
		internal XDocument XDocument
		{
			get
			{
				if (CurrentState.XDocument == null)
				{
					try
					{
						CurrentState.XDocument = CurrentHtml.ParseHtml();
					}
					catch (Exception ex)
					{
						Log("Error converting HTML to XML for URL " + Url, LogMessageType.Error);
						Log(ex.Message, LogMessageType.Error);
						Log("<b>Exception Stack Trace:</b><br />" + ex.StackTrace.Replace(Environment.NewLine, "<br />"), LogMessageType.StackTrace);
						CurrentState.XDocument = HtmlParser.CreateBlankHtmlDocument();
					}
					// check if we need to create sob-browsers for frames
					foreach (var frame in this.FindAll("iframe"))
					{
						Log("found iframe +" + frame.CurrentElement.Value);  
					}
					
				}
				return CurrentState.XDocument;
			}
		}


		List<NavigationState> _history = new List<NavigationState>();
		int _historyPosition = -1;
		internal class NavigationState
		{
			public Uri Url;
			public string ContentType;
			public string Html;
			internal XDocument XDocument;
		}
		internal NavigationState CurrentState
		{
			get
			{
				CheckDisposed();
				if (_historyPosition == -1) return null;
				return _history[_historyPosition];
			}
		}

		internal void AddNavigationState(NavigationState state)
		{
			while (_history.Count > _historyPosition + 1)
			{
				_history.Remove(_history.Last());
			}
			_historyPosition++;
			_history.Add(state);
			this.InvalidateAllActiveElements();
			while (_history.Count > 20)
			{
				_history.RemoveAt(0);
				_historyPosition--;
			}
		}



		public event Action<Browser, string> MessageLogged;
		public event Action<Browser, HttpRequestLog> RequestLogged;

		public Browser CreateReferenceView()
		{
			Browser b = new Browser
			{
				Cookies = Cookies,
				_doc = _doc,
				_extraHeaders = _extraHeaders,
				_includeFormValues = _includeFormValues,
				_lastRequestLog = _lastRequestLog,
				_logs = _logs,
				_proxy = _proxy,
				_timeoutMilliseconds = _timeoutMilliseconds,
				Accept = Accept,
				LastWebException = LastWebException,
				MessageLogged = MessageLogged,
				RetainLogs = RetainLogs,
				UserAgent = UserAgent
			};
			b.AddNavigationState(this.CurrentState);
			return b;
		}


		private HashSet<string> _extraHeaders = new HashSet<string>();
		public void SetHeader(string header)
		{
			_extraHeaders.Add(header);
		}

		public string Accept { get; set; }

		public void RemoveHeader(string header)
		{
			_extraHeaders.Remove(header);
		}

		public void SetProxy(string host, int port)
		{
			_proxy = new WebProxy(host, port);
		}

		public void SetProxy(string host, int port, string username, string password)
		{
			SetProxy(host, port);
			_proxy.Credentials = new NetworkCredential(username, password);
		}

		public bool Navigate(string url)
		{
			return Navigate(new Uri(url));
		}

		public bool Navigate(string url, int timeoutMilliseconds)
		{
			return Navigate(new Uri(url), timeoutMilliseconds);
		}

		public bool Navigate(Uri url)
		{
			return DoRequest(url, "GET", null, null, null, null, _timeoutMilliseconds);
		}

		public bool Navigate(Uri url, string postData, string contentType)
		{
			return DoRequest(url, "POST", null, postData, contentType, null, _timeoutMilliseconds);
		}

		public bool Navigate(Uri url, NameValueCollection postData, string contentType = null, string encodingType = null)
		{
			return DoRequest(url, "POST", postData, null, contentType, encodingType, _timeoutMilliseconds);
		}

		public bool Navigate(Uri url, int timeoutMilliseconds)
		{
			_timeoutMilliseconds = timeoutMilliseconds;
			return Navigate(url);
		}

		public bool NavigateBack()
		{
			CheckDisposed();
			if (_historyPosition > 0)
			{
				_historyPosition--;
				InvalidateAllActiveElements();
				return true;
			}
			return false;
		}

		public bool NavigateForward()
		{
			CheckDisposed();
			if (_history.Count > _historyPosition + 1)
			{
				_historyPosition++;
				InvalidateAllActiveElements();
				return true;
			}
			return false;
		}

		//\/////////////////////////////////////////////////
		// RAW HTML BROWSER
		//\/////////////////////////////////////////////////




		const string TARGET_SELF = "_self";
		const string TARGET_BLANK = "_blank";
		const string TARGET_TOP = "_top";

		private bool htmlElement_NavigationRequested(HtmlElement.NavigationArgs args)
		{
			Uri fullUri = new Uri(this.Url, args.Uri);
			if(args.TimeoutMilliseconds == 0)args.TimeoutMilliseconds = _timeoutMilliseconds;

			Browser browserToNav = null;
			if (args.Target == TARGET_SELF || String.IsNullOrEmpty(args.Target))
			{
				browserToNav = this;
			}
			else if (args.Target == TARGET_BLANK)
			{
				browserToNav = new Browser(this._reqFactory);
			}
			else
			{
				browserToNav = Browser.Windows.FirstOrDefault(b => b.WindowHandle == args.Target);
				if (browserToNav == null)
				{
					browserToNav = new Browser(this._reqFactory, args.Target);
				}
			}

			return browserToNav.DoRequest(fullUri, args.Method, args.UserVariables, args.PostData, args.ContentType, args.EncodingType, args.TimeoutMilliseconds);
		}
		private IHttpWebRequest PrepareRequestObject(Uri url, string method, string contentType, int timeoutMilliseconds)
		{
			IHttpWebRequest req = _reqFactory.GetWebRequest(url);
			req.Method = method;
			req.ContentType = contentType; // "application/x-www-form-urlencoded";
			req.UserAgent = UserAgent;
			req.Accept = Accept ?? "*/*";
			req.Timeout = timeoutMilliseconds;
			req.AllowAutoRedirect = false;
            if (UseGZip)
            {
                req.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
            }
            req.CookieContainer = Cookies;
			if (_proxy != null)
				req.Proxy = _proxy;
			if(CurrentState != null)
				req.Referer = this.Url.AbsoluteUri;
			return req;
		}

		internal bool DoRequest(Uri uri, string method, NameValueCollection userVariables, string postData, string contentType, string encodingType,  int timeoutMilliseconds)
		{
			/* IMPORTANT INFORMATION:
			 * HttpWebRequest has a bug where if a 302 redirect is encountered (such as from a Response.Redirect), any cookies
			 * generated during the request are ignored and discarded during the internal redirect process. The headers are in
			 * fact returned, but the normal process where the cookie headers are turned into Cookie objects in the cookie
			 * container is skipped, thus breaking the login processes of half the sites on the internet.
			 * 
			 * The workaround is as follows:
			 * 1. Turn off AllowAutoRedirect so we can intercept the redirect and do things manually
			 * 2. Read the Set-Cookie headers from the response and manually insert them into the cookie container
			 * 3. Get the Location header and redirect to the location specified in the "Location" response header
			 * 
			 * Worth noting that even if this bug has been solved in .Net 4 (I haven't checked) we should still use manual
			 * redirection so that we can properly log responses.
			 * 
			 * OBSOLETE ISSUE: (Bug has been resolved in the .Net 4 framework, which this library is now targeted at)
			 * //CookieContainer also has a horrible bug relating to the specified cookie domain. Basically, if it contains
			 * //a cookie where the "domain" token is specified as ".domain.xxx" and you attempt to request http://domain.ext,
			 * //the cookies are not retrievable via that Uri, as you would expect them to be. CookieContainer is incorrectly
			 * //assuming that the leading dot is a prerequisite specifying that a subdomain is required as opposed to the
			 * //correct behaviour which would be to take it to mean that the domain and all subdomains are valid for the cookie.
			 * //http://channel9.msdn.com/forums/TechOff/260235-Bug-in-CookieContainer-where-do-I-report/?CommentID=397315
			 * //The workaround is as follows:
			 * //When retrieving the response, iterate through the Set-Cookie header and any cookie that explicitly sets
			 * //the domain token with a leading dot should also set the cookie without the leading dot.
			 */

			bool handle301Or302Redirect;
			int maxRedirects = 10;
			string html;
			string responseContentType;
			string postBody = "";
			do
			{
				Debug.WriteLine(uri.ToString());
				if (maxRedirects-- == 0)
				{
					Log("Too many 302 redirects", LogMessageType.Error);
					return false;
				}
				handle301Or302Redirect = false;
				IHttpWebRequest req = null;
				try
				{
					req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds);
				}
				catch (NotSupportedException e)
				{
					// Happens when the URL cannot be parsed (example: 'javascript:')
					return false;
				}
				foreach (var header in _extraHeaders)
					req.Headers.Add(header);
				req.Headers.Add(HttpRequestHeader.ContentEncoding, encodingType);
				if (_includeFormValues != null)
				{
					if (userVariables == null)
						userVariables = _includeFormValues;
					else
						userVariables.Add(_includeFormValues);
				}

				if (userVariables != null)
				{
					if (method == "POST")
					{
						postBody = StringUtil.MakeQueryString(userVariables);
						byte[] data = Encoding.GetEncoding(28591).GetBytes(postBody);
						req.ContentLength = data.Length;
						Stream stream = req.GetRequestStream();
						stream.Write(data, 0, data.Length);
						stream.Close();
					}
					else
					{
						uri = new Uri(
							uri.Scheme + "://" + uri.Host + ":"  + uri.Port + uri.AbsolutePath 
							+ ((userVariables.Count > 0) ? "?" + StringUtil.MakeQueryString(userVariables) : "")
							);
						req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds);
					}
				}
				else if (postData != null)
				{
					if (method == "GET")
						throw new InvalidOperationException("Cannot call DoRequest with method GET and non-null postData");
					postBody = postData;
					byte[] data = Encoding.GetEncoding(28591).GetBytes(postData);
					req.ContentLength = data.Length;
					Stream stream = req.GetRequestStream();
					stream.Write(data, 0, data.Length);
					stream.Close();
				}

				if (contentType != null)
					req.ContentType = contentType;

				_lastRequestLog = new HttpRequestLog
				{
					Method = method,
					PostData = userVariables,
					PostBody = postBody,
					RequestHeaders = req.Headers,
					Url = uri
				};
				try
				{
					using (IHttpWebResponse response = req.GetResponse())
					{
						Encoding responseEncoding = Encoding.UTF8; //default
						string charSet = response.CharacterSet;
						if (!String.IsNullOrEmpty(charSet))
						{
							try
							{
								responseEncoding = Encoding.GetEncoding(charSet);
							}
							catch (ArgumentException)
							{
								responseEncoding = Encoding.UTF8; // try using utf8
							}
						}
						
						StreamReader reader = new StreamReader(response.GetResponseStream(), responseEncoding);
						html = reader.ReadToEnd();
						responseContentType = response.ContentType;
						reader.Close();
						_doc = null;
						_includeFormValues = null;

						_lastRequestLog.Text = html;
						_lastRequestLog.ParsedHtml = html;
						_lastRequestLog.ResponseHeaders = response.Headers;
						_lastRequestLog.StatusCode = (int)response.StatusCode;

						if (method == "GET" && uri.Query.Length > 0 && uri.Query != "?")
							_lastRequestLog.QueryStringData = HttpUtility.ParseQueryString(uri.Query);
						if ((int)response.StatusCode == 302 || (int)response.StatusCode == 301)
						{
							uri = new Uri(uri, response.Headers["Location"]);
							handle301Or302Redirect = true;
							Debug.WriteLine("Redirecting to: " + uri);
							method = "GET";
							postData = null;
							userVariables = null;
						}
					}
				}
				catch (WebException ex)
				{
					_lastRequestLog.StatusCode = (int)ex.Status.GetTypeCode();
					if (ex.Response != null)
					{
						_lastRequestLog.ResponseHeaders = ex.Response.Headers;
						StreamReader reader = new StreamReader(ex.Response.GetResponseStream());
						html = reader.ReadToEnd();
						_lastRequestLog.Text = html;
					}

					LastWebException = ex;

					switch (ex.Status)
					{
						case WebExceptionStatus.Timeout:
							Log("A timeout occurred while trying to load the web page", LogMessageType.Error);
							break;

						case WebExceptionStatus.ReceiveFailure:
							Log("The response was cut short prematurely", LogMessageType.Error);
							break;

						default:
							Log("An exception was thrown while trying to load the page: " + ex.Message, LogMessageType.Error);
							break;
					}
					return false;
				}
				finally
				{
					LogRequestData();
				}
			} while (handle301Or302Redirect);
			this.RemoveChildBrowsers(); //Any frames contained in the previous state should be removed. They will be recreated if we ever navigate back
			this.AddNavigationState(new NavigationState() {Html=html, Url=uri, ContentType=contentType});
			return true;
		}

		public string RenderHtmlLogFile(string title = "SimpleBrowser Session Log")
		{
			var formatter = new HtmlLogFormatter();
			return formatter.Render(_logs, title);
		}

		public WebException LastWebException { get; private set; }
		public void ClearException()
		{
			LastWebException = null;
		}

		private HttpRequestLog AcquireRequestData()
		{
			return _lastRequestLog;
		}

		/// <summary>
		/// Performs a culture-invariant text search on the current document, ignoring whitespace, html elements and case, which reduces the 
		/// </summary>
		/// <param name="text"></param>
		/// <returns></returns>
		public bool ContainsText(string text)
		{
			Log("Looking for text: " + text, LogMessageType.Internal);
			text = HttpUtility.HtmlDecode(text);
			string source = HttpUtility.HtmlDecode(XDocument.Root.Value).Replace("&nbsp;", " ");
			var found = new Regex(Regex.Replace(Regex.Escape(text).Replace(@"\ ", " "), @"\s+", @"\s+"), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant).IsMatch(source);
			Log("&rarr; Text " + (found ? "" : "NOT ") + "found!", LogMessageType.Internal);
			return found;
		}

		/// <summary><commit_msg>Allow public access to AcquireRequestData.<commit_after>﻿using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Web;
using System.Xml.Linq;
using SimpleBrowser.Parser;
using SimpleBrowser.Query;
using SimpleBrowser.Elements;
using SimpleBrowser.Network;

namespace SimpleBrowser
{
	public class Browser
	{
		private WebProxy _proxy;
		private int _timeoutMilliseconds = 30000;
		private NameValueCollection _includeFormValues;
		private XDocument _doc;
		private HttpRequestLog _lastRequestLog;
		private List<LogItem> _logs = new List<LogItem>();
		private IWebRequestFactory _reqFactory;

		static Browser()
		{
			if(ServicePointManager.Expect100Continue)
				ServicePointManager.Expect100Continue = false;
			ServicePointManager.ServerCertificateValidationCallback = delegate { return true; };
		}

		public Browser(IWebRequestFactory requestFactory = null, string name = null)
		{
			UserAgent = "SimpleBrowser (http://github.com/axefrog/SimpleBrowser)";
			RetainLogs = true;
            Cookies = new CookieContainer();
			if (requestFactory == null) requestFactory = new DefaultRequestFactory();
			_reqFactory = requestFactory;
			WindowHandle = name;
			Browser.Register(this);
		}


		private static List<Browser> _allWindows = new List<Browser>();
		public static IEnumerable<Browser> Windows
		{
			get
			{
				return _allWindows;
			}
		}
		public IEnumerable<Browser> Frames
		{
			get
			{
				var dummy = this.XDocument; // Parsing the HTML into XDocument will trigger the loading of frames
				return Browser.Windows.Where(b => b.ParentWindow == this);
			}
		}
		public static void ClearWindows()
		{
			_allWindows.Clear();
		}

		public void Close()
		{
			_history = null;
			_allWindows.Remove(this);
		}
		private void CheckDisposed()
		{
			if (_history == null)
			{
				throw new ObjectDisposedException("This browser has been closed. You cannot access the content or history after closing.");
			}
		}

		public static Browser GetWindowByName(string name)
		{
			return Windows.FirstOrDefault(b => b.WindowHandle == name);
		}
		private static void Register(Browser browser)
		{
			_allWindows.Add(browser);
			if (browser.WindowHandle == null)
			{
				browser.WindowHandle = Guid.NewGuid().ToString().Substring(0, 8);
			}
		}
		public string WindowHandle { get; private set; }
		private Browser ParentWindow { get; set; }
		internal Browser CreateChildBrowser(string name = null)
		{
			Browser child = new Browser(this._reqFactory, name);
			child.ParentWindow = this;
			return child;
		}
		internal void RemoveChildBrowsers()
		{
			_allWindows.RemoveAll((b) => b.ParentWindow == this);
		}

		public string UserAgent { get; set; }
        public bool UseGZip {get; set;}
		public bool RetainLogs { get; set; }
        public CookieContainer Cookies { get; set; }

		public Uri Url { get { return CurrentState.Url; } }
		public string CurrentHtml { get { return CurrentState.Html; } }
        /// <summary>
        /// Obtain the XDocument of the current state to allow XDocument Queries.
        /// </summary>
        public XDocument Document { get  { return XDocument; } }
		public string ResponseText { get { return CurrentState.Html /*TODO What is the difference here?*/; } }
		public string Text { get { return XDocument.Root.Value; } }
		public string ContentType { get { return CurrentState.ContentType; } }

		/// <summary>
		/// Returns a dictionary reflecting the current navigation history. The keys are integers reflecting the position in the history,
		/// where the current page equals 0, the history has negative numbers and the future (pages you can navigate forward to) have positive
		/// numbers.
		/// </summary>
		public IDictionary<int,Uri> NavigationHistory
		{ 
			get
			{
				CheckDisposed();
				return _history.Select((s, i) => new { Index = i, State = s })
					.ToDictionary((i) => i.Index - _historyPosition, (i) => i.State.Url);
			}
		}



		/// <summary>
		/// Returns the current HTML document parsed and converted to a valid XDocument object. Note that the
		/// originating HTML does not have to be valid XML; the parser will use a variety of methods to convert any
		/// invalid markup to valid XML.
		/// </summary>
		internal XDocument XDocument
		{
			get
			{
				if (CurrentState.XDocument == null)
				{
					try
					{
						CurrentState.XDocument = CurrentHtml.ParseHtml();
					}
					catch (Exception ex)
					{
						Log("Error converting HTML to XML for URL " + Url, LogMessageType.Error);
						Log(ex.Message, LogMessageType.Error);
						Log("<b>Exception Stack Trace:</b><br />" + ex.StackTrace.Replace(Environment.NewLine, "<br />"), LogMessageType.StackTrace);
						CurrentState.XDocument = HtmlParser.CreateBlankHtmlDocument();
					}
					// check if we need to create sob-browsers for frames
					foreach (var frame in this.FindAll("iframe"))
					{
						Log("found iframe +" + frame.CurrentElement.Value);  
					}
					
				}
				return CurrentState.XDocument;
			}
		}


		List<NavigationState> _history = new List<NavigationState>();
		int _historyPosition = -1;
		internal class NavigationState
		{
			public Uri Url;
			public string ContentType;
			public string Html;
			internal XDocument XDocument;
		}
		internal NavigationState CurrentState
		{
			get
			{
				CheckDisposed();
				if (_historyPosition == -1) return null;
				return _history[_historyPosition];
			}
		}

		internal void AddNavigationState(NavigationState state)
		{
			while (_history.Count > _historyPosition + 1)
			{
				_history.Remove(_history.Last());
			}
			_historyPosition++;
			_history.Add(state);
			this.InvalidateAllActiveElements();
			while (_history.Count > 20)
			{
				_history.RemoveAt(0);
				_historyPosition--;
			}
		}



		public event Action<Browser, string> MessageLogged;
		public event Action<Browser, HttpRequestLog> RequestLogged;

		public Browser CreateReferenceView()
		{
			Browser b = new Browser
			{
				Cookies = Cookies,
				_doc = _doc,
				_extraHeaders = _extraHeaders,
				_includeFormValues = _includeFormValues,
				_lastRequestLog = _lastRequestLog,
				_logs = _logs,
				_proxy = _proxy,
				_timeoutMilliseconds = _timeoutMilliseconds,
				Accept = Accept,
				LastWebException = LastWebException,
				MessageLogged = MessageLogged,
				RetainLogs = RetainLogs,
				UserAgent = UserAgent
			};
			b.AddNavigationState(this.CurrentState);
			return b;
		}


		private HashSet<string> _extraHeaders = new HashSet<string>();
		public void SetHeader(string header)
		{
			_extraHeaders.Add(header);
		}

		public string Accept { get; set; }

		public void RemoveHeader(string header)
		{
			_extraHeaders.Remove(header);
		}

		public void SetProxy(string host, int port)
		{
			_proxy = new WebProxy(host, port);
		}

		public void SetProxy(string host, int port, string username, string password)
		{
			SetProxy(host, port);
			_proxy.Credentials = new NetworkCredential(username, password);
		}

		public bool Navigate(string url)
		{
			return Navigate(new Uri(url));
		}

		public bool Navigate(string url, int timeoutMilliseconds)
		{
			return Navigate(new Uri(url), timeoutMilliseconds);
		}

		public bool Navigate(Uri url)
		{
			return DoRequest(url, "GET", null, null, null, null, _timeoutMilliseconds);
		}

		public bool Navigate(Uri url, string postData, string contentType)
		{
			return DoRequest(url, "POST", null, postData, contentType, null, _timeoutMilliseconds);
		}

		public bool Navigate(Uri url, NameValueCollection postData, string contentType = null, string encodingType = null)
		{
			return DoRequest(url, "POST", postData, null, contentType, encodingType, _timeoutMilliseconds);
		}

		public bool Navigate(Uri url, int timeoutMilliseconds)
		{
			_timeoutMilliseconds = timeoutMilliseconds;
			return Navigate(url);
		}

		public bool NavigateBack()
		{
			CheckDisposed();
			if (_historyPosition > 0)
			{
				_historyPosition--;
				InvalidateAllActiveElements();
				return true;
			}
			return false;
		}

		public bool NavigateForward()
		{
			CheckDisposed();
			if (_history.Count > _historyPosition + 1)
			{
				_historyPosition++;
				InvalidateAllActiveElements();
				return true;
			}
			return false;
		}

		//\/////////////////////////////////////////////////
		// RAW HTML BROWSER
		//\/////////////////////////////////////////////////




		const string TARGET_SELF = "_self";
		const string TARGET_BLANK = "_blank";
		const string TARGET_TOP = "_top";

		private bool htmlElement_NavigationRequested(HtmlElement.NavigationArgs args)
		{
			Uri fullUri = new Uri(this.Url, args.Uri);
			if(args.TimeoutMilliseconds == 0)args.TimeoutMilliseconds = _timeoutMilliseconds;

			Browser browserToNav = null;
			if (args.Target == TARGET_SELF || String.IsNullOrEmpty(args.Target))
			{
				browserToNav = this;
			}
			else if (args.Target == TARGET_BLANK)
			{
				browserToNav = new Browser(this._reqFactory);
			}
			else
			{
				browserToNav = Browser.Windows.FirstOrDefault(b => b.WindowHandle == args.Target);
				if (browserToNav == null)
				{
					browserToNav = new Browser(this._reqFactory, args.Target);
				}
			}

			return browserToNav.DoRequest(fullUri, args.Method, args.UserVariables, args.PostData, args.ContentType, args.EncodingType, args.TimeoutMilliseconds);
		}
		private IHttpWebRequest PrepareRequestObject(Uri url, string method, string contentType, int timeoutMilliseconds)
		{
			IHttpWebRequest req = _reqFactory.GetWebRequest(url);
			req.Method = method;
			req.ContentType = contentType; // "application/x-www-form-urlencoded";
			req.UserAgent = UserAgent;
			req.Accept = Accept ?? "*/*";
			req.Timeout = timeoutMilliseconds;
			req.AllowAutoRedirect = false;
            if (UseGZip)
            {
                req.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
            }
            req.CookieContainer = Cookies;
			if (_proxy != null)
				req.Proxy = _proxy;
			if(CurrentState != null)
				req.Referer = this.Url.AbsoluteUri;
			return req;
		}

		internal bool DoRequest(Uri uri, string method, NameValueCollection userVariables, string postData, string contentType, string encodingType,  int timeoutMilliseconds)
		{
			/* IMPORTANT INFORMATION:
			 * HttpWebRequest has a bug where if a 302 redirect is encountered (such as from a Response.Redirect), any cookies
			 * generated during the request are ignored and discarded during the internal redirect process. The headers are in
			 * fact returned, but the normal process where the cookie headers are turned into Cookie objects in the cookie
			 * container is skipped, thus breaking the login processes of half the sites on the internet.
			 * 
			 * The workaround is as follows:
			 * 1. Turn off AllowAutoRedirect so we can intercept the redirect and do things manually
			 * 2. Read the Set-Cookie headers from the response and manually insert them into the cookie container
			 * 3. Get the Location header and redirect to the location specified in the "Location" response header
			 * 
			 * Worth noting that even if this bug has been solved in .Net 4 (I haven't checked) we should still use manual
			 * redirection so that we can properly log responses.
			 * 
			 * OBSOLETE ISSUE: (Bug has been resolved in the .Net 4 framework, which this library is now targeted at)
			 * //CookieContainer also has a horrible bug relating to the specified cookie domain. Basically, if it contains
			 * //a cookie where the "domain" token is specified as ".domain.xxx" and you attempt to request http://domain.ext,
			 * //the cookies are not retrievable via that Uri, as you would expect them to be. CookieContainer is incorrectly
			 * //assuming that the leading dot is a prerequisite specifying that a subdomain is required as opposed to the
			 * //correct behaviour which would be to take it to mean that the domain and all subdomains are valid for the cookie.
			 * //http://channel9.msdn.com/forums/TechOff/260235-Bug-in-CookieContainer-where-do-I-report/?CommentID=397315
			 * //The workaround is as follows:
			 * //When retrieving the response, iterate through the Set-Cookie header and any cookie that explicitly sets
			 * //the domain token with a leading dot should also set the cookie without the leading dot.
			 */

			bool handle301Or302Redirect;
			int maxRedirects = 10;
			string html;
			string responseContentType;
			string postBody = "";
			do
			{
				Debug.WriteLine(uri.ToString());
				if (maxRedirects-- == 0)
				{
					Log("Too many 302 redirects", LogMessageType.Error);
					return false;
				}
				handle301Or302Redirect = false;
				IHttpWebRequest req = null;
				try
				{
					req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds);
				}
				catch (NotSupportedException e)
				{
					// Happens when the URL cannot be parsed (example: 'javascript:')
					return false;
				}
				foreach (var header in _extraHeaders)
					req.Headers.Add(header);
				req.Headers.Add(HttpRequestHeader.ContentEncoding, encodingType);
				if (_includeFormValues != null)
				{
					if (userVariables == null)
						userVariables = _includeFormValues;
					else
						userVariables.Add(_includeFormValues);
				}

				if (userVariables != null)
				{
					if (method == "POST")
					{
						postBody = StringUtil.MakeQueryString(userVariables);
						byte[] data = Encoding.GetEncoding(28591).GetBytes(postBody);
						req.ContentLength = data.Length;
						Stream stream = req.GetRequestStream();
						stream.Write(data, 0, data.Length);
						stream.Close();
					}
					else
					{
						uri = new Uri(
							uri.Scheme + "://" + uri.Host + ":"  + uri.Port + uri.AbsolutePath 
							+ ((userVariables.Count > 0) ? "?" + StringUtil.MakeQueryString(userVariables) : "")
							);
						req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds);
					}
				}
				else if (postData != null)
				{
					if (method == "GET")
						throw new InvalidOperationException("Cannot call DoRequest with method GET and non-null postData");
					postBody = postData;
					byte[] data = Encoding.GetEncoding(28591).GetBytes(postData);
					req.ContentLength = data.Length;
					Stream stream = req.GetRequestStream();
					stream.Write(data, 0, data.Length);
					stream.Close();
				}

				if (contentType != null)
					req.ContentType = contentType;

				_lastRequestLog = new HttpRequestLog
				{
					Method = method,
					PostData = userVariables,
					PostBody = postBody,
					RequestHeaders = req.Headers,
					Url = uri
				};
				try
				{
					using (IHttpWebResponse response = req.GetResponse())
					{
						Encoding responseEncoding = Encoding.UTF8; //default
						string charSet = response.CharacterSet;
						if (!String.IsNullOrEmpty(charSet))
						{
							try
							{
								responseEncoding = Encoding.GetEncoding(charSet);
							}
							catch (ArgumentException)
							{
								responseEncoding = Encoding.UTF8; // try using utf8
							}
						}
						
						StreamReader reader = new StreamReader(response.GetResponseStream(), responseEncoding);
						html = reader.ReadToEnd();
						responseContentType = response.ContentType;
						reader.Close();
						_doc = null;
						_includeFormValues = null;

						_lastRequestLog.Text = html;
						_lastRequestLog.ParsedHtml = html;
						_lastRequestLog.ResponseHeaders = response.Headers;
						_lastRequestLog.StatusCode = (int)response.StatusCode;

						if (method == "GET" && uri.Query.Length > 0 && uri.Query != "?")
							_lastRequestLog.QueryStringData = HttpUtility.ParseQueryString(uri.Query);
						if ((int)response.StatusCode == 302 || (int)response.StatusCode == 301)
						{
							uri = new Uri(uri, response.Headers["Location"]);
							handle301Or302Redirect = true;
							Debug.WriteLine("Redirecting to: " + uri);
							method = "GET";
							postData = null;
							userVariables = null;
						}
					}
				}
				catch (WebException ex)
				{
					_lastRequestLog.StatusCode = (int)ex.Status.GetTypeCode();
					if (ex.Response != null)
					{
						_lastRequestLog.ResponseHeaders = ex.Response.Headers;
						StreamReader reader = new StreamReader(ex.Response.GetResponseStream());
						html = reader.ReadToEnd();
						_lastRequestLog.Text = html;
					}

					LastWebException = ex;

					switch (ex.Status)
					{
						case WebExceptionStatus.Timeout:
							Log("A timeout occurred while trying to load the web page", LogMessageType.Error);
							break;

						case WebExceptionStatus.ReceiveFailure:
							Log("The response was cut short prematurely", LogMessageType.Error);
							break;

						default:
							Log("An exception was thrown while trying to load the page: " + ex.Message, LogMessageType.Error);
							break;
					}
					return false;
				}
				finally
				{
					LogRequestData();
				}
			} while (handle301Or302Redirect);
			this.RemoveChildBrowsers(); //Any frames contained in the previous state should be removed. They will be recreated if we ever navigate back
			this.AddNavigationState(new NavigationState() {Html=html, Url=uri, ContentType=contentType});
			return true;
		}

		public string RenderHtmlLogFile(string title = "SimpleBrowser Session Log")
		{
			var formatter = new HtmlLogFormatter();
			return formatter.Render(_logs, title);
		}

		public WebException LastWebException { get; private set; }
		public void ClearException()
		{
			LastWebException = null;
		}

		public HttpRequestLog AcquireRequestData()
		{
			return _lastRequestLog;
		}

		/// <summary>
		/// Performs a culture-invariant text search on the current document, ignoring whitespace, html elements and case, which reduces the 
		/// </summary>
		/// <param name="text"></param>
		/// <returns></returns>
		public bool ContainsText(string text)
		{
			Log("Looking for text: " + text, LogMessageType.Internal);
			text = HttpUtility.HtmlDecode(text);
			string source = HttpUtility.HtmlDecode(XDocument.Root.Value).Replace("&nbsp;", " ");
			var found = new Regex(Regex.Replace(Regex.Escape(text).Replace(@"\ ", " "), @"\s+", @"\s+"), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant).IsMatch(source);
			Log("&rarr; Text " + (found ? "" : "NOT ") + "found!", LogMessageType.Internal);
			return found;
		}

		/// <summary>
{'fraction_non_alphanumeric': 0.10407596865476709, 'fraction_numerical': 0.005278624292555508, 'mean_word_length': 3.1221399730820996, 'pattern_counts': {'":': 4, '<': 63, '<?xml version=': 0, '>': 89, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2192370', 'n_tokens_mistral': 13261, 'n_tokens_neox': 10310, 'n_words': 3025}