<commit_before>


using System;
using System.IO;
using System.Text;
using System.Net;
using System.Net.Sockets;
using System.Collections;
using System.Collections.Generic;

using Mono.Unix.Native;

using Manos.Collections;

namespace Manos.Server {

	public class HttpTransaction : IHttpTransaction {

	        private static readonly long MAX_BUFFERED_CONTENT_LENGTH = 2621440; // 2.5MB (Eventually this will be an environment var)

		public static HttpTransaction BeginTransaction (HttpServer server, IOStream stream, Socket socket, HttpConnectionCallback cb)
		{
			HttpTransaction transaction = new HttpTransaction (server, stream, socket, cb);

			return transaction;
		}

		private bool aborted;
		private bool connection_finished;

		private Queue<IWriteOperation> write_ops;
		
		public HttpTransaction (HttpServer server, IOStream stream, Socket socket, HttpConnectionCallback callback)
		{
			Server = server;
			IOStream = stream;
			Socket = socket;
			ConnectionCallback = callback;

			write_ops = new Queue<IWriteOperation> ();

			stream.OnClose (OnClose);
			stream.ReadUntil ("\r\n\r\n", OnHeaders);
		}

		public HttpServer Server {
			get;
			private set;
		}

		public IOStream IOStream {
			get;
			private set;
		}

		public Socket Socket {
			get;
			private set;
		}

		public  HttpConnectionCallback ConnectionCallback {
			get;
			private set;
		}

		public IHttpRequest Request {
			get;
			private set;
		}

		public IHttpResponse Response {
			get;
			private set;
		}

		public bool Aborted {
			get { return aborted; }	
		}

		// Force the server to disconnect
		public bool NoKeepAlive {
			get;
			set;
		}

		public bool ConnectionFinished {
			get {
				return connection_finished;
			}
		}
		
		public void Abort (int status, string message, params object [] p)
		{
			aborted = true;
		}

		public void Write (List<ArraySegment<byte>> data)
		{
			write_ops.Enqueue (new WriteBytesOperation (data, OnWriteFinished));
			PerformNextWrite ();
		}

		public void SendFile (string file)
		{
			write_ops.Enqueue (new WriteFileOperation (file, OnWriteFinished)); 
			PerformNextWrite ();
		}

		public void Finish ()
		{
			//
			// We mark the connection as finished, and FinishResponse
			// will be raised once all the writing is done.
			//
			connection_finished = true;
		}

		public void Run ()
		{
			ConnectionCallback (this);	
		}
		
		private void OnWriteFinished ()
		{
			if (PerformNextWrite ())
				return;
			
			if (ConnectionFinished)
				FinishResponse ();
		}

		private bool NoWritesQueued {
			get { return write_ops.Count < 1; }	
		}
		
		private bool PerformNextWrite ()
		{
			if (NoWritesQueued)
				return false;
			
			if (IOStream.IsWriting)
				return true;
			
			IWriteOperation op = write_ops.Dequeue ();
			op.Write (IOStream);
			
			return true;
		}
		
		private void FinishResponse ()
		{
			bool disconnect = true;

			if (!NoKeepAlive) {
				string dis;
				if (Request.Http_1_1_Supported && Request.Headers.TryGetValue ("Connection", out dis))
					disconnect = (dis == "close");
			}

			Request = null;
			Response = null;

			if (disconnect) {
			      	IOStream.Close ();
				Server.RemoveTransaction (this);
				return;
			} else 
				IOStream.DisableWriting ();

			IOStream.ReadUntil ("\r\n\r\n", OnHeaders);
		}

		private void OnClose (IOStream stream)
		{
		}

		private void OnHeaders (IOStream stream, byte [] data)
		{
			string h = Encoding.ASCII.GetString (data);
			StringReader reader = new StringReader (h);

			string verb;
			string path;
			string version;

			string line = reader.ReadLine ();
			ParseStartLine (line, out verb, out path, out version);
			
			HttpHeaders headers = new HttpHeaders ();
			headers.Parse (reader);

			Request = new HttpRequest (this, headers, verb, path, Version_1_1_Supported (version));
			Response = new HttpResponse (this, Encoding.ASCII);
			
			
			if (headers.ContentLength != null && headers.ContentLength > 0) {
			        long cl = (long) headers.ContentLength;
				if (cl < MAX_BUFFERED_CONTENT_LENGTH) {
					HandleBody ();
					return;
				} else {
					HandleLargeBody ();
					return;
				}
			}

			stream.DisableReading ();
			Server.RunTransaction (this);
		}

		private void ParseStartLine (string line, out string verb, out string path, out string version)
		{
			int s = 0;
			int e = line.IndexOf (' ');

			verb = line.Substring (s, e);

			s = e + 1;
			e = line.IndexOf (' ', s);
			path = line.Substring (s, e - s);

			s = e + 1;
			version = line.Substring (s);

			if (!version.StartsWith ("HTTP/", StringComparison.InvariantCulture))
				throw new Exception ("Malformed HTTP request, no version specified.");
		}

		private void HandleLargeBody ()
		{
			if (Request.Method != "POST" && Request.Method != "PUT")
				throw new InvalidOperationException ("Large Request bodies are only allowed with PUT or POST operations.");

			string ct = Request.Headers ["Content-Type"];
			if (ct == null || !ct.StartsWith ("multipart/form-data", StringComparison.InvariantCultureIgnoreCase)) {
				// TODO: Maybe someone wants to post large www-form-urlencoded data?
				throw new InvalidOperationException ("Large Request bodies are only allowed with multipart form data.");
			}

			string boundary = ParseBoundary (ct);
			IMFDStream stream = new TempFileMFDStream (IOStream);

			MultipartFormDataParser parser = new MultipartFormDataParser (this.Request, boundary, stream, () => {
				IOStream.DisableReading ();
				Server.RunTransaction (this);
			});

			parser.ParseParts ();
		}

		private void HandleBody ()
		{
			if (Request.Method == "POST" || Request.Method == "PUT") {
				string ct = Request.Headers ["Content-Type"];
				if (ct != null && ct.StartsWith ("application/x-www-form-urlencoded", StringComparison.InvariantCultureIgnoreCase))
					IOStream.ReadBytes ((int) Request.Headers.ContentLength, OnWwwFormData);
				else if (ct != null && ct.StartsWith ("multipart/form-data", StringComparison.InvariantCulture)) {
				        string boundary = ParseBoundary (ct);
					if (boundary == null)
			   		        throw new InvalidOperationException ("Invalid content type boundary.");

					IOStream.ReadBytes ((int) Request.Headers.ContentLength, (ios, data) => {
						OnMultiPartFormData (boundary, data);
					});
				}
			}
		}

		private void OnWwwFormData (IOStream stream, byte [] data)
		{
			//
			// The best I can tell, you can't actually set the content-type of
			// the url-encoded form data.  Looking at the source of apache
			// seems to confirm this.  So for now I wont worry about the
			// encoding type and I'll just use ASCII.
			//

			string post = Encoding.ASCII.GetString (data);

			if (!String.IsNullOrEmpty (post)) {
				// TODO: pass this to the encoder to populate
				DataDictionary post_data = HttpUtility.ParseUrlEncodedData (post);
				if (post_data != null)
					Request.SetWwwFormData (post_data);
			}

			IOStream.DisableReading ();
			Server.RunTransaction (this);
		}

		private void OnMultiPartFormData (string boundary, byte [] data)
		{
			IMFDStream stream = new InMemoryMFDStream (data);
			MultipartFormDataParser parser = new MultipartFormDataParser (this.Request, boundary, stream, () => {
				IOStream.DisableReading ();
				Server.RunTransaction (this);
			});

			parser.ParseParts ();
		}
		
		private bool Version_1_1_Supported (string version)
		{
			return version == "HTTP/1.1";
		}

		public static string ParseBoundary (string ct)
		{
			if (ct == null)
				return null;

			int start = ct.IndexOf ("boundary=");
			if (start < 1)
				return null;
			
			return ct.Substring (start + "boundary=".Length);
		}
	}
}

<commit_msg>Dont null out the request/response objects unless we are actually disconnecting the connection.<commit_after>


using System;
using System.IO;
using System.Text;
using System.Net;
using System.Net.Sockets;
using System.Collections;
using System.Collections.Generic;

using Mono.Unix.Native;

using Manos.Collections;

namespace Manos.Server {

	public class HttpTransaction : IHttpTransaction {

	        private static readonly long MAX_BUFFERED_CONTENT_LENGTH = 2621440; // 2.5MB (Eventually this will be an environment var)

		public static HttpTransaction BeginTransaction (HttpServer server, IOStream stream, Socket socket, HttpConnectionCallback cb)
		{
			HttpTransaction transaction = new HttpTransaction (server, stream, socket, cb);

			return transaction;
		}

		private bool aborted;
		private bool connection_finished;

		private Queue<IWriteOperation> write_ops;
		
		public HttpTransaction (HttpServer server, IOStream stream, Socket socket, HttpConnectionCallback callback)
		{
			Server = server;
			IOStream = stream;
			Socket = socket;
			ConnectionCallback = callback;

			write_ops = new Queue<IWriteOperation> ();

			stream.OnClose (OnClose);
			stream.ReadUntil ("\r\n\r\n", OnHeaders);
		}

		public HttpServer Server {
			get;
			private set;
		}

		public IOStream IOStream {
			get;
			private set;
		}

		public Socket Socket {
			get;
			private set;
		}

		public  HttpConnectionCallback ConnectionCallback {
			get;
			private set;
		}

		public IHttpRequest Request {
			get;
			private set;
		}

		public IHttpResponse Response {
			get;
			private set;
		}

		public bool Aborted {
			get { return aborted; }	
		}

		// Force the server to disconnect
		public bool NoKeepAlive {
			get;
			set;
		}

		public bool ConnectionFinished {
			get {
				return connection_finished;
			}
		}
		
		public void Abort (int status, string message, params object [] p)
		{
			aborted = true;
		}

		public void Write (List<ArraySegment<byte>> data)
		{
			write_ops.Enqueue (new WriteBytesOperation (data, OnWriteFinished));
			PerformNextWrite ();
		}

		public void SendFile (string file)
		{
			write_ops.Enqueue (new WriteFileOperation (file, OnWriteFinished)); 
			PerformNextWrite ();
		}

		public void Finish ()
		{
			//
			// We mark the connection as finished, and FinishResponse
			// will be raised once all the writing is done.
			//
			connection_finished = true;
		}

		public void Run ()
		{
			ConnectionCallback (this);	
		}
		
		private void OnWriteFinished ()
		{
			if (PerformNextWrite ())
				return;
			
			if (ConnectionFinished)
				FinishResponse ();
		}

		private bool NoWritesQueued {
			get { return write_ops.Count < 1; }	
		}
		
		private bool PerformNextWrite ()
		{
			if (NoWritesQueued)
				return false;
			
			if (IOStream.IsWriting)
				return true;
			
			IWriteOperation op = write_ops.Dequeue ();
			op.Write (IOStream);
			
			return true;
		}
		
		private void FinishResponse ()
		{
			bool disconnect = true;

			if (!NoKeepAlive) {
				string dis;
				if (Request.Http_1_1_Supported && Request.Headers.TryGetValue ("Connection", out dis))
					disconnect = (dis == "close");
			}

			if (disconnect) {
				Request = null;
				Response = null;
			      	IOStream.Close ();
				Server.RemoveTransaction (this);
				return;
			} else 
				IOStream.DisableWriting ();

			IOStream.ReadUntil ("\r\n\r\n", OnHeaders);
		}

		private void OnClose (IOStream stream)
		{
		}

		private void OnHeaders (IOStream stream, byte [] data)
		{
			string h = Encoding.ASCII.GetString (data);
			StringReader reader = new StringReader (h);

			string verb;
			string path;
			string version;

			string line = reader.ReadLine ();
			ParseStartLine (line, out verb, out path, out version);
			
			HttpHeaders headers = new HttpHeaders ();
			headers.Parse (reader);

			Request = new HttpRequest (this, headers, verb, path, Version_1_1_Supported (version));
			Response = new HttpResponse (this, Encoding.ASCII);
			
			
			if (headers.ContentLength != null && headers.ContentLength > 0) {
			        long cl = (long) headers.ContentLength;
				if (cl < MAX_BUFFERED_CONTENT_LENGTH) {
					HandleBody ();
					return;
				} else {
					HandleLargeBody ();
					return;
				}
			}

			stream.DisableReading ();
			Server.RunTransaction (this);
		}

		private void ParseStartLine (string line, out string verb, out string path, out string version)
		{
			int s = 0;
			int e = line.IndexOf (' ');

			verb = line.Substring (s, e);

			s = e + 1;
			e = line.IndexOf (' ', s);
			path = line.Substring (s, e - s);

			s = e + 1;
			version = line.Substring (s);

			if (!version.StartsWith ("HTTP/", StringComparison.InvariantCulture))
				throw new Exception ("Malformed HTTP request, no version specified.");
		}

		private void HandleLargeBody ()
		{
			if (Request.Method != "POST" && Request.Method != "PUT")
				throw new InvalidOperationException ("Large Request bodies are only allowed with PUT or POST operations.");

			string ct = Request.Headers ["Content-Type"];
			if (ct == null || !ct.StartsWith ("multipart/form-data", StringComparison.InvariantCultureIgnoreCase)) {
				// TODO: Maybe someone wants to post large www-form-urlencoded data?
				throw new InvalidOperationException ("Large Request bodies are only allowed with multipart form data.");
			}

			string boundary = ParseBoundary (ct);
			IMFDStream stream = new TempFileMFDStream (IOStream);

			MultipartFormDataParser parser = new MultipartFormDataParser (this.Request, boundary, stream, () => {
				IOStream.DisableReading ();
				Server.RunTransaction (this);
			});

			parser.ParseParts ();
		}

		private void HandleBody ()
		{
			if (Request.Method == "POST" || Request.Method == "PUT") {
				string ct = Request.Headers ["Content-Type"];
				if (ct != null && ct.StartsWith ("application/x-www-form-urlencoded", StringComparison.InvariantCultureIgnoreCase))
					IOStream.ReadBytes ((int) Request.Headers.ContentLength, OnWwwFormData);
				else if (ct != null && ct.StartsWith ("multipart/form-data", StringComparison.InvariantCulture)) {
				        string boundary = ParseBoundary (ct);
					if (boundary == null)
			   		        throw new InvalidOperationException ("Invalid content type boundary.");

					IOStream.ReadBytes ((int) Request.Headers.ContentLength, (ios, data) => {
						OnMultiPartFormData (boundary, data);
					});
				}
			}
		}

		private void OnWwwFormData (IOStream stream, byte [] data)
		{
			//
			// The best I can tell, you can't actually set the content-type of
			// the url-encoded form data.  Looking at the source of apache
			// seems to confirm this.  So for now I wont worry about the
			// encoding type and I'll just use ASCII.
			//

			string post = Encoding.ASCII.GetString (data);

			if (!String.IsNullOrEmpty (post)) {
				// TODO: pass this to the encoder to populate
				DataDictionary post_data = HttpUtility.ParseUrlEncodedData (post);
				if (post_data != null)
					Request.SetWwwFormData (post_data);
			}

			IOStream.DisableReading ();
			Server.RunTransaction (this);
		}

		private void OnMultiPartFormData (string boundary, byte [] data)
		{
			IMFDStream stream = new InMemoryMFDStream (data);
			MultipartFormDataParser parser = new MultipartFormDataParser (this.Request, boundary, stream, () => {
				IOStream.DisableReading ();
				Server.RunTransaction (this);
			});

			parser.ParseParts ();
		}
		
		private bool Version_1_1_Supported (string version)
		{
			return version == "HTTP/1.1";
		}

		public static string ParseBoundary (string ct)
		{
			if (ct == null)
				return null;

			int start = ct.IndexOf ("boundary=");
			if (start < 1)
				return null;
			
			return ct.Substring (start + "boundary=".Length);
		}
	}
}


{'fraction_non_alphanumeric': 0.10559854346836596, 'fraction_numerical': 0.002991091748488198, 'mean_word_length': 3.107905982905983, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3094184', 'n_tokens_mistral': 5691, 'n_tokens_neox': 4529, 'n_words': 1352}