<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Security;
using System.Net.Sockets;
using System.Security.Authentication;
using System.Threading;
using System.Threading.Tasks;
using Titanium.Web.Proxy.EventArguments;
using Titanium.Web.Proxy.Exceptions;
using Titanium.Web.Proxy.Extensions;
using Titanium.Web.Proxy.Helpers;
using Titanium.Web.Proxy.Http;
using Titanium.Web.Proxy.Models;
using Titanium.Web.Proxy.Network.Tcp;
using Titanium.Web.Proxy.Shared;

namespace Titanium.Web.Proxy
{
    /// <summary>
    /// Handle the request
    /// </summary>
    partial class ProxyServer
    {
        /// <summary>
        /// This is called when client is aware of proxy
        /// So for HTTPS requests client would send CONNECT header to negotiate a secure tcp tunnel via proxy
        /// </summary>
        /// <param name="endPoint"></param>
        /// <param name="tcpClient"></param>
        /// <returns></returns>
        private async Task HandleClient(ExplicitProxyEndPoint endPoint, TcpClient tcpClient)
        {
            var disposed = false;

            var clientStream = new CustomBufferedStream(tcpClient.GetStream(), BufferSize);

            var clientStreamReader = new CustomBinaryReader(clientStream, BufferSize);
            var clientStreamWriter = new StreamWriter(clientStream) { NewLine = ProxyConstants.NewLine };

            Uri httpRemoteUri;

            try
            {
                //read the first line HTTP command
                var httpCmd = await clientStreamReader.ReadLineAsync();

                if (string.IsNullOrEmpty(httpCmd))
                {
                    return;
                }

                //break up the line into three components (method, remote URL & Http Version)
                var httpCmdSplit = httpCmd.Split(ProxyConstants.SpaceSplit, 3);

                //Find the request Verb
                var httpVerb = httpCmdSplit[0].ToUpper();

                httpRemoteUri = httpVerb == "CONNECT" ? new Uri("http://" + httpCmdSplit[1]) : new Uri(httpCmdSplit[1]);

                //parse the HTTP version
                var version = HttpHeader.Version11;
                if (httpCmdSplit.Length == 3)
                {
                    var httpVersion = httpCmdSplit[2].Trim();

                    if (string.Equals(httpVersion, "HTTP/1.0", StringComparison.OrdinalIgnoreCase))
                    {
                        version = HttpHeader.Version10;
                    }
                }

                //filter out excluded host names
                bool excluded = false;

                if (endPoint.ExcludedHttpsHostNameRegex != null)
                {
                    excluded = endPoint.ExcludedHttpsHostNameRegexList.Any(x => x.IsMatch(httpRemoteUri.Host));
                }

                if (endPoint.IncludedHttpsHostNameRegex != null)
                {
                    excluded = !endPoint.IncludedHttpsHostNameRegexList.Any(x => x.IsMatch(httpRemoteUri.Host));
                }

                List<HttpHeader> connectRequestHeaders = null;

                //Client wants to create a secure tcp tunnel (its a HTTPS request)
                if (httpVerb == "CONNECT" && !excluded 
                    && endPoint.RemoteHttpsPorts.Contains(httpRemoteUri.Port))
                {
                    httpRemoteUri = new Uri("https://" + httpCmdSplit[1]);
                    connectRequestHeaders = new List<HttpHeader>();
                    string tmpLine;
                    while (!string.IsNullOrEmpty(tmpLine = await clientStreamReader.ReadLineAsync()))
                    {
                        var header = tmpLine.Split(ProxyConstants.ColonSplit, 2);

                        var newHeader = new HttpHeader(header[0], header[1]);
                        connectRequestHeaders.Add(newHeader);
                    }

                    if (await CheckAuthorization(clientStreamWriter, connectRequestHeaders) == false)
                    {
                        return;
                    }

                    await WriteConnectResponse(clientStreamWriter, version);

                    SslStream sslStream = null;

                    try
                    {
                        sslStream = new SslStream(clientStream);

                        var certName = HttpHelper.GetWildCardDomainName(httpRemoteUri.Host);

                        var certificate = endPoint.GenericCertificate ??
                                          CertificateManager.CreateCertificate(certName, false);

                        //Successfully managed to authenticate the client using the fake certificate
                        await sslStream.AuthenticateAsServerAsync(certificate, false,
                            SupportedSslProtocols, false);

                        //HTTPS server created - we can now decrypt the client's traffic
                        clientStream = new CustomBufferedStream(sslStream, BufferSize);

                        clientStreamReader.Dispose();
                        clientStreamReader = new CustomBinaryReader(clientStream, BufferSize);
                        clientStreamWriter = new StreamWriter(clientStream) { NewLine = ProxyConstants.NewLine };
                    }
                    catch
                    {
                        sslStream?.Dispose();
                        return;
                    }

                    //Now read the actual HTTPS request line
                    httpCmd = await clientStreamReader.ReadLineAsync();
                }
                //Sorry cannot do a HTTPS request decrypt to port 80 at this time
                else if (httpVerb == "CONNECT")
                {
                    //Siphon out CONNECT request headers
                    await clientStreamReader.ReadAndIgnoreAllLinesAsync();

                    //write back successfull CONNECT response
                    await WriteConnectResponse(clientStreamWriter, version);

                    await TcpHelper.SendRaw(this,
                        httpRemoteUri.Host, httpRemoteUri.Port,
                        null, version, null,
                        false,
                        clientStream, tcpConnectionFactory);

                    return;
                }

                //Now create the request
                disposed = await HandleHttpSessionRequest(tcpClient, httpCmd, clientStream, clientStreamReader, clientStreamWriter,
                    httpRemoteUri.Scheme == Uri.UriSchemeHttps ? httpRemoteUri.Host : null, endPoint,
                    connectRequestHeaders);
            }
            catch (Exception e)
            {
                ExceptionFunc(new Exception("Error whilst authorizing request", e));
            }
            finally
            {
                if (!disposed)
                {
                    Dispose(clientStream, clientStreamReader, clientStreamWriter, null);
                }
            }
        }

        /// <summary>
        /// This is called when this proxy acts as a reverse proxy (like a real http server)
        /// So for HTTPS requests we would start SSL negotiation right away without expecting a CONNECT request from client
        /// </summary>
        /// <param name="endPoint"></param>
        /// <param name="tcpClient"></param>
        /// <returns></returns>
        private async Task HandleClient(TransparentProxyEndPoint endPoint, TcpClient tcpClient)
        {
            bool disposed = false;
            var clientStream = new CustomBufferedStream(tcpClient.GetStream(), BufferSize);

            CustomBinaryReader clientStreamReader = null;
            StreamWriter clientStreamWriter = null;

            try
            {
                if (endPoint.EnableSsl)
                {
                    var sslStream = new SslStream(clientStream);
                    clientStream = new CustomBufferedStream(sslStream, BufferSize);

                    //implement in future once SNI supported by SSL stream, for now use the same certificate
                    var certificate = CertificateManager.CreateCertificate(endPoint.GenericCertificateName, false);

                    //Successfully managed to authenticate the client using the fake certificate
                    await sslStream.AuthenticateAsServerAsync(certificate, false,
                        SslProtocols.Tls, false);

                    //HTTPS server created - we can now decrypt the client's traffic
                }

                clientStreamReader = new CustomBinaryReader(clientStream, BufferSize);
                clientStreamWriter = new StreamWriter(clientStream) { NewLine = ProxyConstants.NewLine };

                //now read the request line
                var httpCmd = await clientStreamReader.ReadLineAsync();

                //Now create the request
                disposed = await HandleHttpSessionRequest(tcpClient, httpCmd, clientStream, clientStreamReader, clientStreamWriter,
                    endPoint.EnableSsl ? endPoint.GenericCertificateName : null, endPoint, null);
            }
            finally
            {
                if (!disposed)
                {
                    Dispose(clientStream, clientStreamReader, clientStreamWriter, null);
                }
            }
        }

        /// <summary>
        /// This is the core request handler method for a particular connection from client
        /// Will create new session (request/response) sequence until 
        /// client/server abruptly terminates connection or by normal HTTP termination
        /// </summary>
        /// <param name="client"></param>
        /// <param name="httpCmd"></param>
        /// <param name="clientStream"></param>
        /// <param name="clientStreamReader"></param>
        /// <param name="clientStreamWriter"></param>
        /// <param name="httpsConnectHostname"></param>
        /// <param name="endPoint"></param>
        /// <param name="connectHeaders"></param>
        /// <returns></returns>
        private async Task<bool> HandleHttpSessionRequest(TcpClient client, string httpCmd, Stream clientStream,
            CustomBinaryReader clientStreamReader, StreamWriter clientStreamWriter, string httpsConnectHostname,
            ProxyEndPoint endPoint, List<HttpHeader> connectHeaders)
        {
            bool disposed = false;

            TcpConnection connection = null;

            //Loop through each subsequest request on this particular client connection
            //(assuming HTTP connection is kept alive by client)
            while (true)
            {
                if (string.IsNullOrEmpty(httpCmd))
                {
                    break;
                }

                var args = new SessionEventArgs(BufferSize, HandleHttpSessionResponse)
                {
                    ProxyClient = { TcpClient = client },
                    WebSession = { ConnectHeaders = connectHeaders }
                };

                args.WebSession.ProcessId = new Lazy<int>(() =>
                {
                    var remoteEndPoint = (IPEndPoint)args.ProxyClient.TcpClient.Client.RemoteEndPoint;

                    //If client is localhost get the process id
                    if (NetworkHelper.IsLocalIpAddress(remoteEndPoint.Address))
                    {
                        return NetworkHelper.GetProcessIdFromPort(remoteEndPoint.Port, endPoint.IpV6Enabled);
                    }

                    //can't access process Id of remote request from remote machine
                    return -1;
                });

                try
                {
                    //break up the line into three components (method, remote URL & Http Version)
                    var httpCmdSplit = httpCmd.Split(ProxyConstants.SpaceSplit, 3);

                    var httpMethod = httpCmdSplit[0];

                    //find the request HTTP version
                    var httpVersion = HttpHeader.Version11;
                    if (httpCmdSplit.Length == 3)
                    {
                        var httpVersionString = httpCmdSplit[2].Trim();

                        if (string.Equals(httpVersionString, "HTTP/1.0", StringComparison.OrdinalIgnoreCase))
                        {
                            httpVersion = HttpHeader.Version10;
                        }
                    }

                    //Read the request headers in to unique and non-unique header collections
                    await HeaderParser.ReadHeaders(clientStreamReader, args.WebSession.Request.NonUniqueRequestHeaders, args.WebSession.Request.RequestHeaders);

                    var httpRemoteUri = new Uri(httpsConnectHostname == null
                        ? httpCmdSplit[1]
                        : string.Concat("https://", args.WebSession.Request.Host ?? httpsConnectHostname, httpCmdSplit[1]));

                    args.WebSession.Request.RequestUri = httpRemoteUri;

                    args.WebSession.Request.Method = httpMethod.Trim().ToUpper();
                    args.WebSession.Request.HttpVersion = httpVersion;
                    args.ProxyClient.ClientStream = clientStream;
                    args.ProxyClient.ClientStreamReader = clientStreamReader;
                    args.ProxyClient.ClientStreamWriter = clientStreamWriter;

                    //proxy authorization check
                    if (await CheckAuthorization(clientStreamWriter,
                            args.WebSession.Request.RequestHeaders.Values) == false)
                    {
                        args.Dispose();
                        break;
                    }

                    PrepareRequestHeaders(args.WebSession.Request.RequestHeaders, args.WebSession);
                    args.WebSession.Request.Host = args.WebSession.Request.RequestUri.Authority;

                    //if win auth is enabled
                    //we need a cache of request body
                    //so that we can send it after authentication in WinAuthHandler.cs
                    if (EnableWinAuth<commit_msg>https already authorized inside when doing CONNECT<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Security;
using System.Net.Sockets;
using System.Security.Authentication;
using System.Threading;
using System.Threading.Tasks;
using Titanium.Web.Proxy.EventArguments;
using Titanium.Web.Proxy.Exceptions;
using Titanium.Web.Proxy.Extensions;
using Titanium.Web.Proxy.Helpers;
using Titanium.Web.Proxy.Http;
using Titanium.Web.Proxy.Models;
using Titanium.Web.Proxy.Network.Tcp;
using Titanium.Web.Proxy.Shared;

namespace Titanium.Web.Proxy
{
    /// <summary>
    /// Handle the request
    /// </summary>
    partial class ProxyServer
    {
        /// <summary>
        /// This is called when client is aware of proxy
        /// So for HTTPS requests client would send CONNECT header to negotiate a secure tcp tunnel via proxy
        /// </summary>
        /// <param name="endPoint"></param>
        /// <param name="tcpClient"></param>
        /// <returns></returns>
        private async Task HandleClient(ExplicitProxyEndPoint endPoint, TcpClient tcpClient)
        {
            var disposed = false;

            var clientStream = new CustomBufferedStream(tcpClient.GetStream(), BufferSize);

            var clientStreamReader = new CustomBinaryReader(clientStream, BufferSize);
            var clientStreamWriter = new StreamWriter(clientStream) { NewLine = ProxyConstants.NewLine };

            Uri httpRemoteUri;

            try
            {
                //read the first line HTTP command
                var httpCmd = await clientStreamReader.ReadLineAsync();

                if (string.IsNullOrEmpty(httpCmd))
                {
                    return;
                }

                //break up the line into three components (method, remote URL & Http Version)
                var httpCmdSplit = httpCmd.Split(ProxyConstants.SpaceSplit, 3);

                //Find the request Verb
                var httpVerb = httpCmdSplit[0].ToUpper();

                httpRemoteUri = httpVerb == "CONNECT" ? new Uri("http://" + httpCmdSplit[1]) : new Uri(httpCmdSplit[1]);

                //parse the HTTP version
                var version = HttpHeader.Version11;
                if (httpCmdSplit.Length == 3)
                {
                    var httpVersion = httpCmdSplit[2].Trim();

                    if (string.Equals(httpVersion, "HTTP/1.0", StringComparison.OrdinalIgnoreCase))
                    {
                        version = HttpHeader.Version10;
                    }
                }

                //filter out excluded host names
                bool excluded = false;

                if (endPoint.ExcludedHttpsHostNameRegex != null)
                {
                    excluded = endPoint.ExcludedHttpsHostNameRegexList.Any(x => x.IsMatch(httpRemoteUri.Host));
                }

                if (endPoint.IncludedHttpsHostNameRegex != null)
                {
                    excluded = !endPoint.IncludedHttpsHostNameRegexList.Any(x => x.IsMatch(httpRemoteUri.Host));
                }

                List<HttpHeader> connectRequestHeaders = null;

                //Client wants to create a secure tcp tunnel (its a HTTPS request)
                if (httpVerb == "CONNECT" && !excluded 
                    && endPoint.RemoteHttpsPorts.Contains(httpRemoteUri.Port))
                {
                    httpRemoteUri = new Uri("https://" + httpCmdSplit[1]);
                    connectRequestHeaders = new List<HttpHeader>();
                    string tmpLine;
                    while (!string.IsNullOrEmpty(tmpLine = await clientStreamReader.ReadLineAsync()))
                    {
                        var header = tmpLine.Split(ProxyConstants.ColonSplit, 2);

                        var newHeader = new HttpHeader(header[0], header[1]);
                        connectRequestHeaders.Add(newHeader);
                    }

                    if (await CheckAuthorization(clientStreamWriter, connectRequestHeaders) == false)
                    {
                        return;
                    }

                    await WriteConnectResponse(clientStreamWriter, version);

                    SslStream sslStream = null;

                    try
                    {
                        sslStream = new SslStream(clientStream);

                        var certName = HttpHelper.GetWildCardDomainName(httpRemoteUri.Host);

                        var certificate = endPoint.GenericCertificate ??
                                          CertificateManager.CreateCertificate(certName, false);

                        //Successfully managed to authenticate the client using the fake certificate
                        await sslStream.AuthenticateAsServerAsync(certificate, false,
                            SupportedSslProtocols, false);

                        //HTTPS server created - we can now decrypt the client's traffic
                        clientStream = new CustomBufferedStream(sslStream, BufferSize);

                        clientStreamReader.Dispose();
                        clientStreamReader = new CustomBinaryReader(clientStream, BufferSize);
                        clientStreamWriter = new StreamWriter(clientStream) { NewLine = ProxyConstants.NewLine };
                    }
                    catch
                    {
                        sslStream?.Dispose();
                        return;
                    }

                    //Now read the actual HTTPS request line
                    httpCmd = await clientStreamReader.ReadLineAsync();
                }
                //Sorry cannot do a HTTPS request decrypt to port 80 at this time
                else if (httpVerb == "CONNECT")
                {
                    //Siphon out CONNECT request headers
                    await clientStreamReader.ReadAndIgnoreAllLinesAsync();

                    //write back successfull CONNECT response
                    await WriteConnectResponse(clientStreamWriter, version);

                    await TcpHelper.SendRaw(this,
                        httpRemoteUri.Host, httpRemoteUri.Port,
                        null, version, null,
                        false,
                        clientStream, tcpConnectionFactory);

                    return;
                }

                //Now create the request
                disposed = await HandleHttpSessionRequest(tcpClient, httpCmd, clientStream, clientStreamReader, clientStreamWriter,
                    httpRemoteUri.Scheme == Uri.UriSchemeHttps ? httpRemoteUri.Host : null, endPoint,
                    connectRequestHeaders);
            }
            catch (Exception e)
            {
                ExceptionFunc(new Exception("Error whilst authorizing request", e));
            }
            finally
            {
                if (!disposed)
                {
                    Dispose(clientStream, clientStreamReader, clientStreamWriter, null);
                }
            }
        }

        /// <summary>
        /// This is called when this proxy acts as a reverse proxy (like a real http server)
        /// So for HTTPS requests we would start SSL negotiation right away without expecting a CONNECT request from client
        /// </summary>
        /// <param name="endPoint"></param>
        /// <param name="tcpClient"></param>
        /// <returns></returns>
        private async Task HandleClient(TransparentProxyEndPoint endPoint, TcpClient tcpClient)
        {
            bool disposed = false;
            var clientStream = new CustomBufferedStream(tcpClient.GetStream(), BufferSize);

            CustomBinaryReader clientStreamReader = null;
            StreamWriter clientStreamWriter = null;

            try
            {
                if (endPoint.EnableSsl)
                {
                    var sslStream = new SslStream(clientStream);
                    clientStream = new CustomBufferedStream(sslStream, BufferSize);

                    //implement in future once SNI supported by SSL stream, for now use the same certificate
                    var certificate = CertificateManager.CreateCertificate(endPoint.GenericCertificateName, false);

                    //Successfully managed to authenticate the client using the fake certificate
                    await sslStream.AuthenticateAsServerAsync(certificate, false,
                        SslProtocols.Tls, false);

                    //HTTPS server created - we can now decrypt the client's traffic
                }

                clientStreamReader = new CustomBinaryReader(clientStream, BufferSize);
                clientStreamWriter = new StreamWriter(clientStream) { NewLine = ProxyConstants.NewLine };

                //now read the request line
                var httpCmd = await clientStreamReader.ReadLineAsync();

                //Now create the request
                disposed = await HandleHttpSessionRequest(tcpClient, httpCmd, clientStream, clientStreamReader, clientStreamWriter,
                    endPoint.EnableSsl ? endPoint.GenericCertificateName : null, endPoint, null);
            }
            finally
            {
                if (!disposed)
                {
                    Dispose(clientStream, clientStreamReader, clientStreamWriter, null);
                }
            }
        }

        /// <summary>
        /// This is the core request handler method for a particular connection from client
        /// Will create new session (request/response) sequence until 
        /// client/server abruptly terminates connection or by normal HTTP termination
        /// </summary>
        /// <param name="client"></param>
        /// <param name="httpCmd"></param>
        /// <param name="clientStream"></param>
        /// <param name="clientStreamReader"></param>
        /// <param name="clientStreamWriter"></param>
        /// <param name="httpsConnectHostname"></param>
        /// <param name="endPoint"></param>
        /// <param name="connectHeaders"></param>
        /// <returns></returns>
        private async Task<bool> HandleHttpSessionRequest(TcpClient client, string httpCmd, Stream clientStream,
            CustomBinaryReader clientStreamReader, StreamWriter clientStreamWriter, string httpsConnectHostname,
            ProxyEndPoint endPoint, List<HttpHeader> connectHeaders)
        {
            bool disposed = false;

            TcpConnection connection = null;

            //Loop through each subsequest request on this particular client connection
            //(assuming HTTP connection is kept alive by client)
            while (true)
            {
                if (string.IsNullOrEmpty(httpCmd))
                {
                    break;
                }

                var args = new SessionEventArgs(BufferSize, HandleHttpSessionResponse)
                {
                    ProxyClient = { TcpClient = client },
                    WebSession = { ConnectHeaders = connectHeaders }
                };

                args.WebSession.ProcessId = new Lazy<int>(() =>
                {
                    var remoteEndPoint = (IPEndPoint)args.ProxyClient.TcpClient.Client.RemoteEndPoint;

                    //If client is localhost get the process id
                    if (NetworkHelper.IsLocalIpAddress(remoteEndPoint.Address))
                    {
                        return NetworkHelper.GetProcessIdFromPort(remoteEndPoint.Port, endPoint.IpV6Enabled);
                    }

                    //can't access process Id of remote request from remote machine
                    return -1;
                });

                try
                {
                    //break up the line into three components (method, remote URL & Http Version)
                    var httpCmdSplit = httpCmd.Split(ProxyConstants.SpaceSplit, 3);

                    var httpMethod = httpCmdSplit[0];

                    //find the request HTTP version
                    var httpVersion = HttpHeader.Version11;
                    if (httpCmdSplit.Length == 3)
                    {
                        var httpVersionString = httpCmdSplit[2].Trim();

                        if (string.Equals(httpVersionString, "HTTP/1.0", StringComparison.OrdinalIgnoreCase))
                        {
                            httpVersion = HttpHeader.Version10;
                        }
                    }

                    //Read the request headers in to unique and non-unique header collections
                    await HeaderParser.ReadHeaders(clientStreamReader, args.WebSession.Request.NonUniqueRequestHeaders, args.WebSession.Request.RequestHeaders);

                    var httpRemoteUri = new Uri(httpsConnectHostname == null
                        ? httpCmdSplit[1]
                        : string.Concat("https://", args.WebSession.Request.Host ?? httpsConnectHostname, httpCmdSplit[1]));

                    args.WebSession.Request.RequestUri = httpRemoteUri;

                    args.WebSession.Request.Method = httpMethod.Trim().ToUpper();
                    args.WebSession.Request.HttpVersion = httpVersion;
                    args.ProxyClient.ClientStream = clientStream;
                    args.ProxyClient.ClientStreamReader = clientStreamReader;
                    args.ProxyClient.ClientStreamWriter = clientStreamWriter;

                    //proxy authorization check
                    if (httpsConnectHostname == null &&
                        await CheckAuthorization(clientStreamWriter,
                            args.WebSession.Request.RequestHeaders.Values) == false)
                    {
                        args.Dispose();
                        break;
                    }

                    PrepareRequestHeaders(args.WebSession.Request.RequestHeaders, args.WebSession);
                    args.WebSession.Request.Host = args.WebSession.Request.RequestUri.Authority;

                    //if win auth is enabled
                    //we need a cache of request body
                    //so that we can send it after authentication in WinAuthHandler.cs
                    if (EnableWinAuth
{'fraction_non_alphanumeric': 0.07676867274712118, 'fraction_numerical': 0.002253759199915484, 'mean_word_length': 1.5357621216180015, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 95, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4134645', 'n_tokens_mistral': 6577, 'n_tokens_neox': 6044, 'n_words': 1603}