<commit_before>var socks = require('socksv5');
var url = require('url');
var net = require('net');
var extend = require('extend');
var EventEmitter = require('events').EventEmitter;
var hparser = require('hparser');
var dispatch = require('./https');
var util = require('./util');
var rules = require('./rules');
var socketMgr = require('./socket-mgr');
var rulesUtil = require('./rules/util');
var existsCustomCert = require('./https/util').existsCustomCert;
var pluginMgr = require('./plugins');
var config = require('./config');

var LOCALHOST = '127.0.0.1';
var TUNNEL_HOST_RE = /^[^:\/]+\.[^:\/]+:\d+$/;
var STATUS_CODES = require('http').STATUS_CODES || {};
var getRawHeaderNames = hparser.getRawHeaderNames;
var formatHeaders = hparser.formatHeaders;
var getRawHeaders = hparser.getRawHeaders;
var ESTABLISHED_CTN = 'HTTP/1.1 200 Connection Established\r\nProxy-Agent: ' + config.name + '\r\n\r\n';

function tunnelProxy(server, proxy) {
  server.on('connect', function(req, reqSocket, head) {//ws, wss, https proxy
    var tunnelUrl = req.fullUrl = util.setProtocol(TUNNEL_HOST_RE.test(req.url) ? req.url : req.headers.host, true);
    var options;
    var parseUrl = function (_url, port) {
      _url = _url || tunnelUrl;
      options = req.options = url.parse(_url);
      options.port = options.port || port || 443;
      return options;
    };
    parseUrl();
    reqSocket.on('error', util.noop);
    tunnelUrl = req.fullUrl = 'tunnel://' + options.host;
    proxy.emit('_request', tunnelUrl);
    var resSocket, responsed, reqEmitter, data, rulesMgr;
    var headers, inspect, reqData, resData, res, rollBackTunnel, buf;
    req.isTunnel = true;
    reqSocket.clientIp = req.clientIp = util.getClientIp(req);
    req.reqId = reqSocket.reqId = util.getReqId();
    var hostname = options.hostname;
    var isICloundCKDB = hostname === 'p22-ckdatabase.icloud.com';
    var isIPHost = !isICloundCKDB && net.isIP(hostname);
    var policy = req.headers[config.WHISTLE_POLICY_HEADER];
    var useTunnelPolicy = policy == 'tunnel';
    var isLocalUIUrl = !useTunnelPolicy && (config.isLocalUIUrl(hostname) || config.isPluginUrl(hostname));
    var _rules = req.rules = (isICloundCKDB || isLocalUIUrl) ? {} : rules.initRules(req);
    rules.resolveRulesFile(req, function() {
      var filter = req.filter;
      var disable = req.disable;
      var isDisabeIntercept = function() {
        return isICloundCKDB || useTunnelPolicy || disable.intercept || disable.https || disable.capture;
      };
      var isEnableIntercept = function() {
        if (filter.https || filter.capture || filter.intercept || policy === 'intercept') {
          return true;
        }
        if (!rulesUtil.properties.get('interceptHttpsConnects')) {
          return existsCustomCert(hostname);
        }
        if (isIPHost) {
          return options.port == 80 || options.port == 443 || util.isLocalAddress(reqSocket.clientIp);
        }
        return true;
      };
      var isIntercept = function() {
        return isLocalUIUrl || (!isDisabeIntercept() && isEnableIntercept());
      };
      var plugin = isIntercept() ? false : pluginMgr.resolveWhistlePlugins(req);
      var handlePluginRules = function(_rulesMgr) {
        if (_rulesMgr) {
          rulesMgr = _rulesMgr;
          util.mergeRules(req, rulesMgr.resolveRules(tunnelUrl));
          plugin = pluginMgr.getPluginByRuleUrl(util.rule.getUrl(_rules.rule));
        }
        filter = req.filter;
        disable = req.disable;
        if (req.headers[config.WEBUI_HEAD]) {
          delete req.headers[config.WEBUI_HEAD];
          reqSocket.destroy();
          return false;
        }
        return true;
      };
      pluginMgr.getTunnelRules(req, function(_rulesMgr) {
        abortIfUnavailable(reqSocket);
        if (!handlePluginRules(_rulesMgr)) {
          return;
        }
        if (isIntercept()) {
          reqSocket.rulesHeaders = req.rulesHeaders;
          dispatch(reqSocket, hostname, function(_resSocket) {
            resSocket = _resSocket;
          }, function(_buf) {
            buf = _buf;
            reqSocket.pause();
            rollBackTunnel = true;
            plugin = pluginMgr.resolveWhistlePlugins(req);
            pluginMgr.getTunnelRules(req, function(_rulesMgr2) {
              if (handlePluginRules(_rulesMgr2)) {
                handleTunnel();
              }
            });
          });
          reqSocket.write(ESTABLISHED_CTN);
          return;
        }
        
        handleTunnel();
        
        function handleTunnel() {
          var reqRawHeaderNames = getRawHeaderNames(req.rawHeaders) || {};
          inspect = req.enable.inspect;
          headers = req.headers;
          reqData = {
            ip: req.clientIp,
            port: req.clientPort,
            method: util.toUpperCase(req.method) || 'CONNECT',
            httpVersion: req.httpVersion || '1.1',
            headers: headers,
            rawHeaderNames: reqRawHeaderNames
          };
          resData = {headers: {}};
          pluginMgr.postStats(req);
          req.whistleTunnelPlugins = req.whistlePlugins;
          req.whistlePlugins = null;
          reqEmitter = new EventEmitter();
          req.clientPort = reqSocket.remotePort;
          data = reqEmitter.data = {
            id: req.reqId,
            url: options.host,
            startTime: Date.now(),
            rules: _rules,
            req: reqData,
            res: resData,
            isHttps: true,
            inspect: inspect,
            rulesHeaders: req.rulesHeaders
          };
          var cId = headers['x-whistle-composer-id'];
          if (cId) {
            data.cId = cId;
            delete headers['x-whistle-composer-id'];
          }
          if (!filter.hide) {
            proxy.emit('request', reqEmitter);
          }

          if (req.enable.abort || filter.abort || disable.tunnel) {
            return reqSocket.destroy();
          }

          var statusCode = util.getMatcherValue(_rules.statusCode);
          if (statusCode) {
            if (statusCode == 200) {
              resSocket = util.getEmptyRes();
              var reqDelay = util.getMatcherValue(_rules.reqDelay);
              data.requestTime = data.dnsTime = Date.now();
              if (reqDelay > 0) {
                setTimeout(handleConnect, reqDelay);
              } else {
                handleConnect();
              }
              return;
            }
            return sendEstablished(statusCode);
          }

          pluginMgr.loadPlugin(plugin, function() {
            var tunnelPort, proxyUrl;
            if(plugin) {
              tunnelPort = plugin.ports && plugin.ports.tunnelPort;
              if (!tunnelPort) {
                return emitError(new Error('No plugin.tunnelServer'));
              }
              proxyUrl = 'proxy://127.0.0.1:' + tunnelPort;
              pluginMgr.addRuleHeaders(req, _rules);
            }

            var realUrl = _rules.rule && _rules.rule.url.replace('https:', 'tunnel:');
            if (/^tunnel:\/\//.test(realUrl) && realUrl != tunnelUrl) {
              tunnelUrl = realUrl;
              data.realUrl = realUrl.replace('tunnel://', '');
              parseUrl();
            }

            rules.getProxy(tunnelUrl, proxyUrl ? null : req, function(err, hostIp, hostPort) {
              if (!proxyUrl) {
                proxyUrl = _rules.proxy ? _rules.proxy.matcher : null;
              }

              if (proxyUrl) {
                var isSocks = /^socks:\/\//.test(proxyUrl);
                var _url = 'http:' + util.removeProtocol(proxyUrl);
                data.proxy = true;
                resolveHost(_url, function(ip) {
                  options = parseUrl(_url, isSocks ? 1080 : 80);
                  var isProxyPort = options.port == config.port;
                  if (isProxyPort && util.isLocalAddress(ip)) {
                    return emitError(new Error('Unable to proxy to itself'));
                  }
                  var onConnect = function(proxySocket, _res) {
                    resSocket = proxySocket;
                    res = _res;
                    handleConnect();
                    abortIfUnavailable(resSocket);
                  };
                  var dstOptions = url.parse(tunnelUrl);
                  dstOptions.proxyHost = ip;
                  dstOptions.proxyPort = parseInt(options.port, 10);
                  resData.port = dstOptions.port = dstOptions.port || 443;
                  dstOptions.host = dstOptions.hostname;
                  var _headers = extend({}, headers);
                  pluginMgr.addRuleHeaders(req, _rules, _headers);
                  _headers.host = dstOptions.hostname + ':' + (dstOptions.port || 443);
                  util.parseRuleJson(_rules.reqHeaders, function(reqHeaders) {
                    extend(headers, reqHeaders);
                    if (disable.clientIp || disable.clientIP
                    || util.isLocalAddress(headers[config.CLIENT_IP_HEAD])) {
                      delete headers[config.CLIENT_IP_HEAD];
                    } else {
                      var forwardedFor = util.getMatcherValue(_rules.forwardedFor);
                      if (forwardedFor) {
                        headers[config.CLIENT_IP_HEAD] = forwardedFor;
                      }
                    }
                    var delReqHeaders = util.parseDeleteProperties(req).reqHeaders;
                    Object.keys(delReqHeaders).forEach(function(name) {
                      delete headers[name];
                    });
                    dstOptions.headers = formatHeaders(headers, reqRawHeaderNames);
                    if (isSocks) {
                      dstOptions.proxyPort = options.port || 1080;
                      dstOptions.localDNS = false;
                      dstOptions.auths = config.getAuths(options);
                    } else {
                      dstOptions.proxyPort = options.port || 80;
                      dstOptions.proxyAuth = options.auth;
                      if (isProxyPort) {
                        _headers[config.WEBUI_HEAD] = 1;
                      }
                    }
                    var netMgr = isSocks ? socks : config;
                    var reqDelay = util.getMatcherValue(_rules.reqDelay);
                    if (reqDelay > 0) {
                      setTimeout(function() {
                        netMgr.connect(dstOptions, onConnect).on('error', emitError);
                      }, reqDelay);
                    } else {
                      netMgr.connect(dstOptions, onConnect).on('error', emitError);
                    }
                  });
                });
              } else {
                tunnel(hostIp, hostPort);
              }
            });
          });
        }

        function tunnel(hostIp, hostPort) {
          resolveHost(tunnelUrl, function(ip, port) {
            if (port) {
              req.hostIp = resData.ip = ip + ':' + port;
              resData.port = port;
            } else {
              req.hostIp = resData.ip = ip;
              resData.port = port = options.port;
            }
            resSocket = net.connect(port, ip, handleConnect);
            abortIfUnavailable(resSocket);
          }, hostIp, hostPort);
        }

        function handleConnect() {
          reqSocket.headers = headers;
          reqSocket.fullUrl = tunnelUrl;
          reqSocket.inspectFrames = useTunnelPolicy || inspect;
          if (buf) {
            resSocket.write(buf);
          }
          socketMgr.handleConnect(reqSocket, resSocket);
          var resDelay = util.getMatcherValue(_rules.resDelay);
          if (resDelay > 0) {
            setTimeout(sendEstablished, resDelay);
          } else {
            sendEstablished();
          }
        }

        function resolveHost(url, callback, hostIp, hostPort) {
          var hostHandler = function(err, ip, port, host) {
            if (host) {
              _rules.host = host;
            }
            data.requestTime = data.dnsTime = Date.now();
            req.hostIp = resData.ip = ip || LOCALHOST;
            reqEmitter.emit('send', data);
            err ? emitError(err) : callback(ip, port);
          };
          if (hostIp) {
            hostHandler(null, hostIp, hostPort);
          } else {<commit_msg>refactor: supports roll back to tunnel<commit_after>var socks = require('socksv5');
var url = require('url');
var net = require('net');
var extend = require('extend');
var EventEmitter = require('events').EventEmitter;
var hparser = require('hparser');
var dispatch = require('./https');
var util = require('./util');
var rules = require('./rules');
var socketMgr = require('./socket-mgr');
var rulesUtil = require('./rules/util');
var existsCustomCert = require('./https/util').existsCustomCert;
var pluginMgr = require('./plugins');
var config = require('./config');

var LOCALHOST = '127.0.0.1';
var TUNNEL_HOST_RE = /^[^:\/]+\.[^:\/]+:\d+$/;
var STATUS_CODES = require('http').STATUS_CODES || {};
var getRawHeaderNames = hparser.getRawHeaderNames;
var formatHeaders = hparser.formatHeaders;
var getRawHeaders = hparser.getRawHeaders;
var ESTABLISHED_CTN = 'HTTP/1.1 200 Connection Established\r\nProxy-Agent: ' + config.name + '\r\n\r\n';

function tunnelProxy(server, proxy) {
  server.on('connect', function(req, reqSocket, head) {//ws, wss, https proxy
    var tunnelUrl = req.fullUrl = util.setProtocol(TUNNEL_HOST_RE.test(req.url) ? req.url : req.headers.host, true);
    var options;
    var parseUrl = function (_url, port) {
      _url = _url || tunnelUrl;
      options = req.options = url.parse(_url);
      options.port = options.port || port || 443;
      return options;
    };
    parseUrl();
    reqSocket.on('error', util.noop);
    tunnelUrl = req.fullUrl = 'tunnel://' + options.host;
    proxy.emit('_request', tunnelUrl);
    var resSocket, responsed, reqEmitter, data, rulesMgr;
    var headers, inspect, reqData, resData, res, rollBackTunnel, buf;
    req.isTunnel = true;
    reqSocket.clientIp = req.clientIp = util.getClientIp(req);
    req.reqId = reqSocket.reqId = util.getReqId();
    var hostname = options.hostname;
    var isICloundCKDB = hostname === 'p22-ckdatabase.icloud.com';
    var isIPHost = !isICloundCKDB && net.isIP(hostname);
    var policy = req.headers[config.WHISTLE_POLICY_HEADER];
    var useTunnelPolicy = policy == 'tunnel';
    var isLocalUIUrl = !useTunnelPolicy && (config.isLocalUIUrl(hostname) || config.isPluginUrl(hostname));
    var _rules = req.rules = (isICloundCKDB || isLocalUIUrl) ? {} : rules.initRules(req);
    rules.resolveRulesFile(req, function() {
      var filter = req.filter;
      var disable = req.disable;
      var isDisabeIntercept = function() {
        return isICloundCKDB || useTunnelPolicy || disable.intercept || disable.https || disable.capture;
      };
      var isEnableIntercept = function() {
        if (filter.https || filter.capture || filter.intercept || policy === 'intercept') {
          return true;
        }
        if (!rulesUtil.properties.get('interceptHttpsConnects')) {
          return existsCustomCert(hostname);
        }
        if (isIPHost) {
          return options.port == 80 || options.port == 443 || util.isLocalAddress(reqSocket.clientIp);
        }
        return true;
      };
      var isIntercept = function() {
        return isLocalUIUrl || (!isDisabeIntercept() && isEnableIntercept());
      };
      var plugin = isIntercept() ? false : pluginMgr.resolveWhistlePlugins(req);
      var handlePluginRules = function(_rulesMgr) {
        if (_rulesMgr) {
          rulesMgr = _rulesMgr;
          util.mergeRules(req, rulesMgr.resolveRules(tunnelUrl));
          plugin = pluginMgr.getPluginByRuleUrl(util.rule.getUrl(_rules.rule));
        }
        filter = req.filter;
        disable = req.disable;
        if (req.headers[config.WEBUI_HEAD]) {
          delete req.headers[config.WEBUI_HEAD];
          reqSocket.destroy();
          return false;
        }
        return true;
      };
      pluginMgr.getTunnelRules(req, function(_rulesMgr) {
        abortIfUnavailable(reqSocket);
        if (!handlePluginRules(_rulesMgr)) {
          return;
        }
        if (isIntercept()) {
          reqSocket.rulesHeaders = req.rulesHeaders;
          dispatch(reqSocket, hostname, function(_resSocket) {
            resSocket = _resSocket;
          }, function(_buf) {
            buf = _buf;
            reqSocket.pause();
            rollBackTunnel = true;
            plugin = pluginMgr.resolveWhistlePlugins(req);
            pluginMgr.getTunnelRules(req, function(_rulesMgr2) {
              if (handlePluginRules(_rulesMgr2)) {
                handleTunnel();
              }
            });
          });
          reqSocket.write(ESTABLISHED_CTN);
          return;
        }
        
        handleTunnel();
        
        function handleTunnel() {
          var reqRawHeaderNames = getRawHeaderNames(req.rawHeaders) || {};
          inspect = req.enable.inspect;
          headers = req.headers;
          reqData = {
            ip: req.clientIp,
            port: req.clientPort,
            method: util.toUpperCase(req.method) || 'CONNECT',
            httpVersion: req.httpVersion || '1.1',
            headers: headers,
            rawHeaderNames: reqRawHeaderNames
          };
          resData = {headers: {}};
          pluginMgr.postStats(req);
          req.whistleTunnelPlugins = req.whistlePlugins;
          req.whistlePlugins = null;
          reqEmitter = new EventEmitter();
          req.clientPort = reqSocket.remotePort;
          data = reqEmitter.data = {
            id: req.reqId,
            url: options.host,
            startTime: Date.now(),
            rules: _rules,
            req: reqData,
            res: resData,
            isHttps: true,
            inspect: inspect,
            rulesHeaders: req.rulesHeaders
          };
          var cId = headers['x-whistle-composer-id'];
          if (cId) {
            data.cId = cId;
            delete headers['x-whistle-composer-id'];
          }
          if (!filter.hide) {
            proxy.emit('request', reqEmitter);
          }

          if (req.enable.abort || filter.abort || disable.tunnel) {
            return reqSocket.destroy();
          }

          var statusCode = util.getMatcherValue(_rules.statusCode);
          if (statusCode) {
            if (statusCode == 200) {
              resSocket = util.getEmptyRes();
              var reqDelay = util.getMatcherValue(_rules.reqDelay);
              data.requestTime = data.dnsTime = Date.now();
              if (reqDelay > 0) {
                setTimeout(handleConnect, reqDelay);
              } else {
                handleConnect();
              }
              return;
            }
            return sendEstablished(statusCode);
          }

          pluginMgr.loadPlugin(plugin, function() {
            var tunnelPort, proxyUrl;
            if(plugin) {
              tunnelPort = plugin.ports && plugin.ports.tunnelPort;
              if (!tunnelPort) {
                return emitError(new Error('No plugin.tunnelServer'));
              }
              proxyUrl = 'proxy://127.0.0.1:' + tunnelPort;
              pluginMgr.addRuleHeaders(req, _rules);
            }

            var realUrl = _rules.rule && _rules.rule.url.replace('https:', 'tunnel:');
            if (/^tunnel:\/\//.test(realUrl) && realUrl != tunnelUrl) {
              tunnelUrl = realUrl;
              data.realUrl = realUrl.replace('tunnel://', '');
              parseUrl();
            }

            rules.getProxy(tunnelUrl, proxyUrl ? null : req, function(err, hostIp, hostPort) {
              if (!proxyUrl) {
                proxyUrl = _rules.proxy ? _rules.proxy.matcher : null;
              }

              if (proxyUrl) {
                var isSocks = /^socks:\/\//.test(proxyUrl);
                var _url = 'http:' + util.removeProtocol(proxyUrl);
                data.proxy = true;
                resolveHost(_url, function(ip) {
                  options = parseUrl(_url, isSocks ? 1080 : 80);
                  var isProxyPort = options.port == config.port;
                  if (isProxyPort && util.isLocalAddress(ip)) {
                    return emitError(new Error('Unable to proxy to itself'));
                  }
                  var onConnect = function(proxySocket, _res) {
                    resSocket = proxySocket;
                    res = _res;
                    handleConnect();
                    abortIfUnavailable(resSocket);
                  };
                  var dstOptions = url.parse(tunnelUrl);
                  dstOptions.proxyHost = ip;
                  dstOptions.proxyPort = parseInt(options.port, 10);
                  resData.port = dstOptions.port = dstOptions.port || 443;
                  dstOptions.host = dstOptions.hostname;
                  var _headers = extend({}, headers);
                  pluginMgr.addRuleHeaders(req, _rules, _headers);
                  _headers.host = dstOptions.hostname + ':' + (dstOptions.port || 443);
                  util.parseRuleJson(_rules.reqHeaders, function(reqHeaders) {
                    extend(headers, reqHeaders);
                    if (disable.clientIp || disable.clientIP
                    || util.isLocalAddress(headers[config.CLIENT_IP_HEAD])) {
                      delete headers[config.CLIENT_IP_HEAD];
                    } else {
                      var forwardedFor = util.getMatcherValue(_rules.forwardedFor);
                      if (forwardedFor) {
                        headers[config.CLIENT_IP_HEAD] = forwardedFor;
                      }
                    }
                    var delReqHeaders = util.parseDeleteProperties(req).reqHeaders;
                    Object.keys(delReqHeaders).forEach(function(name) {
                      delete headers[name];
                    });
                    dstOptions.headers = formatHeaders(headers, reqRawHeaderNames);
                    if (isSocks) {
                      dstOptions.proxyPort = options.port || 1080;
                      dstOptions.localDNS = false;
                      dstOptions.auths = config.getAuths(options);
                    } else {
                      dstOptions.proxyPort = options.port || 80;
                      dstOptions.proxyAuth = options.auth;
                      if (isProxyPort) {
                        _headers[config.WEBUI_HEAD] = 1;
                      }
                    }
                    var netMgr = isSocks ? socks : config;
                    var reqDelay = util.getMatcherValue(_rules.reqDelay);
                    if (reqDelay > 0) {
                      setTimeout(function() {
                        netMgr.connect(dstOptions, onConnect).on('error', emitError);
                      }, reqDelay);
                    } else {
                      netMgr.connect(dstOptions, onConnect).on('error', emitError);
                    }
                  });
                });
              } else {
                tunnel(hostIp, hostPort);
              }
            });
          });
        }

        function tunnel(hostIp, hostPort) {
          resolveHost(tunnelUrl, function(ip, port) {
            if (port) {
              req.hostIp = resData.ip = ip + ':' + port;
              resData.port = port;
            } else {
              req.hostIp = resData.ip = ip;
              resData.port = port = options.port;
            }
            resSocket = net.connect(port, ip, handleConnect);
            abortIfUnavailable(resSocket);
          }, hostIp, hostPort);
        }

        function handleConnect() {
          reqSocket.headers = headers;
          reqSocket.fullUrl = tunnelUrl;
          reqSocket.inspectFrames = useTunnelPolicy || inspect;
          if (buf) {
            resSocket.write(buf);
            buf = null;
          }
          socketMgr.handleConnect(reqSocket, resSocket);
          var resDelay = util.getMatcherValue(_rules.resDelay);
          if (resDelay > 0) {
            setTimeout(sendEstablished, resDelay);
          } else {
            sendEstablished();
          }
        }

        function resolveHost(url, callback, hostIp, hostPort) {
          var hostHandler = function(err, ip, port, host) {
            if (host) {
              _rules.host = host;
            }
            data.requestTime = data.dnsTime = Date.now();
            req.hostIp = resData.ip = ip || LOCALHOST;
            reqEmitter.emit('send', data);
            err ? emitError(err) : callback(ip, port);
          };
          if (hostIp) {
            hostHandler(null, hostIp, hostPort);
          } else {
{'fraction_non_alphanumeric': 0.1160573621490608, 'fraction_numerical': 0.00476671379519289, 'mean_word_length': 1.702915165411071, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '709577', 'n_tokens_mistral': 6842, 'n_tokens_neox': 6697, 'n_words': 1295}