<commit_before>var util = require('../util/util');
var url = require('url');
var net = require('net');
var dns = require('dns');
var extend = require('util')._extend;

var rules = [];
var hosts = [];
var filters = [];

function parse(text) {
	rules = [];
	hosts = [];
	filters = [];
	if (!text || !(text = text.trim())) {
		return;
	}
	
	text.split(/\n|\r\n|\r/g).forEach(parseHost);
}

function parseHost(line) {
	line = line.replace(/#.*$/, '').trim().split(/\s+/);
	var pattern = line[0] && line[0].trim();
	var matcher = line[1] && line[1].trim();
	
	if (!pattern || !matcher) {
		return;
	}
	
	if (net.isIP(pattern) || util.isRegExp(matcher) || (util.hasProtocol(pattern) && !/^https?:\/\//.test(pattern))) {
		var tmp = pattern;
		pattern = matcher;
		matcher = tmp;
	}
	
	var isIP = net.isIP(matcher);
	var isRegExp = util.isRegExp(pattern);
	var protocol;
	if (!isRegExp) {
		protocol = util.getProtocol(pattern);
		
		if (!isIP) {
			if (pattern.indexOf('/', protocol == null ? 0 : pattern.indexOf('://') + 3) == -1) {
				pattern += '/';
			}
		} else if (!(pattern = util.getHost(pattern))) {
			return;
		}
	} else if (!(pattern = util.toRegExp(pattern))) {
		return;
	}
	
	line = {
			isRegExp: isRegExp,
			protocol: protocol,
			pattern: pattern,
			matcher: matcher
		};
	
	if (isIP) {
		hosts.push(line);
	} else if (/^filter:\/\//.test(matcher)) {
		filters.push(line);
	} else {
		rules.push(line);
	}
}

exports.parse = parse;

function resolve(_url, callback) {
	var rule = {};
	var matchedUrl = resolveRule(_url, rule);
	var error, done;
	var matcher = matchedUrl || _url;
	var options = url.parse(matcher);
	var hosts = options.hosts = [];
	options.url = matcher;
	options.rule = rule;
	options.hasMatcher = !!matchedUrl;

	if (matchedUrl) {
		resolveHost(matchedUrl, function(err, ip) {
			hosts[1] = ip;
			error = err;
			execCallback();
		});
		
		hosts[0] = null;
		execCallback();
	} else {
		resolveHost(_url, function(err, ip) {
			hosts[0] = ip;
			if (!matchedUrl) {
				hosts[1] = ip;
				error = err;
			}
			execCallback();
		});
	}
	
	function execCallback() {
		if (!done && (error || (typeof hosts[0] != 'undefined' && typeof hosts[1] != 'undefined'))) {
			done = true;
			hosts[1] = hosts[1] || hosts[0];
			hosts[0] = hosts[0] || hosts[1];
			callback(error, options);
		}
	}
}

exports.resolve = resolve;

function resolveHost(_url, callback) {
	var options = _url ? url.parse(util.setProtocol(_url)) : {};
	var protocol = options.protocol;
	if (!util.isWebProtocol(protocol)) {
		callback(null, null);
		return;
	}
	var hostname = options.hostname;
	for (var i = 0, host; host = hosts[i]; i++) {
		if (host.isRegExp ? host.pattern.test(_url) : (hostname == host.pattern && (!host.protocol || host.protocol == protocol))) {
			callback(null, host.matcher);
			return;
		}
	}
	
	try {
		dns.lookup(hostname, function(err, ip, addressType) {
		      callback(err, err ? null : (ip || (!addressType || addressType === 4 ? '127.0.0.1' : '0:0:0:0:0:0:0:1')));
		  });
	} catch(err) {//如果断网，可能直接抛异常，https代理没有用到error-handler
		callback(err);
	}
}

exports.resolveHost = resolveHost;

function resolveRule(url, data) {
	data = data || {};
	var _url = url.replace(/(?:\?|#).*$/, '');
	for (var i = 0, rule; rule = rules[i]; i++) {
		var pattern = rule.pattern;
		if (rule.isRegExp) {
			if (pattern.test(url)) {
				var regExp = {};
				for (var i = 0; i < 10; i++) {
					regExp['$' + i] = RegExp['$' + i] || '';
				}
				extend(data, rule);
				return rule.matcher.replace(/(^|.)?(\$[1-9])/g, 
						function(matched, $1, $2) {
					return $1 == '\\' ? $2 : ($1 || '') + regExp[$2];
				});
			}
		} else {
			pattern = setProtocol(pattern, url);
			if (_url.indexOf(pattern) === 0) {
				var len = pattern.length;
				if (pattern == _url || isPathSeparator(_url[len]) || isPathSeparator(pattern[len - 1])) {
					extend(data, rule);
					return join(setProtocol(rule.matcher, url), url.substring(len));
				}
			}
		}
	}
}

function setProtocol(target, source) {
	if (util.hasProtocol(target)) {
		return target;
	}
	
	var protocol = util.getProtocol(source);
	if (protocol == null) {
		return target;
	}
	
	return protocol + '//' + target;
}

exports.resolveRule = resolveRule;

function isPathSeparator(ch) {
	return ch == '/' || ch == '\\';
}

function join(first, last) {
	if (!first || !last) {
		return first + last;
	}
	
	var len = first.length - 1;
	if (isPathSeparator(first[len])) {
		return isPathSeparator(last[0]) ? first.substring(0, len) + last : first + last;
	}
	
	return isPathSeparator(last[0]) ? first + last : first + '/' + last;
}

<commit_msg>filters<commit_after>var util = require('../util/util');
var url = require('url');
var net = require('net');
var dns = require('dns');
var extend = require('util')._extend;

var rules = [];
var hosts = [];
var filters = [];

function parse(text) {
	rules = [];
	hosts = [];
	filters = [];
	if (!text || !(text = text.trim())) {
		return;
	}
	
	text.split(/\n|\r\n|\r/g).forEach(parseHost);
}

function parseHost(line) {
	line = line.replace(/#.*$/, '').trim().split(/\s+/);
	var pattern = line[0] && line[0].trim();
	var matcher = line[1] && line[1].trim();
	
	if (!pattern || !matcher) {
		return;
	}
	
	if (net.isIP(pattern) || util.isRegExp(matcher) || (util.hasProtocol(pattern) && !/^https?:\/\//.test(pattern))) {
		var tmp = pattern;
		pattern = matcher;
		matcher = tmp;
	}
	
	var isIP = net.isIP(matcher);
	var isRegExp = util.isRegExp(pattern);
	var protocol;
	if (!isRegExp) {
		protocol = util.getProtocol(pattern);
		
		if (!isIP) {
			if (pattern.indexOf('/', protocol == null ? 0 : pattern.indexOf('://') + 3) == -1) {
				pattern += '/';
			}
		} else if (!(pattern = util.getHost(pattern))) {
			return;
		}
	} else if (!(pattern = util.toRegExp(pattern))) {
		return;
	}
	
	line = {
			isRegExp: isRegExp,
			protocol: protocol,
			pattern: pattern,
			matcher: matcher
		};
	
	if (isIP) {
		hosts.push(line);
	} else if (/^filter:\/\//.test(matcher)) {
		filters.push(line);
	} else {
		rules.push(line);
	}
}

exports.parse = parse;

function resolve(_url, callback) {
	var rule = {};
	var matchedUrl = resolveRule(_url, rule);
	var error, done;
	var matcher = matchedUrl || _url;
	var options = url.parse(matcher);
	var hosts = options.hosts = [];
	options.url = matcher;
	options.rule = rule;
	options.hasMatcher = !!matchedUrl;

	if (matchedUrl) {
		resolveHost(matchedUrl, function(err, ip) {
			hosts[1] = ip;
			error = err;
			execCallback();
		});
		
		hosts[0] = null;
		execCallback();
	} else {
		resolveHost(_url, function(err, ip) {
			hosts[0] = ip;
			if (!matchedUrl) {
				hosts[1] = ip;
				error = err;
			}
			execCallback();
		});
	}
	
	function execCallback() {
		if (!done && (error || (typeof hosts[0] != 'undefined' && typeof hosts[1] != 'undefined'))) {
			done = true;
			hosts[1] = hosts[1] || hosts[0];
			hosts[0] = hosts[0] || hosts[1];
			callback(error, options);
		}
	}
}

exports.resolve = resolve;

function resolveHost(_url, callback) {
	var options = _url ? url.parse(util.setProtocol(_url)) : {};
	var protocol = options.protocol;
	if (!util.isWebProtocol(protocol)) {
		callback(null, null);
		return;
	}
	var hostname = options.hostname;
	for (var i = 0, host; host = hosts[i]; i++) {
		if (host.isRegExp ? host.pattern.test(_url) : (hostname == host.pattern && (!host.protocol || host.protocol == protocol))) {
			callback(null, host.matcher);
			return;
		}
	}
	
	try {
		dns.lookup(hostname, function(err, ip, addressType) {
		      callback(err, err ? null : (ip || (!addressType || addressType === 4 ? '127.0.0.1' : '0:0:0:0:0:0:0:1')));
		  });
	} catch(err) {//如果断网，可能直接抛异常，https代理没有用到error-handler
		callback(err);
	}
}

exports.resolveHost = resolveHost;

function resolveRule(url, data, _rules) {
	data = data || {};
	_rules = _rules || rules;
	var _url = url.replace(/(?:\?|#).*$/, '');
	for (var i = 0, rule; rule = _rules[i]; i++) {
		var pattern = rule.pattern;
		if (rule.isRegExp) {
			if (pattern.test(url)) {
				var regExp = {};
				for (var i = 0; i < 10; i++) {
					regExp['$' + i] = RegExp['$' + i] || '';
				}
				extend(data, rule);
				return rule.matcher.replace(/(^|.)?(\$[1-9])/g, 
						function(matched, $1, $2) {
					return $1 == '\\' ? $2 : ($1 || '') + regExp[$2];
				});
			}
		} else {
			pattern = setProtocol(pattern, url);
			if (_url.indexOf(pattern) === 0) {
				var len = pattern.length;
				if (pattern == _url || isPathSeparator(_url[len]) || isPathSeparator(pattern[len - 1])) {
					extend(data, rule);
					return join(setProtocol(rule.matcher, url), url.substring(len));
				}
			}
		}
	}
}

function setProtocol(target, source) {
	if (util.hasProtocol(target)) {
		return target;
	}
	
	var protocol = util.getProtocol(source);
	if (protocol == null) {
		return target;
	}
	
	return protocol + '//' + target;
}

exports.resolveRule = resolveRule;

function resolveFilter(url, data) {
	return resolveRule(url, data, filters);
}

exports.resolveFilter = resolveFilter;

function isPathSeparator(ch) {
	return ch == '/' || ch == '\\';
}

function join(first, last) {
	if (!first || !last) {
		return first + last;
	}
	
	var len = first.length - 1;
	if (isPathSeparator(first[len])) {
		return isPathSeparator(last[0]) ? first.substring(0, len) + last : first + last;
	}
	
	return isPathSeparator(last[0]) ? first + last : first + '/' + last;
}


{'fraction_non_alphanumeric': 0.20196623210087625, 'fraction_numerical': 0.011327206668091472, 'mean_word_length': 3.3129032258064517, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4065443', 'n_tokens_mistral': 3844, 'n_tokens_neox': 3464, 'n_words': 753}