<commit_before>/*
 * grunt-connect-ssi
 * https://github.com/anguspiv/grunt-connect-ssi
 *
 * Copyright (c) 2014 <NAME>
 * Licensed under the MIT license.
 */

module.exports = function ssi(opt) {

	var path = require('path');
	var fs = require('fs');
	var URL = require('url');

	//options
	var opt = opt || {};
	var fileTypes = opt.fileTypes ||
					opt.includeList ||
					['.html', '.shtml', '.inc', '.incl'];
	var baseDir = opt.baseDir || __dirname;

	if(!Array.isArray(baseDir)) {
		baseDir = [baseDir];
	}

	//var ssiRegex = opt.ssiRegex || /<!--\#include\s+(file|virtual)=["']([^"'<>|\b]+)['"]\s+-->/gi;
	var includeRegex = opt.includeRegex || /<!--\s*\#include\s+(file|virtual)=["']([^"'<>|\b]+)['"]\s*-->/;

	var ssiRegex = new RegExp(includeRegex.source, 'gi');

	var ssiCache = {};

	var html = opt.html || _html;

	var errorMessage = opt.errorMessage !== null ? opt.errorMessage  : '[There was an error processing this include]';

	var fileEncoding = opt.encoding || 'utf8';

	/**
	 * Test the string for HTML
	 */
	function _html(str) {
		if (!str) return false;
		return /<[:_-\w\s\!\/\=\"\']+>/i.test(str);
	}

	function accept(req) {
		var ha = req.headers["accept"];
		if (!ha) {
			return false;
		}
		return (~ha.indexOf("html") || ~ha.indexOf('text'));
	}

	function leave(req) {
		var url = req.url;
		var ignored = true;
		if (!url) {
			return true;
		}
		fileTypes.forEach(function(item) {
			if (~url.indexOf(item)) {
				ignored = false;
			}
		});
		return false;
	}

	/**
	 * Generates an array of the SSI tags in the given html string
	 * @param  {string} html A string of HTML tags
	 * @return {Array}      A list of include tag objects, with their type, path, and original tag
	 */
	function getIncludes(html) {
		var matches = html.match(ssiRegex);

		var includes = [];

		if (matches) {

			matches.forEach(function(match) {
				var includeParts = includeRegex.exec(match);

				if(includeParts) {
					includes.push({
						type: includeParts[1],
						path: includeParts[2],
						original: includeParts[0],
					});
				}
			});

		}

		return includes;
	}

	/**
	 * Creates the full filepath to the include objects path
	 * @param  {object} include The include object to get the full path for
	 * @param  {string} currDir The filepath to the current working directory
	 * @return {string}         The fullpath to a include objects path,
	 *                          null if the file is not found
	 */
	function getFilePath(include, currDir) {

		var fullPath = null;

		baseDir.some(function(dir) {

			var filepath = dir;

			if(include.type.toLowerCase() === 'file' &&
				currDir) {
					filepath = path.join(dir, currDir, include.path);
				} else {
					filepath = path.join(dir, include.path);
				}

			filepath = path.normalize(filepath);

			if(fs.existsSync(filepath)) {
				fullPath = filepath;
				return true;
			}

		});

		return fullPath;
	}

	/**
	 * Creates a key string from the passed in filepath
	 * @param  {string} filePath the filepath to create the key for
	 * @return {string}          A Key string for the filepath
	 */
	function getKey(filepath) {
		var key = filepath.substring(0, filepath.lastIndexOf(path.extname(filepath)));

		return key.split(/[\\\/]+/).join('-');
	}

	/**
	 * Returns the data for a entry in the cache
	 * @param  {string} key Key for the cache entry
	 * @return {string}     The Data for the cache entry
	 */
	function getCache(key) {

		var cache = ssiCache[key];

		return cache.processed ? cache.data : cache.data.replace(ssiRegex, errorMessage);

	}

	/**
	 * Sets an entry into the cache objects
	 * @param {string} key      key for the cache entry
	 * @param {string} data     data for the cache entry
	 * @param {bool} processed  if the data has been processed yet
	 */
	function setCache(key, data, processed) {

		ssiCache[key] = {
			data:data,
			processed: (processed ? true : false)
		};
	}

	function clearCache() {
		ssiCache = {};
	}

	function getFile(filepath) {
		try {
			return fs.readFileSync(filepath, fileEncoding);
		} catch(e) {
			console.log('Could not read file: '+ filepath +'\nERROR::\n'+e.message);
			return null;
		}
	}

	function parseSSI(data, currDir) {
		currDir = currDir || '';

		var body = data instanceof Buffer ? data.toString(fileEncoding) : data;

		var includes = getIncludes(body);

		if(includes) {
			includes.forEach(function(include) {

				//Grab the new Current Directory
				var newDir = path.join(currDir, path.dirname(include.path));

				//Get the includes absolute filePath
				var filepath = getFilePath(include, currDir);

				var includeBuffer = filepath ? getFile(filepath) : null;

				var includeData = includeBuffer !== null ? parseSSI(includeBuffer, newDir) : errorMessage;

				body = body.replace(include.original.trim() , includeData);

			});
		}

		return body;
	}

	return function ssi(req, res, next) {
<commit_msg>Adds the ability to pass proxies/ redirects to include paths<commit_after>/*
 * grunt-connect-ssi
 * https://github.com/anguspiv/grunt-connect-ssi
 *
 * Copyright (c) 2014 <NAME>
 * Licensed under the MIT license.
 */

module.exports = function ssi(opt) {

	var path = require('path');
	var fs = require('fs');
	var URL = require('url');

	//options
	var opt = opt || {};
	var fileTypes = opt.fileTypes ||
					opt.includeList ||
					['.html', '.shtml', '.inc', '.incl'];
	var baseDir = opt.baseDir || __dirname;

	var proxies = opt.proxies || [];

	if(!Array.isArray(baseDir)) {
		baseDir = [baseDir];
	}

	//var ssiRegex = opt.ssiRegex || /<!--\#include\s+(file|virtual)=["']([^"'<>|\b]+)['"]\s+-->/gi;
	var includeRegex = opt.includeRegex || /<!--\s*\#include\s+(file|virtual)=["']([^"'<>|\b]+)['"]\s*-->/;

	var ssiRegex = new RegExp(includeRegex.source, 'gi');

	var ssiCache = {};

	var html = opt.html || _html;

	var errorMessage = opt.errorMessage !== null ? opt.errorMessage  : '[There was an error processing this include]';

	var fileEncoding = opt.encoding || 'utf8';

	/**
	 * Test the string for HTML
	 */
	function _html(str) {
		if (!str) return false;
		return /<[:_-\w\s\!\/\=\"\']+>/i.test(str);
	}

	function accept(req) {
		var ha = req.headers["accept"];
		if (!ha) {
			return false;
		}
		return (~ha.indexOf("html") || ~ha.indexOf('text'));
	}

	function leave(req) {
		var url = req.url;
		var ignored = true;
		if (!url) {
			return true;
		}
		fileTypes.forEach(function(item) {
			if (~url.indexOf(item)) {
				ignored = false;
			}
		});
		return false;
	}

	/**
	 * Generates an array of the SSI tags in the given html string
	 * @param  {string} html A string of HTML tags
	 * @return {Array}      A list of include tag objects, with their type, path, and original tag
	 */
	function getIncludes(html) {
		var matches = html.match(ssiRegex);

		var includes = [];

		if (matches) {

			matches.forEach(function(match) {
				var includeParts = includeRegex.exec(match);

				if(includeParts) {
					includes.push({
						type: includeParts[1],
						path: includeParts[2],
						original: includeParts[0],
					});
				}
			});

		}

		return includes;
	}

	/**
	 * Creates the full filepath to the include objects path
	 * @param  {object} include The include object to get the full path for
	 * @param  {string} currDir The filepath to the current working directory
	 * @return {string}         The fullpath to a include objects path,
	 *                          null if the file is not found
	 */
	function getFilePath(include, currDir) {

		var fullPath = null;

		baseDir.some(function(dir) {

			var filepath = dir;

			if(include.type.toLowerCase() === 'file' &&
				currDir) {
					filepath = path.join(dir, currDir, include.path);
				} else {
					filepath = path.join(dir, include.path);
				}

			filepath = path.normalize(filepath);

			if(fs.existsSync(filepath)) {
				fullPath = filepath;
				return true;
			}

		});

		return fullPath;
	}

	/**
	 * Creates a key string from the passed in filepath
	 * @param  {string} filePath the filepath to create the key for
	 * @return {string}          A Key string for the filepath
	 */
	function getKey(filepath) {
		var key = filepath.substring(0, filepath.lastIndexOf(path.extname(filepath)));

		return key.split(/[\\\/]+/).join('-');
	}

	/**
	 * Returns the data for a entry in the cache
	 * @param  {string} key Key for the cache entry
	 * @return {string}     The Data for the cache entry
	 */
	function getCache(key) {

		var cache = ssiCache[key];

		return cache.processed ? cache.data : cache.data.replace(ssiRegex, errorMessage);

	}

	/**
	 * Sets an entry into the cache objects
	 * @param {string} key      key for the cache entry
	 * @param {string} data     data for the cache entry
	 * @param {bool} processed  if the data has been processed yet
	 */
	function setCache(key, data, processed) {

		ssiCache[key] = {
			data:data,
			processed: (processed ? true : false)
		};
	}

	function clearCache() {
		ssiCache = {};
	}

	function getFile(filepath) {
		try {
			return fs.readFileSync(filepath, fileEncoding);
		} catch(e) {
			console.log('Could not read file: '+ filepath +'\nERROR::\n'+e.message);
			return null;
		}
	}

	function parseSSI(data, currDir) {
		currDir = currDir || '';

		var body = data instanceof Buffer ? data.toString(fileEncoding) : data;

		var includes = getIncludes(body);

		if(includes) {
			includes.forEach(function(include) {

				//Grab the new Current Directory
				var newDir = path.join(currDir, path.dirname(include.path));

				proxies.forEach(function(proxy) {

                    if(include.path.indexOf(proxy.context) > -1) {

                        for(var rule in proxy.rewrite){
                            if(proxy.rewrite.hasOwnProperty(rule)) {

                                var regexRule = new RegExp(rule);

                                include.path = include.path.replace(regexRule, proxy.rewrite[rule]);
                            }
                        }
                    }


                });

				//Get the includes absolute filePath
				var filepath = getFilePath(include, currDir);

				var includeBuffer = filepath ? getFile(filepath) : null;

				var includeData = includeBuffer !== null ? parseSSI(includeBuffer, newDir) : errorMessage;

				body = body.replace(include.original.trim() , includeData);

			});
		}

		return body;
	}

	return function ssi(req, res, next) {

{'fraction_non_alphanumeric': 0.14739996133771505, 'fraction_numerical': 0.0018364585346994007, 'mean_word_length': 2.82655325443787, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 16, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4340288', 'n_tokens_mistral': 3806, 'n_tokens_neox': 3298, 'n_words': 917}