<commit_before>var _ = require('underscore'),
	async = require('async'),
	cast = require('sc-cast'),
	crypto = require('crypto'),
	fs = require('fs'),
	os = require('os'),
	gm = require('gm'),
	http = require('http');

module.exports = function(_config) {
	var config = _.extend({

		cacheFolder: os.tmpdir(),
		ttl: 1000 * 60 * 60 * 24 * 7, // 1 week
		allowProxy: false,
		imageDir: process.cwd(),

	}, _config);

	return function(_req, _res, _next) {
		var dimensions = _req.query.resize || '0x0';
		var crop = _req.query.crop || '0x0';

		var gmOptions = {};
		if (config.imageMagick) {
			gmOptions.imageMagick = config.imageMagick;
		}
		if (config.graphicsMagick) {
			gmOptions.graphicsMagick = config.graphicsMagick;
		}

		var image = {
			location: _req.query.image,
			resize: {
				width: cast(_.first(dimensions.match(/^[^x]+/)), 'number') || null,
				height: cast(_.first(dimensions.match(/[^x]+$/)), 'number') || null
			},
			crop: {
				width: cast(_.first(crop.match(/^[^x]+/)), 'number') || null,
				height: cast(_.first(crop.match(/[^x]+$/)), 'number') || null
			},
			size: {
				width: 0,
				height: 0
			},
			upscale: /true/i.test(_req.query.upscale)
		};

		if (config.allowProxy && !/^https?\:/.test(image.location)) {
			image.location = image.location.trim().replace(/^\//, '');
			image.location = _req.protocol + '://' + _req.headers.host + '/' + image.location;
		}

		image.hash = crypto.createHash('sha1').update(JSON.stringify(image)).digest('hex');

		async.waterfall([

			/**
			 * If the image already exists and it's within the TTL range use it, otherwise continue
			 */
			function(_callback) {
				var stat,
					createdAt = 0,
					now = new Date().getTime();

				try {
					stat = fs.statSync(config.cacheFolder + '/' + image.hash);
					createdAt = cast(stat.mtime, 'date').getTime();
				} catch (e) {};

				if (stat && now - createdAt < config.ttl) {
					gm(config.cacheFolder + '/' + image.hash)
						.options(gmOptions)
						.stream()
						.pipe(_res);
				} else {
					_callback();
				}

			},

			/**
			 * Get the image
			 */
			function(_callback) {
				if (config.allowProxy) {
					http.get(image.location, function(_httpResponse) {
						if (_httpResponse.statusCode >= 400) return _callback(new Error('status ' + _httpResponse.statusCode));
						_callback(null, _httpResponse);
					}).on('error', function(_error) {
						_callback(_error);
					});
				} else {
					var imageStream = fs.createReadStream(config.imageDir + image.location);
					_callback(null, imageStream);
				}
			},

			/**
			 * Perform image resize/crop operations
			 */
			function(_imageSrc, _manipulationDoneCallback) {
				var gmImage = gm(_imageSrc, image.hash).options(gmOptions);

				async.waterfall([

					//Get original size
					function(_callback) {
						gmImage.size({
							bufferStream: true
						}, function(_error, _size) {
							if (_error) {
								console.log(_error);
								return _callback(_error);
							}
							image.size = _size;

							_callback(null);
						});
					},

					// Crop
					function(_callback) {
						// If a crop ratio has been specified
						if (!(!image.crop.width && !image.crop.height)) {

							var newSize = {
								width: image.size.width,
								height: image.size.height
							};
							var offset = {
								x: 0,
								y: 0
							};

							var sourceRatio = image.size.width / image.size.height;
							var targetRatio = image.crop.width / image.crop.height;

							if (sourceRatio < targetRatio) {
								newSize.height = image.size.width / targetRatio;
							} else if (sourceRatio > targetRatio) {
								newSize.width = image.size.height * targetRatio;
							} else {
								//Matching ratios. dont need to do anything.
								return _callback(null);
							}

							offset.x = (image.size.width - newSize.width) / 2;
							offset.y = (image.size.height - newSize.height) / 2;

							// console.log('Cropped', image.size, "to", newSize, "offset", offset);
							gmImage.crop(
								newSize.width,
								newSize.height,
								offset.x,
								offset.y
							);
						}
						_callback(null);
					},

					// Resize
					function(_callback) {
						// If a width or height has been sepcified
						if (!(!image.resize.width && !image.resize.height)) {
							gmImage.resize(image.resize.width, image.resize.height, image.upscale === true ? '' : '>');
						}
						_callback(null);
					},

				], function(_error) {

					//Stream it back.
					gmImage.stream(function(_error, _stdout, _stderr) {
						if (_error) {
							console.log(_error);
							return _manipulationDoneCallback(_error);
						}

						var writeStream = fs.createWriteStream(config.cacheFolder + '/' + image.hash);

						// Pipe to the response
						_stdout.pipe(_res);

						// Pipe to the cache folder
						_stdout.pipe(writeStream);

						//Close up the waterfall when done
						_stdout.on('end', function() {
							_manipulationDoneCallback(null);
						})
					});
				});

			}

		], function(_error) {
			if (_error) {
				_next(_error);
			}
		});

	}
}
<commit_msg>set the statusCode to 304 if the image was pulled from the cache<commit_after>var _ = require('underscore'),
	async = require('async'),
	cast = require('sc-cast'),
	crypto = require('crypto'),
	fs = require('fs'),
	os = require('os'),
	gm = require('gm'),
	http = require('http');

module.exports = function(_config) {
	var config = _.extend({

		cacheFolder: os.tmpdir(),
		ttl: 1000 * 60 * 60 * 24 * 7, // 1 week
		allowProxy: false,
		imageDir: process.cwd(),

	}, _config);

	return function(_req, _res, _next) {
		var dimensions = _req.query.resize || '0x0';
		var crop = _req.query.crop || '0x0';

		var gmOptions = {};
		if (config.imageMagick) {
			gmOptions.imageMagick = config.imageMagick;
		}
		if (config.graphicsMagick) {
			gmOptions.graphicsMagick = config.graphicsMagick;
		}

		var image = {
			location: _req.query.image,
			resize: {
				width: cast(_.first(dimensions.match(/^[^x]+/)), 'number') || null,
				height: cast(_.first(dimensions.match(/[^x]+$/)), 'number') || null
			},
			crop: {
				width: cast(_.first(crop.match(/^[^x]+/)), 'number') || null,
				height: cast(_.first(crop.match(/[^x]+$/)), 'number') || null
			},
			size: {
				width: 0,
				height: 0
			},
			upscale: /true/i.test(_req.query.upscale)
		};

		if (config.allowProxy && !/^https?\:/.test(image.location)) {
			image.location = image.location.trim().replace(/^\//, '');
			image.location = _req.protocol + '://' + _req.headers.host + '/' + image.location;
		}

		image.hash = crypto.createHash('sha1').update(JSON.stringify(image)).digest('hex');

		async.waterfall([

			/**
			 * If the image already exists and it's within the TTL range use it, otherwise continue
			 */
			function(_callback) {
				var stat,
					createdAt = 0,
					now = new Date().getTime();

				try {
					stat = fs.statSync(config.cacheFolder + '/' + image.hash);
					createdAt = cast(stat.mtime, 'date').getTime();
				} catch (e) {};

				if (stat && now - createdAt < config.ttl) {
					_res.statusCode = 304;
					gm(config.cacheFolder + '/' + image.hash)
						.options(gmOptions)
						.stream()
						.pipe(_res);
				} else {
					_callback();
				}

			},

			/**
			 * Get the image
			 */
			function(_callback) {
				if (config.allowProxy) {
					http.get(image.location, function(_httpResponse) {
						if (_httpResponse.statusCode >= 400) return _callback(new Error('status ' + _httpResponse.statusCode));
						_callback(null, _httpResponse);
					}).on('error', function(_error) {
						_callback(_error);
					});
				} else {
					var imageStream = fs.createReadStream(config.imageDir + image.location);
					_callback(null, imageStream);
				}
			},

			/**
			 * Perform image resize/crop operations
			 */
			function(_imageSrc, _manipulationDoneCallback) {
				var gmImage = gm(_imageSrc, image.hash).options(gmOptions);

				async.waterfall([

					//Get original size
					function(_callback) {
						gmImage.size({
							bufferStream: true
						}, function(_error, _size) {
							if (_error) {
								console.log(_error);
								return _callback(_error);
							}
							image.size = _size;

							_callback(null);
						});
					},

					// Crop
					function(_callback) {
						// If a crop ratio has been specified
						if (!(!image.crop.width && !image.crop.height)) {

							var newSize = {
								width: image.size.width,
								height: image.size.height
							};
							var offset = {
								x: 0,
								y: 0
							};

							var sourceRatio = image.size.width / image.size.height;
							var targetRatio = image.crop.width / image.crop.height;

							if (sourceRatio < targetRatio) {
								newSize.height = image.size.width / targetRatio;
							} else if (sourceRatio > targetRatio) {
								newSize.width = image.size.height * targetRatio;
							} else {
								//Matching ratios. dont need to do anything.
								return _callback(null);
							}

							offset.x = (image.size.width - newSize.width) / 2;
							offset.y = (image.size.height - newSize.height) / 2;

							// console.log('Cropped', image.size, "to", newSize, "offset", offset);
							gmImage.crop(
								newSize.width,
								newSize.height,
								offset.x,
								offset.y
							);
						}
						_callback(null);
					},

					// Resize
					function(_callback) {
						// If a width or height has been sepcified
						if (!(!image.resize.width && !image.resize.height)) {
							gmImage.resize(image.resize.width, image.resize.height, image.upscale === true ? '' : '>');
						}
						_callback(null);
					},

				], function(_error) {

					//Stream it back.
					gmImage.stream(function(_error, _stdout, _stderr) {
						if (_error) {
							console.log(_error);
							return _manipulationDoneCallback(_error);
						}

						var writeStream = fs.createWriteStream(config.cacheFolder + '/' + image.hash);

						// Pipe to the response
						_stdout.pipe(_res);

						// Pipe to the cache folder
						_stdout.pipe(writeStream);

						//Close up the waterfall when done
						_stdout.on('end', function() {
							_manipulationDoneCallback(null);
						})
					});
				});

			}

		], function(_error) {
			if (_error) {
				_next(_error);
			}
		});

	}
}

{'fraction_non_alphanumeric': 0.15880579597393757, 'fraction_numerical': 0.005834873091510259, 'mean_word_length': 2.356396866840731, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1399416', 'n_tokens_mistral': 4670, 'n_tokens_neox': 3299, 'n_words': 676}