<commit_before>var async = require("async"),
	AWS = require("aws-sdk"),
	fs = require("fs"),
	im = require("gm").subClass({imageMagick: true}),
	s3 = new AWS.S3(),
	spawn = require('child_process').spawn,
	temp = require('temp');
	piexif = require("piexifjs");

//temp.track();

const FORMAT_JPEG = "jpeg";
const FORMAT_PNG = "png";

var CONFIG = require("./config.json");

function getImageType(objectContentType) {
	if (objectContentType === "image/jpeg") {
		return FORMAT_JPEG;
	} else if (objectContentType === "image/png") {
		return FORMAT_PNG;
	} else {
		throw new Error("unsupported objectContentType " + objectContentType);
	}
}

function cross(left, right) {
	var res = [];
	left.forEach(function(l) {
		right.forEach(function(r) {
			res.push([l, r]);
		});
	});
	return res;
}

/**
 * remove double quoutes at the beginning and the end of an etag
 *
 * @param $etag
 * @returns {void|XML|string|*}
 */
function cleanETag($etag) {
	return $etag.replace(/"/g, "");
}

exports.handler = function(event, context) {
	var defaults =  {
			bucketOriginals : "",
			bucketThumbnails : "",
			autorotate: true,
			sharpImage: true,
			sharpOriginalImage: false,
			doWatermark: false,
			watermark : {
				src : ""
			},
			sizes: [],
			folder: "",
			files: [],
			concurrency: 1,
			savelocal: false
		},
		settings = Object.assign({}, defaults, event),
		resultETags = {original: {}, thumbnails: {}};


	console.log(JSON.stringify(settings, null, 2));

	async.mapLimit(settings.files, settings.concurrency,
		function(file, cb) {

			async.waterfall(<commit_msg>added watermark option to the API<commit_after>var async = require("async"),
	AWS = require("aws-sdk"),
	fs = require("fs"),
	im = require("gm").subClass({imageMagick: true}),
	s3 = new AWS.S3(),
	spawn = require('child_process').spawn,
	temp = require('temp');
	piexif = require("piexifjs");

//temp.track();

const FORMAT_JPEG = "jpeg";
const FORMAT_PNG = "png";

var CONFIG = require("./config.json");

function getImageType(objectContentType) {
	if (objectContentType === "image/jpeg") {
		return FORMAT_JPEG;
	} else if (objectContentType === "image/png") {
		return FORMAT_PNG;
	} else {
		throw new Error("unsupported objectContentType " + objectContentType);
	}
}

function cross(left, right) {
	var res = [];
	left.forEach(function(l) {
		right.forEach(function(r) {
			res.push([l, r]);
		});
	});
	return res;
}

/**
 * remove double quoutes at the beginning and the end of an etag
 *
 * @param $etag
 * @returns {void|XML|string|*}
 */
function cleanETag($etag) {
	return $etag.replace(/"/g, "");
}

exports.handler = function(event, context) {
	var defaults =  {
			bucketOriginals : "",
			bucketThumbnails : "",
			autorotate: true,
			sharpImage: true,
			sharpOriginalImage: false,
			doWatermark: false,
			watermark : {
				src : "",
				margin : {
					horizontal: 0,
					vertical: 0
				},
				opacity: 0,
				position: "mc", //top middle bottom, left center right
				mode: "prop", //prop, fill, fit
				mode_prop: 50, // 50% of the image
				thumbnailThreshold: 0 // don't apply a watermarkt to images smaller than this value
			},
			sizes: [],
			folder: "",
			files: [],
			concurrency: 1,
			savelocal: false
		},
		settings = Object.assign({}, defaults, event),
		resultETags = {original: {}, thumbnails: {}};


	console.log(JSON.stringify(settings, null, 2));

	async.mapLimit(settings.files, settings.concurrency,
		function(file, cb) {

			async.waterfall(
{'fraction_non_alphanumeric': 0.16647398843930636, 'fraction_numerical': 0.004624277456647399, 'mean_word_length': 3.8405594405594408, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3149522', 'n_tokens_mistral': 1333, 'n_tokens_neox': 1199, 'n_words': 278}