<commit_before>var AWS = require('aws-sdk'),
	Promise = require("bluebird"),
	shortid = require('shortid');

var utils = require("./utils");

module.exports = function(awsconfig, dynamodboptions) {

	AWS.config.update(awsconfig);

	var dynamodb = new Promise.promisifyAll(new AWS.DynamoDB(dynamodboptions));

	var query = function(table, conditions, index, options) {

		var params = {
			TableName: table
		};

		if (index) {
			params.IndexName = index;
		}

		// Legacy KeyConditions are in use
		if (conditions.constructor === Array) {

			var keys = {};
			conditions.forEach(function(condition) {

				keys[condition.key] = {
					ComparisonOperator: condition.operator,
					AttributeValueList: [utils.itemize(condition.value)],
				};

			});

			params.KeyConditions = keys;

		}

		// KeyConditionExpression are in use
		if (conditions.constructor === Object) {

			params.KeyConditionExpression = conditions.KeyConditionExpression;
			params.ExpressionAttributeNames = conditions.ExpressionAttributeNames;

			var ExpressionAttributeValues = {};

			var values = conditions.ExpressionAttributeValues;

			for (var p in values) {
				if (!values.hasOwnProperty(p)) {
					continue;
				}
				ExpressionAttributeValues[p] = utils.itemize(values[p]);
			}

			params.ExpressionAttributeValues = ExpressionAttributeValues;

		}

		if (options && options.reverse) {
			params.ScanIndexForward = false;
		}

		if (options && options.limit) {
			params.Limit = options.limit;
		}

		if (options && options.filter) {

			var queryFilter = {};

			options.filter.forEach(function(condition) {

				queryFilter[condition.key] = {
					ComparisonOperator: condition.operator,
					AttributeValueList: [utils.itemize(condition.value)],
				};

			});

			params.QueryFilter = queryFilter;

		}

		if (options && options.paginate) {

			var items = [];

			var paginate = function(lastItem) {

				params.Limit = options.paginate;

				if (lastItem) {
					params.ExclusiveStartKey = lastItem;
				}

				return dynamodb.queryAsync(params).then(function(data) {

					data.Items.forEach(function(item) {
						items.push(utils.deitemize(item));
					});

					if (data.LastEvaluatedKey) {

						return paginate(data.LastEvaluatedKey);

					}

					return items;

				});

			};

			return paginate();

		}

		return dynamodb.queryAsync(params).then(function(data) {

			var items = [];
			data.Items.forEach(function(item) {
				items.push(utils.deitemize(item));
			});
			return items;

		});

	};

	var list = function(table, conditions) {

		var params = {
			TableName: table
		};

		var keys = {};
		conditions.forEach(function(condition) {

			keys[condition.key] = {
				ComparisonOperator: condition.operator,
				AttributeValueList: [utils.itemize(condition.value)],
			};

		});

		params.ScanFilter = keys;

		return dynamodb.scanAsync(params).then(function(data) {

			var items = [];
			data.Items.forEach(function(item) {
				items.push(utils.deitemize(item));
			});

			return items;

		});

	};

	var get = function(table, id) {

		var params = {
			TableName: table,
			Key: {
				"id": {
					"S": id
				}
			}
		};

		return dynamodb.getItemAsync(params).then(function(data) {

			return utils.deitemize(data.Item);

		});

	};

	var getItems = function(table, ids, options) {

		if (ids.constructor !== Array) {
			throw Error("ids is not an Array");
		}

		if (ids.length === 0) {
			return Promise.resolve([]);
		}

 		var params = {
			RequestItems: {}
		};

		var keys = ids.map(function(id) {
			return {
				"id": utils.itemize(id)
			};
		});

		var consistendRead = false;
		if (options && options.consistentRead) {
			params.consistentRead = true;
		}

		params.RequestItems[table] = {
			ConsistentRead: params.consistentRead,
			Keys: keys
		};

		return dynamodb.batchGetItemAsync(params).then(function(data) {

			var output = [];
			if (!data.Responses) {
				return output;
			}<commit_msg>Cleanup<commit_after>var AWS = require('aws-sdk'),
	Promise = require("bluebird"),
	shortid = require('shortid');

var utils = require("./utils");

module.exports = function(awsconfig, dynamodboptions) {

	AWS.config.update(awsconfig);

	var dynamodb = new Promise.promisifyAll(new AWS.DynamoDB(dynamodboptions));

	var query = function(table, conditions, index, options) {

		var params = {
			TableName: table
		};

		if (index) {
			params.IndexName = index;
		}

		// Legacy KeyConditions are in use
		if (conditions.constructor === Array) {

			var keys = {};
			conditions.forEach(function(condition) {

				keys[condition.key] = {
					ComparisonOperator: condition.operator,
					AttributeValueList: [utils.itemize(condition.value)],
				};

			});

			params.KeyConditions = keys;

		}

		// KeyConditionExpression are in use
		if (conditions.constructor === Object) {

			params.KeyConditionExpression = conditions.KeyConditionExpression;
			params.ExpressionAttributeNames = conditions.ExpressionAttributeNames;

			var ExpressionAttributeValues = {};

			var values = conditions.ExpressionAttributeValues;

			for (var p in values) {
				if (!values.hasOwnProperty(p)) {
					continue;
				}
				ExpressionAttributeValues[p] = utils.itemize(values[p]);
			}

			params.ExpressionAttributeValues = ExpressionAttributeValues;

		}

		if (options && options.reverse) {
			params.ScanIndexForward = false;
		}

		if (options && options.limit) {
			params.Limit = options.limit;
		}

		if (options && options.filter) {

			var queryFilter = {};

			options.filter.forEach(function(condition) {

				queryFilter[condition.key] = {
					ComparisonOperator: condition.operator,
					AttributeValueList: [utils.itemize(condition.value)],
				};

			});

			params.QueryFilter = queryFilter;

		}

		if (options && options.paginate) {

			var items = [];

			var paginate = function(lastItem) {

				params.Limit = options.paginate;

				if (lastItem) {
					params.ExclusiveStartKey = lastItem;
				}

				return dynamodb.queryAsync(params).then(function(data) {

					data.Items.forEach(function(item) {
						items.push(utils.deitemize(item));
					});

					if (data.LastEvaluatedKey) {

						return paginate(data.LastEvaluatedKey);

					}

					return items;

				});

			};

			return paginate();

		}

		return dynamodb.queryAsync(params).then(function(data) {

			var items = [];
			data.Items.forEach(function(item) {
				items.push(utils.deitemize(item));
			});
			return items;

		});

	};

	var list = function(table, conditions) {

		var params = {
			TableName: table
		};

		var keys = {};
		conditions.forEach(function(condition) {

			keys[condition.key] = {
				ComparisonOperator: condition.operator,
				AttributeValueList: [utils.itemize(condition.value)],
			};

		});

		params.ScanFilter = keys;

		return dynamodb.scanAsync(params).then(function(data) {

			var items = [];
			data.Items.forEach(function(item) {
				items.push(utils.deitemize(item));
			});

			return items;

		});

	};

	var get = function(table, id) {

		var params = {
			TableName: table,
			Key: {
				"id": {
					"S": id
				}
			}
		};

		return dynamodb.getItemAsync(params).then(function(data) {

			return utils.deitemize(data.Item);

		});

	};

	var getItems = function(table, ids, options) {

		if (ids.constructor !== Array) {
			throw Error("ids is not an Array");
		}

		if (ids.length === 0) {
			return Promise.resolve([]);
		}

 		var params = {
			RequestItems: {}
		};

		var keys = ids.map(function(id) {
			return {
				"id": utils.itemize(id)
			};
		});

		if (options && options.consistentRead) {
			params.consistentRead = true;
		}

		params.RequestItems[table] = {
			ConsistentRead: params.consistentRead,
			Keys: keys
		};

		return dynamodb.batchGetItemAsync(params).then(function(data) {

			var output = [];
			if (!data.Responses) {
				return output;
			}
{'fraction_non_alphanumeric': 0.13836317135549872, 'fraction_numerical': 0.0002557544757033248, 'mean_word_length': 3.1756540309663643, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '543577', 'n_tokens_mistral': 3080, 'n_tokens_neox': 2411, 'n_words': 451}