<commit_before>/*
    ***** BEGIN LICENSE BLOCK *****
    
    Copyright © 2009 Center for History and New Media
                     George Mason University, Fairfax, Virginia, USA
                     http://zotero.org
    
    This file is part of Zotero.
    
    Zotero is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
    
    Zotero is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.
    
    You should have received a copy of the GNU Affero General Public License
    along with Zotero.  If not, see <http://www.gnu.org/licenses/>.
    
    ***** END LICENSE BLOCK *****
*/

// Exclusive locking mode (default) prevents access to Zotero database while Zotero is open
// and speeds up DB access (http://www.sqlite.org/pragma.html#pragma_locking_mode).
// Normal mode is more convenient for development, but risks database corruption, particularly if
// the same database is accessed simultaneously by multiple Zotero instances.
const DB_LOCK_EXCLUSIVE = true;

Zotero.DBConnection = function(dbName) {
	if (!dbName) {
		throw ('DB name not provided in Zotero.DBConnection()');
	}
	
	this.MAX_BOUND_PARAMETERS = 999;
	
	Components.utils.import("resource://gre/modules/Sqlite.jsm", this);
	
	this.skipBackup = false;
	this.transactionVacuum = false;
	
	// JS Date
	this.__defineGetter__('transactionDate', function () {
		if (this._transactionDate) {
			this._lastTransactionDate = this._transactionDate;
			return this._transactionDate;
		}
		
		throw new Error("Transaction not in progress");
		
		// Use second granularity rather than millisecond
		// for comparison purposes
		var d = new Date(Math.floor(new Date / 1000) * 1000);
		this._lastTransactionDate = d;
		return d;
	});
	// SQL DATETIME
	this.__defineGetter__('transactionDateTime', function () {
		var d = this.transactionDate;
		return Zotero.Date.dateToSQL(d, true);
	});
	// Unix timestamp
	this.__defineGetter__('transactionTimestamp', function () {
		var d = this.transactionDate;
		return Zotero.Date.toUnixTimestamp(d);
	});
	
	// Private members
	this._dbName = dbName;
	this._shutdown = false;
	this._connectionAsync = null;
	this._transactionDate = null;
	this._lastTransactionDate = null;
	this._transactionRollback = false;
	this._transactionNestingLevel = 0;
	this._asyncTransactionNestingLevel = 0;
	this._callbacks = {
		begin: [],
		commit: [],
		rollback: [],
		current: {
			commit: [],
			rollback: []
		}
	};
	this._dbIsCorrupt = null
	this._self = this;
	
	this._transactionPromise = null;
}

/////////////////////////////////////////////////////////////////
//
// Public methods
//
/////////////////////////////////////////////////////////////////

/**
 * Test a read-only connection to the database, throwing any errors that occur
 *
 * @return	void
 */
Zotero.DBConnection.prototype.test = function () {
	return this._getConnectionAsync().return();
}

Zotero.DBConnection.prototype.getAsyncStatement = Zotero.Promise.coroutine(function* (sql) {
	var conn = yield this._getConnectionAsync();
	conn = conn._connection;
	
	try {
		this._debug(sql, 5);
		return conn.createAsyncStatement(sql);
	}
	catch (e) {
		var dberr = (conn.lastErrorString != 'not an error')
			? ' [ERROR: ' + conn.lastErrorString + ']' : '';
		throw new Error(e + ' [QUERY: ' + sql + ']' + dberr); 
	}
});


Zotero.DBConnection.prototype.parseQueryAndParams = function (sql, params, options) {
	// If single scalar value, wrap in an array
	if (!Array.isArray(params)) {
		if (typeof params == 'string' || typeof params == 'number' || params === null) {
			params = [params];
		}
		else {
			params = [];
		}
	}
	// Otherwise, since we might make changes, only work on a copy of the array
	else {
		params = params.concat();
	}
	
	// Find placeholders
	var placeholderRE = /\s*[=,(]\s*\?/g;
	
	if (params.length) {
		if (options && options.checkParams) {
			let matches = sql.match(placeholderRE);
			
			if (!matches) {
				throw new Error("Parameters provided for query without placeholders "
					+ "[QUERY: " + sql + "]");
			}
			else if (matches.length != params.length) {
				throw new Error("Incorrect number of parameters provided for query "
					+ "(" + params.length + ", expecting " + matches.length + ") "
					+ "[QUERY: " + sql + "]");
			}
		}
		
		// First, determine the type of query using first word
		var matches = sql.match(/^[^\s\(]*/);
		var queryMethod = matches[0].toLowerCase();
		
		// Reset lastIndex, since regexp isn't recompiled dynamically
		placeholderRE.lastIndex = 0;
		var lastNullParamIndex = -1;
		for (var i=0; i<params.length; i++) {
			if (typeof params[i] == 'boolean') {
				throw new Error("Invalid boolean parameter " + i + " '" + params[i] + "' "
					+ "[QUERY: " + sql + "]");
			}
			else if (params[i] === undefined) {
				throw new Error('Parameter ' + i + ' is undefined [QUERY: ' + sql + ']');
			}
			
			if (params[i] !== null) {
				// Force parameter type if specified
				
				// Int
				if (typeof params[i]['int'] != 'undefined') {
					params[i] = parseInt(params[i]['int']);
					if (isNaN(params[i])) {
						throw new Error("Invalid bound parameter " + i + " integer value '" + params[i] + "' "
							+ "[QUERY: " + sql + "]")
					}
				}
				// String
				else if (typeof params[i]['string'] != 'undefined') {
					params[i] = params[i]['string'] + "";
				}
				
				continue;
			}
			
			//
			// Replace NULL bound parameters with hard-coded NULLs
			//
			
			// Find index of this parameter, skipping previous ones
			do {
				var matches = placeholderRE.exec(sql);
				lastNullParamIndex++;
			}
			while (lastNullParamIndex < i);
			lastNullParamIndex = i;
			
			if (!matches) {
				throw new Error("Null parameter provided for a query without placeholders "
					+ "-- use false or undefined [QUERY: " + sql + "]");
			}
			
			if (matches[0].indexOf('=') == -1) {
				// mozStorage supports null bound parameters in value lists (e.g., "(?,?)") natively
				continue;
			}
			else if (queryMethod == 'select') {
				var repl = ' IS NULL';
			}
			else {
				var repl = '=NULL';
			}
			
			var subpos = matches.index;
			var sublen = matches[0].length;
			sql = sql.substring(0, subpos) + repl + sql.substr(subpos + sublen);
			
			//Zotero.debug("Hard-coding null bound parameter " + i);
			
			params.splice(i, 1);
			i--;
			lastNullParamIndex--;
			continue;
		}
		if (!params.length) {
			params = [];
		}
	}
	else if (options && options.checkParams && placeholderRE.test(sql)) {
		throw new Error("Parameters not provided for query containing placeholders "
			+ "[QUERY: " + sql + "]");
	}
	
	return [sql, params];
};


/**
 * Execute an asynchronous statement with previously bound parameters
 *
 * Warning: This will freeze if used with a write statement within executeTransaction()!
 *
 * @param {mozIStorageAsyncStatement} statement - Statement to run
 * @param {Function} [progressHandler] - Function to pass each available row to for SELECT queries
 * @return {Promise} - Resolved on completion, rejected with a reason on error
 */
Zotero.DBConnection.prototype.executeAsyncStatement = Zotero.Promise.method(function (statement, progressHandler) {
	var resolve;
	var reject;
	statement.executeAsync({
		handleResult: function (resultSet) {
			if (progressHandler) {
				progressHandler(resultSet.getNextRow());
			}
		},
		
		handleError: function (e) {
			reject(e);
		},
		
		handleCompletion: function (reason) {
			if (reason != Components.interfaces.mozIStorageStatementCallback.REASON_FINISHED) {
				reject(reason);
			}
			resolve();
		}
	});
	return new Zotero.Promise(function () {
		resolve = arguments[0];
		reject = arguments[1];
	});
});



Zotero.DBConnection.prototype.addCallback = function (type, cb) {
	switch (type) {
		case 'begin':
		case 'commit':
		case 'rollback':
			break;
			
		default:
			throw ("Invalid callback type '" + type + "' in DB.addCallback()");
	}
	
	var id = this._callbacks[type].length;
	this._callbacks[type][id] = cb;
	return id;
}


Zotero.DBConnection.prototype.addCurrentCallback = function (type, cb) {
	this.requireTransaction();
	this._callbacks.current[type].push(cb);
}


Zotero.DBConnection.prototype.removeCallback = function (type, id) {
	switch (type) {
		case 'begin':
		case 'commit':
		case 'rollback':
			break;
			
		default:
			throw ("Invalid callback type '" + type + "' in DB.removeCallback()");
	}
	
	delete this._callbacks[type][id];
}


/*
 * Used on shutdown to rollback all open transactions
 *
 * TODO: update or remove
 */
Zotero.DBConnection.prototype.rollbackAllTransactions = function () {
	if (this.transactionInProgress()) {
		var level = this._transactionNestingLevel;
		this._transactionNestingLevel = 0;
		try {
			this.rollbackTransaction();
		}
		catch (e) {}
		return level ? level : true;
	}
	return false;
}


Zotero.DBConnection.prototype.getColumns = function (table) {
	return Zotero.DB.queryAsync("PRAGMA table_info(" + table + ")")
	.then(function (rows) {
		return [row.name for each (row in rows)];
	})
	.catch(function (e) {
		this._debug(e, 1);
		return false;
	});
}


/**
* Find the next lowest numeric suffix for a value in table column
*
* For example, if "Untitled" and "Untitled 2" and "Untitled 4",
* returns "Untitled 3"
*
* If _name_ alone is available, returns that
**/
Zotero.DBConnection.prototype.getNextName = Zotero.Promise.coroutine(function* (libraryID, table, field, name)
{
	if (typeof name == 'undefined') {
		Zotero.debug("WARNING: The parameters of Zotero.DB.getNextName() have changed -- update your code", 2);
		[libraryID, table, field, name] = [null, libraryID, table, field];
	}
	
	var sql = "SELECT SUBSTR(" + field + ", " + (name.length + 1) + ") "
				+ "FROM " + table + " "
				+ "WHERE libraryID=? AND "
				+ field + " LIKE '" + name + "%' "
				+ " ORDER BY " + field;
	var params = [libraryID];
	var suffixes = yield this.columnQueryAsync(sql, params);
	suffixes.filter(function (x) x.match(/^( [0-9]+)?$/));
	
	// If none found or first one has a suffix, use default name
	if (!suffixes.length || suffixes[0]) {
		return name;
	}
	
	suffixes.sort(function (a, b) {
		return parseInt(a) - parseInt(b);
	});
	
	var i = 1;
	while (suffixes[i] === "") {
		i++;
	}
	var num = 2;
	while (suffixes[i] == num) {
		while (suffixes[i+1] && suffixes[i] == suffixes[i+1]) {
			i++;
		}
		i++;
		num++;
	}
	return name + ' ' + num;
});


//
// Async methods
//
//
// Zotero.DB.executeTransaction(function* (conn) {
//     var created = yield Zotero.DB.queryAsync("CREATE TEMPORARY TABLE tmpFoo (foo TEXT, bar INT)");
//     
//     // created == true
//     
//     var result = yield Zotero.DB.queryAsync("INSERT INTO tmpFoo VALUES ('a', ?)", 1);
//     
//     // result == 1
//     
//     yield Zotero.DB.queryAsync("INSERT INTO tmpFoo VALUES ('b', 2)");
//     yield Zotero.DB.queryAsync("INSERT INTO tmpFoo VALUES ('c', 3)");
//     yield Zotero.DB.queryAsync("INSERT INTO tmpFoo VALUES ('d', 4)");
//     
//     var value = yield Zotero.DB.valueQueryAsync("SELECT foo FROM tmpFoo WHERE bar=?", 2);
//     
//     // value == "b"
//     
//     var vals = yield Zotero.DB.columnQueryAsync("SELECT foo FROM tmpFoo");
//     
//     // '0' => "a"
//     // '1' => "b"
//     // '2' => "c"
//     // '3' => "d"
//     
//     let rows = yield Zotero.DB.queryAsync("SELECT * FROM tmpFoo");
//     for (let i=0; i<rows.length; i++) {
//         let row = rows[i];
//         // row.foo == 'a', row.bar == 1
//         // row.foo == 'b', row.bar == 2
//         // row.foo == 'c', row.bar == 3
//         // row.foo == 'd', row.bar == 4
//     }
//     
//    return rows;
// })
// .then(function (rows) {
//     // rows == same as above
// });
//
/**
 * @param {Function} func - Generator function that yields promises,
 *                          generally from queryAsync() and similar
 * @return {Promise} - Promise for result of generator function
 */
Zotero.DBConnection.prototype.executeTransaction = Zotero.Promise.coroutine(function* (func, options) {
	options = options || {};
	var resolve;
	
	// Set temporary options for this transaction that will be reset at the end
	var origOptions = {};
	if (options) {
		for (let option in options) {
			origOptions[option] = this[option];
			this[option] = options[option];
		}
	}
	
	var startedTransaction = false;
	
	try {
		while (this._inTransaction) {
			yield Zotero.DB.waitForTransaction().timeout(options.waitTimeout || 10000);
		}
		this._inTransaction = startedTransaction = true;
		
		Zotero.debug("Beginning async DB transaction", 5);
		
		this._transactionPromise = new Zotero.Promise(function () {
			resolve = arguments[0];
		});
		
		// Set a timestamp for this transaction
		this._transactionDate = new Date(Math.floor(new Date / 1000) * 1000);
		
		// Run begin callbacks
		for (var i=0; i<this._callbacks.begin.length; i++) {
			if (this._callbacks.begin[i]) {
				this._callbacks.begin[i]();
			}
		}
		var conn = this._getConnection(options) || (yield this._getConnectionAsync(options));
		var result = yield conn.executeTransaction(func);
		Zotero.debug("Committed async DB transaction", 5);
		
		// Clear transaction time
		if (this._transactionDate) {
			this._transactionDate = null;
		}
		
		if (options.vacuumOnCommit) {
			Zotero.debug('Vacuuming database');
			yield Zotero.DB.queryAsync('VACUUM');
		}
		
		this._inTransaction = false;
		
		// Function to run once transaction has been committed but before any
		// permanent callbacks
		if (options.onCommit) {
			this._callbacks.current.commit.push(options.onCommit);
		}
		this._callbacks.current.rollback = [];
		
		// Run temporary commit callbacks
		var f;
		while (f = this._callbacks.current.commit.shift()) {
			yield Zotero.Promise.resolve(f());
		}
		
		// Run commit callbacks
		for (var i=0; i<this._callbacks.commit.length; i++) {
			if (this._callbacks.commit[i]) {
				yield this._callbacks.commit[i]();
			}
		}
		
		return result;
	}
	catch (e) {
		if (e.name == "TimeoutError") {
			Zotero.debug("Timed out waiting for transaction", 1);
		}
		else {
			Zotero.debug("Rolled back async DB transaction", 5);
			Zotero.debug(e, 1);
		}
		if (startedTransaction) {
			this._inTransaction = false;
		}
		
		// Function to run once transaction has been committed but before any
		// permanent callbacks
		if (options.onRollback) {
			this._callbacks.current.rollback.push(options.onRollback);
		}
		
		// Run temporary commit callbacks
		var f;
		while (f = this._callbacks.current.rollback.shift()) {
			yield Zotero.Promise.resolve(f());
		}
		
		// Run rollback callbacks
		for (var i=0; i<this._callbacks.rollback.length; i++) {
			if (this._callbacks.rollback[i]) {
				yield Zotero.Promise.resolve(this._callbacks.rollback[i]());
			}
		}
		
		throw e;
	}
	finally {
		// Reset options back to their previous values
		if (options) {
			for (let option in options) {
				this[option] = origOptions[option];
			}
		}
		
		// Process all resolvers
		if (resolve) {
			resolve.call();
		}
	}
});


Zotero.DBConnection.prototype.inTransaction = function () {
	return this._inTransaction;
}


Zotero.DBConnection.prototype.waitForTransaction = function () {
	if (!this._inTransaction) {
		return Zotero.Promise.resolve().cancellable();
	}
	Zotero.debug("Waiting for transaction to finish");
	Zotero.debug((new Error).stack);
	return this._transactionPromise;
};


Zotero.DBConnection.prototype.requireTransaction = function () {
	if (!this._inTransaction) {
		throw new Error("Not in transaction");
	}
};


/**
 * @param {String} sql SQL statement to run
 * @param {Array|String|Integer} [params] SQL parameters to bind
 * @return {Promise|Array} A promise for an array of rows. The individual
 *                         rows are Proxy objects that return values from the
 *                         underlying mozIStorageRows based on column names.
 */
Zotero.DBConnection.prototype.queryAsync = function (sql, params, options) {
	let conn;
	let self = this;
	let onRow = null;
	return this._getConnectionAsync(options)
	.then(function (c) {
		conn = c;
		[sql, params] = self.parseQueryAndParams(sql, params);
		if (Zotero.Debug.enabled && (!options || options.debug === undefined || options.debug === true)) {
			Zotero.debug(sql, 5);
			for each(let param in params) {
				let paramType = typeof param;
				let msg = "Binding parameter of type " + paramType + ": ";
				msg += paramType == 'string' ? "'" + param + "'" : param;
				Zotero.debug(msg, 5);
			}
		}
		if (options && options.onRow) {
			// Errors in onRow don't stop the query unless StopIteration is thrown
			onRow = function (row) {
				try {
					options.onRow(row);
				}
				catch (e) {
					if (e instanceof StopIteration) {
						Zotero.debug("Query cancelled");
						throw e;
					}
					Zotero.debug(e, 1);
					Components.utils.reportError(e);
					throw StopIteration;
				}
			}
		}
		return conn.executeCached(sql, params, onRow);
	})
	.then(function (rows) {
		// Parse out the SQL command being used
		var op = sql.match(/^[^a-z]*[^ ]+/i);
		if (op) {
			op = op.toString().toLowerCase();
		}
		
		// If SELECT statement, return result
		if (op == 'select' || op == 'pragma') {
			if (onRow) {
				return;
			}
			// Fake an associative array with a proxy
			let handler = {
				get: function(target, name) {
					// Ignore promise check
					if (name == 'then') {
						return undefined;
					}
					
					try {
						return target.getResultByName(name);
					}
					catch (e) {
						Zotero.debug(e, 1);
						var msg = "DB column '" + name + "' not found";
						Zotero.debug(msg, 1);
						throw new Error(msg);
					}
				}
			};
			for (let i=0, len=rows.length; i<len; i++) {
				rows[i] = new Proxy(rows[i], handler);
			}
			return rows;
		}
		else {
			if (op == 'insert' || op == 'replace') {
				return conn.lastInsertRowID;
			}
			else if (op == 'create') {
				return true;
			}
			else {
				return conn.affectedRows;
			}
		}
	})
	.catch(function (e) {
		if (e.errors && e.errors[0]) {
			var eStr = e + "";
			eStr = eStr.indexOf("Error: ") == 0 ? eStr.substr(7): e;
			throw new Error(eStr + ' [QUERY: ' + sql + '] [ERROR: ' + e.errors[0].message + ']');
		}
		else {
			throw e;
		}
	});
};


Zotero.DBConnection.prototype.queryTx = function (sql, params, options) {
	return this.executeTransaction(function* () {
		options = options || {};
		delete options.tx;
		return this.queryAsync(sql, params, options);
	}.bind(this));
};


/**
 * @param {String} sql  SQL statement to run
 * @param {Array|String|Integer} [params]  SQL parameters to bind
 * @return {Promise<Array|Boolean>}  A promise for either the value or FALSE if no result
 */
Zotero.DBConnection.prototype.valueQueryAsync = function (sql, params) {
	let self = this;
	return this._getConnectionAsync()
	.then(function (conn) {
		[sql, params] = self.parseQueryAndParams(sql, params);
		if (Zotero.Debug.enabled) {
			Zotero.debug(sql, 5);
			for each(let param in params) {
				let paramType = typeof param;
				let msg = "Binding parameter of type " + paramType + ": ";
				msg += paramType == 'string' ? "'" + param + "'" : param;
				Zotero.debug(msg, 5);
			}
		}
		return conn.executeCached(sql, params);
	})
	.then(function (rows) {
		return rows.length ? rows[0].getResultByIndex(0) : false;
	})
	.catch(function (e) {
		if (e.errors && e.errors[0]) {
			var eStr = e + "";
			eStr = eStr.indexOf("Error: ") == 0 ? eStr.substr(7): e;
			throw new Error(eStr + ' [QUERY: ' + sql + '] [ERROR: ' + e.errors[0].message + ']');
		}
		else {
			throw e;
		}
	});
};


/**
 * @param {String} sql SQL statement to run
 * @param {Array|String|Integer} [params] SQL parameters to bind
 * @return {Promise<Object>}  A promise for a proxied storage row
 */
Zotero.DBConnection.prototype.rowQueryAsync = function (sql, params) {
	return this.queryAsync(sql, params)
	.then(function (rows) {
		return rows.length ? rows[0] : false;
	});
};


/**
 * @param {String} sql SQL statement to run
 * @param {Array|String|Integer} [params] SQL parameters to bind
 * @return {Promise<Array>}  A promise for an array of values in the column
 */
Zotero.DBConnection.prototype.columnQueryAsync = function (sql, params) {
	let conn;
	let self = this;
	return this._getConnectionAsync().
	then(function (c) {
		conn = c;
		[sql, params] = self.parseQueryAndParams(sql, params);
		if (Zotero.Debug.enabled) {
			Zotero.debug(sql, 5);
			for each(let param in params) {
				let paramType = typeof param;
				let msg = "Binding parameter of type " + paramType + ": ";
				msg += paramType == 'string' ? "'" + param + "'" : param;
				Zotero.debug(msg, 5);
			}
		}
		return conn.executeCached(sql, params);
	})
	.then(function (rows) {
		var column = [];
		for (let i=0, len=rows.length; i<len; i++) {
			column.push(rows[i].getResultByIndex(0));
		}
		return column;
	})
	.catch(function (e) {
		if (e.errors && e.errors[0]) {
			var eStr = e + "";
			eStr = eStr.indexOf("Error: ") == 0 ? eStr.substr(7): e;
			throw new Error(eStr + ' [QUERY: ' + sql + '] [ERROR: ' + e.errors[0].message + ']');
		}
		else {
			throw e;
		}
	});
};


Zotero.DBConnection.prototype.tableExists = function (table) {
	return this._getConnectionAsync()
	.then(function () {
		var sql = "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND tbl_name=?";
		return Zotero.DB.valueQueryAsync(sql, [table]);
	})
	.then(function (count) {
		return !!count;
	});
}


/**
 * Parse SQL string and execute transaction with all statements
 *
 * @return {Promise}
 */<commit_msg>Use more concise 1-line format for SQL logging<commit_after>/*
    ***** BEGIN LICENSE BLOCK *****
    
    Copyright © 2009 Center for History and New Media
                     George Mason University, Fairfax, Virginia, USA
                     http://zotero.org
    
    This file is part of Zotero.
    
    Zotero is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
    
    Zotero is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.
    
    You should have received a copy of the GNU Affero General Public License
    along with Zotero.  If not, see <http://www.gnu.org/licenses/>.
    
    ***** END LICENSE BLOCK *****
*/

// Exclusive locking mode (default) prevents access to Zotero database while Zotero is open
// and speeds up DB access (http://www.sqlite.org/pragma.html#pragma_locking_mode).
// Normal mode is more convenient for development, but risks database corruption, particularly if
// the same database is accessed simultaneously by multiple Zotero instances.
const DB_LOCK_EXCLUSIVE = true;

Zotero.DBConnection = function(dbName) {
	if (!dbName) {
		throw ('DB name not provided in Zotero.DBConnection()');
	}
	
	this.MAX_BOUND_PARAMETERS = 999;
	
	Components.utils.import("resource://gre/modules/Sqlite.jsm", this);
	
	this.skipBackup = false;
	this.transactionVacuum = false;
	
	// JS Date
	this.__defineGetter__('transactionDate', function () {
		if (this._transactionDate) {
			this._lastTransactionDate = this._transactionDate;
			return this._transactionDate;
		}
		
		throw new Error("Transaction not in progress");
		
		// Use second granularity rather than millisecond
		// for comparison purposes
		var d = new Date(Math.floor(new Date / 1000) * 1000);
		this._lastTransactionDate = d;
		return d;
	});
	// SQL DATETIME
	this.__defineGetter__('transactionDateTime', function () {
		var d = this.transactionDate;
		return Zotero.Date.dateToSQL(d, true);
	});
	// Unix timestamp
	this.__defineGetter__('transactionTimestamp', function () {
		var d = this.transactionDate;
		return Zotero.Date.toUnixTimestamp(d);
	});
	
	// Private members
	this._dbName = dbName;
	this._shutdown = false;
	this._connectionAsync = null;
	this._transactionDate = null;
	this._lastTransactionDate = null;
	this._transactionRollback = false;
	this._transactionNestingLevel = 0;
	this._asyncTransactionNestingLevel = 0;
	this._callbacks = {
		begin: [],
		commit: [],
		rollback: [],
		current: {
			commit: [],
			rollback: []
		}
	};
	this._dbIsCorrupt = null
	this._self = this;
	
	this._transactionPromise = null;
}

/////////////////////////////////////////////////////////////////
//
// Public methods
//
/////////////////////////////////////////////////////////////////

/**
 * Test a read-only connection to the database, throwing any errors that occur
 *
 * @return	void
 */
Zotero.DBConnection.prototype.test = function () {
	return this._getConnectionAsync().return();
}

Zotero.DBConnection.prototype.getAsyncStatement = Zotero.Promise.coroutine(function* (sql) {
	var conn = yield this._getConnectionAsync();
	conn = conn._connection;
	
	try {
		this._debug(sql, 5);
		return conn.createAsyncStatement(sql);
	}
	catch (e) {
		var dberr = (conn.lastErrorString != 'not an error')
			? ' [ERROR: ' + conn.lastErrorString + ']' : '';
		throw new Error(e + ' [QUERY: ' + sql + ']' + dberr); 
	}
});


Zotero.DBConnection.prototype.parseQueryAndParams = function (sql, params, options) {
	// If single scalar value, wrap in an array
	if (!Array.isArray(params)) {
		if (typeof params == 'string' || typeof params == 'number' || params === null) {
			params = [params];
		}
		else {
			params = [];
		}
	}
	// Otherwise, since we might make changes, only work on a copy of the array
	else {
		params = params.concat();
	}
	
	// Find placeholders
	var placeholderRE = /\s*[=,(]\s*\?/g;
	
	if (params.length) {
		if (options && options.checkParams) {
			let matches = sql.match(placeholderRE);
			
			if (!matches) {
				throw new Error("Parameters provided for query without placeholders "
					+ "[QUERY: " + sql + "]");
			}
			else if (matches.length != params.length) {
				throw new Error("Incorrect number of parameters provided for query "
					+ "(" + params.length + ", expecting " + matches.length + ") "
					+ "[QUERY: " + sql + "]");
			}
		}
		
		// First, determine the type of query using first word
		var matches = sql.match(/^[^\s\(]*/);
		var queryMethod = matches[0].toLowerCase();
		
		// Reset lastIndex, since regexp isn't recompiled dynamically
		placeholderRE.lastIndex = 0;
		var lastNullParamIndex = -1;
		for (var i=0; i<params.length; i++) {
			if (typeof params[i] == 'boolean') {
				throw new Error("Invalid boolean parameter " + i + " '" + params[i] + "' "
					+ "[QUERY: " + sql + "]");
			}
			else if (params[i] === undefined) {
				throw new Error('Parameter ' + i + ' is undefined [QUERY: ' + sql + ']');
			}
			
			if (params[i] !== null) {
				// Force parameter type if specified
				
				// Int
				if (typeof params[i]['int'] != 'undefined') {
					params[i] = parseInt(params[i]['int']);
					if (isNaN(params[i])) {
						throw new Error("Invalid bound parameter " + i + " integer value '" + params[i] + "' "
							+ "[QUERY: " + sql + "]")
					}
				}
				// String
				else if (typeof params[i]['string'] != 'undefined') {
					params[i] = params[i]['string'] + "";
				}
				
				continue;
			}
			
			//
			// Replace NULL bound parameters with hard-coded NULLs
			//
			
			// Find index of this parameter, skipping previous ones
			do {
				var matches = placeholderRE.exec(sql);
				lastNullParamIndex++;
			}
			while (lastNullParamIndex < i);
			lastNullParamIndex = i;
			
			if (!matches) {
				throw new Error("Null parameter provided for a query without placeholders "
					+ "-- use false or undefined [QUERY: " + sql + "]");
			}
			
			if (matches[0].indexOf('=') == -1) {
				// mozStorage supports null bound parameters in value lists (e.g., "(?,?)") natively
				continue;
			}
			else if (queryMethod == 'select') {
				var repl = ' IS NULL';
			}
			else {
				var repl = '=NULL';
			}
			
			var subpos = matches.index;
			var sublen = matches[0].length;
			sql = sql.substring(0, subpos) + repl + sql.substr(subpos + sublen);
			
			//Zotero.debug("Hard-coding null bound parameter " + i);
			
			params.splice(i, 1);
			i--;
			lastNullParamIndex--;
			continue;
		}
		if (!params.length) {
			params = [];
		}
	}
	else if (options && options.checkParams && placeholderRE.test(sql)) {
		throw new Error("Parameters not provided for query containing placeholders "
			+ "[QUERY: " + sql + "]");
	}
	
	return [sql, params];
};


/**
 * Execute an asynchronous statement with previously bound parameters
 *
 * Warning: This will freeze if used with a write statement within executeTransaction()!
 *
 * @param {mozIStorageAsyncStatement} statement - Statement to run
 * @param {Function} [progressHandler] - Function to pass each available row to for SELECT queries
 * @return {Promise} - Resolved on completion, rejected with a reason on error
 */
Zotero.DBConnection.prototype.executeAsyncStatement = Zotero.Promise.method(function (statement, progressHandler) {
	var resolve;
	var reject;
	statement.executeAsync({
		handleResult: function (resultSet) {
			if (progressHandler) {
				progressHandler(resultSet.getNextRow());
			}
		},
		
		handleError: function (e) {
			reject(e);
		},
		
		handleCompletion: function (reason) {
			if (reason != Components.interfaces.mozIStorageStatementCallback.REASON_FINISHED) {
				reject(reason);
			}
			resolve();
		}
	});
	return new Zotero.Promise(function () {
		resolve = arguments[0];
		reject = arguments[1];
	});
});



Zotero.DBConnection.prototype.addCallback = function (type, cb) {
	switch (type) {
		case 'begin':
		case 'commit':
		case 'rollback':
			break;
			
		default:
			throw ("Invalid callback type '" + type + "' in DB.addCallback()");
	}
	
	var id = this._callbacks[type].length;
	this._callbacks[type][id] = cb;
	return id;
}


Zotero.DBConnection.prototype.addCurrentCallback = function (type, cb) {
	this.requireTransaction();
	this._callbacks.current[type].push(cb);
}


Zotero.DBConnection.prototype.removeCallback = function (type, id) {
	switch (type) {
		case 'begin':
		case 'commit':
		case 'rollback':
			break;
			
		default:
			throw ("Invalid callback type '" + type + "' in DB.removeCallback()");
	}
	
	delete this._callbacks[type][id];
}


/*
 * Used on shutdown to rollback all open transactions
 *
 * TODO: update or remove
 */
Zotero.DBConnection.prototype.rollbackAllTransactions = function () {
	if (this.transactionInProgress()) {
		var level = this._transactionNestingLevel;
		this._transactionNestingLevel = 0;
		try {
			this.rollbackTransaction();
		}
		catch (e) {}
		return level ? level : true;
	}
	return false;
}


Zotero.DBConnection.prototype.getColumns = function (table) {
	return Zotero.DB.queryAsync("PRAGMA table_info(" + table + ")")
	.then(function (rows) {
		return [row.name for each (row in rows)];
	})
	.catch(function (e) {
		this._debug(e, 1);
		return false;
	});
}


/**
* Find the next lowest numeric suffix for a value in table column
*
* For example, if "Untitled" and "Untitled 2" and "Untitled 4",
* returns "Untitled 3"
*
* If _name_ alone is available, returns that
**/
Zotero.DBConnection.prototype.getNextName = Zotero.Promise.coroutine(function* (libraryID, table, field, name)
{
	if (typeof name == 'undefined') {
		Zotero.debug("WARNING: The parameters of Zotero.DB.getNextName() have changed -- update your code", 2);
		[libraryID, table, field, name] = [null, libraryID, table, field];
	}
	
	var sql = "SELECT SUBSTR(" + field + ", " + (name.length + 1) + ") "
				+ "FROM " + table + " "
				+ "WHERE libraryID=? AND "
				+ field + " LIKE '" + name + "%' "
				+ " ORDER BY " + field;
	var params = [libraryID];
	var suffixes = yield this.columnQueryAsync(sql, params);
	suffixes.filter(function (x) x.match(/^( [0-9]+)?$/));
	
	// If none found or first one has a suffix, use default name
	if (!suffixes.length || suffixes[0]) {
		return name;
	}
	
	suffixes.sort(function (a, b) {
		return parseInt(a) - parseInt(b);
	});
	
	var i = 1;
	while (suffixes[i] === "") {
		i++;
	}
	var num = 2;
	while (suffixes[i] == num) {
		while (suffixes[i+1] && suffixes[i] == suffixes[i+1]) {
			i++;
		}
		i++;
		num++;
	}
	return name + ' ' + num;
});


//
// Async methods
//
//
// Zotero.DB.executeTransaction(function* (conn) {
//     var created = yield Zotero.DB.queryAsync("CREATE TEMPORARY TABLE tmpFoo (foo TEXT, bar INT)");
//     
//     // created == true
//     
//     var result = yield Zotero.DB.queryAsync("INSERT INTO tmpFoo VALUES ('a', ?)", 1);
//     
//     // result == 1
//     
//     yield Zotero.DB.queryAsync("INSERT INTO tmpFoo VALUES ('b', 2)");
//     yield Zotero.DB.queryAsync("INSERT INTO tmpFoo VALUES ('c', 3)");
//     yield Zotero.DB.queryAsync("INSERT INTO tmpFoo VALUES ('d', 4)");
//     
//     var value = yield Zotero.DB.valueQueryAsync("SELECT foo FROM tmpFoo WHERE bar=?", 2);
//     
//     // value == "b"
//     
//     var vals = yield Zotero.DB.columnQueryAsync("SELECT foo FROM tmpFoo");
//     
//     // '0' => "a"
//     // '1' => "b"
//     // '2' => "c"
//     // '3' => "d"
//     
//     let rows = yield Zotero.DB.queryAsync("SELECT * FROM tmpFoo");
//     for (let i=0; i<rows.length; i++) {
//         let row = rows[i];
//         // row.foo == 'a', row.bar == 1
//         // row.foo == 'b', row.bar == 2
//         // row.foo == 'c', row.bar == 3
//         // row.foo == 'd', row.bar == 4
//     }
//     
//    return rows;
// })
// .then(function (rows) {
//     // rows == same as above
// });
//
/**
 * @param {Function} func - Generator function that yields promises,
 *                          generally from queryAsync() and similar
 * @return {Promise} - Promise for result of generator function
 */
Zotero.DBConnection.prototype.executeTransaction = Zotero.Promise.coroutine(function* (func, options) {
	options = options || {};
	var resolve;
	
	// Set temporary options for this transaction that will be reset at the end
	var origOptions = {};
	if (options) {
		for (let option in options) {
			origOptions[option] = this[option];
			this[option] = options[option];
		}
	}
	
	var startedTransaction = false;
	
	try {
		while (this._inTransaction) {
			yield Zotero.DB.waitForTransaction().timeout(options.waitTimeout || 10000);
		}
		this._inTransaction = startedTransaction = true;
		
		Zotero.debug("Beginning async DB transaction", 5);
		
		this._transactionPromise = new Zotero.Promise(function () {
			resolve = arguments[0];
		});
		
		// Set a timestamp for this transaction
		this._transactionDate = new Date(Math.floor(new Date / 1000) * 1000);
		
		// Run begin callbacks
		for (var i=0; i<this._callbacks.begin.length; i++) {
			if (this._callbacks.begin[i]) {
				this._callbacks.begin[i]();
			}
		}
		var conn = this._getConnection(options) || (yield this._getConnectionAsync(options));
		var result = yield conn.executeTransaction(func);
		Zotero.debug("Committed async DB transaction", 5);
		
		// Clear transaction time
		if (this._transactionDate) {
			this._transactionDate = null;
		}
		
		if (options.vacuumOnCommit) {
			Zotero.debug('Vacuuming database');
			yield Zotero.DB.queryAsync('VACUUM');
		}
		
		this._inTransaction = false;
		
		// Function to run once transaction has been committed but before any
		// permanent callbacks
		if (options.onCommit) {
			this._callbacks.current.commit.push(options.onCommit);
		}
		this._callbacks.current.rollback = [];
		
		// Run temporary commit callbacks
		var f;
		while (f = this._callbacks.current.commit.shift()) {
			yield Zotero.Promise.resolve(f());
		}
		
		// Run commit callbacks
		for (var i=0; i<this._callbacks.commit.length; i++) {
			if (this._callbacks.commit[i]) {
				yield this._callbacks.commit[i]();
			}
		}
		
		return result;
	}
	catch (e) {
		if (e.name == "TimeoutError") {
			Zotero.debug("Timed out waiting for transaction", 1);
		}
		else {
			Zotero.debug("Rolled back async DB transaction", 5);
			Zotero.debug(e, 1);
		}
		if (startedTransaction) {
			this._inTransaction = false;
		}
		
		// Function to run once transaction has been committed but before any
		// permanent callbacks
		if (options.onRollback) {
			this._callbacks.current.rollback.push(options.onRollback);
		}
		
		// Run temporary commit callbacks
		var f;
		while (f = this._callbacks.current.rollback.shift()) {
			yield Zotero.Promise.resolve(f());
		}
		
		// Run rollback callbacks
		for (var i=0; i<this._callbacks.rollback.length; i++) {
			if (this._callbacks.rollback[i]) {
				yield Zotero.Promise.resolve(this._callbacks.rollback[i]());
			}
		}
		
		throw e;
	}
	finally {
		// Reset options back to their previous values
		if (options) {
			for (let option in options) {
				this[option] = origOptions[option];
			}
		}
		
		// Process all resolvers
		if (resolve) {
			resolve.call();
		}
	}
});


Zotero.DBConnection.prototype.inTransaction = function () {
	return this._inTransaction;
}


Zotero.DBConnection.prototype.waitForTransaction = function () {
	if (!this._inTransaction) {
		return Zotero.Promise.resolve().cancellable();
	}
	Zotero.debug("Waiting for transaction to finish");
	Zotero.debug((new Error).stack);
	return this._transactionPromise;
};


Zotero.DBConnection.prototype.requireTransaction = function () {
	if (!this._inTransaction) {
		throw new Error("Not in transaction");
	}
};


/**
 * @param {String} sql SQL statement to run
 * @param {Array|String|Integer} [params] SQL parameters to bind
 * @return {Promise|Array} A promise for an array of rows. The individual
 *                         rows are Proxy objects that return values from the
 *                         underlying mozIStorageRows based on column names.
 */
Zotero.DBConnection.prototype.queryAsync = function (sql, params, options) {
	let conn;
	let self = this;
	let onRow = null;
	return this._getConnectionAsync(options)
	.then(function (c) {
		conn = c;
		[sql, params] = self.parseQueryAndParams(sql, params);
		if (Zotero.Debug.enabled) {
			self.logQuery(sql, params, options);
		}
		if (options && options.onRow) {
			// Errors in onRow don't stop the query unless StopIteration is thrown
			onRow = function (row) {
				try {
					options.onRow(row);
				}
				catch (e) {
					if (e instanceof StopIteration) {
						Zotero.debug("Query cancelled");
						throw e;
					}
					Zotero.debug(e, 1);
					Components.utils.reportError(e);
					throw StopIteration;
				}
			}
		}
		return conn.executeCached(sql, params, onRow);
	})
	.then(function (rows) {
		// Parse out the SQL command being used
		var op = sql.match(/^[^a-z]*[^ ]+/i);
		if (op) {
			op = op.toString().toLowerCase();
		}
		
		// If SELECT statement, return result
		if (op == 'select' || op == 'pragma') {
			if (onRow) {
				return;
			}
			// Fake an associative array with a proxy
			let handler = {
				get: function(target, name) {
					// Ignore promise check
					if (name == 'then') {
						return undefined;
					}
					
					try {
						return target.getResultByName(name);
					}
					catch (e) {
						Zotero.debug(e, 1);
						var msg = "DB column '" + name + "' not found";
						Zotero.debug(msg, 1);
						throw new Error(msg);
					}
				}
			};
			for (let i=0, len=rows.length; i<len; i++) {
				rows[i] = new Proxy(rows[i], handler);
			}
			return rows;
		}
		else {
			if (op == 'insert' || op == 'replace') {
				return conn.lastInsertRowID;
			}
			else if (op == 'create') {
				return true;
			}
			else {
				return conn.affectedRows;
			}
		}
	})
	.catch(function (e) {
		if (e.errors && e.errors[0]) {
			var eStr = e + "";
			eStr = eStr.indexOf("Error: ") == 0 ? eStr.substr(7): e;
			throw new Error(eStr + ' [QUERY: ' + sql + '] [ERROR: ' + e.errors[0].message + ']');
		}
		else {
			throw e;
		}
	});
};


Zotero.DBConnection.prototype.queryTx = function (sql, params, options) {
	return this.executeTransaction(function* () {
		options = options || {};
		delete options.tx;
		return this.queryAsync(sql, params, options);
	}.bind(this));
};


/**
 * @param {String} sql  SQL statement to run
 * @param {Array|String|Integer} [params]  SQL parameters to bind
 * @return {Promise<Array|Boolean>}  A promise for either the value or FALSE if no result
 */
Zotero.DBConnection.prototype.valueQueryAsync = function (sql, params) {
	let self = this;
	return this._getConnectionAsync()
	.then(function (conn) {
		[sql, params] = self.parseQueryAndParams(sql, params);
		if (Zotero.Debug.enabled) {
			self.logQuery(sql, params);
		}
		return conn.executeCached(sql, params);
	})
	.then(function (rows) {
		return rows.length ? rows[0].getResultByIndex(0) : false;
	})
	.catch(function (e) {
		if (e.errors && e.errors[0]) {
			var eStr = e + "";
			eStr = eStr.indexOf("Error: ") == 0 ? eStr.substr(7): e;
			throw new Error(eStr + ' [QUERY: ' + sql + '] [ERROR: ' + e.errors[0].message + ']');
		}
		else {
			throw e;
		}
	});
};


/**
 * @param {String} sql SQL statement to run
 * @param {Array|String|Integer} [params] SQL parameters to bind
 * @return {Promise<Object>}  A promise for a proxied storage row
 */
Zotero.DBConnection.prototype.rowQueryAsync = function (sql, params) {
	return this.queryAsync(sql, params)
	.then(function (rows) {
		return rows.length ? rows[0] : false;
	});
};


/**
 * @param {String} sql SQL statement to run
 * @param {Array|String|Integer} [params] SQL parameters to bind
 * @return {Promise<Array>}  A promise for an array of values in the column
 */
Zotero.DBConnection.prototype.columnQueryAsync = function (sql, params) {
	let conn;
	let self = this;
	return this._getConnectionAsync().
	then(function (c) {
		conn = c;
		[sql, params] = self.parseQueryAndParams(sql, params);
		if (Zotero.Debug.enabled) {
			self.logQuery(sql, params);
		}
		return conn.executeCached(sql, params);
	})
	.then(function (rows) {
		var column = [];
		for (let i=0, len=rows.length; i<len; i++) {
			column.push(rows[i].getResultByIndex(0));
		}
		return column;
	})
	.catch(function (e) {
		if (e.errors && e.errors[0]) {
			var eStr = e + "";
			eStr = eStr.indexOf("Error: ") == 0 ? eStr.substr(7): e;
			throw new Error(eStr + ' [QUERY: ' + sql + '] [ERROR: ' + e.errors[0].message + ']');
		}
		else {
			throw e;
		}
	});
};


Zotero.DBConnection.prototype.logQuery = function (sql, params, options) {
	var msg = sql;
	if (params.length && (!options || options.debug === undefined || options.debug === true)) {
		msg += " [";
		for (let i = 0; i < params.length; i++) {
			let param = params[i];
			let paramType = typeof param;
			if (paramType == 'string') {
				msg += "'" + param + "', ";
			}
			else {
				msg += param + ", ";
			}
		}
		msg = msg.substr(0, msg.length - 2) + "]";
	}
	Zotero.debug(msg, 5);
}


Zotero.DBConnection.prototype.tableExists = function (table) {
	return this._getConnectionAsync()
	.then(function () {
		var sql = "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND tbl_name=?";
		return Zotero.DB.valueQueryAsync(sql, [table]);
	})
	.then(function (count) {
		return !!count;
	});
}


/**
 * Parse SQL string and execute transaction with all statements
 *
 * @return {Promise}
 */
{'fraction_non_alphanumeric': 0.15494287820158467, 'fraction_numerical': 0.0049520913948774645, 'mean_word_length': 2.966834170854271, 'pattern_counts': {'":': 3, '<': 28, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3201818', 'n_tokens_mistral': 16009, 'n_tokens_neox': 13974, 'n_words': 3790}