<commit_before>/*
    ***** BEGIN LICENSE BLOCK *****
    
    Copyright © 2009 Center for History and New Media
                     George Mason University, Fairfax, Virginia, USA
                     http://zotero.org
    
    This file is part of Zotero.
    
    Zotero is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
    
    Zotero is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
    
    You should have received a copy of the GNU General Public License
    along with Zotero.  If not, see <http://www.gnu.org/licenses/>.
    
    ***** END LICENSE BLOCK *****
*/

// Enumeration of types of translators
const TRANSLATOR_TYPES = {"import":1, "export":2, "web":4, "search":8};

// Byte order marks for various character sets
const BOMs = {
	"UTF-8":"\xEF\xBB\xBF",
	"UTF-16BE":"\xFE\xFF",
	"UTF-16LE":"\xFF\xFE",
	"UTF-32BE":"\x00\x00\xFE\xFF",
	"UTF-32LE":"\xFF\xFE\x00\x00"
}

/**
 * Singleton to handle loading and caching of translators
 * @namespace
 */
Zotero.Translators = new function() {
	var _cache, _translators;
	var _initialized = false;
	
	/**
	 * Initializes translator cache, loading all relevant translators into memory
	 */
	this.init = function() {
		_initialized = true;
		
		var start = (new Date()).getTime();
		var transactionStarted = false;
		
		_cache = {"import":[], "export":[], "web":[], "search":[]};
		_translators = {};
		
		var dbCacheResults = Zotero.DB.query("SELECT leafName, translatorJSON, "+
			"code, lastModifiedTime FROM translatorCache");
		var dbCache = {};
		for each(var cacheEntry in dbCacheResults) {
			dbCache[cacheEntry.leafName] = cacheEntry;
		}
		
		var i = 0;
		var filesInCache = {};
		var contents = Zotero.getTranslatorsDirectory().directoryEntries;
		while(contents.hasMoreElements()) {
			var file = contents.getNext().QueryInterface(Components.interfaces.nsIFile);
			var leafName = file.leafName;
			if(!leafName || leafName[0] == ".") continue;
			var lastModifiedTime = file.lastModifiedTime;
			
			var dbCacheEntry = false;
			if(dbCache[leafName]) {
				filesInCache[leafName] = true;
				if(dbCache[leafName].lastModifiedTime == lastModifiedTime) {
					dbCacheEntry = dbCache[file.leafName];
				}
			}
			
			if(dbCacheEntry) {
				// get JSON from cache if possible
				var translator = new Zotero.Translator(file, dbCacheEntry.translatorJSON, dbCacheEntry.code);
				filesInCache[leafName] = true;
			} else {
				// otherwise, load from file
				var translator = new Zotero.Translator(file);
			}
			
			if(translator.translatorID) {
				if(_translators[translator.translatorID]) {
					// same translator is already cached
					translator.logError('Translator with ID '+
						translator.translatorID+' already loaded from "'+
						_translators[translator.translatorID].file.leafName+'"');
				} else {
					// add to cache
					_translators[translator.translatorID] = translator;
					for(var type in TRANSLATOR_TYPES) {
						if(translator.translatorType & TRANSLATOR_TYPES[type]) {
							_cache[type].push(translator);
						}
					}
					
					if(!dbCacheEntry) {
						// Add cache misses to DB
						if(!transactionStarted) {
							transactionStarted = true;
							Zotero.DB.beginTransaction();
						}
						Zotero.Translators.cacheInDB(leafName, translator.metadataString, translator.cacheCode ? translator.code : null, lastModifiedTime);
						delete translator.metadataString;
					}
				}
			}
			
			i++;
		}
		
		// Remove translators from DB as necessary
		for(var leafName in dbCache) {
			if(!filesInCache[leafName]) {
				Zotero.DB.query("DELETE FROM translatorCache WHERE leafName = ?", [leafName]);
			}
		}
		
		// Close transaction
		if(transactionStarted) {
			Zotero.DB.commitTransaction();
		}
		
		// Sort by priority
		var collation = Zotero.getLocaleCollation();
		var cmp = function (a, b) {
			if (a.priority > b.priority) {
				return 1;
			}
			else if (a.priority < b.priority) {
				return -1;
			}
			return collation.compareString(1, a.label, b.label);
		}
		for(var type in _cache) {
			_cache[type].sort(cmp);
		}
		
		Zotero.debug("Cached "+i+" translators in "+((new Date()).getTime() - start)+" ms");
	}
	
	/**
	 * Gets the translator that corresponds to a given ID
	 */
	this.get = function(id) {
		if(!_initialized) this.init();
		return _translators[id] ? _translators[id] : false;
	}
	
	/**
	 * Gets all translators for a specific type of translation
	 */
	this.getAllForType = function(type) {
		if(!_initialized) this.init();
		return _cache[type].slice(0);
	}
	
	/**
	 * @param	{String}		label
	 * @return	{String}
	 */
	this.getFileNameFromLabel = function(label) {
		return Zotero.File.getValidFileName(label) + ".js";
	}
	
	
	/**
	 * @param	{String}		metadata
	 * @param	{String}		metadata.translatorID		Translator GUID
	 * @param	{Integer}		metadata.translatorType		See TRANSLATOR_TYPES in translate.js
	 * @param	{String}		metadata.label				Translator title
	 * @param	{String}		metadata.creator			Translator author
	 * @param	{String|Null}	metadata.target				Target regexp
	 * @param	{String|Null}	metadata.minVersion
	 * @param	{String}		metadata.maxVersion
	 * @param	{Integer}		metadata.priority
	 * @param	{Boolean}		metadata.inRepository
	 * @param	{String}		metadata.lastUpdated		SQL date
	 * @param	{String}		code
	 * @return	{nsIFile}
	 */
	this.save = function(metadata, code) {
		if (!metadata.translatorID) {
			throw ("metadata.translatorID not provided in Zotero.Translators.save()");
		}
		
		if (!metadata.translatorType) {
			var found = false;
			for each(var type in TRANSLATOR_TYPES) {
				if (metadata.translatorType & type) {
					found = true;
					break;
				}
			}
			if (!found) {
				throw ("Invalid translatorType '" + metadata.translatorType + "' in Zotero.Translators.save()");
			}
		}
		
		if (!metadata.label) {
			throw ("metadata.label not provided in Zotero.Translators.save()");
		}
		
		if (!metadata.priority) {
			throw ("metadata.priority not provided in Zotero.Translators.save()");
		}
		
		if (!metadata.lastUpdated) {
			throw ("metadata.lastUpdated not provided in Zotero.Translators.save()");
		}
		
		if (!code) {
			throw ("code not provided in Zotero.Translators.save()");
		}
		
		var fileName = Zotero.Translators.getFileNameFromLabel(metadata.label);
		var destFile = Zotero.getTranslatorsDirectory();
		destFile.append(fileName);
		
		var metadataJSON;
		
		// JSON.stringify (FF 3.5.4 and up) has the benefit of indenting JSON
		if (typeof JSON != "undefined" && 'function' == typeof JSON.stringify) {
			metadataJSON = JSON.stringify(metadata,null,8);
		} else {
			var nsIJSON = Cc["@mozilla.org/dom/json;1"].createInstance(Ci.nsIJSON);
			metadataJSON = nsIJSON.encode(metadata);
		}
		
		var str = metadataJSON + "\n\n" + code;
		
		var translator = Zotero.Translators.get(metadata.translatorID);
		if (translator && destFile.equals(translator.file)) {
			var sameFile = true;
		}
		
		if (!sameFile && destFile.exists()) {
			var msg = "Overwriting translator with same filename '"
				+ fileName + "'";
			Zotero.debug(msg, 1);
			Zotero.debug(metadata, 1);
			Components.utils.reportError(msg + " in Zotero.Translators.save()");
		}
		
		if (translator && translator.file.exists()) {
			translator.file.remove(false);
		}
		
		Zotero.debug("Saving translator '" + metadata.label + "'");
		Zotero.debug(str);
		Zotero.File.putContents(destFile, str);
		
		return destFile;
	}
	
	this.cacheInDB = function(fileName, metadataJSON, code, lastModifiedTime) {
		Zotero.DB.query("REPLACE INTO translatorCache VALUES (?, ?, ?, ?)",
			[fileName, metadataJSON, code, lastModifiedTime]);
	}
}

/**
 * @class Represents an individual translator
 * @constructor
 * @param {nsIFile} file File from which to generate a translator object
 * @property {String} translatorID Unique GUID of the translator
 * @property {Integer} translatorType Type of the translator (use bitwise & with TRANSLATOR_TYPES to read)
 * @property {String} label Human-readable name of the translator
 * @property {String} creator Author(s) of the translator
 * @property {String} target Location that the translator processes
 * @property {String} minVersion Minimum Zotero version
 * @property {String} maxVersion Minimum Zotero version
 * @property {Integer} priority Lower-priority translators will be selected first
 * @property {Boolean} inRepository Whether the translator may be found in the repository
 * @property {String} lastUpdated SQL-style date and time of translator's last update
 * @property {String} code The executable JavaScript for the translator
 */
Zotero.Translator = function(file, json, code) {
	const codeGetterFunction = function() { return Zotero.File.getContents(this.file); }
	// Maximum length for the info JSON in a translator
	const MAX_INFO_LENGTH = 4096;
	const infoRe = /{(?:(?:"(?:[^"\r\n]*(?:\\")?)*")*[^}"]*)*}/;
	
	this.file = file;
	
	if(json) {
		var info = Zotero.JSON.unserialize(json);
	} else {
		var fStream = Components.classes["@mozilla.org/network/file-input-stream;1"].
			createInstance(Components.interfaces.nsIFileInputStream);
		var cStream = Components.classes["@mozilla.org/intl/converter-input-stream;1"].
			createInstance(Components.interfaces.nsIConverterInputStream);
		fStream.init(file, -1, -1, 0);
		cStream.init(fStream, "UTF-8", 8192,
			Components.interfaces.nsIConverterInputStream.DEFAULT_REPLACEMENT_CHARACTER);
		
		var str = {};
		cStream.readString(MAX_INFO_LENGTH, str);
		
		// We assume lastUpdated is at the end to avoid running the regexp on more than necessary
		var lastUpdatedIndex = str.value.indexOf('"lastUpdated"');
		if (lastUpdatedIndex == -1) {
			this.logError("Invalid or missing translator metadata JSON object in " + file.leafName);
			fStream.close();
			return;
		}
		
		// Add 50 characters to clear lastUpdated timestamp and final "}"
		var header = str.value.substr(0, lastUpdatedIndex + 50);
		var m = infoRe.exec(header);
		if (!m) {
			this.logError("Invalid or missing translator metadata JSON object in " + file.leafName);
			fStream.close();
			return;
		}
		
		this.metadataString = m[0];
		
		try {
			var info = Zotero.JSON.unserialize(this.metadataString);
		} catch(e) {
			this.logError("Invalid or missing translator metadata JSON object in " + file.leafName);
			fStream.close();
			return;
		}
	}
	
	var haveMetadata = true;
	// make sure we have all the properties
	for each(var property in ["translatorID", "translatorType", "label", "creator", "target", "minVersion", "maxVersion", "priority", "lastUpdated", "inRepository"]) {
		if(info[property] === undefined) {
			this.logError('Missing property "'+property+'" in translator metadata JSON object in ' + file.leafName);
			haveMetadata = false;
			break;
		} else {
			this[property] = info[property];
		}
	}
	if(!haveMetadata) {
		fStream.close();
		return;
	}
	
	this.detectXPath = info["detectXPath"] ? info["detectXPath"] : null;
	
	/**
	 * g = Gecko (Firefox)
	 * c = Google Chrome (WebKit & V8)
	 * s = Safari (WebKit & Nitro/Squirrelfish Extreme)
	 * i = Internet Explorer
	 * a = All
	 */
	this.browserSupport = info["browserSupport"] ? info["browserSupport"] : "g";
	
	if(this.translatorType & TRANSLATOR_TYPES["import"]) {
		// compile import regexp to match only file extension
		this.importRegexp = this.target ? new RegExp("\\."+this.target+"$", "i") : null;
	}
		
	this.cacheCode = false;
	if(this.translatorType & TRANSLATOR_TYPES["web"]) {
		// compile web regexp
		this.webRegexp = this.target ? new RegExp(this.target, "i") : null;
		
		if(!this.target) {
			this.cacheCode = true;
			
			if(json) {
				// if have JSON, also have code
				this.code = code;
			} else {
				// for translators used on every page, cache code in memory
				var strs = [str.value];
				var amountRead;
				while(amountRead = cStream.readString(8192, str)) strs.push(str.value);
				this.code = strs.join("");
			}
		}
	}
	
	if(!this.cacheCode) this.__defineGetter__("code", codeGetterFunction);
	if(!json) cStream.close();
}

/**
 * Log a translator-related error
 * @param {String} message The error message
 * @param {String} [type] The error type ("error", "warning", "exception", or "strict")
 * @param {String} [line] The text of the line on which the error occurred
 * @param {Integer} lineNumber
 * @param {Integer} colNumber
 */
Zotero.Translator.prototype.logError = function(message, type, line, lineNumber, colNumber) {
	var ios = Components.classes["@mozilla.org/network/io-service;1"].
		getService(Components.interfaces.nsIIOService);
	Zotero.log(message, type ? type : "error", ios.newFileURI(this.file).spec);
}


/*
 * Zotero.Translate: a class for translation of Zotero metadata from and to
 * other formats
 *
 * type can be: 
 * export
 * import
 * web
 * search
 *
 * a typical export process:
 * var translatorObj = new Zotero.Translate();
 * var possibleTranslators = translatorObj.getTranslators();
 * // do something involving nsIFilePicker; remember, each possibleTranslator
 * // object has properties translatorID, label, and targetID
 * translatorObj.setLocation(myNsILocalFile);
 * translatorObj.setTranslator(possibleTranslators[x]); // also accepts only an ID
 * translatorObj.setHandler("done", _translationDone);
 * translatorObj.translate();
 *
 *
 * PUBLIC PROPERTIES:
 *
 * type - the text type of translator (set by constructor, should be read only)
 * document - the document object to be used for web scraping (read-only; set
 *           with setDocument)
 * translator - the translator currently in use (read-only; set with
 *               setTranslator)
 * location - the location of the target (read-only; set with setLocation)
 *            for import/export - this is an instance of nsILocalFile
 *            for web - this is a URL
 * search - item (in toArray() format) to extrapolate data for (read-only; set
 *          with setSearch).
 * items - items (in Zotero.Item format) to be exported. if this is empty,
 *         Zotero will export all items in the library (read-only; set with
 *         setItems). setting items disables export of collections.
 * path - the path to the target; for web, this is the same as location
 * string - the string content to be used as a file.
 * libraryID - libraryID (e.g., of a group) of saved database items. null for local items,
 *           or false not to save. defaults to null; set using second argument of constructor.
 * newItems - items created when translate() was called
 * newCollections - collections created when translate() was called
 *
 * PSEUDO-PRIVATE PROPERTIES (used only by other objects in this file):
 *
 * waitForCompletion - whether to wait for asynchronous completion, or return
 *                     immediately when script has finished executing
 * configOptions - options set by translator modifying behavior of
 *                  Zotero.Translate
 * displayOptions - options available to user for this specific translator
 *
 * PRIVATE PROPERTIES:
 * 
 * _charset - character set
 * _numericTypes - possible numeric types as a comma-delimited string
 * _handlers - handlers for various events (see setHandler)
 * _sandbox - sandbox in which translators will be executed
 * _streams - streams that need to be closed when execution is complete
 * _IDMap - a map from IDs as specified in Zotero.Item() to IDs of actual items
 * _parentTranslator - set when a translator is called from another translator. 
 *                     among other things, disables passing of the translate
 *                     object to handlers and modifies complete() function on 
 *                     returned items
 * _storage - the stored string to be treated as input
 * _storageLength - the length of the stored string
 * _exportFileDirectory - the directory to which files will be exported
 *
 * WEB-ONLY PROPERTIES:
 *
 * locationIsProxied - whether the URL being scraped is going through
 *                      an EZProxy
 * _downloadAssociatedFiles - whether to download content, according to
 *                            preferences
 *
 * EXPORT-ONLY PROPERTIES:
 *
 * output - export output (if no location has been specified)
 */
Zotero.Translate = function(type) {
	if (arguments.length > 1) {
		throw ("Zotero.Translate only takes one parameter");
	}
	
	this.type = type;
	
	// import = 0001 = 1
	// export = 0010 = 2
	// web    = 0100 = 4
	// search = 1000 = 8
	
	// combination types determined by addition or bitwise AND
	// i.e., import+export = 1+2 = 3
	this._numericTypes = "";
	for(var i=0; i<=1; i++) {
		for(var j=0; j<=1; j++) {
			for(var k=0; k<=1; k++) {
				if(type == "import") {
					this._numericTypes += ","+parseInt(i.toString()+j.toString()+k.toString()+"1", 2);
				} else if(type == "export") {
					this._numericTypes += ","+parseInt(i.toString()+j.toString()+"1"+k.toString(), 2);
				} else if(type == "web") {
					this._numericTypes += ","+parseInt(i.toString()+"1"+j.toString()+k.toString(), 2);
				} else if(type == "search") {
					this._numericTypes += ","+parseInt("1"+i.toString()+j.toString()+k.toString(), 2);
				} else {
					throw("invalid import type");
				}
			}
		}
	}
	this._numericTypes = this._numericTypes.substr(1);
	
	this._handlers = new Array();
	this._streams = new Array();
}

/*
 * sets the browser to be used for web translation; also sets the location
 */
Zotero.Translate.prototype.setDocument = function(doc) {
	this.document = doc;
	this.setLocation(doc.location.href);
}

/*
 * sets the item to be used for searching
 */
Zotero.Translate.prototype.setSearch = function(search) {
	this.search = search;
}

/*
 * sets the items to be used for export
 */
Zotero.Translate.prototype.setItems = function(items) {
	this.items = items;
}

/*
 * sets the collection to be used for export (overrides setItems)
 */
Zotero.Translate.prototype.setCollection = function(collection) {
	this.collection = collection;
}

/*
 * sets the location to operate upon (file should be an nsILocalFile object or
 * web address)
 */
Zotero.Translate.prototype.setLocation = function(location) {
	if(this.type == "web") {
		// account for proxies
		this.location = Zotero.Proxies.proxyToProper(location);
		if(this.location != location) {
			// figure out if this URL is being proxies
			this.locationIsProxied = true;
		}
		this.path = this.location;
	} else {
		this.location = location;
		if(this.location instanceof Components.interfaces.nsIFile) {	// if a file
			this.path = location.path;
		} else {														// if a url
			this.path = location;
		}
	}
}

/*
 * sets the string to be used as a file
 */
Zotero.Translate.prototype.setString = function(string) {
	this._storage = string;
	this._storageLength = string.length;
	this._storagePointer = 0;
}

/*
 * sets translator display options. you can also pass a translator (not ID) to
 * setTranslator that includes a displayOptions argument
 */
Zotero.Translate.prototype.setDisplayOptions = function(displayOptions) {
	this._setDisplayOptions = displayOptions;
}

/*
 * sets the translator to be used for import/export
 *
 * accepts either the object from getTranslators() or an ID
 */
Zotero.Translate.prototype.setTranslator = function(translator) {
	if(!translator) {
		throw("cannot set translator: invalid value");
	}
	
	this.translator = null;
	this._setDisplayOptions = null;
	
	if(typeof(translator) == "object") {	// passed an object and not an ID
		if(translator.translatorID) {
			if(translator.displayOptions) {
				this._setDisplayOptions = translator.displayOptions;
			}
			
			this.translator = [translator];
		} else {
			// we have an array of translators
			if(this.type != "search") {
				throw("cannot set translator: a single translator must be specified when doing "+this.type+" translation");
			}
			
			// accept a list of objects
			this.translator = [];
			for(var i in translator) {
				if(typeof(translator[i]) == "object") {
					this.translator.push(translator[i]);
				} else {
					this.translator.push(Zotero.Translators.get(translator[i]));
				}
			}
		}
	} else {
		this.translator = [Zotero.Translators.get(translator)];
	}
	
	return !!this.translator;
}

/*
 * registers a handler function to be called when translation is complete
 * 
 * as the first argument, all handlers will be passed the current function. the
 * second argument is dependent on the handler.
 *
 * select
 *   valid: web
 *   called: when the user needs to select from a list of available items
 *   passed: an associative array in the form id => text
 *   returns: a numerically indexed array of ids, as extracted from the passed
 *            string
 *
 * itemDone
 *   valid: import, web, search
 *   called: when an item has been processed; may be called asynchronously
 *   passed: an item object (see Zotero.Item)
 *   returns: N/A
 *
 * collectionDone
 *   valid: import
 *   called: when a collection has been processed, after all items have been
 *           added; may be called asynchronously
 *   passed: a collection object (see Zotero.Collection)
 *   returns: N/A
 * 
 * done
 *   valid: all
 *   called: when all processing is finished
 *   passed: true if successful, false if an error occurred
 *   returns: N/A
 *
 * debug
 *   valid: all
 *   called: when Zotero.debug() is called
 *   passed: string debug message
 *   returns: true if message should be logged to the console, false if not
 *
 * error
 *   valid: all
 *   called: when a fatal error occurs
 *   passed: error object (or string)
 *   returns: N/A
 *
 * translators
 *   valid: all
 *   called: when a translator search initiated with Zotero.getTranslators() is
 *           complete
 *   passed: an array of appropriate translators
 *   returns: N/A
 */
Zotero.Translate.prototype.setHandler = function(type, handler) {
	if(!this._handlers[type]) {
		this._handlers[type] = new Array();
	}
	this._handlers[type].push(handler);
}

/*
 * clears all handlers for a given function
 */
Zotero.Translate.prototype.clearHandlers = function(type) {
	this._handlers[type] = new Array();
}

/*
 * calls a handler (see setHandler above)
 */
Zotero.Translate.prototype.runHandler = function(type, argument) {
	var returnValue = undefined;
	if(this._handlers[type]) {
		for(var i in this._handlers[type]) {
			Zotero.debug("Translate: running handler "+i+" for "+type, 5);
			try {
				if(this._parentTranslator) {
					returnValue = this._handlers[type][i](null, argument);
				} else {
					returnValue = this._handlers[type][i](this, argument);
				}
			} catch(e) {
				if(this._parentTranslator) {
					// throw handler errors if they occur when a translator is
					// called from another translator, so that the
					// "Could Not Translate" dialog will appear if necessary
					throw(e);
				} else {
					// otherwise, fail silently, so as not to interfere with
					// interface cleanup
					Zotero.debug("Translate: "+e+' in handler '+i+' for '+type, 5);
				}
			}
		}
	}
	return returnValue;
}

/*
 * gets all applicable translators
 *
 * for import, you should call this after setLocation; otherwise, you'll just get
 * a list of all import filters, not filters equipped to handle a specific file
 *
 * this returns a list of translator objects, of which the following fields
 * are useful:
 *
 * translatorID - the GUID of the translator
 * label - the name of the translator
 * itemType - the type of item this scraper says it will scrape
 */
Zotero.Translate.prototype.getTranslators = function() {
	// do not allow simultaneous instances of getTranslators
	if(this._translatorSearch) this._translatorSearch.running = false;
	
	var translators = Zotero.Translators.getAllForType(this.type);
	
	// create a new sandbox
	this._generateSandbox();
	this._setSandboxMode("detect");
	
	var possibleTranslators = new Array();
	Zotero.debug("Translate: Searching for translators for "+(this.path ? this.path : "an undisclosed location"), 3);
	
	// see which translators can translate
	this._translatorSearch = new Zotero.Translate.TranslatorSearch(this, translators);
	
	if(this._translatorSearch.asyncMode) {	
		// erroring should call complete
		var me = this;
		this.error = function(value, error) { me._translatorSearch.complete(value, error); };
	} else {
		// return translators if synchronous
		return this._translatorSearch.foundTranslators;
	}
}

/*
 * loads a translator into a sandbox
 */
Zotero.Translate.prototype._loadTranslator = function() {
	if(!this._sandbox || this.type == "search") {
		// create a new sandbox if none exists, or for searching (so that it's
		// bound to the correct url)
		this._generateSandbox();
	}
	this._setSandboxMode("translate");
	
	// parse detect code for the translator
	if(!this._parseCode(this.translator[0])) {
		this._translationComplete(false, "parse error");
		return false;
	}
	
	return true;
}

/**
 * does the actual translation
 *
 * @param 	{NULL|Integer|FALSE}	[libraryID=null]		Library in which to save items,
 *																or NULL for default library;
 *																if FALSE, don't save items
 * @param 	{Boolean}				[saveAttachments=true]	Exclude attachments (e.g., snapshots) on import
 */
Zotero.Translate.prototype.translate = function(libraryID, saveAttachments) {
	/*
	 * initialize properties
	 */
	this.newItems = new Array();
	this.newCollections = new Array();
	this._IDMap = new Array();
	this._complete = false;
	this._itemsDone = false;
	
	if(!this.translator || !this.translator.length) {
		throw("cannot translate: no translator specified");
	}
	
	if(!this.location && this.type != "search" && this.type != "export" && !this._storage) {
		// searches operate differently, because we could have an array of
		// translators and have to go through each
		throw("cannot translate: no location specified");
	}
	
	if(libraryID === false) {
		this.libraryID = false;
	}
	else if(libraryID === true || libraryID == undefined) {
		this.libraryID = null;
	}
	else {
		this.libraryID = libraryID;
	}
	this.saveAttachments = !(saveAttachments === false);
	this.saveFiles = this.saveAttachments;
	
	// If group filesEditable==false, don't save attachments
	if (typeof this.libraryID == 'number') {
		var type = Zotero.Libraries.getType(this.libraryID);
		switch (type) {
			case 'group':
				var groupID = Zotero.Groups.getGroupIDFromLibraryID(this.libraryID);
				var group = Zotero.Groups.get(groupID);
				if (!group.filesEditable) {
					this.saveFiles = false;
				}
				break;
		}
	}
	
	// erroring should end
	var me = this;
	this.error = function(value, error) { me._translationComplete(value, error); }
	
	if(!this._loadTranslator()) {
		return;
	}
	
	if(this._setDisplayOptions) {
		this.displayOptions = this._setDisplayOptions;
	}
	
	if(this._storage) {
		// enable reading from storage, which we can't do until the translator
		// is loaded
		this._storageFunctions(true);
	}
	
	var returnValue;
	if(this.type == "web") {
		returnValue = this._web();
	} else if(this.type == "import") {
		returnValue = this._import();
	} else if(this.type == "export") {
		returnValue = this._export();
	} else if(this.type == "search") {
		returnValue = this._search();
	}
	
	if(returnValue && !this.waitForCompletion) {
		// if synchronous, call _translationComplete();
		this._translationComplete(true);
	}
}

/*
 * parses translator detect code
 */
Zotero.Translate.prototype._parseCode = function(translator) {
	this.configOptions = {};
	this.displayOptions = {};
	
	Zotero.debug("Translate: Parsing code for "+translator.label, 4);
	
	try {
		Components.utils.evalInSandbox("var translatorInfo = "+translator.code, this._sandbox);
		return true;
	} catch(e) {
		if('function' == typeof translator.logError) {
			translator.logError(e.toString());
		} else {
			Components.utils.reportError(e);
		}
		this._debug(e+' in parsing code for '+translator.label, 3);
		return false;
	}
}

/*
 * generates a sandbox for scraping/scraper detection
 */
Zotero.Translate.prototype._generateSandbox = function() {
	var me = this;
	
	// get sandbox URL
	var sandboxLocation = "http://www.example.com/";
	if(this.type == "web") {
		// use real URL, not proxied version, to create sandbox
		sandboxLocation = this.document.defaultView;
		Zotero.debug("Translate: Binding sandbox to "+this.document.location.href, 4);
	} else {
		if (this.type == "search") {
			// generate sandbox for search by extracting domain from translator target
			if(this.translator && this.translator[0] && this.translator[0].target) {
				// so that web translators work too
				const searchSandboxRe = /^http:\/\/[\w.]+\//;
				var tempURL = this.translator[0].target.replace(/\\/g, "").replace(/\^/g, "");
				var m = searchSandboxRe.exec(tempURL);
				if(m) sandboxLocation = m[0];
			}
		}
		Zotero.debug("Translate: Binding sandbox to "+sandboxLocation, 4);
	}
	
	// set up sandbox
	this._sandbox = new Components.utils.Sandbox(sandboxLocation);
	Components.utils.evalInSandbox("var Zotero = {};"+
	"Zotero.Item = function (itemType) {"+
			"this.itemType = itemType;"+
			"this.creators = [];"+
			"this.notes = [];"+
			"this.tags = [];"+
			"this.seeAlso = [];"+
			"this.attachments = [];"+
	"};"+
	"Zotero.Item.prototype.complete = function() { Zotero._itemDone(this); };", this._sandbox)
	
	// add utilities
	this._sandbox.Zotero.Utilities = new Zotero.Utilities.Translate(this);
	this._sandbox.Zotero.Utilities.HTTP = this._sandbox.Zotero.Utilities;
	
	if(this.type == "export") {
		// add routines to retrieve items and collections
		this._sandbox.Zotero.nextItem = function() { return me._exportGetItem() };
		this._sandbox.Zotero.nextCollection = function() { return me._exportGetCollection() }
	} else {
		// copy routines to add new items
		this._sandbox.Zotero._itemDone = function(a) {me._itemDone(a)};
		
		if(this.type == "import") {
			// add routines to add new collections
			this._sandbox.Zotero.Collection = Zotero.Translate.GenerateZoteroCollectionClass();
			// attach the function to be run when a collection is done
			this._sandbox.Zotero.Collection.prototype.complete = function() {me._collectionDone(this)};
		} else if(this.type == "web" || this.type == "search") {	
			// set up selectItems handler
			this._sandbox.Zotero.selectItems = function(options) { return me._selectItems(options) };
		}
	}
	
	this._sandbox.XPathResult = Components.interfaces.nsIDOMXPathResult;
	
	// for debug messages
	this._sandbox.Zotero.debug = function(string) {me._debug(string, 4)};
	
	// for adding configuration options
	this._sandbox.Zotero.configure = function(option, value) {me._configure(option, value) };
	// for adding displayed options
	this._sandbox.Zotero.addOption = function(option, value) {me._addOption(option, value) };
	// for getting the value of displayed options
	this._sandbox.Zotero.getOption = function(option) { return me._getOption(option) };

	// for loading other translators and accessing their methods
	this._sandbox.Zotero.loadTranslator = function(type) {
		if(typeof type !== "string") {
			Zotero.debug("loadTranslator: type must be a string");
			return;
		}
		
		var translation = new Zotero.Translate(type);
		translation._parentTranslator = me;
		
		if(type == "export" && (me.type == "web" || me.type == "search")) {
			throw("for security reasons, web and search translators may not call export translators");
		}
		
		// for security reasons, safeTranslator wraps the translator object.
		// note that setLocation() is not allowed
		var safeTranslator = new Object();
		safeTranslator.setSearch = function(arg) { return translation.setSearch(arg) };
		safeTranslator.setDocument = function(arg) { return translation.setDocument(arg) };
		safeTranslator.setHandler = function(arg1, arg2) {
			translation.setHandler(arg1, 
				function(obj, item) {
					try {
						arg2(obj, item);
					} catch(e) {
						me.error(false, e);
					}
				}
			);
		};
		safeTranslator.setString = function(arg) { translation.setString(arg) };
		safeTranslator.setTranslator = function(arg) { return translation.setTranslator(arg) };
		safeTranslator.getTranslators = function() { return translation.getTranslators() };
		safeTranslator.translate = function() {
			var noHandlers = true;
			for(var i in translation._handlers) {
				noHandlers = false;
				break;
			}
			if(noHandlers) {
				if(type != "export") {
					translation.setHandler("itemDone", function(obj, item) { item.complete() });
				}
				if(type == "web") {
					translation.setHandler("selectItems", me._handlers["selectItems"]);
				}
			}
			return translation.translate(false);
		};
		safeTranslator.getTranslatorObject = function() {
			// load the translator into our sandbox
			translation._loadTranslator();
			// initialize internal IO
			translation._initializeInternalIO();
			
			var noHandlers = true;
			for(var i in translation._handlers) {
				noHandlers = false;
				break;
			}
			if(noHandlers) {
				if(type != "export") {
					translation.setHandler("itemDone", function(obj, item) { item.complete() });
				}
				if(type == "web") {
					translation.setHandler("selectItems", me._handlers["selectItems"]);
				}
			}
			
			// return sandbox
			return translation._sandbox;
		};
		
		return safeTranslator;
	}
}

/*
 * Adds appropriate methods for detect/translate modes
 */
Zotero.Translate.prototype._setSandboxMode = function(mode) {
	var me = this;
	
	// erase waitForCompletion status and done function
	this.waitForCompletion = false;
	this._sandbox.Zotero.done = undefined;
	
	if(mode == "detect") {
		// for asynchronous operation, use wait()
		// done() is implemented after wait() is called
		this._sandbox.Zotero.wait = function() { me._enableAsynchronousDetect() };
	} else {
		// for asynchronous operation, use wait()
		// done() is implemented after wait() is called
		this._sandbox.Zotero.wait = function() { me._enableAsynchronousTranslate() };
	}
}

/*
 * sets an option that modifies the way the translator is executed
 * 
 * called as configure() in translator detectCode
 *
 * current options:
 *
 * dataMode
 *   valid: import, export
 *   options: rdf, block, line
 *   purpose: selects whether write/read behave as standard text functions or
 *            use an RDF data source
 *
 * getCollections
 *   valid: export
 *   options: true, false
 *   purpose: selects whether export translator will receive an array of
 *            collections and children in addition to the array of items and
 *            children
 */
Zotero.Translate.prototype._configure = function(option, value) {
	if(typeof option !== "string") {
		Zotero.debug ("configure: option must be a string");
		return;
	}
	if(typeof value === "object" && value !== null) {
		value = new XPCSafeJSObjectWrapper(value);
	}
	
	this.configOptions[option] = value;
	Zotero.debug("Translate: Setting configure option "+option+" to "+value, 4);
}

/*
 * adds translator options to be displayed in a dialog
 *
 * called as addOption() in detect code
 *
 * current options are exportNotes and exportFileData
 */
Zotero.Translate.prototype._addOption = function(option, value) {
	if(typeof option !== "string") {
		Zotero.debug ("addOption: option must be a string");
		return;
	}
	if(typeof value === "object" && value !== null) {
		value = new XPCSafeJSObjectWrapper(value);
	}
	
	this.displayOptions[option] = value;
	Zotero.debug("Translate: Setting display option "+option+" to "+value, 4);
}

/*
 * gets translator options that were displayed in a dialog
 *
 * called as getOption() in detect code
 *
 */
Zotero.Translate.prototype._getOption = function(option) {
	if(typeof option !== "string") {
		Zotero.debug ("getOption: option must be a string");
		return;
	}
	return this.displayOptions[option];
}

/*
 * makes translation API wait until done() has been called from the translator
 * before executing _translationComplete
 * 
 * called as wait() in translator code
 */
Zotero.Translate.prototype._enableAsynchronousDetect = function() {
	var me = this;
	this.waitForCompletion = true;
	this._sandbox.Zotero.done = function(arg) {
		if(arg !== null && typeof arg === "object") {
			Zotero.debug("done: argument must not be an object");
			me._translatorSearch.complete(false);
		} else {
			me._translatorSearch.complete(arg);
		}
	};
}

Zotero.Translate.prototype._enableAsynchronousTranslate = function() {
	var me = this;
	this.waitForCompletion = true;
	this._sandbox.Zotero.done = function(val, error) {
		if(val !== null && typeof val === "object") {
			val = new XPCSafeJSObjectWrapper(val);
		}
		if(error !== null && typeof error === "object") {
			error = new XPCSafeJSObjectWrapper(error);
		}
		me._translationComplete(val == undefined ? true : val, (error ? error : ""))
	};
}

/*
 * lets user pick which items s/he wants to put in his/her library
 * 
 * called as selectItems() in translator code
 */
Zotero.Translate.prototype._selectItems = function(options) {
	if(!options instanceof XPCSafeJSObjectWrapper) {
		options = new XPCSafeJSObjectWrapper(options);
	}
	
	// hack to see if there are options
	var haveOptions = false;
	for(var i in options) {
		haveOptions = true;
		break;
	}
	
	if(!haveOptions) {
		throw "translator called select items with no items";
	}
	
	if(this._handlers.select) {
		return this.runHandler("select", options);
	} else {	// no handler defined; assume they want all of them
		return options;
	}
}

/*
 * executed on translator completion, either automatically from a synchronous
 * scraper or as done() from an asynchronous scraper
 *
 * finishes things up and calls callback function(s)
 */
Zotero.Translate.prototype._translationComplete = function(returnValue, error) {
	// to make sure this isn't called twice
	if(returnValue === undefined) {
		returnValue = this._itemsDone;
	}
	
	if(!this._complete) {
		this._complete = true;
		
		if(this.type == "search" && !this._itemsDone) {
			// if we're performing a search and didn't get any results, go on
			// to the next translator
			Zotero.debug("Translate: Could not find a result using "+this.translator[0].label+": \n"
			              +this._generateErrorString(error), 3);
			if(this.translator.length > 1) {
				this.translator.shift();
				this.translate();
				return;
			}
			
			returnValue = false;
		} else {
			// close open streams
			this._closeStreams();
			
			if(!returnValue) {
				// report error to console
				if(typeof this.translator[0] != "undefined" && 'function' == typeof this.translator[0].logError) {
					this.translator[0].logError(error.toString(), "exception");
				} else {
					Components.utils.reportError(error);
				}
				
				// report error to debug log
				var errorString = this._generateErrorString(error);
				this._debug("Translation using "+(this.translator && this.translator[0] && this.translator[0].label ? this.translator[0].label : "no translator")+" failed: \n"+errorString, 2);
				
				// report translation error for webpages to mothership
				if(this.type == "web") this._reportTranslationFailure(errorString);
				
				this.runHandler("error", error);
			} else {
				this._debug("Translation successful");
			}
		}		
		
		// call handlers
		this.runHandler("done", returnValue);
	}
}

/*
 * generates a useful error string, for submitting and debugging purposes
 */
Zotero.Translate.prototype._generateErrorString = function(error) {
	var errorString = "";
	if(typeof(error) == "string") {
		errorString = "\nthrown exception => "+error;
	} else {
		for(var i in error) {
			if(typeof(error[i]) != "object") {
				errorString += "\n"+i+' => '+error[i];
			}
		}
	}
	
	errorString += "\nurl => "+this.path
		+ "\ndownloadAssociatedFiles => "+Zotero.Prefs.get("downloadAssociatedFiles")
		+ "\nautomaticSnapshots => "+Zotero.Prefs.get("automaticSnapshots");
	return errorString.substr(1);
}

/*
 * runs an HTTP request to report a translation error
 */
Zotero.Translate.prototype._reportTranslationFailure = function(errorData) {
	if(this.translator[0].inRepository && Zotero.Prefs.get("reportTranslationFailure")) {
		var postBody = "id=" + encodeURIComponent(this.translator[0].translatorID) +
					   "&lastUpdated=" + encodeURIComponent(this.translator[0].lastUpdated) +
					   "&diagnostic=" + encodeURIComponent(Zotero.getSystemInfo()) +
					   "&errorData=" + encodeURIComponent(errorData);
		Zotero.Utilities.HTTP.doPost("http://www.zotero.org/repo/report", postBody);
	}
}

/*
 * closes open file streams, if any exist
 */
Zotero.Translate.prototype._closeStreams = function() {
	// serialize RDF and unregister dataSource
	if(this._rdf) {
		if(this.type == "export") {
			// initialize serializer and add prefixes
			var serializer = Serializer();
			for(var prefix in this._rdf.namespaces) {
				serializer.suggestPrefix(prefix, this._rdf.namespaces[prefix]);
			}
			
			// serialize in appropriate format
			if(this.configOptions.dataMode == "rdf/n3") {
				var output = serializer.statementsToN3(this._rdf.statements);
			} else {
				var output = serializer.statementsToXML(this._rdf.statements);
			}
			
			if(this._streams.length) {
				Zotero.debug("writing to stream");
				// write serialized data to file
				var intlStream = Components.classes["@mozilla.org/intl/converter-output-stream;1"]
									   .createInstance(Components.interfaces.nsIConverterOutputStream);
				intlStream.init(this._streams[0], "UTF-8", 4096, "?".charCodeAt(0));
				this._streams.push(intlStream);
				intlStream.writeString(output);
			} else {
				this.output = output;
			}
		}
		
		delete this._rdf;
	}
	
	if(this._streams.length) {
		for(var i in this._streams) {
			var stream = this._streams[i];
			
			// stream could be either an input stream or an output stream
			try {
				stream.QueryInterface(Components.interfaces.nsIFileInputStream);
			} catch(e) {
				try {
					stream.QueryInterface(Components.interfaces.nsIFileOutputStream);
				} catch(e) {
				}
			}
			
			// encase close in try block, because it's possible it's already
			// closed
			try {
				stream.close();
			} catch(e) {
			}
		}
	}
	
	delete this._streams;
	this._streams = new Array();
	this._inputStream = null;
}

/*
 * handles tags and see also data for notes and attachments
 */
Zotero.Translate.prototype._itemTagsAndSeeAlso = function(item, newItem) {
	// add to ID map
	if(item.itemID) {
		this._IDMap[item.itemID] = newItem.id;
	}
	// add see alsos
	if(item.seeAlso) {
		for each(var seeAlso in item.seeAlso) {
			if(this._IDMap[seeAlso]) {
				newItem.addRelatedItem(this._IDMap[seeAlso]);
			}
		}
		newItem.save();
	}
	if(item.tags) {
		var tagsToAdd = {};
		tagsToAdd[0] = []; // user tags
		tagsToAdd[1] = []; // automatic tags
		
		for each(var tag in item.tags) {
			if(typeof(tag) == "string") {
				// accept strings in tag array as automatic tags, or, if
				// importing, as non-automatic tags
				if (this.type == 'import') {
					tagsToAdd[0].push(tag);
				}
				else {
					tagsToAdd[1].push(tag);
				}
			} else if(typeof(tag) == "object") {
				// also accept objects
				if(tag.tag) {
					if (!tagsToAdd[tag.type]) {
						tagsToAdd[tag.type] = [];
					}
					tagsToAdd[tag.type].push(tag.tag);
				}
			}
		}
		
		for (var type in tagsToAdd) {
			if (tagsToAdd[type].length) {
				newItem.addTags(tagsToAdd[type], type);
			}
		}
	}
}

/*
 * executed when an item is done and ready to be loaded into the database
 */
Zotero.Translate._urlRe = /(([A-Za-z]+):\/\/[^\s]*)/i;
Zotero.Translate.prototype._itemDone = function(item, attachedTo) {
	// warn if itemDone called after translation completed
	if(this._complete) {
		Zotero.debug("Translate: WARNING: Zotero.Item.complete() called after Zotero.done(); please fix your code", 2);
	}
	
	// make item safe to access
	if(!item instanceof XPCSafeJSObjectWrapper) {
		item = new XPCSafeJSObjectWrapper(item);
	}
	
	if(this.type == "web") {
		// store library catalog if this item was captured from a website, and
		// libraryCatalog is truly undefined (not false or "")
		if(typeof item.repository != 'undefined') {
			Zotero.debug("Translate: 'repository' field is now 'libraryCatalog'; please fix your code", 2);
			item.libraryCatalog = item.repository;
			delete item.repository;
		}
		
		if(typeof item.libraryCatalog == 'undefined') {
			item.libraryCatalog = this.translator[0].label;
		}
	}
	
	this._itemsDone = true;
	
	// if we're not supposed to save the item or we're in a child translator,
	// just return the item array
	if(this.libraryID === false || this._parentTranslator) {
		
		// if a parent sandbox exists, use complete() function from that sandbox
		if(this._parentTranslator) {
			var pt = this._parentTranslator;
			item.complete = function() { pt._itemDone(this) };
			Zotero.debug("Translate: Calling done from parent sandbox", 4);
		}
		this.runHandler("itemDone", item);
		return;
	}
	
	// Get typeID, defaulting to "webpage"
	var type = (item.itemType ? item.itemType : "webpage");
	
	if(type == "note") {	// handle notes differently
		var newItem = new Zotero.Item('note');
		newItem.libraryID = this.libraryID ? this.libraryID : null;
		newItem.setNote(item.note);
		var myID = newItem.save();
		// re-retrieve the item
		var newItem = Zotero.Items.get(myID);
	} else {
		if(!item.title && this.type == "web") {
			throw("item has no title");
		}
		
		// if item was accessed through a proxy, ensure that the proxy
		// address remains in the accessed version
		if(this.locationIsProxied && item.url) {
			item.url = Zotero.Proxies.properToProxy(item.url);
		}
		
		// create new item
		if(type == "attachment") {
			if(this.type != "import") {
				Zotero.debug("Translate: Discarding standalone attachment", 2);
				return;
			}
			
			Zotero.debug("Translate: Adding attachment", 4);
			
			if(!item.url && !item.path) {
				Zotero.debug("Translate: Ignoring attachment: no path or URL specified", 2);
				return;
			}
			
			if(!item.path) {
				// see if this is actually a file URL
				var m = Zotero.Translate._urlRe.exec(item.url);
				var protocol = m ? m[2].toLowerCase() : "";
				if(protocol == "file") {
					item.path = item.url;
					item.url = false;
				} else if(protocol != "http" && protocol != "https") {
					Zotero.debug("Translate: Unrecognized protocol "+protocol, 2);
					return;
				}
			}
			
			if(!item.path) {
				// create from URL
				try {
					var myID = Zotero.Attachments.linkFromURL(item.url, attachedTo,
							(item.mimeType ? item.mimeType : undefined),
							(item.title ? item.title : undefined));
				} catch(e) {
					Zotero.debug("Translate: Error adding attachment "+item.url, 2);
					return;
				}
				Zotero.debug("Translate: Created attachment; id is "+myID, 4);
				var newItem = Zotero.Items.get(myID);
			} else {
				// generate nsIFile
				var IOService = Components.classes["@mozilla.org/network/io-service;1"].
								getService(Components.interfaces.nsIIOService);
				var uri = IOService.newURI(item.path, "", null);
				var file = uri.QueryInterface(Components.interfaces.nsIFileURL).file;
				
				if (file.path == '/') {
					Zotero.debug("Translate: Ignoring attachment '" + item.path + "': error parsing path", 2);
					return;
				}
				
				if (!file.exists()) {
					// use item title if possible, or else file leaf name
					var title = item.title;
					if(!title) title = file.leafName;<commit_msg>Handle invalid filenames without failure during import<commit_after>/*
    ***** BEGIN LICENSE BLOCK *****
    
    Copyright © 2009 Center for History and New Media
                     George Mason University, Fairfax, Virginia, USA
                     http://zotero.org
    
    This file is part of Zotero.
    
    Zotero is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
    
    Zotero is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
    
    You should have received a copy of the GNU General Public License
    along with Zotero.  If not, see <http://www.gnu.org/licenses/>.
    
    ***** END LICENSE BLOCK *****
*/

// Enumeration of types of translators
const TRANSLATOR_TYPES = {"import":1, "export":2, "web":4, "search":8};

// Byte order marks for various character sets
const BOMs = {
	"UTF-8":"\xEF\xBB\xBF",
	"UTF-16BE":"\xFE\xFF",
	"UTF-16LE":"\xFF\xFE",
	"UTF-32BE":"\x00\x00\xFE\xFF",
	"UTF-32LE":"\xFF\xFE\x00\x00"
}

/**
 * Singleton to handle loading and caching of translators
 * @namespace
 */
Zotero.Translators = new function() {
	var _cache, _translators;
	var _initialized = false;
	
	/**
	 * Initializes translator cache, loading all relevant translators into memory
	 */
	this.init = function() {
		_initialized = true;
		
		var start = (new Date()).getTime();
		var transactionStarted = false;
		
		_cache = {"import":[], "export":[], "web":[], "search":[]};
		_translators = {};
		
		var dbCacheResults = Zotero.DB.query("SELECT leafName, translatorJSON, "+
			"code, lastModifiedTime FROM translatorCache");
		var dbCache = {};
		for each(var cacheEntry in dbCacheResults) {
			dbCache[cacheEntry.leafName] = cacheEntry;
		}
		
		var i = 0;
		var filesInCache = {};
		var contents = Zotero.getTranslatorsDirectory().directoryEntries;
		while(contents.hasMoreElements()) {
			var file = contents.getNext().QueryInterface(Components.interfaces.nsIFile);
			var leafName = file.leafName;
			if(!leafName || leafName[0] == ".") continue;
			var lastModifiedTime = file.lastModifiedTime;
			
			var dbCacheEntry = false;
			if(dbCache[leafName]) {
				filesInCache[leafName] = true;
				if(dbCache[leafName].lastModifiedTime == lastModifiedTime) {
					dbCacheEntry = dbCache[file.leafName];
				}
			}
			
			if(dbCacheEntry) {
				// get JSON from cache if possible
				var translator = new Zotero.Translator(file, dbCacheEntry.translatorJSON, dbCacheEntry.code);
				filesInCache[leafName] = true;
			} else {
				// otherwise, load from file
				var translator = new Zotero.Translator(file);
			}
			
			if(translator.translatorID) {
				if(_translators[translator.translatorID]) {
					// same translator is already cached
					translator.logError('Translator with ID '+
						translator.translatorID+' already loaded from "'+
						_translators[translator.translatorID].file.leafName+'"');
				} else {
					// add to cache
					_translators[translator.translatorID] = translator;
					for(var type in TRANSLATOR_TYPES) {
						if(translator.translatorType & TRANSLATOR_TYPES[type]) {
							_cache[type].push(translator);
						}
					}
					
					if(!dbCacheEntry) {
						// Add cache misses to DB
						if(!transactionStarted) {
							transactionStarted = true;
							Zotero.DB.beginTransaction();
						}
						Zotero.Translators.cacheInDB(leafName, translator.metadataString, translator.cacheCode ? translator.code : null, lastModifiedTime);
						delete translator.metadataString;
					}
				}
			}
			
			i++;
		}
		
		// Remove translators from DB as necessary
		for(var leafName in dbCache) {
			if(!filesInCache[leafName]) {
				Zotero.DB.query("DELETE FROM translatorCache WHERE leafName = ?", [leafName]);
			}
		}
		
		// Close transaction
		if(transactionStarted) {
			Zotero.DB.commitTransaction();
		}
		
		// Sort by priority
		var collation = Zotero.getLocaleCollation();
		var cmp = function (a, b) {
			if (a.priority > b.priority) {
				return 1;
			}
			else if (a.priority < b.priority) {
				return -1;
			}
			return collation.compareString(1, a.label, b.label);
		}
		for(var type in _cache) {
			_cache[type].sort(cmp);
		}
		
		Zotero.debug("Cached "+i+" translators in "+((new Date()).getTime() - start)+" ms");
	}
	
	/**
	 * Gets the translator that corresponds to a given ID
	 */
	this.get = function(id) {
		if(!_initialized) this.init();
		return _translators[id] ? _translators[id] : false;
	}
	
	/**
	 * Gets all translators for a specific type of translation
	 */
	this.getAllForType = function(type) {
		if(!_initialized) this.init();
		return _cache[type].slice(0);
	}
	
	/**
	 * @param	{String}		label
	 * @return	{String}
	 */
	this.getFileNameFromLabel = function(label) {
		return Zotero.File.getValidFileName(label) + ".js";
	}
	
	
	/**
	 * @param	{String}		metadata
	 * @param	{String}		metadata.translatorID		Translator GUID
	 * @param	{Integer}		metadata.translatorType		See TRANSLATOR_TYPES in translate.js
	 * @param	{String}		metadata.label				Translator title
	 * @param	{String}		metadata.creator			Translator author
	 * @param	{String|Null}	metadata.target				Target regexp
	 * @param	{String|Null}	metadata.minVersion
	 * @param	{String}		metadata.maxVersion
	 * @param	{Integer}		metadata.priority
	 * @param	{Boolean}		metadata.inRepository
	 * @param	{String}		metadata.lastUpdated		SQL date
	 * @param	{String}		code
	 * @return	{nsIFile}
	 */
	this.save = function(metadata, code) {
		if (!metadata.translatorID) {
			throw ("metadata.translatorID not provided in Zotero.Translators.save()");
		}
		
		if (!metadata.translatorType) {
			var found = false;
			for each(var type in TRANSLATOR_TYPES) {
				if (metadata.translatorType & type) {
					found = true;
					break;
				}
			}
			if (!found) {
				throw ("Invalid translatorType '" + metadata.translatorType + "' in Zotero.Translators.save()");
			}
		}
		
		if (!metadata.label) {
			throw ("metadata.label not provided in Zotero.Translators.save()");
		}
		
		if (!metadata.priority) {
			throw ("metadata.priority not provided in Zotero.Translators.save()");
		}
		
		if (!metadata.lastUpdated) {
			throw ("metadata.lastUpdated not provided in Zotero.Translators.save()");
		}
		
		if (!code) {
			throw ("code not provided in Zotero.Translators.save()");
		}
		
		var fileName = Zotero.Translators.getFileNameFromLabel(metadata.label);
		var destFile = Zotero.getTranslatorsDirectory();
		destFile.append(fileName);
		
		var metadataJSON;
		
		// JSON.stringify (FF 3.5.4 and up) has the benefit of indenting JSON
		if (typeof JSON != "undefined" && 'function' == typeof JSON.stringify) {
			metadataJSON = JSON.stringify(metadata,null,8);
		} else {
			var nsIJSON = Cc["@mozilla.org/dom/json;1"].createInstance(Ci.nsIJSON);
			metadataJSON = nsIJSON.encode(metadata);
		}
		
		var str = metadataJSON + "\n\n" + code;
		
		var translator = Zotero.Translators.get(metadata.translatorID);
		if (translator && destFile.equals(translator.file)) {
			var sameFile = true;
		}
		
		if (!sameFile && destFile.exists()) {
			var msg = "Overwriting translator with same filename '"
				+ fileName + "'";
			Zotero.debug(msg, 1);
			Zotero.debug(metadata, 1);
			Components.utils.reportError(msg + " in Zotero.Translators.save()");
		}
		
		if (translator && translator.file.exists()) {
			translator.file.remove(false);
		}
		
		Zotero.debug("Saving translator '" + metadata.label + "'");
		Zotero.debug(str);
		Zotero.File.putContents(destFile, str);
		
		return destFile;
	}
	
	this.cacheInDB = function(fileName, metadataJSON, code, lastModifiedTime) {
		Zotero.DB.query("REPLACE INTO translatorCache VALUES (?, ?, ?, ?)",
			[fileName, metadataJSON, code, lastModifiedTime]);
	}
}

/**
 * @class Represents an individual translator
 * @constructor
 * @param {nsIFile} file File from which to generate a translator object
 * @property {String} translatorID Unique GUID of the translator
 * @property {Integer} translatorType Type of the translator (use bitwise & with TRANSLATOR_TYPES to read)
 * @property {String} label Human-readable name of the translator
 * @property {String} creator Author(s) of the translator
 * @property {String} target Location that the translator processes
 * @property {String} minVersion Minimum Zotero version
 * @property {String} maxVersion Minimum Zotero version
 * @property {Integer} priority Lower-priority translators will be selected first
 * @property {Boolean} inRepository Whether the translator may be found in the repository
 * @property {String} lastUpdated SQL-style date and time of translator's last update
 * @property {String} code The executable JavaScript for the translator
 */
Zotero.Translator = function(file, json, code) {
	const codeGetterFunction = function() { return Zotero.File.getContents(this.file); }
	// Maximum length for the info JSON in a translator
	const MAX_INFO_LENGTH = 4096;
	const infoRe = /{(?:(?:"(?:[^"\r\n]*(?:\\")?)*")*[^}"]*)*}/;
	
	this.file = file;
	
	if(json) {
		var info = Zotero.JSON.unserialize(json);
	} else {
		var fStream = Components.classes["@mozilla.org/network/file-input-stream;1"].
			createInstance(Components.interfaces.nsIFileInputStream);
		var cStream = Components.classes["@mozilla.org/intl/converter-input-stream;1"].
			createInstance(Components.interfaces.nsIConverterInputStream);
		fStream.init(file, -1, -1, 0);
		cStream.init(fStream, "UTF-8", 8192,
			Components.interfaces.nsIConverterInputStream.DEFAULT_REPLACEMENT_CHARACTER);
		
		var str = {};
		cStream.readString(MAX_INFO_LENGTH, str);
		
		// We assume lastUpdated is at the end to avoid running the regexp on more than necessary
		var lastUpdatedIndex = str.value.indexOf('"lastUpdated"');
		if (lastUpdatedIndex == -1) {
			this.logError("Invalid or missing translator metadata JSON object in " + file.leafName);
			fStream.close();
			return;
		}
		
		// Add 50 characters to clear lastUpdated timestamp and final "}"
		var header = str.value.substr(0, lastUpdatedIndex + 50);
		var m = infoRe.exec(header);
		if (!m) {
			this.logError("Invalid or missing translator metadata JSON object in " + file.leafName);
			fStream.close();
			return;
		}
		
		this.metadataString = m[0];
		
		try {
			var info = Zotero.JSON.unserialize(this.metadataString);
		} catch(e) {
			this.logError("Invalid or missing translator metadata JSON object in " + file.leafName);
			fStream.close();
			return;
		}
	}
	
	var haveMetadata = true;
	// make sure we have all the properties
	for each(var property in ["translatorID", "translatorType", "label", "creator", "target", "minVersion", "maxVersion", "priority", "lastUpdated", "inRepository"]) {
		if(info[property] === undefined) {
			this.logError('Missing property "'+property+'" in translator metadata JSON object in ' + file.leafName);
			haveMetadata = false;
			break;
		} else {
			this[property] = info[property];
		}
	}
	if(!haveMetadata) {
		fStream.close();
		return;
	}
	
	this.detectXPath = info["detectXPath"] ? info["detectXPath"] : null;
	
	/**
	 * g = Gecko (Firefox)
	 * c = Google Chrome (WebKit & V8)
	 * s = Safari (WebKit & Nitro/Squirrelfish Extreme)
	 * i = Internet Explorer
	 * a = All
	 */
	this.browserSupport = info["browserSupport"] ? info["browserSupport"] : "g";
	
	if(this.translatorType & TRANSLATOR_TYPES["import"]) {
		// compile import regexp to match only file extension
		this.importRegexp = this.target ? new RegExp("\\."+this.target+"$", "i") : null;
	}
		
	this.cacheCode = false;
	if(this.translatorType & TRANSLATOR_TYPES["web"]) {
		// compile web regexp
		this.webRegexp = this.target ? new RegExp(this.target, "i") : null;
		
		if(!this.target) {
			this.cacheCode = true;
			
			if(json) {
				// if have JSON, also have code
				this.code = code;
			} else {
				// for translators used on every page, cache code in memory
				var strs = [str.value];
				var amountRead;
				while(amountRead = cStream.readString(8192, str)) strs.push(str.value);
				this.code = strs.join("");
			}
		}
	}
	
	if(!this.cacheCode) this.__defineGetter__("code", codeGetterFunction);
	if(!json) cStream.close();
}

/**
 * Log a translator-related error
 * @param {String} message The error message
 * @param {String} [type] The error type ("error", "warning", "exception", or "strict")
 * @param {String} [line] The text of the line on which the error occurred
 * @param {Integer} lineNumber
 * @param {Integer} colNumber
 */
Zotero.Translator.prototype.logError = function(message, type, line, lineNumber, colNumber) {
	var ios = Components.classes["@mozilla.org/network/io-service;1"].
		getService(Components.interfaces.nsIIOService);
	Zotero.log(message, type ? type : "error", ios.newFileURI(this.file).spec);
}


/*
 * Zotero.Translate: a class for translation of Zotero metadata from and to
 * other formats
 *
 * type can be: 
 * export
 * import
 * web
 * search
 *
 * a typical export process:
 * var translatorObj = new Zotero.Translate();
 * var possibleTranslators = translatorObj.getTranslators();
 * // do something involving nsIFilePicker; remember, each possibleTranslator
 * // object has properties translatorID, label, and targetID
 * translatorObj.setLocation(myNsILocalFile);
 * translatorObj.setTranslator(possibleTranslators[x]); // also accepts only an ID
 * translatorObj.setHandler("done", _translationDone);
 * translatorObj.translate();
 *
 *
 * PUBLIC PROPERTIES:
 *
 * type - the text type of translator (set by constructor, should be read only)
 * document - the document object to be used for web scraping (read-only; set
 *           with setDocument)
 * translator - the translator currently in use (read-only; set with
 *               setTranslator)
 * location - the location of the target (read-only; set with setLocation)
 *            for import/export - this is an instance of nsILocalFile
 *            for web - this is a URL
 * search - item (in toArray() format) to extrapolate data for (read-only; set
 *          with setSearch).
 * items - items (in Zotero.Item format) to be exported. if this is empty,
 *         Zotero will export all items in the library (read-only; set with
 *         setItems). setting items disables export of collections.
 * path - the path to the target; for web, this is the same as location
 * string - the string content to be used as a file.
 * libraryID - libraryID (e.g., of a group) of saved database items. null for local items,
 *           or false not to save. defaults to null; set using second argument of constructor.
 * newItems - items created when translate() was called
 * newCollections - collections created when translate() was called
 *
 * PSEUDO-PRIVATE PROPERTIES (used only by other objects in this file):
 *
 * waitForCompletion - whether to wait for asynchronous completion, or return
 *                     immediately when script has finished executing
 * configOptions - options set by translator modifying behavior of
 *                  Zotero.Translate
 * displayOptions - options available to user for this specific translator
 *
 * PRIVATE PROPERTIES:
 * 
 * _charset - character set
 * _numericTypes - possible numeric types as a comma-delimited string
 * _handlers - handlers for various events (see setHandler)
 * _sandbox - sandbox in which translators will be executed
 * _streams - streams that need to be closed when execution is complete
 * _IDMap - a map from IDs as specified in Zotero.Item() to IDs of actual items
 * _parentTranslator - set when a translator is called from another translator. 
 *                     among other things, disables passing of the translate
 *                     object to handlers and modifies complete() function on 
 *                     returned items
 * _storage - the stored string to be treated as input
 * _storageLength - the length of the stored string
 * _exportFileDirectory - the directory to which files will be exported
 *
 * WEB-ONLY PROPERTIES:
 *
 * locationIsProxied - whether the URL being scraped is going through
 *                      an EZProxy
 * _downloadAssociatedFiles - whether to download content, according to
 *                            preferences
 *
 * EXPORT-ONLY PROPERTIES:
 *
 * output - export output (if no location has been specified)
 */
Zotero.Translate = function(type) {
	if (arguments.length > 1) {
		throw ("Zotero.Translate only takes one parameter");
	}
	
	this.type = type;
	
	// import = 0001 = 1
	// export = 0010 = 2
	// web    = 0100 = 4
	// search = 1000 = 8
	
	// combination types determined by addition or bitwise AND
	// i.e., import+export = 1+2 = 3
	this._numericTypes = "";
	for(var i=0; i<=1; i++) {
		for(var j=0; j<=1; j++) {
			for(var k=0; k<=1; k++) {
				if(type == "import") {
					this._numericTypes += ","+parseInt(i.toString()+j.toString()+k.toString()+"1", 2);
				} else if(type == "export") {
					this._numericTypes += ","+parseInt(i.toString()+j.toString()+"1"+k.toString(), 2);
				} else if(type == "web") {
					this._numericTypes += ","+parseInt(i.toString()+"1"+j.toString()+k.toString(), 2);
				} else if(type == "search") {
					this._numericTypes += ","+parseInt("1"+i.toString()+j.toString()+k.toString(), 2);
				} else {
					throw("invalid import type");
				}
			}
		}
	}
	this._numericTypes = this._numericTypes.substr(1);
	
	this._handlers = new Array();
	this._streams = new Array();
}

/*
 * sets the browser to be used for web translation; also sets the location
 */
Zotero.Translate.prototype.setDocument = function(doc) {
	this.document = doc;
	this.setLocation(doc.location.href);
}

/*
 * sets the item to be used for searching
 */
Zotero.Translate.prototype.setSearch = function(search) {
	this.search = search;
}

/*
 * sets the items to be used for export
 */
Zotero.Translate.prototype.setItems = function(items) {
	this.items = items;
}

/*
 * sets the collection to be used for export (overrides setItems)
 */
Zotero.Translate.prototype.setCollection = function(collection) {
	this.collection = collection;
}

/*
 * sets the location to operate upon (file should be an nsILocalFile object or
 * web address)
 */
Zotero.Translate.prototype.setLocation = function(location) {
	if(this.type == "web") {
		// account for proxies
		this.location = Zotero.Proxies.proxyToProper(location);
		if(this.location != location) {
			// figure out if this URL is being proxies
			this.locationIsProxied = true;
		}
		this.path = this.location;
	} else {
		this.location = location;
		if(this.location instanceof Components.interfaces.nsIFile) {	// if a file
			this.path = location.path;
		} else {														// if a url
			this.path = location;
		}
	}
}

/*
 * sets the string to be used as a file
 */
Zotero.Translate.prototype.setString = function(string) {
	this._storage = string;
	this._storageLength = string.length;
	this._storagePointer = 0;
}

/*
 * sets translator display options. you can also pass a translator (not ID) to
 * setTranslator that includes a displayOptions argument
 */
Zotero.Translate.prototype.setDisplayOptions = function(displayOptions) {
	this._setDisplayOptions = displayOptions;
}

/*
 * sets the translator to be used for import/export
 *
 * accepts either the object from getTranslators() or an ID
 */
Zotero.Translate.prototype.setTranslator = function(translator) {
	if(!translator) {
		throw("cannot set translator: invalid value");
	}
	
	this.translator = null;
	this._setDisplayOptions = null;
	
	if(typeof(translator) == "object") {	// passed an object and not an ID
		if(translator.translatorID) {
			if(translator.displayOptions) {
				this._setDisplayOptions = translator.displayOptions;
			}
			
			this.translator = [translator];
		} else {
			// we have an array of translators
			if(this.type != "search") {
				throw("cannot set translator: a single translator must be specified when doing "+this.type+" translation");
			}
			
			// accept a list of objects
			this.translator = [];
			for(var i in translator) {
				if(typeof(translator[i]) == "object") {
					this.translator.push(translator[i]);
				} else {
					this.translator.push(Zotero.Translators.get(translator[i]));
				}
			}
		}
	} else {
		this.translator = [Zotero.Translators.get(translator)];
	}
	
	return !!this.translator;
}

/*
 * registers a handler function to be called when translation is complete
 * 
 * as the first argument, all handlers will be passed the current function. the
 * second argument is dependent on the handler.
 *
 * select
 *   valid: web
 *   called: when the user needs to select from a list of available items
 *   passed: an associative array in the form id => text
 *   returns: a numerically indexed array of ids, as extracted from the passed
 *            string
 *
 * itemDone
 *   valid: import, web, search
 *   called: when an item has been processed; may be called asynchronously
 *   passed: an item object (see Zotero.Item)
 *   returns: N/A
 *
 * collectionDone
 *   valid: import
 *   called: when a collection has been processed, after all items have been
 *           added; may be called asynchronously
 *   passed: a collection object (see Zotero.Collection)
 *   returns: N/A
 * 
 * done
 *   valid: all
 *   called: when all processing is finished
 *   passed: true if successful, false if an error occurred
 *   returns: N/A
 *
 * debug
 *   valid: all
 *   called: when Zotero.debug() is called
 *   passed: string debug message
 *   returns: true if message should be logged to the console, false if not
 *
 * error
 *   valid: all
 *   called: when a fatal error occurs
 *   passed: error object (or string)
 *   returns: N/A
 *
 * translators
 *   valid: all
 *   called: when a translator search initiated with Zotero.getTranslators() is
 *           complete
 *   passed: an array of appropriate translators
 *   returns: N/A
 */
Zotero.Translate.prototype.setHandler = function(type, handler) {
	if(!this._handlers[type]) {
		this._handlers[type] = new Array();
	}
	this._handlers[type].push(handler);
}

/*
 * clears all handlers for a given function
 */
Zotero.Translate.prototype.clearHandlers = function(type) {
	this._handlers[type] = new Array();
}

/*
 * calls a handler (see setHandler above)
 */
Zotero.Translate.prototype.runHandler = function(type, argument) {
	var returnValue = undefined;
	if(this._handlers[type]) {
		for(var i in this._handlers[type]) {
			Zotero.debug("Translate: running handler "+i+" for "+type, 5);
			try {
				if(this._parentTranslator) {
					returnValue = this._handlers[type][i](null, argument);
				} else {
					returnValue = this._handlers[type][i](this, argument);
				}
			} catch(e) {
				if(this._parentTranslator) {
					// throw handler errors if they occur when a translator is
					// called from another translator, so that the
					// "Could Not Translate" dialog will appear if necessary
					throw(e);
				} else {
					// otherwise, fail silently, so as not to interfere with
					// interface cleanup
					Zotero.debug("Translate: "+e+' in handler '+i+' for '+type, 5);
				}
			}
		}
	}
	return returnValue;
}

/*
 * gets all applicable translators
 *
 * for import, you should call this after setLocation; otherwise, you'll just get
 * a list of all import filters, not filters equipped to handle a specific file
 *
 * this returns a list of translator objects, of which the following fields
 * are useful:
 *
 * translatorID - the GUID of the translator
 * label - the name of the translator
 * itemType - the type of item this scraper says it will scrape
 */
Zotero.Translate.prototype.getTranslators = function() {
	// do not allow simultaneous instances of getTranslators
	if(this._translatorSearch) this._translatorSearch.running = false;
	
	var translators = Zotero.Translators.getAllForType(this.type);
	
	// create a new sandbox
	this._generateSandbox();
	this._setSandboxMode("detect");
	
	var possibleTranslators = new Array();
	Zotero.debug("Translate: Searching for translators for "+(this.path ? this.path : "an undisclosed location"), 3);
	
	// see which translators can translate
	this._translatorSearch = new Zotero.Translate.TranslatorSearch(this, translators);
	
	if(this._translatorSearch.asyncMode) {	
		// erroring should call complete
		var me = this;
		this.error = function(value, error) { me._translatorSearch.complete(value, error); };
	} else {
		// return translators if synchronous
		return this._translatorSearch.foundTranslators;
	}
}

/*
 * loads a translator into a sandbox
 */
Zotero.Translate.prototype._loadTranslator = function() {
	if(!this._sandbox || this.type == "search") {
		// create a new sandbox if none exists, or for searching (so that it's
		// bound to the correct url)
		this._generateSandbox();
	}
	this._setSandboxMode("translate");
	
	// parse detect code for the translator
	if(!this._parseCode(this.translator[0])) {
		this._translationComplete(false, "parse error");
		return false;
	}
	
	return true;
}

/**
 * does the actual translation
 *
 * @param 	{NULL|Integer|FALSE}	[libraryID=null]		Library in which to save items,
 *																or NULL for default library;
 *																if FALSE, don't save items
 * @param 	{Boolean}				[saveAttachments=true]	Exclude attachments (e.g., snapshots) on import
 */
Zotero.Translate.prototype.translate = function(libraryID, saveAttachments) {
	/*
	 * initialize properties
	 */
	this.newItems = new Array();
	this.newCollections = new Array();
	this._IDMap = new Array();
	this._complete = false;
	this._itemsDone = false;
	
	if(!this.translator || !this.translator.length) {
		throw("cannot translate: no translator specified");
	}
	
	if(!this.location && this.type != "search" && this.type != "export" && !this._storage) {
		// searches operate differently, because we could have an array of
		// translators and have to go through each
		throw("cannot translate: no location specified");
	}
	
	if(libraryID === false) {
		this.libraryID = false;
	}
	else if(libraryID === true || libraryID == undefined) {
		this.libraryID = null;
	}
	else {
		this.libraryID = libraryID;
	}
	this.saveAttachments = !(saveAttachments === false);
	this.saveFiles = this.saveAttachments;
	
	// If group filesEditable==false, don't save attachments
	if (typeof this.libraryID == 'number') {
		var type = Zotero.Libraries.getType(this.libraryID);
		switch (type) {
			case 'group':
				var groupID = Zotero.Groups.getGroupIDFromLibraryID(this.libraryID);
				var group = Zotero.Groups.get(groupID);
				if (!group.filesEditable) {
					this.saveFiles = false;
				}
				break;
		}
	}
	
	// erroring should end
	var me = this;
	this.error = function(value, error) { me._translationComplete(value, error); }
	
	if(!this._loadTranslator()) {
		return;
	}
	
	if(this._setDisplayOptions) {
		this.displayOptions = this._setDisplayOptions;
	}
	
	if(this._storage) {
		// enable reading from storage, which we can't do until the translator
		// is loaded
		this._storageFunctions(true);
	}
	
	var returnValue;
	if(this.type == "web") {
		returnValue = this._web();
	} else if(this.type == "import") {
		returnValue = this._import();
	} else if(this.type == "export") {
		returnValue = this._export();
	} else if(this.type == "search") {
		returnValue = this._search();
	}
	
	if(returnValue && !this.waitForCompletion) {
		// if synchronous, call _translationComplete();
		this._translationComplete(true);
	}
}

/*
 * parses translator detect code
 */
Zotero.Translate.prototype._parseCode = function(translator) {
	this.configOptions = {};
	this.displayOptions = {};
	
	Zotero.debug("Translate: Parsing code for "+translator.label, 4);
	
	try {
		Components.utils.evalInSandbox("var translatorInfo = "+translator.code, this._sandbox);
		return true;
	} catch(e) {
		if('function' == typeof translator.logError) {
			translator.logError(e.toString());
		} else {
			Components.utils.reportError(e);
		}
		this._debug(e+' in parsing code for '+translator.label, 3);
		return false;
	}
}

/*
 * generates a sandbox for scraping/scraper detection
 */
Zotero.Translate.prototype._generateSandbox = function() {
	var me = this;
	
	// get sandbox URL
	var sandboxLocation = "http://www.example.com/";
	if(this.type == "web") {
		// use real URL, not proxied version, to create sandbox
		sandboxLocation = this.document.defaultView;
		Zotero.debug("Translate: Binding sandbox to "+this.document.location.href, 4);
	} else {
		if (this.type == "search") {
			// generate sandbox for search by extracting domain from translator target
			if(this.translator && this.translator[0] && this.translator[0].target) {
				// so that web translators work too
				const searchSandboxRe = /^http:\/\/[\w.]+\//;
				var tempURL = this.translator[0].target.replace(/\\/g, "").replace(/\^/g, "");
				var m = searchSandboxRe.exec(tempURL);
				if(m) sandboxLocation = m[0];
			}
		}
		Zotero.debug("Translate: Binding sandbox to "+sandboxLocation, 4);
	}
	
	// set up sandbox
	this._sandbox = new Components.utils.Sandbox(sandboxLocation);
	Components.utils.evalInSandbox("var Zotero = {};"+
	"Zotero.Item = function (itemType) {"+
			"this.itemType = itemType;"+
			"this.creators = [];"+
			"this.notes = [];"+
			"this.tags = [];"+
			"this.seeAlso = [];"+
			"this.attachments = [];"+
	"};"+
	"Zotero.Item.prototype.complete = function() { Zotero._itemDone(this); };", this._sandbox)
	
	// add utilities
	this._sandbox.Zotero.Utilities = new Zotero.Utilities.Translate(this);
	this._sandbox.Zotero.Utilities.HTTP = this._sandbox.Zotero.Utilities;
	
	if(this.type == "export") {
		// add routines to retrieve items and collections
		this._sandbox.Zotero.nextItem = function() { return me._exportGetItem() };
		this._sandbox.Zotero.nextCollection = function() { return me._exportGetCollection() }
	} else {
		// copy routines to add new items
		this._sandbox.Zotero._itemDone = function(a) {me._itemDone(a)};
		
		if(this.type == "import") {
			// add routines to add new collections
			this._sandbox.Zotero.Collection = Zotero.Translate.GenerateZoteroCollectionClass();
			// attach the function to be run when a collection is done
			this._sandbox.Zotero.Collection.prototype.complete = function() {me._collectionDone(this)};
		} else if(this.type == "web" || this.type == "search") {	
			// set up selectItems handler
			this._sandbox.Zotero.selectItems = function(options) { return me._selectItems(options) };
		}
	}
	
	this._sandbox.XPathResult = Components.interfaces.nsIDOMXPathResult;
	
	// for debug messages
	this._sandbox.Zotero.debug = function(string) {me._debug(string, 4)};
	
	// for adding configuration options
	this._sandbox.Zotero.configure = function(option, value) {me._configure(option, value) };
	// for adding displayed options
	this._sandbox.Zotero.addOption = function(option, value) {me._addOption(option, value) };
	// for getting the value of displayed options
	this._sandbox.Zotero.getOption = function(option) { return me._getOption(option) };

	// for loading other translators and accessing their methods
	this._sandbox.Zotero.loadTranslator = function(type) {
		if(typeof type !== "string") {
			Zotero.debug("loadTranslator: type must be a string");
			return;
		}
		
		var translation = new Zotero.Translate(type);
		translation._parentTranslator = me;
		
		if(type == "export" && (me.type == "web" || me.type == "search")) {
			throw("for security reasons, web and search translators may not call export translators");
		}
		
		// for security reasons, safeTranslator wraps the translator object.
		// note that setLocation() is not allowed
		var safeTranslator = new Object();
		safeTranslator.setSearch = function(arg) { return translation.setSearch(arg) };
		safeTranslator.setDocument = function(arg) { return translation.setDocument(arg) };
		safeTranslator.setHandler = function(arg1, arg2) {
			translation.setHandler(arg1, 
				function(obj, item) {
					try {
						arg2(obj, item);
					} catch(e) {
						me.error(false, e);
					}
				}
			);
		};
		safeTranslator.setString = function(arg) { translation.setString(arg) };
		safeTranslator.setTranslator = function(arg) { return translation.setTranslator(arg) };
		safeTranslator.getTranslators = function() { return translation.getTranslators() };
		safeTranslator.translate = function() {
			var noHandlers = true;
			for(var i in translation._handlers) {
				noHandlers = false;
				break;
			}
			if(noHandlers) {
				if(type != "export") {
					translation.setHandler("itemDone", function(obj, item) { item.complete() });
				}
				if(type == "web") {
					translation.setHandler("selectItems", me._handlers["selectItems"]);
				}
			}
			return translation.translate(false);
		};
		safeTranslator.getTranslatorObject = function() {
			// load the translator into our sandbox
			translation._loadTranslator();
			// initialize internal IO
			translation._initializeInternalIO();
			
			var noHandlers = true;
			for(var i in translation._handlers) {
				noHandlers = false;
				break;
			}
			if(noHandlers) {
				if(type != "export") {
					translation.setHandler("itemDone", function(obj, item) { item.complete() });
				}
				if(type == "web") {
					translation.setHandler("selectItems", me._handlers["selectItems"]);
				}
			}
			
			// return sandbox
			return translation._sandbox;
		};
		
		return safeTranslator;
	}
}

/*
 * Adds appropriate methods for detect/translate modes
 */
Zotero.Translate.prototype._setSandboxMode = function(mode) {
	var me = this;
	
	// erase waitForCompletion status and done function
	this.waitForCompletion = false;
	this._sandbox.Zotero.done = undefined;
	
	if(mode == "detect") {
		// for asynchronous operation, use wait()
		// done() is implemented after wait() is called
		this._sandbox.Zotero.wait = function() { me._enableAsynchronousDetect() };
	} else {
		// for asynchronous operation, use wait()
		// done() is implemented after wait() is called
		this._sandbox.Zotero.wait = function() { me._enableAsynchronousTranslate() };
	}
}

/*
 * sets an option that modifies the way the translator is executed
 * 
 * called as configure() in translator detectCode
 *
 * current options:
 *
 * dataMode
 *   valid: import, export
 *   options: rdf, block, line
 *   purpose: selects whether write/read behave as standard text functions or
 *            use an RDF data source
 *
 * getCollections
 *   valid: export
 *   options: true, false
 *   purpose: selects whether export translator will receive an array of
 *            collections and children in addition to the array of items and
 *            children
 */
Zotero.Translate.prototype._configure = function(option, value) {
	if(typeof option !== "string") {
		Zotero.debug ("configure: option must be a string");
		return;
	}
	if(typeof value === "object" && value !== null) {
		value = new XPCSafeJSObjectWrapper(value);
	}
	
	this.configOptions[option] = value;
	Zotero.debug("Translate: Setting configure option "+option+" to "+value, 4);
}

/*
 * adds translator options to be displayed in a dialog
 *
 * called as addOption() in detect code
 *
 * current options are exportNotes and exportFileData
 */
Zotero.Translate.prototype._addOption = function(option, value) {
	if(typeof option !== "string") {
		Zotero.debug ("addOption: option must be a string");
		return;
	}
	if(typeof value === "object" && value !== null) {
		value = new XPCSafeJSObjectWrapper(value);
	}
	
	this.displayOptions[option] = value;
	Zotero.debug("Translate: Setting display option "+option+" to "+value, 4);
}

/*
 * gets translator options that were displayed in a dialog
 *
 * called as getOption() in detect code
 *
 */
Zotero.Translate.prototype._getOption = function(option) {
	if(typeof option !== "string") {
		Zotero.debug ("getOption: option must be a string");
		return;
	}
	return this.displayOptions[option];
}

/*
 * makes translation API wait until done() has been called from the translator
 * before executing _translationComplete
 * 
 * called as wait() in translator code
 */
Zotero.Translate.prototype._enableAsynchronousDetect = function() {
	var me = this;
	this.waitForCompletion = true;
	this._sandbox.Zotero.done = function(arg) {
		if(arg !== null && typeof arg === "object") {
			Zotero.debug("done: argument must not be an object");
			me._translatorSearch.complete(false);
		} else {
			me._translatorSearch.complete(arg);
		}
	};
}

Zotero.Translate.prototype._enableAsynchronousTranslate = function() {
	var me = this;
	this.waitForCompletion = true;
	this._sandbox.Zotero.done = function(val, error) {
		if(val !== null && typeof val === "object") {
			val = new XPCSafeJSObjectWrapper(val);
		}
		if(error !== null && typeof error === "object") {
			error = new XPCSafeJSObjectWrapper(error);
		}
		me._translationComplete(val == undefined ? true : val, (error ? error : ""))
	};
}

/*
 * lets user pick which items s/he wants to put in his/her library
 * 
 * called as selectItems() in translator code
 */
Zotero.Translate.prototype._selectItems = function(options) {
	if(!options instanceof XPCSafeJSObjectWrapper) {
		options = new XPCSafeJSObjectWrapper(options);
	}
	
	// hack to see if there are options
	var haveOptions = false;
	for(var i in options) {
		haveOptions = true;
		break;
	}
	
	if(!haveOptions) {
		throw "translator called select items with no items";
	}
	
	if(this._handlers.select) {
		return this.runHandler("select", options);
	} else {	// no handler defined; assume they want all of them
		return options;
	}
}

/*
 * executed on translator completion, either automatically from a synchronous
 * scraper or as done() from an asynchronous scraper
 *
 * finishes things up and calls callback function(s)
 */
Zotero.Translate.prototype._translationComplete = function(returnValue, error) {
	// to make sure this isn't called twice
	if(returnValue === undefined) {
		returnValue = this._itemsDone;
	}
	
	if(!this._complete) {
		this._complete = true;
		
		if(this.type == "search" && !this._itemsDone) {
			// if we're performing a search and didn't get any results, go on
			// to the next translator
			Zotero.debug("Translate: Could not find a result using "+this.translator[0].label+": \n"
			              +this._generateErrorString(error), 3);
			if(this.translator.length > 1) {
				this.translator.shift();
				this.translate();
				return;
			}
			
			returnValue = false;
		} else {
			// close open streams
			this._closeStreams();
			
			if(!returnValue) {
				// report error to console
				if(typeof this.translator[0] != "undefined" && 'function' == typeof this.translator[0].logError) {
					this.translator[0].logError(error.toString(), "exception");
				} else {
					Components.utils.reportError(error);
				}
				
				// report error to debug log
				var errorString = this._generateErrorString(error);
				this._debug("Translation using "+(this.translator && this.translator[0] && this.translator[0].label ? this.translator[0].label : "no translator")+" failed: \n"+errorString, 2);
				
				// report translation error for webpages to mothership
				if(this.type == "web") this._reportTranslationFailure(errorString);
				
				this.runHandler("error", error);
			} else {
				this._debug("Translation successful");
			}
		}		
		
		// call handlers
		this.runHandler("done", returnValue);
	}
}

/*
 * generates a useful error string, for submitting and debugging purposes
 */
Zotero.Translate.prototype._generateErrorString = function(error) {
	var errorString = "";
	if(typeof(error) == "string") {
		errorString = "\nthrown exception => "+error;
	} else {
		for(var i in error) {
			if(typeof(error[i]) != "object") {
				errorString += "\n"+i+' => '+error[i];
			}
		}
	}
	
	errorString += "\nurl => "+this.path
		+ "\ndownloadAssociatedFiles => "+Zotero.Prefs.get("downloadAssociatedFiles")
		+ "\nautomaticSnapshots => "+Zotero.Prefs.get("automaticSnapshots");
	return errorString.substr(1);
}

/*
 * runs an HTTP request to report a translation error
 */
Zotero.Translate.prototype._reportTranslationFailure = function(errorData) {
	if(this.translator[0].inRepository && Zotero.Prefs.get("reportTranslationFailure")) {
		var postBody = "id=" + encodeURIComponent(this.translator[0].translatorID) +
					   "&lastUpdated=" + encodeURIComponent(this.translator[0].lastUpdated) +
					   "&diagnostic=" + encodeURIComponent(Zotero.getSystemInfo()) +
					   "&errorData=" + encodeURIComponent(errorData);
		Zotero.Utilities.HTTP.doPost("http://www.zotero.org/repo/report", postBody);
	}
}

/*
 * closes open file streams, if any exist
 */
Zotero.Translate.prototype._closeStreams = function() {
	// serialize RDF and unregister dataSource
	if(this._rdf) {
		if(this.type == "export") {
			// initialize serializer and add prefixes
			var serializer = Serializer();
			for(var prefix in this._rdf.namespaces) {
				serializer.suggestPrefix(prefix, this._rdf.namespaces[prefix]);
			}
			
			// serialize in appropriate format
			if(this.configOptions.dataMode == "rdf/n3") {
				var output = serializer.statementsToN3(this._rdf.statements);
			} else {
				var output = serializer.statementsToXML(this._rdf.statements);
			}
			
			if(this._streams.length) {
				Zotero.debug("writing to stream");
				// write serialized data to file
				var intlStream = Components.classes["@mozilla.org/intl/converter-output-stream;1"]
									   .createInstance(Components.interfaces.nsIConverterOutputStream);
				intlStream.init(this._streams[0], "UTF-8", 4096, "?".charCodeAt(0));
				this._streams.push(intlStream);
				intlStream.writeString(output);
			} else {
				this.output = output;
			}
		}
		
		delete this._rdf;
	}
	
	if(this._streams.length) {
		for(var i in this._streams) {
			var stream = this._streams[i];
			
			// stream could be either an input stream or an output stream
			try {
				stream.QueryInterface(Components.interfaces.nsIFileInputStream);
			} catch(e) {
				try {
					stream.QueryInterface(Components.interfaces.nsIFileOutputStream);
				} catch(e) {
				}
			}
			
			// encase close in try block, because it's possible it's already
			// closed
			try {
				stream.close();
			} catch(e) {
			}
		}
	}
	
	delete this._streams;
	this._streams = new Array();
	this._inputStream = null;
}

/*
 * handles tags and see also data for notes and attachments
 */
Zotero.Translate.prototype._itemTagsAndSeeAlso = function(item, newItem) {
	// add to ID map
	if(item.itemID) {
		this._IDMap[item.itemID] = newItem.id;
	}
	// add see alsos
	if(item.seeAlso) {
		for each(var seeAlso in item.seeAlso) {
			if(this._IDMap[seeAlso]) {
				newItem.addRelatedItem(this._IDMap[seeAlso]);
			}
		}
		newItem.save();
	}
	if(item.tags) {
		var tagsToAdd = {};
		tagsToAdd[0] = []; // user tags
		tagsToAdd[1] = []; // automatic tags
		
		for each(var tag in item.tags) {
			if(typeof(tag) == "string") {
				// accept strings in tag array as automatic tags, or, if
				// importing, as non-automatic tags
				if (this.type == 'import') {
					tagsToAdd[0].push(tag);
				}
				else {
					tagsToAdd[1].push(tag);
				}
			} else if(typeof(tag) == "object") {
				// also accept objects
				if(tag.tag) {
					if (!tagsToAdd[tag.type]) {
						tagsToAdd[tag.type] = [];
					}
					tagsToAdd[tag.type].push(tag.tag);
				}
			}
		}
		
		for (var type in tagsToAdd) {
			if (tagsToAdd[type].length) {
				newItem.addTags(tagsToAdd[type], type);
			}
		}
	}
}

/*
 * executed when an item is done and ready to be loaded into the database
 */
Zotero.Translate._urlRe = /(([A-Za-z]+):\/\/[^\s]*)/i;
Zotero.Translate.prototype._itemDone = function(item, attachedTo) {
	// warn if itemDone called after translation completed
	if(this._complete) {
		Zotero.debug("Translate: WARNING: Zotero.Item.complete() called after Zotero.done(); please fix your code", 2);
	}
	
	// make item safe to access
	if(!item instanceof XPCSafeJSObjectWrapper) {
		item = new XPCSafeJSObjectWrapper(item);
	}
	
	if(this.type == "web") {
		// store library catalog if this item was captured from a website, and
		// libraryCatalog is truly undefined (not false or "")
		if(typeof item.repository != 'undefined') {
			Zotero.debug("Translate: 'repository' field is now 'libraryCatalog'; please fix your code", 2);
			item.libraryCatalog = item.repository;
			delete item.repository;
		}
		
		if(typeof item.libraryCatalog == 'undefined') {
			item.libraryCatalog = this.translator[0].label;
		}
	}
	
	this._itemsDone = true;
	
	// if we're not supposed to save the item or we're in a child translator,
	// just return the item array
	if(this.libraryID === false || this._parentTranslator) {
		
		// if a parent sandbox exists, use complete() function from that sandbox
		if(this._parentTranslator) {
			var pt = this._parentTranslator;
			item.complete = function() { pt._itemDone(this) };
			Zotero.debug("Translate: Calling done from parent sandbox", 4);
		}
		this.runHandler("itemDone", item);
		return;
	}
	
	// Get typeID, defaulting to "webpage"
	var type = (item.itemType ? item.itemType : "webpage");
	
	if(type == "note") {	// handle notes differently
		var newItem = new Zotero.Item('note');
		newItem.libraryID = this.libraryID ? this.libraryID : null;
		newItem.setNote(item.note);
		var myID = newItem.save();
		// re-retrieve the item
		var newItem = Zotero.Items.get(myID);
	} else {
		if(!item.title && this.type == "web") {
			throw("item has no title");
		}
		
		// if item was accessed through a proxy, ensure that the proxy
		// address remains in the accessed version
		if(this.locationIsProxied && item.url) {
			item.url = Zotero.Proxies.properToProxy(item.url);
		}
		
		// create new item
		if(type == "attachment") {
			if(this.type != "import") {
				Zotero.debug("Translate: Discarding standalone attachment", 2);
				return;
			}
			
			Zotero.debug("Translate: Adding attachment", 4);
			
			if(!item.url && !item.path) {
				Zotero.debug("Translate: Ignoring attachment: no path or URL specified", 2);
				return;
			}
			
			if(!item.path) {
				// see if this is actually a file URL
				var m = Zotero.Translate._urlRe.exec(item.url);
				var protocol = m ? m[2].toLowerCase() : "";
				if(protocol == "file") {
					item.path = item.url;
					item.url = false;
				} else if(protocol != "http" && protocol != "https") {
					Zotero.debug("Translate: Unrecognized protocol "+protocol, 2);
					return;
				}
			}
			
			if(!item.path) {
				// create from URL
				try {
					var myID = Zotero.Attachments.linkFromURL(item.url, attachedTo,
							(item.mimeType ? item.mimeType : undefined),
							(item.title ? item.title : undefined));
				} catch(e) {
					Zotero.debug("Translate: Error adding attachment "+item.url, 2);
					return;
				}
				Zotero.debug("Translate: Created attachment; id is "+myID, 4);
				var newItem = Zotero.Items.get(myID);
			} else {
				var uri, file;
				
				// generate nsIFile
				var IOService = Components.classes["@mozilla.org/network/io-service;1"].
								getService(Components.interfaces.nsIIOService);
				try {
					var uri = IOService.newURI(item.path, "", null);
				}
				catch (e) {
					Components.utils.reportError("Error parsing attachment path: " + item.path);
					Zotero.debug("Translate: Error parsing attachment path '" + item.path + "'", 2);
				}
				
				if (uri) {
					var file = uri.QueryInterface(Components.interfaces.nsIFileURL).file;
					
					if (file.path == '/') {
						Components.utils.reportError("Error parsing attachment path: " + item.path);
						Zotero.debug("Translate: Error parsing attachment attachment '" + item.path + "'", 2);
					}
				}
				
				if (!file || !file.exists()) {
					// use item title if possible, or else file leaf name
					var title = item.title;
					if(!title) {
						title = file ? file.leafName : '';
					}
{'fraction_non_alphanumeric': 0.1280966132843266, 'fraction_numerical': 0.003496980834864794, 'mean_word_length': 3.4556428972487367, 'pattern_counts': {'":': 28, '<': 13, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4726313', 'n_tokens_mistral': 33756, 'n_tokens_neox': 29449, 'n_words': 8042}