<commit_before>var async = require('async'),
	fileType = require('file-type'),
	EventEmitter2 = require('eventemitter2').EventEmitter2,
	_ = require('underscore'),
	nodeExtend = require('node.extend'),
	fs = require('fs-extra'),
	path = require('path'),
	nconf = require('nconf'),

	utils = require('../public/js/utils.js'),
	Data = require('./data.js'),

	MAX_INT = -1 >>> 1,

	Groups = require('../../../src/groups.js'),
	Favourites = require('../../../src/favourites.js'),
	privileges = require('../../../src/privileges.js'),
	Meta = require('../../../src/meta.js'),
	User = require('../../../src/user.js'),
	Messaging = require('../../../src/messaging.js'),
	File = require('../../../src/file.js'),
	Topics = require('../../../src/topics.js'),
	Posts = require('../../../src/posts.js'),
	Categories = require('../../../src/categories.js'),
	db = module.parent.require('../../../src/database.js'),

	IMPORT_BATCH_SIZE = 10,
	FLUSH_BATCH_SIZE = 10,

//todo use the real one
	LOGGEDIN_UID = 1,

	logPrefix = '[nodebb-plugin-import]',

	BACKUP_CONFIG_FILE = path.join(__dirname, '/tmp/importer.nbb.backedConfig.json'),

	DIRTY_GROUPS_FILE = path.join(__dirname, '/tmp/importer.dirty.groups'),
	DIRTY_USERS_FILE = path.join(__dirname, '/tmp/importer.dirty.users'),
	DIRTY_MESSAGES_FILE = path.join(__dirname, '/tmp/importer.dirty.messages'),
	DIRTY_CATEGORIES_FILE = path.join(__dirname, '/tmp/importer.dirty.categories'),
	DIRTY_TOPICS_FILE = path.join(__dirname, '/tmp/importer.dirty.topics'),
	DIRTY_POSTS_FILE = path.join(__dirname, '/tmp/importer.dirty.posts'),
	DIRTY_VOTES_FILE = path.join(__dirname, '/tmp/importer.dirty.votes'),
	DIRTY_BOOKMARKS_FILE = path.join(__dirname, '/tmp/importer.dirty.bookmarks'),

	areGroupsDirty,
	areUsersDirty,
	areMessagesDirty,
	areCategoriesDirty,
	areTopicsDirty,
	arePostsDirty,
	areVotesDirty,
	areBookmarksDirty,

	isAnythingDirty,

	alreadyImportedAllGroups = false,
	alreadyImportedAllUsers = false,
	alreadyImportedAllMessages = false,
	alreadyImportedAllCategories = false,
	alreadyImportedAllTopics = false,
	alreadyImportedAllPosts = false,
	alreadyImportedAllVotes = false,
	alreadyImportedAllBookmarks = false,

	flushed = false,

	defaults = {
		log: true,
		passwordGen: {
			enabled: false,
			chars: '{}.-_=+qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM1234567890',
			len: 13
		},
		categoriesTextColors: ['#FFFFFF'],
		categoriesBgColors: ['#AB4642', '#DC9656', '#F7CA88', '#A1B56C', '#86C1B9', '#7CAFC2', '#BA8BAF', '#A16946'],
		categoriesIcons: ['fa-comment'],
		autoConfirmEmails: true,
		userReputationMultiplier: 1,

		adminTakeOwnership: {
			enable: false,
			_username: null,
			_uid: null
		},

		nbbTmpConfig: {
			maximumPostLength: MAX_INT,
			maximumChatMessageLength: MAX_INT,
			maximumTitleLength: MAX_INT,
			maximumUsernameLength: MAX_INT,
			postDelay: 0,
			initialPostDelay: 0,
			newbiePostDelay: 0,
			minimumPostLength: 0,
			minimumPasswordLength: 0,
			minimumTitleLength: 0,
			requireEmailConfirmation: 0,
			trackIpPerPost: 1,
			"newbiePostDelayThreshold": 0,
			"minimumTagsPerTopic": 0,
			"maximumTagsPerTopic": MAX_INT,
			"allowGuestSearching": 1,
			"allowTopicsThumbnail": 1,
			"registrationType": "normal",
			"allowLocalLogin": 1,
			"allowAccountDelete": 1,
			"allowFileUploads": 1,
			"allowUserHomePage": 1,
			"maximumFileSize": MAX_INT,
			"minimumUsernameLength": 1,
			"maximumSignatureLength": MAX_INT,
			"maximumAboutMeLength": MAX_INT,
			"maximumProfileImageSize": MAX_INT,
			"maximumCoverImageSize": MAX_INT,
			"profileImageDimension": 128,
			"profile:allowProfileImageUploads": 1,
			"reputation:disabled": 0,
			"downvote:disabled": 0
		}
	};

(function(Importer) {

	var coolDownFn = function (timeout) {
		return function (next) {
			timeout = timeout || 5000;
			Importer.log('cooling down for ' + timeout/1000 + ' seconds');
			setTimeout(next, timeout);
		};
	};

	var groupsJoin = function (groupName, uid, timestamp, callback) {
		Groups.join(groupName, uid, function(err, ret) {
			if (err) {
				return callback(err);
			}
			db.sortedSetAdd('group:' + groupName + ':members', timestamp, uid, function (err) {
				if (err) {
					return callback(err);
				}
				Importer.log(uid, 'joined group:', groupName);
				callback(null, ret);
			});
		});
	};

	Importer._dispatcher = new EventEmitter2({
		wildcard: true
	});

	Importer.init = function(exporter, config, callback) {
		Importer.setup(exporter, config, callback);
	};

	Importer.setup = function(exporter, config, callback) {
		Importer.exporter = exporter;

		Importer._config = nodeExtend(true, {}, defaults, config && config.importer ? config.importer : config || {});

		//todo I don't like this
		Importer._config.serverLog = !!config.log.server;
		Importer._config.clientLog = !!config.log.client;
		Importer._config.verbose = !!config.log.verbose;

		Importer.emit('importer.setup.start');

		flushed = false;

		Importer.emit('importer.setup.done');
		Importer.emit('importer.ready');
		if (_.isFunction(callback)) {
			callback();
		}
	};

	function start (flush, callback) {
		Importer.emit('importer.start');

		var series = [];

		if (flush) {
			series.push(Importer.flushData);
		} else {
			Importer.log('Skipping data flush');
			series.push(makeDirty);
		}
		async.series(series.concat([
			Importer.backupConfig,
			Importer.setTmpConfig,
			Importer.importGroups,
			coolDownFn(5000),
			Importer.importCategories,
			Importer.allowGuestsOnAllCategories,
			Importer.importUsers,
			Importer.importMessages,
			Importer.importTopics,
			Importer.importPosts,
			Importer.importVotes,
			Importer.importBookmarks,
			Importer.fixCategoriesParentsAndAbilities,
			Importer.fixPostsToPids,
			Importer.fixGroupsOwners,
			Importer.relockUnlockedTopics,
			Importer.rebanAndMarkReadForUsers,
			Importer.fixTopicTimestamps,
			Importer.restoreConfig,
			Importer.disallowGuestsWriteOnAllCategories,
			Importer.teardown
		]), callback);
	}

	Importer.start = function(callback) {
		var config = Importer.config();
		start(config.flush, callback);
	};

	Importer.resume = function(callback) {
		Importer.emit('importer.start');
		Importer.emit('importer.resume');

		Importer.isDirty();

		var series = [];
		if (! alreadyImportedAllGroups) {
			series.push(Importer.importGroups);
		} else {
			Importer.warn('alreadyImportedAllGroups=true, skipping importGroups Phase');
		}

		if (! alreadyImportedAllCategories) {
			series.push(Importer.importCategories);
			series.push(Importer.allowGuestsOnAllCategories);
		} else {
			Importer.warn('alreadyImportedAllCategories=true, skipping importCategories Phase');
		}
		if (! alreadyImportedAllUsers) {
			series.push(Importer.importUsers);
		} else {
			Importer.warn('alreadyImportedAllUsers=true, skipping importUsers Phase');
		}
		if (! alreadyImportedAllMessages) {
			series.push(Importer.importMessages);
		} else {
			Importer.warn('alreadyImportedAllMessages=true, skipping importMessages Phase');
		}

		if (! alreadyImportedAllTopics) {
			series.push(Importer.importTopics);
		} else {
			Importer.warn('alreadyImportedAllTopics=true, skipping importTopics Phase');
		}
		if (! alreadyImportedAllPosts) {
			series.push(Importer.importPosts);
		} else {
			Importer.warn('alreadyImportedAllPosts=true, skipping importPosts Phase');
		}

		if (! alreadyImportedAllVotes) {
			series.push(Importer.importVotes);
		} else {
			Importer.warn('alreadyImportedAllVotes=true, skipping importVotes Phase');
		}

		if (! alreadyImportedAllBookmarks) {
			series.push(Importer.importBookmarks);
		} else {
			Importer.warn('alreadyImportedAllBookmarks=true, skipping importBookmarks Phase');
		}

		series.concat([
			Importer.fixCategoriesParentsAndAbilities,
			Importer.fixPostsToPids,
			Importer.fixGroupsOwners,
			Importer.relockUnlockedTopics,
			Importer.rebanAndMarkReadForUsers,
			Importer.fixTopicTimestamps,
			Importer.restoreConfig,
			Importer.disallowGuestsWriteOnAllCategories,
			Importer.teardown
		]);

		async.series(series, callback);
	};

	function makeDirty (done) {

		areGroupsDirty = true;
		areUsersDirty = true;
		areMessagesDirty = true;
		areCategoriesDirty = true;
		areTopicsDirty = true;
		arePostsDirty = true;
		areVotesDirty = true;
		areBookmarksDirty = true;
		isAnythingDirty = true;

		alreadyImportedAllGroups = false;
		alreadyImportedAllUsers = false;
		alreadyImportedAllMessages = false;
		alreadyImportedAllCategories = false;
		alreadyImportedAllTopics = false;
		alreadyImportedAllPosts = false;
		alreadyImportedAllVotes = false;
		alreadyImportedAllBookmarks = false;

		flushed = false;

		return _.isFunction(done) ? done(null, true) : true;
	}

	// todo: really? wtf is this logic
	Importer.isDirty = function(done) {

		areGroupsDirty = !! fs.existsSync(DIRTY_GROUPS_FILE);
		areVotesDirty = !! fs.existsSync(DIRTY_VOTES_FILE);
		areUsersDirty = !! fs.existsSync(DIRTY_USERS_FILE);
		areMessagesDirty = !! fs.existsSync(DIRTY_MESSAGES_FILE);
		areCategoriesDirty = !! fs.existsSync(DIRTY_CATEGORIES_FILE);
		areTopicsDirty = !! fs.existsSync(DIRTY_TOPICS_FILE);
		arePostsDirty = !! fs.existsSync(DIRTY_POSTS_FILE);
		areBookmarksDirty = !! fs.existsSync(DIRTY_BOOKMARKS_FILE);

		isAnythingDirty =
			areGroupsDirty
			|| areVotesDirty
			|| areUsersDirty
			|| areCategoriesDirty
			|| areTopicsDirty
			|| arePostsDirty
			|| areMessagesDirty
			|| areBookmarksDirty;

		// order in start() and resume() matters and must be in sync
		if (areGroupsDirty) {
			alreadyImportedAllGroups = false;
			alreadyImportedAllCategories = false;
			alreadyImportedAllUsers = false;
			alreadyImportedAllMessages = false;
			alreadyImportedAllTopics = false;
			alreadyImportedAllPosts = false;
			alreadyImportedAllVotes = false;
			alreadyImportedAllBookmarks = false;
		} else if (areCategoriesDirty) {
			alreadyImportedAllGroups = true;
			alreadyImportedAllCategories = false;
			alreadyImportedAllUsers = false;
			alreadyImportedAllMessages = false;
			alreadyImportedAllTopics = false;
			alreadyImportedAllPosts = false;
			alreadyImportedAllVotes = false;
			alreadyImportedAllBookmarks = false;
		} else if (areUsersDirty) {
			alreadyImportedAllGroups = true;
			alreadyImportedAllCategories = true;
			alreadyImportedAllUsers = false;
			alreadyImportedAllMessages = false;
			alreadyImportedAllTopics = false;
			alreadyImportedAllPosts = false;
			alreadyImportedAllVotes = false;
			alreadyImportedAllBookmarks = false;
		} else if (areMessagesDirty) {
			alreadyImportedAllGroups = true;
			alreadyImportedAllCategories = true;
			alreadyImportedAllUsers = true;
			alreadyImportedAllMessages = false;
			alreadyImportedAllTopics = false;
			alreadyImportedAllPosts = false;
			alreadyImportedAllVotes = false;
			alreadyImportedAllBookmarks = false;
		} else if (areTopicsDirty) {
			alreadyImportedAllGroups = true;
			alreadyImportedAllCategories = true;
			alreadyImportedAllUsers = true;
			alreadyImportedAllMessages = true;
			alreadyImportedAllTopics = false;
			alreadyImportedAllPosts = false;
			alreadyImportedAllVotes = false;
			alreadyImportedAllBookmarks = false;
		} else if (arePostsDirty) {
			alreadyImportedAllGroups = true;
			alreadyImportedAllCategories = true;
			alreadyImportedAllUsers = true;
			alreadyImportedAllMessages = true;
			alreadyImportedAllTopics = true;
			alreadyImportedAllPosts = false;
			alreadyImportedAllVotes = false;
			alreadyImportedAllBookmarks = false;
		} else if (areVotesDirty) {
			alreadyImportedAllGroups = true;
			alreadyImportedAllCategories = true;
			alreadyImportedAllUsers = true;
			alreadyImportedAllMessages = true;
			alreadyImportedAllTopics = true;
			alreadyImportedAllPosts = true;
			alreadyImportedAllVotes = false;
			alreadyImportedAllBookmarks = false;
		} else if (areBookmarksDirty) {
			alreadyImportedAllGroups = true;
			alreadyImportedAllCategories = true;
			alreadyImportedAllUsers = true;
			alreadyImportedAllMessages = true;
			alreadyImportedAllTopics = true;
			alreadyImportedAllPosts = true;
			alreadyImportedAllVotes = true;
			alreadyImportedAllBookmarks = false;
		}

		return _.isFunction(done) ? done(null, isAnythingDirty) : isAnythingDirty;
	};

	Importer.flushData = function(next) {
		async.series([
			function(done){
				Importer.phase('purgeCategories+Topics+Bookmarks+Posts+VotesStart');
				Importer.progress(0, 1);

				// that will delete, categories, topics, topics.bookmarks, posts and posts.votes
				Data.countCategories(function(err, total) {
					var index = 0;
					Data.processCategoriesCidsSet(
						function (err, ids, nextBatch) {
							async.mapLimit(ids, FLUSH_BATCH_SIZE, function(id, cb) {
								Importer.progress(index++, total);
								Categories.purge(id, cb);
							}, nextBatch);
						},
						{alwaysStartAt: 0},
						function(err) {
							if (err) {
								Importer.warn(Importer._phase + " : " + err.message);
							}
							Importer.progress(1, 1);
							Importer.phase('purgeCategories+Topics+Bookmarks+Posts+VotesDone');
							done()
						});
				});

			},
			function(done) {
				Importer.phase('purgeUsersStart');
				Importer.progress(0, 1);

				Data.countUsers(function(err, total) {
					var index = 0; var count = 0;
					Data.processUsersUidsSet(
						function(err, ids, nextBatch) {
							async.mapLimit(ids, FLUSH_BATCH_SIZE, function(uid, cb) {
								Importer.progress(index++, total);
								if (parseInt(uid, 10) === 1) {
									return cb();
								}
								User.delete(uid, function() {
									count++;
									cb();
								});
							}, nextBatch);
						}, {
							// since we're deleting records the range is always shifting backwards, so need to advance the batch start boundary
							alwaysStartAt: 0,
							// done if the uid=1 in the only one in the db
							doneIf: function(start, end, ids) {
								return ids.length === 1;
							}
						},
						function(err) {
							Importer.progress(1, 1);
							Importer.phase('purgeUsersDone');
							done(err)
						}
					);
				});
			},
			function(done) {
				Importer.phase('purgeGroupsStart');
				Importer.progress(0, 1);

				Data.countGroups(function(err, total) {
					var index = 0; var count = 0;
					Data.processGroupsNamesSet(
						function(err, names, nextBatch) {
							async.mapLimit(names, FLUSH_BATCH_SIZE, function(name, cb) {
								Importer.progress(index++, total);

								// skip if system group
								if (name === 'administrators') {
									return cb();
								}
								Groups.destroy(name, function() {
									count++;
									cb();
								});
							}, nextBatch);
						}, {
							// since we're deleting records the range is always shifting backwards, so need to advance the batch start boundary
							alwaysStartAt: 0,

							// done if the administrators group in the only one in the db
							doneIf: function(start, end, names) {
								return !names.length || names.length === 1;
							}
						},
						function(err) {
							Importer.progress(1, 1);
							Importer.phase('purgeGroupsDone');
							done(err)
						}
					);
				});
			},
			function(done) {
				Importer.phase('purgeMessagesStart');
				Importer.progress(0, 1);

				Data.countMessages(function(err, total) {
					var index = 0;
					Data.eachMessage(
						function(message, next) {
							Importer.progress(index++, total);
							var uids = [message.fromuid, message.touid].sort();
							async.parallel([
								function(nxt) {
									db.delete('message:' + message.mid, function () {
										nxt();
									});
								},
								function(nxt) {
									db.sortedSetRemove('messages:uid:' + uids[0] + ':to:' + uids[1], message.mid, function() {
										nxt();
									});
								},
								function(next) {
									db.sortedSetRemove('uid:' + uids[0] + ':chats', uids[1], next);
								},
								function(next) {
									db.sortedSetRemove('uid:' + uids[1] + ':chats', uids[0], next);
								}
							], next);
						},
						{},
						function(err) {
							Importer.progress(1, 1);
							Importer.phase('purgeMessagesDone');
							done(err)
						}
					);
				});
			},
			function(done) {
				flushed = true;

				Importer.phase('resetGlobalsStart');
				Importer.progress(0, 1);

				db.setObject('global', {
					nextUid: 1,
					userCount: 1,
					nextGid: 1,
					groupCount: 1,
					nextMid: 1,
					nextCid: 1,
					categoryCount: 1,
					nextTid: 1,
					topicCount: 1,
					nextPid: 1,
					postCount: 1,
					nextVid: 1,
					voteCount: 1,
					nextBid: 1,
					bookmarkCount: 1
				}, function(err) {
					if (err) {
						return done(err);
					}
					Importer.progress(1, 1);
					Importer.phase('resetGlobalsDone');
					done();
				});
			},
			Importer.deleteTmpImportedSetsAndObjects
		], function(err) {
			if (err) {
				Importer.error(err);
				next(err);
			}
			next();
		});
	};

	Importer.phasePercentage = 0;

	Importer.progress = function(count, total, interval) {
		interval = interval || 0.01;
		var percentage = count / total * 100;
		if (percentage === 0 || percentage >= 100 || (percentage - Importer.phasePercentage >= interval)) {
			Importer.phasePercentage = percentage;
			Importer.emit('importer.progress', {count: count, total: total, percentage: percentage});
		}
	};

	Importer.phase = function(phase, data) {
		Importer.phasePercentage = 0;
		Importer._phase = phase;
		Importer.emit('importer.phase', {phase: phase, data: data, timestamp: +new Date()});
	};

	var recoverImportedGroup = function(_gid, callback) {
		if (! flushed && (alreadyImportedAllGroups || areGroupsDirty)) {
			return Data.getImportedGroup(_gid, callback);
		}
		return callback(null, null);
	};

	var recoverImportedUser = function(_uid, callback) {
		if (! flushed && (alreadyImportedAllUsers || areUsersDirty)) {
			return Data.getImportedUser(_uid, callback);
		}
		return callback(null, null);
	};

	var recoverImportedMessage = function(_mid, callback) {
		if (! flushed && (alreadyImportedAllMessages || areMessagesDirty)) {
			return Data.getImportedMessage(_mid, callback);
		}
		return callback(null, null);
	};

	var recoverImportedCategory = function(_cid, callback) {
		if (! flushed && (alreadyImportedAllCategories || areCategoriesDirty)) {
			return Data.getImportedCategory(_cid, callback);
		}
		return callback(null, null);
	};

	var recoverImportedTopic = function(_tid, callback) {
		if (! flushed && (alreadyImportedAllTopics || areTopicsDirty)) {
			return Data.getImportedTopic(_tid, callback);
		}
		return callback(null, null);
	};

	var recoverImportedPost = function(_pid, callback) {
		if (! flushed && (alreadyImportedAllPosts || arePostsDirty)) {
			return Data.getImportedPost(_pid, callback);
		}
		return callback(null, null);
	};
	var recoverImportedVote = function(_vid, callback) {
		if (! flushed && (alreadyImportedAllVotes || areVotesDirty)) {
			return Data.getImportedVote(_vid, callback);
		}
		return callback(null, null);
	};
	var recoverImportedBookmark = function(_bid, callback) {
		if (! flushed && (alreadyImportedAllBookmarks || areBookmarksDirty)) {
			return Data.getImportedBookmark(_bid, callback);
		}
		return callback(null, null);
	};

	var writeBlob = function(filepath, blob, callback) {
		var buffer = new Buffer(blob, 'binary');

		var ftype = fileType(buffer) || {mime: "unknown/unkown", extension: ""};
		ftype.filepath = filepath;

		fs.writeFile(filepath, buffer.toString('binary'), 'binary', function (err) {
			callback(err, ftype);
		});
	};

	Importer.importUsers = function(next) {
		Importer._lastPercentage = 0;
		Importer.phase('usersImportStart');
		Importer.progress(0, 1);
		var count = 0,
			imported = 0,
			alreadyImported = 0,
			picturesTmpPath = path.join(__dirname, '/tmp/pictures'),
			folder = '_imported_profiles',
			picturesPublicPath = path.join(nconf.get('base_dir'), nconf.get('upload_path'), '_imported_profiles'),
			config = Importer.config(),
			oldOwnerNotFound = config.adminTakeOwnership.enable,
			startTime = +new Date();

		fs.writeFileSync(DIRTY_USERS_FILE, +new Date(), {encoding: 'utf8'});
		fs.ensureDirSync(picturesTmpPath);
		fs.ensureDirSync(picturesPublicPath);

		Importer.exporter.countUsers(function(err, total) {
			Importer.success('Importing ' + total + ' users.');
			Importer.exporter.exportUsers(function(err, users, usersArr, nextExportBatch) {
					async.eachSeries(usersArr, function(user, done) {
							count++;
							var _uid = user._uid;
							recoverImportedUser(_uid, function(err, _user) {
								if (_user) {
									Importer.progress(count, total);
									imported++;
									alreadyImported++;
									return done();
								}
								var u = Importer.makeValidNbbUsername(user._username || '', user._alternativeUsername || '');

								var p, generatedPassword;

								if (config.passwordGen.enabled) {
									generatedPassword = Importer.genRandPwd(config.passwordGen.len, config.passwordGen.chars);
									p = generatedPassword;
								} else {
									p = user._password;
								}

								var userData = {
									username: u.username,
									email: user._email,
									password: p
								};
								if (!userData.username) {
									Importer.warn('[process-count-at:' + count + '] skipping _username:' + user._username + ':_uid:' + user._uid + ', username is invalid.');
									Importer.progress(count, total);
									return done();
								}
								Importer.log('[process-count-at: ' + count + '] saving user:_uid: ' + _uid);
								var onCreate = function(err, uid) {

									if (err) {
										Importer.warn('[process-count-at: ' + count + '] skipping username: "' + user._username + '" ' + err);
										Importer.progress(count, total);
										done();
									} else {

										var onLevel = function() {

											var onGroups = function () {

												var fields = {
													// preseve the signature, but Nodebb allows a max of 255 chars, so i truncate with an '...' at the end
													signature: user._signature || '',
													website: user._website || '',
													location: user._location || '',
													joindate: user._joindate || startTime,
													reputation: (user._reputation || 0) * config.userReputationMultiplier,
													profileviews: user._profileViews || 0,
													fullname: user._fullname || '',
													birthday: user._birthday || '',
													showemail: user._showemail ? 1 : 0,
													lastposttime: user._lastposttime || 0,

													'email:confirmed': config.autoConfirmEmails ? 1 : 0,

													// this is a migration script, no one is online
													status: 'offline',

													// don't ban the users now, ban them later, if _imported_user:_uid._banned == 1
													banned: 0,

													_imported_readTids: user._readTids && JSON.stringify(user._readTids),
													_imported_readCids: user._readCids && JSON.stringify(user._readCids),
													_imported_path: user._path || '',
													_imported_uid: _uid,
													_imported_username: user._username || '',
													_imported_password: user._password || '',
													_imported_hashed_password: user._hashed_password || '',
													_imported_tmp_autogenerated_password: <PASSWORD> || '',
													_imported_slug: user._slug || user._userslug || '',
													_imported_signature: user._signature
												};

												if (user._lastonline) {
													fields.lastonline = user._lastonline;
												}

												var keptPicture = false;
												var onUserFields = function(err, result) {
													if (err) {
														return done(err);
													}

													user.imported = true;
													imported++;

													fields.uid = uid;
													user = nodeExtend(true, {}, user, fields);
													user.keptPicture = keptPicture;
													user.userslug = u.userslug;
													users[_uid] = user;
													Importer.progress(count, total);

													var series = [];
													if (fields.reputation > 0) {
														series.push(async.apply(db.sortedSetAdd, 'users:reputation', fields.reputation, uid));
													}
													async.series(series, function () {
														Data.setUserImported(_uid, uid, user, done);
													});
												};

												if (user._pictureBlob) {

													var filename = user._pictureFilename ? '_' + uid + '_' + user._pictureFilename : uid + '.png';
													var tmpPath = path.join(picturesTmpPath, filename);
													writeBlob(tmpPath, user._pictureBlob, function(err) {
														if (err) {
															Importer.warn(tmpPath, err);
															User.setUserFields(uid, fields, onUserFields);
														} else {
															File.saveFileToLocal(filename, folder, tmpPath, function(err, ret) {
																if (!err) {
																	fields.uploadedpicture = ret.url;
																	fields.picture = ret.url;
																	keptPicture = true;
																} else {
																	Importer.warn(filename, err);
																}

																User.setUserFields(uid, fields, onUserFields);
															});
														}
													});

												} else {
													if (user._picture) {
														fields.uploadedpicture = user._picture;
														fields.picture = user._picture;
														keptPicture = true;
													}

													User.setUserFields(uid, fields, onUserFields);
												}
											};

											if (user._groups && user._groups.length) {
												async.eachSeries(user._groups, function(_gid, next) {
													Data.getImportedGroup(_gid, function(err, _group) {
														if (_group && _group.name) {
															groupsJoin(_group._name, uid, user._joindate || startTime, function() {
																next();
															});
														} else {
															next();
														}
													});
												}, function() {
													onGroups();
												});
											} else {
												onGroups();
											}
										};

										if (('' + user._level).toLowerCase() == 'moderator') {
											Importer.makeModeratorOnAllCategories(uid, user._joindate || startTime, function () {
												Importer.warn(userData.username + ' just became a moderator on all categories');
												onLevel();
											});
										} else if (('' + user._level).toLowerCase() == 'administrator') {
											groupsJoin('administrators', uid, user._joindate || startTime, function(){
												Importer.warn(userData.username + ' became an Administrator');
												onLevel();
											});
										} else {
											onLevel();
										}

									}
								};  // end onCreate

								if (oldOwnerNotFound
									&& parseInt(user._uid, 10) === parseInt(config.adminTakeOwnership._uid, 10)
									|| (user._username || '').toLowerCase() === config.adminTakeOwnership._username.toLowerCase()
								) {
									Importer.warn('[process-count-at:' + count + '] skipping user: ' + user._username + ':'+ user._uid + ', it was revoked ownership');
									// cache the _uid for the next phases
									Importer.config('adminTakeOwnership', {
										enable: true,
										username: user._username,
										// just an alias in this case
										_username: user._username,
										_uid: user._uid
									});
									// no need to make it a mod or an admin, it already is
									user._level = null;
									// set to false so we don't have to match all users
									oldOwnerNotFound = false;
									// dont create, but set the fields
									return onCreate(null, LOGGEDIN_UID);
								} else {
									User.create(userData, onCreate);
								}
							});
						},
						nextExportBatch);
				},
				{
					// options
				},
				function(err) {
					if (err) {
						throw err;
					}
					Importer.success('Imported ' + imported + '/' + total + ' users' + (alreadyImported ? ' (out of which ' + alreadyImported + ' were already imported at an earlier time)' : ''));
					var nxt = function () {
						fs.remove(picturesTmpPath, function() {
							fs.remove(DIRTY_USERS_FILE, next);
						});
					};
					if (config.autoConfirmEmails && Data.keys) {
						async.parallel([
							function (done) {
								Data.keys('confirm:*', function (err, keys) {
									keys.forEach(function (key) {
										db.delete(key);
									});
									done();
								});
							},
							function (done) {
								Data.keys('email:*:confirm', function (err, keys) {
									keys.forEach(function (key) {
										db.delete(key);
									});
									done();
								});
							}
						], function () {
							Importer.progress(1, 1);
							Importer.phase('usersImportDone');
							nxt();
						});
					} else {
						Importer.progress(1, 1);
						Importer.phase('usersImportDone');
						nxt();
					}
				});
		});
	};


	Importer.importMessages = function(next) {
		Importer.phase('messagesImportStart');
		Importer.progress(0, 1);

		Importer._lastPercentage = 0;
		var count = 0,
			imported = 0,
			alreadyImported = 0,
			startTime = +new Date();

		fs.writeFileSync(DIRTY_MESSAGES_FILE, +new Date(), {encoding: 'utf8'});
		Importer.exporter.countMessages(function(err, total) {
			Importer.success('Importing ' + total + ' messages.');
			Importer.exporter.exportMessages(
				function(err, messages, messagesArr, nextExportBatch) {

					async.eachSeries(messagesArr, function(message, done) {
            count++;
            var _mid = message._mid;

            recoverImportedMessage(_mid, function(err, _message) {
              if (_message) {
                Importer.progress(count, total);
                imported++;
                alreadyImported++;
                return done();
              }

              if (message._fromuid == message._touid) {
                Importer.warn('[process-count-at:' + count + '] skipping message:_mid: ' + _mid + ', because it was send to self');
                Importer.progress(count, total);
                return done();
              }

              async.parallel([
                function(cb) {
                  Data.getImportedUser(message._fromuid, function(err, toUser) {
                    if (err) {
                      Importer.warn('getImportedUser:_fromuid:' + message._fromuid + ' err: ' + err.message);
                    }
                    cb(null, toUser);
                  });
                },
                function(cb) {
                  Data.getImportedUser(message._touid, function(err, toUser) {
                    if (err) {
                      Importer.warn('getImportedUser:_touid:' + message._touid + ' err: ' + err.message);
                    }
                    cb(null, toUser);
                  });
                }
              ], function(err, results) {
                var fromUser = results[0];
                var toUser = results[1];

                if (!fromUser || !toUser) {
                  Importer.warn('[process-count-at: ' + count + '] skipping message:_mid: ' + _mid + ' _fromuid:' + message._fromuid + ':imported: ' + !!fromUser + ', _touid:' + message._touid + ':imported: ' + !!toUser);
                  Importer.progress(count, total);
                  done();
                } else {

                  Importer.log('[process-count-at: ' + count + '] saving message:_mid: ' + _mid + ' _fromuid:' + message._fromuid + ', _touid:' + message._touid);

                  Messaging.addMessage(fromUser.uid, toUser.uid, message._content, message._timestamp, function(err, messageReturn) {
                    if (err || !messageReturn) {
                      Importer.warn('[process-count-at: ' + count + '] skipping message:_mid: ' + _mid + ' _fromuid:' + message._fromuid + ':imported: ' + !!fromUser + ', _touid:' + message._touid + ':imported: ' + !!toUser
                      + (err ? ' err: ' + err.message : ' messageReturn: ' + !!messageReturn));
                      Importer.progress(count, total);
                      return done();
                    }

                    imported++;
                    var mid = messageReturn.mid;
                    var uids =  [messageReturn.fromuid, messageReturn.touid].sort();

                    var _imported_content = message._content;

                    delete messageReturn._key;
                    delete messageReturn.toUser;
                    delete messageReturn.fromUser;

                    async.parallel([
                      function(next) {
                        db.setObjectField('message:' + mid, '_imported_content', _imported_content, next);
                      },
                      function(next) {
                        db.sortedSetAdd('uid:' + messageReturn.fromuid + ':chats', message._timestamp, messageReturn.touid, next);
                      },
                      function(next) {
                        db.sortedSetAdd('uid:' + messageReturn.touid + ':chats', message._timestamp, messageReturn.fromuid, next);
                      },
                      function(next) {
                        db.sortedSetRemove('uid:' + messageReturn.fromuid + ':chats:unread', messageReturn.touid, next);
                      },
                      function(next) {
                        db.sortedSetRemove('uid:' + messageReturn.touid + ':chats:unread', messageReturn.fromuid, next);
                      }
                    ], function(err) {
                      if (err) {
                        Importer.warn('[process-count-at: ' + count + '] message creation error message:_mid: ' + _mid + ':mid:' + mid, err);
                        return done();
                      }

                      Importer.progress(count, total);
                      message = nodeExtend(true, {}, message, messageReturn);
                      Data.setMessageImported(_mid, mid, message, done);
                    });
                  });
                }
              });
            });
          }, nextExportBatch);

				},
				{
					// options
				},
				function() {
					Importer.progress(1, 1);
					Importer.phase('messagesImportDone');
					Importer.success('Imported ' + imported + '/' + total + ' messages' + (alreadyImported ? ' (out of which ' + alreadyImported + ' were already imported at an earlier time)' : ''));
					fs.remove(DIRTY_MESSAGES_FILE, next);
				});
		});
	};


	Importer.importCategories = function(next) {
		Importer.phase('categoriesImportStart');
		Importer.progress(0, 1);

		Importer._lastPercentage = 0;

		var count = 0,
			imported = 0,
			alreadyImported = 0,
			startTime = +new Date(),
			config = Importer.config();

		fs.writeFileSync(DIRTY_CATEGORIES_FILE, +new Date(), {encoding: 'utf8'});

		Importer.exporter.countCategories(function(err, total) {
			Importer.success('Importing ' + total + ' categories.');
			Importer.exporter.exportCategories(
				function(err, categories, categoriesArr, nextExportBatch) {
					var onEach = function(category, done) {
						count++;
						var _cid = category._cid;

						recoverImportedCategory(_cid, function(err, _category) {
							if (_category) {
								imported++;
								alreadyImported++;
								Importer.progress(count, total);
								return done();
							}

							Importer.log('[process-count-at:' + count + '] saving category:_cid: ' + _cid);

							var categoryData = {
								name: category._name || ('Category ' + (count + 1)),
								description: category._description || 'no description available',

								// force all categories Parent to be 0, then after the import is done, we can iterate again and fix them.
								parentCid: 0,
								// same deal with disabled
								disabled: 0,

								// you can fix the order later, nbb/admin
								order: category._order || (count + 1),

								link: category._link || 0
							};

							if (config.categoriesIcons && config.categoriesIcons.length) {
								categoryData.icon = config.categoriesIcons[Math.floor(Math.random() * config.categoriesIcons.length)];
							}
							if (config.categoriesBgColors && config.categoriesBgColors.length) {
								categoryData.bgColor = config.categoriesBgColors[Math.floor(Math.random() * config.categoriesBgColors.length)];
							}
							if (config.categoriesTextColors && config.categoriesTextColors.length) {
								categoryData.color = config.categoriesTextColors[Math.floor(Math.random() * config.categoriesTextColors.length)];
							}

							var onCreate = function(err, categoryReturn) {
								if (err) {
									Importer.warn('skipping category:_cid: ' + _cid + ' : ' + err);
									Importer.progress(count, total);
									return done();
								}

								var onFields = function(err) {
									if (err) {
										Importer.warn(err);
									}

									Importer.progress(count, total);

									category.imported = true;
									imported++;
									category = nodeExtend(true, {}, category, categoryReturn, fields);
									categories[_cid] = category;

									Data.setCategoryImported(_cid, categoryReturn.cid, category, done);
								};

								var fields = {
									_imported_cid: _cid,
									_imported_path: category._path || '',
									_imported_name: category._name || '',
									_imported_slug: category._slug || '',
									_imported_parentCid: category._parent || category._parentCid || '',
									_imported_disabled: category._disabled || 0,
									_imported_description: category._description || ''
								};

								db.setObject('category:' + categoryReturn.cid, fields, onFields);
							};

							Categories.create(categoryData, onCreate);
						});
					};
					async.eachSeries(categoriesArr, onEach, nextExportBatch);
				},
				{
					// options
				},
				function(err) {
					if (err) {
						throw err;
					}
					Importer.success('Imported ' + imported + '/' + total + ' categories' + (alreadyImported ? ' (out of which ' + alreadyImported + ' were already imported at an earlier time)' : ''));
					Importer.progress(1, 1);
					Importer.phase('categoriesImportDone');
					fs.remove(DIRTY_CATEGORIES_FILE, next);
				});
		});
	};

	Importer.importGroups = function(next) {
		Importer.phase('groupsImportStart');
		Importer.progress(0, 1);

		Importer._lastPercentage = 0;

		var count = 0,
			imported = 0,
			alreadyImported = 0,
			startTime = +new Date(),
			config = Importer.config();

		fs.writeFileSync(DIRTY_GROUPS_FILE, +new Date(), {encoding: 'utf8'});

		Importer.exporter.countGroups(function(err, total) {
			Importer.success('Importing ' + total + ' groups.');
			Importer.exporter.exportGroups(
				function(err, groups, groupsArr, nextExportBatch) {

					var onEach = function(group, done) {
						count++;
						var _gid = group._gid;

						recoverImportedGroup(_gid, function(err, _group) {
							if (_group) {
								imported++;
								alreadyImported++;
								Importer.progress(count, total);
								return done();
							}

							Importer.log('[process-count-at:' + count + '] saving group:_gid: ' + _gid);

							var groupData = {
								name: group._name || ('Group ' + (count + 1)),
								description: group._description || 'no description available'
							};

							var onCreate = function(err, groupReturn) {
								if (err) {
									Importer.warn('skipping group:_gid: ' + _gid + ' : ' + err);
									Importer.progress(count, total);
									return done();
								}

								var onFields = function(err) {
									if (err) {
										Importer.warn(err);
									}

									var onTime = function () {

										Importer.progress(count, total);

										group.imported = true;
										imported++;
										group = nodeExtend(true, {}, group, groupReturn, fields);
										groups[_gid] = group;

										Data.setGroupImported(_gid, 0, group, done);
									};

									if (group._createtime || group._timestamp) {
										db.sortedSetAdd('groups:createtime', group._createtime || group._timestamp, groupReturn.name, function() {
											onTime();
										});
									} else {
										onTime();
									}
								};

								var fields = {
									_imported_gid: _gid,
									_imported_name: group._name,
									_imported_ownerUid: group._ownerUid || '',
									_imported_path: group._path || '',
									_imported_slug: group._slug || '',
									_imported_description: group._description || ''
								};

								if (group._createtime || group._timestamp) {
									fields.createtime = group._createtime || group._timestamp;
								}

								db.setObject('group:' + groupReturn.name, fields, onFields);
							};

							Groups.create(groupData, onCreate);
						});
					};
					async.eachSeries(groupsArr, onEach, nextExportBatch);
				},
				{
					// options
				},
				function(err) {
					if (err) {
						throw err;
					}
					Importer.success('Importing ' + imported + '/' + total + ' groups' + (alreadyImported ? ' (out of which ' + alreadyImported + ' were already imported at an earlier time)' : ''));
					Importer.progress(1, 1);
					Importer.phase('groupsImportDone');
					fs.remove(DIRTY_GROUPS_FILE, next);
				});
		});
	};

	Importer.allowGuestsOnAllCategories = function(done) {
		Data.eachCategory(function(category, next) {
				async.parallel([
					function(nxt) {
						Groups.join('cid:' + category.cid + ':privileges:groups:topics:create', 'guests', nxt);
					},
					function(nxt) {
						Groups.join('cid:' + category.cid + ':privileges:groups:topics:reply', 'guests', nxt);
					}
				], next);
			},
			{async: true, eachLimit: 10},
			function() {
				done();
			});
	};

	Importer.disallowGuestsWriteOnAllCategories = function(done) {
		Data.eachCategory(function(category, next) {
				async.parallel([
					function(nxt) {
						// 'cid:' + cids[i] + ':privileges:groups:' + privilege'
						Groups.leave('cid:' + category.cid + ':privileges:groups:topics:create', 'guests', nxt);
					},
					function(nxt) {
						Groups.leave('cid:' + category.cid + ':privileges:groups:topics:reply', 'guests', nxt);
					}
				], next);
			},
			{async: true, eachLimit: 10},
			function() {
				done();
			});
	};

	Importer.importTopics = function(next) {
		Importer.phase('topicsImportStart');
		Importer.progress(0, 1);

		Importer._lastPercentage = 0;
		var count = 0,
			imported = 0,
			alreadyImported = 0,
			startTime = +new Date(),
			attachmentsTmpPath = path.join(__dirname, '/tmp/attachments'),
			folder = '_imported_attachments',
			attachmentsPublicPath = path.join(nconf.get('base_dir'), nconf.get('upload_path'), '_imported_attachments'),
			config = Importer.config();

		fs.writeFileSync(DIRTY_TOPICS_FILE, +new Date(), {encoding: 'utf8'});
		fs.ensureDirSync(attachmentsTmpPath);
		fs.ensureDirSync(attachmentsPublicPath);

		Importer.exporter.countTopics(function(err, total) {
			Importer.success('Importing ' + total + ' topics.');
			Importer.exporter.exportTopics(
				function(err, topics, topicsArr, nextExportBatch) {

					async.eachSeries(topicsArr, function(topic, done) {
						count++;
						var _tid = topic._tid;

						recoverImportedTopic(_tid, function(err, _topic) {
							if (_topic) {
								Importer.progress(count, total);
								imported++;
								alreadyImported++;
								return done();
							}

							async.parallel([
								function(cb) {
									Data.getImportedCategory(topic._cid, function(err, cat) {
										if (err) {
											Importer.warn('getImportedCategory: ' + topic._cid + ' err: ' + err);
										}
										cb(null, cat);
									});
								},
								function(cb) {
									Data.getImportedUser(topic._uid, function(err, usr) {
										if (err) {
											Importer.warn('getImportedUser: ' + topic._uid + ' err: ' + err);
										}
										cb(null, usr);
									});
								}
							], function(err, results) {

								var category = results[0];
								var user = results[1] || {uid: '0'};

								if (!category) {
									Importer.warn('[process-count-at:' + count + '] skipping topic:_tid:"' + _tid + '" --> _cid: ' + topic._cid + ':imported:' + !!category);
									Importer.progress(count, total);
									done();
								} else {
									Importer.log('[process-count-at:' + count + '] saving topic:_tid: ' + _tid);

									if (topic._attachmentsBlobs && topic._attachmentsBlobs.length) {
										var attachmentsIndex = 0;

										topic._attachments = [].concat(topic._attachments || []);
										topic._images = [].concat(topic._images || []);

										async.eachSeries(topic._attachmentsBlobs, function(_attachmentsBlob, next) {
											var filename = 'attachment_t_' + _tid + '_' + attachmentsIndex++ + (_attachmentsBlob.filename ? '_' + _attachmentsBlob.filename : _attachmentsBlob.extension);
											var tmpPath = path.join(attachmentsTmpPath, filename);

											writeBlob(tmpPath, _attachmentsBlob.blob, function(err, ftype) {
												if (err) {
													Importer.warn(tmpPath, err);
													next();
												} else {
													File.saveFileToLocal(filename, folder, tmpPath, function(err, ret) {
														if (!err) {
															if (/image/.test(ftype.mime)) {
																topic._images.push(ret.url);
															} else {
																topic._attachments.push(ret.url);
															}
														} else {
															Importer.warn(filename, err);
														}
														next();
													});
												}
											});

										}, onAttachmentsBlobs);

									} else {
										onAttachmentsBlobs();
									}

									function onAttachmentsBlobs () {
										topic._content = (topic._content || '').trim();

										topic._title = utils.slugify(topic._title) ? topic._title[0].toUpperCase() + topic._title.substr(1) : utils.truncate(topic._content, 100);

										(topic._images || []).forEach(function(_image) {
											topic._content += generateImageTag(_image);
										});
										(topic._attachments || []).forEach(function(_attachment) {
											topic._content += generateAnchorTag(_attachment);
										});

										if (topic._tags && !Array.isArray(topic._tags)) {
											topic._tags = ('' + topic._tags).split(',');
										}

										Topics.post({
											uid: !config.adminTakeOwnership.enable ? user.uid : parseInt(config.adminTakeOwnership._uid, 10) === parseInt(topic._uid, 10) ? LOGGEDIN_UID : user.uid,
											title: topic._title,
											content: topic._content,
											timestamp: topic._timestamp,
											cid: category.cid,
											thumb: topic._thumb,
											tags: topic._tags
										}, function (err, returnTopic) {
											if (err) {
												Importer.warn('[process-count-at:' + count + '] skipping topic:_tid: ' + _tid + ':cid:' + category.cid + ':_cid:' + topic._cid + ':uid:' + user.uid +  ':_uid:' + topic._uid + ' err: ' + err);
												Importer.progress(count, total);
												done();
											} else {

												topic.imported = true;
												imported++;

												var topicFields = {
													viewcount: topic._viewcount || topic._viewscount || 0,

													// assume that this topic not locked for now, but will iterate back again at the end and lock it back after finishing the importPosts()
													// locked: normalizedTopic._locked ? 1 : 0,
													locked: 0,

													deleted: topic._deleted ? 1 : 0,

													// if pinned, we should set the db.sortedSetAdd('cid:' + cid + ':tids', Math.pow(2, 53), tid);
													pinned: topic._pinned ? 1 : 0,

													_imported_tid: _tid,
													_imported_uid: topic._uid || '',
													_imported_cid: topic._cid,
													_imported_slug: topic._slug || '',
													_imported_path: topic._path || '',
													_imported_title: topic._title || '',
													_imported_content: topic._content || '',
													_imported_guest: topic._guest || '',
													_imported_ip: topic._ip || '',
													_imported_user_slug: user._slug || '',
													_imported_user_path: user._path || '',
													_imported_category_path: category._path || '',
													_imported_category_slug: category._slug || ''
												};

												var postFields = {
													votes: topic._votes || 0,
													reputation: topic._reputation || 0,
													edited: topic._edited || 0
												};

												// if (topic._edited) {
												// 	postFields.edited = topic._edited;
												// }

												// console.log("Topic Edited:", postFields.edited, "type:", typeof postFields.edited);

												var onPinned = function() {
													db.setObject('topic:' + returnTopic.topicData.tid, topicFields, function(err, result) {
														Importer.progress(count, total);
														if (err) {
															Importer.warn(err);
														}
														Posts.setPostFields(returnTopic.postData.pid, postFields, function(){
															topic = nodeExtend(true, {}, topic, topicFields, returnTopic.topicData);
															topics[_tid] = topic;
															Data.setTopicImported(_tid, returnTopic.topicData.tid, topic, done);
														});
													});
												};

												// pinned = 1 not enough to float the topic to the top in it's category
												if (topic._pinned) {
													db.sortedSetAdd('cid:' + category.cid + ':tids', Math.pow(2, 53), returnTopic.topicData.tid, onPinned);
												}  else {
													db.sortedSetAdd('cid:' + category.cid + ':tids', timestamp, returnTopic.topicData.tid, onPinned);
												}
											}
										});
									}
								}
							});
						});
					}, nextExportBatch);
				},
				{
					//options
				},
				function(err) {
					if (err) {
						throw err;
					}
					Importer.success('Imported ' + imported + '/' + total + ' topics' + (alreadyImported ? ' (out of which ' + alreadyImported + ' were already imported at an earlier time)' : ''));
					Importer.progress(1, 1);
					Importer.phase('topicsImportDone');

					async.series([
						function(nxt) {<commit_msg>oopsy daisy, also related to #163<commit_after>var async = require('async'),
	fileType = require('file-type'),
	EventEmitter2 = require('eventemitter2').EventEmitter2,
	_ = require('underscore'),
	nodeExtend = require('node.extend'),
	fs = require('fs-extra'),
	path = require('path'),
	nconf = require('nconf'),

	utils = require('../public/js/utils.js'),
	Data = require('./data.js'),

	MAX_INT = -1 >>> 1,

	Groups = require('../../../src/groups.js'),
	Favourites = require('../../../src/favourites.js'),
	privileges = require('../../../src/privileges.js'),
	Meta = require('../../../src/meta.js'),
	User = require('../../../src/user.js'),
	Messaging = require('../../../src/messaging.js'),
	File = require('../../../src/file.js'),
	Topics = require('../../../src/topics.js'),
	Posts = require('../../../src/posts.js'),
	Categories = require('../../../src/categories.js'),
	db = module.parent.require('../../../src/database.js'),

	IMPORT_BATCH_SIZE = 10,
	FLUSH_BATCH_SIZE = 10,

//todo use the real one
	LOGGEDIN_UID = 1,

	logPrefix = '[nodebb-plugin-import]',

	BACKUP_CONFIG_FILE = path.join(__dirname, '/tmp/importer.nbb.backedConfig.json'),

	DIRTY_GROUPS_FILE = path.join(__dirname, '/tmp/importer.dirty.groups'),
	DIRTY_USERS_FILE = path.join(__dirname, '/tmp/importer.dirty.users'),
	DIRTY_MESSAGES_FILE = path.join(__dirname, '/tmp/importer.dirty.messages'),
	DIRTY_CATEGORIES_FILE = path.join(__dirname, '/tmp/importer.dirty.categories'),
	DIRTY_TOPICS_FILE = path.join(__dirname, '/tmp/importer.dirty.topics'),
	DIRTY_POSTS_FILE = path.join(__dirname, '/tmp/importer.dirty.posts'),
	DIRTY_VOTES_FILE = path.join(__dirname, '/tmp/importer.dirty.votes'),
	DIRTY_BOOKMARKS_FILE = path.join(__dirname, '/tmp/importer.dirty.bookmarks'),

	areGroupsDirty,
	areUsersDirty,
	areMessagesDirty,
	areCategoriesDirty,
	areTopicsDirty,
	arePostsDirty,
	areVotesDirty,
	areBookmarksDirty,

	isAnythingDirty,

	alreadyImportedAllGroups = false,
	alreadyImportedAllUsers = false,
	alreadyImportedAllMessages = false,
	alreadyImportedAllCategories = false,
	alreadyImportedAllTopics = false,
	alreadyImportedAllPosts = false,
	alreadyImportedAllVotes = false,
	alreadyImportedAllBookmarks = false,

	flushed = false,

	defaults = {
		log: true,
		passwordGen: {
			enabled: false,
			chars: '{}.-_=+qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM1234567890',
			len: 13
		},
		categoriesTextColors: ['#FFFFFF'],
		categoriesBgColors: ['#AB4642', '#DC9656', '#F7CA88', '#A1B56C', '#86C1B9', '#7CAFC2', '#BA8BAF', '#A16946'],
		categoriesIcons: ['fa-comment'],
		autoConfirmEmails: true,
		userReputationMultiplier: 1,

		adminTakeOwnership: {
			enable: false,
			_username: null,
			_uid: null
		},

		nbbTmpConfig: {
			maximumPostLength: MAX_INT,
			maximumChatMessageLength: MAX_INT,
			maximumTitleLength: MAX_INT,
			maximumUsernameLength: MAX_INT,
			postDelay: 0,
			initialPostDelay: 0,
			newbiePostDelay: 0,
			minimumPostLength: 0,
			minimumPasswordLength: 0,
			minimumTitleLength: 0,
			requireEmailConfirmation: 0,
			trackIpPerPost: 1,
			"newbiePostDelayThreshold": 0,
			"minimumTagsPerTopic": 0,
			"maximumTagsPerTopic": MAX_INT,
			"allowGuestSearching": 1,
			"allowTopicsThumbnail": 1,
			"registrationType": "normal",
			"allowLocalLogin": 1,
			"allowAccountDelete": 1,
			"allowFileUploads": 1,
			"allowUserHomePage": 1,
			"maximumFileSize": MAX_INT,
			"minimumUsernameLength": 1,
			"maximumSignatureLength": MAX_INT,
			"maximumAboutMeLength": MAX_INT,
			"maximumProfileImageSize": MAX_INT,
			"maximumCoverImageSize": MAX_INT,
			"profileImageDimension": 128,
			"profile:allowProfileImageUploads": 1,
			"reputation:disabled": 0,
			"downvote:disabled": 0
		}
	};

(function(Importer) {

	var coolDownFn = function (timeout) {
		return function (next) {
			timeout = timeout || 5000;
			Importer.log('cooling down for ' + timeout/1000 + ' seconds');
			setTimeout(next, timeout);
		};
	};

	var groupsJoin = function (groupName, uid, timestamp, callback) {
		Groups.join(groupName, uid, function(err, ret) {
			if (err) {
				return callback(err);
			}
			db.sortedSetAdd('group:' + groupName + ':members', timestamp, uid, function (err) {
				if (err) {
					return callback(err);
				}
				Importer.log(uid, 'joined group:', groupName);
				callback(null, ret);
			});
		});
	};

	Importer._dispatcher = new EventEmitter2({
		wildcard: true
	});

	Importer.init = function(exporter, config, callback) {
		Importer.setup(exporter, config, callback);
	};

	Importer.setup = function(exporter, config, callback) {
		Importer.exporter = exporter;

		Importer._config = nodeExtend(true, {}, defaults, config && config.importer ? config.importer : config || {});

		//todo I don't like this
		Importer._config.serverLog = !!config.log.server;
		Importer._config.clientLog = !!config.log.client;
		Importer._config.verbose = !!config.log.verbose;

		Importer.emit('importer.setup.start');

		flushed = false;

		Importer.emit('importer.setup.done');
		Importer.emit('importer.ready');
		if (_.isFunction(callback)) {
			callback();
		}
	};

	function start (flush, callback) {
		Importer.emit('importer.start');

		var series = [];

		if (flush) {
			series.push(Importer.flushData);
		} else {
			Importer.log('Skipping data flush');
			series.push(makeDirty);
		}
		async.series(series.concat([
			Importer.backupConfig,
			Importer.setTmpConfig,
			Importer.importGroups,
			coolDownFn(5000),
			Importer.importCategories,
			Importer.allowGuestsOnAllCategories,
			Importer.importUsers,
			Importer.importMessages,
			Importer.importTopics,
			Importer.importPosts,
			Importer.importVotes,
			Importer.importBookmarks,
			Importer.fixCategoriesParentsAndAbilities,
			Importer.fixPostsToPids,
			Importer.fixGroupsOwners,
			Importer.relockUnlockedTopics,
			Importer.rebanAndMarkReadForUsers,
			Importer.fixTopicTimestamps,
			Importer.restoreConfig,
			Importer.disallowGuestsWriteOnAllCategories,
			Importer.teardown
		]), callback);
	}

	Importer.start = function(callback) {
		var config = Importer.config();
		start(config.flush, callback);
	};

	Importer.resume = function(callback) {
		Importer.emit('importer.start');
		Importer.emit('importer.resume');

		Importer.isDirty();

		var series = [];
		if (! alreadyImportedAllGroups) {
			series.push(Importer.importGroups);
		} else {
			Importer.warn('alreadyImportedAllGroups=true, skipping importGroups Phase');
		}

		if (! alreadyImportedAllCategories) {
			series.push(Importer.importCategories);
			series.push(Importer.allowGuestsOnAllCategories);
		} else {
			Importer.warn('alreadyImportedAllCategories=true, skipping importCategories Phase');
		}
		if (! alreadyImportedAllUsers) {
			series.push(Importer.importUsers);
		} else {
			Importer.warn('alreadyImportedAllUsers=true, skipping importUsers Phase');
		}
		if (! alreadyImportedAllMessages) {
			series.push(Importer.importMessages);
		} else {
			Importer.warn('alreadyImportedAllMessages=true, skipping importMessages Phase');
		}

		if (! alreadyImportedAllTopics) {
			series.push(Importer.importTopics);
		} else {
			Importer.warn('alreadyImportedAllTopics=true, skipping importTopics Phase');
		}
		if (! alreadyImportedAllPosts) {
			series.push(Importer.importPosts);
		} else {
			Importer.warn('alreadyImportedAllPosts=true, skipping importPosts Phase');
		}

		if (! alreadyImportedAllVotes) {
			series.push(Importer.importVotes);
		} else {
			Importer.warn('alreadyImportedAllVotes=true, skipping importVotes Phase');
		}

		if (! alreadyImportedAllBookmarks) {
			series.push(Importer.importBookmarks);
		} else {
			Importer.warn('alreadyImportedAllBookmarks=true, skipping importBookmarks Phase');
		}

		series.concat([
			Importer.fixCategoriesParentsAndAbilities,
			Importer.fixPostsToPids,
			Importer.fixGroupsOwners,
			Importer.relockUnlockedTopics,
			Importer.rebanAndMarkReadForUsers,
			Importer.fixTopicTimestamps,
			Importer.restoreConfig,
			Importer.disallowGuestsWriteOnAllCategories,
			Importer.teardown
		]);

		async.series(series, callback);
	};

	function makeDirty (done) {

		areGroupsDirty = true;
		areUsersDirty = true;
		areMessagesDirty = true;
		areCategoriesDirty = true;
		areTopicsDirty = true;
		arePostsDirty = true;
		areVotesDirty = true;
		areBookmarksDirty = true;
		isAnythingDirty = true;

		alreadyImportedAllGroups = false;
		alreadyImportedAllUsers = false;
		alreadyImportedAllMessages = false;
		alreadyImportedAllCategories = false;
		alreadyImportedAllTopics = false;
		alreadyImportedAllPosts = false;
		alreadyImportedAllVotes = false;
		alreadyImportedAllBookmarks = false;

		flushed = false;

		return _.isFunction(done) ? done(null, true) : true;
	}

	// todo: really? wtf is this logic
	Importer.isDirty = function(done) {

		areGroupsDirty = !! fs.existsSync(DIRTY_GROUPS_FILE);
		areVotesDirty = !! fs.existsSync(DIRTY_VOTES_FILE);
		areUsersDirty = !! fs.existsSync(DIRTY_USERS_FILE);
		areMessagesDirty = !! fs.existsSync(DIRTY_MESSAGES_FILE);
		areCategoriesDirty = !! fs.existsSync(DIRTY_CATEGORIES_FILE);
		areTopicsDirty = !! fs.existsSync(DIRTY_TOPICS_FILE);
		arePostsDirty = !! fs.existsSync(DIRTY_POSTS_FILE);
		areBookmarksDirty = !! fs.existsSync(DIRTY_BOOKMARKS_FILE);

		isAnythingDirty =
			areGroupsDirty
			|| areVotesDirty
			|| areUsersDirty
			|| areCategoriesDirty
			|| areTopicsDirty
			|| arePostsDirty
			|| areMessagesDirty
			|| areBookmarksDirty;

		// order in start() and resume() matters and must be in sync
		if (areGroupsDirty) {
			alreadyImportedAllGroups = false;
			alreadyImportedAllCategories = false;
			alreadyImportedAllUsers = false;
			alreadyImportedAllMessages = false;
			alreadyImportedAllTopics = false;
			alreadyImportedAllPosts = false;
			alreadyImportedAllVotes = false;
			alreadyImportedAllBookmarks = false;
		} else if (areCategoriesDirty) {
			alreadyImportedAllGroups = true;
			alreadyImportedAllCategories = false;
			alreadyImportedAllUsers = false;
			alreadyImportedAllMessages = false;
			alreadyImportedAllTopics = false;
			alreadyImportedAllPosts = false;
			alreadyImportedAllVotes = false;
			alreadyImportedAllBookmarks = false;
		} else if (areUsersDirty) {
			alreadyImportedAllGroups = true;
			alreadyImportedAllCategories = true;
			alreadyImportedAllUsers = false;
			alreadyImportedAllMessages = false;
			alreadyImportedAllTopics = false;
			alreadyImportedAllPosts = false;
			alreadyImportedAllVotes = false;
			alreadyImportedAllBookmarks = false;
		} else if (areMessagesDirty) {
			alreadyImportedAllGroups = true;
			alreadyImportedAllCategories = true;
			alreadyImportedAllUsers = true;
			alreadyImportedAllMessages = false;
			alreadyImportedAllTopics = false;
			alreadyImportedAllPosts = false;
			alreadyImportedAllVotes = false;
			alreadyImportedAllBookmarks = false;
		} else if (areTopicsDirty) {
			alreadyImportedAllGroups = true;
			alreadyImportedAllCategories = true;
			alreadyImportedAllUsers = true;
			alreadyImportedAllMessages = true;
			alreadyImportedAllTopics = false;
			alreadyImportedAllPosts = false;
			alreadyImportedAllVotes = false;
			alreadyImportedAllBookmarks = false;
		} else if (arePostsDirty) {
			alreadyImportedAllGroups = true;
			alreadyImportedAllCategories = true;
			alreadyImportedAllUsers = true;
			alreadyImportedAllMessages = true;
			alreadyImportedAllTopics = true;
			alreadyImportedAllPosts = false;
			alreadyImportedAllVotes = false;
			alreadyImportedAllBookmarks = false;
		} else if (areVotesDirty) {
			alreadyImportedAllGroups = true;
			alreadyImportedAllCategories = true;
			alreadyImportedAllUsers = true;
			alreadyImportedAllMessages = true;
			alreadyImportedAllTopics = true;
			alreadyImportedAllPosts = true;
			alreadyImportedAllVotes = false;
			alreadyImportedAllBookmarks = false;
		} else if (areBookmarksDirty) {
			alreadyImportedAllGroups = true;
			alreadyImportedAllCategories = true;
			alreadyImportedAllUsers = true;
			alreadyImportedAllMessages = true;
			alreadyImportedAllTopics = true;
			alreadyImportedAllPosts = true;
			alreadyImportedAllVotes = true;
			alreadyImportedAllBookmarks = false;
		}

		return _.isFunction(done) ? done(null, isAnythingDirty) : isAnythingDirty;
	};

	Importer.flushData = function(next) {
		async.series([
			function(done){
				Importer.phase('purgeCategories+Topics+Bookmarks+Posts+VotesStart');
				Importer.progress(0, 1);

				// that will delete, categories, topics, topics.bookmarks, posts and posts.votes
				Data.countCategories(function(err, total) {
					var index = 0;
					Data.processCategoriesCidsSet(
						function (err, ids, nextBatch) {
							async.mapLimit(ids, FLUSH_BATCH_SIZE, function(id, cb) {
								Importer.progress(index++, total);
								Categories.purge(id, cb);
							}, nextBatch);
						},
						{alwaysStartAt: 0},
						function(err) {
							if (err) {
								Importer.warn(Importer._phase + " : " + err.message);
							}
							Importer.progress(1, 1);
							Importer.phase('purgeCategories+Topics+Bookmarks+Posts+VotesDone');
							done()
						});
				});

			},
			function(done) {
				Importer.phase('purgeUsersStart');
				Importer.progress(0, 1);

				Data.countUsers(function(err, total) {
					var index = 0; var count = 0;
					Data.processUsersUidsSet(
						function(err, ids, nextBatch) {
							async.mapLimit(ids, FLUSH_BATCH_SIZE, function(uid, cb) {
								Importer.progress(index++, total);
								if (parseInt(uid, 10) === 1) {
									return cb();
								}
								User.delete(uid, function() {
									count++;
									cb();
								});
							}, nextBatch);
						}, {
							// since we're deleting records the range is always shifting backwards, so need to advance the batch start boundary
							alwaysStartAt: 0,
							// done if the uid=1 in the only one in the db
							doneIf: function(start, end, ids) {
								return ids.length === 1;
							}
						},
						function(err) {
							Importer.progress(1, 1);
							Importer.phase('purgeUsersDone');
							done(err)
						}
					);
				});
			},
			function(done) {
				Importer.phase('purgeGroupsStart');
				Importer.progress(0, 1);

				Data.countGroups(function(err, total) {
					var index = 0; var count = 0;
					Data.processGroupsNamesSet(
						function(err, names, nextBatch) {
							async.mapLimit(names, FLUSH_BATCH_SIZE, function(name, cb) {
								Importer.progress(index++, total);

								// skip if system group
								if (name === 'administrators') {
									return cb();
								}
								Groups.destroy(name, function() {
									count++;
									cb();
								});
							}, nextBatch);
						}, {
							// since we're deleting records the range is always shifting backwards, so need to advance the batch start boundary
							alwaysStartAt: 0,

							// done if the administrators group in the only one in the db
							doneIf: function(start, end, names) {
								return !names.length || names.length === 1;
							}
						},
						function(err) {
							Importer.progress(1, 1);
							Importer.phase('purgeGroupsDone');
							done(err)
						}
					);
				});
			},
			function(done) {
				Importer.phase('purgeMessagesStart');
				Importer.progress(0, 1);

				Data.countMessages(function(err, total) {
					var index = 0;
					Data.eachMessage(
						function(message, next) {
							Importer.progress(index++, total);
							var uids = [message.fromuid, message.touid].sort();
							async.parallel([
								function(nxt) {
									db.delete('message:' + message.mid, function () {
										nxt();
									});
								},
								function(nxt) {
									db.sortedSetRemove('messages:uid:' + uids[0] + ':to:' + uids[1], message.mid, function() {
										nxt();
									});
								},
								function(next) {
									db.sortedSetRemove('uid:' + uids[0] + ':chats', uids[1], next);
								},
								function(next) {
									db.sortedSetRemove('uid:' + uids[1] + ':chats', uids[0], next);
								}
							], next);
						},
						{},
						function(err) {
							Importer.progress(1, 1);
							Importer.phase('purgeMessagesDone');
							done(err)
						}
					);
				});
			},
			function(done) {
				flushed = true;

				Importer.phase('resetGlobalsStart');
				Importer.progress(0, 1);

				db.setObject('global', {
					nextUid: 1,
					userCount: 1,
					nextGid: 1,
					groupCount: 1,
					nextMid: 1,
					nextCid: 1,
					categoryCount: 1,
					nextTid: 1,
					topicCount: 1,
					nextPid: 1,
					postCount: 1,
					nextVid: 1,
					voteCount: 1,
					nextBid: 1,
					bookmarkCount: 1
				}, function(err) {
					if (err) {
						return done(err);
					}
					Importer.progress(1, 1);
					Importer.phase('resetGlobalsDone');
					done();
				});
			},
			Importer.deleteTmpImportedSetsAndObjects
		], function(err) {
			if (err) {
				Importer.error(err);
				next(err);
			}
			next();
		});
	};

	Importer.phasePercentage = 0;

	Importer.progress = function(count, total, interval) {
		interval = interval || 0.01;
		var percentage = count / total * 100;
		if (percentage === 0 || percentage >= 100 || (percentage - Importer.phasePercentage >= interval)) {
			Importer.phasePercentage = percentage;
			Importer.emit('importer.progress', {count: count, total: total, percentage: percentage});
		}
	};

	Importer.phase = function(phase, data) {
		Importer.phasePercentage = 0;
		Importer._phase = phase;
		Importer.emit('importer.phase', {phase: phase, data: data, timestamp: +new Date()});
	};

	var recoverImportedGroup = function(_gid, callback) {
		if (! flushed && (alreadyImportedAllGroups || areGroupsDirty)) {
			return Data.getImportedGroup(_gid, callback);
		}
		return callback(null, null);
	};

	var recoverImportedUser = function(_uid, callback) {
		if (! flushed && (alreadyImportedAllUsers || areUsersDirty)) {
			return Data.getImportedUser(_uid, callback);
		}
		return callback(null, null);
	};

	var recoverImportedMessage = function(_mid, callback) {
		if (! flushed && (alreadyImportedAllMessages || areMessagesDirty)) {
			return Data.getImportedMessage(_mid, callback);
		}
		return callback(null, null);
	};

	var recoverImportedCategory = function(_cid, callback) {
		if (! flushed && (alreadyImportedAllCategories || areCategoriesDirty)) {
			return Data.getImportedCategory(_cid, callback);
		}
		return callback(null, null);
	};

	var recoverImportedTopic = function(_tid, callback) {
		if (! flushed && (alreadyImportedAllTopics || areTopicsDirty)) {
			return Data.getImportedTopic(_tid, callback);
		}
		return callback(null, null);
	};

	var recoverImportedPost = function(_pid, callback) {
		if (! flushed && (alreadyImportedAllPosts || arePostsDirty)) {
			return Data.getImportedPost(_pid, callback);
		}
		return callback(null, null);
	};
	var recoverImportedVote = function(_vid, callback) {
		if (! flushed && (alreadyImportedAllVotes || areVotesDirty)) {
			return Data.getImportedVote(_vid, callback);
		}
		return callback(null, null);
	};
	var recoverImportedBookmark = function(_bid, callback) {
		if (! flushed && (alreadyImportedAllBookmarks || areBookmarksDirty)) {
			return Data.getImportedBookmark(_bid, callback);
		}
		return callback(null, null);
	};

	var writeBlob = function(filepath, blob, callback) {
		var buffer = new Buffer(blob, 'binary');

		var ftype = fileType(buffer) || {mime: "unknown/unkown", extension: ""};
		ftype.filepath = filepath;

		fs.writeFile(filepath, buffer.toString('binary'), 'binary', function (err) {
			callback(err, ftype);
		});
	};

	Importer.importUsers = function(next) {
		Importer._lastPercentage = 0;
		Importer.phase('usersImportStart');
		Importer.progress(0, 1);
		var count = 0,
			imported = 0,
			alreadyImported = 0,
			picturesTmpPath = path.join(__dirname, '/tmp/pictures'),
			folder = '_imported_profiles',
			picturesPublicPath = path.join(nconf.get('base_dir'), nconf.get('upload_path'), '_imported_profiles'),
			config = Importer.config(),
			oldOwnerNotFound = config.adminTakeOwnership.enable,
			startTime = +new Date();

		fs.writeFileSync(DIRTY_USERS_FILE, +new Date(), {encoding: 'utf8'});
		fs.ensureDirSync(picturesTmpPath);
		fs.ensureDirSync(picturesPublicPath);

		Importer.exporter.countUsers(function(err, total) {
			Importer.success('Importing ' + total + ' users.');
			Importer.exporter.exportUsers(function(err, users, usersArr, nextExportBatch) {
					async.eachSeries(usersArr, function(user, done) {
							count++;
							var _uid = user._uid;
							recoverImportedUser(_uid, function(err, _user) {
								if (_user) {
									Importer.progress(count, total);
									imported++;
									alreadyImported++;
									return done();
								}
								var u = Importer.makeValidNbbUsername(user._username || '', user._alternativeUsername || '');

								var p, generatedPassword;

								if (config.passwordGen.enabled) {
									generatedPassword = Importer.genRandPwd(config.passwordGen.len, config.passwordGen.chars);
									p = generatedPassword;
								} else {
									p = user._password;
								}

								var userData = {
									username: u.username,
									email: user._email,
									password: p
								};
								if (!userData.username) {
									Importer.warn('[process-count-at:' + count + '] skipping _username:' + user._username + ':_uid:' + user._uid + ', username is invalid.');
									Importer.progress(count, total);
									return done();
								}
								Importer.log('[process-count-at: ' + count + '] saving user:_uid: ' + _uid);
								var onCreate = function(err, uid) {

									if (err) {
										Importer.warn('[process-count-at: ' + count + '] skipping username: "' + user._username + '" ' + err);
										Importer.progress(count, total);
										done();
									} else {

										var onLevel = function() {

											var onGroups = function () {

												var fields = {
													// preseve the signature, but Nodebb allows a max of 255 chars, so i truncate with an '...' at the end
													signature: user._signature || '',
													website: user._website || '',
													location: user._location || '',
													joindate: user._joindate || startTime,
													reputation: (user._reputation || 0) * config.userReputationMultiplier,
													profileviews: user._profileViews || 0,
													fullname: user._fullname || '',
													birthday: user._birthday || '',
													showemail: user._showemail ? 1 : 0,
													lastposttime: user._lastposttime || 0,

													'email:confirmed': config.autoConfirmEmails ? 1 : 0,

													// this is a migration script, no one is online
													status: 'offline',

													// don't ban the users now, ban them later, if _imported_user:_uid._banned == 1
													banned: 0,

													_imported_readTids: user._readTids && JSON.stringify(user._readTids),
													_imported_readCids: user._readCids && JSON.stringify(user._readCids),
													_imported_path: user._path || '',
													_imported_uid: _uid,
													_imported_username: user._username || '',
													_imported_password: user._password || '',
													_imported_hashed_password: user._hashed_password || '',
													_imported_tmp_autogenerated_password: <PASSWORD> || '',
													_imported_slug: user._slug || user._userslug || '',
													_imported_signature: user._signature
												};

												if (user._lastonline) {
													fields.lastonline = user._lastonline;
												}

												var keptPicture = false;
												var onUserFields = function(err, result) {
													if (err) {
														return done(err);
													}

													user.imported = true;
													imported++;

													fields.uid = uid;
													user = nodeExtend(true, {}, user, fields);
													user.keptPicture = keptPicture;
													user.userslug = u.userslug;
													users[_uid] = user;
													Importer.progress(count, total);

													var series = [];
													if (fields.reputation > 0) {
														series.push(async.apply(db.sortedSetAdd, 'users:reputation', fields.reputation, uid));
													}
													async.series(series, function () {
														Data.setUserImported(_uid, uid, user, done);
													});
												};

												if (user._pictureBlob) {

													var filename = user._pictureFilename ? '_' + uid + '_' + user._pictureFilename : uid + '.png';
													var tmpPath = path.join(picturesTmpPath, filename);
													writeBlob(tmpPath, user._pictureBlob, function(err) {
														if (err) {
															Importer.warn(tmpPath, err);
															User.setUserFields(uid, fields, onUserFields);
														} else {
															File.saveFileToLocal(filename, folder, tmpPath, function(err, ret) {
																if (!err) {
																	fields.uploadedpicture = ret.url;
																	fields.picture = ret.url;
																	keptPicture = true;
																} else {
																	Importer.warn(filename, err);
																}

																User.setUserFields(uid, fields, onUserFields);
															});
														}
													});

												} else {
													if (user._picture) {
														fields.uploadedpicture = user._picture;
														fields.picture = user._picture;
														keptPicture = true;
													}

													User.setUserFields(uid, fields, onUserFields);
												}
											};

											if (user._groups && user._groups.length) {
												async.eachSeries(user._groups, function(_gid, next) {
													Data.getImportedGroup(_gid, function(err, _group) {
														if (_group && _group.name) {
															groupsJoin(_group._name, uid, user._joindate || startTime, function() {
																next();
															});
														} else {
															next();
														}
													});
												}, function() {
													onGroups();
												});
											} else {
												onGroups();
											}
										};

										if (('' + user._level).toLowerCase() == 'moderator') {
											Importer.makeModeratorOnAllCategories(uid, user._joindate || startTime, function () {
												Importer.warn(userData.username + ' just became a moderator on all categories');
												onLevel();
											});
										} else if (('' + user._level).toLowerCase() == 'administrator') {
											groupsJoin('administrators', uid, user._joindate || startTime, function(){
												Importer.warn(userData.username + ' became an Administrator');
												onLevel();
											});
										} else {
											onLevel();
										}

									}
								};  // end onCreate

								if (oldOwnerNotFound
									&& parseInt(user._uid, 10) === parseInt(config.adminTakeOwnership._uid, 10)
									|| (user._username || '').toLowerCase() === config.adminTakeOwnership._username.toLowerCase()
								) {
									Importer.warn('[process-count-at:' + count + '] skipping user: ' + user._username + ':'+ user._uid + ', it was revoked ownership');
									// cache the _uid for the next phases
									Importer.config('adminTakeOwnership', {
										enable: true,
										username: user._username,
										// just an alias in this case
										_username: user._username,
										_uid: user._uid
									});
									// no need to make it a mod or an admin, it already is
									user._level = null;
									// set to false so we don't have to match all users
									oldOwnerNotFound = false;
									// dont create, but set the fields
									return onCreate(null, LOGGEDIN_UID);
								} else {
									User.create(userData, onCreate);
								}
							});
						},
						nextExportBatch);
				},
				{
					// options
				},
				function(err) {
					if (err) {
						throw err;
					}
					Importer.success('Imported ' + imported + '/' + total + ' users' + (alreadyImported ? ' (out of which ' + alreadyImported + ' were already imported at an earlier time)' : ''));
					var nxt = function () {
						fs.remove(picturesTmpPath, function() {
							fs.remove(DIRTY_USERS_FILE, next);
						});
					};
					if (config.autoConfirmEmails && Data.keys) {
						async.parallel([
							function (done) {
								Data.keys('confirm:*', function (err, keys) {
									keys.forEach(function (key) {
										db.delete(key);
									});
									done();
								});
							},
							function (done) {
								Data.keys('email:*:confirm', function (err, keys) {
									keys.forEach(function (key) {
										db.delete(key);
									});
									done();
								});
							}
						], function () {
							Importer.progress(1, 1);
							Importer.phase('usersImportDone');
							nxt();
						});
					} else {
						Importer.progress(1, 1);
						Importer.phase('usersImportDone');
						nxt();
					}
				});
		});
	};


	Importer.importMessages = function(next) {
		Importer.phase('messagesImportStart');
		Importer.progress(0, 1);

		Importer._lastPercentage = 0;
		var count = 0,
			imported = 0,
			alreadyImported = 0,
			startTime = +new Date();

		fs.writeFileSync(DIRTY_MESSAGES_FILE, +new Date(), {encoding: 'utf8'});
		Importer.exporter.countMessages(function(err, total) {
			Importer.success('Importing ' + total + ' messages.');
			Importer.exporter.exportMessages(
				function(err, messages, messagesArr, nextExportBatch) {

					async.eachSeries(messagesArr, function(message, done) {
            count++;
            var _mid = message._mid;

            recoverImportedMessage(_mid, function(err, _message) {
              if (_message) {
                Importer.progress(count, total);
                imported++;
                alreadyImported++;
                return done();
              }

              if (message._fromuid == message._touid) {
                Importer.warn('[process-count-at:' + count + '] skipping message:_mid: ' + _mid + ', because it was send to self');
                Importer.progress(count, total);
                return done();
              }

              async.parallel([
                function(cb) {
                  Data.getImportedUser(message._fromuid, function(err, toUser) {
                    if (err) {
                      Importer.warn('getImportedUser:_fromuid:' + message._fromuid + ' err: ' + err.message);
                    }
                    cb(null, toUser);
                  });
                },
                function(cb) {
                  Data.getImportedUser(message._touid, function(err, toUser) {
                    if (err) {
                      Importer.warn('getImportedUser:_touid:' + message._touid + ' err: ' + err.message);
                    }
                    cb(null, toUser);
                  });
                }
              ], function(err, results) {
                var fromUser = results[0];
                var toUser = results[1];

                if (!fromUser || !toUser) {
                  Importer.warn('[process-count-at: ' + count + '] skipping message:_mid: ' + _mid + ' _fromuid:' + message._fromuid + ':imported: ' + !!fromUser + ', _touid:' + message._touid + ':imported: ' + !!toUser);
                  Importer.progress(count, total);
                  done();
                } else {

                  Importer.log('[process-count-at: ' + count + '] saving message:_mid: ' + _mid + ' _fromuid:' + message._fromuid + ', _touid:' + message._touid);

                  Messaging.addMessage(fromUser.uid, toUser.uid, message._content, message._timestamp, function(err, messageReturn) {
                    if (err || !messageReturn) {
                      Importer.warn('[process-count-at: ' + count + '] skipping message:_mid: ' + _mid + ' _fromuid:' + message._fromuid + ':imported: ' + !!fromUser + ', _touid:' + message._touid + ':imported: ' + !!toUser
                      + (err ? ' err: ' + err.message : ' messageReturn: ' + !!messageReturn));
                      Importer.progress(count, total);
                      return done();
                    }

                    imported++;
                    var mid = messageReturn.mid;
                    var uids =  [messageReturn.fromuid, messageReturn.touid].sort();

                    var _imported_content = message._content;

                    delete messageReturn._key;
                    delete messageReturn.toUser;
                    delete messageReturn.fromUser;

                    async.parallel([
                      function(next) {
                        db.setObjectField('message:' + mid, '_imported_content', _imported_content, next);
                      },
                      function(next) {
                        db.sortedSetAdd('uid:' + messageReturn.fromuid + ':chats', message._timestamp, messageReturn.touid, next);
                      },
                      function(next) {
                        db.sortedSetAdd('uid:' + messageReturn.touid + ':chats', message._timestamp, messageReturn.fromuid, next);
                      },
                      function(next) {
                        db.sortedSetRemove('uid:' + messageReturn.fromuid + ':chats:unread', messageReturn.touid, next);
                      },
                      function(next) {
                        db.sortedSetRemove('uid:' + messageReturn.touid + ':chats:unread', messageReturn.fromuid, next);
                      }
                    ], function(err) {
                      if (err) {
                        Importer.warn('[process-count-at: ' + count + '] message creation error message:_mid: ' + _mid + ':mid:' + mid, err);
                        return done();
                      }

                      Importer.progress(count, total);
                      message = nodeExtend(true, {}, message, messageReturn);
                      Data.setMessageImported(_mid, mid, message, done);
                    });
                  });
                }
              });
            });
          }, nextExportBatch);

				},
				{
					// options
				},
				function() {
					Importer.progress(1, 1);
					Importer.phase('messagesImportDone');
					Importer.success('Imported ' + imported + '/' + total + ' messages' + (alreadyImported ? ' (out of which ' + alreadyImported + ' were already imported at an earlier time)' : ''));
					fs.remove(DIRTY_MESSAGES_FILE, next);
				});
		});
	};


	Importer.importCategories = function(next) {
		Importer.phase('categoriesImportStart');
		Importer.progress(0, 1);

		Importer._lastPercentage = 0;

		var count = 0,
			imported = 0,
			alreadyImported = 0,
			startTime = +new Date(),
			config = Importer.config();

		fs.writeFileSync(DIRTY_CATEGORIES_FILE, +new Date(), {encoding: 'utf8'});

		Importer.exporter.countCategories(function(err, total) {
			Importer.success('Importing ' + total + ' categories.');
			Importer.exporter.exportCategories(
				function(err, categories, categoriesArr, nextExportBatch) {
					var onEach = function(category, done) {
						count++;
						var _cid = category._cid;

						recoverImportedCategory(_cid, function(err, _category) {
							if (_category) {
								imported++;
								alreadyImported++;
								Importer.progress(count, total);
								return done();
							}

							Importer.log('[process-count-at:' + count + '] saving category:_cid: ' + _cid);

							var categoryData = {
								name: category._name || ('Category ' + (count + 1)),
								description: category._description || 'no description available',

								// force all categories Parent to be 0, then after the import is done, we can iterate again and fix them.
								parentCid: 0,
								// same deal with disabled
								disabled: 0,

								// you can fix the order later, nbb/admin
								order: category._order || (count + 1),

								link: category._link || 0
							};

							if (config.categoriesIcons && config.categoriesIcons.length) {
								categoryData.icon = config.categoriesIcons[Math.floor(Math.random() * config.categoriesIcons.length)];
							}
							if (config.categoriesBgColors && config.categoriesBgColors.length) {
								categoryData.bgColor = config.categoriesBgColors[Math.floor(Math.random() * config.categoriesBgColors.length)];
							}
							if (config.categoriesTextColors && config.categoriesTextColors.length) {
								categoryData.color = config.categoriesTextColors[Math.floor(Math.random() * config.categoriesTextColors.length)];
							}

							var onCreate = function(err, categoryReturn) {
								if (err) {
									Importer.warn('skipping category:_cid: ' + _cid + ' : ' + err);
									Importer.progress(count, total);
									return done();
								}

								var onFields = function(err) {
									if (err) {
										Importer.warn(err);
									}

									Importer.progress(count, total);

									category.imported = true;
									imported++;
									category = nodeExtend(true, {}, category, categoryReturn, fields);
									categories[_cid] = category;

									Data.setCategoryImported(_cid, categoryReturn.cid, category, done);
								};

								var fields = {
									_imported_cid: _cid,
									_imported_path: category._path || '',
									_imported_name: category._name || '',
									_imported_slug: category._slug || '',
									_imported_parentCid: category._parent || category._parentCid || '',
									_imported_disabled: category._disabled || 0,
									_imported_description: category._description || ''
								};

								db.setObject('category:' + categoryReturn.cid, fields, onFields);
							};

							Categories.create(categoryData, onCreate);
						});
					};
					async.eachSeries(categoriesArr, onEach, nextExportBatch);
				},
				{
					// options
				},
				function(err) {
					if (err) {
						throw err;
					}
					Importer.success('Imported ' + imported + '/' + total + ' categories' + (alreadyImported ? ' (out of which ' + alreadyImported + ' were already imported at an earlier time)' : ''));
					Importer.progress(1, 1);
					Importer.phase('categoriesImportDone');
					fs.remove(DIRTY_CATEGORIES_FILE, next);
				});
		});
	};

	Importer.importGroups = function(next) {
		Importer.phase('groupsImportStart');
		Importer.progress(0, 1);

		Importer._lastPercentage = 0;

		var count = 0,
			imported = 0,
			alreadyImported = 0,
			startTime = +new Date(),
			config = Importer.config();

		fs.writeFileSync(DIRTY_GROUPS_FILE, +new Date(), {encoding: 'utf8'});

		Importer.exporter.countGroups(function(err, total) {
			Importer.success('Importing ' + total + ' groups.');
			Importer.exporter.exportGroups(
				function(err, groups, groupsArr, nextExportBatch) {

					var onEach = function(group, done) {
						count++;
						var _gid = group._gid;

						recoverImportedGroup(_gid, function(err, _group) {
							if (_group) {
								imported++;
								alreadyImported++;
								Importer.progress(count, total);
								return done();
							}

							Importer.log('[process-count-at:' + count + '] saving group:_gid: ' + _gid);

							var groupData = {
								name: group._name || ('Group ' + (count + 1)),
								description: group._description || 'no description available'
							};

							var onCreate = function(err, groupReturn) {
								if (err) {
									Importer.warn('skipping group:_gid: ' + _gid + ' : ' + err);
									Importer.progress(count, total);
									return done();
								}

								var onFields = function(err) {
									if (err) {
										Importer.warn(err);
									}

									var onTime = function () {

										Importer.progress(count, total);

										group.imported = true;
										imported++;
										group = nodeExtend(true, {}, group, groupReturn, fields);
										groups[_gid] = group;

										Data.setGroupImported(_gid, 0, group, done);
									};

									if (group._createtime || group._timestamp) {
										db.sortedSetAdd('groups:createtime', group._createtime || group._timestamp, groupReturn.name, function() {
											onTime();
										});
									} else {
										onTime();
									}
								};

								var fields = {
									_imported_gid: _gid,
									_imported_name: group._name,
									_imported_ownerUid: group._ownerUid || '',
									_imported_path: group._path || '',
									_imported_slug: group._slug || '',
									_imported_description: group._description || ''
								};

								if (group._createtime || group._timestamp) {
									fields.createtime = group._createtime || group._timestamp;
								}

								db.setObject('group:' + groupReturn.name, fields, onFields);
							};

							Groups.create(groupData, onCreate);
						});
					};
					async.eachSeries(groupsArr, onEach, nextExportBatch);
				},
				{
					// options
				},
				function(err) {
					if (err) {
						throw err;
					}
					Importer.success('Importing ' + imported + '/' + total + ' groups' + (alreadyImported ? ' (out of which ' + alreadyImported + ' were already imported at an earlier time)' : ''));
					Importer.progress(1, 1);
					Importer.phase('groupsImportDone');
					fs.remove(DIRTY_GROUPS_FILE, next);
				});
		});
	};

	Importer.allowGuestsOnAllCategories = function(done) {
		Data.eachCategory(function(category, next) {
				async.parallel([
					function(nxt) {
						Groups.join('cid:' + category.cid + ':privileges:groups:topics:create', 'guests', nxt);
					},
					function(nxt) {
						Groups.join('cid:' + category.cid + ':privileges:groups:topics:reply', 'guests', nxt);
					}
				], next);
			},
			{async: true, eachLimit: 10},
			function() {
				done();
			});
	};

	Importer.disallowGuestsWriteOnAllCategories = function(done) {
		Data.eachCategory(function(category, next) {
				async.parallel([
					function(nxt) {
						// 'cid:' + cids[i] + ':privileges:groups:' + privilege'
						Groups.leave('cid:' + category.cid + ':privileges:groups:topics:create', 'guests', nxt);
					},
					function(nxt) {
						Groups.leave('cid:' + category.cid + ':privileges:groups:topics:reply', 'guests', nxt);
					}
				], next);
			},
			{async: true, eachLimit: 10},
			function() {
				done();
			});
	};

	Importer.importTopics = function(next) {
		Importer.phase('topicsImportStart');
		Importer.progress(0, 1);

		Importer._lastPercentage = 0;
		var count = 0,
			imported = 0,
			alreadyImported = 0,
			startTime = +new Date(),
			attachmentsTmpPath = path.join(__dirname, '/tmp/attachments'),
			folder = '_imported_attachments',
			attachmentsPublicPath = path.join(nconf.get('base_dir'), nconf.get('upload_path'), '_imported_attachments'),
			config = Importer.config();

		fs.writeFileSync(DIRTY_TOPICS_FILE, +new Date(), {encoding: 'utf8'});
		fs.ensureDirSync(attachmentsTmpPath);
		fs.ensureDirSync(attachmentsPublicPath);

		Importer.exporter.countTopics(function(err, total) {
			Importer.success('Importing ' + total + ' topics.');
			Importer.exporter.exportTopics(
				function(err, topics, topicsArr, nextExportBatch) {

					async.eachSeries(topicsArr, function(topic, done) {
						count++;
						var _tid = topic._tid;

						recoverImportedTopic(_tid, function(err, _topic) {
							if (_topic) {
								Importer.progress(count, total);
								imported++;
								alreadyImported++;
								return done();
							}

							async.parallel([
								function(cb) {
									Data.getImportedCategory(topic._cid, function(err, cat) {
										if (err) {
											Importer.warn('getImportedCategory: ' + topic._cid + ' err: ' + err);
										}
										cb(null, cat);
									});
								},
								function(cb) {
									Data.getImportedUser(topic._uid, function(err, usr) {
										if (err) {
											Importer.warn('getImportedUser: ' + topic._uid + ' err: ' + err);
										}
										cb(null, usr);
									});
								}
							], function(err, results) {

								var category = results[0];
								var user = results[1] || {uid: '0'};

								if (!category) {
									Importer.warn('[process-count-at:' + count + '] skipping topic:_tid:"' + _tid + '" --> _cid: ' + topic._cid + ':imported:' + !!category);
									Importer.progress(count, total);
									done();
								} else {
									Importer.log('[process-count-at:' + count + '] saving topic:_tid: ' + _tid);

									if (topic._attachmentsBlobs && topic._attachmentsBlobs.length) {
										var attachmentsIndex = 0;

										topic._attachments = [].concat(topic._attachments || []);
										topic._images = [].concat(topic._images || []);

										async.eachSeries(topic._attachmentsBlobs, function(_attachmentsBlob, next) {
											var filename = 'attachment_t_' + _tid + '_' + attachmentsIndex++ + (_attachmentsBlob.filename ? '_' + _attachmentsBlob.filename : _attachmentsBlob.extension);
											var tmpPath = path.join(attachmentsTmpPath, filename);

											writeBlob(tmpPath, _attachmentsBlob.blob, function(err, ftype) {
												if (err) {
													Importer.warn(tmpPath, err);
													next();
												} else {
													File.saveFileToLocal(filename, folder, tmpPath, function(err, ret) {
														if (!err) {
															if (/image/.test(ftype.mime)) {
																topic._images.push(ret.url);
															} else {
																topic._attachments.push(ret.url);
															}
														} else {
															Importer.warn(filename, err);
														}
														next();
													});
												}
											});

										}, onAttachmentsBlobs);

									} else {
										onAttachmentsBlobs();
									}

									function onAttachmentsBlobs () {
										topic._content = (topic._content || '').trim();

										topic._title = utils.slugify(topic._title) ? topic._title[0].toUpperCase() + topic._title.substr(1) : utils.truncate(topic._content, 100);

										(topic._images || []).forEach(function(_image) {
											topic._content += generateImageTag(_image);
										});
										(topic._attachments || []).forEach(function(_attachment) {
											topic._content += generateAnchorTag(_attachment);
										});

										if (topic._tags && !Array.isArray(topic._tags)) {
											topic._tags = ('' + topic._tags).split(',');
										}

										Topics.post({
											uid: !config.adminTakeOwnership.enable ? user.uid : parseInt(config.adminTakeOwnership._uid, 10) === parseInt(topic._uid, 10) ? LOGGEDIN_UID : user.uid,
											title: topic._title,
											content: topic._content,
											timestamp: topic._timestamp,
											cid: category.cid,
											thumb: topic._thumb,
											tags: topic._tags
										}, function (err, returnTopic) {
											if (err) {
												Importer.warn('[process-count-at:' + count + '] skipping topic:_tid: ' + _tid + ':cid:' + category.cid + ':_cid:' + topic._cid + ':uid:' + user.uid +  ':_uid:' + topic._uid + ' err: ' + err);
												Importer.progress(count, total);
												done();
											} else {

												topic.imported = true;
												imported++;

												var topicFields = {
													viewcount: topic._viewcount || topic._viewscount || 0,

													// assume that this topic not locked for now, but will iterate back again at the end and lock it back after finishing the importPosts()
													// locked: normalizedTopic._locked ? 1 : 0,
													locked: 0,

													deleted: topic._deleted ? 1 : 0,

													// if pinned, we should set the db.sortedSetAdd('cid:' + cid + ':tids', Math.pow(2, 53), tid);
													pinned: topic._pinned ? 1 : 0,

													_imported_tid: _tid,
													_imported_uid: topic._uid || '',
													_imported_cid: topic._cid,
													_imported_slug: topic._slug || '',
													_imported_path: topic._path || '',
													_imported_title: topic._title || '',
													_imported_content: topic._content || '',
													_imported_guest: topic._guest || '',
													_imported_ip: topic._ip || '',
													_imported_user_slug: user._slug || '',
													_imported_user_path: user._path || '',
													_imported_category_path: category._path || '',
													_imported_category_slug: category._slug || ''
												};

												var postFields = {
													votes: topic._votes || 0,
													reputation: topic._reputation || 0,
													edited: topic._edited || 0
												};

												// if (topic._edited) {
												// 	postFields.edited = topic._edited;
												// }

												// console.log("Topic Edited:", postFields.edited, "type:", typeof postFields.edited);

												var onPinned = function() {
													db.setObject('topic:' + returnTopic.topicData.tid, topicFields, function(err, result) {
														Importer.progress(count, total);
														if (err) {
															Importer.warn(err);
														}
														Posts.setPostFields(returnTopic.postData.pid, postFields, function(){
															topic = nodeExtend(true, {}, topic, topicFields, returnTopic.topicData);
															topics[_tid] = topic;
															Data.setTopicImported(_tid, returnTopic.topicData.tid, topic, done);
														});
													});
												};

												// pinned = 1 not enough to float the topic to the top in it's category
												if (topic._pinned) {
													db.sortedSetAdd('cid:' + category.cid + ':tids', Math.pow(2, 53), returnTopic.topicData.tid, onPinned);
												}  else {
													db.sortedSetAdd('cid:' + category.cid + ':tids', topic._timestamp, returnTopic.topicData.tid, onPinned);
												}
											}
										});
									}
								}
							});
						});
					}, nextExportBatch);
				},
				{
					//options
				},
				function(err) {
					if (err) {
						throw err;
					}
					Importer.success('Imported ' + imported + '/' + total + ' topics' + (alreadyImported ? ' (out of which ' + alreadyImported + ' were already imported at an earlier time)' : ''));
					Importer.progress(1, 1);
					Importer.phase('topicsImportDone');

					async.series([
						function(nxt) {
{'fraction_non_alphanumeric': 0.13171297917688143, 'fraction_numerical': 0.005293334800924582, 'mean_word_length': 2.412367261925086, 'pattern_counts': {'":': 40, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1024880', 'n_tokens_mistral': 43312, 'n_tokens_neox': 30069, 'n_words': 5693}