<commit_before>/// <reference path="../typings/DefinitelyTyped/node/node.d.ts" />
/// <reference path="../typings/DefinitelyTyped/q/Q.d.ts" />

// local_store.Store implements an IndexedDB-based store
// of encryption keys and encrypted item data.

// Local Store Schema (v2)
//
// keys: // encryption key and password hint storage
//   hint: string // password hint as plain text
//   key/<ID>: key_agent.Key // encrypted encryption key, plus metadata
//                           // for PBKDF2
//
//
// items: // encrypted item content and sync metadata
//   index: OverviewMap // current overview and revision data for all items
//   lastSynced/<Item ID>: string // last-synced revision of item <Item ID>
//   revision/<rev ID>: ItemRevision // item overview and content (both encrypted) for
//                                   // a particular revision of an item

import assert = require('assert');
import Q = require('q');
import underscore = require('underscore');

import agile_keychain_crypto = require('./agile_keychain_crypto');
import asyncutil = require('./base/asyncutil');
import cached = require('./base/cached');
import collectionutil = require('./base/collectionutil');
import event_stream = require('./base/event_stream');
import item_store = require('./item_store');
import key_agent = require('./key_agent');
import key_value_store = require('./base/key_value_store');

// JSON structure that stores the current overview
// data and revision IDs for all items in the database
interface OverviewMap {
	[index: string]: ItemIndexEntry;
}

// JSON structure used to store item revisions
// in the database
interface ItemRevision {
	parentRevision: string;
	overview: ItemOverview;
	content: item_store.ItemContent;
}

// JSON structure used to store item overview
// data in the database
interface ItemOverview {
	title: string;
	updatedAt: number;
	createdAt: number;
	trashed: boolean;
	typeName: string;
	openContents: item_store.ItemOpenContents;

	locations: string[];
	account: string;

	revision: string;
	parentRevision: string;
}

// JSON structure used to store entries in the item
// index. This consists of item overview data
// plus the last-sync timestamp
interface ItemIndexEntry extends ItemOverview {
	lastSyncedAt?: number;
}

interface LastSyncEntry {
	timestamp: number;
	/** Revision of the item in the local store. */
	local: string;
	/** Revision of the item in the cloud store. */
	external: string;
}

var SCHEMA_VERSION = 3;

// prefix for encryption key entries in the encryption key
// object store
var KEY_ID_PREFIX = 'key/';

export class Store implements item_store.SyncableStore {
	private crypto: agile_keychain_crypto.Crypto;
	private database: key_value_store.Database;
	private name: string;
	private keyAgent: key_agent.KeyAgent;
	private keyStore: key_value_store.ObjectStore;
	private itemStore: key_value_store.ObjectStore;
	private indexUpdateQueue: collectionutil.BatchedUpdateQueue<item_store.Item>;
	private itemIndex: cached.Cached<OverviewMap>;

	onItemUpdated: event_stream.EventStream<item_store.Item>;
	onUnlock: event_stream.EventStream<void>;

	constructor(database: key_value_store.Database, name: string, keyAgent: key_agent.KeyAgent) {
		this.database = database;
		this.keyAgent = keyAgent;
		this.crypto = agile_keychain_crypto.defaultCrypto;
		this.name = name;

		this.onItemUpdated = new event_stream.EventStream<item_store.Item>();
		this.onUnlock = new event_stream.EventStream<void>();

		this.initDatabase();

		this.indexUpdateQueue = new collectionutil.BatchedUpdateQueue((updates: item_store.Item[]) => {
			return this.updateIndex(updates);
		});

		this.itemIndex = new cached.Cached<OverviewMap>(
			() => { return this.readItemIndex() },
			(update) => { return this.writeItemIndex(update) }
			);
		this.keyAgent.onLock().listen(() => {
			this.itemIndex.clear();
		});
	}

	private resetDatabase(schemaUpdater: key_value_store.DatabaseSchemaModifier) {
		schemaUpdater.storeNames().forEach((name) => {
			schemaUpdater.deleteStore(name);
		});
		schemaUpdater.createStore('keys');
		schemaUpdater.createStore('items');
	}

	private initDatabase() {
		this.database.open(this.name, SCHEMA_VERSION, schemaUpdater => {
			// when opening databases with schema versions prior to the
			// first public beta release, we just reset the database
			// and re-sync from the cloud
			if (schemaUpdater.currentVersion() < 3) {
				this.resetDatabase(schemaUpdater);
			}
		});

		this.keyStore = this.database.store('keys');
		this.itemStore = this.database.store('items');
	}

	clear() {
		this.itemIndex.clear();
		return this.database.delete().then(() => {
			this.initDatabase();
		});
	}

	unlock(pwd: string): Q.Promise<void> {
		return item_store.unlockStore(this, this.keyAgent, pwd);
	}

	listItemStates(): Q.Promise<item_store.ItemState[]> {
		return item_store.itemStates(this);
	}

	listItems(opts: item_store.ListItemsOptions = {}): Q.Promise<item_store.Item[]> {
		return this.itemIndex.get().then((overviewMap) => {
			var items: item_store.Item[] = [];
			Object.keys(overviewMap).forEach((key) => {
				var overview = overviewMap[key];
				var item = this.itemFromOverview(key, overview);
				if (!item.isTombstone() || opts.includeTombstones) {
					items.push(item);
				}
			});

			return items;
		});
	}

	private getLastSyncEntries(ids: string[]): Q.Promise<LastSyncEntry[]> {
		return Q.all(ids.map((id) => {
			return this.itemStore.get<LastSyncEntry>('lastSynced/' + id);
		}));
	}

	private updateIndex(updatedItems: item_store.Item[]) {
		var overviewMap = <OverviewMap>{};
		var updatedItemIds = updatedItems.map((item) => {
			return item.uuid;
		});

		return Q.all([this.itemIndex.get(), this.getLastSyncEntries(updatedItemIds)])
		.then((result) => {
			overviewMap = <OverviewMap>result[0];
			if (!overviewMap) {
				overviewMap = {};
			}
			var lastSyncTimes = (<LastSyncEntry[]>result[1]).map((entry) => {
				return entry ? entry.timestamp : 0;
			});
			updatedItems.forEach((item, index) => {
				var entry: ItemIndexEntry = this.overviewFromItem(item);
				entry.lastSyncedAt = lastSyncTimes[index];
				assert(entry.lastSyncedAt !== null);
				overviewMap[item.uuid] = entry;
			});
			return this.itemIndex.set(overviewMap);
		});
	}

	private itemFromOverview(uuid: string, overview: ItemOverview) {
		var item = new item_store.Item(this, uuid);
		item.title = overview.title;
		item.updatedAt = new Date(overview.updatedAt);
		item.createdAt = new Date(overview.createdAt);
		item.trashed = overview.trashed;
		item.typeName = overview.typeName;
		item.openContents = overview.openContents;

		item.account = overview.account;
		item.locations = overview.locations;

		item.revision = overview.revision;
		item.parentRevision = overview.parentRevision;

		return item;
	}

	private overviewFromItem(item: item_store.Item) {
		return <ItemOverview>{
			title: item.title,
			updatedAt: item.updatedAt.getTime(),
			createdAt: item.createdAt.getTime(),
			trashed: item.trashed,
			typeName: <string>item.typeName,
			openContents: item.openContents,

			locations: item.locations,
			account: item.account,

			revision: item.revision,
			parentRevision: item.parentRevision
		};
	}

	loadItem(uuid: string, revision?: string): Q.Promise<item_store.ItemAndContent> {
		if (revision) {
			return Q.all([this.overviewKey(), this.itemStore.get<string>('revisions/' + revision)])
			.then(keyAndRevision => {
				var key = <string>keyAndRevision[0];
				var revisionData = <string>keyAndRevision[1];
				return this.decrypt<ItemRevision>(key, revisionData);
			}).then(revision => {
				var item = this.itemFromOverview(uuid, revision.overview);
				assert.equal(item.revision, revision.overview.revision);
				item.parentRevision = revision.parentRevision;
				return {
					item: item,
					content: revision.content
				};
			});
		} else {
			return this.itemIndex.get().then(overviewMap => {
				if (uuid in overviewMap) {
					return this.loadItem(uuid, overviewMap[uuid].revision);
				} else {
					throw new Error('No such item ' + uuid);
				}
			});
		}
	}
<commit_msg>Fix item sync<commit_after>/// <reference path="../typings/DefinitelyTyped/node/node.d.ts" />
/// <reference path="../typings/DefinitelyTyped/q/Q.d.ts" />

// local_store.Store implements an IndexedDB-based store
// of encryption keys and encrypted item data.

// Local Store Schema (v2)
//
// keys: // encryption key and password hint storage
//   hint: string // password hint as plain text
//   key/<ID>: key_agent.Key // encrypted encryption key, plus metadata
//                           // for PBKDF2
//
//
// items: // encrypted item content and sync metadata
//   index: OverviewMap // current overview and revision data for all items
//   lastSynced/<Item ID>: string // last-synced revision of item <Item ID>
//   revision/<rev ID>: ItemRevision // item overview and content (both encrypted) for
//                                   // a particular revision of an item

import assert = require('assert');
import Q = require('q');
import underscore = require('underscore');

import agile_keychain_crypto = require('./agile_keychain_crypto');
import assign = require('./base/assign');
import asyncutil = require('./base/asyncutil');
import cached = require('./base/cached');
import collectionutil = require('./base/collectionutil');
import event_stream = require('./base/event_stream');
import item_store = require('./item_store');
import key_agent = require('./key_agent');
import key_value_store = require('./base/key_value_store');

// JSON structure that stores the current overview
// data and revision IDs for all items in the database
interface OverviewMap {
	[index: string]: ItemIndexEntry;
}

// JSON structure used to store item revisions
// in the database
interface ItemRevision {
	parentRevision: string;
	overview: ItemOverview;
	content: item_store.ItemContent;
}

// JSON structure used to store item overview
// data in the database
interface ItemOverview {
	title: string;
	updatedAt: number;
	createdAt: number;
	trashed: boolean;
	typeName: string;
	openContents: item_store.ItemOpenContents;

	locations: string[];
	account: string;

	revision: string;
	parentRevision: string;
}

// JSON structure used to store entries in the item
// index. This consists of item overview data
// plus the last-sync timestamp
interface ItemIndexEntry extends ItemOverview {
	lastSyncedAt?: number;
}

interface LastSyncEntry {
	timestamp: number;
	/** Revision of the item in the local store. */
	local: string;
	/** Revision of the item in the cloud store. */
	external: string;
}

var SCHEMA_VERSION = 3;

// prefix for encryption key entries in the encryption key
// object store
var KEY_ID_PREFIX = 'key/';

export class Store implements item_store.SyncableStore {
	private crypto: agile_keychain_crypto.Crypto;
	private database: key_value_store.Database;
	private name: string;
	private keyAgent: key_agent.KeyAgent;
	private keyStore: key_value_store.ObjectStore;
	private itemStore: key_value_store.ObjectStore;
	private indexUpdateQueue: collectionutil.BatchedUpdateQueue<item_store.Item>;
	private itemIndex: cached.Cached<OverviewMap>;

	onItemUpdated: event_stream.EventStream<item_store.Item>;
	onUnlock: event_stream.EventStream<void>;

	constructor(database: key_value_store.Database, name: string, keyAgent: key_agent.KeyAgent) {
		this.database = database;
		this.keyAgent = keyAgent;
		this.crypto = agile_keychain_crypto.defaultCrypto;
		this.name = name;

		this.onItemUpdated = new event_stream.EventStream<item_store.Item>();
		this.onUnlock = new event_stream.EventStream<void>();

		this.initDatabase();

		this.indexUpdateQueue = new collectionutil.BatchedUpdateQueue((updates: item_store.Item[]) => {
			return this.updateIndex(updates);
		});

		this.itemIndex = new cached.Cached<OverviewMap>(
			() => { return this.readItemIndex() },
			(update) => { return this.writeItemIndex(update) }
			);
		this.keyAgent.onLock().listen(() => {
			this.itemIndex.clear();
		});
	}

	private resetDatabase(schemaUpdater: key_value_store.DatabaseSchemaModifier) {
		schemaUpdater.storeNames().forEach((name) => {
			schemaUpdater.deleteStore(name);
		});
		schemaUpdater.createStore('keys');
		schemaUpdater.createStore('items');
	}

	private initDatabase() {
		this.database.open(this.name, SCHEMA_VERSION, schemaUpdater => {
			// when opening databases with schema versions prior to the
			// first public beta release, we just reset the database
			// and re-sync from the cloud
			if (schemaUpdater.currentVersion() < 3) {
				this.resetDatabase(schemaUpdater);
			}
		});

		this.keyStore = this.database.store('keys');
		this.itemStore = this.database.store('items');
	}

	clear() {
		this.itemIndex.clear();
		return this.database.delete().then(() => {
			this.initDatabase();
		});
	}

	unlock(pwd: string): Q.Promise<void> {
		return item_store.unlockStore(this, this.keyAgent, pwd);
	}

	listItemStates(): Q.Promise<item_store.ItemState[]> {
		return item_store.itemStates(this);
	}

	listItems(opts: item_store.ListItemsOptions = {}): Q.Promise<item_store.Item[]> {
		return this.itemIndex.get().then((overviewMap) => {
			var items: item_store.Item[] = [];
			Object.keys(overviewMap).forEach((key) => {
				var overview = overviewMap[key];
				var item = this.itemFromOverview(key, overview);
				if (!item.isTombstone() || opts.includeTombstones) {
					items.push(item);
				}
			});

			return items;
		});
	}

	private getLastSyncEntries(ids: string[]): Q.Promise<LastSyncEntry[]> {
		return Q.all(ids.map((id) => {
			return this.itemStore.get<LastSyncEntry>('lastSynced/' + id);
		}));
	}

	private updateIndex(updatedItems: item_store.Item[]) {
		var overviewMap = <OverviewMap>{};
		var updatedItemIds = updatedItems.map((item) => {
			return item.uuid;
		});

		return Q.all([this.itemIndex.get(), this.getLastSyncEntries(updatedItemIds)])
		.then((result) => {
			overviewMap = <OverviewMap>result[0];
			if (!overviewMap) {
				overviewMap = {};
			}
			var lastSyncTimes = (<LastSyncEntry[]>result[1]).map((entry) => {
				return entry ? entry.timestamp : 0;
			});
			updatedItems.forEach((item, index) => {
				var entry: ItemIndexEntry = this.overviewFromItem(item);
				entry.lastSyncedAt = lastSyncTimes[index];
				assert(entry.lastSyncedAt !== null);
				overviewMap[item.uuid] = entry;
			});
			return this.itemIndex.set(overviewMap);
		});
	}

	private itemFromOverview(uuid: string, overview: ItemOverview) {
		var item = new item_store.Item(this, uuid);
		item.title = overview.title;
		item.updatedAt = new Date(overview.updatedAt);
		item.createdAt = new Date(overview.createdAt);
		item.trashed = overview.trashed;
		item.typeName = overview.typeName;
		item.openContents = overview.openContents;

		item.account = overview.account;
		item.locations = overview.locations;

		item.revision = overview.revision;
		item.parentRevision = overview.parentRevision;

		return item;
	}

	private overviewFromItem(item: item_store.Item) {
		return <ItemOverview>{
			title: item.title,
			updatedAt: item.updatedAt.getTime(),
			createdAt: item.createdAt.getTime(),
			trashed: item.trashed,
			typeName: <string>item.typeName,
			openContents: item.openContents,

			locations: item.locations,
			account: item.account,

			revision: item.revision,
			parentRevision: item.parentRevision
		};
	}

	loadItem(uuid: string, revision?: string): Q.Promise<item_store.ItemAndContent> {
		if (revision) {
			return Q.all([this.overviewKey(), this.itemStore.get<string>('revisions/' + revision)])
			.then(keyAndRevision => {
				var key = <string>keyAndRevision[0];
				var revisionData = <string>keyAndRevision[1];
				return this.decrypt<ItemRevision>(key, revisionData);
			}).then(revision => {
				var item = this.itemFromOverview(uuid, revision.overview);
				assert.equal(item.revision, revision.overview.revision);
				item.parentRevision = revision.parentRevision;

				// Restore the prototype of the de-serialized ItemContent
				// FIXME - Refactor ItemContent and fields to make it
				// (de-)serializable without needing a hack to restore prototypes
				let content = new item_store.ItemContent();
				assign(content, revision.content);

				return {
					item: item,
					content: content
				};
			});
		} else {
			return this.itemIndex.get().then(overviewMap => {
				if (uuid in overviewMap) {
					return this.loadItem(uuid, overviewMap[uuid].revision);
				} else {
					throw new Error('No such item ' + uuid);
				}
			});
		}
	}

{'fraction_non_alphanumeric': 0.13170643090710912, 'fraction_numerical': 0.001089984255782972, 'mean_word_length': 4.072174447174447, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 93, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3261037', 'n_tokens_mistral': 5613, 'n_tokens_neox': 4987, 'n_words': 1176}