From f21638fcd6a3b2fc3bc333abcc342e56dcf8ba7d Mon Sep 17 00:00:00 2001 From: Ewout Stortenbeker Date: Sat, 29 Oct 2022 11:49:22 +0200 Subject: [PATCH] moved IndexedDB code into own files --- src/ts/acebase-browser.ts | 335 +----------------- src/ts/storage/custom/indexed-db/index.ts | 79 ++++- .../storage/custom/indexed-db/transaction.ts | 270 ++++++++++++++ 3 files changed, 350 insertions(+), 334 deletions(-) create mode 100644 src/ts/storage/custom/indexed-db/transaction.ts diff --git a/src/ts/acebase-browser.ts b/src/ts/acebase-browser.ts index edb62d7..5bf2b9a 100644 --- a/src/ts/acebase-browser.ts +++ b/src/ts/acebase-browser.ts @@ -1,14 +1,7 @@ -import { LoggingLevel, SimpleCache } from 'acebase-core'; import { AceBase, AceBaseLocalSettings } from './acebase-local'; -import { IPCPeer } from './ipc'; -import { CustomStorageSettings, CustomStorageTransaction, CustomStorageHelpers, ICustomStorageNode, ICustomStorageNodeMetaData } from './storage/custom'; +import { createIndexedDBInstance } from './storage/custom/indexed-db'; import { IndexedDBStorageSettings } from './storage/custom/indexed-db/settings'; -interface IIndexedDBNodeData { - path: string; - metadata: ICustomStorageNodeMetaData; -} - const deprecatedConstructorError = `Using AceBase constructor in the browser to use localStorage is deprecated! Switch to: IndexedDB implementation (FASTER, MORE RELIABLE): @@ -48,330 +41,6 @@ export class BrowserAceBase extends AceBase { * @param settings optional settings */ static WithIndexedDB(dbname: string, init: Partial = {}) { - - const settings = new IndexedDBStorageSettings(init); - - // We'll create an IndexedDB with name "dbname.acebase" - const IndexedDB: IDBFactory = window.indexedDB || (window as any).mozIndexedDB || (window as any).webkitIndexedDB || (window as any).msIndexedDB; // browser prefixes not really needed, see https://caniuse.com/#feat=indexeddb - const request = IndexedDB.open(`${dbname}.acebase`, 1); - - request.onupgradeneeded = (e) => { - // create datastore - const db = request.result; - - // Create "nodes" object store for metadata - db.createObjectStore('nodes', { keyPath: 'path'}); - - // Create "content" object store with all data - db.createObjectStore('content'); - }; - - let db: IDBDatabase; - const readyPromise = new Promise((resolve, reject) => { - request.onsuccess = e => { - db = request.result; - resolve(); - }; - request.onerror = e => { - reject(e); - }; - }); - - const cache = new SimpleCache(typeof settings.cacheSeconds === 'number' ? settings.cacheSeconds : 60); // 60 second node cache by default - // cache.enabled = false; - - const storageSettings = new CustomStorageSettings({ - name: 'IndexedDB', - locking: true, // IndexedDB transactions are short-lived, so we'll use AceBase's path based locking - removeVoidProperties: settings.removeVoidProperties, - maxInlineValueSize: settings.maxInlineValueSize, - lockTimeout: settings.lockTimeout, - ready() { - return readyPromise; - }, - async getTransaction(target: { path: string; write: boolean }) { - await readyPromise; - const context = { - debug: false, - db, - cache, - ipc, - }; - return new IndexedDBStorageTransaction(context, target); - }, - }); - const acebase: AceBase = new BrowserAceBase(dbname, { - logLevel: settings.logLevel, - storage: storageSettings, - sponsor: settings.sponsor, - }); - const ipc = acebase.api.storage.ipc; - acebase.settings.ipcEvents = settings.multipleTabs === true; - ipc.on('notification', async (notification: { data: any }) => { - const message = notification.data; - if (typeof message !== 'object') { return; } - if (message.action === 'cache.invalidate') { - // console.warn(`Invalidating cache for paths`, message.paths); - for (const path of message.paths) { - cache.remove(path); - } - } - }); - return acebase; + return createIndexedDBInstance(dbname, init); } } - -function _requestToPromise(request: IDBRequest) { - return new Promise((resolve, reject) => { - request.onsuccess = event => { - return resolve(request.result || null); - }; - request.onerror = reject; - }); -} - -class IndexedDBStorageTransaction extends CustomStorageTransaction { - - production = true; // Improves performance, only set when all works well - - private _pending: Array<{ path: string; action: 'set' | 'update' | 'remove'; node?: ICustomStorageNode }>; - - /** - * Creates a transaction object for IndexedDB usage. Because IndexedDB automatically commits - * transactions when they have not been touched for a number of microtasks (eg promises - * resolving whithout querying data), we will enqueue set and remove operations until commit - * or rollback. We'll create separate IndexedDB transactions for get operations, caching their - * values to speed up successive requests for the same data. - */ - constructor(public context: {debug: boolean, db: IDBDatabase, cache: SimpleCache, ipc: IPCPeer }, target: { path: string, write: boolean }) { - super(target); - this._pending = []; - } - - _createTransaction(write = false) { - const tx = this.context.db.transaction(['nodes', 'content'], write ? 'readwrite' : 'readonly'); - return tx; - } - - _splitMetadata(node: ICustomStorageNode) { - const value = node.value; - const copy: ICustomStorageNode = { ...node }; - delete copy.value; - const metadata = copy as ICustomStorageNodeMetaData; - return { metadata, value }; - } - - async commit() { - // console.log(`*** commit ${this._pending.length} operations ****`); - if (this._pending.length === 0) { return; } - const batch = this._pending.splice(0); - - this.context.ipc.sendNotification({ action: 'cache.invalidate', paths: batch.map(op => op.path) }); - - const tx = this._createTransaction(true); - try { - await new Promise((resolve, reject) => { - let stop = false, processed = 0; - const handleError = (err: any) => { - stop = true; - reject(err); - }; - const handleSuccess = () => { - if (++processed === batch.length) { - resolve(); - } - }; - batch.forEach((op, i) => { - if (stop) { return; } - let r1, r2; - const path = op.path; - if (op.action === 'set') { - const { metadata, value } = this._splitMetadata(op.node); - const nodeInfo: IIndexedDBNodeData = { path, metadata }; - r1 = tx.objectStore('nodes').put(nodeInfo); // Insert into "nodes" object store - r2 = tx.objectStore('content').put(value, path); // Add value to "content" object store - this.context.cache.set(path, op.node); - } - else if (op.action === 'remove') { - r1 = tx.objectStore('content').delete(path); // Remove from "content" object store - r2 = tx.objectStore('nodes').delete(path); // Remove from "nodes" data store - this.context.cache.set(path, null); - } - else { - handleError(new Error(`Unknown pending operation "${op.action}" on path "${path}" `)); - } - let succeeded = 0; - r1.onsuccess = r2.onsuccess = () => { - if (++succeeded === 2) { handleSuccess(); } - }; - r1.onerror = r2.onerror = handleError; - }); - }); - tx.commit && tx.commit(); - } - catch (err) { - console.error(err); - tx.abort && tx.abort(); - throw err; - } - } - - async rollback(err: any) { - // Nothing has committed yet, so we'll leave it like that - this._pending = []; - } - - async get(path: string) { - // console.log(`*** get "${path}" ****`); - if (this.context.cache.has(path)) { - const cache = this.context.cache.get(path); - // console.log(`Using cached node for path "${path}": `, cache); - return cache; - } - const tx = this._createTransaction(false); - const r1 = _requestToPromise(tx.objectStore('nodes').get(path)); // Get metadata from "nodes" object store - const r2 = _requestToPromise(tx.objectStore('content').get(path)); // Get content from "content" object store - try { - const results = await Promise.all([r1, r2]); - tx.commit && tx.commit(); - const info = results[0] as IIndexedDBNodeData; - if (!info) { - // Node doesn't exist - this.context.cache.set(path, null); - return null; - } - const node = info.metadata as ICustomStorageNode; - node.value = results[1]; - this.context.cache.set(path, node); - return node; - } - catch(err) { - console.error(`IndexedDB get error`, err); - tx.abort && tx.abort(); - throw err; - } - } - - set(path: string, node: ICustomStorageNode) { - // Queue the operation until commit - this._pending.push({ action: 'set', path, node }); - } - - remove(path: string) { - // Queue the operation until commit - this._pending.push({ action: 'remove', path }); - } - - async removeMultiple(paths: string[]) { - // Queues multiple items at once, dramatically improves performance for large datasets - paths.forEach(path => { - this._pending.push({ action: 'remove', path }); - }); - } - - childrenOf( - path: string, - include: { - metadata: boolean; - value: boolean; - }, - checkCallback: (childPath: string) => boolean, - addCallback?: (childPath: string, node?: ICustomStorageNodeMetaData|ICustomStorageNode) => boolean, - ) { - // console.log(`*** childrenOf "${path}" ****`); - return this._getChildrenOf(path, { ...include, descendants: false }, checkCallback, addCallback); - } - - descendantsOf( - path: string, - include: { - metadata: boolean; - value: boolean; - }, - checkCallback: (descPath: string, metadata?: ICustomStorageNodeMetaData) => boolean, - addCallback?: (descPath: string, node?: ICustomStorageNodeMetaData|ICustomStorageNode) => boolean, - ) { - // console.log(`*** descendantsOf "${path}" ****`); - return this._getChildrenOf(path, { ...include, descendants: true }, checkCallback, addCallback); - } - - _getChildrenOf( - path: string, - include: { - metadata: boolean; - value: boolean; - descendants: boolean; - }, - checkCallback: (path: string, metadata?: ICustomStorageNodeMetaData) => boolean, - addCallback?: (path: string, node?: ICustomStorageNodeMetaData|ICustomStorageNode) => boolean, - ) { - // Use cursor to loop from path on - return new Promise((resolve, reject) => { - const pathInfo = CustomStorageHelpers.PathInfo.get(path); - const tx = this._createTransaction(false); - const store = tx.objectStore('nodes'); - const query = IDBKeyRange.lowerBound(path, true); - const cursor = include.metadata ? store.openCursor(query) as IDBRequest : store.openKeyCursor(query) as IDBRequest; - cursor.onerror = e => { - tx.abort?.(); - reject(e); - }; - cursor.onsuccess = async e => { - const otherPath = cursor.result?.key as string ?? null; - let keepGoing = true; - if (otherPath === null) { - // No more results - keepGoing = false; - } - else if (!pathInfo.isAncestorOf(otherPath)) { - // Paths are sorted, no more children or ancestors to be expected! - keepGoing = false; - } - else if (include.descendants || pathInfo.isParentOf(otherPath)) { - - let node: ICustomStorageNode|ICustomStorageNodeMetaData; - if (include.metadata) { - const valueCursor = cursor as IDBRequest; - const data = valueCursor.result.value as IIndexedDBNodeData; - node = data.metadata; - } - const shouldAdd = checkCallback(otherPath, node); - if (shouldAdd) { - if (include.value) { - // Load value! - if (this.context.cache.has(otherPath)) { - const cache = this.context.cache.get(otherPath); - (node as ICustomStorageNode).value = cache.value; - } - else { - const req = tx.objectStore('content').get(otherPath); - (node as ICustomStorageNode).value = await new Promise((resolve, reject) => { - req.onerror = e => { - resolve(null); // Value missing? - }; - req.onsuccess = e => { - resolve(req.result); - }; - }); - this.context.cache.set(otherPath, (node as ICustomStorageNode).value === null ? null : node as ICustomStorageNode); - } - } - keepGoing = addCallback(otherPath, node); - } - } - if (keepGoing) { - try { cursor.result.continue(); } - catch(err) { - // We reached the end of the cursor? - keepGoing = false; - } - } - if (!keepGoing) { - tx.commit?.(); - resolve(); - } - }; - }); - } - -} diff --git a/src/ts/storage/custom/indexed-db/index.ts b/src/ts/storage/custom/indexed-db/index.ts index 7a42c60..cff4653 100644 --- a/src/ts/storage/custom/indexed-db/index.ts +++ b/src/ts/storage/custom/indexed-db/index.ts @@ -1 +1,78 @@ -// TODO: Move IndexedDB implementation from acebase-browser.ts here +import { SimpleCache } from 'acebase-core'; +import { CustomStorageSettings, ICustomStorageNode } from '..'; +import { AceBase } from '../../..'; +import { IndexedDBStorageSettings } from './settings'; +import { IndexedDBStorageTransaction, IndexedDBTransactionContext } from './transaction'; + +export function createIndexedDBInstance(dbname: string, init: Partial = {}) { + const settings = new IndexedDBStorageSettings(init); + + // We'll create an IndexedDB with name "dbname.acebase" + const IndexedDB: IDBFactory = window.indexedDB || (window as any).mozIndexedDB || (window as any).webkitIndexedDB || (window as any).msIndexedDB; // browser prefixes not really needed, see https://caniuse.com/#feat=indexeddb + const request = IndexedDB.open(`${dbname}.acebase`, 1); + + request.onupgradeneeded = (e) => { + // create datastore + const db = request.result; + + // Create "nodes" object store for metadata + db.createObjectStore('nodes', { keyPath: 'path'}); + + // Create "content" object store with all data + db.createObjectStore('content'); + }; + + let idb: IDBDatabase; + const readyPromise = new Promise((resolve, reject) => { + request.onsuccess = e => { + idb = request.result; + resolve(); + }; + request.onerror = e => { + reject(e); + }; + }); + + const cache = new SimpleCache(typeof settings.cacheSeconds === 'number' ? settings.cacheSeconds : 60); // 60 second node cache by default + // cache.enabled = false; + + const storageSettings = new CustomStorageSettings({ + name: 'IndexedDB', + locking: true, // IndexedDB transactions are short-lived, so we'll use AceBase's path based locking + removeVoidProperties: settings.removeVoidProperties, + maxInlineValueSize: settings.maxInlineValueSize, + lockTimeout: settings.lockTimeout, + ready() { + return readyPromise; + }, + async getTransaction(target: { path: string; write: boolean }) { + await readyPromise; + const context: IndexedDBTransactionContext = { + debug: false, + db: idb, + cache, + ipc, + }; + return new IndexedDBStorageTransaction(context, target); + }, + }); + const db = new AceBase(dbname, { + logLevel: settings.logLevel, + storage: storageSettings, + sponsor: settings.sponsor, + // isolated: settings.isolated, + }); + const ipc = db.api.storage.ipc; + db.settings.ipcEvents = settings.multipleTabs === true; + ipc.on('notification', async (notification: { data: any }) => { + const message = notification.data; + if (typeof message !== 'object') { return; } + if (message.action === 'cache.invalidate') { + // console.warn(`Invalidating cache for paths`, message.paths); + for (const path of message.paths) { + cache.remove(path); + } + } + }); + return db; +} diff --git a/src/ts/storage/custom/indexed-db/transaction.ts b/src/ts/storage/custom/indexed-db/transaction.ts new file mode 100644 index 0000000..e740b65 --- /dev/null +++ b/src/ts/storage/custom/indexed-db/transaction.ts @@ -0,0 +1,270 @@ +import { SimpleCache } from 'acebase-core'; +import { CustomStorageHelpers, CustomStorageTransaction, ICustomStorageNode, ICustomStorageNodeMetaData } from '..'; +import { IPCPeer } from '../../../ipc'; + +interface IIndexedDBNodeData { + path: string; + metadata: ICustomStorageNodeMetaData; +} + +function _requestToPromise(request: IDBRequest) { + return new Promise((resolve, reject) => { + request.onsuccess = event => { + return resolve(request.result || null); + }; + request.onerror = reject; + }); +} + +export interface IndexedDBTransactionContext { + debug: boolean; + db: IDBDatabase; + cache: SimpleCache; + ipc: IPCPeer; +} + +export class IndexedDBStorageTransaction extends CustomStorageTransaction { + + production = true; // Improves performance, only set when all works well + + private _pending: Array<{ path: string; action: 'set' | 'update' | 'remove'; node?: ICustomStorageNode }>; + + /** + * Creates a transaction object for IndexedDB usage. Because IndexedDB automatically commits + * transactions when they have not been touched for a number of microtasks (eg promises + * resolving whithout querying data), we will enqueue set and remove operations until commit + * or rollback. We'll create separate IndexedDB transactions for get operations, caching their + * values to speed up successive requests for the same data. + */ + constructor(public context: IndexedDBTransactionContext, target: { path: string, write: boolean }) { + super(target); + this._pending = []; + } + + _createTransaction(write = false) { + const tx = this.context.db.transaction(['nodes', 'content'], write ? 'readwrite' : 'readonly'); + return tx; + } + + _splitMetadata(node: ICustomStorageNode) { + const value = node.value; + const copy: ICustomStorageNode = { ...node }; + delete copy.value; + const metadata = copy as ICustomStorageNodeMetaData; + return { metadata, value }; + } + + async commit() { + // console.log(`*** commit ${this._pending.length} operations ****`); + if (this._pending.length === 0) { return; } + const batch = this._pending.splice(0); + + this.context.ipc.sendNotification({ action: 'cache.invalidate', paths: batch.map(op => op.path) }); + + const tx = this._createTransaction(true); + try { + await new Promise((resolve, reject) => { + let stop = false, processed = 0; + const handleError = (err: any) => { + stop = true; + reject(err); + }; + const handleSuccess = () => { + if (++processed === batch.length) { + resolve(); + } + }; + batch.forEach((op, i) => { + if (stop) { return; } + let r1, r2; + const path = op.path; + if (op.action === 'set') { + const { metadata, value } = this._splitMetadata(op.node); + const nodeInfo: IIndexedDBNodeData = { path, metadata }; + r1 = tx.objectStore('nodes').put(nodeInfo); // Insert into "nodes" object store + r2 = tx.objectStore('content').put(value, path); // Add value to "content" object store + this.context.cache.set(path, op.node); + } + else if (op.action === 'remove') { + r1 = tx.objectStore('content').delete(path); // Remove from "content" object store + r2 = tx.objectStore('nodes').delete(path); // Remove from "nodes" data store + this.context.cache.set(path, null); + } + else { + handleError(new Error(`Unknown pending operation "${op.action}" on path "${path}" `)); + } + let succeeded = 0; + r1.onsuccess = r2.onsuccess = () => { + if (++succeeded === 2) { handleSuccess(); } + }; + r1.onerror = r2.onerror = handleError; + }); + }); + tx.commit && tx.commit(); + } + catch (err) { + console.error(err); + tx.abort && tx.abort(); + throw err; + } + } + + async rollback(err: any) { + // Nothing has committed yet, so we'll leave it like that + this._pending = []; + } + + async get(path: string) { + // console.log(`*** get "${path}" ****`); + if (this.context.cache.has(path)) { + const cache = this.context.cache.get(path); + // console.log(`Using cached node for path "${path}": `, cache); + return cache; + } + const tx = this._createTransaction(false); + const r1 = _requestToPromise(tx.objectStore('nodes').get(path)); // Get metadata from "nodes" object store + const r2 = _requestToPromise(tx.objectStore('content').get(path)); // Get content from "content" object store + try { + const results = await Promise.all([r1, r2]); + tx.commit && tx.commit(); + const info = results[0] as IIndexedDBNodeData; + if (!info) { + // Node doesn't exist + this.context.cache.set(path, null); + return null; + } + const node = info.metadata as ICustomStorageNode; + node.value = results[1]; + this.context.cache.set(path, node); + return node; + } + catch(err) { + console.error(`IndexedDB get error`, err); + tx.abort && tx.abort(); + throw err; + } + } + + set(path: string, node: ICustomStorageNode) { + // Queue the operation until commit + this._pending.push({ action: 'set', path, node }); + } + + remove(path: string) { + // Queue the operation until commit + this._pending.push({ action: 'remove', path }); + } + + async removeMultiple(paths: string[]) { + // Queues multiple items at once, dramatically improves performance for large datasets + paths.forEach(path => { + this._pending.push({ action: 'remove', path }); + }); + } + + childrenOf( + path: string, + include: { + metadata: boolean; + value: boolean; + }, + checkCallback: (childPath: string) => boolean, + addCallback?: (childPath: string, node?: ICustomStorageNodeMetaData|ICustomStorageNode) => boolean, + ) { + // console.log(`*** childrenOf "${path}" ****`); + return this._getChildrenOf(path, { ...include, descendants: false }, checkCallback, addCallback); + } + + descendantsOf( + path: string, + include: { + metadata: boolean; + value: boolean; + }, + checkCallback: (descPath: string, metadata?: ICustomStorageNodeMetaData) => boolean, + addCallback?: (descPath: string, node?: ICustomStorageNodeMetaData|ICustomStorageNode) => boolean, + ) { + // console.log(`*** descendantsOf "${path}" ****`); + return this._getChildrenOf(path, { ...include, descendants: true }, checkCallback, addCallback); + } + + _getChildrenOf( + path: string, + include: { + metadata: boolean; + value: boolean; + descendants: boolean; + }, + checkCallback: (path: string, metadata?: ICustomStorageNodeMetaData) => boolean, + addCallback?: (path: string, node?: ICustomStorageNodeMetaData|ICustomStorageNode) => boolean, + ) { + // Use cursor to loop from path on + return new Promise((resolve, reject) => { + const pathInfo = CustomStorageHelpers.PathInfo.get(path); + const tx = this._createTransaction(false); + const store = tx.objectStore('nodes'); + const query = IDBKeyRange.lowerBound(path, true); + const cursor = include.metadata ? store.openCursor(query) as IDBRequest : store.openKeyCursor(query) as IDBRequest; + cursor.onerror = e => { + tx.abort?.(); + reject(e); + }; + cursor.onsuccess = async e => { + const otherPath = cursor.result?.key as string ?? null; + let keepGoing = true; + if (otherPath === null) { + // No more results + keepGoing = false; + } + else if (!pathInfo.isAncestorOf(otherPath)) { + // Paths are sorted, no more children or ancestors to be expected! + keepGoing = false; + } + else if (include.descendants || pathInfo.isParentOf(otherPath)) { + + let node: ICustomStorageNode|ICustomStorageNodeMetaData; + if (include.metadata) { + const valueCursor = cursor as IDBRequest; + const data = valueCursor.result.value as IIndexedDBNodeData; + node = data.metadata; + } + const shouldAdd = checkCallback(otherPath, node); + if (shouldAdd) { + if (include.value) { + // Load value! + if (this.context.cache.has(otherPath)) { + const cache = this.context.cache.get(otherPath); + (node as ICustomStorageNode).value = cache.value; + } + else { + const req = tx.objectStore('content').get(otherPath); + (node as ICustomStorageNode).value = await new Promise((resolve, reject) => { + req.onerror = e => { + resolve(null); // Value missing? + }; + req.onsuccess = e => { + resolve(req.result); + }; + }); + this.context.cache.set(otherPath, (node as ICustomStorageNode).value === null ? null : node as ICustomStorageNode); + } + } + keepGoing = addCallback(otherPath, node); + } + } + if (keepGoing) { + try { cursor.result.continue(); } + catch(err) { + // We reached the end of the cursor? + keepGoing = false; + } + } + if (!keepGoing) { + tx.commit?.(); + resolve(); + } + }; + }); + } + +}