github-starred/acebase
2
0
Fork 0
mirror of https://github.com/appy-one/acebase.git synced 2026-06-30 06:02:02 -06:00
Code Issues 20 Projects Packages Wiki Activity Actions

Improve allocation ranges -> addresses conversion

Browse source 
This moves away from adding all allocated storage addresses for each range which is VERY slow on large nodes spanning multiple pages.  It now only requests target records for specific record numbers, and calculated counts only
This commit is contained in:
Ewout Stortenbeker 2023-10-02 11:03:39 +02:00
parent e3fdade566
commit 3756aaba7e
1 changed files with 36 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

45
src/storage/binary/index.ts
View file

@ -2414,12 +2414,31 @@ class NodeAllocation {
get addresses(): StorageAddress[] {
const addresses = [] as StorageAddress[];
this.ranges.forEach(range => {
for (const range of this.ranges) {
for (let i = 0; i < range.length; i++) {
const address = new StorageAddress(range.pageNr, range.recordNr + i);
addresses.push(address);
}
});
}
return addresses;
}
/**
* Gets individual record addresses from current allocation
* @param start index to start
* @param end index to stop (does not include this record)
*/
getAddresses(start: number, end: number) {
const addresses = [] as StorageAddress[];
let nr = 0;
for (const range of this.ranges) {
for (let i = 0; i < range.length && nr < end; i++, nr++) {
if (nr >= start && nr < end) {
const address = new StorageAddress(range.pageNr, range.recordNr + i);
addresses.push(address);
}
}
if (nr >= end) { break; }
}
return addresses;
}
@ -3056,7 +3075,7 @@ class NodeReader {
if (this.recordInfo.hasKeyIndex) {
return createStreamFromBinaryTree();
}
else if (this.recordInfo.allocation.addresses.length === 1) {
else if (this.recordInfo.allocation.totalAddresses === 1) {
// We have all data in memory (small record)
return createStreamFromLinearData(this.recordInfo.startData, true);
}
@ -3371,6 +3390,7 @@ class NodeReader {
});
}
// #allocatedAddresses = null as StorageAddress[] | null;
async _treeDataWriter(binary: number[] | Buffer, index: number) {
if (binary instanceof Array) {
binary = Buffer.from(binary);
@ -3386,7 +3406,11 @@ class NodeReader {
nr: Math.floor((headerLength + index + length) / recordSize),
offset: (headerLength + index + length) % recordSize,
};
const writeRecords = this.recordInfo.allocation.addresses.slice(startRecord.nr, endRecord.nr + 1);
// if (!this.#allocatedAddresses) {
// this.#allocatedAddresses = this.recordInfo.allocation.addresses;
// }
// const writeRecords = this.#allocatedAddresses.slice(startRecord.nr, endRecord.nr + 1);
const writeRecords = this.recordInfo.allocation.getAddresses(startRecord.nr, endRecord.nr + 1);
const writeRanges = NodeAllocation.fromAdresses(writeRecords).ranges;
const writes = [];
let bOffset = 0;
@ -3421,11 +3445,15 @@ class NodeReader {
nr: Math.floor((headerLength + index + length) / recordSize),
offset: (headerLength + index + length) % recordSize,
};
const readRecords = this.recordInfo.allocation.addresses.slice(startRecord.nr, endRecord.nr + 1);
// if (!this.#allocatedAddresses) {
// this.#allocatedAddresses = this.recordInfo.allocation.addresses;
// }
// const readRecords = this.#allocatedAddresses.slice(startRecord.nr, endRecord.nr + 1);
const readRecords = this.recordInfo.allocation.getAddresses(startRecord.nr, endRecord.nr + 1);
if (readRecords.length === 0) {
throw new Error(
`Attempt to read non-existing records of path "/${this.recordInfo.path}": ${startRecord.nr} to ${endRecord.nr + 1} ` +
`for index ${index} + ${length} bytes. Node has ${this.recordInfo.allocation.addresses.length} allocated records ` +
`for index ${index} + ${length} bytes. Node has ${this.recordInfo.allocation.totalAddresses} allocated records ` +
`in the following ranges: ` + this.recordInfo.allocation.toString()
);
}
@ -3468,7 +3496,6 @@ class NodeReader {
let view = new DataView(data.buffer);
let offset = 1;
const firstRange = new StorageAddressRange(this.address.pageNr, this.address.recordNr, 1);
/** @type {StorageAddressRange[]} */
const ranges = [firstRange];
const allocation = new NodeAllocation(ranges);
let readingRecordIndex = 0;
@ -3478,7 +3505,7 @@ class NodeReader {
if (offset + 9 + 2 >= data.length) {
// Read more data (next record)
readingRecordIndex++;
const address = allocation.addresses[readingRecordIndex];
const [address] = allocation.getAddresses(readingRecordIndex, readingRecordIndex + 1);
const fileIndex = this.storage.getRecordFileIndex(address.pageNr, address.recordNr);
const moreData = new Uint8Array(bytesPerRecord);
await this.storage.readData(fileIndex, moreData.buffer);
@ -4045,7 +4072,7 @@ async function _writeNode(storage: AceBaseStorage, path: string, value: any, loc
// Create a B+tree
const fillFactor =
isArray || serialized.every(kvp => typeof kvp.key === 'string' && /^[0-9]+$/.test(kvp.key))
? BINARY_TREE_FILL_FACTOR_50
? BINARY_TREE_FILL_FACTOR_50 // TODO: Consider removing this, might be better performing now with 95% instead!
: BINARY_TREE_FILL_FACTOR_95;
const treeBuilder = new BPlusTreeBuilder(true, fillFactor);