diff --git a/uppsrc/Core/HeapImp.h b/uppsrc/Core/HeapImp.h index 6d25e5bd9..3946f34de 100644 --- a/uppsrc/Core/HeapImp.h +++ b/uppsrc/Core/HeapImp.h @@ -88,34 +88,30 @@ struct Heap { Page work[NKLASS][1]; // circular list of pages that contain some empty blocks Page full[NKLASS][1]; // circular list of pages that contain NO empty blocks - Page *empty[NKLASS]; // last fully freed page per klass (hot) or global list of empty pages in aux + Page *empty[NKLASS]; // last fully freed page per klass (hot reserve); shared global list of empty pages in aux FreeLink *cache[NKLASS]; // hot frontend cache of small blocks int cachen[NKLASS]; // counter of small blocks that are allowed to be stored in cache bool initialized; - static word BinSz[LBINS]; - static byte SzBin[MAXBLOCK / 8 + 1]; - static byte BlBin[MAXBLOCK / 8 + 1]; + static word BinSz[LBINS]; // block size for bin + static byte SzBin[MAXBLOCK / 8 + 1]; // maps size/8 to bin + static byte BlBin[MAXBLOCK / 8 + 1]; // Largest bin less or equal to size/8 (free -> bin) - DLink large[1]; - int lcount; - DLink freebin[LBINS][1]; - static DLink lempty[1]; + DLink large[1]; // all large chunks that belong to this heap + int lcount; // count of large chunks + DLink freebin[LBINS][1]; // all free blocks by bin + static DLink lempty[1]; // shared global list of all empty large blocks - struct Out { - Heap *heap; - void *ptr; - }; void *out[REMOTE_OUT_SZ / 16 + 1]; void **out_ptr; int out_size; - byte filler1[128]; // make next variable is in distinct cacheline + byte filler1[128]; // make sure the next variable is in distinct cacheline FreeLink *remote_list; // single linked list of remotely released pointers - static DLink big[1]; // List of all big blocks - static Heap aux; // Single global auxiliary heap to store orphans and global list of free pages + static DLink big[1]; // List of all big blocks + static Heap aux; // Single global auxiliary heap to store orphans and global list of free pages #ifdef HEAPDBG static void DbgFreeFill(void *ptr, size_t size); diff --git a/uppsrc/Core/lheap.cpp b/uppsrc/Core/lheap.cpp index ed8bfa2e7..7077fee45 100644 --- a/uppsrc/Core/lheap.cpp +++ b/uppsrc/Core/lheap.cpp @@ -61,7 +61,7 @@ void Heap::LinkFree(DLink *b, int size) } Heap::DLink *Heap::AddChunk(int reqsize) -{ +{ // gets a free chunk DLink *ml; if(lempty->next != lempty) { ml = lempty->next; @@ -159,34 +159,32 @@ void *Heap::TryLAlloc(int ii, size_t size) return NULL; } -int sBig__; - -void *Heap::LAlloc(size_t& size) { +void *Heap::LAlloc(size_t& size) +{ // allocate large or big block LLOG("+++ LAlloc " << size); ASSERT(size > 256); if(!initialized) Init(); - if(size > MAXBLOCK) { + if(size > MAXBLOCK) { // big block allocation Mutex::Lock __(mutex); BigHdr *h = (BigHdr *)SysAllocRaw(size + BIGHDRSZ, size); h->Link(big); h->size = size = ((size + BIGHDRSZ + 4095) & ~4095) - BIGHDRSZ; Header *b = (Header *)((byte *)h + BIGHDRSZ - sizeof(Header)); - b->size = 0; + b->size = 0; // header contains large header with size = 0, to detect big during free b->free = false; - sBig__++; LLOG("Big alloc " << (void *)b->GetBlock()); return b->GetBlock(); } - int bini = SizeToBin((int)size); - size = BinSz[bini]; + int bini = SizeToBin((int)size); // get the bin + size = BinSz[bini]; // get the real bin size LLOG("Binned size " << asString(size)); - void *ptr = TryLAlloc(bini, size); + void *ptr = TryLAlloc(bini, size); // try current working blocks first if(ptr) return ptr; - if(remote_list) { - FreeRemote(); - ptr = TryLAlloc(bini, size); + if(remote_list) { // there might be blocks freed in other threads + FreeRemote(); // free them + ptr = TryLAlloc(bini, size); // try again if(ptr) return ptr; } Mutex::Lock __(mutex); @@ -207,7 +205,8 @@ void *Heap::LAlloc(size_t& size) { return ptr; } -void Heap::LFree(void *ptr) { +void Heap::LFree(void *ptr) +{ // free large or big block DLink *b = (DLink *)ptr; Header *bh = b->GetHeader(); if(bh->size == 0) { @@ -217,7 +216,6 @@ void Heap::LFree(void *ptr) { h->Unlink(); LLOG("Big free " << (void *) ptr << " size " << h->size); SysFreeRaw(h, h->size); - sBig__--; return; } if(bh->heap != this) { diff --git a/uppsrc/Core/srcimp.tpp/Heap$en-us.tpp b/uppsrc/Core/srcimp.tpp/Heap$en-us.tpp index 02869bce9..79ab46102 100644 --- a/uppsrc/Core/srcimp.tpp/Heap$en-us.tpp +++ b/uppsrc/Core/srcimp.tpp/Heap$en-us.tpp @@ -66,7 +66,7 @@ has to be done:&] [s0; When allocating small block, first sWork list is checked for the block. If not available, sFree list is checked to get free block, if even that is empty, new block is obtained from the -system (using SysAllocRaw). Note that allocator keeps the number +system (using AllocRaw4KB). Note that allocator keeps the number of free blocks in the header. Implementation detail: there are two possibilities how free blocks can be recorded in the block header. First, there is a single`-linked list of free blocks. @@ -87,23 +87,24 @@ page moves to sFree.&] [s6; Medium blocks &] [s0; Blocks >256 and < 65504 bytes. Approximate best`-fit allocator is used for these blocks. Memory is organized in 64KB chunks -(obtained using SysAllocRaw). Each allocated block has header -with its size and the size of previous block.&] +(obtained using AllocRaw64KB). Each allocated block has header +with its size and the size of previous block, free flag and pointer +to the Heap.&] [s0; &] [s0; Allocator keeps an array of lists of free blocks of particular sizes. Size distribution is mostly exponential, blocks lower than 2048 are rounded up to 32 bytes, between 2048 and about -35000, rounding exponentially grows up to 2048 and then stays +35000 rounding exponentially grows up to 2048 and then stays at this value. Each such size has its index in the array of free blocks.&] [s0; &] [s0; When allocating, index is decided based on the size and array is searched starting with that index to obtain the smallest free block (best`-fit) greater than required size. Bigger blocks are -divided.&] +divided and the rest of block is put to free block list.&] [s0; &] [s0; When freeing, allocator merges the freed block with previous -or next free block if any.&] +or next free block if any and reassigns in free block list.&] [s0; &] [s0; Note that master header of 64KB blocks and all operations are designed so that resulting pointers are NOT 16 byte aligned (see @@ -121,11 +122,29 @@ memory back to the system.&] [s6; Multithreading&] [s0; Each thread has its own heap (implemented using TLS) and there is also `'aux`' heap, which is basically used to keep track of -completely free blocks, both 4KB pages for small blocks and 64KB -pages for medium blocks.&] -[s0; Most small and medium block allocations are lockless, unless -a there is no free space in existing working blocks, freeing -is lockless as long as memory was allocated in the same thread -(belongs to the same heap). Also, if free&] +completely free 4KB pages or 64KB chunks.&] +[s0; Most small and medium block allocations are lockless. Single +mutex for the whole allocator is locked in following, relatively +rare, situations:&] +[s0; &] +[s0;i150;O0; When freeing the small block that was allocated in different +thread (has different heap). Such blocks are first buffered until +their total size is more than 2000 bytes, then the mutex is locked +and all blocks are, distributed to remote`_free lists of respective +heaps.&] +[s0;i150;O0; When allocating the small block and there is no block +available in partially used pages and there is no `'reserve`' +empty page (of any size class) available `- in that case, mutex +is locked and and situation resolved, either by obtaining the +free page from global storage, by adopting orphaned partially +used page from auxiliary page, or if all else fails, by retrieving +page from the system. As part of this process, remote`_list pointer +is checked (without locking) and if not null, mutex is locked +and remotely freed pages are processed.&] +[s0;i150;O0; When freeing the small block which results in completely +free page and when heap already has reserve empty page for given +size class. In that case, reserve page is put to global list +of empty pages and new free page is used as new reserve (this +is because new page is likely more `'hot`' in cache).&] [s0; &] [s0; ]] \ No newline at end of file