diff --git a/uppsrc/Core/Algo.h b/uppsrc/Core/Algo.h index 67d0b3fec..473e3cf0c 100644 --- a/uppsrc/Core/Algo.h +++ b/uppsrc/Core/Algo.h @@ -180,6 +180,16 @@ Vector FindAll(const Range& r, Predicate match, int from = 0) return ndx; } +template +Vector FindAlli(const Range& r, Predicate match, int from = 0) +{ + Vector ndx; + for(int i = from; i < r.GetCount(); i++) + if(match(i)) + ndx.Add(i); + return ndx; +} + template int FindLowerBound(const Range& r, const T& val, const Less& less) { diff --git a/uppsrc/Core/Core.h b/uppsrc/Core/Core.h index 4fa3771c1..3ecba50ec 100644 --- a/uppsrc/Core/Core.h +++ b/uppsrc/Core/Core.h @@ -240,6 +240,7 @@ typedef int SOCKET; #include #include #include +#include // fix MSC8 beta problem.... #ifdef COMPILER_MSC @@ -349,6 +350,7 @@ class JsonIO; #include "Win32Util.h" #include "Vcont.hpp" +#include "Index.hpp" #include "Map.hpp" #include "InVector.hpp" #include "InMap.hpp" diff --git a/uppsrc/Core/Core.upp b/uppsrc/Core/Core.upp index d9329f8c0..7788101b6 100644 --- a/uppsrc/Core/Core.upp +++ b/uppsrc/Core/Core.upp @@ -51,6 +51,7 @@ file Heap.h, HeapImp.h, heaputil.cpp, + hheap.cpp, sheap.cpp, lheap.cpp, heap.cpp, @@ -110,10 +111,11 @@ file Vcont.hpp, Vcont.cpp, Index.h, + Index.hpp, + Index.cpp, Map.h, - FixedMap.h, Map.hpp, - Hash.cpp, + FixedMap.h, InVector.h, InVector.hpp, InMap.hpp, diff --git a/uppsrc/Core/Debug.cpp b/uppsrc/Core/Debug.cpp index 8613056bd..2dd597585 100644 --- a/uppsrc/Core/Debug.cpp +++ b/uppsrc/Core/Debug.cpp @@ -47,8 +47,8 @@ TimingInspector::TimingInspector(const char *_name) { } TimingInspector::~TimingInspector() { - Mutex::Lock __(mutex); if(this == &s_zero) return; + Mutex::Lock __(mutex); StdLog() << Dump() << "\r\n"; } diff --git a/uppsrc/Core/Defs.h b/uppsrc/Core/Defs.h index 3caad5c15..2f5f55305 100644 --- a/uppsrc/Core/Defs.h +++ b/uppsrc/Core/Defs.h @@ -416,10 +416,13 @@ template class Function; #ifdef COMPILER_MSC #define force_inline __forceinline +#define never_inline __declspec(noinline) #elif defined(COMPILER_GCC) #define force_inline __attribute__((always_inline)) inline +#define never_inline __attribute__((noinline)) #else #define force_inline inline +#define never_inline #endif #define BINARY(i, f) \ diff --git a/uppsrc/Core/Hash.cpp b/uppsrc/Core/Hash.cpp deleted file mode 100644 index a961f6bfb..000000000 --- a/uppsrc/Core/Hash.cpp +++ /dev/null @@ -1,339 +0,0 @@ -#include - -namespace Upp { - -#ifdef _MSC_VER -#pragma inline_depth(255) -#pragma optimize("t", on) -#endif - -unsigned Pow2Bound(unsigned i) -{ - unsigned n = 1; - while(n < i) { - n <<= 1; - if(n == 0) - return 1 << 31; - } - return n; -} - -unsigned PrimeBound(unsigned n) -{ - static unsigned prime_tab[] = { - 17, 29, 61, 127, 257, 509, 1021, 2053, 4093, 8191, 16381, - 32771, 65537, 131071, 262147, 524287, 1048573, 2097143, - 4194301, 8388617, 16777213, 33554467, 67108859, 134217757, - 268435459, 536870909, 1073741827, 2147483647u, 4294967291u - }; -#ifdef DEPRECATED - return *FindUpperBoundIter(prime_tab, prime_tab + __countof(prime_tab), n); -#else - return prime_tab[FindUpperBound(SubRange(prime_tab, prime_tab + __countof(prime_tab)), n)]; -#endif -} - -#ifdef FOLDHASH -inline unsigned HashBound(unsigned i) { return Pow2Bound(i); } -#else -inline unsigned HashBound(unsigned i) { return PrimeBound(i); } -#endif - -static int _EmptyHashBase = -1; - -inline -void HashBase::Zero() -{ - map = &_EmptyHashBase; - mask = 0; - unlinked = -1; -} - -void HashBase::Free() -{ - if(map != &_EmptyHashBase) - delete [](byte *)map; - Zero(); -} - -void HashBase::ClearIndex() -{ - link.Clear(); - Free(); -} - -HashBase::HashBase() -{ - Zero(); -} - -HashBase::HashBase(HashBase&& b) -: hash(pick(b.hash)), - link(pick(b.link)) -{ - map = b.map; - mask = b.mask; - unlinked = b.unlinked; - b.Zero(); -} - -void HashBase::operator=(HashBase&& b) -{ - hash = pick(b.hash); - link = pick(b.link); - Free(); - map = b.map; - mask = b.mask; - unlinked = b.unlinked; - b.Zero(); -} - -HashBase::HashBase(const HashBase& b, int) -: hash(b.hash, 0) -{ - Zero(); - Reindex(); -} - -void HashBase::operator<<=(const HashBase& b) -{ - ClearIndex(); - hash = clone(b.hash); - Reindex(); -} - -HashBase::~HashBase() -{ - Free(); -} - -force_inline -void HashBase::LinkBefore(int i, Link& l, int bi) -{ - Link& bl = link[bi]; - l.next = bi; - l.prev = bl.prev; - bl.prev = i; - link[l.prev].next = i; -} - -void HashBase::Trim(int count) -{ - ASSERT(count <= hash.GetCount() && count >= 0); - for(int i = count; i < link.GetCount(); i++) - Unlink(i, link[i]); - link.Trim(count); - hash.SetCount(count); -} - -void HashBase::Drop(int n) -{ - Trim(hash.GetCount() - n); -} - -void HashBase::FinishIndex() -{ - int q = link.GetCount(); - link.Reserve(hash.GetCount()); - link.AddN(hash.GetCount() - q); - for(int i = q; i < hash.GetCount(); i++) - LinkTo(i, link[i], hash[i] & UNSIGNED_HIBIT ? unlinked : Mapi(i)); -} - -void HashBase::Reindex(int n) -{ - ClearIndex(); - Free(); - n = HashBound(n); - mask = n - 1; - map = (int *)new byte[n * sizeof(int)]; - Fill(map, map + n, -1); - FinishIndex(); -} - -void HashBase::Reindex() -{ - Reindex(hash.GetCount()); -} - -void HashBase::SetUn(int i, unsigned _hash) -{ - if(map) { - Link& lnk = link[i]; - Unlink(i, lnk); - LinkTo(i, lnk, Maph(_hash & ~UNSIGNED_HIBIT)); - } - hash[i] = _hash & ~UNSIGNED_HIBIT; -} - -Vector HashBase::GetUnlinked() const -{ - Vector r; - int q = unlinked; - if(q >= 0) { - do { - r.Add(q); - q = link[q].next; - } - while(q != unlinked); - } - return r; -} - -int HashBase::Put(unsigned _hash) -{ - if(unlinked < 0) return -1; - Link& l = link[unlinked]; - int i = unlinked; - unlinked = link[unlinked].next; - if(i == unlinked) - unlinked = -1; - else { - link[l.next].prev = l.prev; - link[l.prev].next = l.next; - } - LinkTo(i, l, Maph(_hash & ~UNSIGNED_HIBIT)); - hash[i] = _hash & ~UNSIGNED_HIBIT; - return i; -} - -void HashBase::Set(int i, unsigned _hash) { - if(map) { - Link& lnk = link[i]; - Unlink(i, lnk); - int& mi = Maph(_hash & ~UNSIGNED_HIBIT); - int ii = mi; - if(ii < 0) - mi = lnk.prev = lnk.next = i; - else - if(i < ii) { - LinkBefore(i, lnk, ii); - mi = i; - } - else { - int l = ii; - int h = link[ii].prev; - if(h - i < i - l) { - while(i < h) { - h = link[h].prev; - } - LinkBefore(i, lnk, link[h].next); - } - else { - l = link[l].next; - while(i > l && l != ii) { - l = link[l].next; - } - LinkBefore(i, lnk, l); - } - } - } - hash[i] = _hash & ~UNSIGNED_HIBIT; -} - -void HashBase::Shrink() { - hash.Shrink(); - if((int)HashBound(hash.GetCount()) <= mask) { - Reindex(hash.GetCount()); - } - else - link.Shrink(); -} - -void HashBase::Reserve(int n) -{ - hash.Reserve(n); - link.Reserve(n); - if((int)HashBound(n) > mask) - Reindex(n); -} - -void HashBase::Remove(int i) -{ - hash.Remove(i); - ClearIndex(); - Reindex(); -} - -void HashBase::Remove(int i, int count) -{ - hash.Remove(i, count); - ClearIndex(); - Reindex(); -} - -void HashBase::Remove(const int *sorted_list, int count) -{ - hash.Remove(sorted_list, count); - ClearIndex(); - Reindex(); -} - -void HashBase::Insert(int i, unsigned _hash) { - hash.Insert(i, _hash & ~UNSIGNED_HIBIT); - ClearIndex(); - Reindex(); -} - -#ifdef UPP -void HashBase::Serialize(Stream& s) { - int version = 0; - s / version; - if(s.IsLoading()) - ClearIndex(); - hash.Serialize(s); - Reindex(); -} -#endif - -void HashBase::Swap(HashBase& b) { - UPP::Swap(hash, b.hash); - UPP::Swap(link, b.link); - UPP::Swap(map, b.map); - UPP::Swap(mask, b.mask); - UPP::Swap(unlinked, b.unlinked); -} - - -#ifdef CPU_UNALIGNED - -NOUBSAN // CPU supports unaligned memory access -unsigned memhash(const void *ptr, size_t count) -{ - unsigned hash = 1234567890U; - - const unsigned *ds = (unsigned *)ptr; - const unsigned *de = ds + (count >> 2); - while(ds < de) - hash = ((hash << 5) - hash) ^ *ds++; - - const byte *s = (byte *)ds; - const byte *e = s + (count & 3); - while(s < e) - hash = ((hash << 5) - hash) ^ *s++; - - return hash; -} - -#else - -unsigned memhash(const void *ptr, size_t count) -{ - unsigned hash = 1234567890U; - - const byte *s = (byte *)ptr; - const byte *e = s + count; - while(s < e) - hash = ((hash << 5) - hash) ^ *s++; - - return hash; -} - -#endif - -unsigned GetHashValue0(const double& d) -{ - return memhash(&d, sizeof(double)); -} - -} \ No newline at end of file diff --git a/uppsrc/Core/Heap.h b/uppsrc/Core/Heap.h index 6f2452978..6bfcb88ed 100644 --- a/uppsrc/Core/Heap.h +++ b/uppsrc/Core/Heap.h @@ -16,7 +16,20 @@ int MemoryUsedKb(); void MemoryLimitKb(int kb); size_t GetMemoryBlockSize(void *ptr); -bool TryRealloc(void *ptr, size_t newsize); + +bool MemoryTryRealloc__(void *ptr, size_t& newsize); + +#ifdef _DEBUG +inline // in DEBUG test for small block is moved inside, because debug adds diagnostics header +bool MemoryTryRealloc(void *ptr, size_t& newsize) { + return MemoryTryRealloc__(ptr, newsize); +} +#else +inline +bool MemoryTryRealloc(void *ptr, size_t& newsize) { + return (((dword)(uintptr_t)ptr) & 16) && MemoryTryRealloc__(ptr, newsize); +} +#endif #ifdef MEMORY_SHRINK void MemoryShrink(); @@ -27,14 +40,6 @@ void MemoryBreakpoint(dword serial); void MemoryInitDiagnostics(); void MemoryDumpLeaks(); -enum MemoryProbeFlags { - MEMORY_PROBE_FULL = 1, - MEMORY_PROBE_FREE = 2, - MEMORY_PROBE_MIXED = 4, - MEMORY_PROBE_LARGE = 8, - MEMORY_PROBE_SUMMARY = 16, -}; - #ifdef HEAPDBG void MemoryIgnoreLeaksBegin(); void MemoryIgnoreLeaksEnd(); @@ -48,18 +53,22 @@ inline void MemoryCheckDebug() {} #endif struct MemoryProfile { - int allocated[1024]; - int fragmented[1024]; - int freepages; - int large_count; - size_t large_size[1024]; - size_t large_total; - int large_free_count; - size_t large_free_size[1024]; - int large_free_total; - int large_empty; - int big_count; - size_t big_size; + int allocated[1024]; // active small blocks (index is size in bytes) + int fragments[1024]; // unallocated small blocks (index is size in bytes) + int freepages; // empty 4KB pages (can be recycled) + int large_count; // count of large (~ 1 - 64KB) active blocks + size_t large_total; // ^ total size + int large_fragments_count; // count of unused large blocks + size_t large_fragments_total; // ^ total size + int large_fragments[2048]; // * 256 + int huge_count; // bigger blocks managed by U++ heap (<= 32MB) + size_t huge_total; // ^total size + int huge_fragments_count; // count of unused large blocks + size_t huge_fragments_total; // ^ total size + int huge_fragments[65536]; // * 256 + int sys_count; // blocks directly allocated from the system (>32MB + size_t sys_total; // ^total size + int master_chunks; // master blocks MemoryProfile(); }; diff --git a/uppsrc/Core/HeapImp.h b/uppsrc/Core/HeapImp.h index 448d4518e..e0e613d21 100644 --- a/uppsrc/Core/HeapImp.h +++ b/uppsrc/Core/HeapImp.h @@ -3,23 +3,283 @@ void OutOfMemoryPanic(size_t size); void *SysAllocRaw(size_t size, size_t reqsize); void SysFreeRaw(void *ptr, size_t size); -void *AllocRaw4KB(int reqsize); -void *AllocRaw64KB(int reqsize); -void *LAlloc(size_t& size); -void LFree(void *ptr); +const char *asString(int i); +const char *asString(void *ptr); -#ifdef MEMORY_SHRINK -#if 0 -void FreeRaw4KB(void *ptr); // Win32 does not allow simple support here -#endif -void FreeRaw64KB(void *ptr); +struct Heap; + +// This is used internally by U++ to manage large (64KB) and huge (up to 256MB) blocks + +struct BlkPrefix { // this part is at the start of Blk allocated block, client must not touch it + word prev_size; + word size; + bool free; + bool last; + Heap *heap; // we need this for 4KB pages and large blocks, NULL for Huge blocks +#ifdef CPU_32 + dword filler; #endif -struct Heap { + void SetSize(word sz) { size = sz; } + void SetPrevSize(word sz) { prev_size = sz; } + void SetFree(bool b) { free = b; } + void SetLast(bool b) { last = b; } + + word GetSize() { return size; } + word GetPrevSize() { return prev_size; } + bool IsFirst() { return GetPrevSize() == 0; } + bool IsFree() { return free; } + bool IsLast() { return last; } + + BlkPrefix *GetPrevHeader(int BlkSize) { return (BlkPrefix *)((byte *)this - BlkSize * GetPrevSize()); } + BlkPrefix *GetNextHeader(int BlkSize) { return (BlkPrefix *)((byte *)this + BlkSize * GetSize()); } +}; + +template +struct BlkHeader_ : BlkPrefix { // free block record + BlkHeader_ *prev; // linked list of free blocks + BlkHeader_ *next; // linked list of free blocks + + BlkHeader_ *GetPrevHeader() { return (BlkHeader_ *)BlkPrefix::GetPrevHeader(BlkSize); } + BlkHeader_ *GetNextHeader() { return (BlkHeader_ *)BlkPrefix::GetNextHeader(BlkSize); } + + void SetNextPrevSz() { if(!IsLast()) GetNextHeader()->SetPrevSize(GetSize()); } + + void UnlinkFree() { Dbl_Unlink(this); } +}; + +template +struct BlkHeap : Detail { + typedef BlkHeader_ BlkHeader; + typedef Detail D; + + bool JoinNext(BlkHeader *h, word needs_count = 0); + void Split(BlkHeader *h, word wcount, bool fill = false); + void AddChunk(BlkHeader *h, int count); + void *MakeAlloc(BlkHeader *h, word wcount); + BlkHeader *Free(BlkHeader *h); // returns final joined block + bool TryRealloc(void *ptr, size_t count, size_t& n); + void BlkCheck(void *page, int size, bool check_heap = false); + + static void Assert(bool b); +#ifdef HEAPDBG + static void DbgFreeFill(void *ptr, size_t size); + static void DbgFreeCheck(void *ptr, size_t size); + static void FillFree(BlkHeader *h); + static void CheckFree(BlkHeader *h); +#else + static void DbgFreeFill(void *ptr, size_t size) {} + static void DbgFreeCheck(void *ptr, size_t size) {} + static void FillFree(BlkHeader *h) {} + static void CheckFree(BlkHeader *h) {} +#endif +}; + +template +void BlkHeap::Assert(bool b) +{ + if(!b) + Panic("Heap is corrupted!"); +} + +#ifdef HEAPDBG + +template +void BlkHeap::DbgFreeFill(void *p, size_t size) +{ + size_t count = size >> 2; + dword *ptr = (dword *)p; + while(count--) + *ptr++ = 0x65657246; +} + +template +void BlkHeap::DbgFreeCheck(void *p, size_t size) +{ + size_t count = size >> 2; + dword *ptr = (dword *)p; + while(count--) + if(*ptr++ != 0x65657246) + Panic("Writes to freed blocks detected"); +} + +template +void BlkHeap::FillFree(BlkHeader *h) +{ + if(BlkSize == 4096) // it is too slow to check huge blocks + return; + DbgFreeFill(h + 1, h->GetSize() * BlkSize - sizeof(BlkHeader)); +} + +template +void BlkHeap::CheckFree(BlkHeader *h) +{ + if(BlkSize == 4096) // it is too slow to check huge blocks + return; + DbgFreeCheck(h + 1, h->GetSize() * BlkSize - sizeof(BlkHeader)); +} + +#endif + +template +void BlkHeap::BlkCheck(void *page, int page_size, bool check_heap) +{ + #define CLOG(x) // LOG(x) + CLOG("=== " << asString(page_size) << " " << AsString(page)); + BlkPrefix *h = (BlkPrefix *)page; + int size = 0; + int psize = 0; + Assert(h->IsFirst()); + for(;;) { + size += h->GetSize(); + CLOG("h: " << AsString(h) << ", GetSize: " << asString(h->GetSize()) + << ", size: " << asString(size) << ", islast: " << asString(h->IsLast())); + Assert(h->GetSize()); + Assert(h->GetPrevSize() == psize); + Assert(!check_heap || h->IsFree() || h->heap); + if(h->IsFree()) + CheckFree((BlkHeader *)h); + psize = h->GetSize(); + if(h->IsLast() && size == page_size) + return; + Assert(size < page_size); + h = h->GetNextHeader(BlkSize); + } + #undef CLOG +} + +template +force_inline +bool BlkHeap::JoinNext(BlkHeader *h, word needs_count) +{ // try to join with next header if it is free, does not link it to free + if(h->IsLast()) + return false; + BlkHeader *nh = h->GetNextHeader(); + if(!nh->IsFree() || h->GetSize() + nh->GetSize() < needs_count) + return false; + CheckFree(nh); + h->SetLast(nh->IsLast()); + nh->UnlinkFree(); + word nsz = h->GetSize() + nh->GetSize(); + h->SetSize(nsz); + h->SetNextPrevSz(); + return true; +} + +template +force_inline +void BlkHeap::Split(BlkHeader *h, word wcount, bool fill) +{ // splits the block if bigger, links new block to free + ASSERT(BlkSize != 4096 || ((dword)(uintptr_t)h & 4095) == 0); + BlkHeader *h2 = (BlkHeader *)((byte *)h + BlkSize * (int)wcount); + word nsz = h->GetSize() - wcount; + if(nsz == 0) // nothing to split + return; + + h2->SetFree(true); + h2->SetLast(h->IsLast()); + h2->SetSize(nsz); + h2->SetPrevSize(wcount); + h2->SetNextPrevSz(); + D::LinkFree(h2); + if(fill) + FillFree(h2); + + h->SetSize(wcount); + h->SetLast(false); +} + +template +void BlkHeap::AddChunk(BlkHeader *h, int count) +{ + h->SetSize(count); + h->SetPrevSize(0); // is first + h->SetLast(true); + h->SetFree(true); + D::LinkFree(h); + FillFree(h); +} + +template +force_inline +void *BlkHeap::MakeAlloc(BlkHeader *h, word wcount) +{ + h->UnlinkFree(); + h->SetFree(false); + Split(h, wcount); + CheckFree(h); + return h; +} + +template +bool BlkHeap::TryRealloc(void *ptr, size_t count, size_t& n) +{ + ASSERT(count); + + BlkHeader *h = (BlkHeader *)ptr; + if(h->size == 0) + return false; + + word sz = h->GetSize(); + if(sz != count) { + if(!JoinNext(h, (word)count) && count > sz) + return false; + Split(h, (word)count, true); + n = n - sz + count; + } + return true; +} + +template +force_inline +typename BlkHeap::BlkHeader *BlkHeap::Free(BlkHeader *h) +{ + ASSERT(BlkSize != 4096 || ((dword)(uintptr_t)h & 4095) == 0); + JoinNext(h); + if(!h->IsFirst()) { // try to join with previous header if it is free + BlkHeader *ph = h->GetPrevHeader(); + if(ph->IsFree()) { + ph->UnlinkFree(); // remove because size will change, might end in another bucket then + word nsz = ph->GetSize() + h->GetSize(); + ph->SetSize(nsz); + ph->SetLast(h->IsLast()); + ph->SetNextPrevSz(); + h = ph; + } + } + h->SetFree(true); + D::LinkFree(h); // was not joined with previous header + FillFree(h); + return h; +} + +struct HugeHeapDetail { + static BlkHeader_<4096> freelist[2][1]; + + static void LinkFree(BlkHeader_<4096> *h) { Dbl_LinkAfter(h, freelist[h->GetSize() >= 16]); } + static void NewFreeSize(BlkHeader_<4096> *h) {} +}; + +struct Heap : BlkHeap { enum { + #ifdef CPU_64 + HPAGE = 7 * 8192, // number of 4KB pages in huge page (also limit for sys allocation) (224MB) + #else + HPAGE = 8192, // number of 4KB pages in huge page (also limit for sys allocation) (32MB) + #endif + LUNIT = 256, // granularity of large blocks (size always a multiple of this) + LPAGE = LUNIT - 1, // number of LUNITs in large page + LOFFSET = 64, // offset from 64KB start to the first block header + NKLASS = 23, // number of small size classes + + REMOTE_OUT_SZ = 2000, // maximum size of remote frees to be buffered to flush at once }; + void *HugeAlloc(size_t count); // count in 4KB, client needs to not touch HugePrefix + int HugeFree(void *ptr); + bool HugeTryRealloc(void *ptr, size_t count); + static int Ksz(int k) { static int sz[] = { // 0 1 2 3 4 5 6 7 8 9 10 11 @@ -36,8 +296,7 @@ struct Heap { FreeLink *next; }; - struct Page { // small block Page - Heap *heap; // pointer to Heap + struct Page : BlkPrefix { // small block Page byte klass; // size class word active; // number of used (active) blocks in this page FreeLink *freelist; // single linked list of free blocks in Page @@ -54,53 +313,41 @@ struct Heap { byte *End() { return (byte *)this + 4096; } int Count() { return (int)(uintptr_t)(End() - Begin()) / Ksz(klass); } }; + + struct LargeHeapDetail { + BlkHeader_ freelist[6][1]; // <1KB, <2KB, <4KB, <8KB, >= 8KB + static int Cv(int n) { return min(n >> 2, 4); } - struct Header; - - struct DLink { + void Free64KB(BlkHeader_ *h); + void LinkFree(BlkHeader_ *h) { + Dbl_LinkAfter(h, freelist[Cv(h->GetSize())]); + } + }; + + struct LargeHeap : BlkHeap {}; + + typedef LargeHeap::BlkHeader LBlkHeader; + + struct DLink : BlkPrefix { // Large Page header / big block header DLink *next; DLink *prev; + size_t size; // only used for big header + #ifdef CPU_64 + dword filler[6]; // sizeof need to be 64 because of alignment + #else + dword filler[9]; + #endif void LinkSelf() { Dbl_Self(this); } void Unlink() { Dbl_Unlink(this); } void Link(DLink *lnk) { Dbl_LinkAfter(this, lnk); } - - Header *GetHeader() { return (Header *)this - 1; } + + LargeHeap::BlkHeader *GetFirst() { return (LargeHeap::BlkHeader *)((byte *)this + LOFFSET); } // pointer to data area }; - struct Header { // Large block header - byte free; - byte filler1, filler2, filler3; - word size; - word prev; - Heap *heap; - #ifdef CPU_32 - dword filler4; - #endif - - Header *Next() { return (Header *)((byte *)this + size) + 1; } - Header *Prev() { return (Header *)((byte *)this - prev) - 1; } - - DLink *GetBlock() { return (DLink *)(this + 1); } - }; + static_assert(sizeof(BlkPrefix) == 16, "Wrong sizeof(BlkPrefix)"); + static_assert(sizeof(DLink) == 64, "Wrong sizeof(DLink)"); - struct BigHdr : DLink { - size_t size; - }; - - enum { - LARGEHDRSZ = 32, // size of large block header, causes 16 byte disalignment - BIGHDRSZ = 48, // size of huge block header - REMOTE_OUT_SZ = 2000, // maximum size of remote frees to be buffered to flush at once - - MAXBLOCK = 65536 - 2 * sizeof(Header) - LARGEHDRSZ, // maximum size of large block - LBINS = 77, // number of large size bins - }; - - static_assert(sizeof(Header) == 16, "Wrong sizeof(Header)"); - static_assert(sizeof(DLink) <= 16, "Wrong sizeof(DLink)"); - static_assert(sizeof(BigHdr) + sizeof(Header) < BIGHDRSZ, "Big header sizeof issue"); - static StaticMutex mutex; Page work[NKLASS][1]; // circular list of pages that contain some empty blocks @@ -111,14 +358,8 @@ struct Heap { bool initialized; - static word BinSz[LBINS]; // block size for bin - static byte SzBin[MAXBLOCK / 8 + 1]; // maps size/8 to bin - static byte BlBin[MAXBLOCK / 8 + 1]; // Largest bin less or equal to size/8 (free -> bin) - - DLink large[1]; // all large chunks that belong to this heap - int lcount; // count of large chunks - DLink freebin[LBINS][1]; // all free blocks by bin - static DLink lempty[1]; // shared global list of all empty large blocks + LargeHeap lheap; + DLink large[1]; // all large 64KB chunks that belong to this heap void *out[REMOTE_OUT_SZ / 8 + 1]; void **out_ptr; @@ -129,17 +370,28 @@ struct Heap { FreeLink *small_remote_list; // list of remotely freed small blocks for lazy reclamation FreeLink *large_remote_list; // list of remotely freed large blocks for lazy reclamation - static DLink big[1]; // List of all big blocks - static Heap aux; // Single global auxiliary heap to store orphans and global list of free pages + struct HugePage { // to store the list of all huge pages in permanent storage + void *page; + HugePage *next; + }; + + static HugePage *huge_pages; + + static DLink big[1]; // List of all big blocks + static Heap aux; // Single global auxiliary heap to store orphans and global list of free pages + + static size_t huge_4KB_count; // total number of 4KB pages in small/large/huge blocks + static size_t free_4KB; // empty 4KB pages + static size_t big_size; // blocks >~64KB + static size_t big_count; + static size_t sys_size; // blocks allocated directly from system (included in big too) + static size_t sys_count; + static size_t huge_chunks; // 32MB master pages #ifdef HEAPDBG - static void DbgFreeFill(void *ptr, size_t size); - static void DbgFreeCheck(void *ptr, size_t size); static void DbgFreeFillK(void *ptr, int k); static void *DbgFreeCheckK(void *p, int k); #else - static void DbgFreeFill(void *ptr, size_t size) {} - static void DbgFreeCheck(void *ptr, size_t size) {} static void DbgFreeFillK(void *ptr, int k) {} static void *DbgFreeCheckK(void *p, int k) { return p; } #endif @@ -154,7 +406,6 @@ struct Heap { static int CheckFree(FreeLink *l, int k); void Check(); - static void Assert(bool b); static void DblCheck(Page *p); static void AssertLeaks(bool b); @@ -167,20 +418,17 @@ struct Heap { void *Allok(int k); void Free(void *ptr, Page *page, int k); void Free(void *ptr, int k); - void MoveLarge(Heap *dest, DLink *l); - void MoveToEmpty(DLink *l, Header *bh); - static void GlobalLInit(); - static int SizeToBin(int n) { return SzBin[(n - 1) >> 3]; } + static bool FreeSmallEmpty(int size4KB, int count = INT_MAX); - void LinkFree(DLink *b, int size); - DLink *AddChunk(int reqsize); - void *DivideBlock(DLink *b, int size); - void *TryLAlloc(int ii, size_t size); + void LInit(); + void *TryLAlloc(int i0, word wcount); void *LAlloc(size_t& size); + void FreeLargePage(DLink *l); void LFree(void *ptr); + bool LTryRealloc(void *ptr, size_t& newsize); size_t LGetBlockSize(void *ptr); - bool LTryRealloc(void *ptr, size_t newsize); + void Make(MemoryProfile& f); static void Shrink(); @@ -209,7 +457,7 @@ struct Heap { void *Alloc48(); void Free48(void *ptr); - bool TryRealloc(void *ptr, size_t newsize); + bool TryRealloc(void *ptr, size_t& newsize); }; force_inline diff --git a/uppsrc/Core/Index.h b/uppsrc/Core/Index.h index e52e431a2..c9f1c6956 100644 --- a/uppsrc/Core/Index.h +++ b/uppsrc/Core/Index.h @@ -1,167 +1,142 @@ -#define FOLDHASH +struct IndexCommon { + enum { HIBIT = 0x80000000 }; -enum { UNSIGNED_HIBIT = 0x80000000 }; - -class HashBase : Moveable { - struct Link : Moveable { - int next; + struct Hash : Moveable { + int next; // also link for unlinked items... + dword hash; int prev; }; - Vector hash; - Vector link; - int *map; - int mask; - int unlinked; + + int *map; + Hash *hash; + dword mask; + int unlinked; - void LinkBefore(int i, Link& l, int bi); - void LinkTo(int i, Link& l, int& m); - void Unlink(int i, Link& l, int& mi); - void Unlink(int i, Link& l); - int& Maph(unsigned _hash) const; - int& Mapi(int i) const; - void FinishIndex(); - void Zero(); - void Free(); + + static int empty[1]; -public: - void ClearIndex(); - void Reindex(int n); - void Reindex(); + static dword Smear(dword h) { return FoldHash(h) | HIBIT; } - void Add(unsigned hash); - void Set(int i, unsigned hash); - void SetUn(int i, unsigned hash); - unsigned operator [] (int i) const { return hash[i]; } - int Find(unsigned hash) const; - int GetNext(int i) const { return link[i].next; } - int FindNext(int i, int first) const; - int FindNext(int i) const; - int FindLast(unsigned hash) const; - int FindPrev(int i) const; - int Put(unsigned hash); + void Link(int& m, Hash& hh, int ii); + void Link(int ii, dword sh); + void Del(int& m, Hash& hh, int ii); + void Ins(int& m, Hash& hh, int ii); - bool IsUnlinked(int i) const { return hash[i] & UNSIGNED_HIBIT; } - void Unlink(int i); + void MakeMap(int count); + void Remap(int count); + void Reindex(int count); + void GrowMap(int count); + void FreeMap(); + void Free(); + + void Set(int ii, dword h); + Vector GetUnlinked() const; - bool HasUnlinked() const { return unlinked >= 0; } - - void Remove(int i); - void Remove(int i, int count); - void Remove(const int *sorted_list, int count); - void Insert(int i, unsigned hash); - - int GetCount() const { return hash.GetCount(); } - void Trim(int count); - void Drop(int n); - void Clear() { hash.Clear(); ClearIndex(); } - - void Reserve(int n); - void Shrink(); - -#ifdef UPP - void Serialize(Stream& s); -#endif - - HashBase(); - ~HashBase(); - - HashBase(HashBase&& b); - void operator=(HashBase&& b); - HashBase(const HashBase& b, int); - void operator<<=(const HashBase& b); - - const unsigned *begin() const { return hash.begin(); } - const unsigned *end() const { return hash.end(); } - - void Swap(HashBase& b); + + void Clear(); + void Trim(int n, int count); + void Sweep(int n); + void Reserve(int n); + void Shrink(); + void AdjustMap(int count, int alloc); + + void Copy(const IndexCommon& b, int count); + void Pick(IndexCommon& b); + void Swap(IndexCommon& b); + + IndexCommon(); + ~IndexCommon(); }; -template -class Index : MoveableAndDeepCopyOption> { -protected: - Vector key; - HashBase hash; +template class AMap; - int Find0(const T& x, int i) const { - while(i >= 0 && !(x == key[i])) i = hash.FindNext(i); - return i; - } - int FindB(const T& x, int i) const { - while(i >= 0 && !(x == key[i])) i = hash.FindPrev(i); - return i; - } - void Hash(); +template +class Index : MoveableAndDeepCopyOption>, IndexCommon { + Vector key; + + static dword Smear(const T& k) { return IndexCommon::Smear(GetHashValue(k)); } + + int FindFrom(int i, dword sh, const T& k, int end) const; + int FindBack(int i, dword sh, const T& k, int end) const; + + + void ReallocHash(int n); + template void GrowAdd(U&& k, dword sh); + template void AddS(int& m, U&& k, dword sh); + template void AddS(U&& k, dword sh); + + template int FindAdd(U&& k, OP add_op); + template int FindPut0(U&& k); + + template int Put0(U&& k, dword sh); + template void Set0(int i, U&& k); + + template friend class AMap; + + void FixHash(bool makemap = true); public: - unsigned hashfn(const T& x) const { return GetHashValue(x); } + void Add(const T& k) { AddS(k, Smear(k)); } + void Add(T&& k) { AddS(pick(k), Smear(k)); } + Index& operator<<(const T& x) { Add(x); return *this; } + Index& operator<<(T&& x) { Add(pick(x)); return *this; } - T& Add(const T& x, unsigned _hash); - T& Add(T&& x, unsigned _hash); - T& Add(const T& x); - T& Add(T&& x); - Index& operator<<(const T& x) { Add(x); return *this; } - Index& operator<<(T&& x) { Add(pick(x)); return *this; } - - int FindAdd(const T& key, unsigned _hash); - int FindAdd(const T& key); - int FindAdd(T&& key, unsigned _hash); - int FindAdd(T&& key); - - int Put(const T& x, unsigned _hash); - int Put(const T& x); - int Put(T&& x, unsigned _hash); - int Put(T&& x); - - int FindPut(const T& key, unsigned _hash); - int FindPut(const T& key); - int FindPut(T&& key, unsigned _hash); - int FindPut(T&& key); - - int Find(const T& x, unsigned _hash) const; - int Find(const T& x) const; - int FindNext(int i) const; - int FindLast(const T& x, unsigned _hash) const; - int FindLast(const T& x) const; - int FindPrev(int i) const; - - T& Set(int i, const T& x, unsigned _hash); - T& Set(int i, const T& x); - T& Set(int i, T&& x, unsigned _hash); - T& Set(int i, T&& x); - - const T& operator[](int i) const { return key[i]; } - int GetCount() const { return key.GetCount(); } - bool IsEmpty() const { return key.IsEmpty(); } + int Find(const T& k) const; + int FindNext(int i) const; + int FindLast(const T& k) const; + int FindPrev(int i) const; - unsigned GetHash(int i) const { return hash[i]; } + int FindAdd(const T& k) { return FindAdd(k, []{}); } + int FindAdd(T&& k) { return FindAdd(pick(k), []{}); } - void Clear() { hash.Clear(); key.Clear(); } + int Put(const T& k) { return Put0(k, Smear(k)); } + int Put(T&& k) { return Put0(pick(k), Smear(k)); } + int FindPut(const T& k) { return FindPut0(k); } + int FindPut(T&& k) { return FindPut0(pick(k)); } - void Unlink(int i) { hash.Unlink(i); } - int UnlinkKey(const T& k, unsigned h); - int UnlinkKey(const T& k); - bool IsUnlinked(int i) const { return hash.IsUnlinked(i); } - Vector GetUnlinked() const { return hash.GetUnlinked(); } - void Sweep(); - bool HasUnlinked() const { return hash.HasUnlinked(); } + void Unlink(int i); + int UnlinkKey(const T& k); + bool IsUnlinked(int i) const { return hash[i].hash == 0; } + bool HasUnlinked() const { return unlinked >= 0; } + Vector GetUnlinked() const { return IndexCommon::GetUnlinked(); } + + void Sweep(); + + void Set(int i, const T& k) { Set0(i, k); } + void Set(int i, T&& k) { Set0(i, pick(k)); } + + const T& operator[](int i) const { return key[i]; } + int GetCount() const { return key.GetCount(); } + bool IsEmpty() const { return key.IsEmpty(); } + + void Clear() { key.Clear(); IndexCommon::Clear(); } + + void Trim(int n = 0) { IndexCommon::Trim(n, GetCount()); key.Trim(n); } + void Drop(int n = 1) { Trim(GetCount() - 1); } + const T& Top() const { return key.Top(); } + T Pop() { T x = pick(Top()); Drop(); return x; } + + void Reserve(int n); + void Shrink(); + int GetAlloc() const { return key.GetAlloc(); } + + Vector PickKeys() { Vector r = pick(key); Clear(); return r; } + const Vector& GetKeys() const { return key; } - T& Insert(int i, const T& k, unsigned h); - T& Insert(int i, const T& k); - void Remove(int i); - void Remove(int i, int count); void Remove(const int *sorted_list, int count); - void Remove(const Vector& sorted_list); - int RemoveKey(const T& k, unsigned h); - int RemoveKey(const T& k); + void Remove(const Vector& sorted_list) { Remove(sorted_list, sorted_list.GetCount()); } + template void RemoveIf(Pred p) { Remove(FindAlli(key, p)); } + + Index() {} + Index(Index&& s) : key(pick(s.key)) { IndexCommon::Pick(s); } + Index(const Index& s, int) : key(s.key, 0) { ReallocHash(0); IndexCommon::Copy(s, key.GetCount()); } // TODO: Unlinked! + explicit Index(Vector&& s) : key(pick(s)) { FixHash(); } + Index(const Vector& s, int) : key(s, 0) { FixHash(); } - void Trim(int n) { key.SetCount(n); hash.Trim(n); } - void Drop(int n = 1) { key.Drop(n); hash.Drop(n); } - const T& Top() const { return key.Top(); } - T Pop() { T x = pick(Top()); Drop(); return x; } + Index& operator=(Vector&& x) { key = pick(x); FixHash(); return *this; } + Index& operator=(Index&& x) { key = pick(x.key); IndexCommon::Pick(x); return *this; } - void Reserve(int n) { key.Reserve(n); hash.Reserve(n); } - void Shrink() { key.Shrink(); hash.Shrink(); } - int GetAlloc() const { return key.GetAlloc(); } + Index(std::initializer_list init) : key(init) { FixHash(); } void Serialize(Stream& s); void Xmlize(XmlIO& xio, const char *itemtag = "key"); @@ -175,38 +150,30 @@ public: template bool operator<(const B& x) const { return Compare(x) < 0; } template bool operator>(const B& x) const { return Compare(x) > 0; } - Vector PickKeys() { Vector r = pick(key); Clear(); return r; } - const Vector& GetKeys() const { return key; } - - Index() {} - Index(Index&& s) : key(pick(s.key)), hash(pick(s.hash)) {} - Index(const Index& s, int) : key(s.key, 0), hash(s.hash, 0) {} - explicit Index(Vector&& s) : key(pick(s)) { Hash(); } - Index(const Vector& s, int) : key(s, 0) { Hash(); } - - Index& operator=(Vector&& x) { key = pick(x); Hash(); return *this; } - Index& operator=(Index&& x) { key = pick(x.key); hash = pick(x.hash); return *this; } - - Index(std::initializer_list init) : key(init) { Hash(); } - - typedef ConstIteratorOf> ConstIterator; - // Standard container interface - ConstIterator begin() const { return key.Begin(); } - ConstIterator end() const { return key.End(); } + typedef ConstIteratorOf> ConstIterator; + ConstIterator begin() const { return key.begin(); } + ConstIterator end() const { return key.end(); } - friend void Swap(Index& a, Index& b) { UPP::Swap(a.hash, b.hash); - UPP::Swap(a.key, b.key); } + friend void Swap(Index& a, Index& b) { a.IndexCommon::Swap(b); UPP::Swap(a.key, b.key); } +// deprecated: #ifdef DEPRECATED - Index& operator<<=(const Vector& s) { *this = clone(s); return *this; } + T& Insert(int i, const T& k) { key.Insert(i, k); FixHash(); return key[i]; } + void Remove(int i, int count) { key.Remove(i, count); FixHash(); } + void Remove(int i) { Remove(i, 1); } + int RemoveKey(const T& k) { int i = Find(k); if(i >= 0) Remove(i); return i; } + + unsigned GetHash(int i) const { return hash[i].hash; } + + Index& operator<<=(const Vector& s) { *this = clone(s); return *this; } typedef T ValueType; typedef Vector ValueContainer; - ConstIterator GetIter(int pos) const { return key.GetIter(pos); } + ConstIterator GetIter(int pos) const { return key.GetIter(pos); } - void ClearIndex() { hash.ClearIndex(); } - void Reindex(int n) { hash.Reindex(n); } - void Reindex() { hash.Reindex(); } + void ClearIndex() {} + void Reindex(int n) {} + void Reindex() {} typedef T value_type; typedef ConstIterator const_iterator; @@ -219,4 +186,8 @@ public: size_type size() { return GetCount(); } bool empty() const { return IsEmpty(); } #endif + +#ifdef _DEBUG + String Dump() const; +#endif }; diff --git a/uppsrc/Core/Log.cpp b/uppsrc/Core/Log.cpp index b0d070331..1e53878c8 100644 --- a/uppsrc/Core/Log.cpp +++ b/uppsrc/Core/Log.cpp @@ -305,55 +305,6 @@ void LogStream::_Put(const void *data, dword size) sTh.Put(*q++); } -String AsString(const MemoryProfile& mem) -{ - String text; -#ifdef UPP_HEAP - int acount = 0; - size_t asize = 0; - int fcount = 0; - size_t fsize = 0; - for(int i = 0; i < 1024; i++) - if(mem.allocated[i]) { - int sz = 4 * i; - text << Format("%4d B, %6d allocated (%5d KB), %6d fragmented (%5d KB)\n", - sz, mem.allocated[i], (mem.allocated[i] * sz) >> 10, - mem.fragmented[i], (mem.fragmented[i] * sz) >> 10); - acount += mem.allocated[i]; - asize += mem.allocated[i] * sz; - fcount += mem.fragmented[i]; - fsize += mem.fragmented[i] * sz; - } - text << Format(" TOTAL, %6d allocated (%5d KB), %6d fragmented (%5d KB)\n", - acount, int(asize >> 10), fcount, int(fsize >> 10)); - text << "Free 4KB pages " << mem.freepages << " (" << mem.freepages * 4 << " KB)\n"; - text << "Large block count " << mem.large_count - << ", total size " << (mem.large_total >> 10) << " KB\n"; - text << "Large fragments count " << mem.large_free_count - << ", total size " << (mem.large_free_total >> 10) << " KB\n"; - text << "Large free 64KB pages " << mem.large_empty - << ", total size " << 64 * mem.large_empty << " KB\n"; - text << "Big block count " << mem.big_count - << ", total size " << int(mem.big_size >> 10) << " KB\n"; -#endif - return text; -} - -int64 GetAllocatedBytes(const MemoryProfile& mem) -{ - int64 asize = 0; -#ifdef UPP_HEAP - for(int i = 0; i < 1024; i++) - if(mem.allocated[i]) { - int sz = 4 * i; - asize += mem.allocated[i] * sz; - } - asize += mem.large_total; - asize += mem.big_size; -#endif - return asize; -} - #ifdef flagCHECKINIT void InitBlockBegin__(const char *fn, int line) { diff --git a/uppsrc/Core/Map.h b/uppsrc/Core/Map.h index 2332789f3..231f7f54a 100644 --- a/uppsrc/Core/Map.h +++ b/uppsrc/Core/Map.h @@ -34,6 +34,18 @@ class AMap { protected: Index key; V value; + + template int FindAdd_(KK&& k); + template int FindAdd_(KK&& k, TT&& init); + template T& Put_(KK&& k); + template int Put_(KK&& k, TT&& x); + template int PutDefault_(KK&& k); + template int FindPut_(KK&& k); + template int FindPut_(KK&& k, TT&& init); + template T& GetAdd_(KK&& k); + template T& GetAdd_(KK&& k, TT&& init); + template T& GetPut_(KK&& k); + template T& GetPut_(KK&& k, TT&& init); public: T& Add(const K& k, const T& x) { key.Add(k); return value.Add(x); } @@ -43,139 +55,124 @@ public: T& Add(K&& k, T&& x) { key.Add(pick(k)); return value.Add(pick(x)); } T& Add(K&& k) { key.Add(pick(k)); return value.Add(); } - int Find(const K& k, unsigned h) const { return key.Find(k, h); } int Find(const K& k) const { return key.Find(k); } int FindNext(int i) const { return key.FindNext(i); } - int FindLast(const K& k, unsigned h) const { return key.FindLast(k, h); } int FindLast(const K& k) const { return key.FindLast(k); } int FindPrev(int i) const { return key.FindPrev(i); } - int FindAdd(const K& k); - int FindAdd(const K& k, const T& init); - int FindAdd(const K& k, T&& init); - int FindAdd(K&& k); - int FindAdd(K&& k, const T& init); - int FindAdd(K&& k, T&& init); + int FindAdd(const K& k) { return FindAdd_(k); } + int FindAdd(const K& k, const T& init) { return FindAdd_(k, init); } + int FindAdd(const K& k, T&& init) { return FindAdd_(k, pick(init)); } + int FindAdd(K&& k) { return FindAdd_(pick(k)); } + int FindAdd(K&& k, const T& init) { return FindAdd_(pick(k), init); } + int FindAdd(K&& k, T&& init) { return FindAdd_(pick(k), pick(init)); } - T& Put(const K& k); - T& Put(K&& k); + T& Put(const K& k) { return Put_(k); } + int Put(const K& k, const T& x) { return Put_(k, x); } + int Put(const K& k, T&& x) { return Put_(k, pick(x)); } + T& Put(K&& k) { return Put_(pick(k)); } + int Put(K&& k, const T& x) { return Put_(pick(k), x); } + int Put(K&& k, T&& x) { return Put_(pick(k), pick(x)); } - int Put(const K& k, const T& x); - int Put(const K& k, T&& x); - int Put(K&& k, const T& x); - int Put(K&& k, T&& x); + int PutDefault(const K& k) { return PutDefault_(k); } + int PutDefault(K&& k) { return PutDefault_(pick(k)); } - int PutDefault(const K& k); - int PutDefault(K&& k); + int FindPut(const K& k) { return FindPut_(k); } + int FindPut(const K& k, const T& init) { return FindPut_(k, init); } + int FindPut(const K& k, T&& init) { return FindPut_(k, pick(init)); } + int FindPut(K&& k) { return FindPut_(pick(k)); } + int FindPut(K&& k, const T& init) { return FindPut_(pick(k), init); } + int FindPut(K&& k, T&& init) { return FindPut_(pick(k), pick(init)); } - int FindPut(const K& k); - int FindPut(const K& k, const T& init); - int FindPut(const K& k, T&& init); - int FindPut(K&& k); - int FindPut(K&& k, const T& init); - int FindPut(K&& k, T&& init); + T& Get(const K& k) { return value[Find(k)]; } + const T& Get(const K& k) const { return value[Find(k)]; } + const T& Get(const K& k, const T& d) const { int i = Find(k); return i >= 0 ? value[i] : d; } - T& Get(const K& k) { return value[Find(k)]; } - const T& Get(const K& k) const { return value[Find(k)]; } - const T& Get(const K& k, const T& d) const { int i = Find(k); return i >= 0 ? value[i] : d; } + T& GetAdd(const K& k) { return GetAdd_(k); } + T& GetAdd(const K& k, const T& x) { return GetAdd_(k, x); } + T& GetAdd(const K& k, T&& x) { return GetAdd_(k, pick(x)); } + T& GetAdd(K&& k) { return GetAdd_(pick(k)); } + T& GetAdd(K&& k, const T& x) { return GetAdd_(pick(k), x); } + T& GetAdd(K&& k, T&& x) { return GetAdd_(pick(k), pick(x)); } - T& GetAdd(const K& k); - T& GetAdd(const K& k, const T& x); - T& GetAdd(const K& k, T&& x); - T& GetAdd(K&& k); - T& GetAdd(K&& k, const T& x); - T& GetAdd(K&& k, T&& x); + T& GetPut(const K& k) { return GetPut_(k); } + T& GetPut(const K& k, const T& x) { return GetPut_(k, x); } + T& GetPut(const K& k, T&& x) { return GetPut_(k, pick(x)); } + T& GetPut(K&& k) { return GetPut_(pick(k)); } + T& GetPut(K&& k, const T& x) { return GetPut_(pick(k), x); } + T& GetPut(K&& k, T&& x) { return GetPut_(pick(k), pick(x)); } - T& GetPut(const K& k); - T& GetPut(const K& k, const T& x); - T& GetPut(const K& k, T&& x); - T& GetPut(K&& k); - T& GetPut(K&& k, const T& x); - T& GetPut(K&& k, T&& x); + void SetKey(int i, const K& k) { key.Set(i, k); } + void SetKey(int i, K&& k) { key.Set(i, pick(k)); } - void SetKey(int i, const K& k) { key.Set(i, k); } - void SetKey(int i, K&& k) { key.Set(i, pick(k)); } + T *FindPtr(const K& k) { int i = Find(k); return i >= 0 ? &value[i] : NULL; } + const T *FindPtr(const K& k) const { int i = Find(k); return i >= 0 ? &value[i] : NULL; } - T *FindPtr(const K& k) { int i = Find(k); return i >= 0 ? &value[i] : NULL; } - const T *FindPtr(const K& k) const { int i = Find(k); return i >= 0 ? &value[i] : NULL; } + T *FindLastPtr(const K& k) { int i = FindLast(k); return i >= 0 ? &value[i] : NULL; } + const T *FindLastPtr(const K& k) const { int i = FindLast(k); return i >= 0 ? &value[i] : NULL; } - T *FindLastPtr(const K& k) { int i = FindLast(k); return i >= 0 ? &value[i] : NULL; } - const T *FindLastPtr(const K& k) const { int i = FindLast(k); return i >= 0 ? &value[i] : NULL; } - - void Unlink(int i) { key.Unlink(i); } - int UnlinkKey(const K& k, unsigned h) { return key.UnlinkKey(k, h); } - int UnlinkKey(const K& k) { return key.UnlinkKey(k); } - bool IsUnlinked(int i) const { return key.IsUnlinked(i); } + void Unlink(int i) { key.Unlink(i); } + int UnlinkKey(const K& k, unsigned h) { return key.UnlinkKey(k, h); } + int UnlinkKey(const K& k) { return key.UnlinkKey(k); } + bool IsUnlinked(int i) const { return key.IsUnlinked(i); } void Sweep(); - bool HasUnlinked() const { return key.HasUnlinked(); } + bool HasUnlinked() const { return key.HasUnlinked(); } - T& Insert(int i, const K& k) { key.Insert(i, k); return value.Insert(i); } - T& Insert(int i, const K& k, const T& x) { key.Insert(i, k); return value.Insert(i, x); } - T& Insert(int i, const K& k, T&& x) { key.Insert(i, k); return value.Insert(i, pick(x)); } - T& Insert(int i, K&& k) { key.Insert(i, pick(k)); return value.Insert(i); } - T& Insert(int i, K&& k, const T& x) { key.Insert(i, pick(k)); return value.Insert(i, x); } - T& Insert(int i, K&& k, T&& x) { key.Insert(i, pick(k)); return value.Insert(i, pick(x)); } + const T& operator[](int i) const { return value[i]; } + T& operator[](int i) { return value[i]; } + int GetCount() const { return value.GetCount(); } + bool IsEmpty() const { return value.IsEmpty(); } + void Clear() { key.Clear(); value.Clear(); } + void Shrink() { value.Shrink(); key.Shrink(); } + void Reserve(int xtra) { value.Reserve(xtra); key.Reserve(xtra); } + int GetAlloc() const { return value.GetAlloc(); } - void Remove(int i) { key.Remove(i); value.Remove(i); } - void Remove(int i, int count) { key.Remove(i, count); value.Remove(i, count); } - void Remove(const int *sl, int n) { key.Remove(sl, n); value.Remove(sl, n); } - void Remove(const Vector& sl) { Remove(sl, sl.GetCount()); } - int RemoveKey(const K& k); + unsigned GetHash(int i) const { return key.GetHash(i); } - const T& operator[](int i) const { return value[i]; } - T& operator[](int i) { return value[i]; } - int GetCount() const { return value.GetCount(); } - bool IsEmpty() const { return value.IsEmpty(); } - void Clear() { key.Clear(); value.Clear(); } - void Shrink() { value.Shrink(); key.Shrink(); } - void Reserve(int xtra) { value.Reserve(xtra); key.Reserve(xtra); } - int GetAlloc() const { return value.GetAlloc(); } + void Drop(int n = 1) { key.Drop(n); value.Drop(n); } + T& Top() { return value.Top(); } + const T& Top() const { return value.Top(); } + const K& TopKey() const { return key.Top(); } +// T Pop() { T h = Top(); Drop(); return h; } + K PopKey() { K h = TopKey(); Drop(); return h; } + void Trim(int n) { key.Trim(n); value.SetCount(n); } - unsigned GetHash(int i) const { return key.GetHash(i); } + const K& GetKey(int i) const { return key[i]; } - void Drop(int n = 1) { key.Drop(n); value.Drop(n); } - T& Top() { return value.Top(); } - const T& Top() const { return value.Top(); } - const K& TopKey() const { return key.Top(); } -// T Pop() { T h = Top(); Drop(); return h; } - K PopKey() { K h = TopKey(); Drop(); return h; } - void Trim(int n) { key.Trim(n); value.SetCount(n); } + void Remove(const int *sl, int n) { key.Remove(sl, n); value.Remove(sl, n); } + void Remove(const Vector& sl) { Remove(sl, sl.GetCount()); } + template void RemoveIf(P p) { Remove(FindAlli(p)); } - const K& GetKey(int i) const { return key[i]; } - -#ifdef UPP void Serialize(Stream& s); void Xmlize(XmlIO& xio); void Jsonize(JsonIO& jio); String ToString() const; - dword GetHashValue() const { return HashBySerialize(*this); } - bool operator==(const AMap& b) const { ASSERT(!HasUnlinked()); return IsEqualMap(*this, b); } - bool operator!=(const AMap& b) const { return !operator==(b); } - int Compare(const AMap& b) const { ASSERT(!HasUnlinked()); return CompareMap(*this, b); } - bool operator<=(const AMap& x) const { return Compare(x) <= 0; } - bool operator>=(const AMap& x) const { return Compare(x) >= 0; } - bool operator<(const AMap& x) const { return Compare(x) < 0; } - bool operator>(const AMap& x) const { return Compare(x) > 0; } -#endif + dword GetHashValue() const { return HashBySerialize(*this); } + bool operator==(const AMap& b) const { ASSERT(!HasUnlinked()); return IsEqualMap(*this, b); } + bool operator!=(const AMap& b) const { return !operator==(b); } + int Compare(const AMap& b) const { ASSERT(!HasUnlinked()); return CompareMap(*this, b); } + bool operator<=(const AMap& x) const { return Compare(x) <= 0; } + bool operator>=(const AMap& x) const { return Compare(x) >= 0; } + bool operator<(const AMap& x) const { return Compare(x) < 0; } + bool operator>(const AMap& x) const { return Compare(x) > 0; } - void Swap(AMap& x) { UPP::Swap(value, x.value); - UPP::Swap(key, x.key); } - const Index& GetIndex() const { return key; } - Index PickIndex() { return pick(key); } + void Swap(AMap& x) { UPP::Swap(value, x.value); UPP::Swap(key, x.key); } + const Index& GetIndex() const { return key; } + Index PickIndex() { return pick(key); } - const Vector& GetKeys() const { return key.GetKeys(); } - Vector PickKeys() { return key.PickKeys(); } + const Vector& GetKeys() const { return key.GetKeys(); } + Vector PickKeys() { return key.PickKeys(); } - const V& GetValues() const { return value; } - V& GetValues() { return value; } - V PickValues() { return pick(value); } + const V& GetValues() const { return value; } + V& GetValues() { return value; } + V PickValues() { return pick(value); } - MapKVRange operator~() { return MapKVRange(*this); } + MapKVRange operator~() { return MapKVRange(*this); } MapKVRange operator~() const { return MapKVRange(*this); } - AMap& operator()(const K& k, const T& v) { Add(k, v); return *this; } + AMap& operator()(const K& k, const T& v) { Add(k, v); return *this; } - AMap() {} + AMap() {} AMap(const AMap& s, int) : key(s.key, 0), value(s.value, 0) {} AMap(Index&& ndx, V&& val) : key(pick(ndx)), value(pick(val)) {} AMap(Vector&& ndx, V&& val) : key(pick(ndx)), value(pick(val)) {} @@ -184,10 +181,10 @@ public: typedef IteratorOf Iterator; typedef ConstIteratorOf ConstIterator; - Iterator begin() { return value.begin(); } - Iterator end() { return value.end(); } - ConstIterator begin() const { return value.begin(); } - ConstIterator end() const { return value.end(); } + Iterator begin() { return value.begin(); } + Iterator end() { return value.end(); } + ConstIterator begin() const { return value.begin(); } + ConstIterator end() const { return value.end(); } #ifdef DEPRECATED typedef V ValueContainer; @@ -211,6 +208,17 @@ public: KeyConstIterator KeyBegin() const { return key.begin(); } KeyConstIterator KeyEnd() const { return key.end(); } + + T& Insert(int i, const K& k) { key.Insert(i, k); return value.Insert(i); } + T& Insert(int i, const K& k, const T& x) { key.Insert(i, k); return value.Insert(i, x); } + T& Insert(int i, const K& k, T&& x) { key.Insert(i, k); return value.Insert(i, pick(x)); } + T& Insert(int i, K&& k) { key.Insert(i, pick(k)); return value.Insert(i); } + T& Insert(int i, K&& k, const T& x) { key.Insert(i, pick(k)); return value.Insert(i, x); } + T& Insert(int i, K&& k, T&& x) { key.Insert(i, pick(k)); return value.Insert(i, pick(x)); } + + void Remove(int i) { key.Remove(i); value.Remove(i); } + void Remove(int i, int count) { key.Remove(i, count); value.Remove(i, count); } + int RemoveKey(const K& k); #endif }; diff --git a/uppsrc/Core/Map.hpp b/uppsrc/Core/Map.hpp index 4b68b76b2..885bd7d73 100644 --- a/uppsrc/Core/Map.hpp +++ b/uppsrc/Core/Map.hpp @@ -1,571 +1,23 @@ -inline int& HashBase::Maph(unsigned _hash) const -{ - unsigned h = _hash & ~UNSIGNED_HIBIT; -#ifdef FOLDHASH - return map[mask & FoldHash(h)]; -// return map[(mcount - 1) & (((h >> 23) - (h >> 15) - (h >> 7) - h))]; -#else - return map[h % mcount]; -#endif -} - -inline int& HashBase::Mapi(int i) const -{ - return Maph(hash[i]); -} - -inline void HashBase::LinkTo(int i, Link& l, int& m) -{ - if(m >= 0) { - Link& bl = link[m]; - l.next = m; - l.prev = bl.prev; - bl.prev = i; - link[l.prev].next = i; - } - else - m = l.prev = l.next = i; -} - -inline -void HashBase::Add(unsigned _hash) -{ - ASSERT(hash.GetCount() == link.GetCount()); - int i = hash.GetCount(); - hash.Add(_hash & ~UNSIGNED_HIBIT); - if(hash.GetCount() <= mask) - LinkTo(i, link.Add(), Maph(_hash)); - else - Reindex(); -} - -inline void HashBase::Unlink(int i, Link& l, int& m) -{ - if(i == m) { - if(l.next == i) { - m = -1; - return; - } - m = l.next; - } - link[l.next].prev = l.prev; - link[l.prev].next = l.next; -} - -inline void HashBase::Unlink(int i, Link& l) -{ - Unlink(i, l, hash[i] & UNSIGNED_HIBIT ? unlinked : Mapi(i)); -} - -inline int HashBase::Find(unsigned _hash) const -{ - return Maph(_hash); -} - -inline int HashBase::FindNext(int i) const -{ - int q = link[i].next; - return q == Mapi(i) ? -1 : q; -} - -inline int HashBase::FindLast(unsigned _hash) const -{ - if(hash.GetCount() == 0) return - 1; - int i = Find(_hash & ~UNSIGNED_HIBIT); - return i >= 0 ? link[i].prev : -1; -} - -inline int HashBase::FindPrev(int i) const -{ - int q = link[i].prev; - return q == link[Mapi(i)].prev ? -1 : q; -} - -inline void HashBase::Unlink(int i) -{ - ASSERT(!IsUnlinked(i)); - hash[i] |= UNSIGNED_HIBIT; - if(i < link.GetCount()) { - Link& lnk = link[i]; - Unlink(i, lnk, Mapi(i)); - LinkTo(i, lnk, unlinked); - } -} - -template -void Index::Hash() { - hash.Clear(); - for(int i = 0; i < key.GetCount(); i++) - hash.Add(hashfn(key[i])); -} - -template -T& Index::Add(const T& x, unsigned _hash) { - T& t = key.Add(x); - hash.Add(_hash); - return t; -} - -template -T& Index::Add(const T& x) { - return Add(x, hashfn(x)); -} - -template -T& Index::Add(T&& x, unsigned _hash) -{ - T& t = key.Add(pick(x)); - hash.Add(_hash); - return t; -} - -template -T& Index::Add(T&& x) -{ - return Add(pick(x), hashfn(x)); -} - -template -int Index::FindAdd(const T& _key, unsigned _hash) { - int i = Find(_key, _hash); - if(i >= 0) return i; - i = key.GetCount(); - Add(_key, _hash); - return i; -} - -template -int Index::FindAdd(T&& _key, unsigned _hash) -{ - int i = Find(_key, _hash); - if(i >= 0) return i; - i = key.GetCount(); - Add(pick(_key), _hash); - return i; -} - -template -int Index::FindAdd(T&& key) -{ - return FindAdd(pick(key), hashfn(key)); -} - -template -int Index::Put(const T& x, unsigned _hash) -{ - int q = hash.Put(_hash); - if(q < 0) { - q = key.GetCount(); - Add(x, _hash); - } - else - key[q] = x; - return q; -} - -template -int Index::Put(const T& x) -{ - return Put(x, hashfn(x)); -} - -template -int Index::Put(T&& x, unsigned _hash) -{ - int q = hash.Put(_hash); - if(q < 0) { - q = key.GetCount(); - Add(pick(x), _hash); - } - else - key[q] = pick(x); - return q; -} - -template -int Index::Put(T&& x) -{ - return Put(pick(x), hashfn(x)); -} - -template -int Index::FindPut(const T& _key, unsigned _hash) -{ - int i = Find(_key, _hash); - if(i >= 0) return i; - return Put(_key, _hash); -} - -template -int Index::FindPut(const T& key) -{ - return FindPut(key, hashfn(key)); -} - -template -int Index::FindPut(T&& key, unsigned hash) -{ - int i = Find(key, hash); - if(i >= 0) return i; - return Put(pick(key), hash); -} - -template -int Index::FindPut(T&& key) -{ - return FindPut(pick(key), hashfn(key)); -} - -#ifdef flagTESTINDEX -extern int max_collisions; -#endif - -template -inline -int Index::Find(const T& x, unsigned hash_) const { - int i0 = hash.Find(hash_); - if(i0 >= 0) { - int i = i0; -#ifdef flagTESTINDEX - int collisions = 0; -#endif - do { - if(x == key[i]) return i; - i = hash.GetNext(i); -#ifdef flagTESTINDEX - collisions++; -#endif - } - while(i0 != i); -#ifdef flagTESTINDEX - max_collisions = max(max_collisions, collisions); -#endif - } - return -1; -} - -template -int Index::Find(const T& x) const { - return Find(x, hashfn(x)); -} - -template -int Index::FindNext(int i) const { - return Find0(key[i], hash.FindNext(i)); -} - -template -inline int Index::FindLast(const T& x, unsigned _hash) const { - return FindB(x, hash.FindLast(_hash)); -} - -template -int Index::FindLast(const T& x) const { - return FindLast(x, hashfn(x)); -} - -template -int Index::FindPrev(int i) const { - return FindB(key[i], hash.FindPrev(i)); -} - -template -int Index::FindAdd(const T& key) { - return FindAdd(key, hashfn(key)); -} - -template -T& Index::Set(int i, const T& x, unsigned _hash) { - T& t = key[i]; - t = x; - hash.Set(i, _hash); - return t; -} - -template -T& Index::Set(int i, const T& x) { - return Set(i, x, hashfn(x)); -} - -template -T& Index::Set(int i, T&& x, unsigned _hash) -{ - T& t = key[i]; - t = pick(x); - hash.Set(i, _hash); - return t; -} - -template -T& Index::Set(int i, T&& x) -{ - return Set(i, pick(x), hashfn(x)); -} - -#ifdef UPP -template -void Index::Serialize(Stream& s) { - if(s.IsLoading()) hash.ClearIndex(); - key.Serialize(s); - hash.Serialize(s); -} - -template -void Index::Xmlize(XmlIO& xio, const char *itemtag) -{ - XmlizeIndex >(xio, itemtag, *this); -} - -template -void Index::Jsonize(JsonIO& jio) -{ - JsonizeIndex, T>(jio, *this); -} - -template -String Index::ToString() const -{ - return AsStringArray(*this); -} - -#endif - -template -int Index::UnlinkKey(const T& k, unsigned h) -{ - int n = 0; - int q = hash.Find(h); - while(q >= 0) { - int w = q; - q = hash.FindNext(q); - if(k == key[w]) { - hash.Unlink(w); - n++; - } - } - return n; -} - -template -int Index::UnlinkKey(const T& k) -{ - return UnlinkKey(k, hashfn(k)); -} - -template -void Index::Sweep() -{ - Vector b = hash.GetUnlinked(); - Sort(b); - Remove(b); -} - -template -T& Index::Insert(int i, const T& k, unsigned h) { - key.Insert(i, k); - hash.Insert(i, h); - return key[i]; -} - -template -T& Index::Insert(int i, const T& k) { - key.Insert(i, k); - hash.Insert(i, hashfn(k)); - return key[i]; -} - -template -void Index::Remove(int i) -{ - key.Remove(i); - hash.Remove(i); -} - -template -void Index::Remove(int i, int count) -{ - key.Remove(i, count); - hash.Remove(i, count); -} - -template -void Index::Remove(const int *sorted_list, int count) -{ - key.Remove(sorted_list, count); - hash.Remove(sorted_list, count); -} - -template -void Index::Remove(const Vector& sl) -{ - Remove(sl, sl.GetCount()); -} - -template -int Index::RemoveKey(const T& k, unsigned h) -{ - Vector rk; - int q = Find(k, h); - while(q >= 0) { - rk.Add(q); - q = FindNext(q); - } - Remove(rk); - return rk.GetCount(); -} - -template -int Index::RemoveKey(const T& k) -{ - return RemoveKey(k, hashfn(k)); -} - -// ------------------ - -/* -template -T& ArrayIndex::Add(T *newt, unsigned _hash) { - B::hash.Add(_hash); - return B::key.Add(newt); -} - -template -T& ArrayIndex::Add(T *newt) { - return Add(newt, B::hashfn(*newt)); -} - -template -T& ArrayIndex::Set(int i, T *newt, unsigned _hash) { - T& t = B::key.Set(i, newt); - B::hash.Set(i, _hash); - return t; -} - -template -T& ArrayIndex::Set(int i, T *newt) { - return Set(i, newt, B::hashfn(*newt)); -} -*/ - // -------------------- template -int AMap::FindAdd(const K& k) { - unsigned hash = key.hashfn(k); - int i = Find(k, hash); - if(i < 0) { - i = GetCount(); - key.Add(k, hash); - value.Add(); - } - return i; +template +int AMap::FindAdd_(KK&& k) { + return key.FindAdd(std::forward(k), [&] { value.Add(); }); } template -int AMap::FindAdd(const K& k, const T& x) { - unsigned hash = key.hashfn(k); - int i = Find(k, hash); - if(i < 0) { - i = GetCount(); - key.Add(k, hash); - value.Add(x); - } - return i; +template +int AMap::FindAdd_(KK&& k, TT&& x) { + return key.FindAdd(std::forward(k), [&] { value.Add(std::forward(x)); }); } -template -int AMap::FindAdd(const K& k, T&& x) { - unsigned hash = key.hashfn(k); - int i = Find(k, hash); - if(i < 0) { - i = GetCount(); - key.Add(k, hash); - value.Add(pick(x)); - } - return i; -} template -int AMap::FindAdd(K&& k) { - unsigned hash = key.hashfn(k); - int i = Find(k, hash); - if(i < 0) { - i = GetCount(); - key.Add(pick(k), hash); - value.Add(); - } - return i; -} - -template -int AMap::FindAdd(K&& k, const T& x) { - unsigned hash = key.hashfn(k); - int i = Find(k, hash); - if(i < 0) { - i = GetCount(); - key.Add(pick(k), hash); - value.Add(x); - } - return i; -} - -template -int AMap::FindAdd(K&& k, T&& x) { - unsigned hash = key.hashfn(k); - int i = Find(k, hash); - if(i < 0) { - i = GetCount(); - key.Add(pick(k), hash); - value.Add(pick(x)); - } - return i; -} - -template -int AMap::Put(const K& k, const T& x) +template +T& AMap::Put_(KK&& k) { - int i = key.Put(k); - if(i < value.GetCount()) - value[i] = x; - else { - ASSERT(i == value.GetCount()); - value.Add(x); - } - return i; -} - -template -int AMap::PutDefault(const K& k) -{ - int i = key.Put(k); - if(i >= value.GetCount()) { - ASSERT(i == value.GetCount()); - value.Add(); - } - else { - Destroy(&value[i], &value[i] + 1); - Construct(&value[i], &value[i] + 1); - } - return i; -} - -template -int AMap::Put(const K& k, T&& x) -{ - int i = key.Put(k); - if(i < value.GetCount()) - value[i] = pick(x); - else { - ASSERT(i == value.GetCount()); - value.Add(pick(x)); - } - return i; -} - -template -T& AMap::Put(const K& k) -{ - int i = key.Put(k); + int i = key.Put(std::forward(k)); if(i < value.GetCount()) { Destroy(&value[i], &value[i] + 1); Construct(&value[i], &value[i] + 1); @@ -578,22 +30,24 @@ T& AMap::Put(const K& k) } template -int AMap::Put(K&& k, const T& x) +template +int AMap::Put_(KK&& k, TT&& x) { - int i = key.Put(pick(k)); + int i = key.Put(std::forward(k)); if(i < value.GetCount()) - value[i] = x; + value[i] = std::forward(x); else { ASSERT(i == value.GetCount()); - value.Add(x); + value.Add(std::forward(x)); } return i; } template -int AMap::PutDefault(K&& k) +template +int AMap::PutDefault_(KK&& k) { - int i = key.Put(pick(k)); + int i = key.Put(std::forward(k)); if(i >= value.GetCount()) { ASSERT(i == value.GetCount()); value.Add(); @@ -606,200 +60,59 @@ int AMap::PutDefault(K&& k) } template -int AMap::Put(K&& k, T&& x) +template +int AMap::FindPut_(KK&& k) { - int i = key.Put(pick(k)); - if(i < value.GetCount()) - value[i] = pick(x); + int i = key.FindPut(std::forward(k)); + if(i >= value.GetCount()) + value.Add(); else { - ASSERT(i == value.GetCount()); - value.Add(pick(x)); - } - return i; -} - -template -T& AMap::Put(K&& k) -{ - int i = key.Put(pick(k)); - if(i < value.GetCount()) { Destroy(&value[i], &value[i] + 1); Construct(&value[i], &value[i] + 1); - return value[i]; - } - else { - ASSERT(i == value.GetCount()); - return value.Add(); - } -} - -template -int AMap::FindPut(const K& k) -{ - unsigned hash = key.hashfn(k); - int i = Find(k, hash); - return i < 0 ? PutDefault(k) : i; -} - -template -int AMap::FindPut(const K& k, const T& init) -{ - unsigned hash = key.hashfn(k); - int i = Find(k, hash); - if(i < 0) { - i = key.Put(k, hash); - if(i >= value.GetCount()) { - ASSERT(i == value.GetCount()); - i = value.GetCount(); - value.Add(init); - } - else - value[i] = init; } return i; } template -int AMap::FindPut(const K& k, T&& init) +template +int AMap::FindPut_(KK&& k, TT&& init) { - unsigned hash = key.hashfn(k); - int i = Find(k, hash); - if(i < 0) { - i = key.Put(k, hash); - value.At(i) = pick(init); - } + int i = key.FindPut(std::forward(k)); + if(i >= value.GetCount()) + value.Add(std::forward(init)); return i; } template -int AMap::FindPut(K&& k) +template +T& AMap::GetAdd_(KK&& k) { - unsigned hash = key.hashfn(k); - int i = Find(k, hash); - return i < 0 ? PutDefault(pick(k)) : i; + return value[FindAdd(std::forward(k))]; } template -int AMap::FindPut(K&& k, const T& init) +template +T& AMap::GetAdd_(KK&& k, TT&& x) { - unsigned hash = key.hashfn(k); - int i = Find(k, hash); - if(i < 0) { - i = key.Put(pick(k), hash); - if(i >= value.GetCount()) { - ASSERT(i == value.GetCount()); - i = value.GetCount(); - value.Add(init); - } - else - value[i] = init; - } - return i; + return value[FindAdd(std::forward(k), std::forward(x))]; } template -int AMap::FindPut(K&& k, T&& init) +template +T& AMap::GetPut_(KK&& k) { - unsigned hash = key.hashfn(k); - int i = Find(k, hash); - if(i < 0) { - i = key.Put(pick(k), hash); - value.At(i) = pick(init); - } - return i; + int i = key.FindPut(std::forward(k)); + return i < value.GetCount() ? value[i] : value.Add(); } template -T& AMap::GetAdd(const K& k) { - unsigned hash = key.hashfn(k); - int i = key.Find(k, hash); - if(i >= 0) - return value[i]; - key.Add(k, hash); - return value.Add(); +template +T& AMap::GetPut_(KK&& k, TT&& x) +{ + int i = key.FindPut(std::forward(k)); + return i < value.GetCount() ? value[i] : value.Add(std::forward(x)); } -template -T& AMap::GetAdd(const K& k, const T& x) { - unsigned hash = key.hashfn(k); - int i = Find(k, hash); - if(i >= 0) return value[i]; - key.Add(k, hash); - value.Add(x); - return value.Top(); -} - -template -T& AMap::GetAdd(const K& k, T&& x) { - unsigned hash = key.hashfn(k); - int i = Find(k, hash); - if(i >= 0) return value[i]; - key.Add(k, hash); - value.Add(pick(x)); - return value.Top(); -} - -template -T& AMap::GetAdd(K&& k) { - unsigned hash = key.hashfn(k); - int i = key.Find(k, hash); - if(i >= 0) - return value[i]; - key.Add(pick(k), hash); - return value.Add(); -} - -template -T& AMap::GetAdd(K&& k, const T& x) { - unsigned hash = key.hashfn(k); - int i = Find(k, hash); - if(i >= 0) return value[i]; - key.Add(pick(k), hash); - value.Add(x); - return value.Top(); -} - -template -T& AMap::GetAdd(K&& k, T&& x) { - unsigned hash = key.hashfn(k); - int i = Find(k, hash); - if(i >= 0) return value[i]; - key.Add(pick(k), hash); - value.Add(pick(x)); - return value.Top(); -} - -template -T& AMap::GetPut(const K& k) { - return value[FindPut(k)]; -} - -template -T& AMap::GetPut(const K& k, const T& x) { - return value[FindPut(k, x)]; -} - -template -T& AMap::GetPut(const K& k, T&& x) { - return value[FindPut(k, pick(x))]; -} - -template -T& AMap::GetPut(K&& k) { - return value[FindPut(pick(k))]; -} - -template -T& AMap::GetPut(K&& k, const T& x) { - return value[FindPut(pick(k), x)]; -} - -template -T& AMap::GetPut(K&& k, T&& x) { - return value[FindPut(pick(k), pick(x))]; -} - -#ifdef UPP template void AMap::Serialize(Stream& s) { int version = 0; @@ -834,8 +147,6 @@ String AMap::ToString() const return r; } -#endif - template int AMap::RemoveKey(const K& k) { @@ -852,10 +163,8 @@ int AMap::RemoveKey(const K& k) template void AMap::Sweep() { - Vector b = key.GetUnlinked(); - Sort(b); - key.Remove(b); - value.Remove(b); + value.RemoveIf([&](int i) { return key.IsUnlinked(i); }); + key.Sweep(); } #ifdef UPP diff --git a/uppsrc/Core/String.cpp b/uppsrc/Core/String.cpp index 1e6d54e83..7e116da00 100644 --- a/uppsrc/Core/String.cpp +++ b/uppsrc/Core/String.cpp @@ -104,13 +104,10 @@ char *String0::Alloc(int count, char& kind) } size_t sz = sizeof(Rc) + count + 1; Rc *rc = (Rc *)MemoryAllocSz(sz); - if(count == INT_MAX) - rc->alloc = INT_MAX; - else - rc->alloc = (int)sz - sizeof(Rc) - 1; + rc->alloc = count == INT_MAX ? INT_MAX : (int)sz - sizeof(Rc) - 1; rc->refcount = 1; kind = min(rc->alloc, 255); - return (char *)(rc + 1); + return rc->GetPtr(); } char *String0::Insert(int pos, int count, const char *s) @@ -385,23 +382,38 @@ void StringBuffer::Realloc(dword n, const char *cat, int l) size_t ep = pend - pbegin; if(n > INT_MAX) n = INT_MAX; - char *p = Alloc(n, al); - memcpy(p, pbegin, min((dword)GetLength(), n)); + bool realloced = false; + char *p; + if((int)(limit - pbegin) > 800) { + size_t sz = sizeof(Rc) + n + 1; + Rc *rc = (Rc *)pbegin - 1; + if(MemoryTryRealloc(rc, sz)) { + realloced = true; + al = rc->alloc = (int)min((size_t)INT_MAX, sz - sizeof(Rc) - 1); + p = pbegin; + } + } + if(!realloced) { + p = Alloc(n, al); + memcpy(p, pbegin, min((dword)GetLength(), n)); + } if(cat) { if(ep + l > INT_MAX) - Panic("StringBuffer is too big!"); + Panic("StringBuffer is too big (>2GB)!"); memcpy(p + ep, cat, l); ep += l; } - Free(); - pbegin = p; + if(!realloced) { + Free(); + pbegin = p; + } pend = pbegin + ep; limit = pbegin + al; } void StringBuffer::Expand() { - Realloc(GetLength() * 2); + Realloc(GetLength() * 3 / 2); if(pend == limit) Panic("StringBuffer is too big!"); } diff --git a/uppsrc/Core/Topt.h b/uppsrc/Core/Topt.h index 9b99edbe0..697ace1b4 100644 --- a/uppsrc/Core/Topt.h +++ b/uppsrc/Core/Topt.h @@ -425,24 +425,22 @@ inline unsigned GetHashValue(const T& x) { return x.G struct CombineHash { unsigned hash; - enum { INIT = 1234567890 }; - template CombineHash& Do(const T& x) { Put(GetHashValue(x)); return *this; } public: - CombineHash& Put(unsigned h) { hash = ((hash << 4) + hash) ^ h; return *this; } + CombineHash& Put(unsigned h) { hash = (0xacf34ce7 * hash) ^ h; return *this; } operator unsigned() const { return hash; } - CombineHash() { hash = INIT; } + CombineHash() { hash = 0; } template - CombineHash(const T& h1) { hash = INIT; Do(h1); } + CombineHash(const T& h1) { hash = 0; Do(h1); } template - CombineHash(const T& h1, const U& h2) { hash = INIT; Do(h1); Do(h2); } + CombineHash(const T& h1, const U& h2) { hash = 0; Do(h1); Do(h2); } template - CombineHash(const T& h1, const U& h2, const V& h3) { hash = INIT; Do(h1); Do(h2); Do(h3); } + CombineHash(const T& h1, const U& h2, const V& h3) { hash = 0; Do(h1); Do(h2); Do(h3); } template - CombineHash(const T& h1, const U& h2, const V& h3, const W& h4) { hash = INIT; Do(h1); Do(h2); Do(h3); Do(h4); } + CombineHash(const T& h1, const U& h2, const V& h3, const W& h4) { hash = 0; Do(h1); Do(h2); Do(h3); Do(h4); } template CombineHash& operator<<(const T& x) { Do(x); return *this; } }; diff --git a/uppsrc/Core/Vcont.cpp b/uppsrc/Core/Vcont.cpp index b57185f9e..1b5070341 100644 --- a/uppsrc/Core/Vcont.cpp +++ b/uppsrc/Core/Vcont.cpp @@ -4,73 +4,6 @@ namespace Upp { -struct Vector_ { - void *vector; - int items; - int alloc; -}; - -#ifdef _DEBUG - -void *break_when_picked; - -void BreakWhenPicked(void *ptr) -{ - if(ptr == break_when_picked) - __BREAK__; -} - -void BREAK_WHEN_PICKED__(void *ptr) -{ - break_when_picked = ptr; -} - -#endif - -void VectorReAlloc_(void *v_, int newalloc, int sizeofT) -{ - Vector_ *v = (Vector_*)v_; - ASSERT(newalloc >= v->items); - ASSERT(v->items >= 0); // Pick semantics broken - size_t sz0 = (size_t)newalloc * sizeofT; - size_t sz = sz0; - void *newvector = newalloc ? MemoryAllocSz(sz) : NULL; - v->alloc = newalloc == INT_MAX ? INT_MAX // maximum alloc reached - : (int)((sz - sz0) / sizeofT + newalloc); // adjust alloc to real memory size - if(v->vector) - memcpy(newvector, v->vector, (size_t)v->items * sizeofT); - v->vector = newvector; -} - -void VectorReAllocF_(void *v_, int newalloc, int sizeofT) -{ - Vector_ *v = (Vector_*)v_; - void *prev = v->vector; - VectorReAlloc_(v, newalloc, sizeofT); - MemoryFree(prev); -} - -void VectorGrow_(void *v_, int sizeofT) -{ - Vector_ *v = (Vector_*)v_; - if(v->alloc == INT_MAX) - Panic("Too many items in container!"); -#ifdef _DEBUG - VectorReAlloc_(v, max(v->alloc + 1, v->alloc >= INT_MAX / 2 ? INT_MAX : 2 * v->alloc), sizeofT); -#else - VectorReAlloc_(v, max(v->alloc + 1, v->alloc >= int((int64)2 * INT_MAX / 3) ? INT_MAX : v->alloc + (v->alloc >> 1)), sizeofT); -#endif -} - -void VectorGrowF_(void *v_, int sizeofT) -{ - Vector_ *v = (Vector_*)v_; - void *prev = v->vector; - VectorGrow_(v, sizeofT); - MemoryFree(prev); -} - - int64 NewInVectorSerial() { static int64 x; diff --git a/uppsrc/Core/Vcont.h b/uppsrc/Core/Vcont.h index 24d2c338d..488a2427d 100644 --- a/uppsrc/Core/Vcont.h +++ b/uppsrc/Core/Vcont.h @@ -84,19 +84,21 @@ class Vector : public MoveableAndDeepCopyOption< Vector > { void Free(); void __DeepCopy(const Vector& src); - T& Get(int i) const { ASSERT(i >= 0 && i < items); return vector[i]; } - void ReAlloc(int alloc); + T& Get(int i) const { ASSERT(i >= 0 && i < items); return vector[i]; } + bool ReAlloc(int alloc); void ReAllocF(int alloc); - void Grow(); + bool GrowSz(); void GrowF(); T& GrowAdd(const T& x); - T& GrowAddPick(T&& x); + T& GrowAdd(T&& x); void RawInsert(int q, int count); + template friend class Index; + public: T& Add() { if(items >= alloc) GrowF(); return *(::new(Rdd()) T); } T& Add(const T& x) { return items < alloc ? *(new(Rdd()) T(clone(x))) : GrowAdd(x); } - T& Add(T&& x) { return items < alloc ? *(::new(Rdd()) T(pick(x))) : GrowAddPick(pick(x)); } + T& Add(T&& x) { return items < alloc ? *(::new(Rdd()) T(pick(x))) : GrowAdd(pick(x)); } template T& Create(Args&&... args) { if(items >= alloc) GrowF(); return *(::new(Rdd()) T(std::forward(args)...)); } void AddN(int n); @@ -129,6 +131,8 @@ public: void Remove(int i, int count = 1); void Remove(const int *sorted_list, int n); void Remove(const Vector& sorted_list); + template + void RemoveIf(Condition c); void InsertN(int i, int count = 1); T& Insert(int i) { InsertN(i); return Get(i); } @@ -161,11 +165,11 @@ public: Vector& operator<<(const T& x) { Add(x); return *this; } #ifdef UPP - void Serialize(Stream& s) { StreamContainer(s, *this); } + void Serialize(Stream& s) { StreamContainer(s, *this); } void Xmlize(XmlIO& xio, const char *itemtag = "item"); void Jsonize(JsonIO& jio); String ToString() const; - dword GetHashValue() const { return HashBySerialize(*this); } + dword GetHashValue() const { return HashBySerialize(*this); } template bool operator==(const B& b) const { return IsEqualRange(*this, b); } template bool operator!=(const B& b) const { return !operator==(b); } template int Compare(const B& b) const { return CompareRanges(*this, b); } @@ -273,6 +277,8 @@ public: void Remove(int i, int count = 1); void Remove(const int *sorted_list, int n); void Remove(const Vector& sorted_list); + template + void RemoveIf(Condition c); void InsertN(int i, int count = 1); T& Insert(int i) { InsertN(i); return Get(i); } void Insert(int i, const T& x, int count); diff --git a/uppsrc/Core/Vcont.hpp b/uppsrc/Core/Vcont.hpp index 287bb3af5..8030ff333 100644 --- a/uppsrc/Core/Vcont.hpp +++ b/uppsrc/Core/Vcont.hpp @@ -39,11 +39,6 @@ bool IsEqualBySerialize(const T& a, const T& b) return sa.GetResult() == sb.GetResult(); } -void VectorReAlloc_(void *vector_, int newalloc, int sizeofT); -void VectorReAllocF_(void *vector_, int newalloc, int sizeofT); -void VectorGrow_(void *vector_, int sizeofT); -void VectorGrowF_(void *vector_, int sizeofT); - template T * Vector::RawAlloc(int& n) { @@ -55,27 +50,54 @@ T * Vector::RawAlloc(int& n) } template -void Vector::ReAlloc(int newalloc) +bool Vector::ReAlloc(int newalloc) { - VectorReAlloc_(this, newalloc, sizeof(T)); + ASSERT(newalloc >= items); + size_t sz0 = (size_t)newalloc * sizeof(T); + size_t sz = sz0; + void *newvector; + bool alloced = true; + if(MemoryTryRealloc(vector, sz)) { + newvector = vector; + vector = NULL; + alloced = false; + } + else + newvector = newalloc ? MemoryAllocSz(sz) : NULL; + alloc = newalloc == INT_MAX ? INT_MAX // maximum alloc reached + : (int)((sz - sz0) / sizeof(T) + newalloc); // adjust alloc to real memory size + if(vector) + memcpy(newvector, vector, (size_t)items * sizeof(T)); + vector = (T *)newvector; + return alloced; } template void Vector::ReAllocF(int newalloc) { - VectorReAllocF_(this, newalloc, sizeof(T)); + void *prev = vector; + if(ReAlloc(newalloc)) + MemoryFree(prev); } template -void Vector::Grow() +bool Vector::GrowSz() { - VectorGrow_(this, sizeof(T)); + if(alloc == INT_MAX) + Panic("Too many items in container!"); +#ifdef _DEBUG + return ReAlloc(max(alloc + 1, alloc >= INT_MAX / 2 ? INT_MAX : 2 * alloc)); +#else + return ReAlloc(max(alloc + 1, alloc >= int((int64)2 * INT_MAX / 3) ? INT_MAX : alloc + (alloc >> 1))); +#endif } template void Vector::GrowF() { - VectorGrowF_(this, sizeof(T)); + void *prev = vector; + if(GrowSz()) + MemoryFree(prev); } template @@ -132,18 +154,18 @@ void Vector::__DeepCopy(const Vector& src) { template T& Vector::GrowAdd(const T& x) { T *prev = vector; - Grow(); + bool b = GrowSz(); T *q = new(Rdd()) T(clone(x)); - RawFree(prev); + if(b) RawFree(prev); return *q; } template -T& Vector::GrowAddPick(T&& x) { +T& Vector::GrowAdd(T&& x) { T *prev = vector; - Grow(); + bool b = GrowSz(); T *q = ::new(Rdd()) T(pick(x)); - RawFree(prev); + if(b) RawFree(prev); return *q; } @@ -189,9 +211,9 @@ void Vector::SetCount(int n, const T& init) { else { if(n > alloc) { T *prev = vector; - ReAlloc(n); + bool b = ReAlloc(n); // because init can be in original vector DeepCopyConstructFill(vector + items, vector + n, init); - RawFree(prev); + if(b) RawFree(prev); } else DeepCopyConstructFill(vector + items, vector + n, init); @@ -215,9 +237,9 @@ void Vector::SetCountR(int n, const T& init) { else if(n > alloc) { T *prev = vector; - ReAlloc(alloc + max(alloc, n - items)); + bool b = ReAlloc(alloc + max(alloc, n - items)); // because init can be in original vector DeepCopyConstructFill(vector + items, vector + n, init); - RawFree(prev); + if(b) RawFree(prev); } else DeepCopyConstructFill(vector + items, vector + n, init); @@ -263,6 +285,19 @@ void Vector::Remove(const Vector& v) Remove(v, v.GetCount()); } +template +template +void Vector::RemoveIf(Condition c) +{ + int ti = 0; + for(int i = 0; i < items; i++) + if(c(i)) + (vector + i)->~T(); + else + *((Data_S_*)vector + ti++) = *((Data_S_*)vector + i); + items = ti; +} + template void Vector::RawInsert(int q, int count) { @@ -528,6 +563,20 @@ void Array::Remove(const Vector& sorted_list) Remove(sorted_list, sorted_list.GetCount()); } +template +template +void Array::RemoveIf(Condition c) +{ + int ti = 0; + for(int i = 0; i < vector.GetCount(); i++) + if(c(i)) + delete (T *)vector[i]; + else + vector[ti++] = vector[i]; + vector.Trim(ti); +} + + template void Array::Set(int i, const T& x, int count) { ASSERT(i >= 0 && count >= 0); @@ -891,4 +940,3 @@ void Bits::SetN(int i, bool b, int count) mask = (dword)-1 >> (32 - count); bp[q] = (bp[q] & ~mask) | (bits & mask); } - diff --git a/uppsrc/Core/heap.cpp b/uppsrc/Core/heap.cpp index 61c38d9aa..2d9c03e04 100644 --- a/uppsrc/Core/heap.cpp +++ b/uppsrc/Core/heap.cpp @@ -8,6 +8,25 @@ namespace Upp { #define LLOG(x) // LOG((void *)this << ' ' << x) +const char *asString(int i) +{ + static thread_local char h[4][1024]; + static thread_local int ii; + ii = (ii + 1) & 3; + sprintf(h[ii], "%d", i); + return h[ii]; +} + +const char *asString(void *ptr) +{ + static thread_local char h[4][1024]; + static thread_local int ii; + ii = (ii + 1) & 3; + sprintf(h[ii], "%p", ptr); + return h[ii]; +} + + Heap::DLink Heap::big[1]; Heap Heap::aux; StaticMutex Heap::mutex; @@ -26,11 +45,8 @@ void Heap::Init() work[i]->klass = i; cachen[i] = 3500 / Ksz(i); } - GlobalLInit(); - for(int i = 0; i < LBINS; i++) - freebin[i]->LinkSelf(); + LInit(); large->LinkSelf(); - lcount = 0; if(this != &aux && !aux.initialized) { Mutex::Lock __(mutex); aux.Init(); @@ -38,7 +54,6 @@ void Heap::Init() initialized = true; out_ptr = out; out_size = 0; - PROFILEMT(mutex); } void Heap::FreeRemoteRaw() @@ -86,23 +101,21 @@ void Heap::Shutdown() LLOG("Orphan empty " << (void *)empty[i]); } } - while(large != large->next) { - Header *bh = (Header *)((byte *)large->next + LARGEHDRSZ); - LLOG("Orphan large block " << (void *)large->next << " size: " << bh->size); - if(bh->size == MAXBLOCK && bh->free) - MoveToEmpty(large->next, bh); - else - MoveLarge(&aux, large->next); + while(large != large->next) { // Move all large pages to aux (some heap will pick them later) + DLink *ml = large->next; + ml->Unlink(); + ml->Link(aux.large); + LBlkHeader *h = ml->GetFirst(); + for(;;) { + h->heap = &aux; + if(h->IsLast()) + break; + h = h->GetNextHeader(); + } } memset(this, 0, sizeof(Heap)); } -void Heap::Assert(bool b) -{ - if(!b) - Panic("Heap is corrupted!"); -} - void Heap::DblCheck(Page *p) { Page *l = p; @@ -157,17 +170,19 @@ void Heap::Check() { } CheckFree(cache[i], i); } + DLink *l = large->next; while(l != large) { - Header *bh = (Header *)((byte *)l + LARGEHDRSZ); - while(bh->size) { - Assert((byte *)bh >= (byte *)l + LARGEHDRSZ && (byte *)bh < (byte *)l + 65536); - if(bh->free) - DbgFreeCheck(bh->GetBlock() + 1, bh->size - sizeof(DLink)); - bh = bh->Next(); - } + lheap.BlkCheck(l->GetFirst(), 255, true); l = l->next; } + + HugePage *pg = huge_pages; + while(pg) { + BlkCheck(pg->page, HPAGE); + pg = pg->next; + } + if(this != &aux) aux.Check(); } diff --git a/uppsrc/Core/heapdbg.cpp b/uppsrc/Core/heapdbg.cpp index 5f468a21a..f406653a6 100644 --- a/uppsrc/Core/heapdbg.cpp +++ b/uppsrc/Core/heapdbg.cpp @@ -79,17 +79,11 @@ void MemoryBreakpoint(dword serial) s_allocbreakpoint = serial; } -void *MemoryAlloc_(size_t size); +void *MemoryAllocSz_(size_t& size); -void *MemoryAlloc(size_t size) +void DbgSet(DbgBlkHeader *p, size_t size) { - if(PanicMode) - return malloc(size); -#ifdef _MULTITHREADED - sHeapLock2.Enter(); -#endif static dword serial_number = 0; - DbgBlkHeader *p = (DbgBlkHeader *)MemoryAlloc_(sizeof(DbgBlkHeader) + size + sizeof(dword)); #if (defined(TESTLEAKS) || defined(HEAPDBG)) && defined(COMPILER_GCC) && defined(UPP_HEAP) p->serial = sMemDiagInitCount == 0 || s_ignoreleaks ? 0 : ~ ++serial_number ^ (dword)(uintptr_t) p; #else @@ -100,9 +94,17 @@ void *MemoryAlloc(size_t size) __BREAK__; dbg_live.Insert(p); Poke32le((byte *)(p + 1) + p->size, p->serial); -#ifdef _MULTITHREADED - sHeapLock2.Leave(); -#endif +} + +void *MemoryAllocSz(size_t& size) +{ + if(PanicMode) + return malloc(size); + Mutex::Lock __(sHeapLock2); + size += sizeof(DbgBlkHeader) + sizeof(dword); + DbgBlkHeader *p = (DbgBlkHeader *)MemoryAllocSz_(size); + size -= sizeof(DbgBlkHeader) + sizeof(dword); + DbgSet(p, size); #ifdef LOGAF char h[200]; sprintf(h, "ALLOCATED %d at %p - %p", size, p + 1, (byte *)(p + 1) + size); @@ -111,14 +113,19 @@ void *MemoryAlloc(size_t size) return p + 1; } -void *MemoryAllocSz(size_t& size) +void *MemoryAlloc(size_t size) { - size = (size + 15) & ~((size_t)15); - return MemoryAlloc(size); + return MemoryAllocSz(size); } void MemoryFree_(void *ptr); +void DbgCheck(DbgBlkHeader *p) +{ + if((dword)Peek32le((byte *)(p + 1) + p->size) != p->serial) + DbgHeapPanic("Heap is corrupted ", p); +} + void MemoryFree(void *ptr) { #ifdef LOGAF @@ -129,18 +136,32 @@ void MemoryFree(void *ptr) if(PanicMode) return; if(!ptr) return; -#ifdef _MULTITHREADED Mutex::Lock __(sHeapLock2); -#endif DbgBlkHeader *p = (DbgBlkHeader *)ptr - 1; - if((dword)Peek32le((byte *)(p + 1) + p->size) != p->serial) { - sHeapLock2.Leave(); - DbgHeapPanic("Heap is corrupted ", p); - } + DbgCheck(p); p->Unlink(); MemoryFree_(p); } +bool MemoryTryRealloc_(void *ptr, size_t& newsize); + +bool MemoryTryRealloc__(void *ptr, size_t& newsize) +{ + if(!ptr) return false; + Mutex::Lock __(sHeapLock2); + DbgBlkHeader *p = (DbgBlkHeader *)ptr - 1; + DbgCheck(p); + size_t sz = newsize; + sz += sizeof(DbgBlkHeader) + sizeof(dword); + if(MemoryTryRealloc_((DbgBlkHeader *)ptr - 1, sz)) { + newsize = sz - sizeof(DbgBlkHeader) - sizeof(dword); + p->Unlink(); + DbgSet(p, newsize); + return true; + } + return false; +} + size_t GetMemoryBlockSize_(void *ptr); size_t GetMemoryBlockSize(void *ptr) @@ -149,11 +170,6 @@ size_t GetMemoryBlockSize(void *ptr) return ((DbgBlkHeader *)ptr - 1)->size; } -bool TryRealloc(void *ptr, size_t newsize) -{ - return false; -} - void *MemoryAlloc32() { return MemoryAlloc(32); } void MemoryFree32(void *ptr) { return MemoryFree(ptr); } void *MemoryAlloc48() { return MemoryAlloc(48); } @@ -165,10 +181,8 @@ void MemoryCheckDebug() Mutex::Lock __(sHeapLock2); DbgBlkHeader *p = dbg_live.next; while(p != &dbg_live) { - if((dword)Peek32le((byte *)(p + 1) + p->size) != p->serial) { - sHeapLock2.Leave(); + if((dword)Peek32le((byte *)(p + 1) + p->size) != p->serial) DbgHeapPanic("HEAP CHECK: Heap is corrupted ", p); - } p = p->next; } while(p != &dbg_live); diff --git a/uppsrc/Core/heaputil.cpp b/uppsrc/Core/heaputil.cpp index 0805c7c5f..1f002c834 100644 --- a/uppsrc/Core/heaputil.cpp +++ b/uppsrc/Core/heaputil.cpp @@ -10,28 +10,6 @@ namespace Upp { #include "HeapImp.h" -void *MemoryAllocPermanentRaw(size_t size) -{ - if(size >= 256) - return malloc(size); - static byte *ptr = NULL; - static byte *limit = NULL; - ASSERT(size < INT_MAX); - if(ptr + size >= limit) { - ptr = (byte *)AllocRaw4KB((int)size); - limit = ptr + 4096; - } - void *p = ptr; - ptr += size; - return p; -} - -void *MemoryAllocPermanent(size_t size) -{ - Mutex::Lock __(Heap::mutex); - return MemoryAllocPermanentRaw(size); -} - void OutOfMemoryPanic(size_t size) { char h[200]; @@ -40,9 +18,18 @@ void OutOfMemoryPanic(size_t size) Panic(h); } -int sKB; +size_t Heap::huge_4KB_count; +size_t Heap::free_4KB; +size_t Heap::big_size; +size_t Heap::big_count; +size_t Heap::sys_size; +size_t Heap::sys_count; +size_t Heap::huge_chunks; -int MemoryUsedKb() { return sKB; } +int MemoryUsedKb() +{ + return int(4 * (Heap::huge_4KB_count - Heap::free_4KB)); +} int sKBLimit = INT_MAX; @@ -57,36 +44,27 @@ void *SysAllocRaw(size_t size, size_t reqsize) { size_t rsz = int(((size + 4095) & ~4095) >> 10); void *ptr = NULL; - for(int pass = 0; pass < 2; pass++) { - if(sKB + (int)rsz < sKBLimit) { - #ifdef PLATFORM_WIN32 - ptr = VirtualAlloc(NULL, size, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); - #elif PLATFORM_LINUX - ptr = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if(ptr == MAP_FAILED) - ptr = NULL; - #else - ptr = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0); - if(ptr == MAP_FAILED) - ptr = NULL; - #endif - if(ptr) - break; - } - #ifdef MEMORY_SHRINK - MemoryShrink(); // Freeing large / small empty might help + if(MemoryUsedKb() < sKBLimit) { + #ifdef PLATFORM_WIN32 + ptr = VirtualAlloc(NULL, size, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); + #elif PLATFORM_LINUX + ptr = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if(ptr == MAP_FAILED) + ptr = NULL; + #else + ptr = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0); + if(ptr == MAP_FAILED) + ptr = NULL; #endif } if(!ptr) OutOfMemoryPanic(size/*reqsize*/); - sKB += (int)rsz; DoPeakProfile(); return ptr; } void SysFreeRaw(void *ptr, size_t size) { - sKB -= int(((size + 4095) & ~4095) >> 10); #ifdef PLATFORM_WIN32 VirtualFree(ptr, 0, MEM_RELEASE); #else @@ -94,112 +72,23 @@ void SysFreeRaw(void *ptr, size_t size) #endif } -int s4kb__; -int s64kb__; -int s256kb__; - -#ifdef MEMORY_SHRINK -void *AllocRaw4KB(int reqsize) +void *MemoryAllocPermanent(size_t size) { - static int left; - static byte *ptr; - static int n = 32; - if(left == 0) { - left = n >> 5; - ptr = (byte *)SysAllocRaw(left * 4096, reqsize); + Mutex::Lock __(Heap::mutex); + if(size > 10000) + return SysAllocRaw(size, size); + static byte *ptr = NULL; + static byte *limit = NULL; + ASSERT(size < INT_MAX); + if(ptr + size >= limit) { + ptr = (byte *)SysAllocRaw(16384, 16384); + limit = ptr + 16384; } - n = n + 1; - if(n > 4096) n = 4096; void *p = ptr; - ptr += 4096; - left--; - s4kb__++; - DoPeakProfile(); + ptr += size; return p; } -void *AllocRaw64KB(int reqsize) -{ - void *ptr = (byte *)SysAllocRaw(65536, reqsize); - s64kb__++; - DoPeakProfile(); - return ptr; -} - -#if 0 -void FreeRaw4KB(void *ptr) -{ - SysFreeRaw(ptr, 4096); - s4kb__--; -} -#endif - -void FreeRaw64KB(void *ptr) -{ - SysFreeRaw(ptr, 65536); - s64kb__--; -} - -#else - -void *AllocRaw4KB(int reqsize) -{ - static int left; - static byte *ptr; - static int n = 32; - if(left == 0) { - left = n >> 5; - ptr = (byte *)SysAllocRaw(left * 4096, reqsize); - } - n = n + 1; - if(n > 4096) n = 4096; - void *p = ptr; - ptr += 4096; - left--; - s4kb__++; - DoPeakProfile(); - return p; -} - -void *AllocRaw64KB(int reqsize) -{ - static int left; - static byte *ptr; - static int n = 32; - if(left == 0) { - left = n >> 5; - ptr = (byte *)SysAllocRaw(left * 65536, reqsize); - } - n = n + 1; - if(n > 256) n = 256; - void *p = ptr; - ptr += 65536; - left--; - s64kb__++; - DoPeakProfile(); - return p; -} - -void *AllocRaw256KB(int reqsize) -{ - static int left; - static byte *ptr; - static int n = 32; - if(left == 0) { - left = n >> 5; - ptr = (byte *)SysAllocRaw(left * 256*1024, reqsize); - } - n = n + 1; - if(n > 64) n = 64; - void *p = ptr; - ptr += 256*1024; - left--; - s256kb__++; - DoPeakProfile(); - return p; -} -#endif - void HeapPanic(const char *text, void *pos, int size) { RLOG("\n\n" << text << "\n"); @@ -209,23 +98,6 @@ void HeapPanic(const char *text, void *pos, int size) #ifdef HEAPDBG -void Heap::DbgFreeFill(void *p, size_t size) -{ - size_t count = size >> 2; - dword *ptr = (dword *)p; - while(count--) - *ptr++ = 0x65657246; -} - -void Heap::DbgFreeCheck(void *p, size_t size) -{ - size_t count = size >> 2; - dword *ptr = (dword *)p; - while(count--) - if(*ptr++ != 0x65657246) - HeapPanic("Writes to freed blocks detected", p, (int)(uintptr_t)size); -} - void *Heap::DbgFreeCheckK(void *p, int k) { Page *page = GetPage(p); @@ -253,7 +125,7 @@ void Heap::Make(MemoryProfile& f) Page *p = work[i]->next; while(p != work[i]) { f.allocated[qq] += p->active; - f.fragmented[qq] += p->Count() - p->active; + f.fragments[qq] += p->Count() - p->active; p = p->next; } p = full[i]->next; @@ -271,39 +143,97 @@ void Heap::Make(MemoryProfile& f) } int ii = 0; int fi = 0; - DLink *m = big->next; - while(m != big) { - f.big_count++; - f.big_size += ((BigHdr *)m)->size; - m = m->next; - } - m = large->next; + DLink *m = large->next; while(m != large) { - Header *h = (Header *)((byte *)m + LARGEHDRSZ); - while(h->size) { - if(h->free) { - f.large_free_count++; - f.large_free_total += h->size; - if(fi < 1024) - f.large_free_size[fi++] = h->size; + LargeHeap::BlkHeader *h = m->GetFirst(); + for(;;) { + if(h->IsFree()) { + f.large_fragments_count++; + int sz = LUNIT * h->GetSize(); + f.large_fragments_total += sz; + if(h->size < 2048) + f.large_fragments[sz >> 8]++; } else { f.large_count++; - f.large_total += h->size; - if(ii < 1024) - f.large_size[ii++] = h->size; + f.large_total += LUNIT * h->size; } - h = h->Next(); + if(h->IsLast()) + break; + h = h->GetNextHeader(); } m = m->next; } - m = lempty->next; - while(m != lempty) { - f.large_empty++; - m = m->next; + + f.sys_count = (int)sys_count; + f.sys_total = sys_size; + + f.huge_count = int(big_count - sys_count); + f.huge_total = 4096 * max((int)big_size - (int)sys_size, 0); // this is not 100% correct, but approximate + + f.master_chunks = (int)huge_chunks; + + HugePage *pg = huge_pages; + while(pg) { + BlkPrefix *h = (BlkPrefix *)pg->page; + for(;;) { + if(h->IsFree()) { + word sz = h->GetSize(); + f.huge_fragments[sz]++; + f.huge_fragments_count++; + f.huge_fragments_total += sz; + } + if(h->IsLast()) + break; + h = h->GetNextHeader(4096); + } + pg = pg->next; } } +String AsString(const MemoryProfile& mem) +{ + String text; +#ifdef UPP_HEAP + int acount = 0; + size_t asize = 0; + int fcount = 0; + size_t fsize = 0; + for(int i = 0; i < 1024; i++) + if(mem.allocated[i]) { + int sz = 4 * i; + text << Format("%4d B, %7d allocated (%6d KB), %6d fragments (%6d KB)\n", + sz, mem.allocated[i], (mem.allocated[i] * sz) >> 10, + mem.fragments[i], (mem.fragments[i] * sz) >> 10); + acount += mem.allocated[i]; + asize += mem.allocated[i] * sz; + fcount += mem.fragments[i]; + fsize += mem.fragments[i] * sz; + } + text << Format(" TOTAL, %7d allocated (%6d KB), %6d fragments (%6d KB)\n", + acount, int(asize >> 10), fcount, int(fsize >> 10)); + text << "Empty 4KB pages " << mem.freepages << " (" << mem.freepages * 4 << " KB)\n"; + text << "Large block count " << mem.large_count + << ", total size " << (mem.large_total >> 10) << " KB\n"; + text << "Large fragments count " << mem.large_fragments_count + << ", total size " << (mem.large_fragments_total >> 10) << " KB\n"; + text << "Huge block count " << mem.huge_count + << ", total size " << int(mem.huge_total >> 10) << " KB\n"; + text << "Sys block count " << mem.sys_count + << ", total size " << int(mem.sys_total >> 10) << " KB\n"; + text << Heap::HPAGE * 4 / 1024 << "MB master blocks " << mem.master_chunks << "\n"; + text << "\nLarge fragments:\n"; + for(int i = 0; i < 2048; i++) + if(mem.large_fragments[i]) + text << 256.0 * i / 1024 << " KB: " << mem.large_fragments[i] << "\n"; + text << "\nHuge fragments:\n"; + for(int i = 0; i < 65535; i++) + if(mem.huge_fragments[i]) + text << i * 4 << " KB: " << mem.huge_fragments[i] << "\n"; +#endif + return text; +} + #ifdef flagHEAPSTAT int stat[65536]; int bigstat; diff --git a/uppsrc/Core/lheap.cpp b/uppsrc/Core/lheap.cpp index 0f431536a..e04967bd2 100644 --- a/uppsrc/Core/lheap.cpp +++ b/uppsrc/Core/lheap.cpp @@ -1,4 +1,7 @@ #include "Core.h" +#include "Core.h" + +#define LTIMING(x) // RTIMING(x) namespace Upp { @@ -8,285 +11,200 @@ namespace Upp { #include "HeapImp.h" -word Heap::BinSz[LBINS]; -byte Heap::SzBin[MAXBLOCK / 8 + 1]; // Minimal bin for size (request -> bin) -byte Heap::BlBin[MAXBLOCK / 8 + 1]; // Largest bin less or equal to size (free -> bin) - -Heap::DLink Heap::lempty[1]; - -const char *asString(int i) +void Heap::LInit() { - static thread_local char h[4][1024]; - static thread_local int ii; - ii = (ii + 1) & 3; - sprintf(h[ii], "%d", i); - return h[ii]; + for(int i = 0; i <= __countof(lheap.freelist); i++) + Dbl_Self(lheap.freelist[i]); + big->LinkSelf(); } -void Heap::GlobalLInit() +void *Heap::TryLAlloc(int i0, word wcount) { - ONCELOCK { - int p = 64; - int bi = 0; - while(p < MAXBLOCK) { - BinSz[bi++] = p - 16; - int add = minmax(8 * p / 100 / 64 * 64, 64, INT_MAX); - p += add; + for(int i = i0; i < __countof(lheap.freelist); i++) { + LBlkHeader *l = lheap.freelist[i]; + LBlkHeader *h = l->next; + while(h != l) { + word sz = h->GetSize(); + if(sz >= wcount) { + lheap.MakeAlloc(h, wcount); + h->heap = this; + return (BlkPrefix *)h + 1; + } + h = h->next; } - ASSERT(bi == LBINS - 1); - BinSz[LBINS - 1] = MAXBLOCK; - int k = 0; - for(int i = 0; i < MAXBLOCK / 8; i++) { - while(i * 8 + 7 > BinSz[k]) - k++; - SzBin[i] = k; - } - k = LBINS - 1; - for(int i = MAXBLOCK / 8; i >= 0; i--) { - while(k >= 0 && i * 8 < BinSz[k]) k--; - BlBin[i] = k; - } - BlBin[0] = 0; - big->LinkSelf(); - lempty->LinkSelf(); - } -} - -inline -void Heap::LinkFree(DLink *b, int size) -{ - int q = BlBin[size >> 3]; - b->Link(freebin[q]); - LLOG("Linked " << asString(size) << " to freebin " << asString(q)); -} - -Heap::DLink *Heap::AddChunk(int reqsize) -{ // gets a free chunk, returns pointer to heap block - DLink *ml; - if(lempty->next != lempty) { - ml = lempty->next; - ml->Unlink(); - LLOG("Retrieved empty large " << (void *)ml); - } - else { - ml = (DLink *)AllocRaw64KB(reqsize); - LLOG("AllocRaw64KB " << (void *)ml); - } - lcount++; - LLOG("lcount = " << lcount); - if(!ml) return NULL; - ml->Link(large); - Header *bh = (Header *)((byte *)ml + LARGEHDRSZ); - bh->size = MAXBLOCK; - bh->prev = 0; - bh->free = true; - bh->heap = this; - DLink *b = bh->GetBlock(); - LinkFree(b, MAXBLOCK); - DbgFreeFill(b + 1, MAXBLOCK - sizeof(DLink)); - bh = bh->Next(); - bh->prev = MAXBLOCK; - bh->size = 0; - bh->free = false; - bh->heap = this; - return b; -} - -inline -void *Heap::DivideBlock(DLink *b, int size) -{ // unlink from free and truncate block, move truncated part back to free blocks - b->Unlink(); - Header *bh = b->GetHeader(); - ASSERT(bh->size >= (dword)size && size > 0); - bh->free = false; - int sz2 = bh->size - size - sizeof(Header); - if(sz2 >= 48) { - Header *bh2 = (Header *)((byte *)b + size); - bh2->prev = size; - bh2->free = true; - bh2->heap = this; - LinkFree(bh2->GetBlock(), sz2); - bh->Next()->prev = bh2->size = sz2; - bh->size = size; - } - DbgFreeCheck(b + 1, size - sizeof(DLink)); - return b; -} - -void Heap::MoveLarge(Heap *dest, DLink *l) -{ // move large block page (64KB) current heap (aux) to dest - dest->lcount++; - LLOG("Moving large " << (void *)l << " to " << (void *)dest << " lcount " << dest->lcount); - Mutex::Lock __(mutex); - l->Unlink(); - l->Link(dest->large); - Header *h = (Header *)((byte *)l + LARGEHDRSZ); - while(h->size) { - h->heap = dest; - if(h->free) { - DLink *b = h->GetBlock(); - b->Unlink(); - dest->LinkFree(b, h->size); - } - h = h->Next(); - } - aux.lcount = 10000; -} - -void Heap::MoveToEmpty(DLink *l, Header *bh) -{ - LLOG("Moving empty large " << (void *)l << " to global empty storage, lcount " << lcount); - bh->GetBlock()->Unlink(); - l->Unlink(); - Mutex::Lock __(mutex); - l->Link(lempty); - aux.lcount = 10000; -} - -inline -void *Heap::TryLAlloc(int ii, size_t size) -{ - LLOG("TryLAlloc bin: " << asString(ii) << " size: " << asString(size)); - while(ii < LBINS) { - if(freebin[ii] != freebin[ii]->next) { - void *ptr = DivideBlock(freebin[ii]->next, (int)size); - LLOG("TryLAlloc succeeded " << (void *)ptr); - ASSERT((size_t)ptr & 16); - return ptr; - } - ii++; } return NULL; } +#if 0 +int stat[65536]; + +EXITBLOCK { + int cnt = 0; + for(int i = 0; i < 65536; i++) { + cnt += stat[i]; + if(stat[i]) + RLOG(i * 256 << ": " << stat[i] << " / " << cnt); + } +} +#endif + void *Heap::LAlloc(size_t& size) -{ // allocate large or big block - LLOG("+++ LAlloc " << size); - ASSERT(size > 256); +{ if(!initialized) Init(); - if(size > MAXBLOCK) { // big block allocation + + if(size > LUNIT * LPAGE - sizeof(BlkPrefix)) { // big block allocation + LTIMING("Big alloc"); Mutex::Lock __(mutex); - BigHdr *h = (BigHdr *)SysAllocRaw(size + BIGHDRSZ, size); - h->Link(big); - h->size = size = ((size + BIGHDRSZ + 4095) & ~4095) - BIGHDRSZ; - Header *b = (Header *)((byte *)h + BIGHDRSZ - sizeof(Header)); - b->size = 0; // header contains large header with size = 0, to detect big during free - b->free = false; - LLOG("Big alloc " << size << ": " << (void *)b->GetBlock()); - return b->GetBlock(); + size_t count = (size + sizeof(DLink) + sizeof(BlkPrefix) + 4095) >> 12; + DLink *d = (DLink *)HugeAlloc(count); + d->Link(big); + d->size = size = (count << 12) - sizeof(DLink) - sizeof(BlkPrefix); + BlkPrefix *h = (BlkPrefix *)(d + 1); + h->heap = NULL; // mark this as huge block + big_size += size; + big_count++; + LLOG("Big alloc " << size << ": " << h + 1); + return h + 1; } - int bini = SizeToBin((int)size); // get the bin - size = BinSz[bini]; // get the real bin size - LLOG("Binned size " << asString(size)); - void *ptr = TryLAlloc(bini, size); // try current working blocks first + + RTIMING("Large Alloc"); + + word wcount = word((size + sizeof(BlkPrefix) + LUNIT - 1) >> 8); + +#if 0 + stat[wcount]++; +#endif + + size = ((int)wcount * LUNIT) - sizeof(BlkPrefix); + int i0 = lheap.Cv(wcount); + + if(large_remote_list) // there might be blocks of this heap freed in other threads + LargeFreeRemote(); // free them first + + void *ptr = TryLAlloc(i0, wcount); if(ptr) return ptr; - if(large_remote_list) { // there might be blocks freed in other threads - LargeFreeRemote(); // free them - ptr = TryLAlloc(bini, size); // try again - if(ptr) return ptr; - } + + RTIMING("Large Alloc 2"); Mutex::Lock __(mutex); aux.LargeFreeRemoteRaw(); - while(aux.large->next != aux.large) { - LLOG("Adopting large block " << (void *)aux.large->next); - MoveLarge(this, aux.large->next); - lcount++; - ptr = TryLAlloc(bini, size); - if(ptr) return ptr; + if(aux.large->next != aux.large) { + while(aux.large->next != aux.large) { // adopt all abandoned large blocks + DLink *ml = aux.large->next; + ml->Unlink(); + ml->Link(large); + } + ptr = TryLAlloc(i0, wcount); + if(ptr) + return ptr; } - DLink *n = AddChunk((int)size); - if(!n) - Panic("Out of memory!"); - ptr = DivideBlock(n, (int)size); - LLOG("LAlloc via AddChunk " << (void *)ptr); - ASSERT((size_t)ptr & 16); - return ptr; + + LTIMING("Large More"); + DLink *ml = (DLink *)HugeAlloc(((LPAGE + 1) * LUNIT) / 4096); + ml->Link(large); + LBlkHeader *h = ml->GetFirst(); + lheap.AddChunk(h, LPAGE); + lheap.MakeAlloc(h, wcount); + h->heap = this; + return (BlkPrefix *)h + 1; +} + +void Heap::FreeLargePage(DLink *l) +{ + LLOG("Moving empty large " << (void *)l << " to global storage, lcount " << lcount); + l->Unlink(); + Mutex::Lock __(mutex); + HugeFree(l); } void Heap::LFree(void *ptr) -{ // free large or big block - DLink *b = (DLink *)ptr; - Header *bh = b->GetHeader(); - if(bh->size == 0) { - Mutex::Lock __(mutex); - ASSERT(((dword)(uintptr_t)bh & 4095) == BIGHDRSZ - sizeof(Header)); - BigHdr *h = (BigHdr *)((byte *)ptr - BIGHDRSZ); - h->Unlink(); - LLOG("Big free " << (void *) ptr << " size " << h->size); - SysFreeRaw(h, h->size); - return; - } - if(bh->heap != this) { - LLOG("Remote large, heap " << (void *)bh->heap); - Mutex::Lock __(mutex); // TODO: Replace with SpinLock - FreeLink *f = (FreeLink *)ptr; - f->next = bh->heap->large_remote_list; - bh->heap->large_remote_list = f; - return; - } - LLOG("--- LFree " << asString(bh->size)); - if(bh->prev) { // there is previous block - Header *p = bh->Prev(); - if(p->free) { // previous block is free, join - b = p->GetBlock(); - b->Unlink(); // remove previous block from free list - p->size += bh->size + sizeof(Header); - p->Next()->prev = p->size; - bh = p; +{ + BlkPrefix *h = (BlkPrefix *)ptr - 1; + + if(h->heap == this) { + LTIMING("Large Free"); + LBlkHeader *fh = lheap.Free((LBlkHeader *)h); + if(fh->GetSize() == LPAGE) { + LTIMING("FreeLargePage"); + fh->UnlinkFree(); + FreeLargePage((DLink *)((byte *)fh - LOFFSET)); } + return; } - Header *n = bh->Next(); - if(n->free) { // next block block is free, join - n->GetBlock()->Unlink(); // remove next block from free list - bh->size += n->size + sizeof(Header); - n->Next()->prev = bh->size; - } - bh->free = true; - LinkFree(b, bh->size); - DbgFreeFill(b + 1, bh->size - sizeof(DLink)); - LLOG("Freed, joined size " << asString(bh->size) << " lcount " << asString(lcount)); - if(bh->size == MAXBLOCK && lcount > 1) { - DLink *l = (DLink *)((byte *)bh - LARGEHDRSZ); - lcount--; - MoveToEmpty(l, bh); + + Mutex::Lock __(mutex); + if(h->heap == NULL) { // this is big block + LTIMING("Big Free"); + DLink *d = (DLink *)h - 1; + big_size -= h->size; + big_count--; + d->Unlink(); + LLOG("Big free " << (void *) ptr << " size " << h->size); + HugeFree(d); + return; } + + LTIMING("Remote Free"); + // this is remote heap + FreeLink *f = (FreeLink *)ptr; + f->next = h->heap->large_remote_list; + h->heap->large_remote_list = f; } -bool Heap::LTryRealloc(void *ptr, size_t newsize) +bool Heap::TryRealloc(void *ptr, size_t& newsize) { - DLink *b = (DLink *)ptr; - Header *bh = b->GetHeader(); - if(bh->size == 0) { - Mutex::Lock __(mutex); - ASSERT(((dword)(uintptr_t)bh & 4095) == BIGHDRSZ - sizeof(Header)); - BigHdr *h = (BigHdr *)((byte *)ptr - BIGHDRSZ); - return newsize <= h->size; + ASSERT(ptr); + +#ifdef _DEBUG + if(IsSmall(ptr)) + return false; +#endif + + BlkPrefix *h = (BlkPrefix *)ptr - 1; + + if(h->heap == this) { + if(newsize > LUNIT * LPAGE - sizeof(BlkPrefix)) + return false; + word wcount = word(((newsize ? newsize : 1) + sizeof(BlkPrefix) + LUNIT - 1) >> 8); + size_t dummy; + if(wcount == h->GetSize() || lheap.TryRealloc(h, wcount, dummy)) { + newsize = ((int)wcount * LUNIT) - sizeof(BlkPrefix); + return true; + } } - if(bh->heap != this) // if another thread's heap, do not bother to be smart - return newsize <= bh->size; - if(bh->size >= newsize) - return true; - LLOG("--- TryRealloc " << asString(bh->size)); - Header *n = bh->Next(); - if(n->free && newsize <= (size_t)n->size + (size_t)bh->size) { - DivideBlock(n->GetBlock(), int(newsize - n->size)); - bh->size += n->size + sizeof(Header); - n->Next()->prev = bh->size; - return true; + + Mutex::Lock __(mutex); + if(h->heap == NULL) { // this is big block + LTIMING("Big Free"); + + DLink *d = (DLink *)h - 1; + BlkPrefix *h = (BlkPrefix *)(d + 1); + + size_t count = (newsize + sizeof(DLink) + sizeof(BlkPrefix) + 4095) >> 12; + + if(HugeTryRealloc(d, count)) { + big_size -= h->size; + d->size = newsize = (count << 12) - sizeof(DLink) - sizeof(BlkPrefix); + big_size += h->size; + return true; + } } + return false; } size_t Heap::LGetBlockSize(void *ptr) { - DLink *b = (DLink *)ptr; - Header *bh = b->GetHeader(); - if(bh->size == 0) { - BigHdr *h = (BigHdr *)((byte *)ptr - BIGHDRSZ); + LBlkHeader *h = (LBlkHeader *)ptr - 1; + + if(h->heap == NULL) { // huge block + Mutex::Lock __(mutex); + DLink *h = (DLink *)ptr - 1; return h->size; } - return bh->size; + + return h->GetSize(); } #endif diff --git a/uppsrc/Core/sheap.cpp b/uppsrc/Core/sheap.cpp index 6e2118a26..bcc7e558e 100644 --- a/uppsrc/Core/sheap.cpp +++ b/uppsrc/Core/sheap.cpp @@ -60,6 +60,7 @@ Heap::Page *Heap::WorkPage(int k) // get a new workpage with empty blocks if(!page && aux.empty[k]) { // Try hot empty page of the same klass page = aux.empty[k]; aux.empty[k] = page->next; + free_4KB--; LLOG("AllocK - empty aux page available of the same format " << k << " page: " << (void *)page << ", free " << (void *)page->freelist); } if(!page) @@ -67,12 +68,13 @@ Heap::Page *Heap::WorkPage(int k) // get a new workpage with empty blocks if(aux.empty[i]) { page = aux.empty[i]; aux.empty[i] = page->next; + free_4KB--; page->Format(k); LLOG("AllocK - empty aux page available for reformatting " << k << " page: " << (void *)page << ", free " << (void *)page->freelist); break; } - if(!page) { // No free memory was found, ask system for the new page - page = (Page *)AllocRaw4KB(Ksz(k)); + if(!page) { // No free memory was found, ask huge for the new page + page = (Page *)HugeAlloc(1); LLOG("AllocK - allocated new system page " << (void *)page << " " << k); page->Format(k); } @@ -185,6 +187,7 @@ void Heap::FreeK(void *ptr, Page *page, int k) empty[k]->heap = &aux; empty[k]->next = aux.empty[k]; aux.empty[k] = empty[k]; + free_4KB++; } empty[k] = page; } @@ -196,12 +199,10 @@ void Heap::Free(void *ptr, Page *page, int k) { LLOG("Small free page: " << (void *)page << ", k: " << k << ", ksz: " << Ksz(k)); ASSERT((4096 - ((uintptr_t)ptr & (uintptr_t)4095)) % Ksz(k) == 0); -#ifdef _MULTITHREADED if(page->heap != this) { // freeing page allocated in different thread RemoteFree(ptr, Ksz(k)); // add to originating heap's list of free pages to be properly freed later return; } -#endif DbgFreeFillK(ptr, k); if(cachen[k]) { cachen[k]--; @@ -238,17 +239,6 @@ size_t Heap::GetBlockSize(void *ptr) return LGetBlockSize(ptr); } -bool Heap::TryRealloc(void *ptr, size_t newsize) -{ - if(!ptr) return 0; - LLOG("GetBlockSize " << ptr); - if(IsSmall(ptr)) { - Page *page = GetPage(ptr); - int k = page->klass; - return newsize <= (size_t)Ksz(k); - } - return LTryRealloc(ptr, newsize); -} void Heap::SmallFreeDirect(void *ptr) { // does not need to check for target heap or small vs large @@ -262,6 +252,25 @@ void Heap::SmallFreeDirect(void *ptr) FreeK(ptr, page, k); } +bool Heap::FreeSmallEmpty(int size4KB, int count) +{ // attempt to release small 4KB pages to gain count4KB space or count of releases + bool released; + do { + released = false; + for(int i = 0; i < NKLASS; i++) + if(aux.empty[i]) { + Page *q = aux.empty[i]; + aux.empty[i] = q->next; + free_4KB--; + if(aux.HugeFree(q) >= size4KB || --count <= 0) // HugeFree is really static, aux needed just to compile + return true; + released = true; + } + } + while(released); + return false; +} + force_inline void *Heap::Alloc32() { @@ -294,14 +303,56 @@ void Heap::Free48(void *ptr) Free(ptr, KLASS_48); } +#if defined(COMPILER_GCC) && defined(PLATFORM_WIN32) // Workaround for MINGW inneficient storage +struct TEB_ { + PVOID Reserved1[12]; + PVOID ProcessEnvironmentBlock; + PVOID Reserved2[399]; + BYTE Reserved3[1952]; + PVOID TlsSlots[64]; + BYTE Reserved4[8]; + PVOID Reserved5[26]; + PVOID ReservedForOle; + PVOID Reserved6[4]; + PVOID TlsExpansionSlots; +}; + +dword TlsPointerNdx = TlsAlloc(); + +void SetTlsPointer(void *ptr) +{ + TlsSetValue(TlsPointerNdx, ptr); +} + +force_inline +void *GetTlsPointer() +{ +#ifdef CPU_64 + TEB_ *teb = (TEB_ *)__readgsqword(0x30); +#else + TEB_ *teb = (TEB_ *)__readfsdword(0x18); +#endif + return teb->TlsSlots[TlsPointerNdx]; +} +#endif + force_inline Heap *ThreadHeap() { #if defined(COMPILER_GCC) && defined(PLATFORM_WIN32) // Workaround for MINGW bug thread_local byte sHeap[sizeof(Heap)]; +#if 1 + Heap *heap = (Heap *)GetTlsPointer(); + if(!heap) { + heap = (Heap *)sHeap; + SetTlsPointer(heap); + ASSERT(GetTlsPointer() == heap); + } +#else thread_local Heap *heap; if(!heap) heap = (Heap *)sHeap; +#endif return heap; #else thread_local Heap heap[1]; @@ -321,7 +372,7 @@ void *MemoryAllok__(int klass) #if defined(HEAPDBG) -void *MemoryAlloc_(size_t sz) +void *MemoryAllocSz_(size_t& sz) { return ThreadHeap()->AllocSz(sz); } @@ -331,6 +382,11 @@ void MemoryFree_(void *ptr) ThreadHeap()->Free(ptr); } +bool MemoryTryRealloc_(void *ptr, size_t& size) +{ + return ThreadHeap()->TryRealloc(ptr, size); +} + size_t GetMemoryBlockSize_(void *ptr) { return ThreadHeap()->GetBlockSize(ptr); @@ -363,7 +419,7 @@ size_t GetMemoryBlockSize(void *ptr) return ThreadHeap()->GetBlockSize(ptr); } -bool TryRealloc(void *ptr, size_t size) +bool MemoryTryRealloc__(void *ptr, size_t& size) { return ThreadHeap()->TryRealloc(ptr, size); } diff --git a/uppsrc/Core/src.tpp/Array_en-us.tpp b/uppsrc/Core/src.tpp/Array_en-us.tpp index 9bcb88d89..2a10d0e89 100644 --- a/uppsrc/Core/src.tpp/Array_en-us.tpp +++ b/uppsrc/Core/src.tpp/Array_en-us.tpp @@ -321,6 +321,14 @@ sorted`_list.GetCount())].&] [s7; [*C@3 sorted`_list]-|Sorted Vector of positions to remove.&] [s3;%- &] [s4;%- &] +[s5;:Upp`:`:Array`:`:RemoveIf`(Condition`):%- [@(0.0.255) template]_<[@(0.0.255) class]_[*@4 C +ondition]>_[@(0.0.255) void]_[* RemoveIf]([*@4 Condition]_[*@3 c])&] +[s2; Removes elements where condition [%-*@3 c] is satisfied. Condition +is a callable (usually lambda) that accepts (int i) as parameter +and returns either true or false.&] +[s6; Invalidates iterators to the Array.&] +[s3; &] +[s4;%- &] [s5;:Array`:`:InsertN`(int`,int`):%- [@(0.0.255) void]_[* InsertN]([@(0.0.255) int]_[*@3 i], [@(0.0.255) int]_[*@3 count]_`=_[@3 1])&] [s2; Inserts a specified number of default constructed elements at diff --git a/uppsrc/Core/src.tpp/Huge_en-us.tpp b/uppsrc/Core/src.tpp/Huge_en-us.tpp index 8884a7168..fd0bfd295 100644 --- a/uppsrc/Core/src.tpp/Huge_en-us.tpp +++ b/uppsrc/Core/src.tpp/Huge_en-us.tpp @@ -1,5 +1,4 @@ topic "Huge"; -[2 $$0,0#00000000000000000000000000000000:Default] [i448;a25;kKO9;2 $$1,0#37138531426314131252341829483380:class] [l288;2 $$2,2#27521748481378242620020725143825:desc] [0 $$3,0#96390100711032703541132217272105:end] @@ -9,16 +8,17 @@ topic "Huge"; [l288;i1121;b17;O9;~~~.1408;2 $$7,0#10431211400427159095818037425705:param] [i448;b42;O9;2 $$8,8#61672508125594000341940100500538:tparam] [b42;2 $$9,9#13035079074754324216151401829390:normal] +[2 $$0,0#00000000000000000000000000000000:Default] [{_}%EN-US [ {{10000@(113.42.0) [s0; [*@7;4 Huge]]}}&] [s3;%- &] [s1;:Upp`:`:Huge`:`:class:%- [@(0.0.255)3 class][3 _][*3 Huge]&] -[s2; This class is intended for dealing of raw binary data bigger -than 2GB. The main use is supposed to be in serialization loading, -where it is used to preload large binary data (e.g. String or -Image) as means for protection against invalid input `- Huge -is storing data in 1MB chunks, so is likely to run to the end -of stream without allocating invalid amounts of data.&] +[s2; This class is intended for dealing of large raw binary data. +The main use is supposed to be in serialization loading, where +it is used to preload large binary data (e.g. String or Image) +as means for protection against invalid input `- Huge is storing +data in 1MB chunks, so is likely to run to the end of stream +without allocating invalid amounts of data.&] [s2; To explain further, the problem we are trying to solve here is&] [s2; &] diff --git a/uppsrc/Core/src.tpp/algo_en-us.tpp b/uppsrc/Core/src.tpp/algo_en-us.tpp index 1421f2bd3..b27568433 100644 --- a/uppsrc/Core/src.tpp/algo_en-us.tpp +++ b/uppsrc/Core/src.tpp/algo_en-us.tpp @@ -190,7 +190,18 @@ nt]>_[* FindAll]([@(0.0.255) const]_[*@4 Range][@(0.0.255) `&]_[*@3 r], [*@4 Predicate]_[*@3 match], [@(0.0.255) int]_[*@3 from]_`=_[@3 0])&] [s2;%% Returns the Vector of indices of [/ ALL] elements for which [%-*@3 match] is true. Returned Vector is sorted in ascending order. -Search starts at index [%-*@3 from].&] +Search starts at index [%-*@3 from]. The [%-*@3 match] parameter +is an element.&] +[s3;%% &] +[s4; &] +[s5;:Upp`:`:FindAlli`(const Range`&`,Predicate`,int`): [@(0.0.255) template]_<[@(0.0.255) c +lass]_[*@4 Range], [@(0.0.255) class]_[*@4 Predicate]>_[_^Upp`:`:Vector^ Vector]<[@(0.0.255) i +nt]>_[* FindAlli]([@(0.0.255) const]_[*@4 Range][@(0.0.255) `&]_[*@3 r], +[*@4 Predicate]_[*@3 match], [@(0.0.255) int]_[*@3 from]_`=_[@3 0])&] +[s2;%% Returns the Vector of indices of [/ ALL] elements for which +[%-*@3 match] is true. Returned Vector is sorted in ascending order. +Search starts at index [%-*@3 from]. Unlike FindAll, the parameter +to [%-*@3 match] is index of element.&] [s3;%% &] [s4; &] [s5;:Upp`:`:FindLowerBound`(const Range`&`,const T`&`,const Less`&`): [@(0.0.255) templ diff --git a/uppsrc/CppBase/Base.cpp b/uppsrc/CppBase/Base.cpp index 851589200..6eeb09595 100644 --- a/uppsrc/CppBase/Base.cpp +++ b/uppsrc/CppBase/Base.cpp @@ -68,41 +68,21 @@ void CppBase::Dump(Stream& s) } } -void CppBase::Sweep(const Index& keep_file) +void CppBase::Sweep(const Index& file, bool keep) { - int ni = 0; - while(ni < GetCount()) { - Array& n = (*this)[ni]; - Vector nr; - for(int i = 0; i < n.GetCount(); i++) { - if(keep_file.Find(n[i].file) < 0) - nr.Add(i); - } - if(nr.GetCount() == n.GetCount()) - Remove(ni); - else { - n.Remove(nr); - ni++; - } - } -} - -void CppBase::RemoveFiles(const Index& remove_file) -{ - int ni = 0; - while(ni < GetCount()) { + Vector remove; + for(int ni = 0; ni < GetCount(); ni++) { Array& n = (*this)[ni]; Vector nr; for(int i = 0; i < n.GetCount(); i++) - if(remove_file.Find(n[i].file) >= 0) + if((file.Find(n[i].file) < 0) == keep) nr.Add(i); if(nr.GetCount() == n.GetCount()) - Remove(ni); - else { - n.Remove(nr); - ni++; - } + remove.Add(ni); // remove whole array (later) + else + n.Remove(nr); // only remove some items } + Remove(remove); } void CppBase::RemoveFile(int filei) diff --git a/uppsrc/CppBase/CppBase.h b/uppsrc/CppBase/CppBase.h index ffc6f312b..53dddbd83 100644 --- a/uppsrc/CppBase/CppBase.h +++ b/uppsrc/CppBase/CppBase.h @@ -422,8 +422,8 @@ struct CppBase : ArrayMap > { Index namespaces; bool IsType(int i) const; - void Sweep(const Index& keep_file); - void RemoveFiles(const Index& remove_file); + void Sweep(const Index& file, bool keep = true); + void RemoveFiles(const Index& remove_file) { Sweep(remove_file, false); } void RemoveFile(int filei); void Dump(Stream& s); diff --git a/uppsrc/TextDiffCtrl/TextDiff.cpp b/uppsrc/TextDiffCtrl/TextDiff.cpp index b440bc13c..54daf6547 100644 --- a/uppsrc/TextDiffCtrl/TextDiff.cpp +++ b/uppsrc/TextDiffCtrl/TextDiff.cpp @@ -63,8 +63,8 @@ private: void Split(Array& dest, int start1, int end1, int start2, int end2) const; private: - Vector hash1; - Vector hash2; + Vector> hash1; + Vector> hash2; const Vector& file1; const Vector& file2; }; @@ -74,10 +74,10 @@ Array CompareLineMaps(const Vector& s1, const Vector& hash, const Vector& file, int limit) +static void CalcHash(Vector>& hash, const Vector& file, int limit) { { // 1st row - HashBase& first = hash.Add(); + Index& first = hash.Add(); for(int i = 0; i < file.GetCount(); i++) first.Add(GetHashValue(file[i])); } @@ -90,8 +90,8 @@ static void CalcHash(Vector& hash, const Vector& file, int lim }; const int *pp = prime; for(int l = 1; l < limit; l <<= 1) { - HashBase& nhash = hash.Add(); - const HashBase& ohash = hash[hash.GetCount() - 2]; + Index& nhash = hash.Add(); + const Index& ohash = hash[hash.GetCount() - 2]; int pri = *pp++; int t; for(t = l; t < ohash.GetCount(); t++) @@ -143,9 +143,9 @@ bool TextComparator::Find(int start1, int end1, int start2, int end2, int& best_ int lvl = GetHashLevel(best_count + 1, hash1.GetCount()); int chunk = 1 << lvl; int last = max(best_count - chunk + 1, 0); - const HashBase *hp1 = &hash1[lvl]; - const HashBase *hp2 = &hash2[lvl]; - const unsigned *h1 = hp1->begin() + start1; + const Index *hp1 = &hash1[lvl]; + const Index *hp2 = &hash2[lvl]; + const dword *h1 = hp1->begin() + start1; int i = hp2->Find(*h1); while(i >= 0) diff --git a/uppsrc/ide/idefile.cpp b/uppsrc/ide/idefile.cpp index eb8ebcf47..b666ea25a 100644 --- a/uppsrc/ide/idefile.cpp +++ b/uppsrc/ide/idefile.cpp @@ -789,6 +789,8 @@ void Ide::CheckFileUpdate() ReloadFile(); } +typedef Index HashBase; + static void GetLineIndex(String file, HashBase& hash, Vector& lines) { const char *p = file; diff --git a/uppsrc/plugin/dbf/dbf.cpp b/uppsrc/plugin/dbf/dbf.cpp index 3a12d23b7..a4a3b7002 100644 --- a/uppsrc/plugin/dbf/dbf.cpp +++ b/uppsrc/plugin/dbf/dbf.cpp @@ -709,7 +709,6 @@ Vector DbfStream::FetchRow(int row) void DbfStream::WriteRow(int row, const Vector& values) { - RTIMING("DbfStream::WriteRow"); ASSERT(row >= 0); FlushRow(); if(values.GetCount() < fields.GetCount())