From f4964d53d0eee30868520a697b05f3f6525ca28a Mon Sep 17 00:00:00 2001 From: cxl Date: Fri, 7 Jun 2019 07:09:32 +0000 Subject: [PATCH] Core2019 merged to trunk (missing files) git-svn-id: svn://ultimatepp.org/upp/trunk@13360 f0d560ea-af0d-0410-9eb7-867de7ffcac7 --- uppsrc/Core/Index.cpp | 204 ++++++++++++++++++++++ uppsrc/Core/Index.hpp | 390 ++++++++++++++++++++++++++++++++++++++++++ uppsrc/Core/hheap.cpp | 118 +++++++++++++ 3 files changed, 712 insertions(+) create mode 100644 uppsrc/Core/Index.cpp create mode 100644 uppsrc/Core/Index.hpp create mode 100644 uppsrc/Core/hheap.cpp diff --git a/uppsrc/Core/Index.cpp b/uppsrc/Core/Index.cpp new file mode 100644 index 000000000..4b339bb05 --- /dev/null +++ b/uppsrc/Core/Index.cpp @@ -0,0 +1,204 @@ +#include + +namespace Upp { + +int IndexCommon::empty[1] = { -1 }; + +IndexCommon::IndexCommon() +{ + hash = NULL; + map = empty; + mask = 0; + unlinked = -1; +} + +void IndexCommon::Pick(IndexCommon& b) +{ + Free(); + + hash = b.hash; + map = b.map; + mask = b.mask; + unlinked = b.unlinked; + + b.hash = NULL; + b.map = empty; + b.mask = 0; + b.unlinked = -1; +} + +void IndexCommon::Copy(const IndexCommon& b, int count) +{ + memcpy(hash, b.hash, sizeof(Hash) * count); + mask = b.mask; + unlinked = b.unlinked; + + FreeMap(); + map = (int *)MemoryAlloc((mask + 1) * sizeof(int)); + memcpy(map, b.map, (mask + 1) * sizeof(int)); +} + +void IndexCommon::Swap(IndexCommon& b) +{ + UPP::Swap(hash, b.hash); + UPP::Swap(map, b.map); + UPP::Swap(mask, b.mask); + UPP::Swap(unlinked, b.unlinked); +} + +IndexCommon::~IndexCommon() +{ + Free(); +} + +void IndexCommon::FreeMap() +{ + if(map != empty) + MemoryFree(map); +} + +void IndexCommon::Free() +{ + if(hash) + MemoryFree(hash); + FreeMap(); +} + +void IndexCommon::Remap(int count) +{ + Fill(map, map + mask + 1, -1); + for(int i = 0; i < count; i++) // todo: unlinked + if(hash[i].hash) + Link(i, hash[i].hash); +} + +void IndexCommon::Reindex(int count) +{ + FreeMap(); + map = (int *)MemoryAlloc((mask + 1) * sizeof(int)); + Remap(count); +} + +void IndexCommon::Clear() +{ + Free(); + hash = NULL; + map = empty; + mask = 0; + unlinked = -1; +} + +void IndexCommon::GrowMap(int count) +{ + mask = (mask << 1) | 3; + Reindex(count); +} + +Vector IndexCommon::GetUnlinked() const +{ + Vector r; + int i = unlinked; + if(i >= 0) { + do { + i = hash[i].prev; + r.Add(i); + } + while(i != unlinked); + } + return r; +} + +void IndexCommon::AdjustMap(int count, int alloc) +{ + if(alloc == 0) { + FreeMap(); + map = empty; + mask = 0; + return; + } + dword msk = 0; + while(msk < (dword)alloc) + msk = (msk << 1) | 3; + if(msk != mask) { + mask = msk; + Reindex(count); + } +} + +void IndexCommon::MakeMap(int count) +{ + mask = 0; + AdjustMap(count, count); +} + +void IndexCommon::Trim(int n, int count) +{ + if(n == 0) { + int n = (int)(mask + 1); + for(int i = 0; i < n; i++) + map[i] = -1; + unlinked = -1; + return; + } + + for(int i = n; i < count; i++) { // remove items in trimmed area from buckets / unlinked + Hash& hh = hash[i]; + if(hh.hash) + Del(map[hh.hash & mask], hh, i); + else + Del(unlinked, hh, i); + } +} + +void IndexCommon::Sweep(int n) +{ + int ti = 0; + for(int i = 0; i < n; i++) + if(hash[i].hash) + hash[ti++].hash = hash[i].hash; + Remap(ti); + unlinked = -1; +} + +#ifdef CPU_UNALIGNED + +NOUBSAN // CPU supports unaligned memory access +unsigned memhash(const void *ptr, size_t count) +{ + unsigned hash = 1234567890U; + + const unsigned *ds = (unsigned *)ptr; + const unsigned *de = ds + (count >> 2); + while(ds < de) + hash = ((hash << 5) - hash) ^ *ds++; + + const byte *s = (byte *)ds; + const byte *e = s + (count & 3); + while(s < e) + hash = ((hash << 5) - hash) ^ *s++; + + return hash; +} + +#else + +unsigned memhash(const void *ptr, size_t count) +{ + unsigned hash = 1234567890U; + + const byte *s = (byte *)ptr; + const byte *e = s + count; + while(s < e) + hash = ((hash << 5) - hash) ^ *s++; + + return hash; +} + +#endif + +unsigned GetHashValue0(const double& d) +{ + return memhash(&d, sizeof(double)); +} + +} \ No newline at end of file diff --git a/uppsrc/Core/Index.hpp b/uppsrc/Core/Index.hpp new file mode 100644 index 000000000..c9bfe75fa --- /dev/null +++ b/uppsrc/Core/Index.hpp @@ -0,0 +1,390 @@ +force_inline +void IndexCommon::Link(int& m, Hash& hh, int ii) +{ + if(m < 0) + m = hh.prev = hh.next = ii; + else { + hh.next = m; + hh.prev = hash[m].prev; + hash[hh.prev].next = ii; + hash[m].prev = ii; + } +} + +force_inline +void IndexCommon::Link(int ii, dword sh) +{ + Link(map[sh & mask], hash[ii], ii); +} + +force_inline +void IndexCommon::Del(int& m, Hash& hh, int ii) +{ // unlink from m + if(ii == m) { // this is item pointed by map + if(hh.next == ii) { // this is the only item in the bucket + m = -1; // bucket is now empty + return; + } + m = hh.next; // move bucket pointer to the next item + } + hash[hh.next].prev = hh.prev; // unlink + hash[hh.prev].next = hh.next; +} + +template +never_inline +void Index::ReallocHash(int n) +{ // realloc hash to have the same capacity as key, copy n elements from previous alloc + if(key.GetAlloc()) { + size_t sz = key.GetAlloc() * sizeof(Hash); + if(!MemoryTryRealloc(hash, sz)) { + Hash *h = (Hash *)MemoryAlloc(sz); + if(hash) { + if(n) + memcpy(h, hash, sizeof(Hash) * n); + MemoryFree(hash); + } + hash = h; + } + } + else { + MemoryFree(hash); + hash = NULL; + } +} + +template +never_inline +void Index::FixHash(bool makemap) +{ + ReallocHash(0); + unlinked = -1; + for(int i = 0; i < key.GetCount(); i++) + hash[i].hash = Smear(key[i]); + if(makemap) + MakeMap(key.GetCount()); + else + Remap(key.GetCount()); +} + +template +template +never_inline +void Index::GrowAdd(U&& k, dword sh) +{ + int n = key.GetCount(); + key.GrowAdd(std::forward(k)); + ReallocHash(n); +} + +template +template +void Index::AddS(int& m, U&& k, dword sh) +{ + int ii = key.GetCount(); + if(ii >= key.GetAlloc()) + GrowAdd(std::forward(k), sh); + else + new(key.Rdd()) T(std::forward(k)); + Hash& hh = hash[ii]; + hh.hash = sh; + if(ii >= (int)mask) + GrowMap(key.GetCount()); + else + Link(m, hh, ii); +} + +template +template +void Index::AddS(U&& k, dword sh) +{ + AddS(map[sh & mask], std::forward(k), sh); +} + +template +force_inline +int Index::FindFrom(int i, dword sh, const T& k, int end) const +{ + if(i >= 0) + do { + if(key[i] == k) + return i; + i = hash[i].next; + } + while(i != end); + return -1; +} + +template +int Index::Find(const T& k) const +{ + dword sh = Smear(k); + int& m = map[sh & mask]; + return FindFrom(m, sh, k, m); +} + +template +int Index::FindNext(int i) const +{ + const Hash& hh = hash[i]; + int end = map[hash[i].hash & mask]; + return hh.next == end ? -1 : FindFrom(hh.next, hh.hash, key[i], end); +} + +template +force_inline +int Index::FindBack(int i, dword sh, const T& k, int end) const +{ + do { + const Hash& ih = hash[i]; + if(key[i] == k) + return i; + i = ih.prev; + } + while(i != end); + return -1; +} + +template +int Index::FindLast(const T& k) const +{ + dword sh = Smear(k); + int& m = map[sh & mask]; + return m < 0 ? -1 : FindBack(hash[m].prev, sh, k, hash[m].prev); +} + +template +int Index::FindPrev(int i) const +{ + const Hash& hh = hash[i]; + int end = map[hash[i].hash & mask]; + return hh.prev == hash[end].prev ? -1 : FindBack(hh.prev, hh.hash, key[i], hash[end].prev); +} + +template +template +force_inline +int Index::FindAdd(U&& k, OP op) { + dword sh = Smear(k); + int& m = map[sh & mask]; + int i = m; + if(i >= 0) + do { + if(key[i] == k) + return i; + i = hash[i].next; + } + while(i != m); + i = key.GetCount(); + AddS(m, std::forward(k), sh); + op(); + return i; +} + +template +void Index::Unlink(int ii) +{ + Hash& hh = hash[ii]; + Del(map[hh.hash & mask], hh, ii); + Link(unlinked, hh, ii); + hh.hash = 0; +} + +template +int Index::UnlinkKey(const T& k) +{ + dword sh = Smear(k); + int& m = map[sh & mask]; + int i = m; + int n = 0; + if(i >= 0) + for(;;) { + Hash& hh = hash[i]; + int ni = hh.next; + if(key[i] == k) { + Del(m, hh, i); + Link(unlinked, hh, i); + n++; + hh.hash = 0; + if(ni == i) // last item removed + break; + i = ni; + } + else { + i = ni; + if(i == m) + break; + } + } + return n; +} + +template +template +int Index::Put0(U&& k, dword sh) +{ + int i; + if(HasUnlinked()) { + i = hash[unlinked].prev; + Hash& hh = hash[i]; + Del(unlinked, hh, i); + Link(map[sh & mask], hh, i); + hh.hash = sh; + key[i] = std::forward(k); + } + else { + i = GetCount(); + AddS(std::forward(k), sh); + } + return i; +} + +template +template +force_inline +int Index::FindPut0(U&& k) { + dword sh = Smear(k); + int& m = map[sh & mask]; + int i = m; + if(i >= 0) + do { + if(key[i] == k) + return i; + i = hash[i].next; + } + while(i != m); + return Put0(std::forward(k), sh); +} + +template +template +void Index::Set0(int ii, U&& k) +{ + Hash& hh = hash[ii]; + if(IsUnlinked(ii)) + Del(unlinked, hh, ii); + else + Del(map[hh.hash & mask], hh, ii); + + dword sh = Smear(k); + hh.hash = sh; + Link(map[sh & mask], hh, ii); + key[ii] = std::forward(k); +} + +template +never_inline +void Index::Sweep() +{ + if(unlinked >= 0) { + int n = key.GetCount(); + key.RemoveIf([&](int i) { return hash[i].hash == 0; }); + IndexCommon::Sweep(n); + } +} + +template +never_inline +void Index::Reserve(int n) +{ + int a = key.GetAlloc(); + key.Reserve(n); + if(a != key.GetAlloc()) { + ReallocHash(key.GetCount()); + AdjustMap(key.GetCount(), n); + } +} + +template +never_inline +void Index::Shrink() +{ + int a = key.GetAlloc(); + key.Shrink(); + if(a != key.GetAlloc()) { + ReallocHash(key.GetCount()); + AdjustMap(key.GetCount(), key.GetCount()); + } +} + +template +void Index::Remove(const int *sorted_list, int count) +{ + if(HasUnlinked()) { + Vector u; + u.SetCount(GetCount()); + for(int i = 0; i < GetCount(); i++) + u[i] = IsUnlinked(i); + key.Remove(sorted_list, count); + u.Remove(sorted_list, count); + FixHash(false); + for(int i = 0; i < GetCount(); i++) + if(u[i]) + Unlink(i); + } + else { + key.Remove(sorted_list, count); + FixHash(false); + } +} + +template +never_inline +void Index::Serialize(Stream& s) +{ + key.Serialize(s); + if(s.IsLoading()) + FixHash(); + + int version = 1; + s / version; + if(version == 0) { // support previous version + Vector h; + h.Serialize(s); + if(s.IsLoading()) + for(int i = 0; i < h.GetCount(); i++) + if(h[i] & 0x80000000) + Unlink(i); + } + else { + Vector u = GetUnlinked(); + u.Serialize(s); + if(s.IsLoading()) + for(int i : ReverseRange(u)) // Reverse range to ensure the correct order of Put + Unlink(i); + } +} + +template +void Index::Xmlize(XmlIO& xio, const char *itemtag) +{ + XmlizeIndex >(xio, itemtag, *this); +} + +template +void Index::Jsonize(JsonIO& jio) +{ + JsonizeIndex, T>(jio, *this); +} + +template +String Index::ToString() const +{ + return AsStringArray(*this); +} + +#ifdef _DEBUG +template +String Index::Dump() const +{ + String h; + for(int i = 0; i < key.GetCount(); i++) { + if(i) + h << "; "; + if(IsUnlinked(i)) + h << "#"; + h << i << ": " << key[i] << '/' << (hash[i].hash & mask) << " -> " << hash[i].prev << ":" << hash[i].next; + } + return h; +} +#endif \ No newline at end of file diff --git a/uppsrc/Core/hheap.cpp b/uppsrc/Core/hheap.cpp new file mode 100644 index 000000000..daf93d68d --- /dev/null +++ b/uppsrc/Core/hheap.cpp @@ -0,0 +1,118 @@ +#include + +#define LTIMING(x) // RTIMING(x) +// #define LSTAT + +namespace Upp { + +#ifdef UPP_HEAP + +#include "HeapImp.h" + +// this part reserves very large (224MB for 64bit CPUs, 32MB otherwise) +// chunks form the system and then serves as 4KB rounded allocator +// used as manager of huge memory blocks. 4KB and 64KB blocks are allocated from here too +// also able to deal with bigger blocks, those are directly allocated / freed from system + +BlkHeader_<4096> HugeHeapDetail::freelist[2][1]; // only single global Huge heap... +Heap::HugePage *Heap::huge_pages; + +#ifdef LSTAT +static int hstat[65536]; + +EXITBLOCK { + int cnt = 0; + for(int i = 0; i < 65536; i++) { + cnt += hstat[i]; + if(hstat[i]) + RLOG(i * 4 << " KB: " << hstat[i] << " / " << cnt); + } +} +#endif + +void *Heap::HugeAlloc(size_t count) // count in 4kb pages +{ + ASSERT(count); + +#ifdef LSTAT + if(count < 65536) + hstat[count]++; +#endif + + huge_4KB_count += count; + + if(!D::freelist[0]->next) { // initialization + for(int i = 0; i < 2; i++) + Dbl_Self(D::freelist[i]); + } + + if(count > HPAGE) { // we are wasting 4KB to store just 4 bytes here, but this is >32MB after all.. + LTIMING("SysAlloc"); + byte *sysblk = (byte *)SysAllocRaw((count + 1) * 4096, 0); + BlkHeader *h = (BlkHeader *)(sysblk + 4096); + h->size = 0; + *((size_t *)sysblk) = count; + sys_count++; + sys_size += 4096 * count; + return h; + } + + LTIMING("Huge Alloc"); + + word wcount = (word)count; + + if(16 * free_4KB > huge_4KB_count) // keep number of free 4KB blocks in check + FreeSmallEmpty(INT_MAX, int(free_4KB - huge_4KB_count / 32)); + + for(int pass = 0; pass < 2; pass++) { + for(int i = count >= 16; i < 2; i++) { + BlkHeader *l = D::freelist[i]; + BlkHeader *h = l->next; + while(h != l) { + word sz = h->GetSize(); + if(sz >= count) + return MakeAlloc(h, wcount); + h = h->next; + } + } + + if(!FreeSmallEmpty(wcount, INT_MAX)) { // try to coalesce 4KB small free blocks back to huge storage + void *ptr = SysAllocRaw(HPAGE * 4096, 0); + HugePage *pg = (HugePage *)MemoryAllocPermanent(sizeof(HugePage)); + pg->page = ptr; + pg->next = huge_pages; + huge_pages = pg; + AddChunk((BlkHeader *)ptr, HPAGE); // failed, add 32MB from the system + huge_chunks++; + } + } + Panic("Out of memory"); + return NULL; +} + +int Heap::HugeFree(void *ptr) +{ + BlkHeader *h = (BlkHeader *)ptr; + if(h->size == 0) { + LTIMING("Sys Free"); + byte *sysblk = (byte *)h - 4096; + size_t count = *((size_t *)sysblk); + SysFreeRaw(sysblk, count); + huge_4KB_count -= count; + sys_count--; + sys_size -= 4096 * count; + return 0; + } + LTIMING("Huge Free"); + huge_4KB_count -= h->GetSize(); + return BlkHeap::Free(h)->GetSize(); +} + +bool Heap::HugeTryRealloc(void *ptr, size_t count) +{ + return count <= HPAGE && BlkHeap::TryRealloc(ptr, count, huge_4KB_count); +} + +#endif + +} \ No newline at end of file