From dae458db060cb31f34de3e11508175a8eb4115df Mon Sep 17 00:00:00 2001 From: cxl Date: Wed, 27 May 2020 16:39:08 +0000 Subject: [PATCH] Core: Mem.h improvements git-svn-id: svn://ultimatepp.org/upp/trunk@14505 f0d560ea-af0d-0410-9eb7-867de7ffcac7 --- uppsrc/Core/AString.hpp | 2 +- uppsrc/Core/BlockStream.cpp | 14 +- uppsrc/Core/Core.upp | 3 +- uppsrc/Core/Cpu.cpp | 165 -------------- uppsrc/Core/FilterStream.cpp | 8 +- uppsrc/Core/Format.cpp | 2 +- uppsrc/Core/Huge.cpp | 2 +- uppsrc/Core/Mem.cpp | 97 ++++++++ uppsrc/Core/Mem.h | 405 +++++++++++++++++---------------- uppsrc/Core/Stream.cpp | 12 +- uppsrc/Core/Stream.h | 4 +- uppsrc/Core/String.cpp | 28 +-- uppsrc/Core/Topt.h | 12 +- uppsrc/Core/Util.h | 11 +- uppsrc/Core/WString.cpp | 2 +- uppsrc/CtrlCore/ImageWin32.cpp | 2 +- uppsrc/Draw/FontWin32.cpp | 8 +- uppsrc/Draw/Image.h | 2 +- uppsrc/Painter/BufferPainter.h | 2 +- uppsrc/RichText/ParaData.cpp | 4 +- uppsrc/RichText/ParseQtf.cpp | 2 +- 21 files changed, 366 insertions(+), 421 deletions(-) create mode 100644 uppsrc/Core/Mem.cpp diff --git a/uppsrc/Core/AString.hpp b/uppsrc/Core/AString.hpp index f756050e2..23c511c06 100644 --- a/uppsrc/Core/AString.hpp +++ b/uppsrc/Core/AString.hpp @@ -290,7 +290,7 @@ void String0::Set(const char *s, int len) { Clear(); if(len < 14) { - svo_memcpy(chr, s, len); + memcpy8(chr, s, len); SLen() = len; Dsyn(); return; diff --git a/uppsrc/Core/BlockStream.cpp b/uppsrc/Core/BlockStream.cpp index 00f6352ba..025dce011 100644 --- a/uppsrc/Core/BlockStream.cpp +++ b/uppsrc/Core/BlockStream.cpp @@ -155,12 +155,12 @@ void BlockStream::_Put(const void *data, dword size) { wrlim = buffer + pagesize; pagedirty = true; if(pg0 == pg1) { - svo_memcpy(buffer + pos0 - pos, data, size); + memcpy8(buffer + pos0 - pos, data, size); ptr = buffer + pos1 - pos; } else { int n = int(pos + pagesize - pos0); - svo_memcpy(buffer + pos0 - pos, s, n); + memcpy8(buffer + pos0 - pos, s, n); s += n; n = dword(pg1 - pg0) - pagesize; streamsize = max(pos + pagesize + n, streamsize); @@ -173,7 +173,7 @@ void BlockStream::_Put(const void *data, dword size) { if(pos1 > pg1) { wrlim = buffer + pagesize; pagedirty = true; - svo_memcpy(buffer, s, int(pos1 - pg1)); + memcpy8(buffer, s, int(pos1 - pg1)); } } } @@ -191,19 +191,19 @@ dword BlockStream::_Get(void *data, dword size) { int64 pg1 = pos1 & pagemask; if(pg0 == pg1) { SyncPage(); - svo_memcpy(data, buffer + pos0 - pos, size); + memcpy8(data, buffer + pos0 - pos, size); ptr = buffer + pos1 - pos; _Term(); } else { int last = int(pos1 - pg1); if(pagepos == pg1) { - svo_memcpy(t + size - last, buffer, last); + memcpy8(t + size - last, buffer, last); last = 0; } SyncPage(); int n = int(pos + pagesize - pos0); - svo_memcpy(t, buffer + pos0 - pos, n); + memcpy8(t, buffer + pos0 - pos, n); dword q = dword(pg1 - pg0) - pagesize; if(q && Read(pos + pagesize, t + n, q) != q) { SetError(); @@ -212,7 +212,7 @@ dword BlockStream::_Get(void *data, dword size) { SetPos(pos0 + size); if(last) { SyncPage(); - svo_memcpy(t + size - last, buffer, last); + memcpy8(t + size - last, buffer, last); } } return size; diff --git a/uppsrc/Core/Core.upp b/uppsrc/Core/Core.upp index a91b53960..0a434dca4 100644 --- a/uppsrc/Core/Core.upp +++ b/uppsrc/Core/Core.upp @@ -30,8 +30,9 @@ file config.h, Defs.h, Ops.h, - Mem.h, Cpu.cpp, + Mem.h, + Mem.cpp, Atomic.h, Mt.h, Mt.cpp, diff --git a/uppsrc/Core/Cpu.cpp b/uppsrc/Core/Cpu.cpp index 63df239fd..912765495 100644 --- a/uppsrc/Core/Cpu.cpp +++ b/uppsrc/Core/Cpu.cpp @@ -154,169 +154,4 @@ void EndianSwap(int *v, size_t count) ENDIAN_SWAP void EndianSwap(int64 *v, size_t count) ENDIAN_SWAP void EndianSwap(uint64 *v, size_t count) ENDIAN_SWAP -#ifdef CPU_X86 -void huge_memsetd(void *p, dword c, size_t len) -{ // bypasses the cache, good for >4MB - dword *t = (dword *)p; - if(((uintptr_t)t & 3) == 0 && len > 64) { - __m128i val4 = _mm_set1_epi32(c); - auto Set4S = [&](int at) { _mm_stream_si128((__m128i *)(t + at), val4); }; - while((uintptr_t)t & 15) { // align to 16 bytes for SSE - *t++ = c; - len--; - } - while(len >= 16) { - Set4S(0); Set4S(4); Set4S(8); Set4S(12); - t += 16; - len -= 16; - } - _mm_sfence(); - } - while(len--) - *t++ = c; -} - -void memsetd_l(dword *t, dword data, size_t len) -{ - __m128i val4 = _mm_set1_epi32(data); - auto Set4 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), val4); }; - Set4(len - 4); // fill tail - if(len >= 32) { - if(len >= 1024*1024) { // for really huge data, bypass the cache - huge_memsetd(t, data, len); - return; - } - Set4(0); // align up on 16 bytes boundary - const dword *e = t + len; - t = (dword *)(((uintptr_t)t | 15) + 1); - len = e - t; - e -= 32; - while(t <= e) { - Set4(0); Set4(4); Set4(8); Set4(12); - Set4(16); Set4(20); Set4(24); Set4(28); - t += 32; - } - } - if(len & 16) { - Set4(0); Set4(4); Set4(8); Set4(12); - t += 16; - } - if(len & 8) { - Set4(0); Set4(4); - t += 8; - } - if(len & 4) - Set4(0); -} - -void memcpyd_l(dword *t, const dword *s, size_t len) -{ - auto Copy4 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); }; - - Copy4(0); // align target data up on next 16 bytes boundary - const dword *e = t + len; - dword *t1 = (dword *)(((uintptr_t)t | 15) + 1); - s += t1 - t; - t = t1; - len = e - t; - e -= 16; - if(len >= 1024*1024) { // for really huge data, call memcpy to bypass the cache - memcpy(t, s, 4 * len); - return; - } - while(t <= e) { - Copy4(0); Copy4(4); Copy4(8); Copy4(12); - t += 16; - s += 16; - } - if(len & 8) { - Copy4(0); Copy4(4); - t += 8; - s += 8; - } - if(len & 4) - Copy4(0); -} - -void memcpyq_l(qword *t, const qword *s, size_t len) -{ - auto Copy4 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); }; - - if(len >= 512*1024) { // for really huge data, call memcpy to bypass the cache - memcpy(t, s, 8 * len); - return; - } - Copy4(0); // align target data up on next 16 bytes boundary - const qword *e = t + len; - qword *t1 = (qword *)(((uintptr_t)t | 15) + 1); - s += t1 - t; - t = t1; - len = e - t; - e -= 8; - while(t <= e) { - Copy4(0); Copy4(2); Copy4(4); Copy4(6); - t += 8; - s += 8; - } - if(len & 4) { - Copy4(0); Copy4(2); - t += 4; - s += 4; - } - if(len & 2) - Copy4(0); -} - -void memcpydq_l(dqword *t, const dqword *s, size_t len) -{ - auto Copy4 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); }; - - if(len >= 256*1024) { // for really huge data, call memcpy to bypass the cache - memcpy(t, s, 16 * len); - return; - } - Copy4(0); // align target data up on next 16 bytes boundary - const dqword *e = t + len; - dqword *t1 = (dqword *)(((uintptr_t)t | 15) + 1); - s += t1 - t; - t = t1; - len = e - t; - e -= 4; - while(t <= e) { - Copy4(0); Copy4(1); Copy4(2); Copy4(3); - t += 4; - s += 4; - } - if(len & 2) { - Copy4(0); Copy4(1); - t += 2; - s += 2; - } - if(len & 1) - Copy4(0); -} -#endif - -#ifdef CPU_UNALIGNED -never_inline -void svo_memset_l(byte *t, dword val4, size_t len) -{ - const byte *e = t + len; - t = (byte *)(((uintptr_t)t | 3) + 1); - len = e - t; - memsetd(t, val4, len >> 2); -} - -never_inline -void svo_memcpy_l(byte *t, byte *s, size_t len) -{ - const byte *e = t + len; - byte *t2 = (byte *)(((uintptr_t)t | 3) + 1); - s += t2 - t; - t = t2; - len = e - t; - memcpyd((dword *)t, (dword *)s, len >> 2); -} -#endif - } diff --git a/uppsrc/Core/FilterStream.cpp b/uppsrc/Core/FilterStream.cpp index ccfaf31ac..cc1929efb 100644 --- a/uppsrc/Core/FilterStream.cpp +++ b/uppsrc/Core/FilterStream.cpp @@ -43,7 +43,7 @@ dword InFilterStream::_Get(void *data, dword size) { t = (byte *)data; dword sz0 = min(dword(rdlim - ptr), size); - svo_memcpy(t, ptr, sz0); + memcpy8(t, ptr, sz0); t += sz0; ptr += sz0; todo = size - sz0; @@ -57,7 +57,7 @@ void InFilterStream::Out(const void *p, int size) const byte *s = (byte *)p; if(todo) { dword sz = min(todo, (dword)size); - svo_memcpy(t, s, sz); + memcpy8(t, s, sz); t += sz; s += sz; todo -= sz; @@ -67,7 +67,7 @@ void InFilterStream::Out(const void *p, int size) if(size) { int l = buffer.GetCount(); buffer.SetCountR(l + size); - svo_memcpy(buffer.begin() + l, s, size); + memcpy8(buffer.begin() + l, s, size); } WhenOut(); } @@ -153,7 +153,7 @@ void OutFilterStream::_Put(const void *data, dword size) const byte *p = (const byte *)data; for(;;) { int n = min(Avail(), size); - svo_memcpy(ptr, p, n); + memcpy8(ptr, p, n); size -= n; p += n; ptr += n; diff --git a/uppsrc/Core/Format.cpp b/uppsrc/Core/Format.cpp index d3e4d3e75..59f3f7fc8 100644 --- a/uppsrc/Core/Format.cpp +++ b/uppsrc/Core/Format.cpp @@ -63,7 +63,7 @@ String FormatIntBase(int i, int base, int width, char lpad, int sign, bool upper char *o = out; if(dwd < width) memset(o, lpad, pad); - svo_memcpy(o + pad, p, dwd); + memcpy8(o + pad, p, dwd); return out; } diff --git a/uppsrc/Core/Huge.cpp b/uppsrc/Core/Huge.cpp index d780b5bcd..f69f26a16 100644 --- a/uppsrc/Core/Huge.cpp +++ b/uppsrc/Core/Huge.cpp @@ -27,7 +27,7 @@ void Huge::Get(void *t_, size_t pos, size_t sz) const while(sz > 0) { size_t m = min(sz, CHUNK - blkpos); - svo_memcpy(t, data[blki].data + blkpos, m); + memcpy8(t, data[blki].data + blkpos, m); t += m; sz -= m; blkpos = 0; diff --git a/uppsrc/Core/Mem.cpp b/uppsrc/Core/Mem.cpp new file mode 100644 index 000000000..ca617a9e9 --- /dev/null +++ b/uppsrc/Core/Mem.cpp @@ -0,0 +1,97 @@ +#include "Core.h" + +namespace Upp { + +#ifdef CPU_X86 +void huge_memsetd(void *p, dword c, size_t len) +{ // bypasses the cache, good for >4MB + dword *t = (dword *)p; +} + +String AsString(__m128i x) +{ + dword h[4]; + memcpy(h, &x, 16); + word w[8]; + memcpy(w, &x, 16); + return Sprintf("_%08x_%08x_%08x_%08x (%d, %d, %d, %d) : (%d, %d, %d, %d)", + h[3], h[2], h[1], h[0], w[6], w[5], w[4], w[7], w[2], w[1], w[0], w[3]); +} + +void memset8__(void *p, __m128i data, size_t len) +{ + ASSERT(len >= 16); + byte *t = (byte *)p; + auto Set4 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), data); }; + Set4(len - 16); // fill tail + Set4(0); // align up on the next 16 bytes boundary + const byte *e = t + len; + t = (byte *)(((uintptr_t)t | 15) + 1); + len = e - t; + e -= 128; + if(len >= 1024*1024) { // for really huge data, bypass the cache + auto Set4S = [&](int at) { _mm_stream_si128((__m128i *)(t + at), data); }; + while(len >= 64) { + Set4S(0*16); Set4S(1*16); Set4S(2*16); Set4S(3*16); + t += 64; + len -= 64; + } + _mm_sfence(); + e = t - 1; + } + while(t <= e) { + Set4(0*16); Set4(1*16); Set4(2*16); Set4(3*16); + Set4(4*16); Set4(5*16); Set4(6*16); Set4(7*16); + t += 128; + } + if(len & 64) { + Set4(0*16); Set4(1*16); Set4(2*16); Set4(3*16); + t += 64; + } + if(len & 32) { + Set4(0*16); Set4(1*16); + t += 32; + } + if(len & 16) + Set4(0); +} + +never_inline +void memcpy8__(void *p, const void *q, size_t len) +{ + ASSERT(len >= 16); + + byte *t = (byte *)p; + const byte *s = (const byte *)q; + + if(len > 4*1024*1024) { // for really huge data, call memcpy to bypass the cache + memcpy(t, s, len); + return; + } + + auto Copy128 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); }; + Copy128(len - 16); // copy tail + Copy128(0); // align target data up on the next 16 bytes boundary + const byte *e = t + len; + byte *t1 = (byte *)(((uintptr_t)t | 15) + 1); + s += t1 - t; + t = t1; + len = e - t; + e -= 64; + while(t <= e) { + Copy128(0*16); Copy128(1*16); Copy128(2*16), Copy128(3*16); + t += 64; + s += 64; + } + if(len & 32) { + Copy128(0*16); Copy128(1*16); + t += 32; + s += 32; + } + if(len & 16) + Copy128(0*16); +} + +#endif + +}; \ No newline at end of file diff --git a/uppsrc/Core/Mem.h b/uppsrc/Core/Mem.h index b2bd15229..3662937e6 100644 --- a/uppsrc/Core/Mem.h +++ b/uppsrc/Core/Mem.h @@ -1,13 +1,59 @@ -struct dqword { - qword a, b; -}; - #ifdef CPU_X86 -void huge_memsetd(void *p, dword data, size_t len); -void memsetd_l(dword *t, dword data, size_t len); +void memset8__(void *t, __m128i data, size_t len); inline -void memsetd(void *p, dword data, size_t len) +void memset8(void *p, byte data, size_t len) +{ + byte *t = (byte *)p; + if(len < 2) { + if(len) + t[0] = data; + return; + } + dword val4 = 0x1010101 * data; + if(len <= 4) { + *(word *)t = *(word *)(t + len - 2) = (word)val4; + return; + } + if(len > 16) { + memset8__(t, _mm_set1_epi32(val4), len); + return; + } + *(dword *)t = *(dword *)(t + len - 4) = val4; + if(len > 8) + *(dword *)(t + 4) = *(dword *)(t + len - 8) = val4; +} + +inline +void memset16(void *p, word data, size_t len) +{ + word *t = (word *)p; + if(len < 2) { + if(len) + t[0] = data; + return; + } + dword val4 = 0x10001 * data; + if(len >= 16) { + memset8__(t, _mm_set1_epi32(val4), 2 * len); + return; + } + *(dword *)(t + len - 2) = val4; + if(len & 8) { + _mm_storeu_si128((__m128i *)t, _mm_set1_epi32(val4)); + t += 8; + } + if(len & 4) { + *(dword *)t = val4; + *(dword *)(t + 2) = val4; + t += 4; + } + if(len & 2) + *(dword *)t = val4; +} + +inline +void memset32(void *p, dword data, size_t len) { dword *t = (dword *)p; if(len < 4) { @@ -19,13 +65,11 @@ void memsetd(void *p, dword data, size_t len) t[0] = data; return; } - + __m128i val4 = _mm_set1_epi32(data); if(len >= 16) { - memsetd_l(t, data, len); + memset8__(t, val4, 4 * len); return; } - - __m128i val4 = _mm_set1_epi32(data); auto Set4 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), val4); }; Set4(len - 4); // fill tail if(len & 8) { @@ -36,11 +80,71 @@ void memsetd(void *p, dword data, size_t len) Set4(0); } -void memcpyd_l(dword *t, const dword *s, size_t len); +void memcpy8__(void *p, const void *q, size_t len); inline -void memcpyd(dword *t, const dword *s, size_t len) +void memcpy8(void *p, const void *q, size_t len) { + byte *t = (byte *)p; + byte *s = (byte *)q; + if(len <= 4) { + if(len < 2) { + if(len) + t[0] = s[0]; + return; + } + *(word *)t = *(word *)s; + *(word *)(t + len - 2) = *(word *)(s + len - 2); + return; + } + if(len <= 16) { + if(len <= 8) { + *(dword *)(t) = *(dword *)(s); + *(dword *)(t + len - 4) = *(dword *)(s + len - 4); + return; + } + *(uint64 *)t = *(uint64 *)s; + *(uint64 *)(t + len - 8) = *(uint64 *)(s + len - 8); + return; + } + memcpy8__(t, s, len); +} + +inline +void memcpy16(void *p, const void *q, size_t len) +{ + word *t = (word *)p; + word *s = (word *)q; + if(len <= 4) { + if(len < 2) { + if(len) + t[0] = s[0]; + return; + } + *(dword *)t = *(dword *)s; + *(dword *)(t + len - 2) = *(dword *)(s + len - 2); + return; + } + if(len <= 16) { + if(len <= 8) { + *(uint64 *)(t) = *(uint64 *)(s); + *(uint64 *)(t + len - 4) = *(uint64 *)(s + len - 4); + return; + } + auto Copy128 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); }; + Copy128(0); + Copy128(len - 8); + return; + } + memcpy8__(t, s, 2 * len); +} + +inline +void memcpy32(void *p, const void *q, size_t len) +{ + dword *t = (dword *)p; + dword *s = (dword *)q; + #ifdef CPU_64 if(len <= 4) { if(len) { @@ -68,27 +172,28 @@ void memcpyd(dword *t, const dword *s, size_t len) } #endif - auto Copy4 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); }; + auto Copy128 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); }; - Copy4(len - 4); // copy tail if(len >= 16) { - memcpyd_l(t, s, len); + memcpy8__(t, s, 4 * len); return; } + Copy128(len - 4); // copy tail if(len & 8) { - Copy4(0); Copy4(4); + Copy128(0); Copy128(4); t += 8; s += 8; } if(len & 4) - Copy4(0); + Copy128(0); } -void memcpyq_l(qword *t, const qword *s, size_t len); - inline -void memcpyq(qword *t, const qword *s, size_t len) +void memcpy64(void *p, const void *q, size_t len) { + qword *t = (qword *)p; + qword *s = (qword *)q; + if(len <= 2) { if(len) { if(len > 1) { @@ -105,7 +210,7 @@ void memcpyq(qword *t, const qword *s, size_t len) Copy4(len - 2); // copy tail if(len >= 8) { - memcpyq_l(t, s, len); + memcpy8__(t, s, 8 * len); return; } if(len & 4) { @@ -117,17 +222,20 @@ void memcpyq(qword *t, const qword *s, size_t len) Copy4(0); } -static_assert(sizeof(dqword) == 16, "dqword sizeof"); - -void memcpydq_l(dqword *t, const dqword *s, size_t len); - inline -void memcpydq(dqword *t, const dqword *s, size_t len) +void memcpy128(void *p, const void *q, size_t len) { + struct dqword { qword x[2]; }; + + static_assert(sizeof(dqword) == 16, "dqword sizeof"); + + dqword *t = (dqword *)p; + dqword *s = (dqword *)q; + auto Copy4 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); }; if(len >= 8) { - memcpydq_l(t, s, len); + memcpy8__(t, s, 16 * len); return; } if(len & 4) { @@ -144,199 +252,112 @@ void memcpydq(dqword *t, const dqword *s, size_t len) Copy4(0); } -#else -inline -void memsetd(void *p, dword c, size_t len) +template +void memcpy_t(void *t, const T *s, size_t count) { - dword *t = (dword *)p; +#ifdef CPU_X86 + if((sizeof(T) & 15) == 0) + memcpy128(t, s, count * (sizeof(T) >> 4)); + else + if((sizeof(T) & 7) == 0) + memcpy64(t, s, count * (sizeof(T) >> 3)); + else +#endif + if((sizeof(T) & 3) == 0) + memcpy32(t, s, count * (sizeof(T) >> 2)); + else + if((sizeof(T) & 1) == 0) + memcpy16(t, s, count * (sizeof(T) >> 1)); + else + memcpy8(t, s, count * sizeof(T)); +} + +#else + +template +void memset__(void *p, T data, size_t len) +{ + T *t = (T *)p; while(len >= 16) { - t[0] = c; t[1] = c; t[2] = c; t[3] = c; - t[4] = c; t[5] = c; t[6] = c; t[7] = c; - t[8] = c; t[9] = c; t[10] = c; t[11] = c; - t[12] = c; t[13] = c; t[14] = c; t[15] = c; + t[0] = data; t[1] = data; t[2] = data; t[3] = data; + t[4] = data; t[5] = data; t[6] = data; t[7] = data; + t[8] = data; t[9] = data; t[10] = data; t[11] = data; + t[12] = data; t[13] = data; t[14] = data; t[15] = data; t += 16; len -= 16; } if(len & 8) { - t[0] = t[1] = t[2] = t[3] = t[4] = t[5] = t[6] = t[7] = c; + t[0] = t[1] = t[2] = t[3] = t[4] = t[5] = t[6] = t[7] = data; t += 8; } if(len & 4) { - t[0] = t[1] = t[2] = t[3] = c; + t[0] = t[1] = t[2] = t[3] = data; t += 4; } if(len & 2) { - t[0] = t[1] = c; + t[0] = t[1] = data; t += 2; } if(len & 1) - t[0] = c; + t[0] = data; } inline -void memcpyd(dword *t, const dword *s, size_t len) +void memset8(void *p, byte val, size_t len) { - if(len >= 16) { - memcpy(t, s, 4 * len); - return; - } - if(len & 8) { - t[0] = s[0]; t[1] = s[1]; t[2] = s[2]; t[3] = s[3]; - t[4] = s[4]; t[5] = s[5]; t[6] = s[6]; t[7] = s[7]; - t += 8; - s += 8; - len -= 8; - } - if(len & 4) { - t[0] = s[0]; t[1] = s[1]; t[2] = s[2]; t[3] = s[3]; - s += 4; - t += 4; - } - if(len & 2) { - t[0] = s[0]; t[1] = s[1]; - s += 2; - t += 2; - } - if(len & 1) - t[0] = s[0]; -} - -#endif - -#ifdef CPU_UNALIGNED - -void svo_memset_l(byte *t, dword val4, size_t len); - -inline -void svo_memset(void *p, byte val, size_t len) -{ - byte *t = (byte *)p; - if(len < 2) { - if(len) - t[0] = val; - return; - } - dword val4 = 0x1010101 * val; - if(len <= 4) { - *(word *)t = *(word *)(t + len - 2) = (word)val4; - return; - } - *(dword *)t = *(dword *)(t + len - 4) = val4; // alignment & fill tail - if(len > 16) { - svo_memset_l(t, val4, len); - return; - } - if(len > 8) - *(dword *)(t + 4) = *(dword *)(t + len - 8) = val4; -} - -void svo_memcpy_l(byte *t, byte *s, size_t len); - -inline -void svo_memcpy(void *p, const void *q, size_t len) -{ - byte *t = (byte *)p; - byte *s = (byte *)q; - if(len < 2) { - if(len) - t[0] = s[0]; - return; - } - if(len <= 4) { - *(word *)t = *(word *)s; - *(word *)(t + len - 2) = *(word *)(s + len - 2); - return; - } - // TODO: IMPROVE! use int64 - *(dword *)t = *(dword *)s; - *(dword *)(t + len - 4) = *(dword *)(s + len - 4); - if(len > 16) { - svo_memcpy_l(t, s, len); - return; - } - if(len > 8) { - *(dword *)(t + 4) = *(dword *)(s + 4); - *(dword *)(t + len - 8) = *(dword *)(s + len - 8); - } -} - -#else - -inline -void svo_memset(void *p, byte val, size_t len) -{ - if(len >= 32) { + if(len >= 64) { memset(p, val, len); return; } - byte *t = (byte *)p; - if(len & 16) { - t[0] = val; t[1] = val; t[2] = val; t[3] = val; - t[4] = val; t[5] = val; t[6] = val; t[7] = val; - t[8] = val; t[9] = val; t[10] = val; t[11] = val; - t[12] = val; t[13] = val; t[14] = val; t[15] = val; - t += 16; - } - if(len & 8) { - t[0] = t[1] = t[2] = t[3] = t[4] = t[5] = t[6] = t[7] = val; - t += 8; - } - if(len & 4) { - t[0] = t[1] = t[2] = t[3] = val; - t += 4; - } - if(len & 2) { - t[0] = t[1] = val; - t += 2; - } - if(len & 1) - t[0] = val; + memset__(p, data, len); } inline -void svo_memcpy(void *p, const void *q, size_t len) +void memset16(void *p, word val, size_t len) { - byte *t = (byte *)p; - byte *s = (byte *)q; - if(len >= 16) { - memcpy(t, s, len); - return; - } - if(len & 8) { - t[0] = s[0]; t[1] = s[1]; t[2] = s[2]; t[3] = s[3]; - t[4] = s[4]; t[5] = s[5]; t[6] = s[6]; t[7] = s[7]; - t += 8; - s += 8; - } - if(len & 4) { - t[0] = s[0]; t[1] = s[1]; t[2] = s[2]; t[3] = s[3]; - t += 4; - s += 4; - } - if(len & 2) { - t[0] = s[0]; t[1] = s[1]; - t += 2; - s += 2; - } - if(len & 1) - t[0] = s[0]; + memset__(p, data, len); } -#endif +inline +void memset32(void *p, word val, size_t len) +{ + memset__(p, data, len); +} + +inline +void memcpy8(void *p, const void *q, size_t len) +{ + memcpy(p, q, len); +} + +inline +void memcpy16(void *p, const void *q, size_t len) +{ + memcpy(p, q, 2 * len); +} + +inline +void memcpy32(void *p, const void *q, size_t len) +{ + memcpy(p, q, 4 * len); +} + +inline +void memcpy64(void *p, const void *q, size_t len) +{ + memcpy(p, q, 8 * len); +} + +inline +void memcpy128(void *p, const void *q, size_t len) +{ + memcpy(p, q, 16 * len); +} template -void memcpy_t(T *t, const T *s, size_t count) +void memcpy_t(void *t, const T *s, size_t count) { -#ifdef CPU_X86 - if((sizeof(T) & 15) == 0) - memcpydq((dqword *)t, (const dqword *)s, count * (sizeof(T) >> 4)); - else - if((sizeof(T) & 7) == 0) - memcpyq((qword *)t, (const qword *)s, count * (sizeof(T) >> 3)); - else -#endif - if((sizeof(T) & 3) == 0) - memcpyd((dword *)t, (const dword *)s, count * (sizeof(T) >> 2)); - else - svo_memcpy((void *)t, (void *)s, count * sizeof(T)); + memcpy8(t, s, count * sizeof(T)); } + +#endif diff --git a/uppsrc/Core/Stream.cpp b/uppsrc/Core/Stream.cpp index a97d20423..e67a0316a 100644 --- a/uppsrc/Core/Stream.cpp +++ b/uppsrc/Core/Stream.cpp @@ -645,7 +645,7 @@ Stream& Stream::operator%(WString& s) { LoadError(); else { WStringBuffer sb(len); - svo_memcpy(~sb, ~h, len * sizeof(wchar)); + memcpy8(~sb, ~h, len * sizeof(wchar)); s = sb; } } @@ -761,7 +761,7 @@ void StringStream::_Put(const void *d, dword sz) SetWriteBuffer(); ptr = buffer + p; } - svo_memcpy(ptr, d, sz); + memcpy8(ptr, d, sz); ptr += sz; } @@ -784,7 +784,7 @@ dword StringStream::_Get(void *data, dword sz) { SetReadMode(); dword read = min((dword)(uintptr_t)(rdlim - ptr), sz); - svo_memcpy(data, ptr, read); + memcpy8(data, ptr, read); ptr += read; return read; } @@ -836,7 +836,7 @@ int64 MemStream::GetSize() const { dword MemStream::_Get(void *data, dword size) { if(size > (dword)(intptr_t)(rdlim - ptr)) size = (dword)(intptr_t)(rdlim - ptr); - svo_memcpy(data, ptr, size); + memcpy8(data, ptr, size); ptr += size; return size; } @@ -846,7 +846,7 @@ void MemStream::_Put(const void *data, dword size) { SetError(ERROR_NOT_ENOUGH_SPACE); return; } - svo_memcpy(ptr, data, size); + memcpy8(ptr, data, size); ptr += size; } @@ -1037,7 +1037,7 @@ void OutStream::_Put(const void *data, dword size) Out(data, size); else if(ptr + size < wrlim) { - svo_memcpy(ptr, data, size); + memcpy8(ptr, data, size); ptr += size; } else { diff --git a/uppsrc/Core/Stream.h b/uppsrc/Core/Stream.h index 3192d157b..7a6b7ea56 100644 --- a/uppsrc/Core/Stream.h +++ b/uppsrc/Core/Stream.h @@ -91,8 +91,8 @@ public: byte *PutPtr(int size = 1) { ASSERT(size > 0); if(ptr + size <= wrlim) { byte *p = ptr; ptr += size; return p; }; return NULL; } const byte *GetSzPtr(int& size) { Term(); size = int(rdlim - ptr); byte *p = ptr; ptr += size; return p; } - void Put(const void *data, int size) { ASSERT(size >= 0); if(size) { if(ptr + size <= wrlim) { svo_memcpy(ptr, data, size); ptr += size; } else _Put(data, size); } } - int Get(void *data, int size) { ASSERT(size >= 0); if(ptr + size <= rdlim) { svo_memcpy(data, ptr, size); ptr += size; return size; } return _Get(data, size); } + void Put(const void *data, int size) { ASSERT(size >= 0); if(size) { if(ptr + size <= wrlim) { memcpy8(ptr, data, size); ptr += size; } else _Put(data, size); } } + int Get(void *data, int size) { ASSERT(size >= 0); if(ptr + size <= rdlim) { memcpy8(data, ptr, size); ptr += size; return size; } return _Get(data, size); } void Put(const String& s) { Put((const char *) s, s.GetLength()); } String Get(int size); diff --git a/uppsrc/Core/String.cpp b/uppsrc/Core/String.cpp index 91771cdf0..3c4cf7d29 100644 --- a/uppsrc/Core/String.cpp +++ b/uppsrc/Core/String.cpp @@ -79,7 +79,7 @@ unsigned String0::LHashValue() const if(l < 15) { dword w[4]; w[0] = w[1] = w[2] = w[3] = 0; - svo_memcpy((char *)w, ptr, l); + memcpy8((char *)w, ptr, l); ((byte *)w)[SLEN] = l; return CombineHash(w[0], w[1], w[2], w[3]); } @@ -127,7 +127,7 @@ char *String0::Insert(int pos, int count, const char *s) LLen() = newlen; str[newlen] = 0; if(s) - svo_memcpy(str + pos, s, count); + memcpy8(str + pos, s, count); Dsyn(); return str + pos; } @@ -135,11 +135,11 @@ char *String0::Insert(int pos, int count, const char *s) char *p = Alloc(max(len >= int((int64)2 * INT_MAX / 3) ? INT_MAX : len + (len >> 1), newlen), kind); if(pos > 0) - svo_memcpy(p, str, pos); + memcpy8(p, str, pos); if(pos < len) - svo_memcpy(p + pos + count, str + pos, len - pos); + memcpy8(p + pos + count, str + pos, len - pos); if(s) - svo_memcpy(p + pos, s, count); + memcpy8(p + pos, s, count); p[newlen] = 0; Free(); ptr = p; @@ -156,7 +156,7 @@ void String0::UnShare() int len = LLen(); char kind; char *p = Alloc(len, kind); - svo_memcpy(p, ptr, len + 1); + memcpy8(p, ptr, len + 1); Free(); chr[KIND] = kind; ptr = p; @@ -231,7 +231,7 @@ void String0::Cat(const char *s, int len) { if(IsSmall()) { if(SLen() + len < 14) { - svo_memcpy(chr + SLen(), s, len); + memcpy8(chr + SLen(), s, len); SLen() += len; chr[(int)SLen()] = 0; Dsyn(); @@ -240,7 +240,7 @@ void String0::Cat(const char *s, int len) } else if((int)LLen() + len < LAlloc() && !IsSharedRef()) { - svo_memcpy(ptr + LLen(), s, len); + memcpy8(ptr + LLen(), s, len); LLen() += len; ptr[LLen()] = 0; Dsyn(); @@ -259,7 +259,7 @@ void String0::Reserve(int r) void String0::SetL(const char *s, int len) { char *p = Alloc(len, chr[KIND]); - svo_memcpy(p, s, len); + memcpy8(p, s, len); p[len] = 0; ptr = p; LLen() = len; @@ -328,7 +328,7 @@ String::String(StringBuffer& b) int l = b.GetLength(); if(l <= 14) { Zero(); - svo_memcpy(chr, b.pbegin, l); + memcpy8(chr, b.pbegin, l); SLen() = l; b.Free(); } @@ -395,12 +395,12 @@ void StringBuffer::Realloc(dword n, const char *cat, int l) } if(!realloced) { p = Alloc(n, al); - svo_memcpy(p, pbegin, min((dword)GetLength(), n)); + memcpy8(p, pbegin, min((dword)GetLength(), n)); } if(cat) { if(ep + l > INT_MAX) Panic("StringBuffer is too big (>2GB)!"); - svo_memcpy(p + ep, cat, l); + memcpy8(p + ep, cat, l); ep += l; } if(!realloced) { @@ -438,7 +438,7 @@ void StringBuffer::Cat(const char *s, int l) if(pend + l > limit) Realloc(max(GetLength(), l) + GetLength(), s, l); else { - svo_memcpy(pend, s, l); + memcpy8(pend, s, l); pend += l; } } @@ -458,7 +458,7 @@ void StringBuffer::Set(String& s) if(s.GetAlloc() == 14) { pbegin = (char *)MemoryAlloc32(); limit = pbegin + 31; - svo_memcpy(pbegin, s.Begin(), l); + memcpy8(pbegin, s.Begin(), l); pend = pbegin + l; } else { diff --git a/uppsrc/Core/Topt.h b/uppsrc/Core/Topt.h index 37e2dc92a..74d8c0675 100644 --- a/uppsrc/Core/Topt.h +++ b/uppsrc/Core/Topt.h @@ -116,19 +116,19 @@ inline void Copy(T *dst, const T *src, int n) { } inline void Fill(char *t, const char *lim, const char& x) -{ svo_memset(t, x, size_t(lim - t)); } +{ memset8(t, x, size_t(lim - t)); } inline void Copy(char *dst, const char *src, const char *lim) -{ svo_memcpy(dst, src, size_t((byte *)lim - (byte *)src)); } +{ memcpy8(dst, src, size_t((byte *)lim - (byte *)src)); } inline void Fill(signed char *t, const signed char *lim, const signed char& x) -{ svo_memset(t, x, size_t(lim - t)); } +{ memset8(t, x, size_t(lim - t)); } inline void Copy(signed char *dst, const signed char *src, const signed char *lim) -{ svo_memcpy(dst, src, size_t((byte *)lim - (byte *)src)); } +{ memcpy8(dst, src, size_t((byte *)lim - (byte *)src)); } inline void Fill(unsigned char *t, const unsigned char *lim, const unsigned char& x) -{ svo_memset(t, x, size_t(lim - t)); } +{ memset8(t, x, size_t(lim - t)); } inline void Copy(unsigned char *dst, const unsigned char *src, const unsigned char *lim) -{ svo_memcpy(dst, src, size_t((byte *)lim - (byte *)src)); } +{ memcpy8(dst, src, size_t((byte *)lim - (byte *)src)); } #ifdef NO_MOVEABLE_CHECK diff --git a/uppsrc/Core/Util.h b/uppsrc/Core/Util.h index d57cb3faf..5668a94ec 100644 --- a/uppsrc/Core/Util.h +++ b/uppsrc/Core/Util.h @@ -187,21 +187,12 @@ void BeepExclamation(); void BeepQuestion(); void BeepError(); -inline -void memsetw(void *t, word value, int count) -{ - word *w = (word *)t; - word *lim = w + count; - while(w < lim) - *w++ = value; -} - inline void memsetex(void *t, const void *item, int item_size, int count) { ASSERT(count >= 0); byte *q = (byte *)t; while(count--) { - svo_memcpy(q, item, item_size); + memcpy8(q, item, item_size); q += item_size; } } diff --git a/uppsrc/Core/WString.cpp b/uppsrc/Core/WString.cpp index 50f4c174e..71f539610 100644 --- a/uppsrc/Core/WString.cpp +++ b/uppsrc/Core/WString.cpp @@ -357,7 +357,7 @@ void WStringBuffer::Cat(int c, int l) { if(pend + l > limit) Expand(max(GetLength(), l) + GetLength(), NULL, l); - memsetw(pend, c, l); + memset16(pend, c, l); pend += l; } diff --git a/uppsrc/CtrlCore/ImageWin32.cpp b/uppsrc/CtrlCore/ImageWin32.cpp index e2bbdeba5..10f7874d4 100644 --- a/uppsrc/CtrlCore/ImageWin32.cpp +++ b/uppsrc/CtrlCore/ImageWin32.cpp @@ -28,7 +28,7 @@ BitmapInfo32__::BitmapInfo32__(int cx, int cy) { data.Alloc(sizeof(BITMAPINFOHEADER) + sizeof(RGBQUAD)*256); BITMAPINFOHEADER *hi = (BITMAPINFOHEADER *) ~data;; - svo_memset(hi, 0, sizeof(BITMAPINFOHEADER)); + memset8(hi, 0, sizeof(BITMAPINFOHEADER)); hi->biSize = sizeof(BITMAPINFOHEADER); hi->biPlanes = 1; #ifdef PLATFORM_WINCE diff --git a/uppsrc/Draw/FontWin32.cpp b/uppsrc/Draw/FontWin32.cpp index b5ed3ff50..302959acc 100644 --- a/uppsrc/Draw/FontWin32.cpp +++ b/uppsrc/Draw/FontWin32.cpp @@ -70,13 +70,13 @@ static void Win32_GetGlyphIndices(HDC hdc, LPCWSTR s, int n, LPWORD r, DWORD fla if(fn) fn(hdc, s, n, r, flag); else - svo_memset(r, 0, n * sizeof(WORD)); + memset8(r, 0, n * sizeof(WORD)); } CommonFontInfo GetFontInfoSys(Font font) { CommonFontInfo fi; - svo_memset(&fi, 0, sizeof(fi)); + memset8(&fi, 0, sizeof(fi)); HFONT hfont = GetWin32Font(font, 0); if(hfont) { HDC hdc = ::CreateIC("DISPLAY", NULL, NULL, NULL); @@ -201,7 +201,7 @@ GlyphInfo GetGlyphInfoSys(Font font, int chr) HFONT hfont = GetWin32Font(font, 0); if(!hfont) { GlyphInfo n; - svo_memset(&n, 0, sizeof(GlyphInfo)); + memset8(&n, 0, sizeof(GlyphInfo)); return n; } HDC hdc = CreateIC("DISPLAY", NULL, NULL, NULL); @@ -322,7 +322,7 @@ void RenderCharacterSys(FontGlyphConsumer& sw, double x, double y, int ch, Font HFONT ohfont = (HFONT) ::SelectObject(hdc, hfont); GLYPHMETRICS gm; MAT2 m_matrix; - svo_memset(&m_matrix, 0, sizeof(m_matrix)); + memset8(&m_matrix, 0, sizeof(m_matrix)); m_matrix.eM11.value = 1; m_matrix.eM22.value = 1; int gsz = GetGlyphOutlineW(hdc, ch, GGO_NATIVE|GGO_UNHINTED, &gm, 0, NULL, &m_matrix); diff --git a/uppsrc/Draw/Image.h b/uppsrc/Draw/Image.h index 652e5270a..bd4810b88 100644 --- a/uppsrc/Draw/Image.h +++ b/uppsrc/Draw/Image.h @@ -20,7 +20,7 @@ inline bool operator!=(const RGBA& a, const RGBA& b) inline RGBA RGBAZero() { RGBA c; c.r = c.g = c.b = c.a = 0; return c; } -inline void Fill(RGBA *t, RGBA c, int n) { memsetd(t, *(dword *)&c, n); } +inline void Fill(RGBA *t, RGBA c, int n) { memset32(t, *(dword *)&c, n); } inline void Copy(RGBA *t, const RGBA *s, int n) { memcpy_t(t, s, n); } int Premultiply(RGBA *t, const RGBA *s, int len); diff --git a/uppsrc/Painter/BufferPainter.h b/uppsrc/Painter/BufferPainter.h index 3c77bb717..975cad513 100644 --- a/uppsrc/Painter/BufferPainter.h +++ b/uppsrc/Painter/BufferPainter.h @@ -8,7 +8,7 @@ class ClippingLine : NoCopy { public: void Clear() { if(!IsFull()) MemoryFree(data); data = NULL; } - void Set(const byte *s, int len) { data = (byte *)MemoryAlloc(len); svo_memcpy(data, s, len); } + void Set(const byte *s, int len) { data = (byte *)MemoryAlloc(len); memcpy8(data, s, len); } void SetFull() { ASSERT(!data); data = (byte *)1; } bool IsEmpty() const { return !data; } diff --git a/uppsrc/RichText/ParaData.cpp b/uppsrc/RichText/ParaData.cpp index 4904919e7..b04a0a168 100644 --- a/uppsrc/RichText/ParaData.cpp +++ b/uppsrc/RichText/ParaData.cpp @@ -123,7 +123,7 @@ void RichPara::Number::Next(const RichPara::NumberFormat& fmt) RichPara::Number::Number() { - svo_memset(n, 0, sizeof(n)); + memset8(n, 0, sizeof(n)); } bool RichPara::NumberFormat::IsNumbered() const @@ -174,7 +174,7 @@ RichPara::Format::Format() bullet = 0; keep = newpage = keepnext = orphan = newhdrftr = false; tabsize = 296; - svo_memset(number, 0, sizeof(number)); + memset8(number, 0, sizeof(number)); reset_number = false; linespacing = 0; tab.Clear(); diff --git a/uppsrc/RichText/ParseQtf.cpp b/uppsrc/RichText/ParseQtf.cpp index 1b2aaa632..0f125e9ec 100644 --- a/uppsrc/RichText/ParseQtf.cpp +++ b/uppsrc/RichText/ParseQtf.cpp @@ -723,7 +723,7 @@ void RichQtfParser::Parse(const char *qtf, int _accesskey) case 'n': format.before_number = GetText(';'); break; case 'm': format.after_number = GetText(';'); break; case 'N': { - svo_memset(format.number, 0, sizeof(format.number)); + memset8(format.number, 0, sizeof(format.number)); format.reset_number = false; int i = 0; while(i < 8) {