diff --git a/uppsrc/Core/Core.h b/uppsrc/Core/Core.h index 4a85bdfa1..10530d791 100644 --- a/uppsrc/Core/Core.h +++ b/uppsrc/Core/Core.h @@ -279,6 +279,26 @@ class JsonIO; #include "Ops.h" #include "Fn.h" + +#ifdef flagNOSIMD + #ifdef CPU_SSE2 + #undef CPU_SSE2 + #endif + #ifdef CPU_NEON + #undef CPU_NEON + #endif +#endif + +#ifdef CPU_SSE2 +#include "SIMD_SSE2.h" +#define CPU_SIMD 1 +#endif + +#ifdef CPU_NEON +#include "SIMD_NEON.h" +#define CPU_SIMD 1 +#endif + #include "Mem.h" #include "Atomic.h" #include "Topt.h" @@ -369,6 +389,13 @@ class JsonIO; #include "ValueCache.h" +#ifdef CPU_SIMD +String AsString(const f32x4& x); +String AsString(const i32x4& x); +String AsString(const i16x8& x); +String AsString(const i8x16& x); +#endif + #ifdef PLATFORM_WIN32 NTL_MOVEABLE(POINT) NTL_MOVEABLE(SIZE) diff --git a/uppsrc/Core/Core.upp b/uppsrc/Core/Core.upp index 8cc1099c2..f65d4307a 100644 --- a/uppsrc/Core/Core.upp +++ b/uppsrc/Core/Core.upp @@ -34,6 +34,8 @@ file Cpu.cpp, Mem.h, Mem.cpp, + SIMD_SSE2.h, + SIMD.cpp, Atomic.h, Mt.h, Mt.cpp, diff --git a/uppsrc/Core/Mem.cpp b/uppsrc/Core/Mem.cpp index 10afa1512..c084ffbd3 100644 --- a/uppsrc/Core/Mem.cpp +++ b/uppsrc/Core/Mem.cpp @@ -4,11 +4,11 @@ namespace Upp { #ifdef CPU_X86 -void memset8__(void *p, __m128i data, size_t len) +void memset8__(void *p, i16x8 data, size_t len) { ASSERT(len >= 16); byte *t = (byte *)p; - auto Set4 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), data); }; + auto Set4 = [&](size_t at) { data.Store(t + at); }; Set4(len - 16); // fill tail Set4(0); // align up on the next 16 bytes boundary if(len <= 32) @@ -18,7 +18,7 @@ void memset8__(void *p, __m128i data, size_t len) len = e - t; e -= 128; if(len >= 1024*1024) { // for really huge data, bypass the cache - auto Set4S = [&](int at) { _mm_stream_si128((__m128i *)(t + at), data); }; + auto Set4S = [&](int at) { data.Stream(t + at); }; while(len >= 64) { Set4S(0*16); Set4S(1*16); Set4S(2*16); Set4S(3*16); t += 64; @@ -57,7 +57,7 @@ void memcpy8__(void *p, const void *q, size_t len) return; } - auto Copy128 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); }; + auto Copy128 = [&](size_t at) { i16x8(s + at).Store(t + at); }; Copy128(len - 16); // copy tail Copy128(0); // align target data up on the next 16 bytes boundary if(len <= 32) diff --git a/uppsrc/Core/Mem.h b/uppsrc/Core/Mem.h index c06353198..f1b3fc6f9 100644 --- a/uppsrc/Core/Mem.h +++ b/uppsrc/Core/Mem.h @@ -1,6 +1,6 @@ -#ifdef CPU_X86 +#ifdef CPU_SIMD -void memset8__(void *t, __m128i data, size_t len); +void memset8__(void *t, i16x8 data, size_t len); inline void memset8(void *p, byte data, size_t len) @@ -17,7 +17,7 @@ void memset8(void *p, byte data, size_t len) return; } if(len > 16) { - memset8__(t, _mm_set1_epi32(val4), len); + memset8__(t, i32all(val4), len); return; } *(dword *)t = *(dword *)(t + len - 4) = val4; @@ -36,12 +36,12 @@ void memset16(void *p, word data, size_t len) } dword val4 = 0x10001 * data; if(len >= 16) { - memset8__(t, _mm_set1_epi32(val4), 2 * len); + memset8__(t, i32all(val4), 2 * len); return; } *(dword *)(t + len - 2) = val4; if(len & 8) { - _mm_storeu_si128((__m128i *)t, _mm_set1_epi32(val4)); + i32all(val4).Store(t); t += 8; } if(len & 4) { @@ -66,19 +66,19 @@ void memset32(void *p, dword data, size_t len) t[0] = data; return; } - __m128i val4 = _mm_set1_epi32(data); + i32x4 val4 = i32all(data); if(len >= 16) { memset8__(t, val4, 4 * len); return; } - auto Set4 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), val4); }; - Set4(len - 4); // fill tail + auto Set128 = [&](size_t at) { val4.Store(t + at); }; + Set128(len - 4); // fill tail if(len & 8) { - Set4(0); Set4(4); + Set128(0); Set128(4); t += 8; } if(len & 4) - Set4(0); + Set128(0); } inline @@ -90,12 +90,12 @@ void memset64(void *p, qword data, size_t len) t[0] = data; return; } - __m128i val2 = _mm_set1_epi64x(data); + i16x8 val2 = i64all(data); if(len >= 8) { memset8__(t, val2, 8 * len); return; } - auto Set128 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), val2); }; + auto Set128 = [&](size_t at) { val2.Store(t + at); }; Set128(len - 2); // fill tail if(len & 4) { Set128(0); Set128(2); @@ -109,7 +109,7 @@ inline void memset128(void *t, m128 data, size_t len) { if(len) - memset8__(t, *(__m128i *)&data, 16 * len); + memset8__(t, i16x8().Load(&data), 16 * len); } void memcpy8__(void *p, const void *q, size_t len); @@ -163,7 +163,7 @@ void memcpy16(void *p, const void *q, size_t len) *(uint64 *)(t + len - 4) = *(uint64 *)(s + len - 4); return; } - auto Copy128 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); }; + auto Copy128 = [&](size_t at) { i16x8(s + at).Store(t + at); }; Copy128(0); Copy128(len - 8); return; @@ -204,7 +204,7 @@ void memcpy32(void *p, const void *q, size_t len) } #endif - auto Copy128 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); }; + auto Copy128 = [&](size_t at) { i16x8(s + at).Store(t + at); }; if(len >= 16) { memcpy8__(t, s, 4 * len); @@ -238,20 +238,20 @@ void memcpy64(void *p, const void *q, size_t len) return; } - auto Copy4 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); }; + auto Copy128 = [&](size_t at) { i16x8(s + at).Store(t + at); }; - Copy4(len - 2); // copy tail + Copy128(len - 2); // copy tail if(len >= 8) { memcpy8__(t, s, 8 * len); return; } if(len & 4) { - Copy4(0); Copy4(2); + Copy128(0); Copy128(2); t += 4; s += 4; } if(len & 2) - Copy4(0); + Copy128(0); } inline @@ -264,24 +264,24 @@ void memcpy128(void *p, const void *q, size_t len) dqword *t = (dqword *)p; dqword *s = (dqword *)q; - auto Copy4 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); }; + auto Copy128 = [&](size_t at) { i16x8(s + at).Store(t + at); }; if(len >= 8) { memcpy8__(t, s, 16 * len); return; } if(len & 4) { - Copy4(0); Copy4(1); Copy4(2); Copy4(3); + Copy128(0); Copy128(1); Copy128(2); Copy128(3); t += 4; s += 4; } if(len & 2) { - Copy4(0); Copy4(1); + Copy128(0); Copy128(1); t += 2; s += 2; } if(len & 1) - Copy4(0); + Copy128(0); } template @@ -311,11 +311,9 @@ bool memeq8__(const void *p, const void *q, size_t len) const byte *t = (byte *)p; const byte *s = (byte *)q; - auto Cmp128 = [&](size_t at) { return _mm_cmpeq_epi32(_mm_loadu_si128((__m128i *)(s + at)), _mm_loadu_si128((__m128i *)(t + at))); }; - auto Neq = [](__m128i v) { return _mm_movemask_epi8(v) != 0xffff; }; - auto And = [](__m128i a, __m128i b) { return _mm_and_si128(a, b); }; + auto Cmp128 = [&](size_t at) { return i16x8(s + at) == i16x8(t + at); }; - if(Neq(And(Cmp128(len - 16), Cmp128(0)))) // tail & alignment, also <= 32 + if(!Test(Cmp128(len - 16) & Cmp128(0))) // tail & alignment, also <= 32 return false; if(len <= 32) @@ -329,13 +327,13 @@ bool memeq8__(const void *p, const void *q, size_t len) len = e - t; e -= 32; while(t <= e) { - if(Neq(And(Cmp128(0), Cmp128(1*16)))) + if(!Test(Cmp128(0) & Cmp128(1*16))) return false; s += 32; t += 32; } if(len & 16) - if(Neq(Cmp128(0))) + if(!Test(Cmp128(0))) return false; return true; } diff --git a/uppsrc/Core/SIMD.cpp b/uppsrc/Core/SIMD.cpp new file mode 100644 index 000000000..f0162766a --- /dev/null +++ b/uppsrc/Core/SIMD.cpp @@ -0,0 +1,35 @@ +#include "Core.h" + +#ifdef CPU_SIMD + +namespace Upp { + +String AsString(const f32x4& x) +{ + float *f = (float *)&x; + return Format("%g %g %g %g", f[3], f[2], f[1], f[0]); +} + +String AsString(const i32x4& x) +{ + int *f = (int *)&x; + return Format("%d %d %d %d", f[3], f[2], f[1], f[0]); +} + +String AsString(const i16x8& x) +{ + int16 *f = (int16 *)&x; + return Format("%d %d %d %d %d %d %d %d", f[7], f[6], f[5], f[4], f[3], f[2], f[1], f[0]); +} + +String AsString(const i8x16& x) +{ + int8 *f = (int8 *)&x; + return Format("%d %d %d %d . %d %d %d %d . %d %d %d %d . %d %d %d %d", + f[15], f[14], f[13], f[12], f[11], f[10], f[9], f[8], + f[7], f[6], f[5], f[4], f[3], f[2], f[1], f[0]); +} + +}; + +#endif diff --git a/uppsrc/Core/SIMD_SSE2.h b/uppsrc/Core/SIMD_SSE2.h new file mode 100644 index 000000000..8003e439b --- /dev/null +++ b/uppsrc/Core/SIMD_SSE2.h @@ -0,0 +1,175 @@ +struct f32x4 { + __m128 data; + + f32x4& Load(void *ptr) { data = _mm_loadu_ps((float *)ptr); return *this; } + f32x4& Load64(void *ptr) { data = _mm_castpd_ps(_mm_load_sd((double *)ptr)); return *this; } + f32x4& Load32(void *ptr) { data = _mm_load_ss((float *)ptr); return *this; } + + void Store(void *ptr) { _mm_storeu_ps((float *)ptr, data); } + void Store64(void *ptr) { _mm_store_sd((double *)ptr, _mm_castps_pd(data)); } + void Store32(void *ptr) { _mm_store_ss((float *)ptr, data); } + + f32x4() {} + f32x4(void *ptr) { Load(ptr); } + f32x4(__m128 d) { data = d; } + f32x4(double f) { data = _mm_set_ss((float)f); } + f32x4(float f) { data = _mm_set_ss(f); } + f32x4(int f) { data = _mm_set_ss((float)f); } + f32x4(double a, double b, double c, double d) { data = _mm_set_ps((float)a, (float)b, (float)c, (float)d); } + + operator __m128() { return data; } +}; + +force_inline f32x4 f32all(double f) { return _mm_set1_ps((float)f); } + +force_inline f32x4 operator+(f32x4 a, f32x4 b) { return _mm_add_ps(a.data, b.data); } +force_inline f32x4& operator+=(f32x4& a, f32x4 b) { return a = a + b; } +force_inline f32x4 operator-(f32x4 a, f32x4 b) { return _mm_sub_ps(a.data, b.data); } +force_inline f32x4& operator-=(f32x4& a, f32x4 b) { return a = a - b; } +force_inline f32x4 operator*(f32x4 a, f32x4 b) { return _mm_mul_ps(a.data, b.data); } +force_inline f32x4& operator*=(f32x4& a, f32x4 b) { return a = a * b; } +force_inline f32x4 operator/(f32x4 a, f32x4 b) { return _mm_div_ps(a.data, b.data); } +force_inline f32x4& operator/=(f32x4& a, f32x4 b) { return a = a + b; } + +force_inline f32x4 operator==(f32x4 a, f32x4 b) { return _mm_cmpeq_ps(a.data, b.data); } +force_inline f32x4 operator!=(f32x4 a, f32x4 b) { return _mm_cmpneq_ps(a.data, b.data); } +force_inline f32x4 operator<(f32x4 a, f32x4 b) { return _mm_cmplt_ps(a.data, b.data); } +force_inline f32x4 operator>(f32x4 a, f32x4 b) { return _mm_cmpgt_ps(a.data, b.data); } +force_inline f32x4 operator<=(f32x4 a, f32x4 b) { return _mm_cmple_ps(a.data, b.data); } +force_inline f32x4 operator>=(f32x4 a, f32x4 b) { return _mm_cmpge_ps(a.data, b.data); } +force_inline bool Test(f32x4 a) { return _mm_movemask_ps(a.data) == 0xf; } + +force_inline f32x4 min(f32x4 a, f32x4 b) { return _mm_min_ps(a.data, b.data); } +force_inline f32x4 max(f32x4 a, f32x4 b) { return _mm_max_ps(a.data, b.data); } + +#define _MM_BCAST(a) _MM_SHUFFLE(a,a,a,a) + +force_inline f32x4 Broadcast0(f32x4 a) { return _mm_shuffle_ps(a.data, a.data, _MM_BCAST(0)); } +force_inline f32x4 Broadcast1(f32x4 a) { return _mm_shuffle_ps(a.data, a.data, _MM_BCAST(1)); } +force_inline f32x4 Broadcast2(f32x4 a) { return _mm_shuffle_ps(a.data, a.data, _MM_BCAST(2)); } +force_inline f32x4 Broadcast3(f32x4 a) { return _mm_shuffle_ps(a.data, a.data, _MM_BCAST(3)); } + +struct i16x8 { // 8xint16 + __m128i data; + + i16x8& Load(const void *ptr) { data = _mm_loadu_si128((__m128i *)ptr); return *this; } + i16x8& Load64(const void *ptr) { data = _mm_castpd_si128(_mm_load_sd((double *)ptr)); return *this; } + i16x8& Load32(const void *ptr) { data = _mm_castps_si128(_mm_load_ss((float *)ptr)); return *this; } + + void Store(void *ptr) { _mm_storeu_si128((__m128i *)ptr, data); } + void Store64(void *ptr) { _mm_store_sd((double *)ptr, _mm_castsi128_pd(data)); } + void Store32(void *ptr) { _mm_store_ss((float *)ptr, _mm_castsi128_ps(data)); } + void Stream(void *ptr) { _mm_stream_si128((__m128i *)ptr, data); }; + + i16x8() {} + i16x8(const void *ptr) { Load(ptr); } + i16x8(__m128i d) { data = d; } + i16x8(int v) { data = _mm_set_epi16(0, 0, 0, 0, 0, 0, 0, v); } + i16x8(int a, int b, int c, int d, int e, int f, int g, int h) { data = _mm_set_epi16(a, b, c, d, e, f, g, h); } + operator __m128i() { return data; } +}; + +force_inline i16x8 i16all(int v) { return _mm_set1_epi16(v); } + +force_inline i16x8 operator+(i16x8 a, i16x8 b) { return _mm_add_epi16(a.data, b.data); } +force_inline i16x8& operator+=(i16x8& a, i16x8 b) { return a = a + b; } +force_inline i16x8 operator-(i16x8 a, i16x8 b) { return _mm_sub_epi16(a.data, b.data); } +force_inline i16x8& operator-=(i16x8& a, i16x8 b) { return a = a - b; } +force_inline i16x8 operator*(i16x8 a, i16x8 b) { return _mm_mullo_epi16(a.data, b.data); } +force_inline i16x8& operator*=(i16x8& a, i16x8 b) { return a = a * b; } + +force_inline i16x8 operator&(i16x8 a, i16x8 b) { return _mm_and_si128(a.data, b.data); } +force_inline i16x8& operator&=(i16x8& a, i16x8 b) { return a = a & b; } +force_inline i16x8 operator|(i16x8 a, i16x8 b) { return _mm_or_si128(a.data, b.data); } +force_inline i16x8& operator|=(i16x8& a, i16x8 b) { return a = a | b; } +force_inline i16x8 operator^(i16x8 a, i16x8 b) { return _mm_xor_si128(a.data, b.data); } +force_inline i16x8& operator^=(i16x8& a, i16x8 b) { return a = a ^ b; } +force_inline i16x8 operator~(i16x8 a) { return _mm_xor_si128(a.data, i16all(0xffff).data); } + +force_inline i16x8 operator>>(i16x8 a, int b) { return _mm_srli_epi16(a.data, b); } +force_inline i16x8& operator>>=(i16x8& a, int b) { return a = a >> b; } +force_inline i16x8 operator<<(i16x8 a, int b) { return _mm_slli_epi16(a.data, b); } +force_inline i16x8& operator<<=(i16x8& a, int b) { return a = a << b; } + +force_inline i16x8 operator==(i16x8 a, i16x8 b) { return _mm_cmpeq_epi16(a.data, b.data); } +force_inline i16x8 operator<(i16x8 a, i16x8 b) { return _mm_cmplt_epi16(a.data, b.data); } +force_inline i16x8 operator>(i16x8 a, i16x8 b) { return _mm_cmpgt_epi16(a.data, b.data); } +force_inline bool Test(i16x8 a) { return _mm_movemask_epi8(a.data) == 0xffff; } + +struct i32x4 : i16x8 { // 4xint32 + i32x4() {} + i32x4(void *ptr) { Load(ptr); } + i32x4(__m128i d) { data = d; } + i32x4(int v) { data = _mm_set_epi32(0, 0, 0, v); } + i32x4(int a, int b, int c, int d) { data = _mm_set_epi32(a, b, c, d); } + operator int() { return _mm_cvtsi128_si32(data); } +}; + +force_inline i32x4 i32all(int v) { return _mm_set1_epi32(v); } + +force_inline i32x4 operator+(i32x4 a, i32x4 b) { return _mm_add_epi32(a.data, b.data); } +force_inline i32x4& operator+=(i32x4& a, i32x4 b) { return a = a + b; } +force_inline i32x4 operator-(i32x4 a, i32x4 b) { return _mm_sub_epi32(a.data, b.data); } +force_inline i32x4& operator-=(i32x4& a, i32x4 b) { return a = a - b; } + +force_inline i32x4 operator&(i32x4 a, i32x4 b) { return _mm_and_si128(a.data, b.data); } +force_inline i32x4& operator&=(i32x4& a, i32x4 b) { return a = a & b; } +force_inline i32x4 operator|(i32x4 a, i32x4 b) { return _mm_or_si128(a.data, b.data); } +force_inline i32x4& operator|=(i32x4& a, i32x4 b) { return a = a | b; } +force_inline i32x4 operator^(i32x4 a, i32x4 b) { return _mm_xor_si128(a.data, b.data); } +force_inline i32x4& operator^=(i32x4& a, i32x4 b) { return a = a ^ b; } +force_inline i32x4 operator~(i32x4 a) { return _mm_xor_si128(a.data, i32all(0xffffffff).data); } + +force_inline i32x4 operator>>(i32x4 a, int b) { return _mm_srli_epi32(a.data, b); } +force_inline i32x4& operator>>=(i32x4& a, int b) { return a = a >> b; } +force_inline i32x4 operator<<(i32x4 a, int b) { return _mm_slli_epi32(a.data, b); } +force_inline i32x4& operator<<=(i32x4& a, int b) { return a = a << b; } + +force_inline i32x4 operator==(i32x4 a, i32x4 b) { return _mm_cmpeq_epi32(a.data, b.data); } +force_inline i32x4 operator<(i32x4 a, i32x4 b) { return _mm_cmplt_epi32(a.data, b.data); } +force_inline i32x4 operator>(i32x4 a, i32x4 b) { return _mm_cmpgt_epi32(a.data, b.data); } +force_inline bool Test(i32x4 a) { return _mm_movemask_epi8(a.data) == 0xffff; } + +struct i8x16 : i16x8 { // 16xint8 + i8x16() {} + i8x16(void *ptr) { Load(ptr); } + i8x16(__m128i d) { data = d; } + i8x16(int v) { data = _mm_set_epi8(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,v); } + i8x16(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k, int l, int m, int n, int o, int p) + { data = _mm_set_epi8(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p); } +}; + +force_inline i8x16 i8all(int v) { return _mm_set1_epi8(v); } + +force_inline i8x16 operator+(i8x16 a, i8x16 b) { return _mm_add_epi8(a.data, b.data); } +force_inline i8x16& operator+=(i8x16& a, i8x16 b) { return a = a + b; } +force_inline i8x16 operator-(i8x16 a, i8x16 b) { return _mm_sub_epi8(a.data, b.data); } +force_inline i8x16& operator-=(i8x16& a, i8x16 b) { return a = a - b; } + +force_inline i8x16 operator&(i8x16 a, i8x16 b) { return _mm_and_si128(a.data, b.data); } +force_inline i8x16& operator&=(i8x16& a, i8x16 b) { return a = a & b; } +force_inline i8x16 operator|(i8x16 a, i8x16 b) { return _mm_or_si128(a.data, b.data); } +force_inline i8x16& operator|=(i8x16& a, i8x16 b) { return a = a | b; } +force_inline i8x16 operator^(i8x16 a, i8x16 b) { return _mm_xor_si128(a.data, b.data); } +force_inline i8x16& operator^=(i8x16& a, i8x16 b) { return a = a ^ b; } +force_inline i8x16 operator~(i8x16 a) { return _mm_xor_si128(a.data, i8all(0xff).data); } + +force_inline f32x4 ToFloat(i32x4 a) { return _mm_cvtepi32_ps(a.data); } +force_inline i32x4 Truncate(f32x4 a) { return _mm_cvttps_epi32(a.data); } +force_inline i32x4 Round(f32x4 a) { return _mm_cvtps_epi32(a.data); } + +force_inline i16x8 Unpack8L(i16x8 a) { return _mm_unpacklo_epi8(a.data, _mm_setzero_si128()); } +force_inline i16x8 Unpack8H(i16x8 a) { return _mm_unpackhi_epi8(a.data, _mm_setzero_si128()); } +force_inline i32x4 Unpack16L(i16x8 a) { return _mm_unpacklo_epi16(a.data, _mm_setzero_si128()); } +force_inline i32x4 Unpack16H(i16x8 a) { return _mm_unpackhi_epi16(a.data, _mm_setzero_si128()); } + +force_inline i8x16 Pack16(i16x8 l, i16x8 h) { return _mm_packus_epi16(l.data, h.data); } +force_inline i8x16 Pack16(i16x8 l) { return _mm_packus_epi16(l.data, _mm_setzero_si128()); } +force_inline i16x8 Pack32(i32x4 a) { return _mm_packs_epi32(a.data, _mm_setzero_si128()); } + +force_inline i16x8 BroadcastLH0(i16x8 a) { return _mm_shufflelo_epi16(_mm_shufflehi_epi16(a.data, _MM_BCAST(0)), _MM_BCAST(0)); } +force_inline i16x8 BroadcastLH1(i16x8 a) { return _mm_shufflelo_epi16(_mm_shufflehi_epi16(a.data, _MM_BCAST(1)), _MM_BCAST(1)); } +force_inline i16x8 BroadcastLH2(i16x8 a) { return _mm_shufflelo_epi16(_mm_shufflehi_epi16(a.data, _MM_BCAST(2)), _MM_BCAST(2)); } +force_inline i16x8 BroadcastLH3(i16x8 a) { return _mm_shufflelo_epi16(_mm_shufflehi_epi16(a.data, _MM_BCAST(3)), _MM_BCAST(3)); } + +force_inline i16x8 i64all(qword data) { return _mm_set1_epi64x(data); } diff --git a/uppsrc/Draw/Mify.cpp b/uppsrc/Draw/Mify.cpp index 3074bbcee..f276094f1 100644 --- a/uppsrc/Draw/Mify.cpp +++ b/uppsrc/Draw/Mify.cpp @@ -2,7 +2,7 @@ namespace Upp { -#ifdef CPU_SSE2 +#ifdef CPU_SIMD Image Minify(const Image& img, int nx, int ny, bool co) { @@ -11,11 +11,11 @@ Image Minify(const Image& img, int nx, int ny, bool co) Size tsz = Size((ssz.cx + nx - 1) / nx, (ssz.cy + ny - 1) / ny); ImageBuffer ib(tsz); int scx0 = ssz.cx / nx * nx; - auto do_line = [&](int ty, __m128 *b, __m128 *div) { - memset(b, 0, tsz.cx * sizeof(__m128)); - memset(div, 0, tsz.cx * sizeof(__m128)); - __m128 v1 = _mm_set1_ps(1); - __m128 vnx = _mm_set1_ps((float)nx); + auto do_line = [&](int ty, f32x4 *b, f32x4 *div) { + memset(b, 0, tsz.cx * sizeof(f32x4)); + memset(div, 0, tsz.cx * sizeof(f32x4)); + f32x4 v1 = f32all(1); + f32x4 vnx = f32all(nx); int yy = ny * ty; for(int yi = 0; yi < ny; yi++) { int y = yy + yi; @@ -23,44 +23,44 @@ Image Minify(const Image& img, int nx, int ny, bool co) const RGBA *s = img[yy + yi]; const RGBA *e = s + scx0; const RGBA *e2 = s + ssz.cx; - __m128 *t = b; - __m128 *d = div; + f32x4 *t = b; + f32x4 *d = div; while(s < e) { - __m128 px = _mm_setzero_ps(); + f32x4 px = 0; for(int n = nx; n--;) - px = _mm_add_ps(px, LoadRGBAF(s++)); - *t = _mm_add_ps(*t, px); - *d = _mm_add_ps(*d, vnx); + px += LoadRGBAF(s++); + *t += px; + *d += vnx; t++; d++; } if(s < e2) { - __m128 px = _mm_setzero_ps(); - __m128 dv = _mm_setzero_ps(); + f32x4 px = 0; + f32x4 dv = 0; while(s < e2) { - px = _mm_add_ps(px, LoadRGBAF(s++)); - dv = _mm_add_ps(px, v1); + px += LoadRGBAF(s++); + dv += v1; } - *t = _mm_add_ps(*t, px); - *d = _mm_add_ps(*d, dv); + *t += px; + *d += dv; t++; d++; } ASSERT(t == b + tsz.cx); } } - __m128 *s = b; - __m128 *d = div; + f32x4 *s = b; + f32x4 *d = div; RGBA *t = ~ib + ty * tsz.cx; RGBA *e = t + tsz.cx; while(t < e) - StoreRGBAF(t++, _mm_div_ps(*s++, *d++)); + StoreRGBAF(t++, *s++ / *d++); }; if(co) { CoWork cw; cw * [&] { - Buffer<__m128> div(tsz.cx); - Buffer<__m128> b(tsz.cx); + Buffer div(tsz.cx); + Buffer b(tsz.cx); for(;;) { int y = cw.Next(); if(y >= tsz.cy) @@ -70,8 +70,8 @@ Image Minify(const Image& img, int nx, int ny, bool co) }; } else { - Buffer<__m128> div(tsz.cx); - Buffer<__m128> b(tsz.cx); + Buffer div(tsz.cx); + Buffer b(tsz.cx); for(int y = 0; y < tsz.cy; y++) do_line(y, b, div); } diff --git a/uppsrc/Draw/RescaleFilter.cpp b/uppsrc/Draw/RescaleFilter.cpp index c3b2c8f1c..2820537d7 100644 --- a/uppsrc/Draw/RescaleFilter.cpp +++ b/uppsrc/Draw/RescaleFilter.cpp @@ -91,23 +91,23 @@ Image RescaleFilter(const Image& img, Size sz, const Rect& sr, } RGBA *t = ib[y]; - #ifdef CPU_X86 + #ifdef CPU_SIMD for(int x = 0; x < sz.cx; x++) { - __m128 rgbaf = _mm_setzero_ps(); - __m128 w = _mm_setzero_ps(); + f32x4 rgbaf = 0; + f32x4 w = 0; yd = py; int hasalpha = 0; for(int yy = 2 * ay; yy-- > 0;) { int ky = *yd++; const RGBA *l = img[*yd++]; for(int xx = 2 * ax; xx-- > 0;) { - __m128 s = LoadRGBAF(&l[*xd++]); - __m128 weight = _mm_set1_ps(float(ky * *xd++)); - rgbaf = _mm_add_ps(rgbaf, _mm_mul_ps(weight, s)); - w = _mm_add_ps(w, weight); + f32x4 s = LoadRGBAF(&l[*xd++]); + f32x4 weight = f32all(float(ky * *xd++)); + rgbaf += weight * s; + w += weight; } } - StoreRGBAF(t++, ClampRGBAF(_mm_div_ps(rgbaf, w))); + StoreRGBAF(t++, ClampRGBAF(rgbaf / w)); } #else for(int x = 0; x < sz.cx; x++) { diff --git a/uppsrc/Draw/SSE2.h b/uppsrc/Draw/SSE2.h index dcf5835f1..997b895e4 100644 --- a/uppsrc/Draw/SSE2.h +++ b/uppsrc/Draw/SSE2.h @@ -1,5 +1,103 @@ #ifdef CPU_X86 +#ifdef CPU_SIMD + +force_inline +i16x8 LoadRGBA(const RGBA *s) +{ + return Unpack8L(i32x4(*(dword *)s)); +} + +force_inline +i16x8 LoadRGBA2(const RGBA *s0, const RGBA *s1) +{ + return Unpack8L(i32x4(0, 0, *(dword *)s1, *(dword *)s0)); +} + +force_inline +i16x8 LoadRGBA2(const RGBA& c) +{ + return LoadRGBA2(&c, &c); +} + +force_inline +i16x8 LoadRGBA2(const RGBA *s) +{ + i16x8 h; + h.Load64(s); + return Unpack8L(h); +} + +force_inline +i16x8 LoadRGBAL(i16x8 x) +{ + return Unpack8L(x); +} + +force_inline +i16x8 LoadRGBAH(i16x8 x) +{ + return Unpack8H(x); +} + +force_inline +void LoadRGBA4(const RGBA *s, i16x8& l, i16x8& h) +{ + i16x8 t4(s); + l = LoadRGBAL(t4); + h = LoadRGBAH(t4); +} + +force_inline +i8x16 PackRGBA(i16x8 l, i16x8 h) +{ + return Pack16(l, h); +} + +force_inline +void StoreRGBA(RGBA *rgba, i16x8 x) +{ + PackRGBA(x, 0).Store32(rgba); +} + +force_inline +void StoreRGBA2(RGBA *rgba, i16x8 x) +{ + PackRGBA(x, 0).Store64(rgba); +} + +force_inline +void StoreRGBA4(RGBA *rgba, i16x8 l, i16x8 h) +{ + PackRGBA(l, h).Store(rgba); +} + +force_inline +f32x4 LoadRGBAF(const RGBA *s) +{ + return ToFloat(Unpack16L(Unpack8L(i16x8().Load32(s)))); +} + +force_inline +void StoreRGBAF(RGBA *t, f32x4 s) +{ + Pack16(Pack32(Truncate(s))).Store32(t); +} + +force_inline +f32x4 ClampRGBAF(f32x4 p) +{return p; +#ifdef PLATFORM_MACOS + f32x4 alpha = Broadcast0(p); +#else + f32x4 alpha = Broadcast3(p); +#endif + alpha = min(alpha, f32all(255.0)); + return min(p, alpha); +} + +#else + force_inline __m128i LoadRGBA(const RGBA *s) { @@ -92,4 +190,6 @@ __m128 ClampRGBAF(__m128 p) return _mm_min_ps(p, alpha); } +#endif + #endif \ No newline at end of file diff --git a/uppsrc/Painter/AlphaBlend.h b/uppsrc/Painter/AlphaBlend.h index 32d5f5ed7..62cc2569b 100644 --- a/uppsrc/Painter/AlphaBlend.h +++ b/uppsrc/Painter/AlphaBlend.h @@ -9,6 +9,145 @@ void AlphaBlend(RGBA *t, const RGBA& c, int alpha, int len); #if defined(CPU_SSE2) && !defined(flagNOSIMD) +#ifdef CPU_SIMD + +force_inline +i16x8 BroadcastAlpha(i16x8 x) +{ +#ifdef PLATFORM_MACOS + return BroadcastLH0(x); +#else + return BroadcastLH3(x); +#endif +} + +force_inline +i16x8 Mul8(i16x8 x, int alpha) +{ + return i16all(alpha) * x >> 8; +} + +force_inline +i16x8 MakeAlpha(i16x8 x) +{ + x = BroadcastAlpha(x); +#ifdef PLATFORM_MACOS + x = i16x8(129, 129, 129, 128, 129, 129, 129, 128) * x >> 7; +#else + x = i16x8(128, 129, 129, 129, 128, 129, 129, 129) * x >> 7; // a for alpha, 256*a/255 for color +#endif + return i16all(256) - x; // 256 - a for alpha, 256 - 256*a/255 for color; +} + +force_inline +i16x8 AlphaBlendSSE2(i16x8 t, i16x8 s, i16x8 alpha) +{ + return s + (t * alpha >> 8); +} + +force_inline +void AlphaBlend1(RGBA *t, i16x8 s, i16x8 alpha) +{ + StoreRGBA(t, AlphaBlendSSE2(LoadRGBA(t), s, alpha)); +} + +force_inline +void AlphaBlend2(RGBA *t, i16x8 s, i16x8 alpha) +{ + StoreRGBA2(t, AlphaBlendSSE2(LoadRGBA2(t), s, alpha)); +} + +force_inline +void AlphaBlend4(RGBA *t, i16x8 sl, i16x8 al, i16x8 sh, i16x8 ah) +{ + i16x8 t4(t); + PackRGBA( + AlphaBlendSSE2(LoadRGBAL(t4), sl, al), + AlphaBlendSSE2(LoadRGBAH(t4), sh, ah) + ).Store(t); +} + +force_inline +void AlphaBlend(RGBA *t, const RGBA& c) +{ + i16x8 s = LoadRGBA(&c); + StoreRGBA(t, AlphaBlendSSE2(LoadRGBA(t), s, MakeAlpha(s))); +} + +force_inline +void AlphaBlend(RGBA *t, const RGBA& c, int alpha) +{ + i16x8 s = Mul8(LoadRGBA(&c), alpha); + StoreRGBA(t, AlphaBlendSSE2(LoadRGBA(t), s, MakeAlpha(s))); +} + +force_inline +void AlphaBlend(RGBA *t, const RGBA& c, int alpha, int len) +{ + i16x8 s = Mul8(LoadRGBA2(c), alpha); + i16x8 a = MakeAlpha(s); + while(len >= 4) { + AlphaBlend4(t, s, a, s, a); + t += 4; + len -= 4; + } + if(len & 2) { + AlphaBlend2(t, s, a); + t += 2; + } + if(len & 1) + AlphaBlend1(t, s, a); +} + +force_inline +void AlphaBlend(RGBA *t, const RGBA *s, int alpha, int len) +{ + if(alpha == 256) { + while(len >= 4) { + i16x8 m(s); + i16x8 s0 = LoadRGBAL(m); + i16x8 s1 = LoadRGBAH(m); + AlphaBlend4(t, s0, MakeAlpha(s0), s1, MakeAlpha(s1)); + t += 4; + s += 4; + len -= 4; + } + if(len & 2) { + i16x8 s0 = LoadRGBA2(s); + AlphaBlend2(t, s0, MakeAlpha(s0)); + t += 2; + s += 2; + } + if(len & 1) { + i16x8 s0 = LoadRGBA(s); + AlphaBlend1(t, s0, MakeAlpha(s0)); + } + } + else { + while(len >= 4) { + i16x8 m(s); + i16x8 s0 = Mul8(LoadRGBAL(m), alpha); + i16x8 s1 = Mul8(LoadRGBAH(m), alpha); + AlphaBlend4(t, s0, MakeAlpha(s0), s1, MakeAlpha(s1)); + t += 4; + s += 4; + len -= 4; + } + if(len & 2) { + i16x8 s0 = Mul8(LoadRGBA2(s), alpha); + AlphaBlend2(t, s0, MakeAlpha(s0)); + t += 2; + s += 2; + } + if(len & 1) { + i16x8 s0 = Mul8(LoadRGBA(s), alpha); + AlphaBlend1(t, s0, MakeAlpha(s0)); + } + } +} + +#else + force_inline __m128i BroadcastAlpha(__m128i x) { @@ -145,6 +284,8 @@ void AlphaBlend(RGBA *t, const RGBA *s, int alpha, int len) } } +#endif + #else force_inline diff --git a/uppsrc/Painter/Image.cpp b/uppsrc/Painter/Image.cpp index 3f06f0a28..2a45cf8b9 100644 --- a/uppsrc/Painter/Image.cpp +++ b/uppsrc/Painter/Image.cpp @@ -2,8 +2,183 @@ namespace Upp { -#ifdef CPU_X86 +#ifdef CPU_SIMD +force_inline +int IntAndFraction(f32x4 x, f32x4& fraction) +{ + x = x + f32all(8000); // Truncate truncates toward 0, need to fix negatives + i32x4 m = Truncate(x); + fraction = x - ToFloat(m); + return (int)m - 8000; +} + +force_inline +int Int(f32x4 x) +{ + return (int)Truncate(x + f32all(8000)) - 8000; +} + +struct PainterImageSpanData { + int ax, ay, cx, cy, maxx, maxy; + byte style; + byte hstyle, vstyle; + bool fast; + bool fixed; + Image image; + Xform2D xform; + + PainterImageSpanData(dword flags, const Xform2D& m, const Image& img, bool co, bool imagecache) { + style = byte(flags & 15); + hstyle = byte(flags & 3); + vstyle = byte(flags & 12); + fast = flags & FILL_FAST; + image = img; + int nx = 1; + int ny = 1; + if(!fast) { + Pointf sc = m.GetScaleXY(); + if(sc.x >= 0.01 && sc.y >= 0.01) { + nx = (int)max(1.0, 1.0 / sc.x); + ny = (int)max(1.0, 1.0 / sc.y); + } + } + if(nx == 1 && ny == 1) + xform = Inverse(m); + else { + if(!fast) + image = (imagecache ? MinifyCached : Minify)(image, nx, ny, co); + xform = Inverse(m) * Xform2D::Scale(1.0 / nx, 1.0 / ny); + } + cx = image.GetWidth(); + cy = image.GetHeight(); + maxx = cx - 1; + maxy = cy - 1; + ax = 6000000 / cx * cx * 2; + ay = 6000000 / cy * cy * 2; + } + + PainterImageSpanData() {} +}; + + +struct PainterImageSpan : SpanSource, PainterImageSpanData { + PainterImageSpan(const PainterImageSpanData& f) + : PainterImageSpanData(f) {} + + const RGBA *Pixel(int x, int y) { return &image[y][x]; } + + const RGBA *GetPixel(int x, int y) { + if(hstyle == FILL_HPAD) + x = minmax(x, 0, maxx); + else + if(hstyle == FILL_HREFLECT) + x = (x + ax) / cx & 1 ? (ax - x - 1) % cx : (x + ax) % cx; + else + if(hstyle == FILL_HREPEAT) + x = (x + ax) % cx; + if(vstyle == FILL_VPAD) + y = minmax(y, 0, maxy); + else + if(vstyle == FILL_VREFLECT) + y = (y + ay) / cy & 1 ? (ay - y - 1) % cy : (y + ay) % cy; + else + if(vstyle == FILL_VREPEAT) + y = (y + ay) % cy; + static RGBA zero; + return fixed || (x >= 0 && x < cx && y >= 0 && y < cy) ? &image[y][x] : &zero; + } + + virtual void Get(RGBA *span, int x, int y, unsigned len) + { + PAINTER_TIMING("ImageSpan::Get"); + + Pointf p0 = xform.Transform(Pointf(x, y)); + Pointf dd = xform.Transform(Pointf(x + 1, y)) - p0; + + f32x4 x0 = f32all(p0.x); + f32x4 y0 = f32all(p0.y); + f32x4 dx = f32all(dd.x); + f32x4 dy = f32all(dd.y); + f32x4 ii = 0; + f32x4 v1 = f32all(1); + f32x4 ix, iy; + + auto GetIXY = [&] { + ix = x0 + ii * dx; + iy = y0 + ii * dy; + ii += v1; + }; + + fixed = hstyle && vstyle; + if(hstyle + vstyle == 0 && fast) { + while(len--) { + GetIXY(); + Point l(Int(ix), Int(iy)); + if(l.x > 0 && l.x < maxx && l.y > 0 && l.y < maxy) + *span = *Pixel(l.x, l.y); + else + if(style == 0 && (l.x < -1 || l.x > cx || l.y < -1 || l.y > cy)) + *span = RGBAZero(); + else + *span = *GetPixel(l.x, l.y); + ++span; + } + return; + } + while(len--) { + GetIXY(); + f32x4 fx, fy; + Point l(IntAndFraction(ix, fx), IntAndFraction(iy, fy)); + if(hstyle == FILL_HREPEAT) + l.x = (l.x + ax) % cx; + if(vstyle == FILL_VREPEAT) + l.y = (l.y + ay) % cy; + if(style == 0 && (l.x < -1 || l.x > cx || l.y < -1 || l.y > cy)) + *span = RGBAZero(); + else + if(fast) { + if(l.x > 0 && l.x < maxx && l.y > 0 && l.y < maxy) + *span = *Pixel(l.x, l.y); + else + *span = *GetPixel(l.x, l.y); + } + else { + f32x4 p00, p01, p10, p11; + if(l.x > 0 && l.x < maxx && l.y > 0 && l.y < maxy) { + p00 = LoadRGBAF(Pixel(l.x + 0, l.y + 0)); + p01 = LoadRGBAF(Pixel(l.x + 0, l.y + 1)); + p10 = LoadRGBAF(Pixel(l.x + 1, l.y + 0)); + p11 = LoadRGBAF(Pixel(l.x + 1, l.y + 1)); + } + else { + p00 = LoadRGBAF(GetPixel(l.x + 0, l.y + 0)); + p01 = LoadRGBAF(GetPixel(l.x + 0, l.y + 1)); + p10 = LoadRGBAF(GetPixel(l.x + 1, l.y + 0)); + p11 = LoadRGBAF(GetPixel(l.x + 1, l.y + 1)); + } + + p01 = p01 * fy; + p11 = p11 * fy; + p10 = p10 * fx; + p11 = p11 * fx; + + fx = v1 - fx; + fy = v1 - fy; + + p00 = p00 * fy; + p10 = p10 * fy; + p00 = p00 * fx; + p01 = p01 * fx; + + StoreRGBAF(span, p00 + p01 + p10 + p11); + } + ++span; + } + } +}; + +#if 0 force_inline int IntAndFraction(__m128 x, __m128& fraction) { @@ -177,6 +352,7 @@ struct PainterImageSpan : SpanSource, PainterImageSpanData { } } }; +#endif void BufferPainter::RenderImage(double width, const Image& image, const Xform2D& transsrc, dword flags) {