github-starred/ultimatepp
2
0
Fork 0
mirror of https://github.com/ultimatepp/ultimatepp.git synced 2026-07-11 22:03:01 -06:00
Code Issues 33 Projects Packages Wiki Activity Actions

Core: SIMD

Browse source 
git-svn-id: svn://ultimatepp.org/upp/trunk@14650 f0d560ea-af0d-0410-9eb7-867de7ffcac7
This commit is contained in:
cxl 2020-07-03 07:29:31 +00:00
parent a864cffc19
commit 4d220b24de
11 changed files with 721 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

27
uppsrc/Core/Core.h
View file

@ -279,6 +279,26 @@ class JsonIO;
#include "Ops.h"
#include "Fn.h"
#ifdef flagNOSIMD
#ifdef CPU_SSE2
#undef CPU_SSE2
#endif
#ifdef CPU_NEON
#undef CPU_NEON
#endif
#endif
#ifdef CPU_SSE2
#include "SIMD_SSE2.h"
#define CPU_SIMD 1
#endif
#ifdef CPU_NEON
#include "SIMD_NEON.h"
#define CPU_SIMD 1
#endif
#include "Mem.h"
#include "Atomic.h"
#include "Topt.h"
@ -369,6 +389,13 @@ class JsonIO;
#include "ValueCache.h"
#ifdef CPU_SIMD
String AsString(const f32x4& x);
String AsString(const i32x4& x);
String AsString(const i16x8& x);
String AsString(const i8x16& x);
#endif
#ifdef PLATFORM_WIN32
NTL_MOVEABLE(POINT)
NTL_MOVEABLE(SIZE)

2
uppsrc/Core/Core.upp
View file

@ -34,6 +34,8 @@ file
Cpu.cpp,
Mem.h,
Mem.cpp,
SIMD_SSE2.h,
SIMD.cpp,
Atomic.h,
Mt.h,
Mt.cpp,

8
uppsrc/Core/Mem.cpp
View file

@ -4,11 +4,11 @@ namespace Upp {
#ifdef CPU_X86
void memset8__(void *p, __m128i data, size_t len)
void memset8__(void *p, i16x8 data, size_t len)
{
ASSERT(len >= 16);
byte *t = (byte *)p;
auto Set4 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), data); };
auto Set4 = [&](size_t at) { data.Store(t + at); };
Set4(len - 16); // fill tail
Set4(0); // align up on the next 16 bytes boundary
if(len <= 32)
@ -18,7 +18,7 @@ void memset8__(void *p, __m128i data, size_t len)
len = e - t;
e -= 128;
if(len >= 1024*1024) { // for really huge data, bypass the cache
auto Set4S = [&](int at) { _mm_stream_si128((__m128i *)(t + at), data); };
auto Set4S = [&](int at) { data.Stream(t + at); };
while(len >= 64) {
Set4S(0*16); Set4S(1*16); Set4S(2*16); Set4S(3*16);
t += 64;
@ -57,7 +57,7 @@ void memcpy8__(void *p, const void *q, size_t len)
return;
}
auto Copy128 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); };
auto Copy128 = [&](size_t at) { i16x8(s + at).Store(t + at); };
Copy128(len - 16); // copy tail
Copy128(0); // align target data up on the next 16 bytes boundary
if(len <= 32)

56
uppsrc/Core/Mem.h
View file

@ -1,6 +1,6 @@
#ifdef CPU_X86
#ifdef CPU_SIMD
void memset8__(void *t, __m128i data, size_t len);
void memset8__(void *t, i16x8 data, size_t len);
inline
void memset8(void *p, byte data, size_t len)
@ -17,7 +17,7 @@ void memset8(void *p, byte data, size_t len)
return;
}
if(len > 16) {
memset8__(t, _mm_set1_epi32(val4), len);
memset8__(t, i32all(val4), len);
return;
}
*(dword *)t = *(dword *)(t + len - 4) = val4;
@ -36,12 +36,12 @@ void memset16(void *p, word data, size_t len)
}
dword val4 = 0x10001 * data;
if(len >= 16) {
memset8__(t, _mm_set1_epi32(val4), 2 * len);
memset8__(t, i32all(val4), 2 * len);
return;
}
*(dword *)(t + len - 2) = val4;
if(len & 8) {
_mm_storeu_si128((__m128i *)t, _mm_set1_epi32(val4));
i32all(val4).Store(t);
t += 8;
}
if(len & 4) {
@ -66,19 +66,19 @@ void memset32(void *p, dword data, size_t len)
t[0] = data;
return;
}
__m128i val4 = _mm_set1_epi32(data);
i32x4 val4 = i32all(data);
if(len >= 16) {
memset8__(t, val4, 4 * len);
return;
}
auto Set4 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), val4); };
Set4(len - 4); // fill tail
auto Set128 = [&](size_t at) { val4.Store(t + at); };
Set128(len - 4); // fill tail
if(len & 8) {
Set4(0); Set4(4);
Set128(0); Set128(4);
t += 8;
}
if(len & 4)
Set4(0);
Set128(0);
}
inline
@ -90,12 +90,12 @@ void memset64(void *p, qword data, size_t len)
t[0] = data;
return;
}
__m128i val2 = _mm_set1_epi64x(data);
i16x8 val2 = i64all(data);
if(len >= 8) {
memset8__(t, val2, 8 * len);
return;
}
auto Set128 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), val2); };
auto Set128 = [&](size_t at) { val2.Store(t + at); };
Set128(len - 2); // fill tail
if(len & 4) {
Set128(0); Set128(2);
@ -109,7 +109,7 @@ inline
void memset128(void *t, m128 data, size_t len)
{
if(len)
memset8__(t, *(__m128i *)&data, 16 * len);
memset8__(t, i16x8().Load(&data), 16 * len);
}
void memcpy8__(void *p, const void *q, size_t len);
@ -163,7 +163,7 @@ void memcpy16(void *p, const void *q, size_t len)
*(uint64 *)(t + len - 4) = *(uint64 *)(s + len - 4);
return;
}
auto Copy128 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); };
auto Copy128 = [&](size_t at) { i16x8(s + at).Store(t + at); };
Copy128(0);
Copy128(len - 8);
return;
@ -204,7 +204,7 @@ void memcpy32(void *p, const void *q, size_t len)
}
#endif
auto Copy128 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); };
auto Copy128 = [&](size_t at) { i16x8(s + at).Store(t + at); };
if(len >= 16) {
memcpy8__(t, s, 4 * len);
@ -238,20 +238,20 @@ void memcpy64(void *p, const void *q, size_t len)
return;
}
auto Copy4 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); };
auto Copy128 = [&](size_t at) { i16x8(s + at).Store(t + at); };
Copy4(len - 2); // copy tail
Copy128(len - 2); // copy tail
if(len >= 8) {
memcpy8__(t, s, 8 * len);
return;
}
if(len & 4) {
Copy4(0); Copy4(2);
Copy128(0); Copy128(2);
t += 4;
s += 4;
}
if(len & 2)
Copy4(0);
Copy128(0);
}
inline
@ -264,24 +264,24 @@ void memcpy128(void *p, const void *q, size_t len)
dqword *t = (dqword *)p;
dqword *s = (dqword *)q;
auto Copy4 = [&](size_t at) { _mm_storeu_si128((__m128i *)(t + at), _mm_loadu_si128((__m128i *)(s + at))); };
auto Copy128 = [&](size_t at) { i16x8(s + at).Store(t + at); };
if(len >= 8) {
memcpy8__(t, s, 16 * len);
return;
}
if(len & 4) {
Copy4(0); Copy4(1); Copy4(2); Copy4(3);
Copy128(0); Copy128(1); Copy128(2); Copy128(3);
t += 4;
s += 4;
}
if(len & 2) {
Copy4(0); Copy4(1);
Copy128(0); Copy128(1);
t += 2;
s += 2;
}
if(len & 1)
Copy4(0);
Copy128(0);
}
template <class T>
@ -311,11 +311,9 @@ bool memeq8__(const void *p, const void *q, size_t len)
const byte *t = (byte *)p;
const byte *s = (byte *)q;
auto Cmp128 = [&](size_t at) { return _mm_cmpeq_epi32(_mm_loadu_si128((__m128i *)(s + at)), _mm_loadu_si128((__m128i *)(t + at))); };
auto Neq = [](__m128i v) { return _mm_movemask_epi8(v) != 0xffff; };
auto And = [](__m128i a, __m128i b) { return _mm_and_si128(a, b); };
auto Cmp128 = [&](size_t at) { return i16x8(s + at) == i16x8(t + at); };
if(Neq(And(Cmp128(len - 16), Cmp128(0)))) // tail & alignment, also <= 32
if(!Test(Cmp128(len - 16) & Cmp128(0))) // tail & alignment, also <= 32
return false;
if(len <= 32)
@ -329,13 +327,13 @@ bool memeq8__(const void *p, const void *q, size_t len)
len = e - t;
e -= 32;
while(t <= e) {
if(Neq(And(Cmp128(0), Cmp128(1*16))))
if(!Test(Cmp128(0) & Cmp128(1*16)))
return false;
s += 32;
t += 32;
}
if(len & 16)
if(Neq(Cmp128(0)))
if(!Test(Cmp128(0)))
return false;
return true;
}

35
uppsrc/Core/SIMD.cpp Normal file
View file

@ -0,0 +1,35 @@
#include "Core.h"
#ifdef CPU_SIMD
namespace Upp {
String AsString(const f32x4& x)
{
float *f = (float *)&x;
return Format("%g %g %g %g", f[3], f[2], f[1], f[0]);
}
String AsString(const i32x4& x)
{
int *f = (int *)&x;
return Format("%d %d %d %d", f[3], f[2], f[1], f[0]);
}
String AsString(const i16x8& x)
{
int16 *f = (int16 *)&x;
return Format("%d %d %d %d %d %d %d %d", f[7], f[6], f[5], f[4], f[3], f[2], f[1], f[0]);
}
String AsString(const i8x16& x)
{
int8 *f = (int8 *)&x;
return Format("%d %d %d %d . %d %d %d %d . %d %d %d %d . %d %d %d %d",
f[15], f[14], f[13], f[12], f[11], f[10], f[9], f[8],
f[7], f[6], f[5], f[4], f[3], f[2], f[1], f[0]);
}
};
#endif

175
uppsrc/Core/SIMD_SSE2.h Normal file
View file

@ -0,0 +1,175 @@
struct f32x4 {
__m128 data;
f32x4& Load(void *ptr) { data = _mm_loadu_ps((float *)ptr); return *this; }
f32x4& Load64(void *ptr) { data = _mm_castpd_ps(_mm_load_sd((double *)ptr)); return *this; }
f32x4& Load32(void *ptr) { data = _mm_load_ss((float *)ptr); return *this; }
void Store(void *ptr) { _mm_storeu_ps((float *)ptr, data); }
void Store64(void *ptr) { _mm_store_sd((double *)ptr, _mm_castps_pd(data)); }
void Store32(void *ptr) { _mm_store_ss((float *)ptr, data); }
f32x4() {}
f32x4(void *ptr) { Load(ptr); }
f32x4(__m128 d) { data = d; }
f32x4(double f) { data = _mm_set_ss((float)f); }
f32x4(float f) { data = _mm_set_ss(f); }
f32x4(int f) { data = _mm_set_ss((float)f); }
f32x4(double a, double b, double c, double d) { data = _mm_set_ps((float)a, (float)b, (float)c, (float)d); }
operator __m128() { return data; }
};
force_inline f32x4 f32all(double f) { return _mm_set1_ps((float)f); }
force_inline f32x4 operator+(f32x4 a, f32x4 b) { return _mm_add_ps(a.data, b.data); }
force_inline f32x4& operator+=(f32x4& a, f32x4 b) { return a = a + b; }
force_inline f32x4 operator-(f32x4 a, f32x4 b) { return _mm_sub_ps(a.data, b.data); }
force_inline f32x4& operator-=(f32x4& a, f32x4 b) { return a = a - b; }
force_inline f32x4 operator*(f32x4 a, f32x4 b) { return _mm_mul_ps(a.data, b.data); }
force_inline f32x4& operator*=(f32x4& a, f32x4 b) { return a = a * b; }
force_inline f32x4 operator/(f32x4 a, f32x4 b) { return _mm_div_ps(a.data, b.data); }
force_inline f32x4& operator/=(f32x4& a, f32x4 b) { return a = a + b; }
force_inline f32x4 operator==(f32x4 a, f32x4 b) { return _mm_cmpeq_ps(a.data, b.data); }
force_inline f32x4 operator!=(f32x4 a, f32x4 b) { return _mm_cmpneq_ps(a.data, b.data); }
force_inline f32x4 operator<(f32x4 a, f32x4 b) { return _mm_cmplt_ps(a.data, b.data); }
force_inline f32x4 operator>(f32x4 a, f32x4 b) { return _mm_cmpgt_ps(a.data, b.data); }
force_inline f32x4 operator<=(f32x4 a, f32x4 b) { return _mm_cmple_ps(a.data, b.data); }
force_inline f32x4 operator>=(f32x4 a, f32x4 b) { return _mm_cmpge_ps(a.data, b.data); }
force_inline bool Test(f32x4 a) { return _mm_movemask_ps(a.data) == 0xf; }
force_inline f32x4 min(f32x4 a, f32x4 b) { return _mm_min_ps(a.data, b.data); }
force_inline f32x4 max(f32x4 a, f32x4 b) { return _mm_max_ps(a.data, b.data); }
#define _MM_BCAST(a) _MM_SHUFFLE(a,a,a,a)
force_inline f32x4 Broadcast0(f32x4 a) { return _mm_shuffle_ps(a.data, a.data, _MM_BCAST(0)); }
force_inline f32x4 Broadcast1(f32x4 a) { return _mm_shuffle_ps(a.data, a.data, _MM_BCAST(1)); }
force_inline f32x4 Broadcast2(f32x4 a) { return _mm_shuffle_ps(a.data, a.data, _MM_BCAST(2)); }
force_inline f32x4 Broadcast3(f32x4 a) { return _mm_shuffle_ps(a.data, a.data, _MM_BCAST(3)); }
struct i16x8 { // 8xint16
__m128i data;
i16x8& Load(const void *ptr) { data = _mm_loadu_si128((__m128i *)ptr); return *this; }
i16x8& Load64(const void *ptr) { data = _mm_castpd_si128(_mm_load_sd((double *)ptr)); return *this; }
i16x8& Load32(const void *ptr) { data = _mm_castps_si128(_mm_load_ss((float *)ptr)); return *this; }
void Store(void *ptr) { _mm_storeu_si128((__m128i *)ptr, data); }
void Store64(void *ptr) { _mm_store_sd((double *)ptr, _mm_castsi128_pd(data)); }
void Store32(void *ptr) { _mm_store_ss((float *)ptr, _mm_castsi128_ps(data)); }
void Stream(void *ptr) { _mm_stream_si128((__m128i *)ptr, data); };
i16x8() {}
i16x8(const void *ptr) { Load(ptr); }
i16x8(__m128i d) { data = d; }
i16x8(int v) { data = _mm_set_epi16(0, 0, 0, 0, 0, 0, 0, v); }
i16x8(int a, int b, int c, int d, int e, int f, int g, int h) { data = _mm_set_epi16(a, b, c, d, e, f, g, h); }
operator __m128i() { return data; }
};
force_inline i16x8 i16all(int v) { return _mm_set1_epi16(v); }
force_inline i16x8 operator+(i16x8 a, i16x8 b) { return _mm_add_epi16(a.data, b.data); }
force_inline i16x8& operator+=(i16x8& a, i16x8 b) { return a = a + b; }
force_inline i16x8 operator-(i16x8 a, i16x8 b) { return _mm_sub_epi16(a.data, b.data); }
force_inline i16x8& operator-=(i16x8& a, i16x8 b) { return a = a - b; }
force_inline i16x8 operator*(i16x8 a, i16x8 b) { return _mm_mullo_epi16(a.data, b.data); }
force_inline i16x8& operator*=(i16x8& a, i16x8 b) { return a = a * b; }
force_inline i16x8 operator&(i16x8 a, i16x8 b) { return _mm_and_si128(a.data, b.data); }
force_inline i16x8& operator&=(i16x8& a, i16x8 b) { return a = a & b; }
force_inline i16x8 operator|(i16x8 a, i16x8 b) { return _mm_or_si128(a.data, b.data); }
force_inline i16x8& operator|=(i16x8& a, i16x8 b) { return a = a | b; }
force_inline i16x8 operator^(i16x8 a, i16x8 b) { return _mm_xor_si128(a.data, b.data); }
force_inline i16x8& operator^=(i16x8& a, i16x8 b) { return a = a ^ b; }
force_inline i16x8 operator~(i16x8 a) { return _mm_xor_si128(a.data, i16all(0xffff).data); }
force_inline i16x8 operator>>(i16x8 a, int b) { return _mm_srli_epi16(a.data, b); }
force_inline i16x8& operator>>=(i16x8& a, int b) { return a = a >> b; }
force_inline i16x8 operator<<(i16x8 a, int b) { return _mm_slli_epi16(a.data, b); }
force_inline i16x8& operator<<=(i16x8& a, int b) { return a = a << b; }
force_inline i16x8 operator==(i16x8 a, i16x8 b) { return _mm_cmpeq_epi16(a.data, b.data); }
force_inline i16x8 operator<(i16x8 a, i16x8 b) { return _mm_cmplt_epi16(a.data, b.data); }
force_inline i16x8 operator>(i16x8 a, i16x8 b) { return _mm_cmpgt_epi16(a.data, b.data); }
force_inline bool Test(i16x8 a) { return _mm_movemask_epi8(a.data) == 0xffff; }
struct i32x4 : i16x8 { // 4xint32
i32x4() {}
i32x4(void *ptr) { Load(ptr); }
i32x4(__m128i d) { data = d; }
i32x4(int v) { data = _mm_set_epi32(0, 0, 0, v); }
i32x4(int a, int b, int c, int d) { data = _mm_set_epi32(a, b, c, d); }
operator int() { return _mm_cvtsi128_si32(data); }
};
force_inline i32x4 i32all(int v) { return _mm_set1_epi32(v); }
force_inline i32x4 operator+(i32x4 a, i32x4 b) { return _mm_add_epi32(a.data, b.data); }
force_inline i32x4& operator+=(i32x4& a, i32x4 b) { return a = a + b; }
force_inline i32x4 operator-(i32x4 a, i32x4 b) { return _mm_sub_epi32(a.data, b.data); }
force_inline i32x4& operator-=(i32x4& a, i32x4 b) { return a = a - b; }
force_inline i32x4 operator&(i32x4 a, i32x4 b) { return _mm_and_si128(a.data, b.data); }
force_inline i32x4& operator&=(i32x4& a, i32x4 b) { return a = a & b; }
force_inline i32x4 operator|(i32x4 a, i32x4 b) { return _mm_or_si128(a.data, b.data); }
force_inline i32x4& operator|=(i32x4& a, i32x4 b) { return a = a | b; }
force_inline i32x4 operator^(i32x4 a, i32x4 b) { return _mm_xor_si128(a.data, b.data); }
force_inline i32x4& operator^=(i32x4& a, i32x4 b) { return a = a ^ b; }
force_inline i32x4 operator~(i32x4 a) { return _mm_xor_si128(a.data, i32all(0xffffffff).data); }
force_inline i32x4 operator>>(i32x4 a, int b) { return _mm_srli_epi32(a.data, b); }
force_inline i32x4& operator>>=(i32x4& a, int b) { return a = a >> b; }
force_inline i32x4 operator<<(i32x4 a, int b) { return _mm_slli_epi32(a.data, b); }
force_inline i32x4& operator<<=(i32x4& a, int b) { return a = a << b; }
force_inline i32x4 operator==(i32x4 a, i32x4 b) { return _mm_cmpeq_epi32(a.data, b.data); }
force_inline i32x4 operator<(i32x4 a, i32x4 b) { return _mm_cmplt_epi32(a.data, b.data); }
force_inline i32x4 operator>(i32x4 a, i32x4 b) { return _mm_cmpgt_epi32(a.data, b.data); }
force_inline bool Test(i32x4 a) { return _mm_movemask_epi8(a.data) == 0xffff; }
struct i8x16 : i16x8 { // 16xint8
i8x16() {}
i8x16(void *ptr) { Load(ptr); }
i8x16(__m128i d) { data = d; }
i8x16(int v) { data = _mm_set_epi8(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,v); }
i8x16(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k, int l, int m, int n, int o, int p)
{ data = _mm_set_epi8(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p); }
};
force_inline i8x16 i8all(int v) { return _mm_set1_epi8(v); }
force_inline i8x16 operator+(i8x16 a, i8x16 b) { return _mm_add_epi8(a.data, b.data); }
force_inline i8x16& operator+=(i8x16& a, i8x16 b) { return a = a + b; }
force_inline i8x16 operator-(i8x16 a, i8x16 b) { return _mm_sub_epi8(a.data, b.data); }
force_inline i8x16& operator-=(i8x16& a, i8x16 b) { return a = a - b; }
force_inline i8x16 operator&(i8x16 a, i8x16 b) { return _mm_and_si128(a.data, b.data); }
force_inline i8x16& operator&=(i8x16& a, i8x16 b) { return a = a & b; }
force_inline i8x16 operator|(i8x16 a, i8x16 b) { return _mm_or_si128(a.data, b.data); }
force_inline i8x16& operator|=(i8x16& a, i8x16 b) { return a = a | b; }
force_inline i8x16 operator^(i8x16 a, i8x16 b) { return _mm_xor_si128(a.data, b.data); }
force_inline i8x16& operator^=(i8x16& a, i8x16 b) { return a = a ^ b; }
force_inline i8x16 operator~(i8x16 a) { return _mm_xor_si128(a.data, i8all(0xff).data); }
force_inline f32x4 ToFloat(i32x4 a) { return _mm_cvtepi32_ps(a.data); }
force_inline i32x4 Truncate(f32x4 a) { return _mm_cvttps_epi32(a.data); }
force_inline i32x4 Round(f32x4 a) { return _mm_cvtps_epi32(a.data); }
force_inline i16x8 Unpack8L(i16x8 a) { return _mm_unpacklo_epi8(a.data, _mm_setzero_si128()); }
force_inline i16x8 Unpack8H(i16x8 a) { return _mm_unpackhi_epi8(a.data, _mm_setzero_si128()); }
force_inline i32x4 Unpack16L(i16x8 a) { return _mm_unpacklo_epi16(a.data, _mm_setzero_si128()); }
force_inline i32x4 Unpack16H(i16x8 a) { return _mm_unpackhi_epi16(a.data, _mm_setzero_si128()); }
force_inline i8x16 Pack16(i16x8 l, i16x8 h) { return _mm_packus_epi16(l.data, h.data); }
force_inline i8x16 Pack16(i16x8 l) { return _mm_packus_epi16(l.data, _mm_setzero_si128()); }
force_inline i16x8 Pack32(i32x4 a) { return _mm_packs_epi32(a.data, _mm_setzero_si128()); }
force_inline i16x8 BroadcastLH0(i16x8 a) { return _mm_shufflelo_epi16(_mm_shufflehi_epi16(a.data, _MM_BCAST(0)), _MM_BCAST(0)); }
force_inline i16x8 BroadcastLH1(i16x8 a) { return _mm_shufflelo_epi16(_mm_shufflehi_epi16(a.data, _MM_BCAST(1)), _MM_BCAST(1)); }
force_inline i16x8 BroadcastLH2(i16x8 a) { return _mm_shufflelo_epi16(_mm_shufflehi_epi16(a.data, _MM_BCAST(2)), _MM_BCAST(2)); }
force_inline i16x8 BroadcastLH3(i16x8 a) { return _mm_shufflelo_epi16(_mm_shufflehi_epi16(a.data, _MM_BCAST(3)), _MM_BCAST(3)); }
force_inline i16x8 i64all(qword data) { return _mm_set1_epi64x(data); }

50
uppsrc/Draw/Mify.cpp
View file

@ -2,7 +2,7 @@
namespace Upp {
#ifdef CPU_SSE2
#ifdef CPU_SIMD
Image Minify(const Image& img, int nx, int ny, bool co)
{
@ -11,11 +11,11 @@ Image Minify(const Image& img, int nx, int ny, bool co)
Size tsz = Size((ssz.cx + nx - 1) / nx, (ssz.cy + ny - 1) / ny);
ImageBuffer ib(tsz);
int scx0 = ssz.cx / nx * nx;
auto do_line = [&](int ty, __m128 *b, __m128 *div) {
memset(b, 0, tsz.cx * sizeof(__m128));
memset(div, 0, tsz.cx * sizeof(__m128));
__m128 v1 = _mm_set1_ps(1);
__m128 vnx = _mm_set1_ps((float)nx);
auto do_line = [&](int ty, f32x4 *b, f32x4 *div) {
memset(b, 0, tsz.cx * sizeof(f32x4));
memset(div, 0, tsz.cx * sizeof(f32x4));
f32x4 v1 = f32all(1);
f32x4 vnx = f32all(nx);
int yy = ny * ty;
for(int yi = 0; yi < ny; yi++) {
int y = yy + yi;
@ -23,44 +23,44 @@ Image Minify(const Image& img, int nx, int ny, bool co)
const RGBA *s = img[yy + yi];
const RGBA *e = s + scx0;
const RGBA *e2 = s + ssz.cx;
__m128 *t = b;
__m128 *d = div;
f32x4 *t = b;
f32x4 *d = div;
while(s < e) {
__m128 px = _mm_setzero_ps();
f32x4 px = 0;
for(int n = nx; n--;)
px = _mm_add_ps(px, LoadRGBAF(s++));
*t = _mm_add_ps(*t, px);
*d = _mm_add_ps(*d, vnx);
px += LoadRGBAF(s++);
*t += px;
*d += vnx;
t++;
d++;
}
if(s < e2) {
__m128 px = _mm_setzero_ps();
__m128 dv = _mm_setzero_ps();
f32x4 px = 0;
f32x4 dv = 0;
while(s < e2) {
px = _mm_add_ps(px, LoadRGBAF(s++));
dv = _mm_add_ps(px, v1);
px += LoadRGBAF(s++);
dv += v1;
}
*t = _mm_add_ps(*t, px);
*d = _mm_add_ps(*d, dv);
*t += px;
*d += dv;
t++;
d++;
}
ASSERT(t == b + tsz.cx);
}
}
__m128 *s = b;
__m128 *d = div;
f32x4 *s = b;
f32x4 *d = div;
RGBA *t = ~ib + ty * tsz.cx;
RGBA *e = t + tsz.cx;
while(t < e)
StoreRGBAF(t++, _mm_div_ps(*s++, *d++));
StoreRGBAF(t++, *s++ / *d++);
};
if(co) {
CoWork cw;
cw * [&] {
Buffer<__m128> div(tsz.cx);
Buffer<__m128> b(tsz.cx);
Buffer<f32x4> div(tsz.cx);
Buffer<f32x4> b(tsz.cx);
for(;;) {
int y = cw.Next();
if(y >= tsz.cy)
@ -70,8 +70,8 @@ Image Minify(const Image& img, int nx, int ny, bool co)
};
}
else {
Buffer<__m128> div(tsz.cx);
Buffer<__m128> b(tsz.cx);
Buffer<f32x4> div(tsz.cx);
Buffer<f32x4> b(tsz.cx);
for(int y = 0; y < tsz.cy; y++)
do_line(y, b, div);
}

16
uppsrc/Draw/RescaleFilter.cpp
View file

@ -91,23 +91,23 @@ Image RescaleFilter(const Image& img, Size sz, const Rect& sr,
}
RGBA *t = ib[y];
#ifdef CPU_X86
#ifdef CPU_SIMD
for(int x = 0; x < sz.cx; x++) {
__m128 rgbaf = _mm_setzero_ps();
__m128 w = _mm_setzero_ps();
f32x4 rgbaf = 0;
f32x4 w = 0;
yd = py;
int hasalpha = 0;
for(int yy = 2 * ay; yy-- > 0;) {
int ky = *yd++;
const RGBA *l = img[*yd++];
for(int xx = 2 * ax; xx-- > 0;) {
__m128 s = LoadRGBAF(&l[*xd++]);
__m128 weight = _mm_set1_ps(float(ky * *xd++));
rgbaf = _mm_add_ps(rgbaf, _mm_mul_ps(weight, s));
w = _mm_add_ps(w, weight);
f32x4 s = LoadRGBAF(&l[*xd++]);
f32x4 weight = f32all(float(ky * *xd++));
rgbaf += weight * s;
w += weight;
}
}
StoreRGBAF(t++, ClampRGBAF(_mm_div_ps(rgbaf, w)));
StoreRGBAF(t++, ClampRGBAF(rgbaf / w));
}
#else
for(int x = 0; x < sz.cx; x++) {

100
uppsrc/Draw/SSE2.h
View file

@ -1,5 +1,103 @@
#ifdef CPU_X86
#ifdef CPU_SIMD
force_inline
i16x8 LoadRGBA(const RGBA *s)
{
return Unpack8L(i32x4(*(dword *)s));
}
force_inline
i16x8 LoadRGBA2(const RGBA *s0, const RGBA *s1)
{
return Unpack8L(i32x4(0, 0, *(dword *)s1, *(dword *)s0));
}
force_inline
i16x8 LoadRGBA2(const RGBA& c)
{
return LoadRGBA2(&c, &c);
}
force_inline
i16x8 LoadRGBA2(const RGBA *s)
{
i16x8 h;
h.Load64(s);
return Unpack8L(h);
}
force_inline
i16x8 LoadRGBAL(i16x8 x)
{
return Unpack8L(x);
}
force_inline
i16x8 LoadRGBAH(i16x8 x)
{
return Unpack8H(x);
}
force_inline
void LoadRGBA4(const RGBA *s, i16x8& l, i16x8& h)
{
i16x8 t4(s);
l = LoadRGBAL(t4);
h = LoadRGBAH(t4);
}
force_inline
i8x16 PackRGBA(i16x8 l, i16x8 h)
{
return Pack16(l, h);
}
force_inline
void StoreRGBA(RGBA *rgba, i16x8 x)
{
PackRGBA(x, 0).Store32(rgba);
}
force_inline
void StoreRGBA2(RGBA *rgba, i16x8 x)
{
PackRGBA(x, 0).Store64(rgba);
}
force_inline
void StoreRGBA4(RGBA *rgba, i16x8 l, i16x8 h)
{
PackRGBA(l, h).Store(rgba);
}
force_inline
f32x4 LoadRGBAF(const RGBA *s)
{
return ToFloat(Unpack16L(Unpack8L(i16x8().Load32(s))));
}
force_inline
void StoreRGBAF(RGBA *t, f32x4 s)
{
Pack16(Pack32(Truncate(s))).Store32(t);
}
force_inline
f32x4 ClampRGBAF(f32x4 p)
{return p;
#ifdef PLATFORM_MACOS
f32x4 alpha = Broadcast0(p);
#else
f32x4 alpha = Broadcast3(p);
#endif
alpha = min(alpha, f32all(255.0));
return min(p, alpha);
}
#else
force_inline
__m128i LoadRGBA(const RGBA *s)
{
@ -92,4 +190,6 @@ __m128 ClampRGBAF(__m128 p)
return _mm_min_ps(p, alpha);
}
#endif
#endif

141
uppsrc/Painter/AlphaBlend.h
View file

@ -9,6 +9,145 @@ void AlphaBlend(RGBA *t, const RGBA& c, int alpha, int len);
#if defined(CPU_SSE2) && !defined(flagNOSIMD)
#ifdef CPU_SIMD
force_inline
i16x8 BroadcastAlpha(i16x8 x)
{
#ifdef PLATFORM_MACOS
return BroadcastLH0(x);
#else
return BroadcastLH3(x);
#endif
}
force_inline
i16x8 Mul8(i16x8 x, int alpha)
{
return i16all(alpha) * x >> 8;
}
force_inline
i16x8 MakeAlpha(i16x8 x)
{
x = BroadcastAlpha(x);
#ifdef PLATFORM_MACOS
x = i16x8(129, 129, 129, 128, 129, 129, 129, 128) * x >> 7;
#else
x = i16x8(128, 129, 129, 129, 128, 129, 129, 129) * x >> 7; // a for alpha, 256*a/255 for color
#endif
return i16all(256) - x; // 256 - a for alpha, 256 - 256*a/255 for color;
}
force_inline
i16x8 AlphaBlendSSE2(i16x8 t, i16x8 s, i16x8 alpha)
{
return s + (t * alpha >> 8);
}
force_inline
void AlphaBlend1(RGBA *t, i16x8 s, i16x8 alpha)
{
StoreRGBA(t, AlphaBlendSSE2(LoadRGBA(t), s, alpha));
}
force_inline
void AlphaBlend2(RGBA *t, i16x8 s, i16x8 alpha)
{
StoreRGBA2(t, AlphaBlendSSE2(LoadRGBA2(t), s, alpha));
}
force_inline
void AlphaBlend4(RGBA *t, i16x8 sl, i16x8 al, i16x8 sh, i16x8 ah)
{
i16x8 t4(t);
PackRGBA(
AlphaBlendSSE2(LoadRGBAL(t4), sl, al),
AlphaBlendSSE2(LoadRGBAH(t4), sh, ah)
).Store(t);
}
force_inline
void AlphaBlend(RGBA *t, const RGBA& c)
{
i16x8 s = LoadRGBA(&c);
StoreRGBA(t, AlphaBlendSSE2(LoadRGBA(t), s, MakeAlpha(s)));
}
force_inline
void AlphaBlend(RGBA *t, const RGBA& c, int alpha)
{
i16x8 s = Mul8(LoadRGBA(&c), alpha);
StoreRGBA(t, AlphaBlendSSE2(LoadRGBA(t), s, MakeAlpha(s)));
}
force_inline
void AlphaBlend(RGBA *t, const RGBA& c, int alpha, int len)
{
i16x8 s = Mul8(LoadRGBA2(c), alpha);
i16x8 a = MakeAlpha(s);
while(len >= 4) {
AlphaBlend4(t, s, a, s, a);
t += 4;
len -= 4;
}
if(len & 2) {
AlphaBlend2(t, s, a);
t += 2;
}
if(len & 1)
AlphaBlend1(t, s, a);
}
force_inline
void AlphaBlend(RGBA *t, const RGBA *s, int alpha, int len)
{
if(alpha == 256) {
while(len >= 4) {
i16x8 m(s);
i16x8 s0 = LoadRGBAL(m);
i16x8 s1 = LoadRGBAH(m);
AlphaBlend4(t, s0, MakeAlpha(s0), s1, MakeAlpha(s1));
t += 4;
s += 4;
len -= 4;
}
if(len & 2) {
i16x8 s0 = LoadRGBA2(s);
AlphaBlend2(t, s0, MakeAlpha(s0));
t += 2;
s += 2;
}
if(len & 1) {
i16x8 s0 = LoadRGBA(s);
AlphaBlend1(t, s0, MakeAlpha(s0));
}
}
else {
while(len >= 4) {
i16x8 m(s);
i16x8 s0 = Mul8(LoadRGBAL(m), alpha);
i16x8 s1 = Mul8(LoadRGBAH(m), alpha);
AlphaBlend4(t, s0, MakeAlpha(s0), s1, MakeAlpha(s1));
t += 4;
s += 4;
len -= 4;
}
if(len & 2) {
i16x8 s0 = Mul8(LoadRGBA2(s), alpha);
AlphaBlend2(t, s0, MakeAlpha(s0));
t += 2;
s += 2;
}
if(len & 1) {
i16x8 s0 = Mul8(LoadRGBA(s), alpha);
AlphaBlend1(t, s0, MakeAlpha(s0));
}
}
}
#else
force_inline
__m128i BroadcastAlpha(__m128i x)
{
@ -145,6 +284,8 @@ void AlphaBlend(RGBA *t, const RGBA *s, int alpha, int len)
}
}
#endif
#else
force_inline

178
uppsrc/Painter/Image.cpp
View file

@ -2,8 +2,183 @@
namespace Upp {
#ifdef CPU_X86
#ifdef CPU_SIMD
force_inline
int IntAndFraction(f32x4 x, f32x4& fraction)
{
x = x + f32all(8000); // Truncate truncates toward 0, need to fix negatives
i32x4 m = Truncate(x);
fraction = x - ToFloat(m);
return (int)m - 8000;
}
force_inline
int Int(f32x4 x)
{
return (int)Truncate(x + f32all(8000)) - 8000;
}
struct PainterImageSpanData {
int ax, ay, cx, cy, maxx, maxy;
byte style;
byte hstyle, vstyle;
bool fast;
bool fixed;
Image image;
Xform2D xform;
PainterImageSpanData(dword flags, const Xform2D& m, const Image& img, bool co, bool imagecache) {
style = byte(flags & 15);
hstyle = byte(flags & 3);
vstyle = byte(flags & 12);
fast = flags & FILL_FAST;
image = img;
int nx = 1;
int ny = 1;
if(!fast) {
Pointf sc = m.GetScaleXY();
if(sc.x >= 0.01 && sc.y >= 0.01) {
nx = (int)max(1.0, 1.0 / sc.x);
ny = (int)max(1.0, 1.0 / sc.y);
}
}
if(nx == 1 && ny == 1)
xform = Inverse(m);
else {
if(!fast)
image = (imagecache ? MinifyCached : Minify)(image, nx, ny, co);
xform = Inverse(m) * Xform2D::Scale(1.0 / nx, 1.0 / ny);
}
cx = image.GetWidth();
cy = image.GetHeight();
maxx = cx - 1;
maxy = cy - 1;
ax = 6000000 / cx * cx * 2;
ay = 6000000 / cy * cy * 2;
}
PainterImageSpanData() {}
};
struct PainterImageSpan : SpanSource, PainterImageSpanData {
PainterImageSpan(const PainterImageSpanData& f)
: PainterImageSpanData(f) {}
const RGBA *Pixel(int x, int y) { return &image[y][x]; }
const RGBA *GetPixel(int x, int y) {
if(hstyle == FILL_HPAD)
x = minmax(x, 0, maxx);
else
if(hstyle == FILL_HREFLECT)
x = (x + ax) / cx & 1 ? (ax - x - 1) % cx : (x + ax) % cx;
else
if(hstyle == FILL_HREPEAT)
x = (x + ax) % cx;
if(vstyle == FILL_VPAD)
y = minmax(y, 0, maxy);
else
if(vstyle == FILL_VREFLECT)
y = (y + ay) / cy & 1 ? (ay - y - 1) % cy : (y + ay) % cy;
else
if(vstyle == FILL_VREPEAT)
y = (y + ay) % cy;
static RGBA zero;
return fixed || (x >= 0 && x < cx && y >= 0 && y < cy) ? &image[y][x] : &zero;
}
virtual void Get(RGBA *span, int x, int y, unsigned len)
{
PAINTER_TIMING("ImageSpan::Get");
Pointf p0 = xform.Transform(Pointf(x, y));
Pointf dd = xform.Transform(Pointf(x + 1, y)) - p0;
f32x4 x0 = f32all(p0.x);
f32x4 y0 = f32all(p0.y);
f32x4 dx = f32all(dd.x);
f32x4 dy = f32all(dd.y);
f32x4 ii = 0;
f32x4 v1 = f32all(1);
f32x4 ix, iy;
auto GetIXY = [&] {
ix = x0 + ii * dx;
iy = y0 + ii * dy;
ii += v1;
};
fixed = hstyle && vstyle;
if(hstyle + vstyle == 0 && fast) {
while(len--) {
GetIXY();
Point l(Int(ix), Int(iy));
if(l.x > 0 && l.x < maxx && l.y > 0 && l.y < maxy)
*span = *Pixel(l.x, l.y);
else
if(style == 0 && (l.x < -1 || l.x > cx || l.y < -1 || l.y > cy))
*span = RGBAZero();
else
*span = *GetPixel(l.x, l.y);
++span;
}
return;
}
while(len--) {
GetIXY();
f32x4 fx, fy;
Point l(IntAndFraction(ix, fx), IntAndFraction(iy, fy));
if(hstyle == FILL_HREPEAT)
l.x = (l.x + ax) % cx;
if(vstyle == FILL_VREPEAT)
l.y = (l.y + ay) % cy;
if(style == 0 && (l.x < -1 || l.x > cx || l.y < -1 || l.y > cy))
*span = RGBAZero();
else
if(fast) {
if(l.x > 0 && l.x < maxx && l.y > 0 && l.y < maxy)
*span = *Pixel(l.x, l.y);
else
*span = *GetPixel(l.x, l.y);
}
else {
f32x4 p00, p01, p10, p11;
if(l.x > 0 && l.x < maxx && l.y > 0 && l.y < maxy) {
p00 = LoadRGBAF(Pixel(l.x + 0, l.y + 0));
p01 = LoadRGBAF(Pixel(l.x + 0, l.y + 1));
p10 = LoadRGBAF(Pixel(l.x + 1, l.y + 0));
p11 = LoadRGBAF(Pixel(l.x + 1, l.y + 1));
}
else {
p00 = LoadRGBAF(GetPixel(l.x + 0, l.y + 0));
p01 = LoadRGBAF(GetPixel(l.x + 0, l.y + 1));
p10 = LoadRGBAF(GetPixel(l.x + 1, l.y + 0));
p11 = LoadRGBAF(GetPixel(l.x + 1, l.y + 1));
}
p01 = p01 * fy;
p11 = p11 * fy;
p10 = p10 * fx;
p11 = p11 * fx;
fx = v1 - fx;
fy = v1 - fy;
p00 = p00 * fy;
p10 = p10 * fy;
p00 = p00 * fx;
p01 = p01 * fx;
StoreRGBAF(span, p00 + p01 + p10 + p11);
}
++span;
}
}
};
#if 0
force_inline
int IntAndFraction(__m128 x, __m128& fraction)
{
@ -177,6 +352,7 @@ struct PainterImageSpan : SpanSource, PainterImageSpanData {
}
}
};
#endif
void BufferPainter::RenderImage(double width, const Image& image, const Xform2D& transsrc, dword flags)
{