mirror of
https://github.com/ultimatepp/ultimatepp.git
synced 2026-05-15 14:16:07 -06:00
308c633a81
Obvious problem since this file is currently under Upp namespace. Also, it polutes all files that include Core/Core.h.
287 lines
16 KiB
C
287 lines
16 KiB
C
#include <arm_neon.h>
|
|
|
|
force_inline
|
|
uint64 cmask16__(uint16x8_t mask) {
|
|
uint8x8_t res = vshrn_n_u16(mask, 4);
|
|
uint64_t matches = vget_lane_u64(vreinterpret_u64_u8(res), 0);
|
|
return matches;
|
|
}
|
|
|
|
// force_inline uint64 cmask16__(uint16x8_t mask) { return vreinterpretq_s16_u16(mask); }
|
|
|
|
const uint64 cmask_all__ = 0xffffffffffffffffull;
|
|
|
|
struct f32x4 {
|
|
float32x4_t data;
|
|
|
|
f32x4& Load(const void *ptr) { data = vld1q_f32((float *)ptr); return *this; }
|
|
f32x4& Load64(const void *ptr) { data = vcombine_f32(vld1_f32((float *)ptr), vdup_n_f32(0)); return *this; }
|
|
f32x4& Load32(const void *ptr) { data = vsetq_lane_f32(*(float *)ptr, vdupq_n_f32(0), 0); return *this; }
|
|
|
|
void Store(void *ptr) { vst1q_f32((float32_t *)ptr, data); }
|
|
void Store64(void *ptr) { vst1_f32((float32_t *)ptr, vget_low_f32(data)); }
|
|
void Store32(void *ptr) { *(float32_t *)ptr = vgetq_lane_f32(data, 0); }
|
|
|
|
f32x4() {}
|
|
f32x4(const void *ptr) { Load(ptr); }
|
|
f32x4(float32x4_t d) { data = d; }
|
|
|
|
f32x4(double f) { data = vsetq_lane_f32((float)f, vdupq_n_f32(0), 0); }
|
|
f32x4(float f) { data = vsetq_lane_f32((float)f, vdupq_n_f32(0), 0); }
|
|
f32x4(int f) { data = vsetq_lane_f32((float)f, vdupq_n_f32(0), 0); }
|
|
f32x4(double a, double b, double c, double d) {
|
|
float __attribute__((aligned(16))) w[4] = { (float)d, (float)c, (float)b, (float)a };
|
|
data = vld1q_f32(w);
|
|
}
|
|
|
|
operator float32x4_t() { return data; }
|
|
};
|
|
|
|
force_inline f32x4 f32all(double f) { return vdupq_n_f32((float)f); }
|
|
|
|
force_inline f32x4 operator+(f32x4 a, f32x4 b) { return vaddq_f32(a, b); }
|
|
force_inline f32x4& operator+=(f32x4& a, f32x4 b) { return a = a + b; }
|
|
force_inline f32x4 operator-(f32x4 a, f32x4 b) { return vsubq_f32(a, b); }
|
|
force_inline f32x4& operator-=(f32x4& a, f32x4 b) { return a = a - b; }
|
|
force_inline f32x4 operator*(f32x4 a, f32x4 b) { return vmulq_f32(a, b); }
|
|
force_inline f32x4& operator*=(f32x4& a, f32x4 b) { return a = a * b; }
|
|
|
|
force_inline f32x4 operator/(f32x4 a, f32x4 b) {
|
|
float32x4_t reciprocal = vrecpeq_f32(b);
|
|
reciprocal = vmulq_f32(vrecpsq_f32(b, reciprocal), reciprocal);
|
|
reciprocal = vmulq_f32(vrecpsq_f32(b, reciprocal), reciprocal);
|
|
return vmulq_f32(a,reciprocal);
|
|
}
|
|
|
|
force_inline f32x4& operator/=(f32x4& a, f32x4 b) { return a = a / b; }
|
|
|
|
force_inline f32x4 operator==(f32x4 a, f32x4 b) { return vreinterpretq_f32_u32(vceqq_f32(a, b)); }
|
|
force_inline f32x4 operator!=(f32x4 a, f32x4 b) { return vreinterpretq_f32_u32(vmvnq_u32(vceqq_f32(a, b))); }
|
|
force_inline f32x4 operator<(f32x4 a, f32x4 b) { return vreinterpretq_f32_u32(vcltq_f32(a, b)); }
|
|
force_inline f32x4 operator>(f32x4 a, f32x4 b) { return vreinterpretq_f32_u32(vcgtq_f32(a, b)); }
|
|
force_inline f32x4 operator<=(f32x4 a, f32x4 b) { return vreinterpretq_f32_u32(vcleq_f32(a, b)); }
|
|
force_inline f32x4 operator>=(f32x4 a, f32x4 b) { return vreinterpretq_f32_u32(vcgeq_f32(a, b)); }
|
|
|
|
force_inline uint64 cmaskf__(float32x4_t mask) { return cmask16__(vreinterpretq_u16_f32(mask)); }
|
|
force_inline bool AllTrue(f32x4 a) { return cmaskf__(a.data) == cmask_all__; }
|
|
force_inline bool AnyTrue(f32x4 a) { return cmaskf__(a.data); }
|
|
force_inline int CountTrue(f32x4 a) { return CountBits64(cmaskf__(a.data)) >> 4; }
|
|
force_inline int FirstTrue(f32x4 a) { return CountTrailingZeroBits64(cmaskf__(a.data)) >> 4; }
|
|
force_inline int FirstFalse(f32x4 a) { return CountTrailingZeroBits64(~cmaskf__(a.data)) >> 4; }
|
|
force_inline bool IsTrue(f32x4 a, int i) { return cmaskf__(a.data) & ((uint64)1 << (i << 4)); }
|
|
|
|
force_inline f32x4 min(f32x4 a, f32x4 b) { return vminq_f32(a, b); }
|
|
force_inline f32x4 max(f32x4 a, f32x4 b) { return vmaxq_f32(a, b); }
|
|
|
|
force_inline f32x4 Broadcast0(f32x4 a) { return vdupq_n_f32(vgetq_lane_f32(a, 0)); }
|
|
force_inline f32x4 Broadcast1(f32x4 a) { return vdupq_n_f32(vgetq_lane_f32(a, 1)); }
|
|
force_inline f32x4 Broadcast2(f32x4 a) { return vdupq_n_f32(vgetq_lane_f32(a, 2)); }
|
|
force_inline f32x4 Broadcast3(f32x4 a) { return vdupq_n_f32(vgetq_lane_f32(a, 3)); }
|
|
|
|
struct i16x8 { // 8xint16
|
|
int16x8_t data;
|
|
|
|
i16x8& Load(const void *ptr) { data = vld1q_s16((int16_t *)ptr); return *this; }
|
|
i16x8& Load64(const void *ptr) { data = vcombine_s16(vld1_s16((int16_t *)ptr), vdup_n_s16(0)); return *this; }
|
|
i16x8& Load32(const void *ptr) { data = vreinterpretq_s16_s32(vsetq_lane_s32(*(int32_t *)ptr, vdupq_n_s32(0), 0)); return *this; }
|
|
|
|
void Store(void *ptr) { vst1q_s16((int16_t *)ptr, data); }
|
|
void Store64(void *ptr) { vst1_s16((int16_t *)ptr, vget_low_s16(data)); }
|
|
void Store32(void *ptr) { *(int32_t *)ptr = vgetq_lane_s32(vreinterpretq_s32_s16(data), 0); }
|
|
|
|
i16x8() {}
|
|
i16x8(const void *ptr) { Load(ptr); }
|
|
i16x8(int16x8_t d) { data = d; }
|
|
i16x8(int8x16_t d) { data = vreinterpretq_s16_s8(d); }
|
|
i16x8(int32x4_t d) { data = vreinterpretq_s16_s32(d); }
|
|
i16x8(int v) { data = vsetq_lane_s16(v, vdupq_n_s16(0), 0); }
|
|
i16x8(int a, int b, int c, int d, int e, int f, int g, int h) {
|
|
int16_t __attribute__((aligned(16))) val[8] = { (int16_t)h, (int16_t)g, (int16_t)f, (int16_t)e, (int16_t)d, (int16_t)c, (int16_t)b, (int16_t)a };
|
|
data = vld1q_s16(val);
|
|
}
|
|
|
|
operator int16x8_t() { return data; }
|
|
};
|
|
|
|
|
|
force_inline i16x8 i16all(int v) { return vdupq_n_s16(v); }
|
|
|
|
force_inline i16x8 operator+(i16x8 a, i16x8 b) { return vaddq_s16(a, b); }
|
|
force_inline i16x8& operator+=(i16x8& a, i16x8 b) { return a = a + b; }
|
|
force_inline i16x8 operator-(i16x8 a, i16x8 b) { return vsubq_s16(a, b); }
|
|
force_inline i16x8& operator-=(i16x8& a, i16x8 b) { return a = a - b; }
|
|
force_inline i16x8 operator*(i16x8 a, i16x8 b) { return vmulq_s16(a, b); }
|
|
force_inline i16x8& operator*=(i16x8& a, i16x8 b) { return a = a * b; }
|
|
|
|
force_inline i16x8 operator&(i16x8 a, i16x8 b) { return vandq_s16(a, b); }
|
|
force_inline i16x8& operator&=(i16x8& a, i16x8 b) { return a = a & b; }
|
|
force_inline i16x8 operator|(i16x8 a, i16x8 b) { return vorrq_s16(a, b); }
|
|
force_inline i16x8& operator|=(i16x8& a, i16x8 b) { return a = a | b; }
|
|
force_inline i16x8 operator^(i16x8 a, i16x8 b) { return veorq_s16(a, b); }
|
|
force_inline i16x8& operator^=(i16x8& a, i16x8 b) { return a = a ^ b; }
|
|
force_inline i16x8 operator~(i16x8 a) { return vmvnq_s16(a); }
|
|
|
|
force_inline i16x8 operator>>(i16x8 a, int b) { return vreinterpretq_s16_u16(vshlq_u16(vreinterpretq_u16_s16(a), vdupq_n_s16(-b))); }
|
|
force_inline i16x8& operator>>=(i16x8& a, int b) { return a = a >> b; }
|
|
force_inline i16x8 operator<<(i16x8 a, int b) { return vshlq_s16(a, vdupq_n_s16(b)); }
|
|
force_inline i16x8& operator<<=(i16x8& a, int b) { return a = a << b; }
|
|
|
|
force_inline i16x8 operator==(i16x8 a, i16x8 b) { return vreinterpretq_s16_u16(vceqq_s16(a, b)); }
|
|
force_inline i16x8 operator<(i16x8 a, i16x8 b) { return vreinterpretq_s16_u16(vcltq_s16(a, b)); }
|
|
force_inline i16x8 operator>(i16x8 a, i16x8 b) { return vreinterpretq_s16_u16(vcgtq_s16(a, b)); }
|
|
|
|
force_inline bool AllTrue(i16x8 a) { return cmask16__(a.data) == cmask_all__; }
|
|
force_inline bool AnyTrue(i16x8 a) { return cmask16__(a.data); }
|
|
force_inline int CountTrue(i16x8 a) { return CountBits64(cmask16__(a.data)) >> 3; }
|
|
force_inline int FirstTrue(i16x8 a) { return CountTrailingZeroBits64(cmask16__(a.data)) >> 3; }
|
|
force_inline int FirstFalse(i16x8 a) { return CountTrailingZeroBits64(~cmask16__(a.data)) >> 3; }
|
|
force_inline bool IsTrue(i16x8 a, int i) { return cmask16__(a.data) & ((uint64)1 << (i << 3)); }
|
|
|
|
struct i32x4 { // 4xint32
|
|
int32x4_t data;
|
|
|
|
i32x4& Load(const void *ptr) { data = vld1q_s32((int32_t *)ptr); return *this; }
|
|
i32x4& Load64(const void *ptr) { data = vcombine_s32(vld1_s32((int32_t *)ptr), vdup_n_s32(0)); return *this; }
|
|
i32x4& Load32(const void *ptr) { data = vsetq_lane_s32(*(int32_t *)ptr, vdupq_n_s32(0), 0); return *this; }
|
|
|
|
void Store(void *ptr) { vst1q_s32((int32_t *)ptr, data); }
|
|
void Store64(void *ptr) { vst1_s32((int32_t *)ptr, vget_low_s32(data)); }
|
|
void Store32(void *ptr) { *(int32_t *)ptr = vgetq_lane_s32(data, 0); }
|
|
|
|
i32x4() {}
|
|
i32x4(const void *ptr) { Load(ptr); }
|
|
i32x4(int32x4_t d) { data = d; }
|
|
i32x4(int8x16_t d) { data = vreinterpretq_s32_s8(d); }
|
|
i32x4(int16x8_t d) { data = vreinterpretq_s32_s16(d); }
|
|
i32x4(int v) { data = vsetq_lane_s32(v, vdupq_n_s32(0), 0); }
|
|
i32x4(int a, int b, int c, int d) {
|
|
int32_t __attribute__((aligned(16))) val[4] = { (int32_t)d, (int32_t)c, (int32_t)b, (int32_t)a };
|
|
data = vld1q_s32(val);
|
|
}
|
|
operator int32x4_t() { return data; }
|
|
operator int() { return vgetq_lane_s32(data, 0); }
|
|
operator i16x8() const { return i16x8(data); }
|
|
};
|
|
|
|
force_inline i32x4 i32all(int v) { return vdupq_n_s32(v); }
|
|
|
|
force_inline i32x4 operator+(i32x4 a, i32x4 b) { return vaddq_s32(a, b); }
|
|
force_inline i32x4& operator+=(i32x4& a, i32x4 b) { return a = a + b; }
|
|
force_inline i32x4 operator-(i32x4 a, i32x4 b) { return vsubq_s32(a, b); }
|
|
force_inline i32x4& operator-=(i32x4& a, i32x4 b) { return a = a - b; }
|
|
|
|
force_inline i32x4 operator&(i32x4 a, i32x4 b) { return vandq_s32(a, b); }
|
|
force_inline i32x4& operator&=(i32x4& a, i32x4 b) { return a = a & b; }
|
|
force_inline i32x4 operator|(i32x4 a, i32x4 b) { return vorrq_s32(a, b); }
|
|
force_inline i32x4& operator|=(i32x4& a, i32x4 b) { return a = a | b; }
|
|
force_inline i32x4 operator^(i32x4 a, i32x4 b) { return veorq_s32(a, b); }
|
|
force_inline i32x4& operator^=(i32x4& a, i32x4 b) { return a = a ^ b; }
|
|
force_inline i32x4 operator~(i32x4 a) { return vmvnq_s32(a); }
|
|
|
|
force_inline i32x4 operator>>(i32x4 a, int b) { return vshlq_s32(a, vdupq_n_s32(-b)); }
|
|
force_inline i32x4& operator>>=(i32x4& a, int b) { return a = a >> b; }
|
|
force_inline i32x4 operator<<(i32x4 a, int b) { return vshlq_s32(a, vdupq_n_s32(b)); }
|
|
force_inline i32x4& operator<<=(i32x4& a, int b) { return a = a << b; }
|
|
|
|
force_inline i32x4 operator==(i32x4 a, i32x4 b) { return vreinterpretq_s32_u32(vceqq_s32(a, b)); }
|
|
force_inline i32x4 operator<(i32x4 a, i32x4 b) { return vreinterpretq_s32_u32(vcltq_s32(a, b)); }
|
|
force_inline i32x4 operator>(i32x4 a, i32x4 b) { return vreinterpretq_s32_u32(vcgtq_s32(a, b)); }
|
|
|
|
force_inline uint64 cmask32__(uint32x4_t mask) { return cmask16__(vreinterpretq_u16_u32(mask)); }
|
|
force_inline bool AllTrue(i32x4 a) { return cmask32__(a.data) == cmask_all__; }
|
|
force_inline bool AnyTrue(i32x4 a) { return cmask32__(a.data); }
|
|
force_inline int CountTrue(i32x4 a) { return CountBits64(cmask32__(a.data)) >> 4; }
|
|
force_inline int FirstTrue(i32x4 a) { return CountTrailingZeroBits64(cmask32__(a.data)) >> 4; }
|
|
force_inline int FirstFalse(i32x4 a) { return CountTrailingZeroBits64(~cmask32__(a.data)) >> 4; }
|
|
force_inline bool IsTrue(i32x4 a, int i) { return cmask32__(a.data) & ((uint64)1 << (i << 4)); }
|
|
|
|
struct i8x16 { // 16*int8
|
|
int8x16_t data;
|
|
|
|
i8x16& Load(const void *ptr) { data = vld1q_s8((int8_t *)ptr); return *this; }
|
|
i8x16& Load64(const void *ptr) { data = vcombine_s8(vld1_s8((int8_t *)ptr), vdup_n_s8(0)); return *this; }
|
|
i8x16& Load32(const void *ptr) { data = vreinterpretq_s8_s32(vsetq_lane_s32(*(int32_t *)ptr, vdupq_n_s32(0), 0)); return *this; }
|
|
|
|
void Store(void *ptr) { vst1q_s8((int8_t *)ptr, data); }
|
|
void Store64(void *ptr) { vst1_s32((int32_t *)ptr, vget_low_s32(vreinterpretq_s32_s8(data))); }
|
|
void Store32(void *ptr) { *(int32_t *)ptr = vgetq_lane_s32(vreinterpretq_s32_s8(data), 0); }
|
|
|
|
i8x16() {}
|
|
i8x16(const void *ptr) { Load(ptr); }
|
|
i8x16(int8x16_t d) { data = d; }
|
|
i8x16(int16x8_t d) { data = vreinterpretq_s8_s16(d); }
|
|
i8x16(int32x4_t d) { data = vreinterpretq_s8_s32(d); }
|
|
i8x16(int v) { data = vsetq_lane_s8(v, vdupq_n_s8(0), 0); }
|
|
i8x16(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k, int l, int m, int n, int o, int p)
|
|
{
|
|
int8_t __attribute__((aligned(16))) val[16] = {
|
|
(int8_t)p, (int8_t)o, (int8_t)n, (int8_t)m,
|
|
(int8_t)l, (int8_t)k, (int8_t)j, (int8_t)i,
|
|
(int8_t)h, (int8_t)g, (int8_t)f, (int8_t)e,
|
|
(int8_t)d, (int8_t)c, (int8_t)b, (int8_t)a,
|
|
};
|
|
data = vld1q_s8(val);
|
|
}
|
|
operator int8x16_t() const { return data; }
|
|
operator i16x8() const { return i16x8(data); }
|
|
};
|
|
|
|
force_inline i8x16 i8all(int v) { return vdupq_n_s8(v); }
|
|
|
|
force_inline i8x16 operator+(i8x16 a, i8x16 b) { return vaddq_s8(a, b); }
|
|
force_inline i8x16& operator+=(i8x16& a, i8x16 b) { return a = a + b; }
|
|
force_inline i8x16 operator-(i8x16 a, i8x16 b) { return vsubq_s8(a, b); }
|
|
force_inline i8x16& operator-=(i8x16& a, i8x16 b) { return a = a - b; }
|
|
|
|
force_inline i8x16 operator&(i8x16 a, i8x16 b) { return vandq_s8(a, b); }
|
|
force_inline i8x16& operator&=(i8x16& a, i8x16 b) { return a = a & b; }
|
|
force_inline i8x16 operator|(i8x16 a, i8x16 b) { return vorrq_s8(a, b); }
|
|
force_inline i8x16& operator|=(i8x16& a, i8x16 b) { return a = a | b; }
|
|
force_inline i8x16 operator^(i8x16 a, i8x16 b) { return veorq_s8(a, b); }
|
|
force_inline i8x16& operator^=(i8x16& a, i8x16 b) { return a = a ^ b; }
|
|
force_inline i8x16 operator~(i8x16 a) { return vmvnq_s8(a); }
|
|
|
|
force_inline i8x16 operator==(i8x16 a, i8x16 b) { return vreinterpretq_s8_u8(vceqq_s8(a, b)); }
|
|
force_inline i8x16 operator<(i8x16 a, i8x16 b) { return vreinterpretq_s8_u8(vcltq_s8(a, b)); }
|
|
force_inline i8x16 operator>(i8x16 a, i8x16 b) { return vreinterpretq_s8_u8(vcgtq_s8(a, b)); }
|
|
|
|
force_inline uint64 cmask8__(uint8x16_t mask) { return cmask16__(vreinterpretq_u16_u8(mask)); }
|
|
force_inline bool AllTrue(i8x16 a) { return cmask8__(a.data) == cmask_all__; }
|
|
force_inline bool AnyTrue(i8x16 a) { return cmask8__(a.data); }
|
|
force_inline int CountTrue(i8x16 a) { return CountBits64(cmask8__(a.data)) >> 2; }
|
|
force_inline int FirstTrue(i8x16 a) { return CountTrailingZeroBits64(cmask8__(a.data)) >> 2; }
|
|
force_inline int FirstFalse(i8x16 a) { return CountTrailingZeroBits64(~cmask8__(a.data)) >> 2; }
|
|
force_inline bool IsTrue(i8x16 a, int i) { return cmask8__(a.data) & ((uint64)1 << (i << 2)); }
|
|
|
|
force_inline f32x4 ToFloat(i32x4 a) { return vcvtq_f32_s32(a); }
|
|
force_inline i32x4 Truncate(f32x4 a) { return vcvtq_s32_f32(a); }
|
|
|
|
force_inline i16x8 Unpack8L(i16x8 a) { return vzipq_s8(vreinterpretq_s8_s16(a), vdupq_n_s8(0)).val[0]; }
|
|
force_inline i16x8 Unpack8H(i16x8 a) { return vzipq_s8(vreinterpretq_s8_s16(a), vdupq_n_s8(0)).val[1]; }
|
|
|
|
force_inline i32x4 Unpack16L(i16x8 a) { return vzipq_s16(a, vdupq_n_s16(0)).val[0]; }
|
|
force_inline i32x4 Unpack16H(i16x8 a) { return vzipq_s16(a, vdupq_n_s16(0)).val[1]; }
|
|
|
|
force_inline i8x16 Pack16(i16x8 l, i16x8 h) { return vreinterpretq_s8_u8(vcombine_u8(vqmovun_s16(l), vqmovun_s16(h))); }
|
|
force_inline i8x16 Pack16(i16x8 l) { return vreinterpretq_s8_u8(vcombine_u8(vqmovun_s16(l), vdup_n_u8(0))); }
|
|
|
|
force_inline i16x8 Pack32(i32x4 a) { return vcombine_s16(vqmovn_s32(a), vdup_n_s16(0)); }
|
|
|
|
force_inline i16x8 BroadcastLH0(i16x8 a) {
|
|
return vcombine_s16(vdup_n_s16(vgetq_lane_s16(a, 0)), vdup_n_s16(vgetq_lane_s16(a, 4)));
|
|
}
|
|
|
|
force_inline i16x8 BroadcastLH1(i16x8 a) {
|
|
return vcombine_s16(vdup_n_s16(vgetq_lane_s16(a, 1)), vdup_n_s16(vgetq_lane_s16(a, 5)));
|
|
}
|
|
|
|
force_inline i16x8 BroadcastLH2(i16x8 a) {
|
|
return vcombine_s16(vdup_n_s16(vgetq_lane_s16(a, 2)), vdup_n_s16(vgetq_lane_s16(a, 6)));
|
|
}
|
|
|
|
force_inline i16x8 BroadcastLH3(i16x8 a) {
|
|
return vcombine_s16(vdup_n_s16(vgetq_lane_s16(a, 3)), vdup_n_s16(vgetq_lane_s16(a, 7)));
|
|
}
|
|
|
|
force_inline i16x8 i64all(qword data) { return vreinterpretq_s16_u64(vdupq_n_u64(data)); }
|
|
|