[Bf-blender-cvs] [d2f0cb67457] master: BLI: Unroll vector loops for better performance on GCC
Hans Goudey
noreply at git.blender.org
Tue Oct 4 18:17:11 CEST 2022
Commit: d2f0cb67457294c0e6479314464e7ec796a39d65
Author: Hans Goudey
Date: Tue Oct 4 11:16:09 2022 -0500
Branches: master
https://developer.blender.org/rBd2f0cb67457294c0e6479314464e7ec796a39d65
BLI: Unroll vector loops for better performance on GCC
On GCC, the loops created by `BLI_VEC_OP_IMPL` were not always
unrolled, leading to branching. For `attribute_math::mix4<float3>`,
this lead to a significant performance regression compared to its
older `interp_v3_v3v3v3v3` counterpart.
Instead of a using macros to create the for loops, use variadic
templates to manually unroll them. The compiler might do it anyway
(I didn't observe any effect on Clang in my tests), but there should
be no reason not to unroll these small loops, and making it explicit
and removing use of macros seems better.
On a Ryzen 3700x, this commits doubles the performance of Catmull
Rom curve position evaluation (from 18-19ms to around 9-10ms).
Differential Revision: https://developer.blender.org/D16136
===================================================================
M source/blender/blenlib/BLI_math_vec_types.hh
===================================================================
diff --git a/source/blender/blenlib/BLI_math_vec_types.hh b/source/blender/blenlib/BLI_math_vec_types.hh
index 5916193894b..0387d0b0440 100644
--- a/source/blender/blenlib/BLI_math_vec_types.hh
+++ b/source/blender/blenlib/BLI_math_vec_types.hh
@@ -40,6 +40,21 @@ template<typename T> struct vec_struct_base<T, 4> {
T x, y, z, w;
};
+template<class Fn, size_t... I> void unroll_impl(Fn fn, std::index_sequence<I...> /*indices*/)
+{
+ (fn(I), ...);
+}
+
+/**
+ * Variadic templates are used to unroll loops manually. This helps GCC avoid branching during math
+ * operations and makes the code generation more explicit and predictable. Unrolling should always
+ * be worth it because the vector size is expected to be small.
+ */
+template<int N, class Fn> void unroll(Fn fn)
+{
+ unroll_impl(fn, std::make_index_sequence<N>());
+}
+
namespace math {
template<typename T> uint64_t vector_hash(const T &vec)
@@ -181,17 +196,13 @@ template<typename T, int Size> struct vec_base : public vec_struct_base<T, Size>
vec_base(const T *ptr)
{
- for (int i = 0; i < Size; i++) {
- (*this)[i] = ptr[i];
- }
+ unroll<Size>([&](auto i) { (*this)[i] = ptr[i]; });
}
template<typename U, BLI_ENABLE_IF((std::is_convertible_v<U, T>))>
explicit vec_base(const U *ptr)
{
- for (int i = 0; i < Size; i++) {
- (*this)[i] = ptr[i];
- }
+ unroll<Size>([&](auto i) { (*this)[i] = ptr[i]; });
}
vec_base(const T (*ptr)[Size]) : vec_base(static_cast<const T *>(ptr[0]))
@@ -202,9 +213,7 @@ template<typename T, int Size> struct vec_base : public vec_struct_base<T, Size>
template<typename U> explicit vec_base(const vec_base<U, Size> &vec)
{
- for (int i = 0; i < Size; i++) {
- (*this)[i] = T(vec[i]);
- }
+ unroll<Size>([&](auto i) { (*this)[i] = T(vec[i]); });
}
/** C-style pointer dereference. */
@@ -239,29 +248,20 @@ template<typename T, int Size> struct vec_base : public vec_struct_base<T, Size>
#define BLI_INT_OP(_T) template<typename U = _T, BLI_ENABLE_IF((std::is_integral_v<U>))>
-#define BLI_VEC_OP_IMPL(_result, _i, _op) \
- vec_base _result; \
- for (int _i = 0; _i < Size; _i++) { \
- _op; \
- } \
- return _result;
-
-#define BLI_VEC_OP_IMPL_SELF(_i, _op) \
- for (int _i = 0; _i < Size; _i++) { \
- _op; \
- } \
- return *this;
-
/** Arithmetic operators. */
friend vec_base operator+(const vec_base &a, const vec_base &b)
{
- BLI_VEC_OP_IMPL(ret, i, ret[i] = a[i] + b[i]);
+ vec_base result;
+ unroll<Size>([&](auto i) { result[i] = a[i] + b[i]; });
+ return result;
}
friend vec_base operator+(const vec_base &a, const T &b)
{
- BLI_VEC_OP_IMPL(ret, i, ret[i] = a[i] + b);
+ vec_base result;
+ unroll<Size>([&](auto i) { result[i] = a[i] + b; });
+ return result;
}
friend vec_base operator+(const T &a, const vec_base &b)
@@ -271,52 +271,69 @@ template<typename T, int Size> struct vec_base : public vec_struct_base<T, Size>
vec_base &operator+=(const vec_base &b)
{
- BLI_VEC_OP_IMPL_SELF(i, (*this)[i] += b[i]);
+ unroll<Size>([&](auto i) { (*this)[i] += b[i]; });
+ return *this;
}
vec_base &operator+=(const T &b)
{
- BLI_VEC_OP_IMPL_SELF(i, (*this)[i] += b);
+ vec_base result;
+ unroll<Size>([&](auto i) { (*this)[i] += b; });
+ return result;
}
friend vec_base operator-(const vec_base &a)
{
- BLI_VEC_OP_IMPL(ret, i, ret[i] = -a[i]);
+ vec_base result;
+ unroll<Size>([&](auto i) { result[i] = -a[i]; });
+ return result;
}
friend vec_base operator-(const vec_base &a, const vec_base &b)
{
- BLI_VEC_OP_IMPL(ret, i, ret[i] = a[i] - b[i]);
+ vec_base result;
+ unroll<Size>([&](auto i) { result[i] = a[i] - b[i]; });
+ return result;
}
friend vec_base operator-(const vec_base &a, const T &b)
{
- BLI_VEC_OP_IMPL(ret, i, ret[i] = a[i] - b);
+ vec_base result;
+ unroll<Size>([&](auto i) { result[i] = a[i] - b; });
+ return result;
}
friend vec_base operator-(const T &a, const vec_base &b)
{
- BLI_VEC_OP_IMPL(ret, i, ret[i] = a - b[i]);
+ vec_base result;
+ unroll<Size>([&](auto i) { result[i] = a - b[i]; });
+ return result;
}
vec_base &operator-=(const vec_base &b)
{
- BLI_VEC_OP_IMPL_SELF(i, (*this)[i] -= b[i]);
+ unroll<Size>([&](auto i) { (*this)[i] -= b[i]; });
+ return *this;
}
vec_base &operator-=(const T &b)
{
- BLI_VEC_OP_IMPL_SELF(i, (*this)[i] -= b);
+ unroll<Size>([&](auto i) { (*this)[i] -= b; });
+ return *this;
}
friend vec_base operator*(const vec_base &a, const vec_base &b)
{
- BLI_VEC_OP_IMPL(ret, i, ret[i] = a[i] * b[i]);
+ vec_base result;
+ unroll<Size>([&](auto i) { result[i] = a[i] * b[i]; });
+ return result;
}
template<typename FactorT> friend vec_base operator*(const vec_base &a, FactorT b)
{
- BLI_VEC_OP_IMPL(ret, i, ret[i] = a[i] * b);
+ vec_base result;
+ unroll<Size>([&](auto i) { result[i] = a[i] * b; });
+ return result;
}
friend vec_base operator*(T a, const vec_base &b)
@@ -326,12 +343,14 @@ template<typename T, int Size> struct vec_base : public vec_struct_base<T, Size>
vec_base &operator*=(T b)
{
- BLI_VEC_OP_IMPL_SELF(i, (*this)[i] *= b);
+ unroll<Size>([&](auto i) { (*this)[i] *= b; });
+ return *this;
}
vec_base &operator*=(const vec_base &b)
{
- BLI_VEC_OP_IMPL_SELF(i, (*this)[i] *= b[i]);
+ unroll<Size>([&](auto i) { (*this)[i] *= b[i]; });
+ return *this;
}
friend vec_base operator/(const vec_base &a, const vec_base &b)
@@ -339,13 +358,17 @@ template<typename T, int Size> struct vec_base : public vec_struct_base<T, Size>
for (int i = 0; i < Size; i++) {
BLI_assert(b[i] != T(0));
}
- BLI_VEC_OP_IMPL(ret, i, ret[i] = a[i] / b[i]);
+ vec_base result;
+ unroll<Size>([&](auto i) { result[i] = a[i] / b[i]; });
+ return result;
}
friend vec_base operator/(const vec_base &a, T b)
{
BLI_assert(b != T(0));
- BLI_VEC_OP_IMPL(ret, i, ret[i] = a[i] / b);
+ vec_base result;
+ unroll<Size>([&](auto i) { result[i] = a[i] / b; });
+ return result;
}
friend vec_base operator/(T a, const vec_base &b)
@@ -353,31 +376,39 @@ template<typename T, int Size> struct vec_base : public vec_struct_base<T, Size>
for (int i = 0; i < Size; i++) {
BLI_assert(b[i] != T(0));
}
- BLI_VEC_OP_IMPL(ret, i, ret[i] = a / b[i]);
+ vec_base result;
+ unroll<Size>([&](auto i) { result[i] = a / b[i]; });
+ return result;
}
vec_base &operator/=(T b)
{
BLI_assert(b != T(0));
- BLI_VEC_OP_IMPL_SELF(i, (*this)[i] /= b);
+ unroll<Size>([&](auto i) { (*this)[i] /= b; });
+ return *this;
}
vec_base &operator/=(const vec_base &b)
{
BLI_assert(b != T(0));
- BLI_VEC_OP_IMPL_SELF(i, (*this)[i] /= b[i]);
+ unroll<Size>([&](auto i) { (*this)[i] /= b[i]; });
+ return *this;
}
/** Binary operators. */
BLI_INT_OP(T) friend vec_base operator&(const vec_base &a, const vec_base &b)
{
- BLI_VEC_OP_IMPL(ret, i, ret[i] = a[i] & b[i]);
+ vec_base result;
+ unroll<Size>([&](auto i) { result[i] = a[i] & b[i]; });
+ return result;
}
BLI_INT_OP(T) friend vec_base operator&(const vec_base &a, T b)
{
- BLI_VEC_OP_IMPL(ret, i, ret[i] = a[i] & b);
+ vec_base result;
+ unroll<Size>([&](auto i) { result[i] = a[i] & b; });
+ return result;
}
BLI_INT_OP(T) friend vec_base operator&(T a, const vec_base &b)
@@ -387,22 +418,28 @@ template<typename T, int Size> struct vec_base : public vec_struct_base<T, Size>
BLI_INT_OP(T) vec_base &operator&=(T b)
{
- BLI_VEC_OP_IMPL_SELF(i, (*this)[i] &= b);
+ unroll<Size>([&](auto i) { (*this)[i] &= b; });
+ return *this;
}
BLI_INT_OP(T) vec_base &operator&=(const vec_base &b)
{
- BLI_VEC_OP_IMPL_SELF(i, (*this)[i] &= b[i]);
+ unroll<Size>([&](auto i) { (*this)[i] &= b[i]; });
+ return *this;
}
BLI_INT_OP(T) friend vec_base operator|(const vec_base &a, const vec_base &b)
{
- BLI_VEC_OP_IMPL(ret, i, ret[i] = a[i] | b[i]);
+ vec_base result;
+ unroll<Size>([&](auto i) { result[i] = a[i] | b[i]; });
+ return result;
}
BLI_INT_OP(T) friend vec_base operator|(const vec_base &a, T b)
{
- BLI_VEC_OP_IMPL(ret, i, ret[i] = a[i] | b);
+ vec_base result;
+ unroll<Size>([&](auto i) { result[i] = a[i] | b; });
+ return result;
}
BLI_INT_OP(T) friend vec_base operator|(T a, const vec_base &b)
@@ -412,22 +449,28 @@ template<typename T, int Size> struct vec_base : public vec_struct_base<T, Size>
BLI_INT_OP(T) vec_base &operator|=(T b)
{
- BLI_VEC_OP_IMPL_SELF(i, (*this)[i] |= b);
+ unroll<Size>([&](auto i) { (*this)[i] |= b; });
+ return *this;
}
BLI_INT_OP(T) vec_base &operator|=(const vec_base &b)
{
- BLI_VEC_OP_IMPL_SELF(i, (*this)[i] |= b[i]);
+ unroll<Size>([&](auto i) { (*this)[i] |= b[i]; });
+ return *this;
}
BLI_INT_OP(T) friend vec_base operator^(const vec_base &a, const vec_base &b)
{
- BLI_VEC_OP_IMPL(ret, i, ret[i] = a[i] ^ b[i]);
+ vec_base result;
+ unroll<Size>([&](auto i) { result[i] = a[i] ^ b[i]; });
+ return result;
}
BLI_INT_OP(T) friend vec_base operator^(const vec_base &a, T b)
{
- BLI_VEC_OP_IMPL(ret, i, ret[i] = a[i] ^ b);
+ vec_base result;
+ unroll<Size>([&](auto i) { result[i] = a[i] ^ b; });
+ return result;
}
BLI_INT_OP(T) friend vec_base operator^(T a, const vec_base &b)
@@ -437,59 +480,75 @@ template<typename T, int Size> struct vec_base : public vec_struct_base<T, Size>
BLI_INT_OP(T) vec_base &operator^=(T b)
{
- BLI_VEC_OP_IMPL_SELF(i, (*this)[i] ^= b);
+ unroll<Size>([&](auto i) { (*this)[i] ^= b; });
+ return *this;
}
BLI_INT_OP(T) vec_base &operator^=(const vec_base &b)
{
- BLI_VEC_OP_IMPL_SELF(i, (*this)[i] ^= b[i]);
+ unroll<Size>([&](auto i) { (*this)[i] ^= b[i]; });
+ return *this;
}
BLI_INT_OP(T) friend vec_base operator~(const vec_base &a)
{
- BLI_VEC_OP_IMPL(ret, i, ret[i] = ~a[i]);
+ vec_base result;
+ unroll<Size>
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list