[Bf-blender-cvs] [1bb2077250a] master: Add locking fallback atomics implementation
Sergey Sharybin
noreply at git.blender.org
Thu Aug 26 14:37:26 CEST 2021
Commit: 1bb2077250aed65e9445468161e01aa219a89624
Author: Sergey Sharybin
Date: Wed Aug 25 11:38:53 2021 +0200
Branches: master
https://developer.blender.org/rB1bb2077250aed65e9445468161e01aa219a89624
Add locking fallback atomics implementation
Is used for platforms for which we do not have native implementation,
such as MIPS, for example.
It is possible to test locking implementation on local computer by
defining `ATOMIC_FORCE_USE_FALLBACK` in the atomic_ops_unix.h file.
Run full regression suit with the locking implementation on amd64
Linux platform and all tests passed.
Having non-optimal but working implementation for odd-ball platforms
seems to be a better choice than failing the build entirely.
Differential Revision: https://developer.blender.org/D12313
===================================================================
M intern/atomic/intern/atomic_ops_unix.h
M intern/atomic/intern/atomic_ops_utils.h
===================================================================
diff --git a/intern/atomic/intern/atomic_ops_unix.h b/intern/atomic/intern/atomic_ops_unix.h
index b08a0e9bc28..dcafbc67949 100644
--- a/intern/atomic/intern/atomic_ops_unix.h
+++ b/intern/atomic/intern/atomic_ops_unix.h
@@ -60,9 +60,108 @@
# define JE_FORCE_SYNC_COMPARE_AND_SWAP_8
#endif
-/******************************************************************************/
-/* 64-bit operations. */
-#if (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) || defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8))
+/* Define the `ATOMIC_FORCE_USE_FALLBACK` to force lock-based fallback implementation to be used
+ * (even on platforms where there is native implementation available via compiler.
+ * Useful for development purposes. */
+#undef ATOMIC_FORCE_USE_FALLBACK
+
+/* -------------------------------------------------------------------- */
+/** \name Spin-lock implementation
+ *
+ * Used to implement atomics on unsupported platforms.
+ * The spin implementation is shared for all platforms to make sure it compiles and tested.
+ * \{ */
+
+typedef struct AtomicSpinLock {
+ volatile int lock;
+
+ /* Pad the structure size to a cache-line, to avoid unwanted sharing with other data. */
+ int pad[32 - sizeof(int)];
+} __attribute__((aligned(32))) AtomicSpinLock;
+
+ATOMIC_INLINE void atomic_spin_lock(volatile AtomicSpinLock *lock)
+{
+ while (__sync_lock_test_and_set(&lock->lock, 1)) {
+ while (lock->lock) {
+ }
+ }
+}
+
+ATOMIC_INLINE void atomic_spin_unlock(volatile AtomicSpinLock *lock)
+{
+ __sync_lock_release(&lock->lock);
+}
+
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name Common part of locking fallback implementation
+ * \{ */
+
+/* Global lock, shared by all atomic operations implementations.
+ *
+ * Could be split into per-size locks, although added complexity and being more error-proone does
+ * not seem to worth it for a fall-back implementation. */
+static _ATOMIC_MAYBE_UNUSED AtomicSpinLock _atomic_global_lock = {0};
+
+#define ATOMIC_LOCKING_OP_AND_FETCH_DEFINE(_type, _op_name, _op) \
+ ATOMIC_INLINE _type##_t atomic_##_op_name##_and_fetch_##_type(_type##_t *p, _type##_t x) \
+ { \
+ atomic_spin_lock(&_atomic_global_lock); \
+ const _type##_t original_value = *(p); \
+ const _type##_t new_value = original_value _op(x); \
+ *(p) = new_value; \
+ atomic_spin_unlock(&_atomic_global_lock); \
+ return new_value; \
+ }
+
+#define ATOMIC_LOCKING_FETCH_AND_OP_DEFINE(_type, _op_name, _op) \
+ ATOMIC_INLINE _type##_t atomic_fetch_and_##_op_name##_##_type(_type##_t *p, _type##_t x) \
+ { \
+ atomic_spin_lock(&_atomic_global_lock); \
+ const _type##_t original_value = *(p); \
+ *(p) = original_value _op(x); \
+ atomic_spin_unlock(&_atomic_global_lock); \
+ return original_value; \
+ }
+
+#define ATOMIC_LOCKING_ADD_AND_FETCH_DEFINE(_type) \
+ ATOMIC_LOCKING_OP_AND_FETCH_DEFINE(_type, add, +)
+
+#define ATOMIC_LOCKING_SUB_AND_FETCH_DEFINE(_type) \
+ ATOMIC_LOCKING_OP_AND_FETCH_DEFINE(_type, sub, -)
+
+#define ATOMIC_LOCKING_FETCH_AND_ADD_DEFINE(_type) \
+ ATOMIC_LOCKING_FETCH_AND_OP_DEFINE(_type, add, +)
+
+#define ATOMIC_LOCKING_FETCH_AND_SUB_DEFINE(_type) \
+ ATOMIC_LOCKING_FETCH_AND_OP_DEFINE(_type, sub, -)
+
+#define ATOMIC_LOCKING_FETCH_AND_OR_DEFINE(_type) ATOMIC_LOCKING_FETCH_AND_OP_DEFINE(_type, or, |)
+
+#define ATOMIC_LOCKING_FETCH_AND_AND_DEFINE(_type) \
+ ATOMIC_LOCKING_FETCH_AND_OP_DEFINE(_type, and, &)
+
+#define ATOMIC_LOCKING_CAS_DEFINE(_type) \
+ ATOMIC_INLINE _type##_t atomic_cas_##_type(_type##_t *v, _type##_t old, _type##_t _new) \
+ { \
+ atomic_spin_lock(&_atomic_global_lock); \
+ const _type##_t original_value = *v; \
+ if (*v == old) { \
+ *v = _new; \
+ } \
+ atomic_spin_unlock(&_atomic_global_lock); \
+ return original_value; \
+ }
+
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name 64-bit operations
+ * \{ */
+
+#if !defined(ATOMIC_FORCE_USE_FALLBACK) && \
+ (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) || defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8))
/* Unsigned */
ATOMIC_INLINE uint64_t atomic_add_and_fetch_uint64(uint64_t *p, uint64_t x)
{
@@ -115,7 +214,7 @@ ATOMIC_INLINE int64_t atomic_cas_int64(int64_t *v, int64_t old, int64_t _new)
return __sync_val_compare_and_swap(v, old, _new);
}
-#elif (defined(__amd64__) || defined(__x86_64__))
+#elif !defined(ATOMIC_FORCE_USE_FALLBACK) && (defined(__amd64__) || defined(__x86_64__))
/* Unsigned */
ATOMIC_INLINE uint64_t atomic_fetch_and_add_uint64(uint64_t *p, uint64_t x)
{
@@ -190,12 +289,36 @@ ATOMIC_INLINE int64_t atomic_cas_int64(int64_t *v, int64_t old, int64_t _new)
return ret;
}
#else
-# error "Missing implementation for 64-bit atomic operations"
+
+/* Unsigned */
+
+ATOMIC_LOCKING_ADD_AND_FETCH_DEFINE(uint64)
+ATOMIC_LOCKING_SUB_AND_FETCH_DEFINE(uint64)
+
+ATOMIC_LOCKING_FETCH_AND_ADD_DEFINE(uint64)
+ATOMIC_LOCKING_FETCH_AND_SUB_DEFINE(uint64)
+
+ATOMIC_LOCKING_CAS_DEFINE(uint64)
+
+/* Signed */
+ATOMIC_LOCKING_ADD_AND_FETCH_DEFINE(int64)
+ATOMIC_LOCKING_SUB_AND_FETCH_DEFINE(int64)
+
+ATOMIC_LOCKING_FETCH_AND_ADD_DEFINE(int64)
+ATOMIC_LOCKING_FETCH_AND_SUB_DEFINE(int64)
+
+ATOMIC_LOCKING_CAS_DEFINE(int64)
+
#endif
-/******************************************************************************/
-/* 32-bit operations. */
-#if (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) || defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4))
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name 32-bit operations
+ * \{ */
+
+#if !defined(ATOMIC_FORCE_USE_FALLBACK) && \
+ (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) || defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4))
/* Unsigned */
ATOMIC_INLINE uint32_t atomic_add_and_fetch_uint32(uint32_t *p, uint32_t x)
{
@@ -228,7 +351,8 @@ ATOMIC_INLINE int32_t atomic_cas_int32(int32_t *v, int32_t old, int32_t _new)
return __sync_val_compare_and_swap(v, old, _new);
}
-#elif (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
+#elif !defined(ATOMIC_FORCE_USE_FALLBACK) && \
+ (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
/* Unsigned */
ATOMIC_INLINE uint32_t atomic_add_and_fetch_uint32(uint32_t *p, uint32_t x)
{
@@ -286,10 +410,25 @@ ATOMIC_INLINE int32_t atomic_cas_int32(int32_t *v, int32_t old, int32_t _new)
}
#else
-# error "Missing implementation for 32-bit atomic operations"
+
+/* Unsigned */
+
+ATOMIC_LOCKING_ADD_AND_FETCH_DEFINE(uint32)
+ATOMIC_LOCKING_SUB_AND_FETCH_DEFINE(uint32)
+
+ATOMIC_LOCKING_CAS_DEFINE(uint32)
+
+/* Signed */
+
+ATOMIC_LOCKING_ADD_AND_FETCH_DEFINE(int32)
+ATOMIC_LOCKING_SUB_AND_FETCH_DEFINE(int32)
+
+ATOMIC_LOCKING_CAS_DEFINE(int32)
+
#endif
-#if (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) || defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4))
+#if !defined(ATOMIC_FORCE_USE_FALLBACK) && \
+ (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) || defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4))
/* Unsigned */
ATOMIC_INLINE uint32_t atomic_fetch_and_add_uint32(uint32_t *p, uint32_t x)
{
@@ -323,12 +462,27 @@ ATOMIC_INLINE int32_t atomic_fetch_and_and_int32(int32_t *p, int32_t x)
}
#else
-# error "Missing implementation for 32-bit atomic operations"
+
+/* Unsigned */
+ATOMIC_LOCKING_FETCH_AND_ADD_DEFINE(uint32)
+ATOMIC_LOCKING_FETCH_AND_OR_DEFINE(uint32)
+ATOMIC_LOCKING_FETCH_AND_AND_DEFINE(uint32)
+
+/* Signed */
+ATOMIC_LOCKING_FETCH_AND_ADD_DEFINE(int32)
+ATOMIC_LOCKING_FETCH_AND_OR_DEFINE(int32)
+ATOMIC_LOCKING_FETCH_AND_AND_DEFINE(int32)
+
#endif
-/******************************************************************************/
-/* 16-bit operations. */
-#if (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2) || defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_2))
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name 16-bit operations
+ * \{ */
+
+#if !defined(ATOMIC_FORCE_USE_FALLBACK) && \
+ (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2) || defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_2))
/* Signed */
ATOMIC_INLINE int16_t atomic_fetch_and_and_int16(int16_t *p, int16_t b)
@@ -341,12 +495,21 @@ ATOMIC_INLINE int16_t atomic_fetch_and_or_int16(int16_t *p, int16_t b)
}
#else
-# error "Missing implementation for 16-bit atomic operations"
+
+ATOMIC_LOCKING_FETCH_AND_AND_DEFINE(int16)
+ATOMIC_LOCKING_FETCH_AND_OR_DEFINE(int16)
+
#endif
-/******************************************************************************/
-/* 8-bit operations. */
-#if (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1) || defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_1))
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name 8-bit operations
+ * \{ */
+
+#if !defined(ATOMIC_FORCE_USE_FALLBACK) && \
+ (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1) || defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_1))
+
/* Unsigned */
ATOMIC_INLINE uint8_t atomic_fetch_and_and_uint8(uint8_t *p, uint8_t b)
{
@@ -368,7 +531,27 @@ ATOMIC_INLINE int8_t atomic_fetch_and_or_int8(int8_t *p, int8_t b)
}
#else
-# error "Missing implementation for 8-bit atomic operations"
+
+/* Unsigned */
+ATOMIC_LOCKING_FETCH_AND_AND_DEFINE(uint8)
+ATOMIC_LOCKING_FETCH_AND_OR_DEFINE(uint8)
+
+/* Signed */
+ATOMIC_LOCKING_FETCH_AND_AND_DEFINE(int8)
+ATOMIC_LOCKING_FETCH_AND_OR_DEFINE(int8)
+
#endif
+/** \} */
+
+#undef ATOMIC_LOCKING_OP_AND_FETCH_DEFINE
+#undef ATOMIC_LOCKING_FETCH_AND_OP_DEFINE
+#undef ATOMIC_LOCKING_ADD_AND_FETCH_DEFINE
+#undef ATOMIC_LOCKING_SUB_AND_FETCH_DEFINE
+#undef ATOMIC_LOCKING_FETCH_AND_ADD_DEFINE
+#undef ATOMIC_LOCKING_FETCH_AND_SUB_DEFINE
+#undef ATOMIC_LOCKING_FETCH_AND_OR_DEFINE
+#undef ATOMIC_LOCKING_FETCH_AND_AND_DEFINE
+#undef ATOMIC_LOCKING_CAS_DEFINE
+
#endif /* __ATOMIC_OPS_UNIX_H__ */
diff --git a/intern/atomic/intern/atomic_ops_utils.h b/intern/atomic/intern/atomic_ops_utils.h
index 57e5b74dd72..01f4284284a 100644
--- a/intern/atomic/intern/atomic_ops_utils.h
+++ b/intern/atomic/intern/atomic_ops_utils.h
@@ -64,9 +64,11 @@
#ifdef __GNUC__
# define _ATOMIC_LIKELY(x) __builtin_expect(!!(x), 1)
# define _ATOMIC_UNLIKELY(x) __builtin_expect(!!(x), 0)
+# define _ATOMIC_MAYBE_UNUSED __attribute__((unused))
#else
# defi
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list