[Bf-blender-cvs] [654e1e901b6] master: Cycles: Use local atomics for faster shader sorting (enabled on Metal)
Michael Jones
noreply at git.blender.org
Mon Feb 6 12:18:28 CET 2023
Commit: 654e1e901b6ae003d8ec7a0ce1bc5926d68a971f
Author: Michael Jones
Date: Mon Feb 6 11:16:02 2023 +0000
Branches: master
https://developer.blender.org/rB654e1e901b6ae003d8ec7a0ce1bc5926d68a971f
Cycles: Use local atomics for faster shader sorting (enabled on Metal)
This patch adds two new kernels: SORT_BUCKET_PASS and SORT_WRITE_PASS. These replace PREFIX_SUM and SORTED_PATHS_ARRAY on supported devices (currently implemented on Metal, but will be trivial to enable on the other backends). The new kernels exploit sort partitioning (see D15331) by sorting each partition separately using local atomics. This can give an overall render speedup of 2-3% depending on architecture. As before, we fall back to the original non-partitioned sorting when the shade [...]
Reviewed By: brecht
Differential Revision: https://developer.blender.org/D16909
===================================================================
M intern/cycles/device/kernel.cpp
M intern/cycles/device/metal/device_impl.h
M intern/cycles/device/metal/device_impl.mm
M intern/cycles/device/metal/kernel.mm
M intern/cycles/device/metal/queue.h
M intern/cycles/device/metal/queue.mm
M intern/cycles/device/queue.h
M intern/cycles/integrator/path_trace_work_gpu.cpp
M intern/cycles/integrator/path_trace_work_gpu.h
M intern/cycles/kernel/device/gpu/kernel.h
M intern/cycles/kernel/device/gpu/parallel_active_index.h
M intern/cycles/kernel/device/gpu/parallel_sorted_index.h
M intern/cycles/kernel/device/metal/compat.h
M intern/cycles/kernel/integrator/state.h
M intern/cycles/kernel/integrator/state_flow.h
M intern/cycles/kernel/types.h
M intern/cycles/util/atomic.h
M intern/cycles/util/debug.cpp
M intern/cycles/util/debug.h
===================================================================
diff --git a/intern/cycles/device/kernel.cpp b/intern/cycles/device/kernel.cpp
index 27ca0d81817..903ad096586 100644
--- a/intern/cycles/device/kernel.cpp
+++ b/intern/cycles/device/kernel.cpp
@@ -73,6 +73,10 @@ const char *device_kernel_as_string(DeviceKernel kernel)
return "integrator_terminated_paths_array";
case DEVICE_KERNEL_INTEGRATOR_SORTED_PATHS_ARRAY:
return "integrator_sorted_paths_array";
+ case DEVICE_KERNEL_INTEGRATOR_SORT_BUCKET_PASS:
+ return "integrator_sort_bucket_pass";
+ case DEVICE_KERNEL_INTEGRATOR_SORT_WRITE_PASS:
+ return "integrator_sort_write_pass";
case DEVICE_KERNEL_INTEGRATOR_COMPACT_PATHS_ARRAY:
return "integrator_compact_paths_array";
case DEVICE_KERNEL_INTEGRATOR_COMPACT_STATES:
diff --git a/intern/cycles/device/metal/device_impl.h b/intern/cycles/device/metal/device_impl.h
index 526535ff132..a10962b4e45 100644
--- a/intern/cycles/device/metal/device_impl.h
+++ b/intern/cycles/device/metal/device_impl.h
@@ -105,6 +105,8 @@ class MetalDevice : public Device {
bool use_adaptive_compilation();
+ bool use_local_atomic_sort() const;
+
bool make_source_and_check_if_compile_needed(MetalPipelineType pso_type);
void make_source(MetalPipelineType pso_type, const uint kernel_features);
diff --git a/intern/cycles/device/metal/device_impl.mm b/intern/cycles/device/metal/device_impl.mm
index 453418386a4..35298822e41 100644
--- a/intern/cycles/device/metal/device_impl.mm
+++ b/intern/cycles/device/metal/device_impl.mm
@@ -271,6 +271,11 @@ bool MetalDevice::use_adaptive_compilation()
return DebugFlags().metal.adaptive_compile;
}
+bool MetalDevice::use_local_atomic_sort() const
+{
+ return DebugFlags().metal.use_local_atomic_sort;
+}
+
void MetalDevice::make_source(MetalPipelineType pso_type, const uint kernel_features)
{
string global_defines;
@@ -278,6 +283,10 @@ void MetalDevice::make_source(MetalPipelineType pso_type, const uint kernel_feat
global_defines += "#define __KERNEL_FEATURES__ " + to_string(kernel_features) + "\n";
}
+ if (use_local_atomic_sort()) {
+ global_defines += "#define __KERNEL_LOCAL_ATOMIC_SORT__\n";
+ }
+
if (use_metalrt) {
global_defines += "#define __METALRT__\n";
if (motion_blur) {
diff --git a/intern/cycles/device/metal/kernel.mm b/intern/cycles/device/metal/kernel.mm
index 6312c5f88ee..2ed230ee657 100644
--- a/intern/cycles/device/metal/kernel.mm
+++ b/intern/cycles/device/metal/kernel.mm
@@ -87,6 +87,9 @@ struct ShaderCache {
break;
}
}
+
+ occupancy_tuning[DEVICE_KERNEL_INTEGRATOR_SORT_BUCKET_PASS] = {1024, 1024};
+ occupancy_tuning[DEVICE_KERNEL_INTEGRATOR_SORT_WRITE_PASS] = {1024, 1024};
}
~ShaderCache();
diff --git a/intern/cycles/device/metal/queue.h b/intern/cycles/device/metal/queue.h
index 2a6c12e2a60..cf3a6175916 100644
--- a/intern/cycles/device/metal/queue.h
+++ b/intern/cycles/device/metal/queue.h
@@ -25,6 +25,7 @@ class MetalDeviceQueue : public DeviceQueue {
virtual int num_concurrent_states(const size_t) const override;
virtual int num_concurrent_busy_states(const size_t) const override;
virtual int num_sort_partition_elements() const override;
+ virtual bool supports_local_atomic_sort() const override;
virtual void init_execution() override;
diff --git a/intern/cycles/device/metal/queue.mm b/intern/cycles/device/metal/queue.mm
index f335844c3f9..9137e9b1fb0 100644
--- a/intern/cycles/device/metal/queue.mm
+++ b/intern/cycles/device/metal/queue.mm
@@ -315,6 +315,11 @@ int MetalDeviceQueue::num_sort_partition_elements() const
return MetalInfo::optimal_sort_partition_elements(metal_device_->mtlDevice);
}
+bool MetalDeviceQueue::supports_local_atomic_sort() const
+{
+ return metal_device_->use_local_atomic_sort();
+}
+
void MetalDeviceQueue::init_execution()
{
/* Synchronize all textures and memory copies before executing task. */
@@ -553,13 +558,24 @@ bool MetalDeviceQueue::enqueue(DeviceKernel kernel,
/* See parallel_active_index.h for why this amount of shared memory is needed.
* Rounded up to 16 bytes for Metal */
shared_mem_bytes = (int)round_up((num_threads_per_block + 1) * sizeof(int), 16);
- [mtlComputeCommandEncoder setThreadgroupMemoryLength:shared_mem_bytes atIndex:0];
break;
+ case DEVICE_KERNEL_INTEGRATOR_SORT_BUCKET_PASS:
+ case DEVICE_KERNEL_INTEGRATOR_SORT_WRITE_PASS: {
+ int key_count = metal_device_->launch_params.data.max_shaders;
+ shared_mem_bytes = (int)round_up(key_count * sizeof(int), 16);
+ break;
+ }
+
default:
break;
}
+ if (shared_mem_bytes) {
+ assert(shared_mem_bytes <= 32 * 1024);
+ [mtlComputeCommandEncoder setThreadgroupMemoryLength:shared_mem_bytes atIndex:0];
+ }
+
MTLSize size_threadgroups_per_dispatch = MTLSizeMake(
divide_up(work_size, num_threads_per_block), 1, 1);
MTLSize size_threads_per_threadgroup = MTLSizeMake(num_threads_per_block, 1, 1);
diff --git a/intern/cycles/device/queue.h b/intern/cycles/device/queue.h
index e27e081a407..3e73724408e 100644
--- a/intern/cycles/device/queue.h
+++ b/intern/cycles/device/queue.h
@@ -112,6 +112,13 @@ class DeviceQueue {
return 65536;
}
+ /* Does device support local atomic sorting kernels (INTEGRATOR_SORT_BUCKET_PASS and
+ * INTEGRATOR_SORT_WRITE_PASS)? */
+ virtual bool supports_local_atomic_sort() const
+ {
+ return false;
+ }
+
/* Initialize execution of kernels on this queue.
*
* Will, for example, load all data required by the kernels from Device to global or path state.
diff --git a/intern/cycles/integrator/path_trace_work_gpu.cpp b/intern/cycles/integrator/path_trace_work_gpu.cpp
index 547e8d50a22..bf0a7b6aae1 100644
--- a/intern/cycles/integrator/path_trace_work_gpu.cpp
+++ b/intern/cycles/integrator/path_trace_work_gpu.cpp
@@ -71,6 +71,8 @@ PathTraceWorkGPU::PathTraceWorkGPU(Device *device,
device, "integrator_shader_mnee_sort_counter", MEM_READ_WRITE),
integrator_shader_sort_prefix_sum_(
device, "integrator_shader_sort_prefix_sum", MEM_READ_WRITE),
+ integrator_shader_sort_partition_key_offsets_(
+ device, "integrator_shader_sort_partition_key_offsets", MEM_READ_WRITE),
integrator_next_main_path_index_(device, "integrator_next_main_path_index", MEM_READ_WRITE),
integrator_next_shadow_path_index_(
device, "integrator_next_shadow_path_index", MEM_READ_WRITE),
@@ -207,33 +209,45 @@ void PathTraceWorkGPU::alloc_integrator_sorting()
integrator_state_gpu_.sort_partition_divisor = (int)divide_up(max_num_paths_,
num_sort_partitions_);
- /* Allocate arrays for shader sorting. */
- const int sort_buckets = device_scene_->data.max_shaders * num_sort_partitions_;
- if (integrator_shader_sort_counter_.size() < sort_buckets) {
- integrator_shader_sort_counter_.alloc(sort_buckets);
- integrator_shader_sort_counter_.zero_to_device();
- integrator_state_gpu_.sort_key_counter[DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE] =
- (int *)integrator_shader_sort_counter_.device_pointer;
-
- integrator_shader_sort_prefix_sum_.alloc(sort_buckets);
- integrator_shader_sort_prefix_sum_.zero_to_device();
+ if (num_sort_partitions_ > 1 && queue_->supports_local_atomic_sort()) {
+ /* Allocate array for partitioned shader sorting using local atomics. */
+ const int num_offsets = (device_scene_->data.max_shaders + 1) * num_sort_partitions_;
+ if (integrator_shader_sort_partition_key_offsets_.size() < num_offsets) {
+ integrator_shader_sort_partition_key_offsets_.alloc(num_offsets);
+ integrator_shader_sort_partition_key_offsets_.zero_to_device();
+ }
+ integrator_state_gpu_.sort_partition_key_offsets =
+ (int *)integrator_shader_sort_partition_key_offsets_.device_pointer;
}
+ else {
+ /* Allocate arrays for shader sorting. */
+ const int sort_buckets = device_scene_->data.max_shaders * num_sort_partitions_;
+ if (integrator_shader_sort_counter_.size() < sort_buckets) {
+ integrator_shader_sort_counter_.alloc(sort_buckets);
+ integrator_shader_sort_counter_.zero_to_device();
+ integrator_state_gpu_.sort_key_counter[DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE] =
+ (int *)integrator_shader_sort_counter_.device_pointer;
+
+ integrator_shader_sort_prefix_sum_.alloc(sort_buckets);
+ integrator_shader_sort_prefix_sum_.zero_to_device();
+ }
- if (device_scene_->data.kernel_features & KERNEL_FEATURE_NODE_RAYTRACE) {
- if (integrator_shader_raytrace_sort_counter_.size() < sort_buckets) {
- integrator_shader_raytrace_sort_counter_.alloc(sort_buckets);
- integrator_shader_raytrace_sort_counter_.zero_to_device();
- integrator_state_gpu_.sort_key_counter[DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE_RAYTRACE] =
- (int *)integrator_shader_raytrace_sort_counter_.device_pointer;
+ if (device_scene_->data.kernel_features & KERNEL_FEATURE_NODE_RAYTRACE) {
+ if (integrator_shader_raytrace_sort_counter_.size() < sort_buckets) {
+ integrator_shader_raytrace_sort_counter_.alloc(sort_buckets);
+ integrator_shader_raytrace_sort_counter_.zero_to_device();
+ integrator_state_gpu_.sort_key_counter[DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE_RAYTRACE] =
+ (int *)integrator_shader_raytrace_sort_counter_.device_pointer;
+ }
}
- }
- if (device_scene_->data.kernel_features & KERNEL_FEATURE_MNEE) {
- if (integrator_shader_mnee_sort_counter_.size() < sort_buckets) {
- integrator_shader_mnee_sort_counter_.alloc(sort_buckets);
- integrator_shader_mnee_sort_counter_.zero_to_device();
- integrator_state_gpu_.sort_key_counter[DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE_MNEE] =
- (int *)integrator_shader_mnee_sort_counter_.device_pointer;
+ if (device_scene_->data.kernel_features & KERNEL_FEATURE_MNEE) {
+ if (integrator_shader_mnee_sort_counter_.size() < sort_buckets) {
+ integrator_shader_mnee_sort_counter_.alloc(sort_buckets);
+ integrator_shader_mnee_sort_counter_.zero_to_device();
+ integrator_state_gpu_.sort_key_counter
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list