[Bf-blender-cvs] [c007e84e9ec] master: Fix: Viewer node overlay alpha broken on NVidia GPUs

[Hans Goudey]

Commit: c007e84e9ec6b7283b871979a8fdb80f8c7819ed
Author: Hans Goudey
Date:   Thu Sep 29 11:28:48 2022 -0500
Branches: master
https://developer.blender.org/rBc007e84e9ec6b7283b871979a8fdb80f8c7819ed

Fix: Viewer node overlay alpha broken on NVidia GPUs

On NVidia GPUs, when the blue channel was between 0 and 0.1,
the overlay's alpha would increase, making it invisible.

With the `overlay_line_fb` frame buffer bound in `overlay_engine.cc`
there are two outputs to write to: the color and the line output, which
is used for making smooth antialiased lines. The overlay is in its
current position in the order of overlays so that it draws on top of
curve wire lines. Not writing to that second output is undefined
behavior, so fix it by writing zeros. In the future, the overlay
could be smoothed on curves using that second texture.

Thanks to Clément for the explanation of the issue!

===================================================================

M	source/blender/draw/engines/overlay/shaders/infos/overlay_viewer_attribute_info.hh
M	source/blender/draw/engines/overlay/shaders/overlay_viewer_attribute_frag.glsl

===================================================================

diff --git a/source/blender/draw/engines/overlay/shaders/infos/overlay_viewer_attribute_info.hh b/source/blender/draw/engines/overlay/shaders/infos/overlay_viewer_attribute_info.hh
index aca80992622..20d4797b76f 100644
--- a/source/blender/draw/engines/overlay/shaders/infos/overlay_viewer_attribute_info.hh
+++ b/source/blender/draw/engines/overlay/shaders/infos/overlay_viewer_attribute_info.hh
@@ -11,6 +11,7 @@ GPU_SHADER_CREATE_INFO(overlay_viewer_attribute_mesh)
     .vertex_source("overlay_viewer_attribute_mesh_vert.glsl")
     .fragment_source("overlay_viewer_attribute_frag.glsl")
     .fragment_out(0, Type::VEC4, "out_color")
+    .fragment_out(1, Type::VEC4, "lineOutput")
     .vertex_in(0, Type::VEC3, "pos")
     .vertex_in(1, Type::VEC4, "attribute_value")
     .vertex_out(overlay_viewer_attribute_iface)
@@ -25,6 +26,7 @@ GPU_SHADER_CREATE_INFO(overlay_viewer_attribute_pointcloud)
     .vertex_source("overlay_viewer_attribute_pointcloud_vert.glsl")
     .fragment_source("overlay_viewer_attribute_frag.glsl")
     .fragment_out(0, Type::VEC4, "out_color")
+    .fragment_out(1, Type::VEC4, "lineOutput")
     .vertex_in(3, Type::VEC4, "attribute_value")
     .vertex_out(overlay_viewer_attribute_iface)
     .additional_info("overlay_viewer_attribute_common", "draw_pointcloud");
@@ -38,6 +40,7 @@ GPU_SHADER_CREATE_INFO(overlay_viewer_attribute_curve)
     .vertex_source("overlay_viewer_attribute_curve_vert.glsl")
     .fragment_source("overlay_viewer_attribute_frag.glsl")
     .fragment_out(0, Type::VEC4, "out_color")
+    .fragment_out(1, Type::VEC4, "lineOutput")
     .vertex_in(0, Type::VEC3, "pos")
     .vertex_in(1, Type::VEC4, "attribute_value")
     .vertex_out(overlay_viewer_attribute_iface)
@@ -52,6 +55,7 @@ GPU_SHADER_CREATE_INFO(overlay_viewer_attribute_curves)
     .vertex_source("overlay_viewer_attribute_curves_vert.glsl")
     .fragment_source("overlay_viewer_attribute_frag.glsl")
     .fragment_out(0, Type::VEC4, "out_color")
+    .fragment_out(1, Type::VEC4, "lineOutput")
     .sampler(0, ImageType::FLOAT_BUFFER, "color_tx")
     .push_constant(Type::BOOL, "is_point_domain")
     .vertex_out(overlay_viewer_attribute_iface)
diff --git a/source/blender/draw/engines/overlay/shaders/overlay_viewer_attribute_frag.glsl b/source/blender/draw/engines/overlay/shaders/overlay_viewer_attribute_frag.glsl
index 6176a6b96ba..e5752ada940 100644
--- a/source/blender/draw/engines/overlay/shaders/overlay_viewer_attribute_frag.glsl
+++ b/source/blender/draw/engines/overlay/shaders/overlay_viewer_attribute_frag.glsl
@@ -3,4 +3,6 @@ void main()
 {
   out_color = finalColor;
   out_color.a *= opacity;
+  /* Writing to this second texture is necessary to avoid undefined behavior. */
+  lineOutput = vec4(0.0);
 }