[Bf-blender-cvs] [458f50ba73b] master: Fix T74780: Face sets operators not aware of SCULPT_FACE_SET_NONE

[Pablo Dobarro]

Commit: 458f50ba73bcd233176f9afadc3273acf05e4f53
Author: Pablo Dobarro
Date:   Thu Mar 26 15:50:03 2020 +0100
Branches: master
https://developer.blender.org/rB458f50ba73bcd233176f9afadc3273acf05e4f53

Fix T74780: Face sets operators not aware of SCULPT_FACE_SET_NONE

SCULPT_FACE_SET_NONE default value is 0 and it is rendered hidden, so
the invert sign operation to show it was not working. Now the show all
function sets this face set to ID 1 before setting its sign.

I also refactored this check in gpu_buffers.

Not related to the reported issue, but the mesh in attached contains non
manifold geometry with hidden loose vertices, so the visibility state
was not syncing correctly to those vertices. Now the toggle operators
checks the current visibility only on the face sets, so no manifold
vertices are ignored (as they are in the rest of operations in sculpt
mode).

Reviewed By: jbakker

Maniphest Tasks: T74780

Differential Revision: https://developer.blender.org/D7188

===================================================================

M	source/blender/editors/sculpt_paint/sculpt.c
M	source/blender/gpu/intern/gpu_buffers.c

===================================================================

diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index c3ea3671195..ea3fb5624dd 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -292,6 +292,14 @@ static void SCULPT_face_sets_visibility_all_set(SculptSession *ss, bool visible)
   switch (BKE_pbvh_type(ss->pbvh)) {
     case PBVH_FACES:
       for (int i = 0; i < ss->totpoly; i++) {
+
+        /* This can run on geometry without a face set assigned, so its ID sign can't be changed to
+         * modify the visibility. Force that geometry to the ID 1 to enable changing the visibility
+         * here. */
+        if (ss->face_sets[i] == SCULPT_FACE_SET_NONE) {
+          ss->face_sets[i] = 1;
+        }
+
         if (visible) {
           ss->face_sets[i] = abs(ss->face_sets[i]);
         }
@@ -11015,19 +11023,25 @@ static int sculpt_face_sets_change_visibility_invoke(bContext *C,
 
   if (mode == SCULPT_FACE_SET_VISIBILITY_TOGGLE) {
     bool hidden_vertex = false;
-    for (int i = 0; i < tot_vert; i++) {
-      if (!SCULPT_vertex_visible_get(ss, i)) {
-        hidden_vertex = true;
-        break;
+
+    /* This can fail with regular meshes with non-manifold geometry as the visibility state can't
+     * be synced from face sets to non-manifold vertices. */
+    if (BKE_pbvh_type(ss->pbvh) == PBVH_GRIDS) {
+      for (int i = 0; i < tot_vert; i++) {
+        if (!SCULPT_vertex_visible_get(ss, i)) {
+          hidden_vertex = true;
+          break;
+        }
       }
     }
 
     for (int i = 0; i < ss->totpoly; i++) {
-      if (ss->face_sets[i] < 0) {
+      if (ss->face_sets[i] <= 0) {
         hidden_vertex = true;
         break;
       }
     }
+
     if (hidden_vertex) {
       SCULPT_face_sets_visibility_all_set(ss, true);
     }
diff --git a/source/blender/gpu/intern/gpu_buffers.c b/source/blender/gpu/intern/gpu_buffers.c
index 3dc5052d472..d06d11023b6 100644
--- a/source/blender/gpu/intern/gpu_buffers.c
+++ b/source/blender/gpu/intern/gpu_buffers.c
@@ -206,6 +206,15 @@ static void face_set_overlay_color_get(const int face_set, const int seed, uchar
   rgba_float_to_uchar(r_color, rgba);
 }
 
+static bool gpu_pbvh_is_looptri_visible(const MLoopTri *lt,
+                                        const MVert *mvert,
+                                        const MLoop *mloop,
+                                        const int *sculpt_face_sets)
+{
+  return (!paint_is_face_hidden(lt, mvert, mloop) && sculpt_face_sets &&
+          sculpt_face_sets[lt->poly] > SCULPT_FACE_SET_NONE);
+}
+
 /* Threaded - do not call any functions that use OpenGL calls! */
 void GPU_pbvh_mesh_buffers_update(GPU_PBVH_Buffers *buffers,
                                   const MVert *mvert,
@@ -315,11 +324,7 @@ void GPU_pbvh_mesh_buffers_update(GPU_PBVH_Buffers *buffers,
               buffers->mloop[lt->tri[2]].v,
           };
 
-          if (paint_is_face_hidden(lt, mvert, buffers->mloop)) {
-            continue;
-          }
-
-          if (sculpt_face_sets[lt->poly] <= 0) {
+          if (!gpu_pbvh_is_looptri_visible(lt, mvert, buffers->mloop, sculpt_face_sets)) {
             continue;
           }
 
@@ -384,15 +389,6 @@ void GPU_pbvh_mesh_buffers_update(GPU_PBVH_Buffers *buffers,
   buffers->mvert = mvert;
 }
 
-static bool gpu_pbvh_is_looptri_visible(const MLoopTri *lt,
-                                        const MVert *mvert,
-                                        const MLoop *mloop,
-                                        const int *sculpt_face_sets)
-{
-  return (!paint_is_face_hidden(lt, mvert, mloop) && sculpt_face_sets &&
-          sculpt_face_sets[lt->poly] > 0);
-}
-
 /* Threaded - do not call any functions that use OpenGL calls! */
 GPU_PBVH_Buffers *GPU_pbvh_mesh_buffers_build(const int (*face_vert_indices)[3],
                                               const MPoly *mpoly,