[Bf-blender-cvs] [c3e919b317e] master: Sculpt: Fix broken multires hidden undo

[Joseph Eagar]

Commit: c3e919b317e3697eec6cbe122f22a8330767b8d3
Author: Joseph Eagar
Date:   Tue Nov 22 11:11:03 2022 -0800
Branches: master
https://developer.blender.org/rBc3e919b317e3697eec6cbe122f22a8330767b8d3

Sculpt: Fix broken multires hidden undo

Wrote a new API method, BKE_pbvh_sync_visibility_from_verts
that flushes vertex hidden flags to edges & faces.

Fixes not being able to sculpt outside a face set after
undoing the fkey hide-all-but-this operator.

===================================================================

M	source/blender/blenkernel/BKE_pbvh.h
M	source/blender/blenkernel/intern/pbvh.c
M	source/blender/editors/sculpt_paint/sculpt_undo.c

===================================================================

diff --git a/source/blender/blenkernel/BKE_pbvh.h b/source/blender/blenkernel/BKE_pbvh.h
index 1c2e4193596..7132dec871e 100644
--- a/source/blender/blenkernel/BKE_pbvh.h
+++ b/source/blender/blenkernel/BKE_pbvh.h
@@ -397,6 +397,8 @@ void BKE_pbvh_bounding_box(const PBVH *pbvh, float min[3], float max[3]);
  */
 unsigned int **BKE_pbvh_grid_hidden(const PBVH *pbvh);
 
+void BKE_pbvh_sync_visibility_from_verts(PBVH *pbvh, struct Mesh *me);
+
 /**
  * Returns the number of visible quads in the nodes' grids.
  */
diff --git a/source/blender/blenkernel/intern/pbvh.c b/source/blender/blenkernel/intern/pbvh.c
index 3d1a20bb688..63dc84e2743 100644
--- a/source/blender/blenkernel/intern/pbvh.c
+++ b/source/blender/blenkernel/intern/pbvh.c
@@ -3633,7 +3633,6 @@ static void pbvh_face_iter_step(PBVHFaceIter *fd, bool do_step)
           fd->verts[i].i = ml->v;
         }
       }
-
       break;
     }
   }
@@ -3702,3 +3701,92 @@ bool BKE_pbvh_face_iter_done(PBVHFaceIter *fd)
       return true;
   }
 }
+
+void BKE_pbvh_sync_visibility_from_verts(PBVH *pbvh, Mesh *mesh)
+{
+  switch (pbvh->header.type) {
+    case PBVH_FACES: {
+      BKE_mesh_flush_hidden_from_verts(mesh);
+      BKE_pbvh_update_hide_attributes_from_mesh(pbvh);
+      break;
+    }
+    case PBVH_BMESH: {
+      BMIter iter;
+      BMVert *v;
+      BMEdge *e;
+      BMFace *f;
+
+      BM_ITER_MESH (f, &iter, pbvh->header.bm, BM_FACES_OF_MESH) {
+        BM_elem_flag_disable(f, BM_ELEM_HIDDEN);
+      }
+
+      BM_ITER_MESH (e, &iter, pbvh->header.bm, BM_EDGES_OF_MESH) {
+        BM_elem_flag_disable(e, BM_ELEM_HIDDEN);
+      }
+
+      BM_ITER_MESH (v, &iter, pbvh->header.bm, BM_VERTS_OF_MESH) {
+        if (!BM_elem_flag_test(v, BM_ELEM_HIDDEN)) {
+          continue;
+        }
+        BMIter iter_l;
+        BMLoop *l;
+
+        BM_ITER_ELEM (l, &iter_l, v, BM_LOOPS_OF_VERT) {
+          BM_elem_flag_enable(l->e, BM_ELEM_HIDDEN);
+          BM_elem_flag_enable(l->f, BM_ELEM_HIDDEN);
+        }
+      }
+      break;
+    }
+    case PBVH_GRIDS: {
+      const MPoly *mp = BKE_mesh_polys(mesh);
+      const MLoop *mloop = BKE_mesh_loops(mesh);
+      CCGKey key = pbvh->gridkey;
+
+      bool *hide_poly = (bool *)CustomData_get_layer_named(
+          &mesh->pdata, CD_PROP_BOOL, ".hide_poly");
+
+      bool delete_hide_poly = true;
+      for (int face_index = 0; face_index < mesh->totpoly; face_index++, mp++) {
+        const MLoop *ml = mloop + mp->loopstart;
+        bool hidden = false;
+
+        for (int loop_index = 0; !hidden && loop_index < mp->totloop; loop_index++, ml++) {
+          int grid_index = mp->loopstart + loop_index;
+
+          if (pbvh->grid_hidden[grid_index] &&
+              BLI_BITMAP_TEST(pbvh->grid_hidden[grid_index], key.grid_area - 1)) {
+            hidden = true;
+
+            break;
+          }
+        }
+
+        if (hidden && !hide_poly) {
+          hide_poly = (bool *)CustomData_get_layer_named(&mesh->pdata, CD_PROP_BOOL, ".hide_poly");
+
+          if (!hide_poly) {
+            CustomData_add_layer_named(
+                &mesh->pdata, CD_PROP_BOOL, CD_CONSTRUCT, NULL, mesh->totpoly, ".hide_poly");
+
+            hide_poly = (bool *)CustomData_get_layer_named(
+                &mesh->pdata, CD_PROP_BOOL, ".hide_poly");
+          }
+        }
+
+        if (hide_poly) {
+          delete_hide_poly = delete_hide_poly && !hidden;
+          hide_poly[face_index] = hidden;
+        }
+      }
+
+      if (delete_hide_poly) {
+        CustomData_free_layer_named(&mesh->pdata, ".hide_poly", mesh->totpoly);
+      }
+
+      BKE_mesh_flush_hidden_from_polys(mesh);
+      BKE_pbvh_update_hide_attributes_from_mesh(pbvh);
+      break;
+    }
+  }
+}
diff --git a/source/blender/editors/sculpt_paint/sculpt_undo.c b/source/blender/editors/sculpt_paint/sculpt_undo.c
index 833f62d4955..339c1a43d77 100644
--- a/source/blender/editors/sculpt_paint/sculpt_undo.c
+++ b/source/blender/editors/sculpt_paint/sculpt_undo.c
@@ -1058,6 +1058,13 @@ static void sculpt_undo_restore_list(bContext *C, Depsgraph *depsgraph, ListBase
     }
 
     if (update_visibility) {
+      if (ELEM(BKE_pbvh_type(ss->pbvh), PBVH_FACES, PBVH_GRIDS)) {
+        Mesh *me = (Mesh *)ob->data;
+        BKE_pbvh_sync_visibility_from_verts(ss->pbvh, me);
+
+        BKE_pbvh_update_hide_attributes_from_mesh(ss->pbvh);
+      }
+
       BKE_pbvh_update_visibility(ss->pbvh);
     }
 
@@ -1080,11 +1087,6 @@ static void sculpt_undo_restore_list(bContext *C, Depsgraph *depsgraph, ListBase
       BKE_sculptsession_free_deformMats(ss);
     }
 
-    if (BKE_pbvh_type(ss->pbvh) == PBVH_FACES && update_visibility) {
-      Mesh *mesh = ob->data;
-      BKE_mesh_flush_hidden_from_verts(mesh);
-    }
-
     if (tag_update) {
       DEG_id_tag_update(&ob->id, ID_RECALC_GEOMETRY);
     }