[Bf-blender-cvs] [f8df47977c3] sculpt-dev: Sculpt: Fix face set boundary flag settings in PBVH_FACES

[Joseph Eagar]

Commit: f8df47977c34392f7233344a0d20d47d9732f765
Author: Joseph Eagar
Date:   Mon Oct 4 11:42:49 2021 -0700
Branches: sculpt-dev
https://developer.blender.org/rBf8df47977c34392f7233344a0d20d47d9732f765

Sculpt: Fix face set boundary flag settings in PBVH_FACES

===================================================================

M	source/blender/blenkernel/intern/paint.c
M	source/blender/blenkernel/intern/pbvh.c
M	source/blender/bmesh/bmesh_class.h
M	source/blender/editors/sculpt_paint/sculpt.c

===================================================================

diff --git a/source/blender/blenkernel/intern/paint.c b/source/blender/blenkernel/intern/paint.c
index 781d299cdc0..91bfb362a1c 100644
--- a/source/blender/blenkernel/intern/paint.c
+++ b/source/blender/blenkernel/intern/paint.c
@@ -2340,6 +2340,9 @@ static void init_mdyntopo_layer_faces(SculptSession *ss, PBVH *pbvh, int totvert
                                         ss->mdyntopo_verts,
                                         ss->pmap,
                                         vertex);
+
+    // can't fully update boundary here, so still flag for update
+    mv->flag |= DYNVERT_NEED_BOUNDARY;
   }
 }
 
@@ -2363,6 +2366,9 @@ static void init_mdyntopo_layer_grids(SculptSession *ss, PBVH *pbvh, int totvert
     SculptVertRef vertex = {.i = i};
 
     BKE_pbvh_update_vert_boundary_grids(pbvh, ss->subdiv_ccg, vertex);
+
+    // can't fully update boundary here, so still flag for update
+    mv->flag |= DYNVERT_NEED_BOUNDARY;
   }
 }
 
diff --git a/source/blender/blenkernel/intern/pbvh.c b/source/blender/blenkernel/intern/pbvh.c
index dea4d48a6d8..98461f6c4c6 100644
--- a/source/blender/blenkernel/intern/pbvh.c
+++ b/source/blender/blenkernel/intern/pbvh.c
@@ -4180,8 +4180,8 @@ void BKE_pbvh_update_vert_boundary_faces(int *face_sets,
   MDynTopoVert *mv = mdyntopo_verts + vertex.i;
   MeshElemMap *vert_map = &pmap[vertex.i];
 
-  int last_fset = 0;
-  int last_fset2 = 0;
+  int last_fset = -1;
+  int last_fset2 = -1;
 
   mv->flag &= ~(DYNVERT_BOUNDARY | DYNVERT_FSET_BOUNDARY | DYNVERT_NEED_BOUNDARY |
                 DYNVERT_FSET_CORNER | DYNVERT_CORNER | DYNVERT_SEAM_BOUNDARY |
@@ -4216,7 +4216,7 @@ void BKE_pbvh_update_vert_boundary_faces(int *face_sets,
       }
     }
 
-    int fset = face_sets ? face_sets[f_i] : -1;
+    int fset = face_sets ? abs(face_sets[f_i]) : -1;
 
     if (fset > 0) {
       visible = true;
@@ -4228,7 +4228,8 @@ void BKE_pbvh_update_vert_boundary_faces(int *face_sets,
     if (i > 0 && fset != last_fset) {
       mv->flag |= DYNVERT_FSET_BOUNDARY;
 
-      if (i > 1 && last_fset2 != last_fset) {
+      if (i > 1 && last_fset2 != last_fset && last_fset != -1 && last_fset2 != -1 && fset != -1 &&
+          last_fset2 != fset) {
         mv->flag |= DYNVERT_FSET_CORNER;
       }
     }
diff --git a/source/blender/bmesh/bmesh_class.h b/source/blender/bmesh/bmesh_class.h
index 94e89ab32e4..7088b93c47d 100644
--- a/source/blender/bmesh/bmesh_class.h
+++ b/source/blender/bmesh/bmesh_class.h
@@ -448,8 +448,7 @@ enum {
       BMElem *, BMElemF *, BMHeader *
 
 #define _BM_GENERIC_TYPE_ELEM_CONST \
-  const void *, const BMVert *, const BMEdge *, const BMLoop *, const BMFace *, \
-      const BMVert_OFlag *, const BMEdge_OFlag *, const BMFace_OFlag *, const BMElem *, \
+  const void *, const BMVert *, const BMEdge *, const BMLoop *, const BMFace *, const BMElem *, \
       const BMElemF *, const BMHeader *, void *const, BMVert *const, BMEdge *const, \
       BMLoop *const, BMFace *const, BMElem *const, BMElemF *const, BMHeader *const
 
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index e1290309203..17ed3846f08 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -2177,11 +2177,26 @@ static void faces_update_boundary_flags(const SculptSession *ss, const SculptVer
   // have to handle boundary here
   MDynTopoVert *mv = ss->mdyntopo_verts + vertex.i;
 
+  mv->flag &= ~(DYNVERT_CORNER | DYNVERT_BOUNDARY);
+
   if (sculpt_check_boundary_vertex_in_base_mesh(ss, vertex)) {
     mv->flag |= DYNVERT_BOUNDARY;
 
     if (ss->pmap[vertex.i].count < 4) {
-      mv->flag |= DYNVERT_CORNER;
+      bool ok = true;
+
+      for (int i = 0; i < ss->pmap[vertex.i].count; i++) {
+        MPoly *mp = ss->mpoly + ss->pmap[vertex.i].indices[i];
+        if (mp->totloop < 4) {
+          ok = false;
+        }
+      }
+      if (ok) {
+        mv->flag |= DYNVERT_CORNER;
+      }
+      else {
+        mv->flag &= ~DYNVERT_CORNER;
+      }
     }
   }
 }