[Bf-blender-cvs] [bbbfd7130ae] master: Fix Boundary Brush not working with partially hidden meshes

[Pablo Dobarro]

Commit: bbbfd7130ae36b9ba1652b919ef6ae426107acd8
Author: Pablo Dobarro
Date:   Sun Sep 6 19:38:37 2020 +0200
Branches: master
https://developer.blender.org/rBbbbfd7130ae36b9ba1652b919ef6ae426107acd8

Fix Boundary Brush not working with partially hidden meshes

Now when a mesh is partially hidden using Face Sets, the open boundary
the hidden geometry produces is also detected by the brush.

Reviewed By: sergey

Differential Revision: https://developer.blender.org/D8819

===================================================================

M	source/blender/editors/sculpt_paint/sculpt_boundary.c

===================================================================

diff --git a/source/blender/editors/sculpt_paint/sculpt_boundary.c b/source/blender/editors/sculpt_paint/sculpt_boundary.c
index 5e01e034715..3051413a405 100644
--- a/source/blender/editors/sculpt_paint/sculpt_boundary.c
+++ b/source/blender/editors/sculpt_paint/sculpt_boundary.c
@@ -74,6 +74,10 @@ static bool boundary_initial_vertex_floodfill_cb(
 {
   BoundaryInitialVertexFloodFillData *data = userdata;
 
+  if (!SCULPT_vertex_visible_get(ss, to_v)) {
+    return false;
+  }
+
   if (!is_duplicate) {
     data->floodfill_steps[to_v] = data->floodfill_steps[from_v] + 1;
   }
@@ -174,13 +178,19 @@ static bool sculpt_boundary_is_vertex_in_editable_boundary(SculptSession *ss,
                                                            const int initial_vertex)
 {
 
+  if (!SCULPT_vertex_visible_get(ss, initial_vertex)) {
+    return false;
+  }
+
   int neighbor_count = 0;
   int boundary_vertex_count = 0;
   SculptVertexNeighborIter ni;
   SCULPT_VERTEX_NEIGHBORS_ITER_BEGIN (ss, initial_vertex, ni) {
-    neighbor_count++;
-    if (SCULPT_vertex_is_boundary(ss, ni.index)) {
-      boundary_vertex_count++;
+    if (SCULPT_vertex_visible_get(ss, ni.index)) {
+      neighbor_count++;
+      if (SCULPT_vertex_is_boundary(ss, ni.index)) {
+        boundary_vertex_count++;
+      }
     }
   }
   SCULPT_VERTEX_NEIGHBORS_ITER_END(ni);
@@ -349,7 +359,9 @@ static void sculpt_boundary_edit_data_init(SculptSession *ss,
 
       SculptVertexNeighborIter ni;
       SCULPT_VERTEX_DUPLICATES_AND_NEIGHBORS_ITER_BEGIN (ss, from_v, ni) {
-        if (boundary->edit_info[ni.index].num_propagation_steps == BOUNDARY_STEPS_NONE) {
+        const bool is_visible = SCULPT_vertex_visible_get(ss, ni.index);
+        if (is_visible &&
+            boundary->edit_info[ni.index].num_propagation_steps == BOUNDARY_STEPS_NONE) {
           boundary->edit_info[ni.index].original_vertex =
               boundary->edit_info[from_v].original_vertex;