[Bf-blender-cvs] [208462e424d] master: Fix T51539: BMesh boolean crash

[Campbell Barton]

Commit: 208462e424d319b3590d0b09cacf0b3807239972
Author: Campbell Barton
Date:   Thu May 18 21:06:45 2017 +1000
Branches: master
https://developer.blender.org/rB208462e424d319b3590d0b09cacf0b3807239972

Fix T51539: BMesh boolean crash

In some cases the vertex visit queue would exceed its max length,
visiting the same vertex multiple times.

===================================================================

M	source/blender/bmesh/intern/bmesh_polygon_edgenet.c

===================================================================

diff --git a/source/blender/bmesh/intern/bmesh_polygon_edgenet.c b/source/blender/bmesh/intern/bmesh_polygon_edgenet.c
index 0cd38f24165..64233151b04 100644
--- a/source/blender/bmesh/intern/bmesh_polygon_edgenet.c
+++ b/source/blender/bmesh/intern/bmesh_polygon_edgenet.c
@@ -512,13 +512,21 @@ bool BM_face_split_edgenet(
 	} while ((l_iter = l_iter->next) != l_first);
 #endif
 
+	/* Note: 'VERT_VISIT' is often not needed at all,
+	 * however in rare cases verts are added multiple times to the queue,
+	 * that on it's own is harmless but in _very_ rare cases,
+	 * the queue will overflow its maximum size,
+	 * so we better be strict about this! see: T51539 */
 
 	for (i = 0; i < edge_net_len; i++) {
 		BM_ELEM_API_FLAG_ENABLE(edge_net[i], EDGE_NET);
+		BM_ELEM_API_FLAG_DISABLE(edge_net[i]->v1, VERT_VISIT);
+		BM_ELEM_API_FLAG_DISABLE(edge_net[i]->v2, VERT_VISIT);
 	}
 	l_iter = l_first = BM_FACE_FIRST_LOOP(f);
 	do {
 		BM_ELEM_API_FLAG_ENABLE(l_iter->e, EDGE_NET);
+		BM_ELEM_API_FLAG_DISABLE(l_iter->v, VERT_VISIT);
 	} while ((l_iter = l_iter->next) != l_first);
 
 	float face_normal_matrix[3][3];
@@ -527,8 +535,10 @@ bool BM_face_split_edgenet(
 
 	/* any vert can be used to begin with */
 	STACK_PUSH(vert_queue, l_first->v);
+	BM_ELEM_API_FLAG_ENABLE(l_first->v, VERT_VISIT);
 
 	while ((v = STACK_POP(vert_queue))) {
+		BM_ELEM_API_FLAG_DISABLE(v, VERT_VISIT);
 		if (bm_face_split_edgenet_find_loop(
 		        v, f->no, face_normal_matrix,
 		        edge_order, edge_order_len, face_verts, &face_verts_len))
@@ -558,8 +568,12 @@ bool BM_face_split_edgenet(
 				 * (verts between boundary and manifold edges) */
 				l_iter = l_first = BM_FACE_FIRST_LOOP(f_new);
 				do {
-					if (bm_face_split_edgenet_find_loop_pair_exists(l_iter->v)) {
+					/* Avoid adding to queue multiple times (not common but happens). */
+					if (BM_ELEM_API_FLAG_TEST(l_iter->v, VERT_VISIT) &&
+					    bm_face_split_edgenet_find_loop_pair_exists(l_iter->v))
+					{
 						STACK_PUSH(vert_queue, l_iter->v);
+						BM_ELEM_API_FLAG_ENABLE(l_iter->v, VERT_VISIT);
 					}
 				} while ((l_iter = l_iter->next) != l_first);
 			}