[Bf-blender-cvs] SVN commit: /data/svn/bf-blender [55334] trunk/blender/source/blender: fix for crash when using BM_face_calc_tessellation(), its not ensured that all tris will be filled in.

[Campbell Barton]

Revision: 55334
          http://projects.blender.org/scm/viewvc.php?view=rev&root=bf-blender&revision=55334
Author:   campbellbarton
Date:     2013-03-16 14:18:32 +0000 (Sat, 16 Mar 2013)
Log Message:
-----------
fix for crash when using BM_face_calc_tessellation(), its not ensured that all tris will be filled in.
(effected knife project and laplacian smooth).

Modified Paths:
--------------
    trunk/blender/source/blender/bmesh/intern/bmesh_polygon.c
    trunk/blender/source/blender/bmesh/intern/bmesh_polygon.h
    trunk/blender/source/blender/bmesh/intern/bmesh_queries.c
    trunk/blender/source/blender/editors/mesh/editmesh_knife.c

Modified: trunk/blender/source/blender/bmesh/intern/bmesh_polygon.c
===================================================================
--- trunk/blender/source/blender/bmesh/intern/bmesh_polygon.c	2013-03-16 08:53:32 UTC (rev 55333)
+++ trunk/blender/source/blender/bmesh/intern/bmesh_polygon.c	2013-03-16 14:18:32 UTC (rev 55334)
@@ -42,6 +42,7 @@
 #include "BLI_math.h"
 #include "BLI_array.h"
 #include "BLI_scanfill.h"
+#include "BLI_listbase.h"
 
 #include "bmesh.h"
 
@@ -160,11 +161,12 @@
  * \param r_loops  Store face loop pointers, (f->len)
  * \param r_index  Store triangle triples, indicies into \a r_loops,  ((f->len - 2) * 3)
  */
-void BM_face_calc_tessellation(BMFace *f, BMLoop **r_loops, int (*_r_index)[3])
+int BM_face_calc_tessellation(BMFace *f, BMLoop **r_loops, int (*_r_index)[3])
 {
 	int *r_index = (int *)_r_index;
 	BMLoop *l_first = BM_FACE_FIRST_LOOP(f);
 	BMLoop *l_iter;
+	int totfilltri;
 
 	if (f->len == 3) {
 		*r_loops++ = (l_iter = l_first);
@@ -174,9 +176,9 @@
 		r_index[0] = 0;
 		r_index[1] = 1;
 		r_index[2] = 2;
+		totfilltri = 1;
 	}
 	else if (f->len == 4) {
-		BMLoop *l_iter;
 		*r_loops++ = (l_iter = l_first);
 		*r_loops++ = (l_iter = l_iter->next);
 		*r_loops++ = (l_iter = l_iter->next);
@@ -189,6 +191,7 @@
 		r_index[3] = 0;
 		r_index[4] = 2;
 		r_index[5] = 3;
+		totfilltri = 2;
 	}
 	else {
 		int j;
@@ -197,7 +200,6 @@
 		ScanFillVert *sf_vert, *sf_vert_last = NULL, *sf_vert_first = NULL;
 		/* ScanFillEdge *e; */ /* UNUSED */
 		ScanFillFace *sf_tri;
-		int totfilltri;
 
 		BLI_scanfill_begin(&sf_ctx);
 
@@ -228,7 +230,7 @@
 
 		totfilltri = BLI_scanfill_calc_ex(&sf_ctx, 0, f->no);
 		BLI_assert(totfilltri <= f->len - 2);
-		(void)totfilltri;
+		BLI_assert(totfilltri == BLI_countlist(&sf_ctx.fillfacebase));
 
 		for (sf_tri = sf_ctx.fillfacebase.first; sf_tri; sf_tri = sf_tri->next) {
 			int i1 = BM_elem_index_get((BMLoop *)sf_tri->v1->tmp.p);
@@ -246,6 +248,8 @@
 
 		BLI_scanfill_end(&sf_ctx);
 	}
+
+	return totfilltri;
 }
 
 /**

Modified: trunk/blender/source/blender/bmesh/intern/bmesh_polygon.h
===================================================================
--- trunk/blender/source/blender/bmesh/intern/bmesh_polygon.h	2013-03-16 08:53:32 UTC (rev 55333)
+++ trunk/blender/source/blender/bmesh/intern/bmesh_polygon.h	2013-03-16 14:18:32 UTC (rev 55334)
@@ -27,7 +27,12 @@
  *  \ingroup bmesh
  */
 
-void  BM_face_calc_tessellation(BMFace *f, BMLoop **r_loops, int (*r_index)[3]);
+int   BM_face_calc_tessellation(BMFace *f, BMLoop **r_loops, int (*r_index)[3])
+#ifdef __GNUC__
+	__attribute__((warn_unused_result))
+	__attribute__((nonnull))
+#endif
+;
 float BM_face_calc_area(BMFace *f);
 float BM_face_calc_perimeter(BMFace *f);
 void  BM_face_calc_center_bounds(BMFace *f, float center[3]);

Modified: trunk/blender/source/blender/bmesh/intern/bmesh_queries.c
===================================================================
--- trunk/blender/source/blender/bmesh/intern/bmesh_queries.c	2013-03-16 08:53:32 UTC (rev 55333)
+++ trunk/blender/source/blender/bmesh/intern/bmesh_queries.c	2013-03-16 14:18:32 UTC (rev 55334)
@@ -1657,12 +1657,13 @@
 
 static void bm_mesh_calc_volume_face(BMFace *f, float *r_vol)
 {
-	const int tottri = f->len - 2;
+	int tottri = f->len - 2;
 	BMLoop **loops     = BLI_array_alloca(loops, f->len);
 	int    (*index)[3] = BLI_array_alloca(index, tottri);
 	int j;
 
-	BM_face_calc_tessellation(f, loops, index);
+	tottri = BM_face_calc_tessellation(f, loops, index);
+	BLI_assert(tottri <= f->len - 2);
 
 	for (j = 0; j < tottri; j++) {
 		const float *p1 = loops[index[j][0]]->v->co;

Modified: trunk/blender/source/blender/editors/mesh/editmesh_knife.c
===================================================================
--- trunk/blender/source/blender/editors/mesh/editmesh_knife.c	2013-03-16 08:53:32 UTC (rev 55333)
+++ trunk/blender/source/blender/editors/mesh/editmesh_knife.c	2013-03-16 14:18:32 UTC (rev 55334)
@@ -3312,15 +3312,17 @@
  */
 static void edvm_mesh_knife_face_point(BMFace *f, float r_cent[3])
 {
-	const int tottri = f->len - 2;
+	int tottri = f->len - 2;
 	BMLoop **loops     = BLI_array_alloca(loops, f->len);
 	int    (*index)[3] = BLI_array_alloca(index, tottri);
 	int j;
 
 	float const *best_co[3] = {NULL};
 	float  best_area  = -1.0f;
+	bool ok = false;
 
-	BM_face_calc_tessellation(f, loops, index);
+	tottri = BM_face_calc_tessellation(f, loops, index);
+	BLI_assert(tottri <= f->len - 2);
 
 	for (j = 0; j < tottri; j++) {
 		const float *p1 = loops[index[j][0]]->v->co;
@@ -3336,10 +3338,16 @@
 			best_co[1] = p2;
 			best_co[2] = p3;
 			best_area = area;
+			ok = true;
 		}
 	}
 
-	mid_v3_v3v3v3(r_cent, best_co[0], best_co[1], best_co[2]);
+	if (ok) {
+		mid_v3_v3v3v3(r_cent, best_co[0], best_co[1], best_co[2]);
+	}
+	else {
+		mid_v3_v3v3v3(r_cent, loops[0]->v->co, loops[1]->v->co, loops[2]->v->co);
+	}
 }
 
 static bool edbm_mesh_knife_face_isect(ARegion *ar, LinkNode *polys, BMFace *f, float projmat[4][4])