[Bf-blender-cvs] SVN commit: /data/svn/bf-blender [56396] trunk/blender/source/blender: fix [#35150] Crash when bmesh operation called from within a Panel draw()

[Campbell Barton]

Revision: 56396
          http://projects.blender.org/scm/viewvc.php?view=rev&root=bf-blender&revision=56396
Author:   campbellbarton
Date:     2013-04-29 20:21:19 +0000 (Mon, 29 Apr 2013)
Log Message:
-----------
fix [#35150] Crash when bmesh operation called from within a Panel draw()

accessing a bmesh from python would reallocate all customdata layers.

add an assert to BM_data_layer_free(), when its called unnecessarily since its reallocating all layers.

Modified Paths:
--------------
    trunk/blender/source/blender/bmesh/intern/bmesh_interp.c
    trunk/blender/source/blender/python/bmesh/bmesh_py_types.c

Modified: trunk/blender/source/blender/bmesh/intern/bmesh_interp.c
===================================================================
--- trunk/blender/source/blender/bmesh/intern/bmesh_interp.c	2013-04-29 19:15:56 UTC (rev 56395)
+++ trunk/blender/source/blender/bmesh/intern/bmesh_interp.c	2013-04-29 20:21:19 UTC (rev 56396)
@@ -804,6 +804,7 @@
 void BM_data_layer_free(BMesh *bm, CustomData *data, int type)
 {
 	CustomData olddata;
+	bool has_layer;
 
 	olddata = *data;
 	olddata.layers = (olddata.layers) ? MEM_dupallocN(olddata.layers): NULL;
@@ -811,7 +812,9 @@
 	/* the pool is now owned by olddata and must not be shared */
 	data->pool = NULL;
 
-	CustomData_free_layer_active(data, type, 0);
+	has_layer = CustomData_free_layer_active(data, type, 0);
+	/* assert because its expensive to realloc - better not do if layer isnt present */
+	BLI_assert(has_layer != false);
 
 	update_data_blocks(bm, &olddata, data);
 	if (olddata.layers) MEM_freeN(olddata.layers);
@@ -820,6 +823,7 @@
 void BM_data_layer_free_n(BMesh *bm, CustomData *data, int type, int n)
 {
 	CustomData olddata;
+	bool has_layer;
 
 	olddata = *data;
 	olddata.layers = (olddata.layers) ? MEM_dupallocN(olddata.layers): NULL;
@@ -827,7 +831,9 @@
 	/* the pool is now owned by olddata and must not be shared */
 	data->pool = NULL;
 
-	CustomData_free_layer(data, type, 0, CustomData_get_layer_index_n(data, type, n));
+	has_layer = CustomData_free_layer(data, type, 0, CustomData_get_layer_index_n(data, type, n));
+	/* assert because its expensive to realloc - better not do if layer isnt present */
+	BLI_assert(has_layer != false);
 	
 	update_data_blocks(bm, &olddata, data);
 	if (olddata.layers) MEM_freeN(olddata.layers);

Modified: trunk/blender/source/blender/python/bmesh/bmesh_py_types.c
===================================================================
--- trunk/blender/source/blender/python/bmesh/bmesh_py_types.c	2013-04-29 19:15:56 UTC (rev 56395)
+++ trunk/blender/source/blender/python/bmesh/bmesh_py_types.c	2013-04-29 20:21:19 UTC (rev 56396)
@@ -2943,10 +2943,10 @@
 	if (bm) {
 		bm_dealloc_editmode_warn(self);
 
-		BM_data_layer_free(bm, &bm->vdata, CD_BM_ELEM_PYPTR);
-		BM_data_layer_free(bm, &bm->edata, CD_BM_ELEM_PYPTR);
-		BM_data_layer_free(bm, &bm->pdata, CD_BM_ELEM_PYPTR);
-		BM_data_layer_free(bm, &bm->ldata, CD_BM_ELEM_PYPTR);
+		if (CustomData_has_layer(&bm->vdata, CD_BM_ELEM_PYPTR)) BM_data_layer_free(bm, &bm->vdata, CD_BM_ELEM_PYPTR);
+		if (CustomData_has_layer(&bm->edata, CD_BM_ELEM_PYPTR)) BM_data_layer_free(bm, &bm->edata, CD_BM_ELEM_PYPTR);
+		if (CustomData_has_layer(&bm->pdata, CD_BM_ELEM_PYPTR)) BM_data_layer_free(bm, &bm->pdata, CD_BM_ELEM_PYPTR);
+		if (CustomData_has_layer(&bm->ldata, CD_BM_ELEM_PYPTR)) BM_data_layer_free(bm, &bm->ldata, CD_BM_ELEM_PYPTR);
 
 		bm->py_handle = NULL;
 
@@ -3370,10 +3370,13 @@
 
 		bm->py_handle = self; /* point back */
 
+		/* avoid allocating layers when we don't have to */
+#if 0
 		BM_data_layer_add(bm, &bm->vdata, CD_BM_ELEM_PYPTR);
 		BM_data_layer_add(bm, &bm->edata, CD_BM_ELEM_PYPTR);
 		BM_data_layer_add(bm, &bm->pdata, CD_BM_ELEM_PYPTR);
 		BM_data_layer_add(bm, &bm->ldata, CD_BM_ELEM_PYPTR);
+#endif
 	}
 
 	return (PyObject *)self;