[Bf-blender-cvs] SVN commit: /data/svn/bf-blender [32732] trunk/blender/source/blender/ editors/space_view3d: recent fix r32626 for [#24339] wasnt correctly clamping the depth rectangle, crashing with grease pencil in some cases.

[Campbell Barton]

Revision: 32732
          http://projects.blender.org/plugins/scmsvn/viewcvs.php?view=rev&root=bf-blender&revision=32732
Author:   campbellbarton
Date:     2010-10-27 10:55:13 +0200 (Wed, 27 Oct 2010)

Log Message:
-----------
recent fix r32626 for [#24339] wasnt correctly clamping the depth rectangle, crashing with grease pencil in some cases.

Modified Paths:
--------------
    trunk/blender/source/blender/editors/space_view3d/view3d_draw.c
    trunk/blender/source/blender/editors/space_view3d/view3d_edit.c

Modified: trunk/blender/source/blender/editors/space_view3d/view3d_draw.c
===================================================================
--- trunk/blender/source/blender/editors/space_view3d/view3d_draw.c	2010-10-27 08:53:40 UTC (rev 32731)
+++ trunk/blender/source/blender/editors/space_view3d/view3d_draw.c	2010-10-27 08:55:13 UTC (rev 32732)
@@ -1573,22 +1573,27 @@
 void view3d_update_depths_rect(ARegion *ar, ViewDepths *d, rcti *rect)
 {
 	int x, y, w, h;	
+	rcti r= {0, ar->winx-1, 0, ar->winy-1};
 	/* clamp rect by area */
-	
+
 	/* Constrain rect to depth bounds */
-	if (rect->xmin < 0) rect->xmin = 0;
-	if (rect->ymin < 0) rect->ymin = 0;
-	if (rect->xmax >= ar->winx) rect->xmax = ar->winx-1;
-	if (rect->ymax >= ar->winy) rect->ymax = ar->winy-1;
+	BLI_isect_rcti(&r, rect, rect);
 
 	/* assign values to compare with the ViewDepths */
-	x= ar->winrct.xmin + rect->xmin;
-	y= ar->winrct.ymin + rect->ymin;
+	x= rect->xmin;
+	y= rect->ymin;
 
 	w= rect->xmax - rect->xmin;
 	h= rect->ymax - rect->ymin;
 
-	if(	d->w != w ||
+	if(w <= 0 || h <= 0) {
+		if(d->depths)
+			MEM_freeN(d->depths);
+		d->depths= NULL;
+
+		d->damaged= FALSE;
+	}
+	else if(	d->w != w ||
 		d->h != h ||
 		d->x != x ||
 		d->y != y ||
@@ -1604,13 +1609,13 @@
 
 		d->depths= MEM_mallocN(sizeof(float)*d->w*d->h,"View depths Subset");
 		
-		d->damaged= 1;		
+		d->damaged= TRUE;
 	}
 
 	if(d->damaged) {
 		glReadPixels(ar->winrct.xmin+d->x,ar->winrct.ymin+d->y, d->w,d->h, GL_DEPTH_COMPONENT,GL_FLOAT, d->depths);
 		glGetDoublev(GL_DEPTH_RANGE,d->depth_range);
-		d->damaged= 0;
+		d->damaged= FALSE;
 	}
 }
 
@@ -1655,7 +1660,7 @@
 
 	const float *depths= d->depths;
 	float depth= FLT_MAX;
-	int i= d->w * d->h;
+	int i= (int)d->w * (int)d->h; /* cast to avoid short overflow */
 
 	/* far is both the starting 'far' value
 	 * and the closest value found. */	

Modified: trunk/blender/source/blender/editors/space_view3d/view3d_edit.c
===================================================================
--- trunk/blender/source/blender/editors/space_view3d/view3d_edit.c	2010-10-27 08:53:40 UTC (rev 32731)
+++ trunk/blender/source/blender/editors/space_view3d/view3d_edit.c	2010-10-27 08:55:13 UTC (rev 32732)
@@ -2575,10 +2575,8 @@
 
 	view3d_update_depths_rect(ar, &depth_temp, &rect);
 	depth_close= view3d_depth_near(&depth_temp);
-
-	MEM_freeN(depth_temp.depths);
-
-	return depth_close;	
+	if(depth_temp.depths) MEM_freeN(depth_temp.depths);
+	return depth_close;
 }
 
 /* XXX todo Zooms in on a border drawn by the user */