[Bf-blender-cvs] SVN commit: /data/svn/bf-blender [49884] trunk/blender/source/blender: fix [#32315] Circle select unreliable when in vertex+edge select mode

[Campbell Barton]

Revision: 49884
          http://projects.blender.org/scm/viewvc.php?view=rev&root=bf-blender&revision=49884
Author:   campbellbarton
Date:     2012-08-14 08:24:49 +0000 (Tue, 14 Aug 2012)
Log Message:
-----------
fix [#32315] Circle select unreliable when in vertex+edge select mode

dist_squared_to_line_segment_v2() was returning the sqrt'd value in some cases.
also use int's for edge_inside_circle() rather then shorts since it was doing int/float/short conversions and we're now using int's for screen vars in more places.

Modified Paths:
--------------
    trunk/blender/source/blender/blenlib/intern/math_geom.c
    trunk/blender/source/blender/editors/include/ED_view3d.h
    trunk/blender/source/blender/editors/space_view3d/view3d_select.c

Modified: trunk/blender/source/blender/blenlib/intern/math_geom.c
===================================================================
--- trunk/blender/source/blender/blenlib/intern/math_geom.c	2012-08-14 00:09:29 UTC (rev 49883)
+++ trunk/blender/source/blender/blenlib/intern/math_geom.c	2012-08-14 08:24:49 UTC (rev 49884)
@@ -188,7 +188,7 @@
 	if (len == 0.0f) {
 		rc[0] = p[0] - l1[0];
 		rc[1] = p[1] - l1[1];
-		return (float)(sqrt(rc[0] * rc[0] + rc[1] * rc[1]));
+		return (rc[0] * rc[0] + rc[1] * rc[1]);
 	}
 
 	labda = (rc[0] * (p[0] - l1[0]) + rc[1] * (p[1] - l1[1])) / len;

Modified: trunk/blender/source/blender/editors/include/ED_view3d.h
===================================================================
--- trunk/blender/source/blender/editors/include/ED_view3d.h	2012-08-14 00:09:29 UTC (rev 49883)
+++ trunk/blender/source/blender/editors/include/ED_view3d.h	2012-08-14 08:24:49 UTC (rev 49884)
@@ -268,7 +268,7 @@
 void view3d_get_transformation(const struct ARegion *ar, struct RegionView3D *rv3d, struct Object *ob, struct bglMats *mats);
 
 /* XXX should move to BLI_math */
-int edge_inside_circle(short centx, short centy, short rad, short x1, short y1, short x2, short y2);
+int edge_inside_circle(int centx, int centy, int rad, int x1, int y1, int x2, int y2);
 
 /* get 3d region from context, also if mouse is in header or toolbar */
 struct RegionView3D *ED_view3d_context_rv3d(struct bContext *C);

Modified: trunk/blender/source/blender/editors/space_view3d/view3d_select.c
===================================================================
--- trunk/blender/source/blender/editors/space_view3d/view3d_select.c	2012-08-14 00:09:29 UTC (rev 49883)
+++ trunk/blender/source/blender/editors/space_view3d/view3d_select.c	2012-08-14 08:24:49 UTC (rev 49884)
@@ -1523,26 +1523,28 @@
 	int select, pass, done;
 } BoxSelectUserData;
 
-int edge_inside_circle(short centx, short centy, short rad, short x1, short y1, short x2, short y2)
+int edge_inside_circle(int centx, int centy, int rad, int x1, int y1, int x2, int y2)
 {
 	int radsq = rad * rad;
-	float v1[2], v2[2], v3[2];
-	
+
 	/* check points in circle itself */
-	if ( (x1 - centx) * (x1 - centx) + (y1 - centy) * (y1 - centy) <= radsq) return 1;
-	if ( (x2 - centx) * (x2 - centx) + (y2 - centy) * (y2 - centy) <= radsq) return 1;
-	
-	/* pointdistline */
-	v3[0] = centx;
-	v3[1] = centy;
-	v1[0] = x1;
-	v1[1] = y1;
-	v2[0] = x2;
-	v2[1] = y2;
-	
-	if (dist_to_line_segment_v2(v3, v1, v2) < (float)rad) return 1;
-	
-	return 0;
+	if ( (x1 - centx) * (x1 - centx) + (y1 - centy) * (y1 - centy) <= radsq) {
+		return TRUE;
+	}
+	else if ( (x2 - centx) * (x2 - centx) + (y2 - centy) * (y2 - centy) <= radsq) {
+		return TRUE;
+	}
+	else {
+		const float cent[2] = {centx, centy};
+		const float v1[2] = {x1, y1};
+		const float v2[2] = {x2, y2};
+		/* pointdistline */
+		if (dist_squared_to_line_segment_v2(cent, v1, v2) < (float)radsq) {
+			return TRUE;
+		}
+	}
+
+	return FALSE;
 }
 
 static void do_nurbs_box_select__doSelect(void *userData, Nurb *UNUSED(nu), BPoint *bp, BezTriple *bezt, int beztindex, int x, int y)
@@ -2192,7 +2194,7 @@
 {
 	CircleSelectUserData *data = userData;
 
-	if (edge_inside_circle(data->mval[0], data->mval[1], (short) data->radius, x0, y0, x1, y1)) {
+	if (edge_inside_circle(data->mval[0], data->mval[1], (int)data->radius, x0, y0, x1, y1)) {
 		BM_edge_select_set(data->vc->em->bm, eed, data->select);
 	}
 }