[Bf-blender-cvs] [918ad17] master: Transform: improve normal orientation

[Campbell Barton]

Commit: 918ad1719f5ece6cd1da9cc3c0933620b83a2526
Author: Campbell Barton
Date:   Thu Nov 28 19:27:53 2013 +1100
http://developer.blender.org/rB918ad1719f5ece6cd1da9cc3c0933620b83a2526

Transform: improve normal orientation

- when 3 verts are selected ensure the normal is flipped on the side of existing vert normals.
  also use the most distant 2 verts to define the tangent.

- when 2 vertices are selected, the normal wasn't aligned with the vert normal.

===================================================================

M	source/blender/blenlib/BLI_utildefines.h
M	source/blender/editors/transform/transform_orientations.c

===================================================================

diff --git a/source/blender/blenlib/BLI_utildefines.h b/source/blender/blenlib/BLI_utildefines.h
index 9bf720c..0f7c0ac 100644
--- a/source/blender/blenlib/BLI_utildefines.h
+++ b/source/blender/blenlib/BLI_utildefines.h
@@ -56,6 +56,15 @@
 #define MAX3(x, y, z)       (MAX2(MAX2((x), (y)), (z)))
 #define MAX4(x, y, z, a)    (MAX2(MAX2((x), (y)), MAX2((z), (a))))
 
+/* min/max that return a value of our choice */
+#define MAX3_PAIR(cmp_a, cmp_b, cmp_c, ret_a, ret_b, ret_c) \
+	((cmp_a > cmp_b) ? ((cmp_a > cmp_c) ? ret_a : ret_c) : \
+	                   ((cmp_b > cmp_c) ? ret_b : ret_c))
+
+#define MIN3_PAIR(cmp_a, cmp_b, cmp_c, ret_a, ret_b, ret_c) \
+	((cmp_a < cmp_b) ? ((cmp_a < cmp_c) ? ret_a : ret_c) : \
+	                   ((cmp_b < cmp_c) ? ret_b : ret_c))
+
 #define INIT_MINMAX(min, max) {                                               \
 		(min)[0] = (min)[1] = (min)[2] =  1.0e30f;                            \
 		(max)[0] = (max)[1] = (max)[2] = -1.0e30f;                            \
diff --git a/source/blender/editors/transform/transform_orientations.c b/source/blender/editors/transform/transform_orientations.c
index 82c2d38..6dc685c 100644
--- a/source/blender/editors/transform/transform_orientations.c
+++ b/source/blender/editors/transform/transform_orientations.c
@@ -577,7 +577,6 @@ int getTransformOrientation(const bContext *C, float normal[3], float plane[3],
 				else if (em->bm->totvertsel == 3) {
 					BMVert *v1 = NULL, *v2 = NULL, *v3 = NULL;
 					BMIter iter;
-					float cotangent[3];
 					
 					BM_ITER_MESH (eve, &iter, em->bm, BM_VERTS_OF_MESH) {
 						if (BM_elem_flag_test(eve, BM_ELEM_SELECT)) {
@@ -588,11 +587,37 @@ int getTransformOrientation(const bContext *C, float normal[3], float plane[3],
 								v2 = eve;
 							}
 							else {
+								float no_test[3];
+
+								float tan_a[3], tan_b[3], tan_c[3];
+								float len_a, len_b, len_c;
+								const float *tan_best;
+
+
 								v3 = eve;
+								sub_v3_v3v3(tan_a, v2->co, v1->co);
+								sub_v3_v3v3(tan_b, v3->co, v2->co);
+								sub_v3_v3v3(tan_c, v1->co, v3->co);
+								cross_v3_v3v3(normal, tan_b, tan_a);
+
+								/* check if the normal is pointing opposite to vert normals */
+								no_test[0] = v1->no[0] + v2->no[0] + v3->no[0];
+								no_test[1] = v1->no[1] + v2->no[1] + v3->no[1];
+								no_test[2] = v1->no[2] + v2->no[2] + v3->no[2];
+								if (dot_v3v3(no_test, normal) < 0.0f) {
+									negate_v3(normal);
+								}
+
+								/* always give the plane to the 2 most distant verts */
+								len_a = len_squared_v3(tan_a);
+								len_b = len_squared_v3(tan_b);
+								len_c = len_squared_v3(tan_c);
+
+								tan_best = MAX3_PAIR(len_a, len_b, len_c,
+								                     tan_a, tan_b, tan_c);
+
+								copy_v3_v3(plane, tan_best);
 
-								sub_v3_v3v3(plane, v2->co, v1->co);
-								sub_v3_v3v3(cotangent, v3->co, v2->co);
-								cross_v3_v3v3(normal, cotangent, plane);
 								break;
 							}
 						}
@@ -639,9 +664,9 @@ int getTransformOrientation(const bContext *C, float normal[3], float plane[3],
 							else {
 								v2 = eve;
 								
-								copy_v3_v3(plane, v1->no);
-								add_v3_v3(plane, v2->no);
-								sub_v3_v3v3(normal, v2->co, v1->co);
+								copy_v3_v3(normal, v1->no);
+								add_v3_v3(normal, v2->no);
+								sub_v3_v3v3(plane, v2->co, v1->co);
 								break; 
 							}
 						}