[Bf-blender-cvs] [c4bbe44e472] blender-v2.92-release: Add Object Tool: workaround non-orthogonal orientation matrices

[Campbell Barton]

Commit: c4bbe44e472b97911127ce354954ff7e9e290a3d
Author: Campbell Barton
Date:   Sat Jan 16 11:29:56 2021 +1100
Branches: blender-v2.92-release
https://developer.blender.org/rBc4bbe44e472b97911127ce354954ff7e9e290a3d

Add Object Tool: workaround non-orthogonal orientation matrices

===================================================================

M	source/blender/editors/space_view3d/view3d_placement.c
M	source/blender/editors/transform/transform_orientations.c

===================================================================

diff --git a/source/blender/editors/space_view3d/view3d_placement.c b/source/blender/editors/space_view3d/view3d_placement.c
index 638c61fc331..5bd1b3458e2 100644
--- a/source/blender/editors/space_view3d/view3d_placement.c
+++ b/source/blender/editors/space_view3d/view3d_placement.c
@@ -795,6 +795,12 @@ static void view3d_interactive_add_calc_plane(bContext *C,
   const RegionView3D *rv3d = region->regiondata;
   ED_transform_calc_orientation_from_type(C, r_matrix_orient);
 
+  /* Non-orthogonal matrices cause the preview and final result not to match.
+   *
+   * While making orthogonal doesn't always work well (especially with gimbal orientation for e.g.)
+   * it's a corner case, without better alternatives as objects don't support shear. */
+  orthogonalize_m3(r_matrix_orient, plane_axis);
+
   SnapObjectContext *snap_context = NULL;
   bool snap_context_free = false;
 
diff --git a/source/blender/editors/transform/transform_orientations.c b/source/blender/editors/transform/transform_orientations.c
index 606862f05aa..90b2ce2a385 100644
--- a/source/blender/editors/transform/transform_orientations.c
+++ b/source/blender/editors/transform/transform_orientations.c
@@ -500,6 +500,15 @@ void ED_transform_calc_orientation_from_type(const bContext *C, float r_mat[3][3
       C, r_mat, scene, rv3d, ob, obedit, orientation_type, orientation_index_custom, pivot_point);
 }
 
+/**
+ * \note The resulting matrix may not be orthogonal,
+ * callers that depend on `r_mat` to be orthogonal should use #orthogonalize_m3.
+ *
+ * A non orthogonal matrix may be returned when:
+ * - #V3D_ORIENT_GIMBAL the result won't be orthogonal unless the object has no rotation.
+ * - #V3D_ORIENT_LOCAL may contain shear from non-uniform scale in parent/child relationships.
+ * - #V3D_ORIENT_CUSTOM may have been created from #V3D_ORIENT_LOCAL.
+ */
 short ED_transform_calc_orientation_from_type_ex(const bContext *C,
                                                  float r_mat[3][3],
                                                  /* extra args (can be accessed from context) */