[Bf-blender-cvs] [dd82a0d6239] master: GPencil: Fix unreported problem with strokes bounding box that makes impossible sculpt

[Antonio Vazquez]

Commit: dd82a0d6239124a14bb554296364f6c951a8491e
Author: Antonio Vazquez
Date:   Wed Oct 21 17:23:46 2020 +0200
Branches: master
https://developer.blender.org/rBdd82a0d6239124a14bb554296364f6c951a8491e

GPencil: Fix unreported problem with strokes bounding box that makes impossible sculpt

When the stroke was not flat, the bounding box projected could not be right and the strokes could not be painted or sculpted.

Now, the 2D bounding box is calculated using the extremes of the 2D bounding box and not the original 3D min and max values.

===================================================================

M	source/blender/editors/gpencil/gpencil_utils.c

===================================================================

diff --git a/source/blender/editors/gpencil/gpencil_utils.c b/source/blender/editors/gpencil/gpencil_utils.c
index 5a45afe9c5a..28a90a26a9d 100644
--- a/source/blender/editors/gpencil/gpencil_utils.c
+++ b/source/blender/editors/gpencil/gpencil_utils.c
@@ -2916,6 +2916,40 @@ void ED_gpencil_sbuffer_vertex_color_set(Depsgraph *depsgraph,
   }
 }
 
+/* Helper to get the bigger 2D bound box points. */
+static void gpencil_projected_2d_bound_box(GP_SpaceConversion *gsc,
+                                           bGPDstroke *gps,
+                                           const float diff_mat[4][4],
+                                           float r_min[2],
+                                           float r_max[2])
+{
+  float bounds[8][2];
+  BoundBox bb;
+  BKE_boundbox_init_from_minmax(&bb, gps->boundbox_min, gps->boundbox_max);
+
+  /* Project 8 vertices in 2D. */
+  for (int i = 0; i < 8; i++) {
+    bGPDspoint pt_dummy, pt_dummy_ps;
+    copy_v3_v3(&pt_dummy.x, bb.vec[i]);
+    gpencil_point_to_parent_space(&pt_dummy, diff_mat, &pt_dummy_ps);
+    gpencil_point_to_xy_fl(gsc, gps, &pt_dummy_ps, &bounds[i][0], &bounds[i][1]);
+  }
+
+  /* Take extremes. */
+  INIT_MINMAX2(r_min, r_max);
+  for (int i = 0; i < 8; i++) {
+    minmax_v2v2_v2(r_min, r_max, bounds[i]);
+  }
+
+  /* Ensure the bounding box is oriented to axis. */
+  if (r_max[0] < r_min[0]) {
+    SWAP(float, r_min[0], r_max[0]);
+  }
+  if (r_max[1] < r_min[1]) {
+    SWAP(float, r_min[1], r_max[1]);
+  }
+}
+
 /* Check if the stroke collides with brush. */
 bool ED_gpencil_stroke_check_collision(GP_SpaceConversion *gsc,
                                        bGPDstroke *gps,
@@ -2924,31 +2958,15 @@ bool ED_gpencil_stroke_check_collision(GP_SpaceConversion *gsc,
                                        const float diff_mat[4][4])
 {
   const int offset = (int)ceil(sqrt((radius * radius) * 2));
-  bGPDspoint pt_dummy, pt_dummy_ps;
-  float boundbox_min[2] = {0.0f};
-  float boundbox_max[2] = {0.0f};
+  float boundbox_min[2];
+  float boundbox_max[2];
 
   /* Check we have something to use (only for old files). */
   if (is_zero_v3(gps->boundbox_min)) {
     BKE_gpencil_stroke_boundingbox_calc(gps);
   }
 
-  /* Convert bound box to 2d */
-  copy_v3_v3(&pt_dummy.x, gps->boundbox_min);
-  gpencil_point_to_parent_space(&pt_dummy, diff_mat, &pt_dummy_ps);
-  gpencil_point_to_xy_fl(gsc, gps, &pt_dummy_ps, &boundbox_min[0], &boundbox_min[1]);
-
-  copy_v3_v3(&pt_dummy.x, gps->boundbox_max);
-  gpencil_point_to_parent_space(&pt_dummy, diff_mat, &pt_dummy_ps);
-  gpencil_point_to_xy_fl(gsc, gps, &pt_dummy_ps, &boundbox_max[0], &boundbox_max[1]);
-
-  /* Ensure the bounding box is oriented to axis. */
-  if (boundbox_max[0] < boundbox_min[0]) {
-    SWAP(float, boundbox_min[0], boundbox_max[0]);
-  }
-  if (boundbox_max[1] < boundbox_min[1]) {
-    SWAP(float, boundbox_min[1], boundbox_max[1]);
-  }
+  gpencil_projected_2d_bound_box(gsc, gps, diff_mat, boundbox_min, boundbox_max);
 
   rcti rect_stroke = {boundbox_min[0], boundbox_max[0], boundbox_min[1], boundbox_max[1]};