[Bf-blender-cvs] [fc8a7a44b20] master: Fix T77560: Bone selection crashes

[Campbell Barton]

Commit: fc8a7a44b200e37066a8753c514564c674eace30
Author: Campbell Barton
Date:   Wed Jun 10 17:50:11 2020 +1000
Branches: master
https://developer.blender.org/rBfc8a7a44b200e37066a8753c514564c674eace30

Fix T77560: Bone selection crashes

The was caused by 8b347fc2cdc67 as the old BONESEL_NOSEL flag
handled the -1 case (used for none).

Instead of checking for -1, remove these elements from the array
as selection code makes decisions based on the number of hits detected.

===================================================================

M	source/blender/editors/armature/armature_select.c
M	source/blender/editors/include/ED_view3d.h
M	source/blender/editors/space_view3d/view3d_view.c
M	source/blender/gpu/GPU_select.h
M	source/blender/gpu/intern/gpu_select.c

===================================================================

diff --git a/source/blender/editors/armature/armature_select.c b/source/blender/editors/armature/armature_select.c
index 8e03dbc7dc3..eb7c1bc74ea 100644
--- a/source/blender/editors/armature/armature_select.c
+++ b/source/blender/editors/armature/armature_select.c
@@ -678,6 +678,8 @@ static EditBone *get_nearest_editbonepoint(
 
   /* matching logic from 'mixed_bones_object_selectbuffer' */
   int hits = 0;
+  /* Don't use hits with this ID, (armature drawing uses this). */
+  const int select_id_ignore = -1;
 
   /* we _must_ end cache before return, use 'goto cache_end' */
   view3d_opengl_select_cache_begin();
@@ -688,8 +690,9 @@ static EditBone *get_nearest_editbonepoint(
 
     rcti rect;
     BLI_rcti_init_pt_radius(&rect, vc->mval, 12);
-    const int hits12 = view3d_opengl_select(
-        vc, buffer, MAXPICKBUF, &rect, select_mode, select_filter);
+    const int hits12 = view3d_opengl_select_with_id_filter(
+        vc, buffer, MAXPICKBUF, &rect, select_mode, select_filter, select_id_ignore);
+
     if (hits12 == 1) {
       hits = selectbuffer_ret_hits_12(buffer, hits12);
       goto cache_end;
@@ -699,8 +702,13 @@ static EditBone *get_nearest_editbonepoint(
 
       offs = 4 * hits12;
       BLI_rcti_init_pt_radius(&rect, vc->mval, 5);
-      const int hits5 = view3d_opengl_select(
-          vc, buffer + offs, MAXPICKBUF - offs, &rect, select_mode, select_filter);
+      const int hits5 = view3d_opengl_select_with_id_filter(vc,
+                                                            buffer + offs,
+                                                            MAXPICKBUF - offs,
+                                                            &rect,
+                                                            select_mode,
+                                                            select_filter,
+                                                            select_id_ignore);
 
       if (hits5 == 1) {
         hits = selectbuffer_ret_hits_5(buffer, hits12, hits5);
diff --git a/source/blender/editors/include/ED_view3d.h b/source/blender/editors/include/ED_view3d.h
index beca517f0a6..5e706856738 100644
--- a/source/blender/editors/include/ED_view3d.h
+++ b/source/blender/editors/include/ED_view3d.h
@@ -544,6 +544,13 @@ int view3d_opengl_select(struct ViewContext *vc,
                          const struct rcti *input,
                          eV3DSelectMode select_mode,
                          eV3DSelectObjectFilter select_filter);
+int view3d_opengl_select_with_id_filter(struct ViewContext *vc,
+                                        unsigned int *buffer,
+                                        unsigned int bufsize,
+                                        const struct rcti *input,
+                                        eV3DSelectMode select_mode,
+                                        eV3DSelectObjectFilter select_filter,
+                                        uint select_id);
 
 /* view3d_select.c */
 float ED_view3d_select_dist_px(void);
diff --git a/source/blender/editors/space_view3d/view3d_view.c b/source/blender/editors/space_view3d/view3d_view.c
index fe77ca05a04..3fc990160d2 100644
--- a/source/blender/editors/space_view3d/view3d_view.c
+++ b/source/blender/editors/space_view3d/view3d_view.c
@@ -1192,6 +1192,24 @@ finally:
   return hits;
 }
 
+int view3d_opengl_select_with_id_filter(ViewContext *vc,
+                                        uint *buffer,
+                                        uint bufsize,
+                                        const rcti *input,
+                                        eV3DSelectMode select_mode,
+                                        eV3DSelectObjectFilter select_filter,
+                                        uint select_id)
+{
+  int hits = view3d_opengl_select(vc, buffer, bufsize, input, select_mode, select_filter);
+
+  /* Selection sometimes uses -1 for an invalid selection ID, remove these as they
+   * interfere with detection of actual number of hits in the selection. */
+  if (hits > 0) {
+    hits = GPU_select_buffer_remove_by_id(buffer, hits, select_id);
+  }
+  return hits;
+}
+
 /** \} */
 
 /* -------------------------------------------------------------------- */
diff --git a/source/blender/gpu/GPU_select.h b/source/blender/gpu/GPU_select.h
index 41d580f9b6d..d9a8e964a3d 100644
--- a/source/blender/gpu/GPU_select.h
+++ b/source/blender/gpu/GPU_select.h
@@ -57,6 +57,7 @@ void GPU_select_cache_end(void);
 
 /* utilities */
 const uint *GPU_select_buffer_near(const uint *buffer, int hits);
+uint GPU_select_buffer_remove_by_id(uint *buffer, int hits, uint select_id);
 void GPU_select_buffer_stride_realign(const struct rcti *src, const struct rcti *dst, uint *r_buf);
 
 #ifdef __cplusplus
diff --git a/source/blender/gpu/intern/gpu_select.c b/source/blender/gpu/intern/gpu_select.c
index 1776f0f391d..5766a176a96 100644
--- a/source/blender/gpu/intern/gpu_select.c
+++ b/source/blender/gpu/intern/gpu_select.c
@@ -213,6 +213,24 @@ const uint *GPU_select_buffer_near(const uint *buffer, int hits)
   return buffer_near;
 }
 
+uint GPU_select_buffer_remove_by_id(uint *buffer, int hits, uint select_id)
+{
+  uint *buffer_src = buffer;
+  uint *buffer_dst = buffer;
+  int hits_final = 0;
+  for (int i = 0; i < hits; i++) {
+    if (buffer_src[3] != select_id) {
+      if (buffer_dst != buffer_src) {
+        memcpy(buffer_dst, buffer_src, sizeof(int[4]));
+      }
+      buffer_dst += 4;
+      hits_final += 1;
+    }
+    buffer_src += 4;
+  }
+  return hits_final;
+}
+
 /* Part of the solution copied from `rect_subregion_stride_calc`. */
 void GPU_select_buffer_stride_realign(const rcti *src, const rcti *dst, uint *r_buf)
 {