[Bf-blender-cvs] [a01ab2b4e09] master: Fix T101492: UV stitch crash (more than 32 objects selected)

[Philipp Oeser]

Commit: a01ab2b4e093a35b56888cf646d84686bccb0fbd
Author: Philipp Oeser
Date:   Fri Sep 30 13:41:07 2022 +0200
Branches: master
https://developer.blender.org/rBa01ab2b4e093a35b56888cf646d84686bccb0fbd

Fix T101492: UV stitch crash (more than 32 objects selected)

Crash happened when adjusting operator props in Adjust Last Operation
panel.

When there are more than 32 objects selected in muti-object-editmode, we
are running into RNA array limit (`objects_selection_count` is defined as
an RNA array (which can only hold 32 entries, see
`RNA_MAX_ARRAY_LENGTH`), leading to reading random memory errors.

While there might be ways to make this work with more than 32 selected
objects (e.g. by instead using a collection, or investigate supporting
dynamic sized arrays for run-time RNA), this patch only cancels the
operator with a report message (instead of crashing).

Maniphest Tasks: T101492

Differential Revision: https://developer.blender.org/D16115

===================================================================

M	source/blender/editors/uvedit/uvedit_smart_stitch.c

===================================================================

diff --git a/source/blender/editors/uvedit/uvedit_smart_stitch.c b/source/blender/editors/uvedit/uvedit_smart_stitch.c
index 05b98ab9627..865262e6947 100644
--- a/source/blender/editors/uvedit/uvedit_smart_stitch.c
+++ b/source/blender/editors/uvedit/uvedit_smart_stitch.c
@@ -14,6 +14,7 @@
 #include "DNA_meshdata_types.h"
 #include "DNA_object_types.h"
 #include "DNA_scene_types.h"
+#include "DNA_windowmanager_types.h"
 
 #include "BLI_ghash.h"
 #include "BLI_math.h"
@@ -28,6 +29,7 @@
 #include "BKE_editmesh.h"
 #include "BKE_layer.h"
 #include "BKE_mesh_mapping.h"
+#include "BKE_report.h"
 
 #include "DEG_depsgraph.h"
 
@@ -2174,6 +2176,28 @@ static int stitch_init_all(bContext *C, wmOperator *op)
   Scene *scene = CTX_data_scene(C);
   ToolSettings *ts = scene->toolsettings;
 
+  ViewLayer *view_layer = CTX_data_view_layer(C);
+  View3D *v3d = CTX_wm_view3d(C);
+  uint objects_len = 0;
+  Object **objects = BKE_view_layer_array_from_objects_in_edit_mode_unique_data_with_uvs(
+      scene, view_layer, v3d, &objects_len);
+
+  if (objects_len == 0) {
+    MEM_freeN(objects);
+    BKE_report(op->reports, RPT_ERROR, "No objects selected");
+    return 0;
+  }
+
+  if (objects_len > RNA_MAX_ARRAY_LENGTH) {
+    MEM_freeN(objects);
+    BKE_reportf(op->reports,
+                RPT_ERROR,
+                "Stitching only works with less than %i objects selected (%u selected)",
+                RNA_MAX_ARRAY_LENGTH,
+                objects_len);
+    return 0;
+  }
+
   StitchStateContainer *ssc = MEM_callocN(sizeof(StitchStateContainer), "stitch collection");
 
   op->customdata = ssc;
@@ -2208,21 +2232,6 @@ static int stitch_init_all(bContext *C, wmOperator *op)
     }
   }
 
-  ssc->objects_len = 0;
-  ssc->states = NULL;
-
-  ViewLayer *view_layer = CTX_data_view_layer(C);
-  View3D *v3d = CTX_wm_view3d(C);
-  uint objects_len = 0;
-  Object **objects = BKE_view_layer_array_from_objects_in_edit_mode_unique_data_with_uvs(
-      scene, view_layer, v3d, &objects_len);
-
-  if (objects_len == 0) {
-    MEM_freeN(objects);
-    state_delete_all(ssc);
-    return 0;
-  }
-
   ssc->objects = MEM_callocN(sizeof(Object *) * objects_len, "Object *ssc->objects");
   ssc->states = MEM_callocN(sizeof(StitchState *) * objects_len, "StitchState");
   ssc->objects_len = 0;
@@ -2288,6 +2297,7 @@ static int stitch_init_all(bContext *C, wmOperator *op)
 
   if (ssc->objects_len == 0) {
     state_delete_all(ssc);
+    BKE_report(op->reports, RPT_ERROR, "Could not initialize stitching on any selected object");
     return 0;
   }