[Bf-blender-cvs] [85fb4708f4c] blender-v3.3-release: Fix T101492: UV stitch crash (more than 32 objects selected)

[Philipp Oeser]

Commit: 85fb4708f4c029db46910c80a07e0e79dc0a9067
Author: Philipp Oeser
Date:   Fri Sep 30 13:41:07 2022 +0200
Branches: blender-v3.3-release
https://developer.blender.org/rB85fb4708f4c029db46910c80a07e0e79dc0a9067

Fix T101492: UV stitch crash (more than 32 objects selected)

Crash happened when adjusting operator props in Adjust Last Operation
panel.

When there are more than 32 objects selected in muti-object-editmode, we
are running into RNA array limit (`objects_selection_count` is defined as
an RNA array (which can only hold 32 entries, see
`RNA_MAX_ARRAY_LENGTH`), leading to reading random memory errors.

While there might be ways to make this work with more than 32 selected
objects (e.g. by instead using a collection, or investigate supporting
dynamic sized arrays for run-time RNA), this patch only cancels the
operator with a report message (instead of crashing).

Maniphest Tasks: T101492

Differential Revision: https://developer.blender.org/D16115

===================================================================

M	source/blender/editors/uvedit/uvedit_smart_stitch.c

===================================================================

diff --git a/source/blender/editors/uvedit/uvedit_smart_stitch.c b/source/blender/editors/uvedit/uvedit_smart_stitch.c
index 579674930a6..beb4d2c261d 100644
--- a/source/blender/editors/uvedit/uvedit_smart_stitch.c
+++ b/source/blender/editors/uvedit/uvedit_smart_stitch.c
@@ -14,6 +14,7 @@
 #include "DNA_meshdata_types.h"
 #include "DNA_object_types.h"
 #include "DNA_scene_types.h"
+#include "DNA_windowmanager_types.h"
 
 #include "BLI_ghash.h"
 #include "BLI_math.h"
@@ -28,6 +29,7 @@
 #include "BKE_editmesh.h"
 #include "BKE_layer.h"
 #include "BKE_mesh_mapping.h"
+#include "BKE_report.h"
 
 #include "DEG_depsgraph.h"
 
@@ -2227,6 +2229,28 @@ static int stitch_init_all(bContext *C, wmOperator *op)
   Scene *scene = CTX_data_scene(C);
   ToolSettings *ts = scene->toolsettings;
 
+  ViewLayer *view_layer = CTX_data_view_layer(C);
+  View3D *v3d = CTX_wm_view3d(C);
+  uint objects_len = 0;
+  Object **objects = BKE_view_layer_array_from_objects_in_edit_mode_unique_data_with_uvs(
+      view_layer, v3d, &objects_len);
+
+  if (objects_len == 0) {
+    MEM_freeN(objects);
+    BKE_report(op->reports, RPT_ERROR, "No objects selected");
+    return 0;
+  }
+
+  if (objects_len > RNA_MAX_ARRAY_LENGTH) {
+    MEM_freeN(objects);
+    BKE_reportf(op->reports,
+                RPT_ERROR,
+                "Stitching only works with less than %i objects selected (%u selected)",
+                RNA_MAX_ARRAY_LENGTH,
+                objects_len);
+    return 0;
+  }
+
   StitchStateContainer *ssc = MEM_callocN(sizeof(StitchStateContainer), "stitch collection");
 
   op->customdata = ssc;
@@ -2261,21 +2285,6 @@ static int stitch_init_all(bContext *C, wmOperator *op)
     }
   }
 
-  ssc->objects_len = 0;
-  ssc->states = NULL;
-
-  ViewLayer *view_layer = CTX_data_view_layer(C);
-  View3D *v3d = CTX_wm_view3d(C);
-  uint objects_len = 0;
-  Object **objects = BKE_view_layer_array_from_objects_in_edit_mode_unique_data_with_uvs(
-      view_layer, v3d, &objects_len);
-
-  if (objects_len == 0) {
-    MEM_freeN(objects);
-    state_delete_all(ssc);
-    return 0;
-  }
-
   ssc->objects = MEM_callocN(sizeof(Object *) * objects_len, "Object *ssc->objects");
   ssc->states = MEM_callocN(sizeof(StitchState *) * objects_len, "StitchState");
   ssc->objects_len = 0;
@@ -2341,6 +2350,7 @@ static int stitch_init_all(bContext *C, wmOperator *op)
 
   if (ssc->objects_len == 0) {
     state_delete_all(ssc);
+    BKE_report(op->reports, RPT_ERROR, "Could not initialize stitching on any selected object");
     return 0;
   }