[Bf-blender-cvs] [503866c6815] blender2.8: Fix T56199: Crash on Annotation in (VSE) Image Preview

[Joshua Leung]

Commit: 503866c68154ef09e6199068391515f81eebaa2d
Author: Joshua Leung
Date:   Thu Aug 2 16:34:27 2018 +1200
Branches: blender2.8
https://developer.blender.org/rB503866c68154ef09e6199068391515f81eebaa2d

Fix T56199: Crash on Annotation in (VSE) Image Preview

In some cases (e.g. using old userpref settings/keymaps)
it was possible to trigger a crash when the wrong GP/Annotation
operators were triggered in the wrong contexts (e.g. using
the old GPENCIL_OT_paint in annotation-only contexts like
all the 2D editors).

This commit resolves several issues that were caused by sloppy
code-churn + features that had been hacked on.

===================================================================

M	source/blender/editors/gpencil/annotate_paint.c
M	source/blender/editors/gpencil/gpencil_paint.c

===================================================================

diff --git a/source/blender/editors/gpencil/annotate_paint.c b/source/blender/editors/gpencil/annotate_paint.c
index 6325052fccd..b551f3d630b 100644
--- a/source/blender/editors/gpencil/annotate_paint.c
+++ b/source/blender/editors/gpencil/annotate_paint.c
@@ -1206,9 +1206,19 @@ static tGPsdata *gp_session_initpaint(bContext *C)
 	/* create new context data */
 	p = MEM_callocN(sizeof(tGPsdata), "Annotation Drawing Data");
 
-	gp_session_initdata(C, p);
+	/* Try to initialise context data
+	 * WARNING: This may not always succeed (e.g. using GP in an annotation-only context)
+	 */
+	if (gp_session_initdata(C, p) == 0) {
+		/* Invalid state - Exit
+		 * NOTE: It should be safe to just free the data, since failing context checks should
+		 * only happen when no data has been allocated.
+		 */
+		MEM_freeN(p);
+		return NULL;
+	}
 
-	/* radius for eraser circle is defined in userprefs now */
+	/* Radius for eraser circle is defined in userprefs */
 	/* NOTE: we do this here, so that if we exit immediately,
 	 *       erase size won't get lost
 	 */
@@ -1548,9 +1558,6 @@ static void gpencil_draw_exit(bContext *C, wmOperator *op)
 {
 	tGPsdata *p = op->customdata;
 
-	/* clear undo stack */
-	gpencil_undo_finish();
-
 	/* restore cursor to indicate end of drawing */
 	WM_cursor_modal_restore(CTX_wm_window(C));
 
@@ -1568,10 +1575,14 @@ static void gpencil_draw_exit(bContext *C, wmOperator *op)
 		 */
 		U.gp_eraser = p->radius;
 
+		/* clear undo stack */
+		gpencil_undo_finish();
+
 		/* cleanup */
 		gp_paint_cleanup(p);
 		gp_session_cleanup(p);
 		gp_session_free(p);
+		p = NULL;
 	}
 
 	op->customdata = NULL;
diff --git a/source/blender/editors/gpencil/gpencil_paint.c b/source/blender/editors/gpencil/gpencil_paint.c
index 0c411db57b4..6a0bf0c4a47 100644
--- a/source/blender/editors/gpencil/gpencil_paint.c
+++ b/source/blender/editors/gpencil/gpencil_paint.c
@@ -1746,18 +1746,20 @@ static tGPsdata *gp_session_initpaint(bContext *C, wmOperator *op)
 {
 	tGPsdata *p = NULL;
 
-	/* create new context data */
+	/* Create new context data */
 	p = MEM_callocN(sizeof(tGPsdata), "GPencil Drawing Data");
 
-	gp_session_initdata(C, op, p);
-
-#if 0
-	/* radius for eraser circle is defined in userprefs now */
-	/* NOTE: we do this here, so that if we exit immediately,
-	 *       erase size won't get lost
+	/* Try to initialise context data
+	 * WARNING: This may not always succeed (e.g. using GP in an annotation-only context)
 	 */
-	p->radius = U.gp_eraser;
-#endif
+	if (gp_session_initdata(C, op, p) == 0) {
+		/* Invalid state - Exit
+		 * NOTE: It should be safe to just free the data, since failing context checks should
+		 * only happen when no data has been allocated.
+		 */
+		MEM_freeN(p);
+		return NULL;
+	}
 
 	/* Random generator, only init once. */
 	uint rng_seed = (uint)(PIL_check_seconds_timer_i() & UINT_MAX);
@@ -2092,26 +2094,7 @@ static bool gpencil_is_tablet_eraser_active(const wmEvent *event)
 static void gpencil_draw_exit(bContext *C, wmOperator *op)
 {
 	tGPsdata *p = op->customdata;
-	bGPdata *gpd = CTX_data_gpencil_data(C);
-
-	/* clear undo stack */
-	gpencil_undo_finish();
-
-	/* restore cursor to indicate end of drawing */
-	if (p->sa->spacetype != SPACE_VIEW3D) {
-		WM_cursor_modal_restore(CTX_wm_window(C));
-	}
-	else {
-		/* or restore paint if 3D view */
-		if ((p) && (p->paintmode == GP_PAINTMODE_ERASER)) {
-			WM_cursor_modal_set(p->win, CURSOR_STD);
-		}
-		/* drawing batch cache is dirty now */
-		if (gpd) {
-			gp_update_cache(gpd);
-		}
 
-	}
 	/* don't assume that operator data exists at all */
 	if (p) {
 		/* check size of buffer before cleanup, to determine if anything happened here */
@@ -2128,6 +2111,26 @@ static void gpencil_draw_exit(bContext *C, wmOperator *op)
 			p->eraser->size = p->radius;
 		}
 
+		/* restore cursor to indicate end of drawing */
+		if (p->sa->spacetype != SPACE_VIEW3D) {
+			WM_cursor_modal_restore(CTX_wm_window(C));
+		}
+		else {
+			/* or restore paint if 3D view */
+			if ((p) && (p->paintmode == GP_PAINTMODE_ERASER)) {
+				WM_cursor_modal_set(p->win, CURSOR_STD);
+			}
+
+			/* drawing batch cache is dirty now */
+			bGPdata *gpd = CTX_data_gpencil_data(C);
+			if (gpd) {
+				gp_update_cache(gpd);
+			}
+		}
+
+		/* clear undo stack */
+		gpencil_undo_finish();
+
 		/* cleanup */
 		gp_paint_cleanup(p);
 		gp_session_cleanup(p);
@@ -2135,6 +2138,7 @@ static void gpencil_draw_exit(bContext *C, wmOperator *op)
 
 		/* finally, free the temp data */
 		gp_session_free(p);
+		p = NULL;
 	}
 
 	op->customdata = NULL;