[Bf-blender-cvs] SVN commit: /data/svn/bf-blender [16406] trunk/blender/source/blender/ render/intern/source: Bugfix #17497

[Ton Roosendaal]

Revision: 16406
          http://projects.blender.org/plugins/scmsvn/viewcvs.php?view=rev&root=bf-blender&revision=16406
Author:   ton
Date:     2008-09-07 15:15:44 +0200 (Sun, 07 Sep 2008)

Log Message:
-----------
Bugfix #17497

FSA didn't support "key alpha", fixed it.
Also noticed edge-enhance didnt work for FSA, added warning for that,
fixing it is too clumsy, this option belongs in compositor.

Modified Paths:
--------------
    trunk/blender/source/blender/render/intern/source/pipeline.c
    trunk/blender/source/blender/render/intern/source/rendercore.c

Modified: trunk/blender/source/blender/render/intern/source/pipeline.c
===================================================================
--- trunk/blender/source/blender/render/intern/source/pipeline.c	2008-09-07 11:36:52 UTC (rev 16405)
+++ trunk/blender/source/blender/render/intern/source/pipeline.c	2008-09-07 13:15:44 UTC (rev 16406)
@@ -2346,6 +2346,12 @@
 		if(re->osa==0)
 			re->r.scemode &= ~R_FULL_SAMPLE;
 		
+		/* no fullsample and edge */
+		if((re->r.scemode & R_FULL_SAMPLE) && (re->r.mode & R_EDGE)) {
+			re->error("Full Sample doesn't support Edge Enhance");
+			return 0;
+		}
+		
 	}
 	else
 		re->r.scemode &= ~R_FULL_SAMPLE;	/* clear to be sure */

Modified: trunk/blender/source/blender/render/intern/source/rendercore.c
===================================================================
--- trunk/blender/source/blender/render/intern/source/rendercore.c	2008-09-07 11:36:52 UTC (rev 16405)
+++ trunk/blender/source/blender/render/intern/source/rendercore.c	2008-09-07 13:15:44 UTC (rev 16406)
@@ -931,17 +931,23 @@
 	}
 }
 
-
-static void convert_to_key_alpha(RenderPart *pa, float *rectf)
+static void convert_to_key_alpha(RenderPart *pa, RenderLayer *rl)
 {
-	int y;
+	RenderLayer *rlpp[RE_MAX_OSA];
+	int y, sample, totsample;
 	
-	for(y= pa->rectx*pa->recty; y>0; y--, rectf+=4) {
-		if(rectf[3] >= 1.0f);
-		else if(rectf[3] > 0.0f) {
-			rectf[0] /= rectf[3];
-			rectf[1] /= rectf[3];
-			rectf[2] /= rectf[3];
+	totsample= get_sample_layers(pa, rl, rlpp);
+	
+	for(sample= 0; sample<totsample; sample++) {
+		float *rectf= rlpp[sample]->rectf;
+		
+		for(y= pa->rectx*pa->recty; y>0; y--, rectf+=4) {
+			if(rectf[3] >= 1.0f);
+			else if(rectf[3] > 0.0f) {
+				rectf[0] /= rectf[3];
+				rectf[1] /= rectf[3];
+				rectf[2] /= rectf[3];
+			}
 		}
 	}
 }
@@ -1229,7 +1235,7 @@
 		
 		/* de-premul alpha */
 		if(R.r.alphamode & R_ALPHAKEY)
-			convert_to_key_alpha(pa, rl->rectf);
+			convert_to_key_alpha(pa, rl);
 		
 		/* free stuff within loop! */
 		MEM_freeN(pa->rectdaps); pa->rectdaps= NULL;
@@ -1393,7 +1399,7 @@
 		
 		/* de-premul alpha */
 		if(R.r.alphamode & R_ALPHAKEY)
-			convert_to_key_alpha(pa, rl->rectf);
+			convert_to_key_alpha(pa, rl);
 		
 		if(edgerect) MEM_freeN(edgerect);
 		edgerect= NULL;