[Bf-blender-cvs] [5bc5b25] temp-cycles-microdisplacement: Fixing two out-of-bounds bugs in the SubPatch code

[Lukas Stockner]

Commit: 5bc5b2570d9f4152e93c89c51fec70e764b6fdae
Author: Lukas Stockner
Date:   Mon Apr 11 20:02:30 2016 +0200
Branches: temp-cycles-microdisplacement
https://developer.blender.org/rB5bc5b2570d9f4152e93c89c51fec70e764b6fdae

Fixing two out-of-bounds bugs in the SubPatch code

Credit for the actual fixes goes to Mai Lavelle.

===================================================================

M	intern/cycles/kernel/geom/geom_cache.cpp
M	intern/cycles/render/mesh.cpp
M	intern/cycles/render/mesh.h
M	intern/cycles/render/mesh_displace.cpp
M	intern/cycles/render/mesh_subdivision.cpp

===================================================================

diff --git a/intern/cycles/kernel/geom/geom_cache.cpp b/intern/cycles/kernel/geom/geom_cache.cpp
index 4433b12..5ac4515 100644
--- a/intern/cycles/kernel/geom/geom_cache.cpp
+++ b/intern/cycles/kernel/geom/geom_cache.cpp
@@ -131,9 +131,9 @@ void geom_cache_set_max_size(GeomCache * geom_cache, uint max_size) {
 	lru->set_max_size(max_size);
 }
 
-static void geom_cache_get_tessellated_subpatch_size(GeomCache* geom_cache, int object, int prim, uint* num_verts, uint* num_tris, int* total_size) {
+static void geom_cache_get_tessellated_subpatch_size(GeomCache* geom_cache, int object, int prim, uint* num_verts, uint* num_tris, int* cached_bvh_size) {
 	Mesh* mesh = geom_cache->scene->objects[object]->mesh;
-	mesh->diced_subpatch_size(prim, num_verts, num_tris, total_size);
+	mesh->diced_subpatch_size(prim, num_verts, num_tris, cached_bvh_size);
 }
 
 static void geom_cache_dice_subpatch(GeomCache* geom_cache, TessellatedSubPatch* subpatch, int object, int prim) {
@@ -143,7 +143,7 @@ static void geom_cache_dice_subpatch(GeomCache* geom_cache, TessellatedSubPatch*
 
 static void geom_cache_update_subpatch_size(GeomCache* geom_cache, int object, int prim, int size) {
 	Mesh* mesh = geom_cache->scene->objects[object]->mesh;
-	mesh->subpatches[prim].cached_tessellated_size = size;
+	mesh->subpatches[prim].cached_bvh_size = size;
 }
 
 static void _geom_cache_sample_subpatch_vert(TessellatedSubPatch *subpatch, int vert, float3 *P, float3 *N, float *u, float *v, int *shader)
@@ -190,13 +190,11 @@ TessellatedSubPatch* geom_cache_get_subpatch(KernelGlobals *kg, int object, int
 
 		// get patch size
 		uint num_verts, num_tris, bvh_size = 0;
-		int total_size = -1;
-		geom_cache_get_tessellated_subpatch_size(geom_cache, object, prim, &num_verts, &num_tris, &total_size);
-		bvh_size = num_tris*2*4 + 4;
+		int cached_bvh_size = -1;
+		geom_cache_get_tessellated_subpatch_size(geom_cache, object, prim, &num_verts, &num_tris, &cached_bvh_size);
+		bvh_size = (cached_bvh_size >= 0)? cached_bvh_size: (num_tris*2*4 + 4);
 
 		size_t size = sizeof(SubPatchWraper) + sizeof(float4)*(num_verts*2 + num_tris + bvh_size);
-		if(total_size >= 0)
-			size = total_size;
 
 		// alloc
 		SubPatchWraper* wraper = (SubPatchWraper*)operator new (size);
@@ -215,6 +213,8 @@ TessellatedSubPatch* geom_cache_get_subpatch(KernelGlobals *kg, int object, int
 		subpatch->tri_offset = num_verts*2;
 		subpatch->bvh_offset = subpatch->tri_offset + num_tris;
 
+		assert((subpatch->bvh_offset + bvh_size)*sizeof(float4) <= (size - sizeof(SubPatchWraper)));
+
 		// dice
 		geom_cache_dice_subpatch(geom_cache, subpatch, object, prim);
 
@@ -281,9 +281,8 @@ TessellatedSubPatch* geom_cache_get_subpatch(KernelGlobals *kg, int object, int
 			bvh_size = subpatch_build_bvh(subpatch, bvh_size);
 
 			// update size for next time
-			if(total_size < 0) {
-				size = sizeof(TessellatedSubPatch) + sizeof(float4)*(num_verts*2 + subpatch->num_triangles + bvh_size);
-				geom_cache_update_subpatch_size(geom_cache, object, prim, size);
+			if(cached_bvh_size < 0) {
+				geom_cache_update_subpatch_size(geom_cache, object, prim, bvh_size);
 			}
 		}
 
diff --git a/intern/cycles/render/mesh.cpp b/intern/cycles/render/mesh.cpp
index d3b075b..7ada360 100644
--- a/intern/cycles/render/mesh.cpp
+++ b/intern/cycles/render/mesh.cpp
@@ -1455,7 +1455,7 @@ void MeshManager::device_update(Device *device, DeviceScene *dscene, Scene *scen
 				mesh->dice_subpatch(diced, i);
 
 				// displace
-				Shader *shader = scene->shaders[mesh->patches[subpatch->patch].shader];
+				Shader *shader = scene->shaders[mesh->patches[subpatch->patch & 0x7fffffff].shader];
 				if(mesh->displacement_method != Mesh::DISPLACE_BUMP && shader->has_displacement)
 					displace_subpatch(device, dscene, scene, mesh, progress, diced, i);
 
diff --git a/intern/cycles/render/mesh.h b/intern/cycles/render/mesh.h
index 4181ddf..9b8baef 100644
--- a/intern/cycles/render/mesh.h
+++ b/intern/cycles/render/mesh.h
@@ -78,9 +78,9 @@ public:
 		int edge_factors[4];
 		float2 uv[4];
 		BoundBox bounds;
-		int cached_num_triangles, cached_tessellated_size;
+		int cached_num_triangles, cached_bvh_size;
 
-		SubPatch() : bounds(BoundBox::empty), cached_num_triangles(-1), cached_tessellated_size(-1) {}
+		SubPatch() : bounds(BoundBox::empty), cached_num_triangles(-1), cached_bvh_size(-1) {}
 
 		bool is_quad() const { return edge_factors[3] != -1; }
 
diff --git a/intern/cycles/render/mesh_displace.cpp b/intern/cycles/render/mesh_displace.cpp
index 7ff36c5..b780c60 100644
--- a/intern/cycles/render/mesh_displace.cpp
+++ b/intern/cycles/render/mesh_displace.cpp
@@ -188,7 +188,7 @@ bool MeshManager::displace_subpatch(Device *device, DeviceScene *dscene, Scene *
 	if(mesh->displacement_method == Mesh::DISPLACE_BUMP)
 		return false;
 
-	Shader *shader = scene->shaders[mesh->patches[subpatch->patch].shader];
+	Shader *shader = scene->shaders[mesh->patches[subpatch->patch & 0x7fffffff].shader];
 
 	if(!shader->has_displacement)
 		return false;
diff --git a/intern/cycles/render/mesh_subdivision.cpp b/intern/cycles/render/mesh_subdivision.cpp
index 06aca14..ae0dc3e 100644
--- a/intern/cycles/render/mesh_subdivision.cpp
+++ b/intern/cycles/render/mesh_subdivision.cpp
@@ -305,8 +305,8 @@ void Mesh::diced_subpatch_size(int subpatch_id, uint* num_verts, uint* num_tris,
 		*num_tris = subpatch.cached_num_triangles;
 	}
 
-	if(total_size && subpatch.cached_tessellated_size >= 0) {
-		*total_size = subpatch.cached_tessellated_size;
+	if(total_size && subpatch.cached_bvh_size >= 0) {
+		*total_size = subpatch.cached_bvh_size;
 	}
 }