[Bf-blender-cvs] SVN commit: /data/svn/bf-blender [41257] trunk/blender: MovieCache implementation

Sergey Sharybin g.ulairi at gmail.com
Mon Oct 24 19:12:29 CEST 2011


Revision: 41257
          http://projects.blender.org/scm/viewvc.php?view=rev&root=bf-blender&revision=41257
Author:   nazgul
Date:     2011-10-24 17:12:28 +0000 (Mon, 24 Oct 2011)
Log Message:
-----------
MovieCache implementation

Implementation of cache for general movie-related areas
such as sequencer and clip editor (in the future)

Some changes in limiter were necessary:
- Limiter counted mapped memory twice when was checking how many memory is used.
- It was using "global" memory usage not memory usage by cached elements.
  It will cause big problems when there's large mesh or plenty of undo steps are
  in memory nothing would be cached in sequencer.
- To solve this problem introduced "callback" to measure cached element size.
  It could be not very accurate in general, but it works well for image buffers.
  And if this callback isn't set old-school memory usage check would be used.
- The whole cache used to get freed when memory limit exceeded, now it'll drop only
  as much elements as necessary to reduce memory usage.

Seqcache is switched to use this new cache code.

Modified Paths:
--------------
    trunk/blender/intern/memutil/MEM_CacheLimiter.h
    trunk/blender/intern/memutil/MEM_CacheLimiterC-Api.h
    trunk/blender/intern/memutil/intern/MEM_CacheLimiterC-Api.cpp
    trunk/blender/source/blender/blenkernel/BKE_sequencer.h
    trunk/blender/source/blender/blenkernel/intern/blender.c
    trunk/blender/source/blender/blenkernel/intern/seqcache.c
    trunk/blender/source/blender/imbuf/CMakeLists.txt
    trunk/blender/source/blender/imbuf/intern/allocimbuf.c

Added Paths:
-----------
    trunk/blender/source/blender/imbuf/IMB_moviecache.h
    trunk/blender/source/blender/imbuf/intern/moviecache.c

Modified: trunk/blender/intern/memutil/MEM_CacheLimiter.h
===================================================================
--- trunk/blender/intern/memutil/MEM_CacheLimiter.h	2011-10-24 17:09:31 UTC (rev 41256)
+++ trunk/blender/intern/memutil/MEM_CacheLimiter.h	2011-10-24 17:12:28 UTC (rev 41257)
@@ -125,6 +125,10 @@
 public:
 	typedef typename std::list<MEM_CacheLimiterHandle<T> *,
 	  MEM_Allocator<MEM_CacheLimiterHandle<T> *> >::iterator iterator;
+	typedef intptr_t (*MEM_CacheLimiter_DataSize_Func) (void *data);
+	MEM_CacheLimiter(MEM_CacheLimiter_DataSize_Func getDataSize_)
+		: getDataSize(getDataSize_) {
+	}
 	~MEM_CacheLimiter() {
 		for (iterator it = queue.begin(); it != queue.end(); it++) {
 			delete *it;
@@ -143,17 +147,36 @@
 	}
 	void enforce_limits() {
 		intptr_t max = MEM_CacheLimiter_get_maximum();
-		intptr_t mem_in_use= MEM_get_memory_in_use();
-		intptr_t mmap_in_use= MEM_get_mapped_memory_in_use();
+		intptr_t mem_in_use, cur_size;
 
 		if (max == 0) {
 			return;
 		}
+
+		if(getDataSize) {
+			mem_in_use = total_size();
+		} else {
+			mem_in_use = MEM_get_memory_in_use();
+		}
+
 		for (iterator it = queue.begin(); 
-		     it != queue.end() && mem_in_use + mmap_in_use > max;) {
+		     it != queue.end() && mem_in_use > max;) {
 			iterator jt = it;
 			++it;
+
+			if(getDataSize) {
+				cur_size= getDataSize((*jt)->get()->get_data());
+			} else {
+				cur_size= mem_in_use;
+			}
+
 			(*jt)->destroy_if_possible();
+
+			if(getDataSize) {
+				mem_in_use-= cur_size;
+			} else {
+				mem_in_use-= cur_size - MEM_get_memory_in_use();
+			}
 		}
 	}
 	void touch(MEM_CacheLimiterHandle<T> * handle) {
@@ -164,8 +187,17 @@
 		handle->me = it;
 	}
 private:
+	intptr_t total_size() {
+		intptr_t size = 0;
+		for (iterator it = queue.begin(); it != queue.end(); it++) {
+			size+= getDataSize((*it)->get()->get_data());
+		}
+		return size;
+	}
+
 	std::list<MEM_CacheLimiterHandle<T>*,
 	  MEM_Allocator<MEM_CacheLimiterHandle<T> *> > queue;
+	MEM_CacheLimiter_DataSize_Func getDataSize;
 };
 
 #endif // MEM_CACHELIMITER_H

Modified: trunk/blender/intern/memutil/MEM_CacheLimiterC-Api.h
===================================================================
--- trunk/blender/intern/memutil/MEM_CacheLimiterC-Api.h	2011-10-24 17:09:31 UTC (rev 41256)
+++ trunk/blender/intern/memutil/MEM_CacheLimiterC-Api.h	2011-10-24 17:12:28 UTC (rev 41257)
@@ -41,6 +41,9 @@
 /* function used to remove data from memory */
 typedef void(*MEM_CacheLimiter_Destruct_Func)(void*);
 
+/* function used to measure stored data element size */
+typedef intptr_t(*MEM_CacheLimiter_DataSize_Func) (void*);
+
 #ifndef MEM_CACHELIMITER_H
 extern void MEM_CacheLimiter_set_maximum(int m);
 extern int MEM_CacheLimiter_get_maximum(void);
@@ -54,7 +57,8 @@
  */
 
 extern MEM_CacheLimiterC * new_MEM_CacheLimiter(
-	MEM_CacheLimiter_Destruct_Func data_destructor);
+	MEM_CacheLimiter_Destruct_Func data_destructor,
+	MEM_CacheLimiter_DataSize_Func data_size);
 
 /** 
  * Delete MEM_CacheLimiter

Modified: trunk/blender/intern/memutil/intern/MEM_CacheLimiterC-Api.cpp
===================================================================
--- trunk/blender/intern/memutil/intern/MEM_CacheLimiterC-Api.cpp	2011-10-24 17:09:31 UTC (rev 41256)
+++ trunk/blender/intern/memutil/intern/MEM_CacheLimiterC-Api.cpp	2011-10-24 17:12:28 UTC (rev 41257)
@@ -54,8 +54,8 @@
 
 class MEM_CacheLimiterCClass {
 public:
-	MEM_CacheLimiterCClass(MEM_CacheLimiter_Destruct_Func data_destructor_)
-		: data_destructor(data_destructor_) { 
+	MEM_CacheLimiterCClass(MEM_CacheLimiter_Destruct_Func data_destructor_, MEM_CacheLimiter_DataSize_Func data_size)
+		: data_destructor(data_destructor_), cache(data_size) {
 	}
         ~MEM_CacheLimiterCClass();
 	
@@ -142,10 +142,12 @@
 }
 
 MEM_CacheLimiterC * new_MEM_CacheLimiter(
-	MEM_CacheLimiter_Destruct_Func data_destructor)
+	MEM_CacheLimiter_Destruct_Func data_destructor,
+	MEM_CacheLimiter_DataSize_Func data_size)
 {
 	return (MEM_CacheLimiterC*) new MEM_CacheLimiterCClass(
-		data_destructor);
+		data_destructor,
+		data_size);
 }
 
 void delete_MEM_CacheLimiter(MEM_CacheLimiterC * This)

Modified: trunk/blender/source/blender/blenkernel/BKE_sequencer.h
===================================================================
--- trunk/blender/source/blender/blenkernel/BKE_sequencer.h	2011-10-24 17:09:31 UTC (rev 41256)
+++ trunk/blender/source/blender/blenkernel/BKE_sequencer.h	2011-10-24 17:12:28 UTC (rev 41257)
@@ -215,9 +215,7 @@
 	SEQ_STRIPELEM_IBUF_ENDSTILL
 } seq_stripelem_ibuf_t;
 
-void seq_stripelem_cache_init(void);
 void seq_stripelem_cache_destruct(void);
-
 void seq_stripelem_cache_cleanup(void);
 
 /* returned ImBuf is properly refed and has to be freed */

Modified: trunk/blender/source/blender/blenkernel/intern/blender.c
===================================================================
--- trunk/blender/source/blender/blenkernel/intern/blender.c	2011-10-24 17:09:31 UTC (rev 41256)
+++ trunk/blender/source/blender/blenkernel/intern/blender.c	2011-10-24 17:12:28 UTC (rev 41257)
@@ -61,6 +61,7 @@
 #include "BLI_callbacks.h"
 
 #include "IMB_imbuf.h"
+#include "IMB_moviecache.h"
 
 #include "BKE_blender.h"
 #include "BKE_context.h"
@@ -111,6 +112,7 @@
 	BLI_cb_finalize();
 
 	seq_stripelem_cache_destruct();
+	IMB_moviecache_destruct();
 	
 	free_nodesystem();	
 }

Modified: trunk/blender/source/blender/blenkernel/intern/seqcache.c
===================================================================
--- trunk/blender/source/blender/blenkernel/intern/seqcache.c	2011-10-24 17:09:31 UTC (rev 41256)
+++ trunk/blender/source/blender/blenkernel/intern/seqcache.c	2011-10-24 17:12:28 UTC (rev 41257)
@@ -17,6 +17,8 @@
  *
  * Peter Schlaile <peter [at] schlaile [dot] de> 2010
  *
+ * Contributor(s): Sergey Sharybin
+ *
  * ***** END GPL LICENSE BLOCK *****
  */
 
@@ -26,22 +28,15 @@
 
 
 #include <stddef.h>
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
 
+#include "BLO_sys_types.h"	/* for intptr_t */
+
 #include "MEM_guardedalloc.h"
-#include "MEM_CacheLimiterC-Api.h"
 
 #include "DNA_sequence_types.h"
 #include "BKE_sequencer.h"
-#include "BLI_utildefines.h"
-#include "BLI_ghash.h"
-#include "BLI_mempool.h"
-#include <pthread.h>
 
-#include "IMB_imbuf.h"
-#include "IMB_imbuf_types.h"
+#include "IMB_moviecache.h"
 
 typedef struct seqCacheKey 
 {
@@ -51,20 +46,9 @@
 	seq_stripelem_ibuf_t type;
 } seqCacheKey;
 
-typedef struct seqCacheEntry
-{
-	ImBuf * ibuf;
-	MEM_CacheLimiterHandleC * c_handle;
-} seqCacheEntry;
+static struct MovieCache *moviecache = NULL;
 
-static GHash * hash = NULL;
-static MEM_CacheLimiterC * limitor = NULL;
-static struct BLI_mempool * entrypool = NULL;
-static struct BLI_mempool * keypool = NULL;
-static int ibufs_in  = 0;
-static int ibufs_rem = 0;
-
-static unsigned int HashHash(const void *key_)
+static unsigned int seqcache_hashhash(const void *key_)
 {
 	const seqCacheKey *key = (seqCacheKey*) key_;
 	unsigned int rval = seq_hash_render_data(&key->context);
@@ -76,7 +60,7 @@
 	return rval;
 }
 
-static int HashCmp(const void *a_, const void *b_)
+static int seqcache_hashcmp(const void *a_, const void *b_)
 {
 	const seqCacheKey * a = (seqCacheKey*) a_;
 	const seqCacheKey * b = (seqCacheKey*) b_;
@@ -105,109 +89,37 @@
 	return seq_cmp_render_data(&a->context, &b->context);
 }
 
-static void HashKeyFree(void *key)
-{
-	BLI_mempool_free(keypool, key);
-}
-
-static void HashValFree(void *val)
-{
-	seqCacheEntry* e = (seqCacheEntry*) val;
-
-	if (e->ibuf) {
-		/* fprintf(stderr, "Removing: %p, cnt: %d\n", e->ibuf, 
-		   e->ibuf->refcounter); */
-		IMB_freeImBuf(e->ibuf);
-		MEM_CacheLimiter_unmanage(e->c_handle);
-		ibufs_rem++;
-	}
-
-	e->ibuf = NULL;
-	e->c_handle = NULL;
-
-	BLI_mempool_free(entrypool, e);
-}
-
-static void IMB_seq_cache_destructor(void * p)
-{
-	seqCacheEntry* e = (seqCacheEntry*) p;
-	
-	if (e && e->ibuf) {
-		/* fprintf(stderr, "Removing: %p, cnt: %d\n", e->ibuf,
-		   e->ibuf->refcounter); */
-		IMB_freeImBuf(e->ibuf);
-		ibufs_rem++;
-
-		e->ibuf = NULL;
-		e->c_handle = NULL;
-	}
-}
-
-void seq_stripelem_cache_init(void)
-{
-	hash = BLI_ghash_new(HashHash, HashCmp, "seq stripelem cache hash");
-	limitor = new_MEM_CacheLimiter( IMB_seq_cache_destructor );
-
-	entrypool = BLI_mempool_create(sizeof(seqCacheEntry), 64, 64, 0);
-	keypool = BLI_mempool_create(sizeof(seqCacheKey), 64, 64, 0);
-}
-
 void seq_stripelem_cache_destruct(void)
 {
-	if (!entrypool) {
-		return;
-	}
-	BLI_ghash_free(hash, HashKeyFree, HashValFree);
-	delete_MEM_CacheLimiter(limitor);
-	BLI_mempool_destroy(entrypool);
-	BLI_mempool_destroy(keypool);
+	if(moviecache)
+		IMB_moviecache_free(moviecache);
 }
 
 void seq_stripelem_cache_cleanup(void)
 {
-	if (!entrypool) {
-		seq_stripelem_cache_init();
+	if(moviecache) {
+		IMB_moviecache_free(moviecache);
+		moviecache = IMB_moviecache_create(sizeof(seqCacheKey), seqcache_hashhash,
+				seqcache_hashcmp, NULL);
 	}
-
-	/* fprintf(stderr, "Stats before cleanup: in: %d rem: %d\n",
-	   ibufs_in, ibufs_rem); */
-
-	BLI_ghash_free(hash, HashKeyFree, HashValFree);
-	hash = BLI_ghash_new(HashHash, HashCmp, "seq stripelem cache hash");
-
-	/* fprintf(stderr, "Stats after cleanup: in: %d rem: %d\n",
-	   ibufs_in, ibufs_rem); */
-
 }
 
 struct ImBuf * seq_stripelem_cache_get(
 	SeqRenderData context, struct Sequence * seq, 
 	float cfra, seq_stripelem_ibuf_t type)
 {
-	seqCacheKey key;
-	seqCacheEntry * e;
 
-	if (!seq) {
-		return NULL;
-	}
+	if(moviecache && seq) {
+		seqCacheKey key;
 
-	if (!entrypool) {
-		seq_stripelem_cache_init();
-	}
+		key.seq = seq;
+		key.context = context;
+		key.cfra = cfra - seq->start;
+		key.type = type;
 
-	key.seq = seq;
-	key.context = context;
-	key.cfra = cfra - seq->start;
-	key.type = type;
-	
-	e = (seqCacheEntry*) BLI_ghash_lookup(hash, &key);
-
-	if (e && e->ibuf) {
-		IMB_refImBuf(e->ibuf);
-
-		MEM_CacheLimiter_touch(e->c_handle);
-		return e->ibuf;
+		return IMB_moviecache_get(moviecache, &key);
 	}
+
 	return NULL;
 }
 
@@ -215,39 +127,21 @@
 	SeqRenderData context, struct Sequence * seq, 
 	float cfra, seq_stripelem_ibuf_t type, struct ImBuf * i)
 {
-	seqCacheKey * key;
-	seqCacheEntry * e;
+	seqCacheKey key;
 
 	if (!i) {
 		return;
 	}
 
-	ibufs_in++;
-
-	if (!entrypool) {
-		seq_stripelem_cache_init();
+	if(!moviecache) {
+		moviecache = IMB_moviecache_create(sizeof(seqCacheKey), seqcache_hashhash,
+				seqcache_hashcmp, NULL);
 	}
 
-	key = (seqCacheKey*) BLI_mempool_alloc(keypool);
+	key.seq = seq;
+	key.context = context;
+	key.cfra = cfra - seq->start;
+	key.type = type;
 
-	key->seq = seq;

@@ Diff output truncated at 10240 characters. @@



More information about the Bf-blender-cvs mailing list