[Bf-blender-cvs] [348d2fa09e0] master: VSE: Disk cache

Richard Antalik noreply at git.blender.org
Thu Mar 19 00:07:52 CET 2020


Commit: 348d2fa09e0c01d62372f5999b62d06ac3b810f9
Author: Richard Antalik
Date:   Thu Mar 19 00:05:18 2020 +0100
Branches: master
https://developer.blender.org/rB348d2fa09e0c01d62372f5999b62d06ac3b810f9

VSE: Disk cache

This patch implements dumping images from cache to HDD.
The main goal of this system is to provide a means to achieve consistent playback speed mainly for strips that are not possible to preview in real time.

How to use:
Disk cache has own settings in user preferences for path to storage, size limit and compression level.
To use disk cache, you need to check `Use Disk Cache` box, set `Disk Cache Directory`, `Disk Cache Limit` and save or open existing .blend file.
By default sequencer output will be cached only. Manual setting is possible in cache panel.

Uses:
 - Replacement or alternative for proxies. Disk cache will work with any strip type, supports float images as well.
 - Storage for strip thumbnails.
 - Less RAM needs to be allocated for preview cache

How it works:
Disk cache is extension of RAM cache. Every image, that is stored or deleted in RAM will be stored or deleted on HDD as well. Images can be compressed to save space and for use on slower drives. Compressed images are slower to write and read though.
Images are stored in bulk of 100 rendered frames per one file. This is to overcome slow file access time for large amount of files. Drawback is, that if one frame needs to be redrawn, all 100 frames are deleted.

Reviewed By: sergey

Differential Revision: https://developer.blender.org/D5524

===================================================================

M	release/datafiles/userdef/userdef_default.c
M	release/scripts/startup/bl_ui/space_userpref.py
M	source/blender/blenkernel/BKE_sequencer.h
M	source/blender/blenkernel/intern/seqcache.c
M	source/blender/blenkernel/intern/sequencer.c
M	source/blender/blenlib/BLI_fileops.h
M	source/blender/blenlib/intern/fileops.c
M	source/blender/makesdna/DNA_sequence_types.h
M	source/blender/makesdna/DNA_userdef_types.h
M	source/blender/makesrna/intern/rna_sequencer.c
M	source/blender/makesrna/intern/rna_userdef.c

===================================================================

diff --git a/release/datafiles/userdef/userdef_default.c b/release/datafiles/userdef/userdef_default.c
index 18750c12d55..ec4b214c5f5 100644
--- a/release/datafiles/userdef/userdef_default.c
+++ b/release/datafiles/userdef/userdef_default.c
@@ -221,6 +221,11 @@ const UserDef U_default = {
             .temp_win_sizey = 600,
         },
 
+    .sequencer_disk_cache_dir = "",
+    .sequencer_disk_cache_compression = 0,
+    .sequencer_disk_cache_size_limit = 100,
+    .sequencer_disk_cache_flag = 0,
+
     .runtime =
         {
             .is_dirty = 0,
diff --git a/release/scripts/startup/bl_ui/space_userpref.py b/release/scripts/startup/bl_ui/space_userpref.py
index 4cccf429179..03828632d64 100644
--- a/release/scripts/startup/bl_ui/space_userpref.py
+++ b/release/scripts/startup/bl_ui/space_userpref.py
@@ -628,6 +628,15 @@ class USERPREF_PT_system_memory(SystemPanel, CenterAlignMixIn, Panel):
 
         flow = layout.grid_flow(row_major=False, columns=0, even_columns=True, even_rows=False, align=False)
 
+        flow.prop(system, "use_sequencer_disk_cache")
+        flow.prop(system, "sequencer_disk_cache_dir")
+        flow.prop(system, "sequencer_disk_cache_size_limit")
+        flow.prop(system, "sequencer_disk_cache_compression")
+
+        layout.separator()
+
+        flow = layout.grid_flow(row_major=False, columns=0, even_columns=True, even_rows=False, align=False)
+
         flow.prop(system, "texture_time_out", text="Texture Time Out")
         flow.prop(system, "texture_collection_rate", text="Garbage Collection Rate")
 
diff --git a/source/blender/blenkernel/BKE_sequencer.h b/source/blender/blenkernel/BKE_sequencer.h
index 31951cc101a..df0f0e730a5 100644
--- a/source/blender/blenkernel/BKE_sequencer.h
+++ b/source/blender/blenkernel/BKE_sequencer.h
@@ -312,19 +312,22 @@ void BKE_sequencer_proxy_set(struct Sequence *seq, bool value);
 struct ImBuf *BKE_sequencer_cache_get(const SeqRenderData *context,
                                       struct Sequence *seq,
                                       float cfra,
-                                      int type);
+                                      int type,
+                                      bool skip_disk_cache);
 void BKE_sequencer_cache_put(const SeqRenderData *context,
                              struct Sequence *seq,
                              float cfra,
                              int type,
                              struct ImBuf *nval,
-                             float cost);
+                             float cost,
+                             bool skip_disk_cache);
 bool BKE_sequencer_cache_put_if_possible(const SeqRenderData *context,
                                          struct Sequence *seq,
                                          float cfra,
                                          int type,
                                          struct ImBuf *nval,
-                                         float cost);
+                                         float cost,
+                                         bool skip_disk_cache);
 bool BKE_sequencer_cache_recycle_item(struct Scene *scene);
 void BKE_sequencer_cache_free_temp_cache(struct Scene *scene, short id, int cfra);
 void BKE_sequencer_cache_destruct(struct Scene *scene);
diff --git a/source/blender/blenkernel/intern/seqcache.c b/source/blender/blenkernel/intern/seqcache.c
index aa4066b643f..f98af4a9406 100644
--- a/source/blender/blenkernel/intern/seqcache.c
+++ b/source/blender/blenkernel/intern/seqcache.c
@@ -22,20 +22,29 @@
 
 #include <stddef.h>
 #include <memory.h>
+#include <time.h>
 
 #include "MEM_guardedalloc.h"
 
 #include "DNA_sequence_types.h"
 #include "DNA_scene_types.h"
+#include "DNA_space_types.h" /* for FILE_MAX. */
 
 #include "IMB_imbuf.h"
 #include "IMB_imbuf_types.h"
+#include "IMB_colormanagement.h"
 
 #include "BLI_mempool.h"
 #include "BLI_threads.h"
 #include "BLI_listbase.h"
 #include "BLI_ghash.h"
+#include "BLI_blenlib.h"
+#include "BLI_endian_switch.h"
+#include "BLI_fileops.h"
+#include "BLI_fileops_types.h"
+#include "BLI_path_util.h"
 
+#include "BKE_global.h"
 #include "BKE_sequencer.h"
 #include "BKE_scene.h"
 #include "BKE_main.h"
@@ -63,15 +72,75 @@
  * entries one by one in reverse order to their creation.
  *
  * User can exclude caching of some images. Such entries will have is_temp_cache set.
+ *
+ *
+ * Disk Cache Design Notes
+ * =======================
+ *
+ * Disk cache uses directory specified in user preferences
+ * For each cached non-temp image, image data and supplementary info are written to HDD.
+ * Multiple(DCACHE_IMAGES_PER_FILE) images share the same file.
+ * Each of these files contains header DiskCacheHeader followed by image data.
+ * Zlib compression with user definable level can be used to compress image data(per image)
+ * Images are written in oreder in which they are rendered.
+ * Overwriting of individual entry is not possible.
+ * Stored images are deleted by invalidation, or when size of all files exceeds maximum
+ * size specified in user preferences.
+ * To distinguish 2 blend files with same name, scene->ed->disk_cache_timestamp
+ * is used as UID. Blend file can still be copied manually which may cause conflict.
+ *
  */
 
+/* <cache type>-<resolution X>x<resolution Y>-<rendersize>%(<view_id>)-<frame no>.dcf */
+#define DCACHE_FNAME_FORMAT "%d-%dx%d-%d%%(%d)-%d.dcf"
+#define DCACHE_IMAGES_PER_FILE 100
+#define DCACHE_CURRENT_VERSION 1
+#define COLORSPACE_NAME_MAX 64 /* XXX: defined in imb intern */
+
+typedef struct DiskCacheHeaderEntry {
+  unsigned char encoding;
+  uint64_t frameno;
+  uint64_t size_compressed;
+  uint64_t size_raw;
+  uint64_t offset;
+  char colorspace_name[COLORSPACE_NAME_MAX];
+} DiskCacheHeaderEntry;
+
+typedef struct DiskCacheHeader {
+  DiskCacheHeaderEntry entry[DCACHE_IMAGES_PER_FILE];
+} DiskCacheHeader;
+
+typedef struct SeqDiskCache {
+  Main *bmain;
+  int64_t timestamp;
+  ListBase files;
+  ThreadMutex read_write_mutex;
+  size_t size_total;
+} SeqDiskCache;
+
+typedef struct DiskCacheFile {
+  struct DiskCacheFile *next, *prev;
+  char path[FILE_MAX];
+  char dir[FILE_MAXDIR];
+  char file[FILE_MAX];
+  BLI_stat_t fstat;
+  int cache_type;
+  int rectx;
+  int recty;
+  int render_size;
+  int view_id;
+  int start_frame;
+} DiskCacheFile;
+
 typedef struct SeqCache {
+  Main *bmain;
   struct GHash *hash;
   ThreadMutex iterator_mutex;
   struct BLI_mempool *keys_pool;
   struct BLI_mempool *items_pool;
   struct SeqCacheKey *last_key;
   size_t memory_used;
+  SeqDiskCache *disk_cache;
 } SeqCache;
 
 typedef struct SeqCacheItem {
@@ -82,8 +151,8 @@ typedef struct SeqCacheItem {
 typedef struct SeqCacheKey {
   struct SeqCache *cache_owner;
   void *userkey;
-  struct SeqCacheKey *link_prev; /* Used for linking intermediate items to final frame */
-  struct SeqCacheKey *link_next; /* Used for linking intermediate items to final frame */
+  struct SeqCacheKey *link_prev; /* Used for linking intermediate items to final frame. */
+  struct SeqCacheKey *link_next; /* Used for linking intermediate items to final frame. */
   struct Sequence *seq;
   SeqRenderData context;
   float nfra;
@@ -95,6 +164,530 @@ typedef struct SeqCacheKey {
 } SeqCacheKey;
 
 static ThreadMutex cache_create_lock = BLI_MUTEX_INITIALIZER;
+static float seq_cache_cfra_to_frame_index(Sequence *seq, float cfra);
+static float seq_cache_frame_index_to_cfra(Sequence *seq, float nfra);
+
+static char *seq_disk_cache_base_dir(void)
+{
+  return U.sequencer_disk_cache_dir;
+}
+
+static int seq_disk_cache_compression_level(void)
+{
+  switch (U.sequencer_disk_cache_compression) {
+    case USER_SEQ_DISK_CACHE_COMPRESSION_NONE:
+      return 0;
+    case USER_SEQ_DISK_CACHE_COMPRESSION_LOW:
+      return 1;
+    case USER_SEQ_DISK_CACHE_COMPRESSION_HIGH:
+      return 9;
+  }
+
+  return U.sequencer_disk_cache_compression;
+}
+
+static size_t seq_disk_cache_size_limit(void)
+{
+  return (size_t)U.sequencer_disk_cache_size_limit * (1024 * 1024 * 1024);
+}
+
+static bool seq_disk_cache_is_enabled(Main *bmain)
+{
+  return (U.sequencer_disk_cache_dir[0] != '\0' && U.sequencer_disk_cache_size_limit != 0 &&
+          (U.sequencer_disk_cache_flag & SEQ_CACHE_DISK_CACHE_ENABLE) != 0 &&
+          bmain->name[0] != '\0');
+}
+
+static DiskCacheFile *seq_disk_cache_add_file_to_list(SeqDiskCache *disk_cache, const char *path)
+{
+
+  DiskCacheFile *cache_file = MEM_callocN(sizeof(DiskCacheFile), "SeqDiskCacheFile");
+  char dir[FILE_MAXDIR], file[FILE_MAX];
+  BLI_split_dirfile(path, dir, file, sizeof(dir), sizeof(file));
+  BLI_strncpy(cache_file->path, path, sizeof(cache_file->path));
+  BLI_strncpy(cache_file->dir, dir, sizeof(cache_file->dir));
+  BLI_strncpy(cache_file->file, file, sizeof(cache_file->file));
+  sscanf(file,
+         DCACHE_FNAME_FORMAT,
+         &cache_file->cache_type,
+         &cache_file->rectx,
+         &cache_file->recty,
+         &cache_file->render_size,
+         &cache_file->view_id,
+         &cache_file->start_frame);
+  cache_file->start_frame *= DCACHE_IMAGES_PER_FILE;
+  BLI_addtail(&disk_cache->files, cache_file);
+  return cache_file;
+}
+
+static void seq_disk_cache_get_files(SeqDiskCache *disk_cache, char *path)
+{
+  struct direntry *filelist, *fl;
+  uint nbr, i;
+  disk_cache->size_total = 0;
+
+  i = nbr = BLI_filelist_dir_contents(path, &filelist);
+  fl = filelist;
+  while (i--) {
+    /* Don't follow links. */
+    const eFileAttributes file_attrs = BLI_file_attributes(fl->path);
+    if (file_attrs & FILE_ATTR_ANY_LINK) {
+      fl++;
+      continue;
+    }
+
+    char file[FILE_MAX];
+    BLI_split_dirfile(fl->path, NULL, file, 0, sizeof(file));
+
+    bool is_dir = BLI_is_dir(fl->path);
+    if (is_dir && !FILENAME_IS_CURRPAR(file)) {
+      char subpath[FILE_MAX];
+      BLI_strncpy(subpath, fl->path, sizeof(subpath));
+      BLI_add_slash(subpath);
+      seq_disk_cache_get_files(disk_cache, subpath);
+    }
+
+    if (!is_dir) {
+      const char *ext = BLI_path_extension(fl->path);
+      if (ext && ext[1] == 'd' && ext[2] == 'c' && ext[3] == 'f') {
+        DiskCacheFile *cache_file = seq_disk_cache_add_file_to_list(disk_cache, fl->path);
+        cache_file->fstat = fl

@@ Diff output truncated at 10240 characters. @@



More information about the Bf-blender-cvs mailing list