[Bf-blender-cvs] [70cc80ea1c7] master: Cleanup: Move VSE disk cache code into own file
Richard Antalik
noreply at git.blender.org
Thu Oct 7 03:06:59 CEST 2021
Commit: 70cc80ea1c7f2678762c8ec31b924f9624172810
Author: Richard Antalik
Date: Thu Oct 7 03:04:34 2021 +0200
Branches: master
https://developer.blender.org/rB70cc80ea1c7f2678762c8ec31b924f9624172810
Cleanup: Move VSE disk cache code into own file
No functional changes.
===================================================================
M source/blender/sequencer/CMakeLists.txt
A source/blender/sequencer/intern/disk_cache.c
A source/blender/sequencer/intern/disk_cache.h
M source/blender/sequencer/intern/image_cache.c
M source/blender/sequencer/intern/image_cache.h
===================================================================
diff --git a/source/blender/sequencer/CMakeLists.txt b/source/blender/sequencer/CMakeLists.txt
index f060e6ad69b..eccc336141a 100644
--- a/source/blender/sequencer/CMakeLists.txt
+++ b/source/blender/sequencer/CMakeLists.txt
@@ -63,6 +63,8 @@ set(SRC
SEQ_utils.h
intern/clipboard.c
+ intern/disk_cache.c
+ intern/disk_cache.h
intern/effects.c
intern/effects.h
intern/image_cache.c
diff --git a/source/blender/sequencer/intern/disk_cache.c b/source/blender/sequencer/intern/disk_cache.c
new file mode 100644
index 00000000000..543c23b184b
--- /dev/null
+++ b/source/blender/sequencer/intern/disk_cache.c
@@ -0,0 +1,698 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2021 Blender Foundation.
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup sequencer
+ */
+
+#include <memory.h>
+#include <stddef.h>
+#include <time.h>
+
+#include "MEM_guardedalloc.h"
+
+#include "DNA_scene_types.h"
+#include "DNA_sequence_types.h"
+#include "DNA_space_types.h" /* for FILE_MAX. */
+
+#include "IMB_colormanagement.h"
+#include "IMB_imbuf.h"
+#include "IMB_imbuf_types.h"
+
+#include "BLI_blenlib.h"
+#include "BLI_endian_defines.h"
+#include "BLI_endian_switch.h"
+#include "BLI_fileops.h"
+#include "BLI_fileops_types.h"
+#include "BLI_ghash.h"
+#include "BLI_listbase.h"
+#include "BLI_mempool.h"
+#include "BLI_path_util.h"
+#include "BLI_threads.h"
+
+#include "BKE_main.h"
+#include "BKE_scene.h"
+
+#include "SEQ_prefetch.h"
+#include "SEQ_relations.h"
+#include "SEQ_render.h"
+#include "SEQ_sequencer.h"
+
+#include "disk_cache.h"
+#include "image_cache.h"
+#include "prefetch.h"
+#include "strip_time.h"
+
+/**
+ * Disk Cache Design Notes
+ * =======================
+ *
+ * Disk cache uses directory specified in user preferences
+ * For each cached non-temp image, image data and supplementary info are written to HDD.
+ * Multiple(DCACHE_IMAGES_PER_FILE) images share the same file.
+ * Each of these files contains header DiskCacheHeader followed by image data.
+ * Zlib compression with user definable level can be used to compress image data(per image)
+ * Images are written in order in which they are rendered.
+ * Overwriting of individual entry is not possible.
+ * Stored images are deleted by invalidation, or when size of all files exceeds maximum
+ * size specified in user preferences.
+ * To distinguish 2 blend files with same name, scene->ed->disk_cache_timestamp
+ * is used as UID. Blend file can still be copied manually which may cause conflict.
+ */
+
+/* <cache type>-<resolution X>x<resolution Y>-<rendersize>%(<view_id>)-<frame no>.dcf */
+#define DCACHE_FNAME_FORMAT "%d-%dx%d-%d%%(%d)-%d.dcf"
+#define DCACHE_IMAGES_PER_FILE 100
+#define DCACHE_CURRENT_VERSION 2
+#define COLORSPACE_NAME_MAX 64 /* XXX: defined in imb intern */
+
+typedef struct DiskCacheHeaderEntry {
+ unsigned char encoding;
+ uint64_t frameno;
+ uint64_t size_compressed;
+ uint64_t size_raw;
+ uint64_t offset;
+ char colorspace_name[COLORSPACE_NAME_MAX];
+} DiskCacheHeaderEntry;
+
+typedef struct DiskCacheHeader {
+ DiskCacheHeaderEntry entry[DCACHE_IMAGES_PER_FILE];
+} DiskCacheHeader;
+
+typedef struct SeqDiskCache {
+ Main *bmain;
+ int64_t timestamp;
+ ListBase files;
+ ThreadMutex read_write_mutex;
+ size_t size_total;
+} SeqDiskCache;
+
+typedef struct DiskCacheFile {
+ struct DiskCacheFile *next, *prev;
+ char path[FILE_MAX];
+ char dir[FILE_MAXDIR];
+ char file[FILE_MAX];
+ BLI_stat_t fstat;
+ int cache_type;
+ int rectx;
+ int recty;
+ int render_size;
+ int view_id;
+ int start_frame;
+} DiskCacheFile;
+
+static ThreadMutex cache_create_lock = BLI_MUTEX_INITIALIZER;
+
+static char *seq_disk_cache_base_dir(void)
+{
+ return U.sequencer_disk_cache_dir;
+}
+
+static int seq_disk_cache_compression_level(void)
+{
+ switch (U.sequencer_disk_cache_compression) {
+ case USER_SEQ_DISK_CACHE_COMPRESSION_NONE:
+ return 0;
+ case USER_SEQ_DISK_CACHE_COMPRESSION_LOW:
+ return 1;
+ case USER_SEQ_DISK_CACHE_COMPRESSION_HIGH:
+ return 9;
+ }
+
+ return U.sequencer_disk_cache_compression;
+}
+
+static size_t seq_disk_cache_size_limit(void)
+{
+ return (size_t)U.sequencer_disk_cache_size_limit * (1024 * 1024 * 1024);
+}
+
+bool seq_disk_cache_is_enabled(Main *bmain)
+{
+ return (U.sequencer_disk_cache_dir[0] != '\0' && U.sequencer_disk_cache_size_limit != 0 &&
+ (U.sequencer_disk_cache_flag & SEQ_CACHE_DISK_CACHE_ENABLE) != 0 &&
+ bmain->name[0] != '\0');
+}
+
+static DiskCacheFile *seq_disk_cache_add_file_to_list(SeqDiskCache *disk_cache, const char *path)
+{
+
+ DiskCacheFile *cache_file = MEM_callocN(sizeof(DiskCacheFile), "SeqDiskCacheFile");
+ char dir[FILE_MAXDIR], file[FILE_MAX];
+ BLI_split_dirfile(path, dir, file, sizeof(dir), sizeof(file));
+ BLI_strncpy(cache_file->path, path, sizeof(cache_file->path));
+ BLI_strncpy(cache_file->dir, dir, sizeof(cache_file->dir));
+ BLI_strncpy(cache_file->file, file, sizeof(cache_file->file));
+ sscanf(file,
+ DCACHE_FNAME_FORMAT,
+ &cache_file->cache_type,
+ &cache_file->rectx,
+ &cache_file->recty,
+ &cache_file->render_size,
+ &cache_file->view_id,
+ &cache_file->start_frame);
+ cache_file->start_frame *= DCACHE_IMAGES_PER_FILE;
+ BLI_addtail(&disk_cache->files, cache_file);
+ return cache_file;
+}
+
+static void seq_disk_cache_get_files(SeqDiskCache *disk_cache, char *path)
+{
+ struct direntry *filelist, *fl;
+ uint nbr, i;
+ disk_cache->size_total = 0;
+
+ i = nbr = BLI_filelist_dir_contents(path, &filelist);
+ fl = filelist;
+ while (i--) {
+ /* Don't follow links. */
+ const eFileAttributes file_attrs = BLI_file_attributes(fl->path);
+ if (file_attrs & FILE_ATTR_ANY_LINK) {
+ fl++;
+ continue;
+ }
+
+ char file[FILE_MAX];
+ BLI_split_dirfile(fl->path, NULL, file, 0, sizeof(file));
+
+ bool is_dir = BLI_is_dir(fl->path);
+ if (is_dir && !FILENAME_IS_CURRPAR(file)) {
+ char subpath[FILE_MAX];
+ BLI_strncpy(subpath, fl->path, sizeof(subpath));
+ BLI_path_slash_ensure(subpath);
+ seq_disk_cache_get_files(disk_cache, subpath);
+ }
+
+ if (!is_dir) {
+ const char *ext = BLI_path_extension(fl->path);
+ if (ext && ext[1] == 'd' && ext[2] == 'c' && ext[3] == 'f') {
+ DiskCacheFile *cache_file = seq_disk_cache_add_file_to_list(disk_cache, fl->path);
+ cache_file->fstat = fl->s;
+ disk_cache->size_total += cache_file->fstat.st_size;
+ }
+ }
+ fl++;
+ }
+ BLI_filelist_free(filelist, nbr);
+}
+
+static DiskCacheFile *seq_disk_cache_get_oldest_file(SeqDiskCache *disk_cache)
+{
+ DiskCacheFile *oldest_file = disk_cache->files.first;
+ if (oldest_file == NULL) {
+ return NULL;
+ }
+ for (DiskCacheFile *cache_file = oldest_file->next; cache_file; cache_file = cache_file->next) {
+ if (cache_file->fstat.st_mtime < oldest_file->fstat.st_mtime) {
+ oldest_file = cache_file;
+ }
+ }
+
+ return oldest_file;
+}
+
+static void seq_disk_cache_delete_file(SeqDiskCache *disk_cache, DiskCacheFile *file)
+{
+ disk_cache->size_total -= file->fstat.st_size;
+ BLI_delete(file->path, false, false);
+ BLI_remlink(&disk_cache->files, file);
+ MEM_freeN(file);
+}
+
+bool seq_disk_cache_enforce_limits(SeqDiskCache *disk_cache)
+{
+ BLI_mutex_lock(&disk_cache->read_write_mutex);
+ while (disk_cache->size_total > seq_disk_cache_size_limit()) {
+ DiskCacheFile *oldest_file = seq_disk_cache_get_oldest_file(disk_cache);
+
+ if (!oldest_file) {
+ /* We shouldn't enforce limits with no files, do re-scan. */
+ seq_disk_cache_get_files(disk_cache, seq_disk_cache_base_dir());
+ continue;
+ }
+
+ if (BLI_exists(oldest_file->path) == 0) {
+ /* File may have been manually deleted during runtime, do re-scan. */
+ BLI_freelistN(&disk_cache->files);
+ seq_disk_cache_get_files(disk_cache, seq_disk_cache_base_dir());
+ continue;
+ }
+
+ seq_disk_cache_delete_file(disk_cache, oldest_file);
+ }
+ BLI_mutex_unlock(&disk_cache->read_write_mutex);
+
+ return true;
+}
+
+static DiskCacheFile *seq_disk_cache_get_file_entry_by_path(SeqDiskCache *disk_cache, char *path)
+{
+ DiskCacheFile *cache_file = disk_cache->files.first;
+
+ for (; cache_file; cache_file = cache_file->next) {
+ if (BLI_strcasecmp(cache_file->path, path) == 0) {
+ return cache_file;
+ }
+ }
+
+ return NULL;
+}
+
+/* Update file size and timestamp. */
+static void seq_disk_cache_update_file(SeqDiskCache *disk_cache, char *path)
+{
+ DiskCacheFile *cache_file;
+ int64_t size_before;
+ int64_t size_after;
+
+ cache_file = seq_disk_cache_get_file_entry_by_path(disk_cache, path);
+ size_before = cache_file->fstat.st_size;
+
+ if (BLI_stat(path, &cache_file->fstat) == -1) {
+ BLI_assert(false);
+ memset(&cache_file->fstat, 0, sizeof(BLI_stat_t));
+ }
+
+ size_after = cache_file->fstat.st_size;
+ disk_cache->size_total += size_after - size_before;
+}
+
+/* Path format:
+ * <cache dir>/<project name>_seq_cache/<scene name>-<timestamp>/<seq name>/DCACHE_FNAME_FORMAT
+ */
+
+static void seq_disk_cache_get_project_dir(SeqDiskCache *disk_cache, char *path, size_t path_len)
+{
+ char cache_dir[FILE_MAX];
+ BLI_split_file_part(BKE_main_blendfile_path(disk_cache->bmain), cache_dir, sizeof(cache_dir));
+ /* Use suffix, so that the cache directory name does not conflict with the bmain's blend file. */
+ const char *suffix = "_seq_cache";
+ strncat(cache_dir, suffi
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list