[Bf-blender-cvs] [2ea66af742b] master: Add support for Zstandard compression for .blend files
Lukas Stockner
noreply at git.blender.org
Sat Aug 21 21:54:38 CEST 2021
Commit: 2ea66af742bca4b427f88de13254414730a33776
Author: Lukas Stockner
Date: Sat Aug 21 03:15:31 2021 +0200
Branches: master
https://developer.blender.org/rB2ea66af742bca4b427f88de13254414730a33776
Add support for Zstandard compression for .blend files
Compressing blendfiles can help save a lot of disk space, but the slowdown
while loading and saving is a major annoyance.
Currently Blender uses Zlib (aka gzip aka Deflate) for compression, but there
are now several more modern algorithms that outperform it in every way.
In this patch, I decided for Zstandard aka Zstd for several reasons:
- It is widely supported, both in other programs and libraries as well as in
general-purpose compression utilities on Unix
- It is extremely flexible - spanning several orders of magnitude of
compression speeds depending on the level setting.
- It is pretty much on the Pareto frontier for all of its configurations
(meaning that no other algorithm is both faster and more efficient).
One downside of course is that older versions of Blender will not be able to
read these files, but one can always just re-save them without compression or
decompress the file manually with an external tool.
The implementation here saves additional metadata into the compressed file in
order to allow for efficient seeking when loading. This is standard-compliant
and will be ignored by other tools that support Zstd.
If the metadata is not present (e.g. because you manually compressed a .blend
file with another tool), Blender will fall back to sequential reading.
Saving is multithreaded to improve performance. Loading is currently not
multithreaded since it's not easy to predict the access patterns of the
loading code when seeking is supported.
In the future, we might want to look into making this more predictable or
disabling seeking for the main .blend file, which would then allow for
multiple background threads that decompress data ahead of time.
The compression level was chosen to get sizes comparable to previous versions
at much higher speeds. In the future, this could be exposed as an option.
Reviewed By: campbellbarton, brecht, mont29
Differential Revision: https://developer.blender.org/D5799
===================================================================
A build_files/cmake/Modules/FindZstd.cmake
M build_files/cmake/platform/platform_apple.cmake
M build_files/cmake/platform/platform_unix.cmake
M build_files/cmake/platform/platform_win32.cmake
M source/blender/blenlib/BLI_fileops.h
M source/blender/blenlib/BLI_filereader.h
M source/blender/blenlib/CMakeLists.txt
M source/blender/blenlib/intern/fileops.c
A source/blender/blenlib/intern/filereader_zstd.c
M source/blender/blenloader/CMakeLists.txt
M source/blender/blenloader/intern/readfile.c
M source/blender/blenloader/intern/writefile.c
M source/blender/windowmanager/CMakeLists.txt
M source/blender/windowmanager/intern/wm_files.c
===================================================================
diff --git a/build_files/cmake/Modules/FindZstd.cmake b/build_files/cmake/Modules/FindZstd.cmake
new file mode 100644
index 00000000000..84606d01f44
--- /dev/null
+++ b/build_files/cmake/Modules/FindZstd.cmake
@@ -0,0 +1,66 @@
+# - Find Zstd library
+# Find the native Zstd includes and library
+# This module defines
+# ZSTD_INCLUDE_DIRS, where to find zstd.h, Set when
+# ZSTD_INCLUDE_DIR is found.
+# ZSTD_LIBRARIES, libraries to link against to use Zstd.
+# ZSTD_ROOT_DIR, The base directory to search for Zstd.
+# This can also be an environment variable.
+# ZSTD_FOUND, If false, do not try to use Zstd.
+#
+# also defined, but not for general use are
+# ZSTD_LIBRARY, where to find the Zstd library.
+
+#=============================================================================
+# Copyright 2019 Blender Foundation.
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+
+# If ZSTD_ROOT_DIR was defined in the environment, use it.
+IF(NOT ZSTD_ROOT_DIR AND NOT $ENV{ZSTD_ROOT_DIR} STREQUAL "")
+ SET(ZSTD_ROOT_DIR $ENV{ZSTD_ROOT_DIR})
+ENDIF()
+
+SET(_zstd_SEARCH_DIRS
+ ${ZSTD_ROOT_DIR}
+)
+
+FIND_PATH(ZSTD_INCLUDE_DIR
+ NAMES
+ zstd.h
+ HINTS
+ ${_zstd_SEARCH_DIRS}
+ PATH_SUFFIXES
+ include
+)
+
+FIND_LIBRARY(ZSTD_LIBRARY
+ NAMES
+ zstd
+ HINTS
+ ${_zstd_SEARCH_DIRS}
+ PATH_SUFFIXES
+ lib64 lib
+ )
+
+# handle the QUIETLY and REQUIRED arguments and set ZSTD_FOUND to TRUE if
+# all listed variables are TRUE
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(Zstd DEFAULT_MSG
+ ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+IF(ZSTD_FOUND)
+ SET(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+ SET(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
+ENDIF()
+
+MARK_AS_ADVANCED(
+ ZSTD_INCLUDE_DIR
+ ZSTD_LIBRARY
+)
diff --git a/build_files/cmake/platform/platform_apple.cmake b/build_files/cmake/platform/platform_apple.cmake
index dceafb236de..2bfdc84ec2a 100644
--- a/build_files/cmake/platform/platform_apple.cmake
+++ b/build_files/cmake/platform/platform_apple.cmake
@@ -441,6 +441,9 @@ if(WITH_HARU)
endif()
endif()
+set(ZSTD_ROOT_DIR ${LIBDIR}/zstd)
+find_package(Zstd REQUIRED)
+
if(EXISTS ${LIBDIR})
without_system_libs_end()
endif()
diff --git a/build_files/cmake/platform/platform_unix.cmake b/build_files/cmake/platform/platform_unix.cmake
index 7f62399ac4f..fc0c37e4c8b 100644
--- a/build_files/cmake/platform/platform_unix.cmake
+++ b/build_files/cmake/platform/platform_unix.cmake
@@ -99,6 +99,7 @@ endif()
find_package_wrapper(JPEG REQUIRED)
find_package_wrapper(PNG REQUIRED)
find_package_wrapper(ZLIB REQUIRED)
+find_package_wrapper(Zstd REQUIRED)
find_package_wrapper(Freetype REQUIRED)
if(WITH_PYTHON)
diff --git a/build_files/cmake/platform/platform_win32.cmake b/build_files/cmake/platform/platform_win32.cmake
index 3773aaaffed..d44ef691d1b 100644
--- a/build_files/cmake/platform/platform_win32.cmake
+++ b/build_files/cmake/platform/platform_win32.cmake
@@ -873,3 +873,6 @@ if(WITH_HARU)
set(WITH_HARU OFF)
endif()
endif()
+
+set(ZSTD_INCLUDE_DIRS ${LIBDIR}/zstd/include)
+set(ZSTD_LIBRARIES ${LIBDIR}/zstd/lib/zstd_static.lib)
diff --git a/source/blender/blenlib/BLI_fileops.h b/source/blender/blenlib/BLI_fileops.h
index 12fa73279c8..377b7bc3bc2 100644
--- a/source/blender/blenlib/BLI_fileops.h
+++ b/source/blender/blenlib/BLI_fileops.h
@@ -168,6 +168,8 @@ size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t g
ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
bool BLI_file_magic_is_gzip(const char header[4]);
+bool BLI_file_magic_is_zstd(const char header[4]);
+
size_t BLI_file_descriptor_size(int file) ATTR_WARN_UNUSED_RESULT;
size_t BLI_file_size(const char *path) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
diff --git a/source/blender/blenlib/BLI_filereader.h b/source/blender/blenlib/BLI_filereader.h
index dbcaa7cb1b6..8d1fa3d1596 100644
--- a/source/blender/blenlib/BLI_filereader.h
+++ b/source/blender/blenlib/BLI_filereader.h
@@ -61,7 +61,7 @@ typedef struct FileReader {
*
* If a FileReader is created, it has to be cleaned up and freed by calling
* its close() function unless another FileReader has taken ownership - for example,
- * Gzip takes over the base FileReader and will clean it up when their clean() is called.
+ * Zstd and Gzip take over the base FileReader and will clean it up when their clean() is called.
*/
/* Create FileReader from raw file descriptor. */
@@ -71,6 +71,8 @@ FileReader *BLI_filereader_new_mmap(int filedes) ATTR_WARN_UNUSED_RESULT;
/* Create FileReader from a region of memory. */
FileReader *BLI_filereader_new_memory(const void *data, size_t len) ATTR_WARN_UNUSED_RESULT
ATTR_NONNULL();
+/* Create FileReader from applying Zstd decompression on an underlying file. */
+FileReader *BLI_filereader_new_zstd(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
/* Create FileReader from applying Gzip decompression on an underlying file. */
FileReader *BLI_filereader_new_gzip(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
diff --git a/source/blender/blenlib/CMakeLists.txt b/source/blender/blenlib/CMakeLists.txt
index d2ba9e74c90..f98d15ad08b 100644
--- a/source/blender/blenlib/CMakeLists.txt
+++ b/source/blender/blenlib/CMakeLists.txt
@@ -31,6 +31,7 @@ set(INC
set(INC_SYS
${ZLIB_INCLUDE_DIRS}
+ ${ZSTD_INCLUDE_DIRS}
${FREETYPE_INCLUDE_DIRS}
${GMP_INCLUDE_DIRS}
)
@@ -78,6 +79,7 @@ set(SRC
intern/filereader_file.c
intern/filereader_gzip.c
intern/filereader_memory.c
+ intern/filereader_zstd.c
intern/fnmatch.c
intern/freetypefont.c
intern/gsqueue.c
@@ -327,6 +329,7 @@ set(LIB
${FREETYPE_LIBRARY}
${ZLIB_LIBRARIES}
+ ${ZSTD_LIBRARIES}
)
if(WITH_MEM_VALGRIND)
diff --git a/source/blender/blenlib/intern/fileops.c b/source/blender/blenlib/intern/fileops.c
index 6fc2222241b..31825c69737 100644
--- a/source/blender/blenlib/intern/fileops.c
+++ b/source/blender/blenlib/intern/fileops.c
@@ -262,6 +262,33 @@ bool BLI_file_magic_is_gzip(const char header[4])
return header[0] == 0x1f && header[1] == 0x8b && header[2] == 0x08;
}
+bool BLI_file_magic_is_zstd(const char header[4])
+{
+ /* ZSTD files consist of concatenated frames, each either a Zstd frame or a skippable frame.
+ * Both types of frames start with a magic number: 0xFD2FB528 for Zstd frames and 0x184D2A5*
+ * for skippable frames, with the * being anything from 0 to F.
+ *
+ * To check whether a file is Zstd-compressed, we just check whether the first frame matches
+ * either. Seeking through the file until a Zstd frame is found would make things more
+ * complicated and the probability of a false positive is rather low anyways.
+ *
+ * Note that LZ4 uses a compatible format, so even though its compressed frames have a
+ * different magic number, a valid LZ4 file might also start with a skippable frame matching
+ * the second check here.
+ *
+ * For more details, see https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md
+ */
+
+ uint32_t magic = *((uint32_t *)header);
+ if (magic == 0xFD2FB528) {
+ return true;
+ }
+ if ((magic >> 4) == 0x184D2A5) {
+ return true;
+ }
+ return false;
+}
+
/**
* Returns true if the file with the specified name can be written.
* This implementation uses access(2), which makes the check according
diff --git a/source/blender/blenlib/intern/filereader_zstd.c b/source/blender/blenlib/intern/filereader_zstd.c
new file mode 100644
index 00000000000..785a40cd1a1
--- /dev/null
+++ b/source/blender/blenlib/intern/filereader_zstd.c
@@ -0,0 +1,335 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2021 Blender Foundation
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ */
+
+#include <string.h>
+#include <zstd.h>
+
+#include "BLI_blenlib.h"
+#include "BLI_endian_switch.h"
+#include "BLI_filereader.h"
+#include "BLI_math_base.h"
+
+#include "MEM_guardedalloc.h"
+
+typedef struct {
+ FileReader reader;
+
+ FileReader *base;
+
+ ZSTD_DCtx *ctx;
+ ZSTD_inBuffer in_buf;
+ size_t in_buf_max_size;
+
+ struct {
+ int num_frames;
+ size_t *compressed_ofs;
+ size_t *uncompressed_ofs;
+
+ char *cached_content;
+ int cached_frame;
+ } seek;
+} ZstdReader;
+
+static bool zstd_read_u32(FileReader *base, uint32_t *val)
+{
+ if (base->read(base, val, sizeof(uint32_t)) != sizeof(uint32_t)) {
+ return false;
+ }
+#ifdef __BIG_ENDIAN__
+ BLI_endian_switch_uint32(val);
+#endif
+ return true;
+}
+
+static bool zstd_read_seek_table(ZstdReader *zstd)
+{
+ FileReader *base = zstd->base;
+
+ /* The seek table frame is at the end of the file, so seek there
+ * and verify that there is enough data. */
+ if (base->seek(base, -4, SEEK_END) < 13) {
+ return false;
+ }
+ uint32_t magic;
+ if (!zstd_read_u32(base, &magic) || magic != 0x8F92EAB1) {
+ return false;
+ }
+
+ uint8_t flags;
+ if (base->seek(base, -5, SEEK_END) < 0 || base->read(base, &flags, 1) != 1) {
+ return false;
+ }
+ /* Bit 7 indicates checksums. Bits 5 and 6 must be zero. */
+ bool has_checksums = (flags & 0x80);
+ if (flags & 0x60) {
+ return false;
+ }
+
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list