[Bf-blender-cvs] [662e37d280e] experimental-build: D5799 Part 2: ZStd support
Lukas Stockner
noreply at git.blender.org
Sat Aug 21 05:08:11 CEST 2021
Commit: 662e37d280e5e085b98500d9c27a41c35a6e65fe
Author: Lukas Stockner
Date: Sat Aug 21 03:15:31 2021 +0200
Branches: experimental-build
https://developer.blender.org/rB662e37d280e5e085b98500d9c27a41c35a6e65fe
D5799 Part 2: ZStd support
===================================================================
A build_files/cmake/Modules/FindZstd.cmake
M build_files/cmake/platform/platform_unix.cmake
M build_files/cmake/platform/platform_win32.cmake
M source/blender/blenlib/BLI_fileops.h
M source/blender/blenlib/BLI_filereader.h
M source/blender/blenlib/CMakeLists.txt
M source/blender/blenlib/intern/fileops.c
A source/blender/blenlib/intern/filereader_zstd.c
M source/blender/blenloader/CMakeLists.txt
M source/blender/blenloader/intern/readfile.c
M source/blender/blenloader/intern/writefile.c
M source/blender/windowmanager/CMakeLists.txt
M source/blender/windowmanager/intern/wm_files.c
===================================================================
diff --git a/build_files/cmake/Modules/FindZstd.cmake b/build_files/cmake/Modules/FindZstd.cmake
new file mode 100644
index 00000000000..84606d01f44
--- /dev/null
+++ b/build_files/cmake/Modules/FindZstd.cmake
@@ -0,0 +1,66 @@
+# - Find Zstd library
+# Find the native Zstd includes and library
+# This module defines
+# ZSTD_INCLUDE_DIRS, where to find zstd.h, Set when
+# ZSTD_INCLUDE_DIR is found.
+# ZSTD_LIBRARIES, libraries to link against to use Zstd.
+# ZSTD_ROOT_DIR, The base directory to search for Zstd.
+# This can also be an environment variable.
+# ZSTD_FOUND, If false, do not try to use Zstd.
+#
+# also defined, but not for general use are
+# ZSTD_LIBRARY, where to find the Zstd library.
+
+#=============================================================================
+# Copyright 2019 Blender Foundation.
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+
+# If ZSTD_ROOT_DIR was defined in the environment, use it.
+IF(NOT ZSTD_ROOT_DIR AND NOT $ENV{ZSTD_ROOT_DIR} STREQUAL "")
+ SET(ZSTD_ROOT_DIR $ENV{ZSTD_ROOT_DIR})
+ENDIF()
+
+SET(_zstd_SEARCH_DIRS
+ ${ZSTD_ROOT_DIR}
+)
+
+FIND_PATH(ZSTD_INCLUDE_DIR
+ NAMES
+ zstd.h
+ HINTS
+ ${_zstd_SEARCH_DIRS}
+ PATH_SUFFIXES
+ include
+)
+
+FIND_LIBRARY(ZSTD_LIBRARY
+ NAMES
+ zstd
+ HINTS
+ ${_zstd_SEARCH_DIRS}
+ PATH_SUFFIXES
+ lib64 lib
+ )
+
+# handle the QUIETLY and REQUIRED arguments and set ZSTD_FOUND to TRUE if
+# all listed variables are TRUE
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(Zstd DEFAULT_MSG
+ ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+IF(ZSTD_FOUND)
+ SET(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+ SET(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
+ENDIF()
+
+MARK_AS_ADVANCED(
+ ZSTD_INCLUDE_DIR
+ ZSTD_LIBRARY
+)
diff --git a/build_files/cmake/platform/platform_unix.cmake b/build_files/cmake/platform/platform_unix.cmake
index 7f62399ac4f..fc0c37e4c8b 100644
--- a/build_files/cmake/platform/platform_unix.cmake
+++ b/build_files/cmake/platform/platform_unix.cmake
@@ -99,6 +99,7 @@ endif()
find_package_wrapper(JPEG REQUIRED)
find_package_wrapper(PNG REQUIRED)
find_package_wrapper(ZLIB REQUIRED)
+find_package_wrapper(Zstd REQUIRED)
find_package_wrapper(Freetype REQUIRED)
if(WITH_PYTHON)
diff --git a/build_files/cmake/platform/platform_win32.cmake b/build_files/cmake/platform/platform_win32.cmake
index 3773aaaffed..d44ef691d1b 100644
--- a/build_files/cmake/platform/platform_win32.cmake
+++ b/build_files/cmake/platform/platform_win32.cmake
@@ -873,3 +873,6 @@ if(WITH_HARU)
set(WITH_HARU OFF)
endif()
endif()
+
+set(ZSTD_INCLUDE_DIRS ${LIBDIR}/zstd/include)
+set(ZSTD_LIBRARIES ${LIBDIR}/zstd/lib/zstd_static.lib)
diff --git a/source/blender/blenlib/BLI_fileops.h b/source/blender/blenlib/BLI_fileops.h
index 12fa73279c8..377b7bc3bc2 100644
--- a/source/blender/blenlib/BLI_fileops.h
+++ b/source/blender/blenlib/BLI_fileops.h
@@ -168,6 +168,8 @@ size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t g
ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
bool BLI_file_magic_is_gzip(const char header[4]);
+bool BLI_file_magic_is_zstd(const char header[4]);
+
size_t BLI_file_descriptor_size(int file) ATTR_WARN_UNUSED_RESULT;
size_t BLI_file_size(const char *path) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
diff --git a/source/blender/blenlib/BLI_filereader.h b/source/blender/blenlib/BLI_filereader.h
index 5391829a548..5daf18deb2f 100644
--- a/source/blender/blenlib/BLI_filereader.h
+++ b/source/blender/blenlib/BLI_filereader.h
@@ -61,7 +61,7 @@ typedef struct FileReader {
*
* If a FileReader is created, it has to be cleaned up and freed by calling
* its close() function unless another FileReader has taken ownership - for example,
- * Gzip takes over the base FileReader and will clean it up when their clean() is called.
+ * Zstd and Gzip take over the base FileReader and will clean it up when their clean() is called.
*/
/* Create FileReader from raw file descriptor. */
@@ -71,6 +71,8 @@ FileReader *BLI_filereader_new_mmap(int filedes) ATTR_WARN_UNUSED_RESULT;
/* Create FileReader from a region of memory. */
FileReader *BLI_filereader_new_memory(const void *data, size_t len) ATTR_WARN_UNUSED_RESULT
ATTR_NONNULL();
+/* Create FileReader from applying Zstd decompression on an underlying file. */
+FileReader *BLI_filereader_new_zstd(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
/* Create FileReader from applying Gzip decompression on an underlying file. */
FileReader *BLI_filereader_new_gzip(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
diff --git a/source/blender/blenlib/CMakeLists.txt b/source/blender/blenlib/CMakeLists.txt
index d2ba9e74c90..f98d15ad08b 100644
--- a/source/blender/blenlib/CMakeLists.txt
+++ b/source/blender/blenlib/CMakeLists.txt
@@ -31,6 +31,7 @@ set(INC
set(INC_SYS
${ZLIB_INCLUDE_DIRS}
+ ${ZSTD_INCLUDE_DIRS}
${FREETYPE_INCLUDE_DIRS}
${GMP_INCLUDE_DIRS}
)
@@ -78,6 +79,7 @@ set(SRC
intern/filereader_file.c
intern/filereader_gzip.c
intern/filereader_memory.c
+ intern/filereader_zstd.c
intern/fnmatch.c
intern/freetypefont.c
intern/gsqueue.c
@@ -327,6 +329,7 @@ set(LIB
${FREETYPE_LIBRARY}
${ZLIB_LIBRARIES}
+ ${ZSTD_LIBRARIES}
)
if(WITH_MEM_VALGRIND)
diff --git a/source/blender/blenlib/intern/fileops.c b/source/blender/blenlib/intern/fileops.c
index 6fc2222241b..31825c69737 100644
--- a/source/blender/blenlib/intern/fileops.c
+++ b/source/blender/blenlib/intern/fileops.c
@@ -262,6 +262,33 @@ bool BLI_file_magic_is_gzip(const char header[4])
return header[0] == 0x1f && header[1] == 0x8b && header[2] == 0x08;
}
+bool BLI_file_magic_is_zstd(const char header[4])
+{
+ /* ZSTD files consist of concatenated frames, each either a Zstd frame or a skippable frame.
+ * Both types of frames start with a magic number: 0xFD2FB528 for Zstd frames and 0x184D2A5*
+ * for skippable frames, with the * being anything from 0 to F.
+ *
+ * To check whether a file is Zstd-compressed, we just check whether the first frame matches
+ * either. Seeking through the file until a Zstd frame is found would make things more
+ * complicated and the probability of a false positive is rather low anyways.
+ *
+ * Note that LZ4 uses a compatible format, so even though its compressed frames have a
+ * different magic number, a valid LZ4 file might also start with a skippable frame matching
+ * the second check here.
+ *
+ * For more details, see https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md
+ */
+
+ uint32_t magic = *((uint32_t *)header);
+ if (magic == 0xFD2FB528) {
+ return true;
+ }
+ if ((magic >> 4) == 0x184D2A5) {
+ return true;
+ }
+ return false;
+}
+
/**
* Returns true if the file with the specified name can be written.
* This implementation uses access(2), which makes the check according
diff --git a/source/blender/blenlib/intern/filereader_zstd.c b/source/blender/blenlib/intern/filereader_zstd.c
new file mode 100644
index 00000000000..785a40cd1a1
--- /dev/null
+++ b/source/blender/blenlib/intern/filereader_zstd.c
@@ -0,0 +1,335 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2021 Blender Foundation
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ */
+
+#include <string.h>
+#include <zstd.h>
+
+#include "BLI_blenlib.h"
+#include "BLI_endian_switch.h"
+#include "BLI_filereader.h"
+#include "BLI_math_base.h"
+
+#include "MEM_guardedalloc.h"
+
+typedef struct {
+ FileReader reader;
+
+ FileReader *base;
+
+ ZSTD_DCtx *ctx;
+ ZSTD_inBuffer in_buf;
+ size_t in_buf_max_size;
+
+ struct {
+ int num_frames;
+ size_t *compressed_ofs;
+ size_t *uncompressed_ofs;
+
+ char *cached_content;
+ int cached_frame;
+ } seek;
+} ZstdReader;
+
+static bool zstd_read_u32(FileReader *base, uint32_t *val)
+{
+ if (base->read(base, val, sizeof(uint32_t)) != sizeof(uint32_t)) {
+ return false;
+ }
+#ifdef __BIG_ENDIAN__
+ BLI_endian_switch_uint32(val);
+#endif
+ return true;
+}
+
+static bool zstd_read_seek_table(ZstdReader *zstd)
+{
+ FileReader *base = zstd->base;
+
+ /* The seek table frame is at the end of the file, so seek there
+ * and verify that there is enough data. */
+ if (base->seek(base, -4, SEEK_END) < 13) {
+ return false;
+ }
+ uint32_t magic;
+ if (!zstd_read_u32(base, &magic) || magic != 0x8F92EAB1) {
+ return false;
+ }
+
+ uint8_t flags;
+ if (base->seek(base, -5, SEEK_END) < 0 || base->read(base, &flags, 1) != 1) {
+ return false;
+ }
+ /* Bit 7 indicates checksums. Bits 5 and 6 must be zero. */
+ bool has_checksums = (flags & 0x80);
+ if (flags & 0x60) {
+ return false;
+ }
+
+ uint32_t num_frames;
+ if (base->seek(base, -9, SEEK_END) < 0 || !zstd_read_u32(base, &num_frames)) {
+ return false;
+ }
+
+ /* Each frame has either 2 or 3 uint32_t, and after that we have
+ * num_frames, flags and magic for another 9 bytes. */
+ uint32_t expected_frame_length = num_frames * (has_checksums ? 12 : 8) + 9;
+ /* The frame starts with another magic number and its length, but these
+ * two fields are not in
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list