[Bf-blender-cvs] [edfcaf0abd2] experimental-build: D5799 Part 1: FileReader refactor
Lukas Stockner
noreply at git.blender.org
Sat Aug 21 05:08:10 CEST 2021
Commit: edfcaf0abd23bbc2898ff5e82ca9bbc20312b5f8
Author: Lukas Stockner
Date: Thu Aug 19 23:57:00 2021 +0200
Branches: experimental-build
https://developer.blender.org/rBedfcaf0abd23bbc2898ff5e82ca9bbc20312b5f8
D5799 Part 1: FileReader refactor
===================================================================
M source/blender/blenlib/BLI_fileops.h
A source/blender/blenlib/BLI_filereader.h
M source/blender/blenlib/CMakeLists.txt
M source/blender/blenlib/intern/fileops.c
A source/blender/blenlib/intern/filereader_file.c
A source/blender/blenlib/intern/filereader_gzip.c
A source/blender/blenlib/intern/filereader_memory.c
M source/blender/blenloader/BLO_undofile.h
M source/blender/blenloader/intern/readfile.c
M source/blender/blenloader/intern/readfile.h
M source/blender/blenloader/intern/undofile.c
M source/blender/blenloader/intern/versioning_250.c
M source/blender/blenloader/intern/versioning_legacy.c
M source/blender/blenloader/intern/writefile.c
M source/blender/windowmanager/intern/wm_files.c
===================================================================
diff --git a/source/blender/blenlib/BLI_fileops.h b/source/blender/blenlib/BLI_fileops.h
index 7cfecc798a7..12fa73279c8 100644
--- a/source/blender/blenlib/BLI_fileops.h
+++ b/source/blender/blenlib/BLI_fileops.h
@@ -166,6 +166,8 @@ size_t BLI_gzip_mem_to_file_at_pos(void *buf,
int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset)
ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
+bool BLI_file_magic_is_gzip(const char header[4]);
+
size_t BLI_file_descriptor_size(int file) ATTR_WARN_UNUSED_RESULT;
size_t BLI_file_size(const char *path) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
diff --git a/source/blender/blenlib/BLI_filereader.h b/source/blender/blenlib/BLI_filereader.h
new file mode 100644
index 00000000000..5391829a548
--- /dev/null
+++ b/source/blender/blenlib/BLI_filereader.h
@@ -0,0 +1,79 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ * \brief Wrapper for reading from various sources (e.g. raw files, compressed files, memory...).
+ */
+
+#pragma once
+
+#ifdef WIN32
+# include "BLI_winstuff.h"
+#endif
+
+#include "BLI_compiler_attrs.h"
+#include "BLI_utildefines.h"
+
+#if defined(_MSC_VER) || defined(__APPLE__) || defined(__HAIKU__) || defined(__NetBSD__)
+typedef int64_t off64_t;
+#else
+# include <sys/types.h>
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct FileReader;
+
+typedef ssize_t (*FileReaderReadFn)(struct FileReader *reader, void *buffer, size_t size);
+typedef off64_t (*FileReaderSeekFn)(struct FileReader *reader, off64_t offset, int whence);
+typedef void (*FileReaderCloseFn)(struct FileReader *reader);
+
+/* General structure for all FileReaders, implementations add custom fields at the end. */
+typedef struct FileReader {
+ FileReaderReadFn read;
+ FileReaderSeekFn seek;
+ FileReaderCloseFn close;
+
+ off64_t offset;
+} FileReader;
+
+/* Functions for opening the various types of FileReader.
+ * They either succeed and return a valid FileReader, or fail and return NULL.
+ *
+ * If a FileReader is created, it has to be cleaned up and freed by calling
+ * its close() function unless another FileReader has taken ownership - for example,
+ * Gzip takes over the base FileReader and will clean it up when their clean() is called.
+ */
+
+/* Create FileReader from raw file descriptor. */
+FileReader *BLI_filereader_new_file(int filedes) ATTR_WARN_UNUSED_RESULT;
+/* Create FileReader from raw file descriptor using memory-mapped IO. */
+FileReader *BLI_filereader_new_mmap(int filedes) ATTR_WARN_UNUSED_RESULT;
+/* Create FileReader from a region of memory. */
+FileReader *BLI_filereader_new_memory(const void *data, size_t len) ATTR_WARN_UNUSED_RESULT
+ ATTR_NONNULL();
+/* Create FileReader from applying Gzip decompression on an underlying file. */
+FileReader *BLI_filereader_new_gzip(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/source/blender/blenlib/CMakeLists.txt b/source/blender/blenlib/CMakeLists.txt
index b6603dce378..d2ba9e74c90 100644
--- a/source/blender/blenlib/CMakeLists.txt
+++ b/source/blender/blenlib/CMakeLists.txt
@@ -75,6 +75,9 @@ set(SRC
intern/endian_switch.c
intern/expr_pylike_eval.c
intern/fileops.c
+ intern/filereader_file.c
+ intern/filereader_gzip.c
+ intern/filereader_memory.c
intern/fnmatch.c
intern/freetypefont.c
intern/gsqueue.c
@@ -194,6 +197,7 @@ set(SRC
BLI_enumerable_thread_specific.hh
BLI_expr_pylike_eval.h
BLI_fileops.h
+ BLI_filereader.h
BLI_fileops_types.h
BLI_float2.hh
BLI_float3.hh
diff --git a/source/blender/blenlib/intern/fileops.c b/source/blender/blenlib/intern/fileops.c
index ac034d2b5cd..6fc2222241b 100644
--- a/source/blender/blenlib/intern/fileops.c
+++ b/source/blender/blenlib/intern/fileops.c
@@ -255,6 +255,13 @@ size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t g
#undef CHUNK
+bool BLI_file_magic_is_gzip(const char header[4])
+{
+ /* GZIP itself starts with the magic bytes 0x1f 0x8b.
+ * The third byte indicates the compression method, which is 0x08 for DEFLATE. */
+ return header[0] == 0x1f && header[1] == 0x8b && header[2] == 0x08;
+}
+
/**
* Returns true if the file with the specified name can be written.
* This implementation uses access(2), which makes the check according
diff --git a/source/blender/blenlib/intern/filereader_file.c b/source/blender/blenlib/intern/filereader_file.c
new file mode 100644
index 00000000000..3a833871e27
--- /dev/null
+++ b/source/blender/blenlib/intern/filereader_file.c
@@ -0,0 +1,80 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2004-2021 Blender Foundation
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ */
+
+#ifndef WIN32
+# include <unistd.h> /* for read close */
+#else
+# include "BLI_winstuff.h"
+# include "winsock2.h"
+# include <io.h> /* for open close read */
+#endif
+
+#include "BLI_blenlib.h"
+#include "BLI_filereader.h"
+
+#include "MEM_guardedalloc.h"
+
+typedef struct {
+ FileReader reader;
+
+ int filedes;
+} RawFileReader;
+
+static ssize_t file_read(FileReader *reader, void *buffer, size_t size)
+{
+ RawFileReader *rawfile = (RawFileReader *)reader;
+ ssize_t readsize = read(rawfile->filedes, buffer, size);
+
+ if (readsize >= 0) {
+ rawfile->reader.offset += readsize;
+ }
+
+ return readsize;
+}
+
+static off64_t file_seek(FileReader *reader, off64_t offset, int whence)
+{
+ RawFileReader *rawfile = (RawFileReader *)reader;
+ rawfile->reader.offset = BLI_lseek(rawfile->filedes, offset, whence);
+ return rawfile->reader.offset;
+}
+
+static void file_close(FileReader *reader)
+{
+ RawFileReader *rawfile = (RawFileReader *)reader;
+ close(rawfile->filedes);
+ MEM_freeN(rawfile);
+}
+
+FileReader *BLI_filereader_new_file(int filedes)
+{
+ RawFileReader *rawfile = MEM_callocN(sizeof(RawFileReader), __func__);
+
+ rawfile->filedes = filedes;
+
+ rawfile->reader.read = file_read;
+ rawfile->reader.seek = file_seek;
+ rawfile->reader.close = file_close;
+
+ return (FileReader *)rawfile;
+}
diff --git a/source/blender/blenlib/intern/filereader_gzip.c b/source/blender/blenlib/intern/filereader_gzip.c
new file mode 100644
index 00000000000..72eb153a8b9
--- /dev/null
+++ b/source/blender/blenlib/intern/filereader_gzip.c
@@ -0,0 +1,108 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2004-2021 Blender Foundation
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ */
+
+#include <zlib.h>
+
+#include "BLI_blenlib.h"
+#include "BLI_filereader.h"
+
+#include "MEM_guardedalloc.h"
+
+typedef struct {
+ FileReader reader;
+
+ FileReader *base;
+
+ z_stream strm;
+
+ void *in_buf;
+ size_t in_size;
+} GzipReader;
+
+static ssize_t gzip_read(FileReader *reader, void *buffer, size_t size)
+{
+ GzipReader *gzip = (GzipReader *)reader;
+
+ gzip->strm.avail_out = size;
+ gzip->strm.next_out = buffer;
+
+ while (gzip->strm.avail_out > 0) {
+ if (gzip->strm.avail_in == 0) {
+ /* Ran out of buffered input data, read some more. */
+ size_t readsize = gzip->base->read(gzip->base, gzip->in_buf, gzip->in_size);
+
+ if (readsize > 0) {
+ /* We got some data, so mark the buffer as refilled. */
+ gzip->strm.avail_in = readsize;
+ gzip->strm.next_in = gzip->in_buf;
+ }
+ else {
+ /* The underlying file is EOF, so return as much as we can. */
+ break;
+ }
+ }
+
+ int ret = inflate(&gzip->strm, Z_NO_FLUSH);
+
+ if (ret != Z_OK && ret != Z_BUF_ERROR) {
+ break;
+ }
+ }
+
+ ssize_t read_len = size - gzip->strm.avail_out;
+ gzip->reader.offset += read_len;
+ return read_len;
+}
+
+static void gzip_close(FileReader *reader)
+{
+ GzipReader *gzip = (GzipReader *)reader;
+
+ if (inflateEnd(&gzip->strm) != Z_OK) {
+ printf("close gzip stream error\n");
+ }
+ MEM_freeN((void *)gzip->in
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list