[Bf-blender-cvs] [329cda95193] experimental-build: Revert "D5799 Part 3: Sequencer cache"
Lukas Stockner
noreply at git.blender.org
Sat Aug 21 06:03:42 CEST 2021
Commit: 329cda9519328e1b67fb4045681d6ab27be1c5a1
Author: Lukas Stockner
Date: Sat Aug 21 06:03:30 2021 +0200
Branches: experimental-build
https://developer.blender.org/rB329cda9519328e1b67fb4045681d6ab27be1c5a1
Revert "D5799 Part 3: Sequencer cache"
This reverts commit 0a6186dcdf3f3e42104ffb39f4ed8bc052eb9fe5.
===================================================================
M source/blender/blenlib/BLI_fileops.h
M source/blender/blenlib/intern/fileops.c
M source/blender/sequencer/intern/image_cache.c
===================================================================
diff --git a/source/blender/blenlib/BLI_fileops.h b/source/blender/blenlib/BLI_fileops.h
index 906a56ce909..377b7bc3bc2 100644
--- a/source/blender/blenlib/BLI_fileops.h
+++ b/source/blender/blenlib/BLI_fileops.h
@@ -154,15 +154,20 @@ bool BLI_file_is_writable(const char *file) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL
bool BLI_file_touch(const char *file) ATTR_NONNULL();
bool BLI_file_alias_target(const char *filepath, char *r_targetpath) ATTR_WARN_UNUSED_RESULT;
+#if 0 /* UNUSED */
+int BLI_file_gzip(const char *from, const char *to) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
+#endif
+char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size) ATTR_WARN_UNUSED_RESULT
+ ATTR_NONNULL();
+size_t BLI_gzip_mem_to_file_at_pos(void *buf,
+ size_t len,
+ FILE *file,
+ size_t gz_stream_offset,
+ int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
+size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset)
+ ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
bool BLI_file_magic_is_gzip(const char header[4]);
-size_t BLI_file_zstd_from_mem_at_pos(void *buf,
- size_t len,
- FILE *file,
- size_t file_offset,
- int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
-size_t BLI_file_unzstd_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t file_offset)
- ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
bool BLI_file_magic_is_zstd(const char header[4]);
size_t BLI_file_descriptor_size(int file) ATTR_WARN_UNUSED_RESULT;
diff --git a/source/blender/blenlib/intern/fileops.c b/source/blender/blenlib/intern/fileops.c
index 532a29b8147..31825c69737 100644
--- a/source/blender/blenlib/intern/fileops.c
+++ b/source/blender/blenlib/intern/fileops.c
@@ -31,7 +31,6 @@
#include <errno.h>
#include "zlib.h"
-#include "zstd.h"
#ifdef WIN32
# include "BLI_fileops_types.h"
@@ -62,90 +61,200 @@
#include "BLI_sys_types.h" /* for intptr_t support */
#include "BLI_utildefines.h"
-size_t BLI_file_zstd_from_mem_at_pos(
- void *buf, size_t len, FILE *file, size_t file_offset, int compression_level)
+#if 0 /* UNUSED */
+/* gzip the file in from and write it to "to".
+ * return -1 if zlib fails, -2 if the originating file does not exist
+ * NOTE: will remove the "from" file
+ */
+int BLI_file_gzip(const char *from, const char *to)
{
- fseek(file, file_offset, SEEK_SET);
-
- ZSTD_CCtx *ctx = ZSTD_createCCtx();
- ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, compression_level);
+ char buffer[10240];
+ int file;
+ int readsize = 0;
+ int rval = 0, err;
+ gzFile gzfile;
- ZSTD_inBuffer input = {buf, len, 0};
+ /* level 1 is very close to 3 (the default) in terms of file size,
+ * but about twice as fast, best use for speedy saving - campbell */
+ gzfile = BLI_gzopen(to, "wb1");
+ if (gzfile == NULL) {
+ return -1;
+ }
+ file = BLI_open(from, O_BINARY | O_RDONLY, 0);
+ if (file == -1) {
+ return -2;
+ }
- size_t out_len = ZSTD_CStreamOutSize();
- void *out_buf = MEM_mallocN(out_len, __func__);
- size_t total_written = 0;
+ while (1) {
+ readsize = read(file, buffer, sizeof(buffer));
- /* Compress block and write it out until the input has been consumed. */
- while (input.pos < input.size) {
- ZSTD_outBuffer output = {out_buf, out_len, 0};
- size_t ret = ZSTD_compressStream2(ctx, &output, &input, ZSTD_e_continue);
- if (ZSTD_isError(ret)) {
+ if (readsize < 0) {
+ rval = -2; /* error happened in reading */
+ fprintf(stderr, "Error reading file %s: %s.\n", from, strerror(errno));
break;
}
- if (fwrite(out_buf, 1, output.pos, file) != output.pos) {
+ else if (readsize == 0) {
+ break; /* done reading */
+ }
+
+ if (gzwrite(gzfile, buffer, readsize) <= 0) {
+ rval = -1; /* error happened in writing */
+ fprintf(stderr, "Error writing gz file %s: %s.\n", to, gzerror(gzfile, &err));
break;
}
- total_written += output.pos;
}
- /* Finalize the Zstd frame. */
- size_t ret = 1;
- while (ret != 0) {
- ZSTD_outBuffer output = {out_buf, out_len, 0};
- ret = ZSTD_compressStream2(ctx, &output, &input, ZSTD_e_end);
- if (ZSTD_isError(ret)) {
- break;
+ gzclose(gzfile);
+ close(file);
+
+ return rval;
+}
+#endif
+
+/* gzip the file in from_file and write it to memory to_mem, at most size bytes.
+ * return the unzipped size
+ */
+char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size)
+{
+ gzFile gzfile;
+ int readsize, size, alloc_size = 0;
+ char *mem = NULL;
+ const int chunk_size = 512 * 1024;
+
+ size = 0;
+
+ gzfile = BLI_gzopen(from_file, "rb");
+ for (;;) {
+ if (mem == NULL) {
+ mem = MEM_callocN(chunk_size, "BLI_ungzip_to_mem");
+ alloc_size = chunk_size;
}
- if (fwrite(out_buf, 1, output.pos, file) != output.pos) {
+ else {
+ mem = MEM_reallocN(mem, size + chunk_size);
+ alloc_size += chunk_size;
+ }
+
+ readsize = gzread(gzfile, mem + size, chunk_size);
+ if (readsize > 0) {
+ size += readsize;
+ }
+ else {
break;
}
- total_written += output.pos;
}
- MEM_freeN(out_buf);
- ZSTD_freeCCtx(ctx);
+ gzclose(gzfile);
+
+ if (size == 0) {
+ MEM_freeN(mem);
+ mem = NULL;
+ }
+ else if (alloc_size != size) {
+ mem = MEM_reallocN(mem, size);
+ }
+
+ *r_size = size;
- return ZSTD_isError(ret) ? 0 : total_written;
+ return mem;
}
-size_t BLI_file_unzstd_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t file_offset)
+#define CHUNK (256 * 1024)
+
+/* gzip byte array from memory and write it to file at certain position.
+ * return size of gzip stream.
+ */
+size_t BLI_gzip_mem_to_file_at_pos(
+ void *buf, size_t len, FILE *file, size_t gz_stream_offset, int compression_level)
{
- fseek(file, file_offset, SEEK_SET);
+ int ret, flush;
+ unsigned have;
+ z_stream strm;
+ unsigned char out[CHUNK];
+
+ BLI_fseek(file, gz_stream_offset, 0);
+
+ strm.zalloc = Z_NULL;
+ strm.zfree = Z_NULL;
+ strm.opaque = Z_NULL;
+ ret = deflateInit(&strm, compression_level);
+ if (ret != Z_OK) {
+ return 0;
+ }
- ZSTD_DCtx *ctx = ZSTD_createDCtx();
+ strm.avail_in = len;
+ strm.next_in = (Bytef *)buf;
+ flush = Z_FINISH;
- size_t in_len = ZSTD_DStreamInSize();
- void *in_buf = MEM_mallocN(in_len, __func__);
- ZSTD_inBuffer input = {in_buf, in_len, 0};
+ do {
+ strm.avail_out = CHUNK;
+ strm.next_out = out;
+ ret = deflate(&strm, flush);
+ if (ret == Z_STREAM_ERROR) {
+ return 0;
+ }
+ have = CHUNK - strm.avail_out;
+ if (fwrite(out, 1, have, file) != have || ferror(file)) {
+ deflateEnd(&strm);
+ return 0;
+ }
+ } while (strm.avail_out == 0);
- ZSTD_outBuffer output = {buf, len, 0};
+ if (strm.avail_in != 0 || ret != Z_STREAM_END) {
+ return 0;
+ }
- size_t ret = 0;
- /* Read and decompress chunks of input data until we have enough output. */
- while (output.pos < output.size && !ZSTD_isError(ret)) {
- input.size = fread(in_buf, 1, in_len, file);
- if (input.size == 0) {
- break;
+ deflateEnd(&strm);
+ return (size_t)strm.total_out;
+}
+
+/* read and decompress gzip stream from file at certain position to buffer.
+ * return size of decompressed data.
+ */
+size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset)
+{
+ int ret;
+ z_stream strm;
+ size_t chunk = 256 * 1024;
+ unsigned char in[CHUNK];
+
+ BLI_fseek(file, gz_stream_offset, 0);
+
+ strm.zalloc = Z_NULL;
+ strm.zfree = Z_NULL;
+ strm.opaque = Z_NULL;
+ strm.avail_in = 0;
+ strm.next_in = Z_NULL;
+ ret = inflateInit(&strm);
+ if (ret != Z_OK) {
+ return 0;
+ }
+
+ do {
+ strm.avail_in = fread(in, 1, chunk, file);
+ strm.next_in = in;
+ if (ferror(file)) {
+ inflateEnd(&strm);
+ return 0;
}
- /* Consume input data until we run out or have enough output. */
- input.pos = 0;
- while (input.pos < input.size && output.pos < output.size) {
- ret = ZSTD_decompressStream(ctx, &output, &input);
+ do {
+ strm.avail_out = len;
+ strm.next_out = (Bytef *)buf + strm.total_out;
- if (ZSTD_isError(ret)) {
- break;
+ ret = inflate(&strm, Z_NO_FLUSH);
+ if (ret == Z_STREAM_ERROR) {
+ return 0;
}
- }
- }
+ } while (strm.avail_out == 0);
- MEM_freeN(in_buf);
- ZSTD_freeDCtx(ctx);
+ } while (ret != Z_STREAM_END);
- return ZSTD_isError(ret) ? 0 : output.pos;
+ inflateEnd(&strm);
+ return (size_t)strm.total_out;
}
+#undef CHUNK
+
bool BLI_file_magic_is_gzip(const char header[4])
{
/* GZIP itself starts with the magic bytes 0x1f 0x8b.
diff --git a/source/blender/sequencer/intern/image_cache.c b/source/blender/sequencer/intern/image_cache.c
index 86bd840ce31..604c9900355 100644
--- a/source/blender/sequencer/intern/image_cache.c
+++ b/source/blender/sequencer/intern/image_cache.c
@@ -102,7 +102,7 @@
/* <cache type>-<resolution X>x<resolution Y>-<rendersize>%(<view_id>)-<frame no>.dcf */
#define DCACHE_FNAME_FORMAT "%d-%dx%d-%d%%(%d)-%d.dcf"
#define DCACHE_IMAGES_PER_FILE 100
-#define DCACHE_CURRENT_VERSION 2
+#define DCACHE_CURRENT_VERSION 1
#define COLORSPACE_NAME_MAX 64 /* XXX: defined in imb intern */
typedef struct DiskCacheHeaderEntry {
@@ -496,34 +496,24 @@ static size_t deflate_imbuf_to_file(ImBuf *ibuf,
int level,
DiskCacheHeaderEntry *header_entry)
{
- void *data = (ibuf->rect != NULL) ? (void *)ibuf->rect : (void *)ibuf->rect_float;
-
- /* Apply compression if wanted, otherwise just write directly to the file. */
- if (level > 0) {
- return BLI_file_zstd_from_mem_at_pos(
- data, header_entry->size_raw, file, header_entry->offset, level);
+ if (ibuf->rect) {
+ return BLI_gzip_mem_to_file_at_pos(
+ ibuf->rect, header_entry->size_raw, file, header_entry->offset, level);
}
- fseek(file, header_entry->offse
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list