[Bf-blender-cvs] [0f2ae614a17] master: Use mmap() IO for reading uncompressed .blends
Lukas Stockner
noreply at git.blender.org
Thu Jan 14 22:20:42 CET 2021
Commit: 0f2ae614a17a7c231b2a6e129633a99d9d1f12e3
Author: Lukas Stockner
Date: Thu Jan 14 22:02:48 2021 +0100
Branches: master
https://developer.blender.org/rB0f2ae614a17a7c231b2a6e129633a99d9d1f12e3
Use mmap() IO for reading uncompressed .blends
Instead of submitting tons of tiny IO syscalls, we can speed things up
significantly by `mmap`ing the .blend file into virtual memory and directly
accessing it.
In my local testing, this speeds up loading the Dweebs file with all its
linked files from 19sec to 10sec (on Linux).
As far as I can see, this should be supported on Linux, OSX and BSD.
For Windows, a second code path uses `CreateFileMapping` and
`MapViewOfFile` to achieve the same result.
Reviewed By: mont29, brecht
Differential Revision: https://developer.blender.org/D8246
===================================================================
M intern/guardedalloc/CMakeLists.txt
D intern/guardedalloc/intern/mmap_win.c
D intern/guardedalloc/mmap_win.h
A source/blender/blenlib/BLI_mmap.h
M source/blender/blenlib/CMakeLists.txt
A source/blender/blenlib/intern/BLI_mmap.c
M source/blender/blenloader/intern/readfile.c
M source/blender/blenloader/intern/readfile.h
M source/blender/imbuf/intern/IMB_allocimbuf.h
M source/blender/imbuf/intern/allocimbuf.c
M source/blender/imbuf/intern/readimage.c
M source/blender/makesdna/intern/CMakeLists.txt
M source/blender/makesrna/intern/CMakeLists.txt
===================================================================
diff --git a/intern/guardedalloc/CMakeLists.txt b/intern/guardedalloc/CMakeLists.txt
index 0d46e81cd87..b47565b8ef9 100644
--- a/intern/guardedalloc/CMakeLists.txt
+++ b/intern/guardedalloc/CMakeLists.txt
@@ -49,12 +49,6 @@ set(LIB
)
if(WIN32 AND NOT UNIX)
- list(APPEND SRC
- intern/mmap_win.c
-
- mmap_win.h
- )
-
list(APPEND INC_SYS
${PTHREADS_INC}
)
diff --git a/intern/guardedalloc/intern/mmap_win.c b/intern/guardedalloc/intern/mmap_win.c
deleted file mode 100644
index a02a0f88fa9..00000000000
--- a/intern/guardedalloc/intern/mmap_win.c
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- *
- * The Original Code is Copyright (C) 2008 Blender Foundation.
- * All rights reserved.
- */
-
-/** \file
- * \ingroup MEM
- */
-
-#ifdef WIN32
-
-# include <errno.h>
-# include <io.h>
-# include <stdio.h>
-# include <sys/types.h>
-# include <windows.h>
-
-# include "mmap_win.h"
-
-# ifndef FILE_MAP_EXECUTE
-// not defined in earlier versions of the Platform SDK (before February 2003)
-# define FILE_MAP_EXECUTE 0x0020
-# endif
-
-/* copied from BLI_utildefines.h, ugh */
-# ifdef __GNUC__
-# define UNUSED(x) UNUSED_##x __attribute__((__unused__))
-# else
-# define UNUSED(x) x
-# endif
-
-/* --------------------------------------------------------------------- */
-/* local storage definitions */
-/* --------------------------------------------------------------------- */
-/* all memory mapped chunks are put in linked lists */
-typedef struct mmapLink {
- struct mmapLink *next, *prev;
-} mmapLink;
-
-typedef struct mmapListBase {
- void *first, *last;
-} mmapListBase;
-
-typedef struct MemMap {
- struct MemMap *next, *prev;
- void *mmap;
- HANDLE fhandle;
- HANDLE maphandle;
-} MemMap;
-
-/* --------------------------------------------------------------------- */
-/* local functions */
-/* --------------------------------------------------------------------- */
-
-static void mmap_addtail(volatile mmapListBase *listbase, void *vlink);
-static void mmap_remlink(volatile mmapListBase *listbase, void *vlink);
-static void *mmap_findlink(volatile mmapListBase *listbase, void *ptr);
-
-static int mmap_get_prot_flags(int flags);
-static int mmap_get_access_flags(int flags);
-
-/* --------------------------------------------------------------------- */
-/* vars */
-/* --------------------------------------------------------------------- */
-volatile static struct mmapListBase _mmapbase;
-volatile static struct mmapListBase *mmapbase = &_mmapbase;
-
-/* --------------------------------------------------------------------- */
-/* implementation */
-/* --------------------------------------------------------------------- */
-
-/* mmap for windows */
-void *mmap(void *UNUSED(start), size_t len, int prot, int flags, int fd, off_t offset)
-{
- HANDLE fhandle = INVALID_HANDLE_VALUE;
- HANDLE maphandle;
- int prot_flags = mmap_get_prot_flags(prot);
- int access_flags = mmap_get_access_flags(prot);
- MemMap *mm = NULL;
- void *ptr = NULL;
-
- if (flags & MAP_FIXED) {
- return MAP_FAILED;
- }
-
-# if 0
- if (fd == -1) {
- _set_errno(EBADF);
- return MAP_FAILED;
- }
-# endif
-
- if (fd != -1) {
- fhandle = (HANDLE)_get_osfhandle(fd);
- }
- if (fhandle == INVALID_HANDLE_VALUE) {
- if (!(flags & MAP_ANONYMOUS)) {
- errno = EBADF;
- return MAP_FAILED;
- }
- }
- else {
- if (!DuplicateHandle(GetCurrentProcess(),
- fhandle,
- GetCurrentProcess(),
- &fhandle,
- 0,
- FALSE,
- DUPLICATE_SAME_ACCESS)) {
- return MAP_FAILED;
- }
- }
-
- /* Split 64 bit size into low and high bits. */
- DWORD len_bits_high = len >> 32;
- DWORD len_bits_low = len & 0xFFFFFFFF;
-
- maphandle = CreateFileMapping(fhandle, NULL, prot_flags, len_bits_high, len_bits_low, NULL);
- if (maphandle == 0) {
- errno = EBADF;
- return MAP_FAILED;
- }
-
- ptr = MapViewOfFile(maphandle, access_flags, 0, offset, 0);
- if (ptr == NULL) {
- DWORD dwLastErr = GetLastError();
- if (dwLastErr == ERROR_MAPPED_ALIGNMENT) {
- errno = EINVAL;
- }
- else {
- errno = EACCES;
- }
- CloseHandle(maphandle);
- return MAP_FAILED;
- }
-
- mm = (MemMap *)malloc(sizeof(MemMap));
- if (!mm) {
- errno = ENOMEM;
- }
- mm->fhandle = fhandle;
- mm->maphandle = maphandle;
- mm->mmap = ptr;
- mmap_addtail(mmapbase, mm);
-
- return ptr;
-}
-
-/* munmap for windows */
-intptr_t munmap(void *ptr, size_t UNUSED(size))
-{
- MemMap *mm = mmap_findlink(mmapbase, ptr);
- if (!mm) {
- errno = EINVAL;
- return -1;
- }
- UnmapViewOfFile(mm->mmap);
- CloseHandle(mm->maphandle);
- CloseHandle(mm->fhandle);
- mmap_remlink(mmapbase, mm);
- free(mm);
- return 0;
-}
-
-/* --------------------------------------------------------------------- */
-/* local functions */
-/* --------------------------------------------------------------------- */
-
-static void mmap_addtail(volatile mmapListBase *listbase, void *vlink)
-{
- struct mmapLink *link = vlink;
-
- if (link == NULL) {
- return;
- }
- if (listbase == NULL) {
- return;
- }
-
- link->next = 0;
- link->prev = listbase->last;
-
- if (listbase->last) {
- ((struct mmapLink *)listbase->last)->next = link;
- }
- if (listbase->first == NULL) {
- listbase->first = link;
- }
- listbase->last = link;
-}
-
-static void mmap_remlink(volatile mmapListBase *listbase, void *vlink)
-{
- struct mmapLink *link = vlink;
-
- if (link == NULL) {
- return;
- }
- if (listbase == NULL) {
- return;
- }
- if (link->next) {
- link->next->prev = link->prev;
- }
- if (link->prev) {
- link->prev->next = link->next;
- }
-
- if (listbase->last == link) {
- listbase->last = link->prev;
- }
- if (listbase->first == link) {
- listbase->first = link->next;
- }
-}
-
-static void *mmap_findlink(volatile mmapListBase *listbase, void *ptr)
-{
- MemMap *mm;
-
- if (ptr == NULL) {
- return NULL;
- }
- if (listbase == NULL) {
- return NULL;
- }
-
- mm = (MemMap *)listbase->first;
- while (mm) {
- if (mm->mmap == ptr) {
- return mm;
- }
- mm = mm->next;
- }
- return NULL;
-}
-
-static int mmap_get_prot_flags(int flags)
-{
- int prot = PAGE_NOACCESS;
-
- if ((flags & PROT_READ) == PROT_READ) {
- if ((flags & PROT_WRITE) == PROT_WRITE) {
- prot = (flags & PROT_EXEC) ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE;
- }
- else {
- prot = (flags & PROT_EXEC) ? PAGE_EXECUTE_READ : PAGE_READONLY;
- }
- }
- else if ((flags & PROT_WRITE) == PROT_WRITE) {
- prot = (flags & PROT_EXEC) ? PAGE_EXECUTE_READ : PAGE_WRITECOPY;
- }
- else if ((flags & PROT_EXEC) == PROT_EXEC) {
- prot = PAGE_EXECUTE_READ;
- }
- return prot;
-}
-
-static int mmap_get_access_flags(int flags)
-{
- int access = 0;
-
- if ((flags & PROT_READ) == PROT_READ) {
- if ((flags & PROT_WRITE) == PROT_WRITE) {
- access = FILE_MAP_WRITE;
- }
- else {
- access = (flags & PROT_EXEC) ? FILE_MAP_EXECUTE : FILE_MAP_READ;
- }
- }
- else if ((flags & PROT_WRITE) == PROT_WRITE) {
- access = FILE_MAP_COPY;
- }
- else if ((flags & PROT_EXEC) == PROT_EXEC) {
- access = FILE_MAP_EXECUTE;
- }
- return access;
-}
-
-#endif // WIN32
diff --git a/intern/guardedalloc/mmap_win.h b/intern/guardedalloc/mmap_win.h
deleted file mode 100644
index c0cbaa0e512..00000000000
--- a/intern/guardedalloc/mmap_win.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- *
- * The Original Code is Copyright (C) 2008 Blender Foundation.
- * All rights reserved.
- */
-
-/** \file
- * \ingroup MEM
- */
-
-#ifndef __MMAP_WIN_H__
-#define __MMAP_WIN_H__
-
-#define PROT_NONE 0
-#define PROT_READ 1
-#define PROT_WRITE 2
-#define PROT_EXEC 4
-
-#define MAP_FILE 0
-#define MAP_SHARED 1
-#define MAP_PRIVATE 2
-#define MAP_TYPE 0xF
-#define MAP_FIXED 0x10
-#define MAP_ANONYMOUS 0x20
-#define MAP_ANON MAP_ANONYMOUS
-
-#define MAP_FAILED ((void *)-1)
-
-/* needed for uintptr_t, exception, dont use BLI anywhere else in MEM_* */
-#include "../../source/blender/blenlib/BLI_sys_types.h"
-
-#include <sys/types.h>
-
-void *mmap(void *start, size_t len, int prot, int flags, int fd, off_t offset);
-intptr_t munmap(void *ptr, size_t size);
-
-#endif
diff --git a/source/blender/blenlib/BLI_mmap.h b/source/blender/blenlib/BLI_mmap.h
new file mode 100644
index 00000000000..385c56bd10a
--- /dev/null
+++ b/source/blender/blenlib
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list