[Bf-blender-cvs] [dbeab82a89c] master: T91406: Asset Library Indexing
Jeroen Bakker
noreply at git.blender.org
Wed Nov 24 08:22:20 CET 2021
Commit: dbeab82a89c4979b7e9e62e740938546394eca3e
Author: Jeroen Bakker
Date: Wed Nov 24 08:20:18 2021 +0100
Branches: master
https://developer.blender.org/rBdbeab82a89c4979b7e9e62e740938546394eca3e
T91406: Asset Library Indexing
Asset library indexing would store indexes of asset files to speed up
asset library browsing.
* Indexes are read when they are up to date
** Index should exist
** Index last modify data should be later than the file it indexes
** Index version should match
* The index of a file containing no assets can be load without opening
the index file. The size of the file should be below a 32 bytes.
* Indexes are stored on a persistent cache folder.
* Unused index files are automatically removed.
The structure of the index files contains all data needed for browsing assets:
```
{
"version": <file version number>,
"entries": [{
"name": "<asset name>",
"catalog_id": "<catalog_id>",
"catalog_name": "<catalog_name>",
"description": "<description>",
"author": "<author>",
"tags": ["<tag>"]
}]
}
```
Reviewed By: sybren, Severin
Maniphest Tasks: T91406
Differential Revision: https://developer.blender.org/D12693
===================================================================
M source/blender/editors/asset/CMakeLists.txt
A source/blender/editors/asset/ED_asset_indexer.h
A source/blender/editors/asset/intern/asset_indexer.cc
M source/blender/editors/asset/intern/asset_list.cc
A source/blender/editors/include/ED_file_indexer.h
M source/blender/editors/space_file/CMakeLists.txt
A source/blender/editors/space_file/file_indexer.cc
A source/blender/editors/space_file/file_indexer.h
M source/blender/editors/space_file/filelist.c
M source/blender/editors/space_file/filelist.h
M source/blender/editors/space_file/space_file.c
===================================================================
diff --git a/source/blender/editors/asset/CMakeLists.txt b/source/blender/editors/asset/CMakeLists.txt
index b9ef8e82bba..2391f4af14d 100644
--- a/source/blender/editors/asset/CMakeLists.txt
+++ b/source/blender/editors/asset/CMakeLists.txt
@@ -24,6 +24,7 @@ set(INC
../../makesdna
../../makesrna
../../windowmanager
+ ../../../../intern/clog
../../../../intern/guardedalloc
)
@@ -34,6 +35,7 @@ set(SRC
intern/asset_catalog.cc
intern/asset_filter.cc
intern/asset_handle.cc
+ intern/asset_indexer.cc
intern/asset_library_reference.cc
intern/asset_library_reference_enum.cc
intern/asset_list.cc
diff --git a/source/blender/editors/asset/ED_asset_indexer.h b/source/blender/editors/asset/ED_asset_indexer.h
new file mode 100644
index 00000000000..33558d8cda5
--- /dev/null
+++ b/source/blender/editors/asset/ED_asset_indexer.h
@@ -0,0 +1,50 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/** \file
+ * \ingroup edasset
+ */
+
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "ED_file_indexer.h"
+
+/**
+ * File Indexer Service for indexing asset files.
+ *
+ * Opening and parsing a large collection of asset files inside a library can take a lot of time.
+ * To reduce the time it takes the files are indexed.
+ *
+ * - Index files are created for each blend file in the asset library, even when the blend file
+ * doesn't contain any assets.
+ * - Indexes are stored in an persistent cache folder (`BKE_appdir_folder_caches` +
+ * `asset_library_indexes/{asset_library_dir}/{asset_index_file.json}`).
+ * - The content of the indexes are used when:
+ * - Index exists and can be opened
+ * - Last modification date is earlier than the file it represents.
+ * - The index file version is the latest.
+ * - Blend files without any assets can be determined by the size of the index file for some
+ * additional performance.
+ */
+extern const FileIndexerType file_indexer_asset;
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/source/blender/editors/asset/intern/asset_indexer.cc b/source/blender/editors/asset/intern/asset_indexer.cc
new file mode 100644
index 00000000000..52b5df14c9f
--- /dev/null
+++ b/source/blender/editors/asset/intern/asset_indexer.cc
@@ -0,0 +1,781 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/** \file
+ * \ingroup edasset
+ */
+
+#include <fstream>
+#include <iomanip>
+#include <optional>
+
+#include "ED_asset_indexer.h"
+
+#include "DNA_asset_types.h"
+#include "DNA_userdef_types.h"
+
+#include "BLI_fileops.h"
+#include "BLI_hash.hh"
+#include "BLI_linklist.h"
+#include "BLI_path_util.h"
+#include "BLI_serialize.hh"
+#include "BLI_set.hh"
+#include "BLI_string_ref.hh"
+#include "BLI_uuid.h"
+
+#include "BKE_appdir.h"
+#include "BKE_asset.h"
+#include "BKE_asset_catalog.hh"
+#include "BKE_preferences.h"
+
+#include "CLG_log.h"
+
+static CLG_LogRef LOG = {"ed.asset"};
+
+namespace blender::ed::asset::index {
+
+using namespace blender::io::serialize;
+using namespace blender::bke;
+
+/**
+ * \file asset_indexer.cc
+ * \brief Indexer for asset libraries.
+ *
+ * Indexes are stored per input file. Each index can contain zero to multiple asset entries.
+ * The indexes are grouped together per asset library. They are stored in
+ * #BKE_appdir_folder_caches +
+ * /asset-library-indices/<asset-library-hash>/<asset-index-hash>_<asset_file>.index.json.
+ *
+ * The structure of an index file is
+ * \code
+ * {
+ * "version": <file version number>,
+ * "entries": [{
+ * "name": "<asset name>",
+ * "catalog_id": "<catalog_id>",
+ * "catalog_name": "<catalog_name>",
+ * "description": "<description>",
+ * "author": "<author>",
+ * "tags": ["<tag>"]
+ * }]
+ * }
+ * \endcode
+ *
+ * NOTE: entries, author, description and tags are optional attributes.
+ *
+ * NOTE: File browser uses name and idcode separate. Inside the index they are joined together like
+ * #ID.name.
+ * NOTE: File browser group name isn't stored in the index as it is a translatable name.
+ */
+constexpr StringRef ATTRIBUTE_VERSION("version");
+constexpr StringRef ATTRIBUTE_ENTRIES("entries");
+constexpr StringRef ATTRIBUTE_ENTRIES_NAME("name");
+constexpr StringRef ATTRIBUTE_ENTRIES_CATALOG_ID("catalog_id");
+constexpr StringRef ATTRIBUTE_ENTRIES_CATALOG_NAME("catalog_name");
+constexpr StringRef ATTRIBUTE_ENTRIES_DESCRIPTION("description");
+constexpr StringRef ATTRIBUTE_ENTRIES_AUTHOR("author");
+constexpr StringRef ATTRIBUTE_ENTRIES_TAGS("tags");
+
+/** Abstract class for #BlendFile and #AssetIndexFile. */
+class AbstractFile {
+ public:
+ virtual ~AbstractFile() = default;
+
+ virtual const char *get_file_path() const = 0;
+
+ bool exists() const
+ {
+ return BLI_exists(get_file_path());
+ }
+
+ size_t get_file_size() const
+ {
+ return BLI_file_size(get_file_path());
+ }
+};
+
+/**
+ * \brief Reference to a blend file that can be indexed.
+ */
+class BlendFile : public AbstractFile {
+ StringRefNull file_path_;
+
+ public:
+ BlendFile(StringRefNull file_path) : file_path_(file_path)
+ {
+ }
+
+ uint64_t hash() const
+ {
+ DefaultHash<StringRefNull> hasher;
+ return hasher(file_path_);
+ }
+
+ std::string get_filename() const
+ {
+ char filename[FILE_MAX];
+ BLI_split_file_part(get_file_path(), filename, sizeof(filename));
+ return std::string(filename);
+ }
+
+ const char *get_file_path() const override
+ {
+ return file_path_.c_str();
+ }
+};
+
+/**
+ * \brief Single entry inside a #AssetIndexFile for reading.
+ */
+struct AssetEntryReader {
+ private:
+ /**
+ * \brief Lookup table containing the elements of the entry.
+ */
+ ObjectValue::Lookup lookup;
+
+ StringRefNull get_name_with_idcode() const
+ {
+ return lookup.lookup(ATTRIBUTE_ENTRIES_NAME)->as_string_value()->value();
+ }
+
+ public:
+ AssetEntryReader(const ObjectValue &entry) : lookup(entry.create_lookup())
+ {
+ }
+
+ ID_Type get_idcode() const
+ {
+ const StringRefNull name_with_idcode = get_name_with_idcode();
+ return GS(name_with_idcode.c_str());
+ }
+
+ StringRef get_name() const
+ {
+ const StringRefNull name_with_idcode = get_name_with_idcode();
+ return name_with_idcode.substr(2);
+ }
+
+ bool has_description() const
+ {
+ return lookup.contains(ATTRIBUTE_ENTRIES_DESCRIPTION);
+ }
+
+ StringRefNull get_description() const
+ {
+ return lookup.lookup(ATTRIBUTE_ENTRIES_DESCRIPTION)->as_string_value()->value();
+ }
+
+ bool has_author() const
+ {
+ return lookup.contains(ATTRIBUTE_ENTRIES_AUTHOR);
+ }
+
+ StringRefNull get_author() const
+ {
+ return lookup.lookup(ATTRIBUTE_ENTRIES_AUTHOR)->as_string_value()->value();
+ }
+
+ StringRefNull get_catalog_name() const
+ {
+ return lookup.lookup(ATTRIBUTE_ENTRIES_CATALOG_NAME)->as_string_value()->value();
+ }
+
+ CatalogID get_catalog_id() const
+ {
+ const std::string &catalog_id =
+ lookup.lookup(ATTRIBUTE_ENTRIES_CATALOG_ID)->as_string_value()->value();
+ CatalogID catalog_uuid(catalog_id);
+ return catalog_uuid;
+ }
+
+ void add_tags_to_meta_data(AssetMetaData *asset_data) const
+ {
+ const ObjectValue::LookupValue *value_ptr = lookup.lookup_ptr(ATTRIBUTE_ENTRIES_TAGS);
+ if (value_ptr == nullptr) {
+ return;
+ }
+
+ const ArrayValue *array_value = (*value_ptr)->as_array_value();
+ const ArrayValue::Items &elements = array_value->elements();
+ for (const ArrayValue::Item &item : elements) {
+ const StringRefNull tag_name = item->as_string_value()->value();
+ BKE_asset_metadata_tag_add(asset_data, tag_name.c_str());
+ }
+ }
+};
+
+struct AssetEntryWriter {
+ private:
+ ObjectValue::Items &attributes;
+
+ public:
+ AssetEntryWriter(ObjectValue &entry) : attributes(entry.elements())
+ {
+ }
+
+ /**
+ * \brief add id + name to the attributes.
+ *
+ * NOTE: id and name are encoded like #ID.name
+ */
+ void add_id_name(const short idcode, const StringRefNull name)
+ {
+ char idcode_prefix[2];
+ /* Similar to `BKE_libblock_alloc`. */
+ *((short *)idcode_prefix) = idcode;
+ std::string name_with_idcode = std::string(idcode_prefix) + name;
+
+ attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_NAME, new StringValue(name_with_idcode)));
+ }
+
+ void add_catalog_id(const CatalogID &catalog_id)
+ {
+ char catalog_id_str[UUID_STRING_LEN];
+ BLI_uuid_format(catalog_id_str, catalog_id);
+ attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_CATALOG_ID, new StringValue(catalog_id_str)));
+ }
+
+ void add_catalog_name(const StringRefNull catalog_name)
+ {
+ attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_CATALOG_NAME, new StringValue(catalog_name)));
+ }
+
+ void add_description(const StringRefNull description)
+ {
+ attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_DESCRIPTION, new StringValue(description)));
+ }
+
+ voi
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list