[Bf-blender-cvs] [7cb1438] asset-engine: Asset update: WIP proof-of-concept async (job-based) update code.

Bastien Montagne noreply at git.blender.org
Tue Apr 19 23:21:15 CEST 2016


Commit: 7cb1438035df74a53d5776e78605c805048b4f21
Author: Bastien Montagne
Date:   Tue Apr 19 13:07:33 2016 +0200
Branches: asset-engine
https://developer.blender.org/rB7cb1438035df74a53d5776e78605c805048b4f21

Asset update: WIP proof-of-concept async (job-based) update code.

Builds, but not tested nor hoocked to anything yet!

===================================================================

M	source/blender/blenkernel/BKE_asset.h
M	source/blender/blenkernel/intern/asset.c
M	source/blender/editors/space_file/filelist.c
M	source/blender/windowmanager/WM_api.h
M	source/blender/windowmanager/intern/wm_operators.c

===================================================================

diff --git a/source/blender/blenkernel/BKE_asset.h b/source/blender/blenkernel/BKE_asset.h
index 82dee54..6a93922 100644
--- a/source/blender/blenkernel/BKE_asset.h
+++ b/source/blender/blenkernel/BKE_asset.h
@@ -97,7 +97,9 @@ typedef int (*ae_ensure_entries)(struct AssetEngine *engine, const int job_id, s
  * Engine should check whether given assets are still valid, if they should be updated, etc.
  * uuids tagged as needing reload will then be reloaded as new ones
  * (ae_load_pre, then actual lib loading, then ae_load_post).
- * \warning DO NOT add or remove (or alter order of) uuids from the list in this callback! */
+ * \warning This callback is expected to handle **real** UUIDS (not 'users' filebrowser ones),
+ *          i.e. calling ae_load_pre with those shall **not** alters them in returned direntries
+ *          (else 'link' between old IDs and reloaded ones would be broken). */
 typedef int (*ae_update_check)(struct AssetEngine *engine, const int job_id, struct AssetUUIDList *uuids);
 
 /* ***** All callbacks below are blocking. They shall be completed upon return. ***** */
@@ -222,11 +224,13 @@ struct FileDirEntry *BKE_filedir_entry_copy(struct FileDirEntry *entry);
 
 void BKE_filedir_entryarr_clear(struct FileDirEntryArr *array);
 
-#define ASSETUUID_SUB_COMPARE(_uuida, _uuidb, _member)                         \
-	(memcmp((_uuida)->#_member, (_uuidb)->#_member, sizeof((_uuida)->#_member)) == 0)
+#define ASSETUUID_SUB_COMPARE(_uuida, _uuidb, _member) \
+	(memcmp((_uuida)->_member, (_uuidb)->_member, sizeof((_uuida)->_member)) == 0)
 
-#define ASSETUUID_COMPARE(_uuida, _uuidb)                                      \
-	(memcmp((_uuida), (_uuidb), sizeof(*(_uuida))) == 0)
+#define ASSETUUID_COMPARE(_uuida, _uuidb) \
+	(ASSETUUID_SUB_COMPARE(_uuida, _uuidb, uuid_asset) && \
+	 ASSETUUID_SUB_COMPARE(_uuida, _uuidb, uuid_variant) && \
+	 ASSETUUID_SUB_COMPARE(_uuida, _uuidb, uuid_revision))
 
 /* GHash helpers */
 unsigned int BKE_asset_uuid_hash(const void *key);
diff --git a/source/blender/blenkernel/intern/asset.c b/source/blender/blenkernel/intern/asset.c
index d0dc664..40b842e 100644
--- a/source/blender/blenkernel/intern/asset.c
+++ b/source/blender/blenkernel/intern/asset.c
@@ -235,7 +235,7 @@ AssetUUIDList *BKE_asset_engine_load_pre(AssetEngine *engine, FileDirEntryArr *r
 	return uuids;
 }
 
-
+/* Note: this is a blocking version! */
 void BKE_assets_update_check(Main *bmain)
 {
 	BKE_main_id_tag_all(bmain, LIB_TAG_DOIT, false);
diff --git a/source/blender/editors/space_file/filelist.c b/source/blender/editors/space_file/filelist.c
index 54545f7..98fc516 100644
--- a/source/blender/editors/space_file/filelist.c
+++ b/source/blender/editors/space_file/filelist.c
@@ -2874,7 +2874,7 @@ static void filelist_readjob_update(void *flrjv)
 
 		flrj->filelist->flags |= (FL_NEED_SORTING | FL_NEED_FILTERING);
 
-		if (flrj->ae_job_id == AE_JOB_ID_INVALID) {
+		if (flrj->ae_job_id == AE_JOB_ID_INVALID) {  /* Immediate execution. */
 			*flrj->progress = 1.0f;
 			*flrj->stop = true;
 		}
@@ -2932,7 +2932,7 @@ static void filelist_readjob_endjob(void *flrjv)
 	flrj->filelist->flags &= ~FL_IS_PENDING;
 	flrj->filelist->flags |= FL_IS_READY;
 
-	if (flrj->filelist->ae && flrj->ae_job_id != 0) {
+	if (flrj->filelist->ae && !ELEM(flrj->ae_job_id, AE_JOB_ID_INVALID, AE_JOB_ID_UNSET)) {
 		AssetEngine *ae = flrj->filelist->ae;
 		ae->type->kill(ae, flrj->ae_job_id);
 	}
diff --git a/source/blender/windowmanager/WM_api.h b/source/blender/windowmanager/WM_api.h
index db2547e..b60c5ef 100644
--- a/source/blender/windowmanager/WM_api.h
+++ b/source/blender/windowmanager/WM_api.h
@@ -446,6 +446,7 @@ enum {
 	WM_JOB_TYPE_SEQ_BUILD_PREVIEW,
 	WM_JOB_TYPE_POINTCACHE,
 	WM_JOB_TYPE_DPAINT_BAKE,
+	WM_JOB_TYPE_ASSET_UPDATECHECK,
 	/* add as needed, screencast, seq proxy build
 	 * if having hard coded values is a problem */
 };
diff --git a/source/blender/windowmanager/intern/wm_operators.c b/source/blender/windowmanager/intern/wm_operators.c
index 6f0afda..61e3e11 100644
--- a/source/blender/windowmanager/intern/wm_operators.c
+++ b/source/blender/windowmanager/intern/wm_operators.c
@@ -2390,6 +2390,256 @@ static void WM_OT_revert_mainfile(wmOperatorType *ot)
 
 /* ****************** assets ****************** */
 
+
+typedef struct AssetUpdateCheckEngine {
+	struct AssetUpdateCheckEngine *next, *prev;
+	AssetEngine *ae;
+
+	/* Note: We cannot store IDs themselves in non-locking async task... so we'll have to check again for
+	 *       UUID/IDs mapping on each update call... Not ideal, but don't think it will be that big of an override
+	 *       in practice. */
+	AssetUUIDList uuids;
+	int ae_job_id;
+	short status;
+} AssetUpdateCheckEngine;
+
+typedef struct AssetUpdateCheckJob {
+	ListBase engines;
+	short flag;
+
+	float *progress;
+	short *stop;
+} AssetUpdateCheckJob;
+
+/* AssetUpdateCheckEngine.status */
+enum {
+	AUCE_UPDATE_CHECK_DONE  = 1 << 0,  /* Update check is finished for this engine. */
+	AUCE_ENSURE_ASSETS_DONE = 1 << 1,  /* Asset ensure is finished for this engine (if applicable). */
+};
+
+/* AssetUpdateCheckJob.flag */
+enum {
+	AUCJ_ENSURE_ASSETS = 1 << 0,  /* Try to perform the 'ensure' task too. */
+};
+
+static void asset_updatecheck_startjob(void *aucjv, short *stop, short *do_update, float *progress)
+{
+	AssetUpdateCheckJob *aucj = aucjv;
+
+	aucj->progress = progress;
+	aucj->stop = stop;
+	/* Using AE engine, worker thread here is just sleeping! */
+	while (!*stop) {
+		*do_update = true;
+		PIL_sleep_ms(100);
+	}
+}
+
+static void asset_updatecheck_update(void *aucjv)
+{
+	AssetUpdateCheckJob *aucj = aucjv;
+	Main *bmain = G.main;
+
+	const bool do_ensure = ((aucj->flag & AUCJ_ENSURE_ASSETS) != 0);
+	bool is_finished = true;
+	int nbr_engines = 0;
+
+	*aucj->progress = 0.0f;
+
+	for (AssetUpdateCheckEngine *auce = aucj->engines.first; auce; auce = auce->next, nbr_engines++) {
+		AssetEngine *ae = auce->ae;
+		AssetEngineType *ae_type = ae->type;
+
+		/* Step 1: we ask asset engine about status of all asset IDs from it. */
+		if (!(auce->status & AUCE_UPDATE_CHECK_DONE)) {
+			auce->ae_job_id = ae_type->update_check(ae, auce->ae_job_id, &auce->uuids);
+			if (auce->ae_job_id == AE_JOB_ID_INVALID) {  /* Immediate execution. */
+				*aucj->progress += 1.0f;
+				auce->status |= AUCE_UPDATE_CHECK_DONE;
+			}
+			else {
+				*aucj->progress += ae_type->progress(ae, auce->ae_job_id);
+				if ((ae_type->status(ae, auce->ae_job_id) & (AE_STATUS_RUNNING | AE_STATUS_VALID)) !=
+					(AE_STATUS_RUNNING | AE_STATUS_VALID))
+				{
+					auce->status |= AUCE_UPDATE_CHECK_DONE;
+				}
+			}
+
+			if (auce->status & AUCE_UPDATE_CHECK_DONE) {
+				auce->ae_job_id = AE_JOB_ID_UNSET;
+
+				for (Library *lib = bmain->library.first; lib; lib = lib->id.next) {
+					if (!lib->asset_repository ||
+					    (BKE_asset_engines_find(lib->asset_repository->asset_engine) != ae_type))
+					{
+						continue;
+					}
+
+					int i = auce->uuids.nbr_uuids;
+					for (AssetUUID *uuid = auce->uuids.uuids; i--; uuid++) {
+						bool done = false;
+						for (AssetRef *aref = lib->asset_repository->assets.first; aref && !done; aref = aref->next) {
+							for (LinkData *ld = aref->id_list.first; ld; ld = ld->next) {
+								ID *id = ld->data;
+								if (id->uuid && ASSETUUID_COMPARE(id->uuid, uuid)) {
+									*id->uuid = *uuid;
+
+									if (id->uuid->tag & UUID_TAG_ENGINE_MISSING) {
+										printf("\t%s uses a currently unknown asset engine!\n", id->name);
+									}
+									else if (id->uuid->tag & UUID_TAG_ASSET_MISSING) {
+										printf("\t%s is currently unknown by asset engine!\n", id->name);
+									}
+									else if (id->uuid->tag & UUID_TAG_ASSET_RELOAD) {
+										printf("\t%s needs to be reloaded/updated!\n", id->name);
+									}
+									done = true;
+									break;
+								}
+							}
+						}
+					}
+				}
+
+			}
+		}
+
+		/* Step 2: If required and supported, we 'ensure' assets tagged as to be reloaded. */
+		if (do_ensure && !(auce->status & AUCE_ENSURE_ASSETS_DONE) && ae_type->ensure_entries != NULL) {
+			/* TODO ensure entries! */
+			*aucj->progress += 1.0f;
+			auce->status |= AUCE_ENSURE_ASSETS_DONE;
+			if (auce->status & AUCE_ENSURE_ASSETS_DONE) {
+				auce->ae_job_id = AE_JOB_ID_UNSET;
+			}
+		}
+
+		if ((auce->status & (AUCE_UPDATE_CHECK_DONE | AUCE_ENSURE_ASSETS_DONE)) !=
+		    (AUCE_UPDATE_CHECK_DONE | AUCE_ENSURE_ASSETS_DONE))
+		{
+			is_finished = false;
+		}
+	}
+
+	*aucj->progress /= (float)(do_ensure ? nbr_engines * 2 : nbr_engines);
+	*aucj->stop = is_finished;
+}
+
+static void asset_updatecheck_endjob(void *aucjv)
+{
+	AssetUpdateCheckJob *aucj = aucjv;
+
+	/* In case there would be some dangling update. */
+	asset_updatecheck_update(aucjv);
+
+	for (AssetUpdateCheckEngine *auce = aucj->engines.first; auce; auce = auce->next) {
+		AssetEngine *ae = auce->ae;
+		if (!ELEM(auce->ae_job_id, AE_JOB_ID_INVALID, AE_JOB_ID_UNSET)) {
+			ae->type->kill(ae, auce->ae_job_id);
+		}
+	}
+}
+
+static void asset_updatecheck_free(void *aucjv)
+{
+	AssetUpdateCheckJob *aucj = aucjv;
+
+	for (AssetUpdateCheckEngine *auce = aucj->engines.first; auce; auce = auce->next) {
+		BKE_asset_engine_free(auce->ae);
+		MEM_freeN(auce->uuids.uuids);
+	}
+	BLI_freelistN(&aucj->engines);
+
+	MEM_freeN(aucj);
+}
+
+static void asset_updatecheck_start(const bContext *C)
+{
+	wmJob *wm_job;
+	AssetUpdateCheckJob *aucj;
+
+	Main *bmain = CTX_data_main(C);
+
+	/* prepare job data */
+	aucj = MEM_callocN(sizeof(*aucj), __func__);
+
+	for (Library *lib = bmain->library.first; lib; lib = lib->id.next) {
+		if (lib->asset_repository) {
+			printf("Handling lib file %s (engine %s, ver. %d)\n", lib->filepath, lib->asset_repository->asset_engine, lib->asset_repository->asset_engine_version);
+
+			AssetUpdateCheckEngine *auce = NULL;
+			AssetEngineType *ae_type = BKE_asset_engines_find(lib->asset_repository->asset_engine);
+			bool copy_engine = false;
+
+			if (ae_type == NULL) {
+				printf("ERROR! Unknown asset engine!\n");
+			}
+			else {
+				for (auce = aucj->engines.first; auce; auce = auce->next) {
+					if (auce->ae->type == ae_type) {
+						/* In case we have several engine versions for the same engine, we create several

@@ Diff output truncated at 10240 characters. @@




More information about the Bf-blender-cvs mailing list