[Bf-extensions-cvs] [69ad4c43] master: BlenderKit: optimize search
Vilem Duha
noreply at git.blender.org
Mon Jul 26 23:28:00 CEST 2021
Commit: 69ad4c43d004c9788ba98e75ade02097284a25f9
Author: Vilem Duha
Date: Mon Jul 26 23:08:13 2021 +0200
Branches: master
https://developer.blender.org/rBA69ad4c43d004c9788ba98e75ade02097284a25f9
BlenderKit: optimize search
quite a large overhaul of how search results are loaded.
this saves time due to not closing the session and enabling small previews to be first.
Also use the dict_param paraameter so that the results responses are a bit smaller.
===================================================================
M blenderkit/__init__.py
M blenderkit/asset_bar_op.py
M blenderkit/search.py
M blenderkit/thumbnails/thumbnail_notready.jpg
M blenderkit/ui.py
M blenderkit/ui_bgl.py
M blenderkit/ui_panels.py
M blenderkit/utils.py
===================================================================
diff --git a/blenderkit/__init__.py b/blenderkit/__init__.py
index 574349b1..ec2268dc 100644
--- a/blenderkit/__init__.py
+++ b/blenderkit/__init__.py
@@ -1704,7 +1704,7 @@ class BlenderKitAddonPreferences(AddonPreferences):
max_assetbar_rows: IntProperty(name="Max Assetbar Rows",
description="max rows of assetbar in the 3D view",
default=1,
- min=0,
+ min=1,
max=20)
thumb_size: IntProperty(name="Assetbar thumbnail Size", default=96, min=-1, max=256)
diff --git a/blenderkit/asset_bar_op.py b/blenderkit/asset_bar_op.py
index 19f51682..8727698d 100644
--- a/blenderkit/asset_bar_op.py
+++ b/blenderkit/asset_bar_op.py
@@ -518,7 +518,7 @@ class BlenderKitAssetBarOperator(BL_UI_OT_draw_operator):
self.active_index = widget.search_index
self.draw_tooltip = True
- self.tooltip = asset_data['tooltip']
+ # self.tooltip = asset_data['tooltip']
ui_props = scene.blenderkitUI
ui_props.active_index = widget.search_index +self.scroll_offset
diff --git a/blenderkit/search.py b/blenderkit/search.py
index ed295e79..6bbbac7f 100644
--- a/blenderkit/search.py
+++ b/blenderkit/search.py
@@ -52,7 +52,6 @@ import urllib
import queue
import logging
-
bk_logger = logging.getLogger('blenderkit')
search_start_time = 0
@@ -77,9 +76,12 @@ def check_errors(rdata):
search_threads = []
-thumb_sml_download_threads = {}
-thumb_full_download_threads = {}
+thumb_workers_sml = []
+thumb_workers_full = []
+thumb_sml_download_threads = queue.Queue()
+thumb_full_download_threads = queue.Queue()
reports_queue = queue.Queue()
+all_thumbs_loaded = True
rtips = ['Click or drag model or material in scene to link/append ',
"Please rate responsively and plentifully. This helps us distribute rewards to the authors.",
@@ -282,7 +284,7 @@ def parse_result(r):
if r['available_resolutions']: # should check only for non-empty sequences
r['max_resolution'] = max(r['available_resolutions'])
- tooltip = generate_tooltip(r)
+ # tooltip = generate_tooltip(r)
# for some reason, the id was still int on some occurances. investigate this.
r['author']['id'] = str(r['author']['id'])
@@ -290,13 +292,13 @@ def parse_result(r):
# so blender's data is same as on server.
asset_data = {'thumbnail': tname,
'thumbnail_small': small_tname,
- 'tooltip': tooltip,
+ # 'tooltip': tooltip,
}
asset_data['downloaded'] = 0
# parse extra params needed for blender here
- params = utils.params_to_dict(r['parameters'])
+ params = r['dictParameters']#utils.params_to_dict(r['parameters'])
if asset_type == 'model':
if params.get('boundBoxMinX') != None:
@@ -343,7 +345,6 @@ def parse_result(r):
# @bpy.app.handlers.persistent
def search_timer():
-
# this makes a first search after opening blender. showing latest assets.
# utils.p('timer search')
# utils.p('start search timer')
@@ -371,6 +372,19 @@ def search_timer():
# check_clipboard()
+ # finish loading thumbs from queues
+ global all_thumbs_loaded
+ if not all_thumbs_loaded:
+ ui_props = bpy.context.scene.blenderkitUI
+ search_name = f'bkit {ui_props.asset_type.lower()} search'
+ wm = bpy.context.window_manager
+ if wm.get(search_name) is not None:
+ all_loaded = True
+ for ri, r in enumerate(wm[search_name]):
+ if not r.get('thumb_small_loaded'):
+ all_loaded = all_loaded and load_preview(r, ri)
+ all_thumbs_loaded = all_loaded
+
global search_threads
if len(search_threads) == 0:
# utils.p('end search timer')
@@ -384,6 +398,8 @@ def search_timer():
return 0.5
+
+
for thread in search_threads:
# TODO this doesn't check all processes when one gets removed,
# but most of the time only one is running anyway
@@ -417,18 +433,16 @@ def search_timer():
rdata = thread[0].result
-
-
ok, error = check_errors(rdata)
if ok:
ui_props = bpy.context.scene.blenderkitUI
-
orig_len = len(result_field)
+
for ri, r in enumerate(rdata['results']):
asset_data = parse_result(r)
if asset_data != None:
result_field.append(asset_data)
- load_preview(asset_data,ri + orig_len)
+ all_thumbs_loaded = all_thumbs_loaded and load_preview(asset_data, ri + orig_len)
# Get ratings from BlenderKit server
user_preferences = bpy.context.preferences.addons['blenderkit'].preferences
@@ -437,13 +451,14 @@ def search_timer():
if utils.profile_is_validator():
for r in rdata['results']:
if ratings_utils.get_rating_local(asset_data['id']) is None:
- rating_thread = threading.Thread(target=ratings_utils.get_rating, args=([r['id'], headers]), daemon=True)
+ rating_thread = threading.Thread(target=ratings_utils.get_rating, args=([r['id'], headers]),
+ daemon=True)
rating_thread.start()
wm[search_name] = result_field
wm['search results'] = result_field
- #rdata=['results']=[]
+ # rdata=['results']=[]
wm[search_name + ' orig'] = rdata
wm['search results orig'] = rdata
@@ -456,7 +471,7 @@ def search_timer():
tasks_queue.add_task((ui.add_report, ('No matching results found.',)))
# undo push
# bpy.ops.wm.undo_push_context(message='Get BlenderKit search')
- #show asset bar automatically, but only on first page - others are loaded also when asset bar is hidden.
+ # show asset bar automatically, but only on first page - others are loaded also when asset bar is hidden.
if not ui_props.assetbar_on and not thread[0].params.get('get_next'):
bpy.ops.object.run_assetbar_fix_context()
@@ -469,9 +484,11 @@ def search_timer():
# print('finished search thread')
mt('preview loading finished')
# utils.p('end search timer')
-
+ if not all_thumbs_loaded:
+ return .1
return .3
+
def load_preview(asset, index):
scene = bpy.context.scene
# FIRST START SEARCH
@@ -479,11 +496,13 @@ def load_preview(asset, index):
directory = paths.get_temp_dir('%s_search' % props.asset_type.lower())
s = bpy.context.scene
results = bpy.context.window_manager.get('search results')
-
+ loaded = True
tpath = os.path.join(directory, asset['thumbnail_small'])
- if not asset['thumbnail_small']:
- tpath = paths.get_addon_thumbnail_path('thumbnail_not_available.jpg')
+ if not asset['thumbnail_small'] or not os.path.exists(tpath):
+ # tpath = paths.get_addon_thumbnail_path('thumbnail_notready.jpg')
+ asset['thumb_small_loaded'] = False
+ loaded = False
iname = utils.previmg_name(index)
@@ -492,14 +511,16 @@ def load_preview(asset, index):
if img is None:
if not os.path.exists(tpath):
- return
+ return False
img = bpy.data.images.load(tpath)
img.name = iname
elif img.filepath != tpath:
if not os.path.exists(tpath):
- return
+ #unload loaded previews from previous results
+ bpy.data.images.remove(img)
+ return False
# had to add this check for autopacking files...
- if img.packed_file is not None:
+ if bpy.data.use_autopack and img.packed_file is not None:
img.unpack(method='USE_ORIGINAL')
img.filepath = tpath
img.reload()
@@ -508,6 +529,8 @@ def load_preview(asset, index):
image_utils.set_colorspace(img, 'Non-Color')
else:
image_utils.set_colorspace(img, 'sRGB')
+ asset['thumb_small_loaded'] = True
+ return loaded
def load_previews():
@@ -521,7 +544,7 @@ def load_previews():
if results is not None:
i = 0
for r in results:
- load_preview(r,i)
+ load_preview(r, i)
i += 1
@@ -635,14 +658,47 @@ def generate_author_textblock(adata):
t = writeblockm(t, adata, key='aboutMe', pretext='', width=col_w)
return t
+def download_image(session, url, filepath):
+ r = None
+ try:
+ r = session.get(url, stream=False)
+ except Exception as e:
+ bk_logger.error('Thumbnail download failed')
+ bk_logger.error(str(e))
+ if r and r.status_code == 200:
+ with open(filepath, 'wb') as f:
+ f.write(r.content)
+
+def thumb_download_worker(queue_sml, queue_full):
+ # print('thumb downloader', self.url)
+ # utils.p('start thumbdownloader thread')
+ while 1:
+ session = None
+ #start a session only for single search usually.
+ if not queue_sml.empty() or not queue_full.empty():
+ session = requests.Session()
+
+ while not queue_sml.empty():
+ url, filepath = queue_sml.get()
+ download_image(session,url, filepath)
+ exit_full = False
+ # download full resolution image, but only if no small thumbs are waiting.
+ while not queue_full.empty() and queue_sml.empty():
+ url, filepath = queue_full.get()
+ download_image(session,url, filepath)
+
+ if queue_sml.empty() and queue_full.empty():
+ if session is not None:
+ session.close()
+ time.sleep(.5)
class ThumbDownloader(threading.Thread):
- query = None
- def _
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-extensions-cvs
mailing list