[Bf-extensions-cvs] [90b3cd61] master: UI translations addon: make some heavy processing parallel.
Bastien Montagne
noreply at git.blender.org
Tue May 5 18:22:27 CEST 2020
Commit: 90b3cd61a92e961563a10bd73a138d6a5bccee00
Author: Bastien Montagne
Date: Tue May 5 18:19:55 2020 +0200
Branches: master
https://developer.blender.org/rBA90b3cd61a92e961563a10bd73a138d6a5bccee00
UI translations addon: make some heavy processing parallel.
Fairly straghtforward, once the data from the i18n utils module are
pickleable... ;)
Gives nice improvements in global speed of i18n update process.
===================================================================
M ui_translate/__init__.py
M ui_translate/update_svn.py
===================================================================
diff --git a/ui_translate/__init__.py b/ui_translate/__init__.py
index 72f63a92..2cbf6cb0 100644
--- a/ui_translate/__init__.py
+++ b/ui_translate/__init__.py
@@ -21,7 +21,7 @@
bl_info = {
"name": "Manage UI translations",
"author": "Bastien Montagne",
- "version": (1, 2, 0),
+ "version": (1, 3, 0),
"blender": (2, 80, 0),
"location": "Main \"File\" menu, text editor, any UI control",
"description": "Allows managing UI translations directly from Blender "
diff --git a/ui_translate/update_svn.py b/ui_translate/update_svn.py
index a3228bac..70b7f0b5 100644
--- a/ui_translate/update_svn.py
+++ b/ui_translate/update_svn.py
@@ -34,6 +34,7 @@ else:
from bl_i18n_utils import utils as utils_i18n
from bl_i18n_utils import utils_languages_menu
+import concurrent.futures
import io
import os
import shutil
@@ -43,6 +44,18 @@ import tempfile
# Operators ###################################################################
+def i18n_updatetranslation_svn_branches_callback(pot, lng, settings):
+ if not lng['use']:
+ return
+ if os.path.isfile(lng['po_path']):
+ po = utils_i18n.I18nMessages(uid=lng['uid'], kind='PO', src=lng['po_path'], settings=settings)
+ po.update(pot)
+ else:
+ po = pot
+ po.write(kind="PO", dest=lng['po_path'])
+ print("{} PO written!".format(lng['uid']))
+
+
class UI_OT_i18n_updatetranslation_svn_branches(Operator):
"""Update i18n svn's branches (po files)"""
bl_idname = "ui.i18n_updatetranslation_svn_branches"
@@ -88,18 +101,15 @@ class UI_OT_i18n_updatetranslation_svn_branches(Operator):
return {'CANCELLED'}
# Now we should have a valid POT file, we have to merge it in all languages po's...
- pot = utils_i18n.I18nMessages(kind='PO', src=self.settings.FILE_NAME_POT, settings=self.settings)
- for progress, lng in enumerate(i18n_sett.langs):
- context.window_manager.progress_update(progress + 2)
- if not lng.use:
- continue
- if os.path.isfile(lng.po_path):
- po = utils_i18n.I18nMessages(uid=lng.uid, kind='PO', src=lng.po_path, settings=self.settings)
- po.update(pot)
- else:
- po = pot
- po.write(kind="PO", dest=lng.po_path)
- print("{} PO written!".format(lng.uid))
+ with concurrent.futures.ProcessPoolExecutor() as exctr:
+ pot = utils_i18n.I18nMessages(kind='PO', src=self.settings.FILE_NAME_POT, settings=self.settings)
+ num_langs = len(i18n_sett.langs)
+ for progress, _ in enumerate(exctr.map(i18n_updatetranslation_svn_branches_callback,
+ (pot,) * num_langs,
+ [dict(lng.items()) for lng in i18n_sett.langs],
+ (self.settings,) * num_langs,
+ chunksize=4)):
+ context.window_manager.progress_update(progress + 2)
context.window_manager.progress_end()
return {'FINISHED'}
@@ -108,6 +118,18 @@ class UI_OT_i18n_updatetranslation_svn_branches(Operator):
return wm.invoke_props_dialog(self)
+def i18n_cleanuptranslation_svn_branches_callback(lng, settings):
+ if not lng['use']:
+ print("Skipping {} language ({}).".format(lng['name'], lng['uid']))
+ return
+ po = utils_i18n.I18nMessages(uid=lng['uid'], kind='PO', src=lng['po_path'], settings=settings)
+ errs = po.check(fix=True)
+ po.write(kind="PO", dest=lng['po_path'])
+ print("Processing {} language ({}).\n"
+ "Cleaned up {} commented messages.\n".format(lng['name'], lng['uid'], po.clean_commented()) +
+ ("Errors in this po, solved as best as possible!\n\t" + "\n\t".join(errs) if errs else "") + "\n")
+
+
class UI_OT_i18n_cleanuptranslation_svn_branches(Operator):
"""Clean up i18n svn's branches (po files)"""
bl_idname = "ui.i18n_cleanuptranslation_svn_branches"
@@ -122,26 +144,41 @@ class UI_OT_i18n_cleanuptranslation_svn_branches(Operator):
context.window_manager.progress_begin(0, len(i18n_sett.langs) + 1)
context.window_manager.progress_update(0)
- for progress, lng in enumerate(i18n_sett.langs):
- context.window_manager.progress_update(progress + 1)
- if not lng.use:
- print("Skipping {} language ({}).".format(lng.name, lng.uid))
- continue
- print("Processing {} language ({}).".format(lng.name, lng.uid))
- po = utils_i18n.I18nMessages(uid=lng.uid, kind='PO', src=lng.po_path, settings=self.settings)
- print("Cleaned up {} commented messages.".format(po.clean_commented()))
- errs = po.check(fix=True)
- if errs:
- print("Errors in this po, solved as best as possible!")
- print("\t" + "\n\t".join(errs))
- po.write(kind="PO", dest=lng.po_path)
- print("\n")
+ with concurrent.futures.ProcessPoolExecutor() as exctr:
+ num_langs = len(i18n_sett.langs)
+ for progress, _ in enumerate(exctr.map(i18n_cleanuptranslation_svn_branches_callback,
+ [dict(lng.items()) for lng in i18n_sett.langs],
+ (self.settings,) * num_langs,
+ chunksize=4)):
+ context.window_manager.progress_update(progress + 1)
context.window_manager.progress_end()
return {'FINISHED'}
+def i18n_updatetranslation_svn_trunk_callback(lng, settings):
+ if lng['uid'] in settings.IMPORT_LANGUAGES_SKIP:
+ print("Skipping {} language ({}), edit settings if you want to enable it.\n".format(lng['name'], lng['uid']))
+ return lng['uid'], 0.0
+ if not lng['use']:
+ print("Skipping {} language ({}).\n".format(lng['name'], lng['uid']))
+ return lng['uid'], 0.0
+ po = utils_i18n.I18nMessages(uid=lng['uid'], kind='PO', src=lng['po_path'], settings=settings)
+ errs = po.check(fix=True)
+ print("Processing {} language ({}).\n"
+ "Cleaned up {} commented messages.\n".format(lng['name'], lng['uid'], po.clean_commented()) +
+ ("Errors in this po, solved as best as possible!\n\t" + "\n\t".join(errs) if errs else "") + "\n")
+ if lng['uid'] in settings.IMPORT_LANGUAGES_RTL:
+ po.write(kind="PO", dest=lng['po_path_trunk'][:-3] + "_raw.po")
+ po.rtl_process()
+ po.write(kind="PO", dest=lng['po_path_trunk'])
+ po.write(kind="PO_COMPACT", dest=lng['po_path_git'])
+ po.write(kind="MO", dest=lng['mo_path_trunk'])
+ po.update_info()
+ return lng['uid'], po.nbr_trans_msgs / po.nbr_msgs
+
+
class UI_OT_i18n_updatetranslation_svn_trunk(Operator):
"""Update i18n svn's branches (po files)"""
bl_idname = "ui.i18n_updatetranslation_svn_trunk"
@@ -156,30 +193,14 @@ class UI_OT_i18n_updatetranslation_svn_trunk(Operator):
context.window_manager.progress_begin(0, len(i18n_sett.langs) + 1)
context.window_manager.progress_update(0)
- for progress, lng in enumerate(i18n_sett.langs):
- context.window_manager.progress_update(progress + 1)
- if lng.uid in self.settings.IMPORT_LANGUAGES_SKIP:
- print("Skipping {} language ({}), edit settings if you want to enable it.".format(lng.name, lng.uid))
- continue
- if not lng.use:
- print("Skipping {} language ({}).".format(lng.name, lng.uid))
- continue
- print("Processing {} language ({}).".format(lng.name, lng.uid))
- po = utils_i18n.I18nMessages(uid=lng.uid, kind='PO', src=lng.po_path, settings=self.settings)
- print("Cleaned up {} commented messages.".format(po.clean_commented()))
- errs = po.check(fix=True)
- if errs:
- print("Errors in this po, solved as best as possible!")
- print("\t" + "\n\t".join(errs))
- if lng.uid in self.settings.IMPORT_LANGUAGES_RTL:
- po.write(kind="PO", dest=lng.po_path_trunk[:-3] + "_raw.po")
- po.rtl_process()
- po.write(kind="PO", dest=lng.po_path_trunk)
- po.write(kind="PO_COMPACT", dest=lng.po_path_git)
- po.write(kind="MO", dest=lng.mo_path_trunk)
- po.update_info()
- stats[lng.uid] = po.nbr_trans_msgs / po.nbr_msgs
- print("\n")
+ with concurrent.futures.ProcessPoolExecutor() as exctr:
+ num_langs = len(i18n_sett.langs)
+ for progress, (lng_uid, stats_val) in enumerate(exctr.map(i18n_updatetranslation_svn_trunk_callback,
+ [dict(lng.items()) for lng in i18n_sett.langs],
+ (self.settings,) * num_langs,
+ chunksize=4)):
+ context.window_manager.progress_update(progress + 1)
+ stats[lng_uid] = stats_val
# Copy pot file from branches to trunk.
shutil.copy2(self.settings.FILE_NAME_POT, self.settings.TRUNK_PO_DIR)
More information about the Bf-extensions-cvs
mailing list