[Bf-translations-svn] SVN commit: /data/svn/bf-translations [396] trunk/po/tools: New ( hopefully enhanced) RTL-preprocess.
bf-translations at blender.org
bf-translations at blender.org
Fri Feb 17 17:39:41 CET 2012
Revision: 396
http://projects.blender.org/scm/viewvc.php?view=rev&root=bf-translations&revision=396
Author: mont29
Date: 2012-02-17 16:39:40 +0000 (Fri, 17 Feb 2012)
Log Message:
-----------
New (hopefully enhanced) RTL-preprocess.
This one uses C library of Fribidi (update your settings.py file!), adding a new tool, rtl_preprocess, which can be either used standalone, or called through import_po_from_branches or update_trunk ones.
That new tool should handle nicely special escapes/formatting snippets currently in Blender's messages (like %d, \", %.4f, etc.), which were previously completely broken by RTL pre-processing.
Note that from now on, po's having an (empty) file named "is_rtl" in their directory will be rtl-preprocessed.
Will make an update of the whole repo after that commit, so that persian/arabic guys can immediately test everything is fine...
Modified Paths:
--------------
trunk/po/tools/_update_msg.py
trunk/po/tools/import_po_from_branches.py
trunk/po/tools/settings_template.py
Added Paths:
-----------
trunk/po/tools/rtl_preprocess.py
Modified: trunk/po/tools/_update_msg.py
===================================================================
--- trunk/po/tools/_update_msg.py 2012-02-16 16:32:48 UTC (rev 395)
+++ trunk/po/tools/_update_msg.py 2012-02-17 16:39:40 UTC (rev 396)
@@ -30,6 +30,9 @@
import settings
+classes = set()
+
+
SOURCE_DIR = settings.SOURCE_DIR
FILE_NAME_MESSAGES = settings.FILE_NAME_MESSAGES
@@ -186,7 +189,7 @@
def process_cls_list(cls_list):
if not cls_list:
- return
+ return 0
def full_class_id(cls):
""" gives us 'ID.Lamp.AreaLamp' which is best for sorting.
@@ -199,13 +202,22 @@
return cls_id
cls_list.sort(key=full_class_id)
+ processed = 0
for cls in cls_list:
walkClass(cls)
+ classes.add(cls)
# Recursively process subclasses.
- process_cls_list(cls.__subclasses__())
+ processed += process_cls_list(cls.__subclasses__()) + 1
+ return processed
# Parse everything (recursively parsing from bpy_struct "class"...).
- process_cls_list(type(bpy.context).__base__.__subclasses__())
+ processed = process_cls_list(type(bpy.context).__base__.__subclasses__())
+ print("{} classes processed!".format(processed))
+# import pickle
+# global classes
+# classes = {str(c) for c in classes}
+# with open("/home/i7deb64/Bureau/tpck_2", "wb") as f:
+# pickle.dump(classes, f, protocol=0)
from bpy_extras.keyconfig_utils import KM_HIERARCHY
Modified: trunk/po/tools/import_po_from_branches.py
===================================================================
--- trunk/po/tools/import_po_from_branches.py 2012-02-16 16:32:48 UTC (rev 395)
+++ trunk/po/tools/import_po_from_branches.py 2012-02-17 16:39:40 UTC (rev 396)
@@ -30,11 +30,14 @@
import settings
import utils
+import rtl_preprocess
TRUNK_PO_DIR = settings.TRUNK_PO_DIR
BRANCHES_DIR = settings.BRANCHES_DIR
+RTL_PREPROCESS_FILE = settings.RTL_PREPROCESS_FILE
+
PY3 = settings.PYTHON3_EXEC
@@ -62,9 +65,8 @@
continue
po = os.path.join(BRANCHES_DIR, lang, ".".join((lang, "po")))
if os.path.exists(po):
- po_script = os.path.join(BRANCHES_DIR, lang,
- "".join((lang, "_to_utf.py")))
- u1, state, stats = utils.parse_messages(po)
+ po_is_rtl = os.path.join(BRANCHES_DIR, lang, RTL_PREPROCESS_FILE)
+ msgs, state, stats = utils.parse_messages(po)
tot_msgs = stats["tot_msg"]
trans_msgs = stats["trans_msg"]
lvl = 0.0
@@ -76,12 +78,24 @@
"".format(lang, lvl))
ret = 1
else:
- if os.path.exists(po_script):
+ if os.path.exists(po_is_rtl):
out_po = os.path.join(TRUNK_PO_DIR,
".".join((lang, "po")))
- t = subprocess.call([PY3, po_script, po, out_po])
- if t:
- ret = t
+ out_raw_po = os.path.join(TRUNK_PO_DIR,
+ "_".join((lang, "raw.po")))
+ keys = []
+ trans = []
+ for k, m in msgs.items():
+ keys.append(k)
+ trans.append("".join(m["msgstr_lines"]))
+ trans = rtl_preprocess.log2vis(trans)
+ for k, t in zip(keys, trans):
+ # Mono-line for now...
+ msgs[k]["msgstr_lines"] = [t]
+ utils.write_messages(out_po, msgs, state["comm_msg"],
+ state["fuzzy_msg"])
+ # Also copies org po!
+ shutil.copy(po, out_raw_po)
print("{:<10}: {:>6.1%} done, enough translated " \
"messages, processed and copied to trunk." \
"".format(lang, lvl))
Added: trunk/po/tools/rtl_preprocess.py
===================================================================
--- trunk/po/tools/rtl_preprocess.py (rev 0)
+++ trunk/po/tools/rtl_preprocess.py 2012-02-17 16:39:40 UTC (rev 396)
@@ -0,0 +1,231 @@
+#!/usr/bin/python3
+
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# ***** END GPL LICENSE BLOCK *****
+
+# <pep8 compliant>
+
+# Preprocess right-to-left languages.
+# You can use it either standalone, or through import_po_from_branches or
+# update_trunk.
+#
+# Notes: This has been tested on Linux, not 100% it will work nicely on
+# Windows or OsX.
+# This uses ctypes, as there is no py3 binding for fribidi currently.
+# This implies you only need the compiled C library to run it.
+# Finally, note that it handles some formating/escape codes (like
+# \", %s, %x12, %.4f, etc.), protecting them from ugly (evil) fribidi,
+# which seems completely unaware of such things (as unicode is...).
+
+import sys
+import ctypes
+
+import settings
+import utils
+
+FRIBIDI_LIB = settings.FRIBIDI_LIB
+
+###### Import C library and recreate "defines". #####
+fbd = ctypes.CDLL(FRIBIDI_LIB)
+
+
+#define FRIBIDI_MASK_NEUTRAL 0x00000040L /* Is neutral */
+FRIBIDI_PAR_ON = 0x00000040
+
+
+#define FRIBIDI_FLAG_SHAPE_MIRRORING 0x00000001
+#define FRIBIDI_FLAG_REORDER_NSM 0x00000002
+
+#define FRIBIDI_FLAG_SHAPE_ARAB_PRES 0x00000100
+#define FRIBIDI_FLAG_SHAPE_ARAB_LIGA 0x00000200
+#define FRIBIDI_FLAG_SHAPE_ARAB_CONSOLE 0x00000400
+
+#define FRIBIDI_FLAG_REMOVE_BIDI 0x00010000
+#define FRIBIDI_FLAG_REMOVE_JOINING 0x00020000
+#define FRIBIDI_FLAG_REMOVE_SPECIALS 0x00040000
+
+#define FRIBIDI_FLAGS_DEFAULT ( \
+# FRIBIDI_FLAG_SHAPE_MIRRORING | \
+# FRIBIDI_FLAG_REORDER_NSM | \
+# FRIBIDI_FLAG_REMOVE_SPECIALS )
+
+#define FRIBIDI_FLAGS_ARABIC ( \
+# FRIBIDI_FLAG_SHAPE_ARAB_PRES | \
+# FRIBIDI_FLAG_SHAPE_ARAB_LIGA )
+
+FRIBIDI_FLAG_SHAPE_MIRRORING = 0x00000001
+FRIBIDI_FLAG_REORDER_NSM = 0x00000002
+FRIBIDI_FLAG_REMOVE_SPECIALS = 0x00040000
+
+FRIBIDI_FLAG_SHAPE_ARAB_PRES = 0x00000100
+FRIBIDI_FLAG_SHAPE_ARAB_LIGA = 0x00000200
+
+FRIBIDI_FLAGS_DEFAULT = FRIBIDI_FLAG_SHAPE_MIRRORING | \
+ FRIBIDI_FLAG_REORDER_NSM | \
+ FRIBIDI_FLAG_REMOVE_SPECIALS
+
+FRIBIDI_FLAGS_ARABIC = FRIBIDI_FLAG_SHAPE_ARAB_PRES | \
+ FRIBIDI_FLAG_SHAPE_ARAB_LIGA
+
+##### Kernel processing funcs. #####
+def protect_format_seq(msg):
+ """
+ Find some specific escaping/formating sequences (like \", %s, etc.,
+ and protect them from any modification!
+ """
+ LRE = "\u202A"
+ PDF = "\u202C"
+ # Most likely incomplete, but seems to cover current needs.
+ format_codes = set("tslfd")
+ digits = set(".0123456789")
+
+ idx = 0
+ ret = []
+ ln = len(msg)
+ while idx < ln:
+ dlt = 1
+ # \" or \'
+ if idx < (ln - 1) and msg[idx] == '\\' and msg[idx + 1] in "\"\'":
+ dlt = 2
+ # %x12
+ elif idx < (ln - 2) and msg[idx] == '%' and msg[idx + 1] in "x" and \
+ msg[idx + 2] in digits:
+ dlt = 2
+ while (idx + dlt + 1) < ln and msg[idx + dlt + 1] in digits:
+ dlt += 1
+ # %.4f
+ elif idx < (ln - 3) and msg[idx] == '%' and msg[idx + 1] in digits:
+ dlt = 2
+ while (idx + dlt + 1) < ln and msg[idx + dlt + 1] in digits:
+ dlt += 1
+ if (idx + dlt + 1) < ln and msg[idx + dlt + 1] in format_codes:
+ dlt += 1
+ else:
+ dlt = 1
+ # %s
+ elif idx < (ln - 1) and msg[idx] == '%' and \
+ msg[idx + 1] in format_codes:
+ dlt = 2
+
+ if dlt > 1:
+ ret.append(LRE)
+ ret += msg[idx:idx + dlt]
+ idx += dlt
+ if dlt > 1:
+ ret.append(PDF)
+
+ return "".join(ret)
+
+
+def log2vis(msgs):
+ """
+ Globally mimics deprecated fribidi_log2vis.
+ msgs should be an iterable of messages to rtl-process.
+ """
+ for msg in msgs:
+ msg = protect_format_seq(msg)
+
+ fbc_str = ctypes.create_unicode_buffer(msg)
+ ln = len(fbc_str) - 1
+# print(fbc_str.value, ln)
+ btypes = (ctypes.c_int * ln)()
+ embed_lvl = (ctypes.c_uint8 * ln)()
+ pbase_dir = ctypes.c_int(FRIBIDI_PAR_ON)
+ jtypes = (ctypes.c_uint8 * ln)()
+ flags = FRIBIDI_FLAGS_DEFAULT | FRIBIDI_FLAGS_ARABIC
+
+ # Find out direction of each char.
+ fbd.fribidi_get_bidi_types(fbc_str, ln, ctypes.byref(btypes))
+
+# print(*btypes)
+
+ fbd.fribidi_get_par_embedding_levels(btypes, ln,
+ ctypes.byref(pbase_dir),
+ embed_lvl)
+
+# print(*embed_lvl)
+
+ # Joinings for arabic chars.
+ fbd.fribidi_get_joining_types(fbc_str, ln, jtypes)
+# print(*jtypes)
+ fbd.fribidi_join_arabic(btypes, ln, embed_lvl, jtypes)
+# print(*jtypes)
+
+ # Final Shaping!
+ fbd.fribidi_shape(flags, embed_lvl, ln, jtypes, fbc_str)
+
+ print(fbc_str.value)
+# print(*(ord(c) for c in fbc_str))
+ # And now, the reordering.
+ # Note that here, we expect a single line, so no need to do
+ # fancy things...
+ fbd.fribidi_reorder_line(flags, btypes, ln, 0, pbase_dir, embed_lvl,
+ fbc_str, None)
+ print(fbc_str.value)
+# print(*(ord(c) for c in fbc_str))
+
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-translations-svn
mailing list