[Bf-docboard-svn] bf-manual: [7419] trunk/blender_docs/tools_maintenance: tools po shortcuts:
Tobias Heinke
noreply at blender.org
Sun Nov 22 00:06:16 CET 2020
Revision: 7419
https://developer.blender.org/rBM7419
Author: TobiasH
Date: 2020-11-22 00:06:16 +0100 (Sun, 22 Nov 2020)
Log Message:
-----------
tools po shortcuts:
relative path
generators for file loop, po parsing
try-catch block for json decode
json fix bug in set creation for duplicates check
remove external folder copy creation
Modified Paths:
--------------
trunk/blender_docs/tools_maintenance/po_shortcuts.py
trunk/blender_docs/tools_maintenance/po_shortcuts_tables.json
Modified: trunk/blender_docs/tools_maintenance/po_shortcuts.py
===================================================================
--- trunk/blender_docs/tools_maintenance/po_shortcuts.py 2020-11-21 21:32:52 UTC (rev 7418)
+++ trunk/blender_docs/tools_maintenance/po_shortcuts.py 2020-11-21 23:06:16 UTC (rev 7419)
@@ -1,156 +1,156 @@
#! /usr/bin/env python3
-# Changes the shortcuts (:kbd:`...`) across all the translated PO files
-# of a spcecific language, according to the provided JSON table.
-# Language code and output directory must be provided.
+"""
+Changes the shortcuts (:kbd:`...`) across all the translated PO files
+of a specific language, according to the provided JSON table.
+Language code must be provided.
+"""
-import json
import sys
import os
import re
+import json
-ROLE = ':kbd:' # Change to any other role if needed
+CURRENT_DIR = os.path.abspath(os.path.dirname(__file__))
+PO_DIR = os.path.normpath(os.path.join(CURRENT_DIR, "..", "locale"))
+ROLE = 'kbd' # Change to any other role if needed
-def find_vcs_root(dirs=(".svn", ".git"), default=None):
- """
- Returns the repo root dir.
- :param dirs: dirs to look for, for repo detection.
- :param default: value to return if not found.
- :return: repo root dir.
- """
- prev, test = None, os.path.abspath(os.path.dirname(__file__))
- while prev != test:
- if any(os.path.isdir(os.path.join(test, d)) for d in dirs):
- return test
- prev, test = test, os.path.abspath(os.path.join(test, os.pardir))
- return default
+def po_files(path):
+ for dirpath, dirnames, filenames in os.walk(path):
+ if dirpath.startswith("."):
+ continue
+ for filename in filenames:
+ if filename.startswith("."):
+ continue
+ ext = os.path.splitext(filename)[1]
+ if ext.lower() == ".po":
+ yield os.path.join(dirpath, filename)
-def msgstr_replace(msgstr):
+def parse_po(text):
"""
- Replaces and returns the provided string based on the substitution table.
+ Parse the given po text for msgstrs.
- Function attribute 'table' contains the substitution table, as a list of
- 2-tuples with replacements for the language of the form
- (regex object, replacement string).
- :param msgstr: string to process.
- :return: replaced string, number of changes made.
+ :param text: the text of the po file.
+ :yield: text of the msgstr, start char position.
"""
- ntotal = 0
- for regex, repl in msgstr_replace.table:
- msgstr, n = regex.subn(repl, msgstr)
- ntotal += n
- return msgstr, ntotal
+ msgstr = []
+ pos = 0
+ for line in text.splitlines(keepends=True):
+ if msgstr:
+ if line.startswith('"'):
+ msgstr.append(line)
+ else:
+ yield "".join(msgstr), start
+ msgstr = []
+ else:
+ if line.startswith('msgstr'):
+ msgstr.append(line)
+ start = pos
+ pos += len(line)
-def file_process(filename):
- """
- Processes the given file. If changes are made, output file is generated.
+ if msgstr:
+ yield "".join(msgstr), start
- :param filename: the path and name of the file to process.
- :return: nothing.
- """
- out_text = ''
- n_changes = 0
- with open(filename, 'rt') as f:
- msgstr = ''
- for line in f:
- if msgstr:
- if line.startswith('"'):
- msgstr += line
- else:
- newtext, n = msgstr_replace(msgstr)
- out_text += newtext + line
- n_changes += n
- msgstr = ''
+
+def read_json(lang):
+ def parse_json(obj):
+ """
+ Function used to parse the json file. Find table and search for duplicates.
+
+ :param obj: tuple passed by 'json.load()'.
+ :return: if 'obj' is top level, it returns the table in a list of 2-tuples,
+ or None if table not found, or duplicate elements present.
+ If not on top level, it just returns the passed object.
+ """
+ retval = obj
+ if isinstance(obj[0][1], list): # top level?
+ retval = None # language table not found so far
+ for tpl in obj:
+ if tpl[0] == lang: # table found?
+ retval = tpl[1]
+ break
+ if retval: # table was found?
+ # Check for duplicates:
+ test_set = set(entry[0] for entry in retval)
+ if len(test_set) < len(retval): # duplicates?
+ raise ValueError("table contains duplicate entries")
else:
- if line.startswith('msgstr'):
- msgstr += line
- else:
- out_text += line
- if msgstr: # is there a last pending msgstr?
- newtext, n = msgstr_replace(msgstr)
- out_text += newtext
- n_changes += n
- # In case there were changes, an output file is generated:
- if n_changes > 0:
- print(filename[filename.find('LC_MESSAGES')+11:] + ':',
- n_changes, 'change(s).')
- file_out = os.path.join(os.path.expanduser(sys.argv[2]),
- 'new_' + filename[filename.find('locale'):])
- os.makedirs(os.path.dirname(file_out), exist_ok=True)
- with open(file_out, 'wt') as f:
- f.write(out_text)
+ raise ValueError("table not found")
+ return retval
+ filename = 'po_shortcuts_tables.json'
+ try:
+ with open(filename, 'r', encoding="utf-8") as json_file:
+ table = json.load(json_file, object_pairs_hook=parse_json)
-def json_parse(obj):
- """
- Function used to parse the json file.
+ except (IOError, OSError) as err:
+ print("{0}: cannot read data file: {1}".format(filename, err))
+ return None
- :param obj: tuple passed by 'json.load()'.
- :return: if 'obj' is top level, it returns the table in a list of 2-tuples,
- or None if table not found, or duplicate elements present.
- If not on top level, it just returns the passed object.
- """
- retval = obj
- if isinstance(obj[0][1], list): # top level?
- retval = None # language table not found so far
- for tpl in obj:
- if tpl[0] == sys.argv[1]: # table found?
- retval = tpl[1]
- break
- if retval: # table was found?
- # Let's check for duplicates:
- test_set = set(retval)
- if len(test_set) < len(retval): # duplicates?
- print(f"Error: '{sys.argv[1]}' table contains duplicate entries.")
- retval = None
- else:
- print(f"Error: '{sys.argv[1]}' table not found.")
- return retval
+ except json.JSONDecodeError as err:
+ print("{0}: cannot decode data file: {1}".format(filename, err))
+ return None
+ except ValueError as err:
+ print("{0}: {1}".format(filename, err))
+ return None
-root_path = find_vcs_root()
+ return table
-# Preliminary checks:
-if root_path is None:
- print('Repository not found. Script must be in a repo subfolder.')
-elif len(sys.argv) != 3:
- print("""\nUsage: po_shortcuts.py <LANGUAGE> <FOLDER>\n
- Examples: po_shortcuts.py es ~/test
- po_shortcuts.py fr d:\\test\n
- Substitution table needed in 'po_shortcuts_tables.json'.
- Script and tables file must be in the same folder.\n
- Output files will be created in 'new_locale' subfolder
- inside the provided folder in your computer.""")
-elif not os.path.isdir(os.path.join(root_path, 'locale', sys.argv[1])):
- print("'<repo_root>/locale/" + sys.argv[1] + "' folder not found. "
- "Script must be in a folder inside the repository.")
-elif not os.path.isfile('po_shortcuts_tables.json'):
- print("'po_shortcuts_tables.json' file not found. "
- "Script and tables file must be in the same folder.")
-elif not os.path.isdir(os.path.expanduser(sys.argv[2])):
- print("'" + sys.argv[2] + "' folder not found.")
-else: # All OK
- # Reading substitution table. It will be stored as a list of tuples
- # (regex object, replace string) in function attribute 'table' of
- # msgstr_replace() function:
- with open('po_shortcuts_tables.json', 'rt') as ftab:
- table = json.load(ftab, object_pairs_hook=json_parse)
- if table: # table found and no duplicates?
- msgstr_replace.table = []
- for src, dst in table:
- pattern = r'(' + ROLE + r')("\n")?(`[^`]*?)\b' + src + r'\b([^`]*?`)'
- replace = r'\1\2\3' + dst + r'\4'
- msgstr_replace.table.append((re.compile(pattern, re.MULTILINE),
- replace))
- # Main loop:
- for wpath, wdirs, wfiles in os.walk(os.path.join(root_path, 'locale',
- sys.argv[1],
- 'LC_MESSAGES')):
- for fname in wfiles:
- path_full = os.path.join(wpath, fname)
- if path_full[-3:] == '.po':
- file_process(path_full)
+
+def main(lang):
+ # todo all available langs for admins
+
+ lang_dir = os.path.join(PO_DIR, lang)
+ if not os.path.exists(lang_dir):
+ print("Language folder not found:", lang)
+ return
+
+ table = read_json(lang)
+ if not table:
+ return
+
+ table_compiled = []
+ for key, value in table:
+ pattern_str = r'(\:' + ROLE + r'\:["\s]*?`[^`]*?)\b' + key + r'\b([^`]*?`)'
+ replace = r'\1' + value + r'\2'
+ table_compiled.append((re.compile(pattern_str, re.MULTILINE),
+ replace))
+
+ for filename in po_files(lang_dir):
+ with open(filename, 'r', encoding="utf-8") as f:
+ text_src = f.read()
+
+ text_dst = []
+ last_end = 0
+ n_total = 0
+ for msgstr, start in parse_po(text_src):
+ text_dst.append(text_src[last_end:start])
+ last_end = start + len(msgstr)
+
+ for pattern, repl in table_compiled:
+ msgstr, n = re.subn(pattern, repl, msgstr)
+ n_total += n
+
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-docboard-svn
mailing list