[Bf-blender-cvs] SVN commit: /data/svn/bf-blender [41162] trunk/blender/source/blender/ blenlib: replace own unicode functions with versions from glib which support more unicode characters .
Campbell Barton
ideasman42 at gmail.com
Fri Oct 21 02:01:26 CEST 2011
Revision: 41162
http://projects.blender.org/scm/viewvc.php?view=rev&root=bf-blender&revision=41162
Author: campbellbarton
Date: 2011-10-21 00:01:22 +0000 (Fri, 21 Oct 2011)
Log Message:
-----------
replace own unicode functions with versions from glib which support more unicode characters.
added BLI_str_utf8_as_unicode(), BLI_str_utf8_from_unicode()
Modified Paths:
--------------
trunk/blender/source/blender/blenlib/BLI_string_utf8.h
trunk/blender/source/blender/blenlib/intern/string_utf8.c
Modified: trunk/blender/source/blender/blenlib/BLI_string_utf8.h
===================================================================
--- trunk/blender/source/blender/blenlib/BLI_string_utf8.h 2011-10-20 20:57:29 UTC (rev 41161)
+++ trunk/blender/source/blender/blenlib/BLI_string_utf8.h 2011-10-21 00:01:22 UTC (rev 41162)
@@ -33,21 +33,28 @@
extern "C" {
#endif
-char *BLI_strncpy_utf8(char *dst, const char *src, size_t maxncpy);
-int BLI_utf8_invalid_byte(const char *str, int length);
-int BLI_utf8_invalid_strip(char *str, int length);
+char *BLI_strncpy_utf8(char *dst, const char *src, size_t maxncpy);
+int BLI_utf8_invalid_byte(const char *str, int length);
+int BLI_utf8_invalid_strip(char *str, int length);
/* copied from glib */
-char *BLI_str_find_prev_char_utf8(const char *str, const char *p);
-char *BLI_str_find_next_char_utf8(const char *p, const char *end);
-char *BLI_str_prev_char_utf8(const char *p);
+unsigned int BLI_str_utf8_as_unicode(const char *p);
+unsigned int BLI_str_utf8_as_unicode_and_size(const char *p, size_t *index);
+unsigned int BLI_str_utf8_as_unicode_step(const char *p, size_t *index);
+size_t BLI_str_utf8_from_unicode(unsigned int c, char *outbuf);
+char *BLI_str_find_prev_char_utf8(const char *str, const char *p);
+char *BLI_str_find_next_char_utf8(const char *p, const char *end);
+char *BLI_str_prev_char_utf8(const char *p);
+
/* wchar_t functions, copied from blenders own font.c originally */
-size_t BLI_wstrlen_utf8(const wchar_t *src);
-size_t BLI_strlen_utf8(const char *strc);
-size_t BLI_strncpy_wchar_as_utf8(char *dst, const wchar_t *src, const size_t maxcpy);
-size_t BLI_strncpy_wchar_from_utf8(wchar_t *dst, const char *src, const size_t maxcpy);
+size_t BLI_wstrlen_utf8(const wchar_t *src);
+size_t BLI_strlen_utf8(const char *strc);
+size_t BLI_strncpy_wchar_as_utf8(char *dst, const wchar_t *src, const size_t maxcpy);
+size_t BLI_strncpy_wchar_from_utf8(wchar_t *dst, const char *src, const size_t maxcpy);
+#define BLI_STRING_MAX_UTF8 6
+
#ifdef __cplusplus
}
#endif
Modified: trunk/blender/source/blender/blenlib/intern/string_utf8.c
===================================================================
--- trunk/blender/source/blender/blenlib/intern/string_utf8.c 2011-10-20 20:57:29 UTC (rev 41161)
+++ trunk/blender/source/blender/blenlib/intern/string_utf8.c 2011-10-21 00:01:22 UTC (rev 41162)
@@ -34,7 +34,7 @@
#include <wchar.h>
#include <wctype.h>
-#include "BLI_string.h"
+#include "BLI_string_utf8.h"
/* from libswish3, originally called u8_isvalid(),
* modified to return the index of the bad character (byte index not utf).
@@ -189,45 +189,11 @@
/* --------------------------------------------------------------------------*/
/* wchar_t / utf8 functions */
-/* UTF-8 <-> wchar transformations */
-static size_t chtoutf8(const unsigned long c, char o[4])
-{
- // Variables and initialization
-/* memset(o, 0, 4); */
-
- // Create the utf-8 string
- if (c < 0x80) {
- o[0] = (char) c;
- return 1;
- }
- else if (c < 0x800) {
- o[0] = (0xC0 | (c>>6));
- o[1] = (0x80 | (c & 0x3f));
- return 2;
- }
- else if (c < 0x10000) {
- o[0] = (0xe0 | (c >> 12));
- o[1] = (0x80 | (c >>6 & 0x3f));
- o[2] = (0x80 | (c & 0x3f));
- return 3;
- }
- else if (c < 0x200000) {
- o[0] = (0xf0 | (c>>18));
- o[1] = (0x80 | (c >>12 & 0x3f));
- o[2] = (0x80 | (c >> 6 & 0x3f));
- o[3] = (0x80 | (c & 0x3f));
- return 4;
- }
-
- /* should we assert here? */
- return 0;
-}
-
size_t BLI_strncpy_wchar_as_utf8(char *dst, const wchar_t *src, const size_t maxcpy)
{
size_t len = 0;
while(*src && len < maxcpy) { /* XXX can still run over the buffer because utf8 size isnt known :| */
- len += chtoutf8(*src++, dst+len);
+ len += BLI_str_utf8_from_unicode(*src++, dst+len);
}
dst[len]= '\0';
@@ -238,11 +204,10 @@
/* wchar len in utf8 */
size_t BLI_wstrlen_utf8(const wchar_t *src)
{
- char ch_dummy[4];
size_t len = 0;
while(*src) {
- len += chtoutf8(*src++, ch_dummy);
+ len += BLI_str_utf8_from_unicode(*src++, NULL);
}
return len;
@@ -272,26 +237,6 @@
return len;
}
-
-/* Converts Unicode to wchar
-
-According to RFC 3629 "UTF-8, a transformation format of ISO 10646"
-(http://tools.ietf.org/html/rfc3629), the valid UTF-8 encoding are:
-
- Char. number range | UTF-8 octet sequence
- (hexadecimal) | (binary)
- --------------------+---------------------------------------------
- 0000 0000-0000 007F | 0xxxxxxx
- 0000 0080-0000 07FF | 110xxxxx 10xxxxxx
- 0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
- 0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
-
-If the encoding incidated by the first character is incorrect (because the
-1 to 3 following characters do not match 10xxxxxx), the output is a '?' and
-only a single input character is consumed.
-
-*/
-
size_t BLI_strncpy_wchar_from_utf8(wchar_t *dst_w, const char *src_c, const size_t maxcpy)
{
int len=0;
@@ -299,32 +244,16 @@
if(dst_w==NULL || src_c==NULL) return(0);
while(*src_c && len < maxcpy) {
- if ((*src_c & 0xe0) == 0xc0) {
- if((src_c[1] & 0x80) && (src_c[1] & 0x40) == 0x00) {
- *dst_w=((src_c[0] &0x1f)<<6) | (src_c[1]&0x3f);
- src_c++;
- } else {
- *dst_w = '?';
- }
- } else if ((*src_c & 0xf0) == 0xe0) {
- if((src_c[1] & src_c[2] & 0x80) && ((src_c[1] | src_c[2]) & 0x40) == 0x00) {
- *dst_w=((src_c[0] & 0x0f)<<12) | ((src_c[1]&0x3f)<<6) | (src_c[2]&0x3f);
- src_c += 2;
- } else {
- *dst_w = '?';
- }
- } else if ((*src_c & 0xf8) == 0xf0) {
- if((src_c[1] & src_c[2] & src_c[3] & 0x80) && ((src_c[1] | src_c[2] | src_c[3]) & 0x40) == 0x00) {
- *dst_w=((src_c[0] & 0x07)<<18) | ((src_c[1]&0x1f)<<12) | ((src_c[2]&0x3f)<<6) | (src_c[3]&0x3f);
- src_c += 3;
- } else {
- *dst_w = '?';
- }
- } else {
- *dst_w=(src_c[0] & 0x7f);
+ size_t step= 0;
+ unsigned int unicode= BLI_str_utf8_as_unicode_and_size(src_c, &step);
+ if (unicode != (unsigned int)-1) {
+ *dst_w= (wchar_t)unicode;
+ src_c += step;
}
-
- src_c++;
+ else {
+ *dst_w = '?';
+ src_c= BLI_str_find_next_char_utf8(src_c, NULL);
+ }
dst_w++;
len++;
}
@@ -334,13 +263,178 @@
/* end wchar_t / utf8 functions */
/* --------------------------------------------------------------------------*/
+/* copied from glib */
+/* note, glib uses unsigned int for unicode, best we do the same,
+ * though we dont typedef it - campbell */
+#define UTF8_COMPUTE(Char, Mask, Len) \
+ if (Char < 128) { \
+ Len = 1; \
+ Mask = 0x7f; \
+ } \
+ else if ((Char & 0xe0) == 0xc0) { \
+ Len = 2; \
+ Mask = 0x1f; \
+ } \
+ else if ((Char & 0xf0) == 0xe0) { \
+ Len = 3; \
+ Mask = 0x0f; \
+ } \
+ else if ((Char & 0xf8) == 0xf0) { \
+ Len = 4; \
+ Mask = 0x07; \
+ } \
+ else if ((Char & 0xfc) == 0xf8) { \
+ Len = 5; \
+ Mask = 0x03; \
+ } \
+ else if ((Char & 0xfe) == 0xfc) { \
+ Len = 6; \
+ Mask = 0x01; \
+ } \
+ else { \
+ Len = -1; \
+ }
-/* copied from glib */
+#define UTF8_GET(Result, Chars, Count, Mask, Len) \
+ (Result) = (Chars)[0] & (Mask); \
+ for ((Count) = 1; (Count) < (Len); ++(Count)) { \
+ if (((Chars)[(Count)] & 0xc0) != 0x80) { \
+ (Result) = -1; \
+ break; \
+ } \
+ (Result) <<= 6; \
+ (Result) |= ((Chars)[(Count)] & 0x3f); \
+ }
+
+
+/* was g_utf8_get_char */
/**
- * g_utf8_find_prev_char:
+ * BLI_str_utf8_as_unicode:
+ * @p: a pointer to Unicode character encoded as UTF-8
+ *
+ * Converts a sequence of bytes encoded as UTF-8 to a Unicode character.
+ * If @p does not point to a valid UTF-8 encoded character, results are
+ * undefined. If you are not sure that the bytes are complete
+ * valid Unicode characters, you should use g_utf8_get_char_validated()
+ * instead.
+ *
+ * Return value: the resulting character
+ **/
+unsigned int BLI_str_utf8_as_unicode(const char *p)
+{
+ int i, mask = 0, len;
+ unsigned int result;
+ unsigned char c = (unsigned char) *p;
+
+ UTF8_COMPUTE (c, mask, len);
+ if (len == -1)
+ return (unsigned int)-1;
+ UTF8_GET (result, p, i, mask, len);
+
+ return result;
+}
+
+/* varient that increments the length */
+unsigned int BLI_str_utf8_as_unicode_and_size(const char *p, size_t *index)
+{
+ int i, mask = 0, len;
+ unsigned int result;
+ unsigned char c = (unsigned char) *p;
+
+ UTF8_COMPUTE (c, mask, len);
+ if (len == -1)
+ return (unsigned int)-1;
+ UTF8_GET (result, p, i, mask, len);
+ *index += len;
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list