From a69fd4e1126083eaf4e7377aae7805df0e0fada3 Mon Sep 17 00:00:00 2001 From: cxl Date: Mon, 5 Jun 2017 15:44:51 +0000 Subject: [PATCH] Core: Utf (8, 16, 32) support git-svn-id: svn://ultimatepp.org/upp/trunk@11139 f0d560ea-af0d-0410-9eb7-867de7ffcac7 --- uppsrc/Core/CharSet.cpp | 43 ------------- uppsrc/Core/CharSet.h | 71 ++++++++++++++++++-- uppsrc/Core/Core.h | 3 +- uppsrc/Core/Core.upp | 2 + uppsrc/Core/Utf.cpp | 139 ++++++++++++++++++++++++++++++++++++++++ uppsrc/Core/Utf.hpp | 109 +++++++++++++++++++++++++++++++ 6 files changed, 318 insertions(+), 49 deletions(-) create mode 100644 uppsrc/Core/Utf.cpp create mode 100644 uppsrc/Core/Utf.hpp diff --git a/uppsrc/Core/CharSet.cpp b/uppsrc/Core/CharSet.cpp index a2d753a91..7b17c0f9c 100644 --- a/uppsrc/Core/CharSet.cpp +++ b/uppsrc/Core/CharSet.cpp @@ -2420,34 +2420,6 @@ int lenAsUtf8(const wchar *s, int len) return len; } -String ToUtf8(const wchar *s, int len) -{ - const wchar *lim = s + len; - int tlen = lenAsUtf8(s, len); - StringBuffer result(tlen); - char *t = result; - while(s < lim) { - word code = *s++; - if(code < 0x80) - *t++ = (char)code; - else - if(code < 0x800) { - *t++ = 0xc0 | (code >> 6); - *t++ = 0x80 | (code & 0x3f); - } - else - if((code & 0xFF00) == 0xEE00) - *t++ = (char) code; - else { - *t++ = 0xe0 | (code >> 12); - *t++ = 0x80 | ((code >> 6) & 0x3f); - *t++ = 0x80 | (code & 0x3f); - } - } - ASSERT(t - ~result == tlen); - return result; -} - int utf8len(const char *s) { return utf8len(s, (int)strlen(s)); @@ -2492,21 +2464,6 @@ String ToUtf8(wchar code) return ToUtf8(&code, 1); } -String ToUtf8(const WString& w) -{ - return ToUtf8(w, w.GetLength()); -} - -String ToUtf8(const wchar *s) -{ - return ToUtf8(s, wstrlen(s)); -} - -bool CheckUtf8(const String& src) -{ - return utf8check(~src, src.GetLength()); -} - WString FromUtf8(const char *s) { return FromUtf8(s, (int)strlen(s)); diff --git a/uppsrc/Core/CharSet.h b/uppsrc/Core/CharSet.h index 1ae2a1521..d1fdedd67 100644 --- a/uppsrc/Core/CharSet.h +++ b/uppsrc/Core/CharSet.h @@ -25,6 +25,72 @@ enum { #endif +#include "Utf.hpp" + +int strlen32(const dword *s); + +bool CheckUtf8(const char *s, int len); +inline bool CheckUtf8(const char *s) { return CheckUtf8(s, strlen(s)); } +inline bool CheckUtf8(const String& s) { return CheckUtf8(~s, s.GetCount()); } + +int Utf8Len(const dword *s, int len); +inline int Utf8Len(const dword *s) { return Utf8Len(s, strlen32(s)); } +inline int Utf8Len(const Vector& s) { return Utf8Len(s, s.GetCount()); } +inline int Utf8Len(dword code) { return Utf8Len(&code, 1); } + +void ToUtf8(char *t, const dword *s, int len); +String ToUtf8(const dword *s, int len); +inline String ToUtf8(const dword *s) { return ToUtf8(s, strlen32(s)); } +inline String ToUtf8(const Vector& s) { return ToUtf8(s, s.GetCount()); } +inline String ToUtf8(dword code) { return ToUtf8(&code, 1); } + +int Utf8Len(const wchar *s, int len); +inline int Utf8Len(const wchar *s) { return Utf8Len(s, wstrlen(s)); } +inline int Utf8Len(const WString& s) { return Utf8Len(~s, s.GetCount()); } + +void ToUtf8(char *t, const wchar *s, int len); +String ToUtf8(const wchar *s, int len); +inline String ToUtf8(const wchar *s) { return ToUtf8(s, wstrlen(s)); } +inline String ToUtf8(const WString& s) { return ToUtf8(~s, s.GetCount()); } + +int Utf16Len(const dword *s, int len); +inline int Utf16Len(const dword *s) { return Utf16Len(s, strlen32(s)); } +inline int Utf16Len(const Vector& s) { return Utf16Len(s, s.GetCount()); } +inline int Utf16Len(dword code) { return Utf16Len(&code, 1); } + +void ToUtf16(wchar *t, const dword *s, int len); +WString ToUtf16(const dword *s, int len); +inline WString ToUtf16(const dword *s) { return ToUtf16(s, strlen32(s)); } +inline WString ToUtf16(const Vector& s) { return ToUtf16(s, s.GetCount()); } +inline WString ToUtf16(dword code) { return ToUtf16(&code, 1); } + +int Utf16Len(const char *s, int len); +inline int Utf16Len(const char *s) { return Utf16Len(s, strlen(s)); } +inline int Utf16Len(const String& s) { return Utf16Len(~s, s.GetCount()); } + +void ToUtf16(wchar *t, const char *s, int len); +WString ToUtf16(const char *s, int len); +inline WString ToUtf16(const char *s) { return ToUtf16(s, strlen(s)); } +inline WString ToUtf16(const String& s) { return ToUtf16(~s, s.GetCount()); } + +int Utf32Len(const wchar *s, int len); +inline int Utf32Len(const wchar *s) { return Utf32Len(s, wstrlen(s)); } +inline int Utf32Len(const WString& s) { return Utf32Len(~s, s.GetCount()); } + +void ToUtf32(dword *t, const wchar *s, int len); +Vector ToUtf32(const wchar *s, int len); +inline Vector ToUtf32(const wchar *s) { return ToUtf32(s, wstrlen(s)); } +inline Vector ToUtf32(const WString& s) { return ToUtf32(~s, s.GetCount()); } + +int Utf32Len(const char *s, int len); +inline int Utf32Len(const char *s) { return Utf32Len(s, strlen(s)); } +inline int Utf32Len(const String& s) { return Utf32Len(~s, s.GetCount()); } + +void ToUtf32(dword *t, const char *s, int len); +Vector ToUtf32(const char *s, int len); +inline Vector ToUtf32(const char *s) { return ToUtf32(s, strlen(s)); } +inline Vector ToUtf32(const String& s) { return ToUtf32(~s, s.GetCount()); } + void SetDefaultCharset(byte charset); byte GetDefaultCharset(); @@ -50,11 +116,6 @@ inline bool IsUtf8Lead(int c) return (c & 0xc0) != 0x80; } -String ToUtf8(int code); -String ToUtf8(const wchar *s, int len); -String ToUtf8(const wchar *s); -String ToUtf8(const WString& w); - WString FromUtf8(const char *_s, int len); WString FromUtf8(const char *_s); WString FromUtf8(const String& s); diff --git a/uppsrc/Core/Core.h b/uppsrc/Core/Core.h index aa2d32776..c2d1e38dc 100644 --- a/uppsrc/Core/Core.h +++ b/uppsrc/Core/Core.h @@ -270,7 +270,6 @@ class JsonIO; #include "Mt.h" #include "String.h" -#include "CharSet.h" #include "TimeDate.h" #include "Path.h" #include "Stream.h" @@ -288,6 +287,8 @@ class JsonIO; #include "FixedMap.h" #include "InVector.h" +#include "CharSet.h" + #include "SplitMerge.h" #include "Other.h" diff --git a/uppsrc/Core/Core.upp b/uppsrc/Core/Core.upp index 6fc5dca13..20403d697 100644 --- a/uppsrc/Core/Core.upp +++ b/uppsrc/Core/Core.upp @@ -63,6 +63,8 @@ file SplitMerge.cpp, CharSet.i, CharSet.h, + Utf.hpp, + Utf.cpp, CharSet.cpp, Bom.cpp, Path.h, diff --git a/uppsrc/Core/Utf.cpp b/uppsrc/Core/Utf.cpp new file mode 100644 index 000000000..64380a171 --- /dev/null +++ b/uppsrc/Core/Utf.cpp @@ -0,0 +1,139 @@ +#include "Core.h" + +namespace Upp { + +int strlen32(const dword *s) +{ + const dword *s0 = s; + while(*s) s++; + return s - s0; +} + +bool CheckUtf8(const char *s, int len) +{ + return FromUtf8_([](const byte *, dword){}, s, len); +} + +int Utf8Len(const dword *s, int len) +{ + int rlen = 0; + for(const dword *lim = s + len; s < lim; s++) + ToUtf8_([&](char) { rlen++; }, *s); + return rlen; +} + +void ToUtf8(char *t, const dword *s, int len) +{ + for(const dword *lim = s + len; s < lim; s++) + ToUtf8_([&](char c) { *t++ = c; }, *s); +} + +String ToUtf8(const dword *s, int len) +{ + String r; + for(const dword *lim = s + len; s < lim; s++) + ToUtf8_([&](char c) { r.Cat(c); }, *s); + return r; +} + +int Utf8Len(const wchar *s, int len) +{ + int rlen = 0; + FromUtf16_([&](const wchar *, dword code) { ToUtf8_([&](char c) { rlen++; }, code); }, s, len); + return rlen; +} + +void ToUtf8(char *t, const wchar *s, int len) +{ + FromUtf16_([&](const wchar *, dword code) { ToUtf8_([&](char c) { *t++ = c; }, code); }, s, len); +} + +String ToUtf8(const wchar *s, int len) +{ + StringBuffer r; + r.Reserve(len); + FromUtf16_([&](const wchar *, dword code) { ToUtf8_([&](char c) { r.Cat(c); }, code); }, s, len); + return r; +} + +int Utf16Len(const dword *s, int len) +{ + int rlen = 0; + for(const dword *lim = s + len; s < lim; s++) + ToUtf16_([&](wchar) { rlen++; }, *s); + return rlen; +} + +void ToUtf16(wchar *t, const dword *s, int len) +{ + for(const dword *lim = s + len; s < lim; s++) + ToUtf16_([&](wchar c) { *t++ = c; }, *s); +} + +WString ToUtf16(const dword *s, int len) +{ + WStringBuffer r; + r.Reserve(len); + for(const dword *lim = s + len; s < lim; s++) + ToUtf16_([&](wchar c) { r.Cat(c); }, *s); + return r; +} + +int Utf16Len(const char *s, int len) +{ + int rlen = 0; + FromUtf8_([&](const byte *, dword code) { ToUtf16_([&](wchar c) { rlen++; }, code); }, s, len); + return rlen; +} + +void ToUtf16(wchar *t, const char *s, int len) +{ + FromUtf8_([&](const byte *, dword code) { ToUtf16_([&](wchar c) { *t++ = c; }, code); }, s, len); +} + +WString ToUtf16(const char *s, int len) +{ + WStringBuffer r; + FromUtf8_([&](const byte *, dword code) { ToUtf16_([&](wchar c) { r.Cat(c); }, code); }, s, len); + return r; +} + +int Utf32Len(const char *s, int len) +{ + int rlen = 0; + FromUtf8_([&](const byte *, dword) { rlen++; }, s, len); + return rlen; +} + +void ToUtf32(dword *t, const char *s, int len) +{ + FromUtf8_([&](const byte *, dword c) { *t++ = c; }, s, len); +} + +Vector ToUtf32(const char *s, int len) +{ + Vector r; + FromUtf8_([&](const byte *, dword c) { r.Add(c); }, s, len); + return r; +} + +int Utf32Len(const wchar *s, int len) +{ + int rlen = 0; + FromUtf16_([&](const wchar *, dword) { rlen++; }, s, len); + return rlen; +} + +void ToUtf32(dword *t, const wchar *s, int len) +{ + FromUtf16_([&](const wchar *, dword c) { *t++ = c; }, s, len); +} + +Vector ToUtf32(const wchar *s, int len) +{ + Vector r; + FromUtf16_([&](const wchar *, dword c) { r.Add(c); }, s, len); + return r; +} + +}; diff --git a/uppsrc/Core/Utf.hpp b/uppsrc/Core/Utf.hpp new file mode 100644 index 000000000..acc69b87b --- /dev/null +++ b/uppsrc/Core/Utf.hpp @@ -0,0 +1,109 @@ +template +force_inline bool ToUtf8_(Target &t, dword codepoint) +{ + if(codepoint < 0x80) + t((char)codepoint); + else + if(codepoint < 0x800) { + t(0xc0 | byte(codepoint >> 6)); + t(0x80 | byte(codepoint & 0x3f)); + } + else + if((codepoint & 0xFFFFFF00) == 0xEE00) // ERROR ESCAPE + t((char) codepoint); + else + if(codepoint < 0x10000) { + t(0xe0 | byte(codepoint >> 12)); + t(0x80 | byte((codepoint >> 6) & 0x3f)); + t(0x80 | byte(codepoint & 0x3f)); + } + else + if(codepoint < 0x110000) { + t(0xf0 | byte(codepoint >> 18)); + t(0x80 | byte((codepoint >> 12) & 0x3f)); + t(0x80 | byte((codepoint >> 6) & 0x3f)); + t(0x80 | byte(codepoint & 0x3f)); + } + else + return false; + return true; +} + +template +force_inline bool FromUtf8_(Target& t, const char *_s, int len) +{ + bool ok = true; + const byte *s = (const byte *)_s; + const byte *lim = s + len; + while(s < lim) { + dword code = *s; + if(code < 0x80) { + t(s++, code); + continue; + } + else + if(code >= 0xC2) { + dword c; + if(code < 0xE0 && s + 1 < lim && + s[1] >= 0x80 && s[1] < 0xc0 && + (c = ((code - 0xC0) << 6) + s[1] - 0x80) >= 0x80 && c < 0x800) { + t(s, c); + s += 2; + continue; + } + else + if(code < 0xF0 && s + 2 < lim && + s[1] >= 0x80 && s[1] < 0xc0 && s[2] >= 0x80 && s[2] < 0xc0 && + (c = ((code - 0xE0) << 12) + ((s[1] - 0x80) << 6) + s[2] - 0x80) >= 0x800 && + !(c >= 0xEE00 && c <= 0xEEFF)) { + t(s, c); + s += 3; + continue; + } + else + if(code < 0xF8 && s + 3 < lim && + s[1] >= 0x80 && s[1] < 0xc0 && s[2] >= 0x80 && s[2] < 0xc0 && s[3] >= 0x80 && s[3] < 0xc0 && + (c = ((code - 0xF0) << 18) + ((s[1] - 0x80) << 12) + ((s[2] - 0x80) << 6) + s[3] - 0x80) >= 0x10000 && + c < 0x110000) { + t(s, c); + s += 4; + continue; + } + } + t(s++, 0xEE00 + code); + ok = false; + } + return ok; +} + +template +force_inline bool ToUtf16_(Target &t, int codepoint) +{ + if(codepoint < 0x10000) + t((wchar)codepoint); + else + if(codepoint < 0x110000) { + codepoint -= 0x10000; + t(wchar(0xD800 + (0x3ff & (codepoint >> 10)))); + t(wchar(0xDC00 + (0x3ff & codepoint))); + } + else + return false; + return true; +} + +template +force_inline void FromUtf16_(Target& t, const wchar *s, int len) +{ + const wchar *lim = s + len; + while(s < lim) { + if((*s & 0XFC00) == 0xD800 && s + 1 < lim && (s[1] & 0xFC00) == 0xDC00) { + t(s, ((dword(s[0] & 0x3ff) << 10) | (s[1] & 0x3ff)) + 0x10000); + s += 2; + } + else { + t((const wchar *)s, *s); + s++; + } + } +}