From bfdc0cd44ea4551582a0e19978719dbd3b028017 Mon Sep 17 00:00:00 2001 From: cxl Date: Thu, 27 Jul 2017 12:38:51 +0000 Subject: [PATCH] Core: JSON now treats string codepoints >0x8000 as surrogate pairs git-svn-id: svn://ultimatepp.org/upp/trunk@11269 f0d560ea-af0d-0410-9eb7-867de7ffcac7 --- uppsrc/Core/CharSet.h | 3 ++ uppsrc/Core/JSON.cpp | 4 +- uppsrc/Core/Utf.hpp | 79 ++++++++++++++++--------------- uppsrc/Core/parser.cpp | 61 ++++++++++++++++-------- uppsrc/Core/src.tpp/Utf$en-us.tpp | 13 +++++ 5 files changed, 100 insertions(+), 60 deletions(-) diff --git a/uppsrc/Core/CharSet.h b/uppsrc/Core/CharSet.h index 5c8d8fc7f..f193d5af1 100644 --- a/uppsrc/Core/CharSet.h +++ b/uppsrc/Core/CharSet.h @@ -34,6 +34,9 @@ inline bool IsUtf8Lead(int c) return (c & 0xc0) != 0x80; } +dword FetchUtf8(const char *&s, const char *lim, bool& ok); +inline dword FetchUtf8(const char *&s, const char *lim) { bool ok; return FetchUtf8(s, lim, ok); } + bool CheckUtf8(const char *s, int len); inline bool CheckUtf8(const char *s) { return CheckUtf8(s, (int)strlen(s)); } inline bool CheckUtf8(const String& s) { return CheckUtf8(~s, s.GetCount()); } diff --git a/uppsrc/Core/JSON.cpp b/uppsrc/Core/JSON.cpp index a6f80ca68..028242d84 100644 --- a/uppsrc/Core/JSON.cpp +++ b/uppsrc/Core/JSON.cpp @@ -78,7 +78,7 @@ Json& Json::CatRaw(const char *key, const String& val) if(text.GetCount()) text << ','; text << AsJSON(key) << ":" << val; - return *this; + return *this; } JsonArray& JsonArray::CatRaw(const String& val) @@ -86,7 +86,7 @@ JsonArray& JsonArray::CatRaw(const String& val) if(text.GetCount()) text << ','; text << val; - return *this; + return *this; } String AsJSON(const Value& v, const String& sep, bool pretty) diff --git a/uppsrc/Core/Utf.hpp b/uppsrc/Core/Utf.hpp index efe158db3..ebeee9339 100644 --- a/uppsrc/Core/Utf.hpp +++ b/uppsrc/Core/Utf.hpp @@ -29,50 +29,53 @@ force_inline bool ToUtf8_(Target t, dword codepoint) return true; } -template -force_inline bool FromUtf8_(Target t, const char *_s, size_t len) +force_inline dword FetchUtf8(const char *&_s, const char *_lim, bool& ok) { - bool ok = true; const byte *s = (const byte *)_s; - const byte *lim = s + len; - while(s < lim) { - dword code = *s; - if(code < 0x80) { - t(s++, code); - continue; + const byte *lim = (const byte *)_lim; + dword code = *s; + if(code < 0x80) { + _s++; + return *s; + } + else + if(code >= 0xC2) { + dword c; + if(code < 0xE0 && s + 1 < lim && + s[1] >= 0x80 && s[1] < 0xc0 && + (c = ((code - 0xC0) << 6) + s[1] - 0x80) >= 0x80 && c < 0x800) { + _s += 2; + return c; } else - if(code >= 0xC2) { - dword c; - if(code < 0xE0 && s + 1 < lim && - s[1] >= 0x80 && s[1] < 0xc0 && - (c = ((code - 0xC0) << 6) + s[1] - 0x80) >= 0x80 && c < 0x800) { - t(s, c); - s += 2; - continue; - } - else - if(code < 0xF0 && s + 2 < lim && - s[1] >= 0x80 && s[1] < 0xc0 && s[2] >= 0x80 && s[2] < 0xc0 && - (c = ((code - 0xE0) << 12) + ((s[1] - 0x80) << 6) + s[2] - 0x80) >= 0x800 && - !(c >= 0xEE00 && c <= 0xEEFF)) { - t(s, c); - s += 3; - continue; - } - else - if(code < 0xF8 && s + 3 < lim && - s[1] >= 0x80 && s[1] < 0xc0 && s[2] >= 0x80 && s[2] < 0xc0 && s[3] >= 0x80 && s[3] < 0xc0 && - (c = ((code - 0xF0) << 18) + ((s[1] - 0x80) << 12) + ((s[2] - 0x80) << 6) + s[3] - 0x80) >= 0x10000 && - c < 0x110000) { - t(s, c); - s += 4; - continue; - } + if(code < 0xF0 && s + 2 < lim && + s[1] >= 0x80 && s[1] < 0xc0 && s[2] >= 0x80 && s[2] < 0xc0 && + (c = ((code - 0xE0) << 12) + ((s[1] - 0x80) << 6) + s[2] - 0x80) >= 0x800 && + !(c >= 0xEE00 && c <= 0xEEFF)) { + _s += 3; + return c; + } + else + if(code < 0xF8 && s + 3 < lim && + s[1] >= 0x80 && s[1] < 0xc0 && s[2] >= 0x80 && s[2] < 0xc0 && s[3] >= 0x80 && s[3] < 0xc0 && + (c = ((code - 0xF0) << 18) + ((s[1] - 0x80) << 12) + ((s[2] - 0x80) << 6) + s[3] - 0x80) >= 0x10000 && + c < 0x110000) { + _s += 4; + return c; } - t(s++, 0xEE00 + code); - ok = false; } + _s++; + ok = false; + return 0xEE00 + code; // ERROR ESCAPE +} + +template +force_inline bool FromUtf8_(Target t, const char *s, size_t len) +{ + bool ok = true; + const char *lim = s + len; + while(s < lim) + t((const byte *)s, FetchUtf8(s, lim, ok)); return ok; } diff --git a/uppsrc/Core/parser.cpp b/uppsrc/Core/parser.cpp index 784a98b4b..858f43a50 100644 --- a/uppsrc/Core/parser.cpp +++ b/uppsrc/Core/parser.cpp @@ -583,7 +583,7 @@ void CParser::Set(const char *_ptr) Set(_ptr, "", 1); } -inline void NextCStringLine(String& t, const char *linepfx, int& pl) +inline void NextCStringLine(StringBuffer& t, const char *linepfx, int& pl) { t << "\"\r\n" << (linepfx ? linepfx : "") << "\""; pl = t.GetLength(); @@ -594,16 +594,30 @@ inline int HexDigit(int c) return "0123456789ABCDEF"[c & 15]; } +static inline void sCatHex(StringBuffer& t, word q) +{ + char h[6]; + h[0] = '\\'; + h[1] = 'u'; + h[2] = HexDigit(q >> 12); + h[3] = HexDigit(q >> 8); + h[4] = HexDigit(q >> 4); + h[5] = HexDigit(q); + t.Cat(h, 6); +} + String AsCString(const char *s, const char *lim, int linemax, const char *linepfx, dword flags) { - String t; + StringBuffer t; t.Cat('\"'); int pl = 0; bool wasspace = false; + byte cs = GetDefaultCharset(); + bool toutf8 = GetDefaultCharset() != CHARSET_UTF8; while(s < lim) { if(t.GetLength() - pl > linemax && (!(flags & ASCSTRING_SMART) || wasspace)) NextCStringLine(t, linepfx, pl); - wasspace = false; + wasspace = *s == ' '; switch(*s) { case '\a': t.Cat("\\a"); break; case '\b': t.Cat("\\b"); break; @@ -615,19 +629,29 @@ String AsCString(const char *s, const char *lim, int linemax, const char *linepf case '\\': t.Cat("\\\\"); break; case '\n': t.Cat("\\n"); wasspace = true; break; default: - if(byte(*s) < 32 || (byte)*s >= 0x7f && (flags & ASCSTRING_OCTALHI) || (byte)*s == 0xff || (byte)*s == 0x7f) { - if(flags & ASCSTRING_JSON) { - char h[6]; - int q = (byte)*s; - h[0] = '\\'; - h[1] = 'u'; - h[2] = '0'; - h[3] = '0'; - h[4] = HexDigit(q >> 4); - h[5] = HexDigit(q); - t.Cat(h, 6); + if(flags & ASCSTRING_JSON) { + if((byte)*s < 32) { + sCatHex(t, (byte)*s++); } - else { + else + if((byte)*s >= 0x7f) { + const char *s0 = s; + dword c = toutf8 ? ToUnicode((byte)*s++, cs) : FetchUtf8(s, lim); + if(c < 0x10000) + t.Cat(s0, s); + else { + WString h = ToUtf16(c); + ASSERT(h.GetCount() == 2); + sCatHex(t, h[0]); + sCatHex(t, h[1]); + } + } + else + t.Cat(*s++); + continue; // skip s++ + } + else { + if(byte(*s) < 32 || (byte)*s >= 0x7f && (flags & ASCSTRING_OCTALHI) || (byte)*s == 0xff || (byte)*s == 0x7f) { char h[4]; int q = (byte)*s; h[0] = '\\'; @@ -636,11 +660,8 @@ String AsCString(const char *s, const char *lim, int linemax, const char *linepf h[3] = (7 & q) + '0'; t.Cat(h, 4); } - } - else { - t.Cat(*s); - if(*s == ' ') - wasspace = true; + else + t.Cat(*s); } break; } diff --git a/uppsrc/Core/src.tpp/Utf$en-us.tpp b/uppsrc/Core/src.tpp/Utf$en-us.tpp index 608a8b120..d7c8d468e 100644 --- a/uppsrc/Core/src.tpp/Utf$en-us.tpp +++ b/uppsrc/Core/src.tpp/Utf$en-us.tpp @@ -16,6 +16,19 @@ topic ""; [s2;%% Tests whether [%-*@3 c ]is lead UTF`-8 byte.&] [s3;%% &] [s4; &] +[s5;:Upp`:`:FetchUtf8`(const char`*`&`,const char`*`,bool`&`): [_^Upp`:`:dword^ dword]_ +[* FetchUtf8]([@(0.0.255) const]_[@(0.0.255) char]_`*`&[*@3 s], [@(0.0.255) const]_[@(0.0.255) c +har]_`*[*@3 lim], [@(0.0.255) bool`&]_[*@3 ok])&] +[s5;:Upp`:`:FetchUtf8`(const char`*`&`,const char`*`): [_^Upp`:`:dword^ dword]_[* FetchUt +f8]([@(0.0.255) const]_[@(0.0.255) char]_`*`&[*@3 s], [@(0.0.255) const]_[@(0.0.255) char]_ +`*[*@3 lim])&] +[s2;%% Reads a single UTF`-32 codepoint from UTF`-8 string [%-*@3 s] +with end at [%-*@3 lim]. [%-*@3 s] must be less than [%-*@3 lim]. [%-*@3 s] +is advanced accordingly. [%-*@3 ok] is set to false if UTF`-8 is +invalid `- in that case, error`-escape of single byte is returned +(but it is NOT set to true if valid UTF`-8 character is read).&] +[s3;%% &] +[s4; &] [s5;:Upp`:`:CheckUtf8`(const char`*`,int`): [@(0.0.255) bool]_[* CheckUtf8]([@(0.0.255) con st]_[@(0.0.255) char]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] [s5;:Upp`:`:CheckUtf8`(const char`*`): [@(0.0.255) bool]_[* CheckUtf8]([@(0.0.255) const]_[@(0.0.255) c