mirror of
https://github.com/ultimatepp/ultimatepp.git
synced 2026-08-31 07:12:39 -06:00
Core: JSON now treats string codepoints >0x8000 as surrogate pairs
git-svn-id: svn://ultimatepp.org/upp/trunk@11269 f0d560ea-af0d-0410-9eb7-867de7ffcac7
This commit is contained in:
cxl
parent
ecf68362c0
commit
bfdc0cd44e
5 changed files with 100 additions and 60 deletions
|
|
@ -34,6 +34,9 @@ inline bool IsUtf8Lead(int c)
|
|||
return (c & 0xc0) != 0x80;
|
||||
}
|
||||
|
||||
dword FetchUtf8(const char *&s, const char *lim, bool& ok);
|
||||
inline dword FetchUtf8(const char *&s, const char *lim) { bool ok; return FetchUtf8(s, lim, ok); }
|
||||
|
||||
bool CheckUtf8(const char *s, int len);
|
||||
inline bool CheckUtf8(const char *s) { return CheckUtf8(s, (int)strlen(s)); }
|
||||
inline bool CheckUtf8(const String& s) { return CheckUtf8(~s, s.GetCount()); }
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ Json& Json::CatRaw(const char *key, const String& val)
|
|||
if(text.GetCount())
|
||||
text << ',';
|
||||
text << AsJSON(key) << ":" << val;
|
||||
return *this;
|
||||
return *this;
|
||||
}
|
||||
|
||||
JsonArray& JsonArray::CatRaw(const String& val)
|
||||
|
|
@ -86,7 +86,7 @@ JsonArray& JsonArray::CatRaw(const String& val)
|
|||
if(text.GetCount())
|
||||
text << ',';
|
||||
text << val;
|
||||
return *this;
|
||||
return *this;
|
||||
}
|
||||
|
||||
String AsJSON(const Value& v, const String& sep, bool pretty)
|
||||
|
|
|
|||
|
|
@ -29,50 +29,53 @@ force_inline bool ToUtf8_(Target t, dword codepoint)
|
|||
return true;
|
||||
}
|
||||
|
||||
template <class Target>
|
||||
force_inline bool FromUtf8_(Target t, const char *_s, size_t len)
|
||||
force_inline dword FetchUtf8(const char *&_s, const char *_lim, bool& ok)
|
||||
{
|
||||
bool ok = true;
|
||||
const byte *s = (const byte *)_s;
|
||||
const byte *lim = s + len;
|
||||
while(s < lim) {
|
||||
dword code = *s;
|
||||
if(code < 0x80) {
|
||||
t(s++, code);
|
||||
continue;
|
||||
const byte *lim = (const byte *)_lim;
|
||||
dword code = *s;
|
||||
if(code < 0x80) {
|
||||
_s++;
|
||||
return *s;
|
||||
}
|
||||
else
|
||||
if(code >= 0xC2) {
|
||||
dword c;
|
||||
if(code < 0xE0 && s + 1 < lim &&
|
||||
s[1] >= 0x80 && s[1] < 0xc0 &&
|
||||
(c = ((code - 0xC0) << 6) + s[1] - 0x80) >= 0x80 && c < 0x800) {
|
||||
_s += 2;
|
||||
return c;
|
||||
}
|
||||
else
|
||||
if(code >= 0xC2) {
|
||||
dword c;
|
||||
if(code < 0xE0 && s + 1 < lim &&
|
||||
s[1] >= 0x80 && s[1] < 0xc0 &&
|
||||
(c = ((code - 0xC0) << 6) + s[1] - 0x80) >= 0x80 && c < 0x800) {
|
||||
t(s, c);
|
||||
s += 2;
|
||||
continue;
|
||||
}
|
||||
else
|
||||
if(code < 0xF0 && s + 2 < lim &&
|
||||
s[1] >= 0x80 && s[1] < 0xc0 && s[2] >= 0x80 && s[2] < 0xc0 &&
|
||||
(c = ((code - 0xE0) << 12) + ((s[1] - 0x80) << 6) + s[2] - 0x80) >= 0x800 &&
|
||||
!(c >= 0xEE00 && c <= 0xEEFF)) {
|
||||
t(s, c);
|
||||
s += 3;
|
||||
continue;
|
||||
}
|
||||
else
|
||||
if(code < 0xF8 && s + 3 < lim &&
|
||||
s[1] >= 0x80 && s[1] < 0xc0 && s[2] >= 0x80 && s[2] < 0xc0 && s[3] >= 0x80 && s[3] < 0xc0 &&
|
||||
(c = ((code - 0xF0) << 18) + ((s[1] - 0x80) << 12) + ((s[2] - 0x80) << 6) + s[3] - 0x80) >= 0x10000 &&
|
||||
c < 0x110000) {
|
||||
t(s, c);
|
||||
s += 4;
|
||||
continue;
|
||||
}
|
||||
if(code < 0xF0 && s + 2 < lim &&
|
||||
s[1] >= 0x80 && s[1] < 0xc0 && s[2] >= 0x80 && s[2] < 0xc0 &&
|
||||
(c = ((code - 0xE0) << 12) + ((s[1] - 0x80) << 6) + s[2] - 0x80) >= 0x800 &&
|
||||
!(c >= 0xEE00 && c <= 0xEEFF)) {
|
||||
_s += 3;
|
||||
return c;
|
||||
}
|
||||
else
|
||||
if(code < 0xF8 && s + 3 < lim &&
|
||||
s[1] >= 0x80 && s[1] < 0xc0 && s[2] >= 0x80 && s[2] < 0xc0 && s[3] >= 0x80 && s[3] < 0xc0 &&
|
||||
(c = ((code - 0xF0) << 18) + ((s[1] - 0x80) << 12) + ((s[2] - 0x80) << 6) + s[3] - 0x80) >= 0x10000 &&
|
||||
c < 0x110000) {
|
||||
_s += 4;
|
||||
return c;
|
||||
}
|
||||
t(s++, 0xEE00 + code);
|
||||
ok = false;
|
||||
}
|
||||
_s++;
|
||||
ok = false;
|
||||
return 0xEE00 + code; // ERROR ESCAPE
|
||||
}
|
||||
|
||||
template <class Target>
|
||||
force_inline bool FromUtf8_(Target t, const char *s, size_t len)
|
||||
{
|
||||
bool ok = true;
|
||||
const char *lim = s + len;
|
||||
while(s < lim)
|
||||
t((const byte *)s, FetchUtf8(s, lim, ok));
|
||||
return ok;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -583,7 +583,7 @@ void CParser::Set(const char *_ptr)
|
|||
Set(_ptr, "", 1);
|
||||
}
|
||||
|
||||
inline void NextCStringLine(String& t, const char *linepfx, int& pl)
|
||||
inline void NextCStringLine(StringBuffer& t, const char *linepfx, int& pl)
|
||||
{
|
||||
t << "\"\r\n" << (linepfx ? linepfx : "") << "\"";
|
||||
pl = t.GetLength();
|
||||
|
|
@ -594,16 +594,30 @@ inline int HexDigit(int c)
|
|||
return "0123456789ABCDEF"[c & 15];
|
||||
}
|
||||
|
||||
static inline void sCatHex(StringBuffer& t, word q)
|
||||
{
|
||||
char h[6];
|
||||
h[0] = '\\';
|
||||
h[1] = 'u';
|
||||
h[2] = HexDigit(q >> 12);
|
||||
h[3] = HexDigit(q >> 8);
|
||||
h[4] = HexDigit(q >> 4);
|
||||
h[5] = HexDigit(q);
|
||||
t.Cat(h, 6);
|
||||
}
|
||||
|
||||
String AsCString(const char *s, const char *lim, int linemax, const char *linepfx, dword flags)
|
||||
{
|
||||
String t;
|
||||
StringBuffer t;
|
||||
t.Cat('\"');
|
||||
int pl = 0;
|
||||
bool wasspace = false;
|
||||
byte cs = GetDefaultCharset();
|
||||
bool toutf8 = GetDefaultCharset() != CHARSET_UTF8;
|
||||
while(s < lim) {
|
||||
if(t.GetLength() - pl > linemax && (!(flags & ASCSTRING_SMART) || wasspace))
|
||||
NextCStringLine(t, linepfx, pl);
|
||||
wasspace = false;
|
||||
wasspace = *s == ' ';
|
||||
switch(*s) {
|
||||
case '\a': t.Cat("\\a"); break;
|
||||
case '\b': t.Cat("\\b"); break;
|
||||
|
|
@ -615,19 +629,29 @@ String AsCString(const char *s, const char *lim, int linemax, const char *linepf
|
|||
case '\\': t.Cat("\\\\"); break;
|
||||
case '\n': t.Cat("\\n"); wasspace = true; break;
|
||||
default:
|
||||
if(byte(*s) < 32 || (byte)*s >= 0x7f && (flags & ASCSTRING_OCTALHI) || (byte)*s == 0xff || (byte)*s == 0x7f) {
|
||||
if(flags & ASCSTRING_JSON) {
|
||||
char h[6];
|
||||
int q = (byte)*s;
|
||||
h[0] = '\\';
|
||||
h[1] = 'u';
|
||||
h[2] = '0';
|
||||
h[3] = '0';
|
||||
h[4] = HexDigit(q >> 4);
|
||||
h[5] = HexDigit(q);
|
||||
t.Cat(h, 6);
|
||||
if(flags & ASCSTRING_JSON) {
|
||||
if((byte)*s < 32) {
|
||||
sCatHex(t, (byte)*s++);
|
||||
}
|
||||
else {
|
||||
else
|
||||
if((byte)*s >= 0x7f) {
|
||||
const char *s0 = s;
|
||||
dword c = toutf8 ? ToUnicode((byte)*s++, cs) : FetchUtf8(s, lim);
|
||||
if(c < 0x10000)
|
||||
t.Cat(s0, s);
|
||||
else {
|
||||
WString h = ToUtf16(c);
|
||||
ASSERT(h.GetCount() == 2);
|
||||
sCatHex(t, h[0]);
|
||||
sCatHex(t, h[1]);
|
||||
}
|
||||
}
|
||||
else
|
||||
t.Cat(*s++);
|
||||
continue; // skip s++
|
||||
}
|
||||
else {
|
||||
if(byte(*s) < 32 || (byte)*s >= 0x7f && (flags & ASCSTRING_OCTALHI) || (byte)*s == 0xff || (byte)*s == 0x7f) {
|
||||
char h[4];
|
||||
int q = (byte)*s;
|
||||
h[0] = '\\';
|
||||
|
|
@ -636,11 +660,8 @@ String AsCString(const char *s, const char *lim, int linemax, const char *linepf
|
|||
h[3] = (7 & q) + '0';
|
||||
t.Cat(h, 4);
|
||||
}
|
||||
}
|
||||
else {
|
||||
t.Cat(*s);
|
||||
if(*s == ' ')
|
||||
wasspace = true;
|
||||
else
|
||||
t.Cat(*s);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,19 @@ topic "";
|
|||
[s2;%% Tests whether [%-*@3 c ]is lead UTF`-8 byte.&]
|
||||
[s3;%% &]
|
||||
[s4; &]
|
||||
[s5;:Upp`:`:FetchUtf8`(const char`*`&`,const char`*`,bool`&`): [_^Upp`:`:dword^ dword]_
|
||||
[* FetchUtf8]([@(0.0.255) const]_[@(0.0.255) char]_`*`&[*@3 s], [@(0.0.255) const]_[@(0.0.255) c
|
||||
har]_`*[*@3 lim], [@(0.0.255) bool`&]_[*@3 ok])&]
|
||||
[s5;:Upp`:`:FetchUtf8`(const char`*`&`,const char`*`): [_^Upp`:`:dword^ dword]_[* FetchUt
|
||||
f8]([@(0.0.255) const]_[@(0.0.255) char]_`*`&[*@3 s], [@(0.0.255) const]_[@(0.0.255) char]_
|
||||
`*[*@3 lim])&]
|
||||
[s2;%% Reads a single UTF`-32 codepoint from UTF`-8 string [%-*@3 s]
|
||||
with end at [%-*@3 lim]. [%-*@3 s] must be less than [%-*@3 lim]. [%-*@3 s]
|
||||
is advanced accordingly. [%-*@3 ok] is set to false if UTF`-8 is
|
||||
invalid `- in that case, error`-escape of single byte is returned
|
||||
(but it is NOT set to true if valid UTF`-8 character is read).&]
|
||||
[s3;%% &]
|
||||
[s4; &]
|
||||
[s5;:Upp`:`:CheckUtf8`(const char`*`,int`): [@(0.0.255) bool]_[* CheckUtf8]([@(0.0.255) con
|
||||
st]_[@(0.0.255) char]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&]
|
||||
[s5;:Upp`:`:CheckUtf8`(const char`*`): [@(0.0.255) bool]_[* CheckUtf8]([@(0.0.255) const]_[@(0.0.255) c
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue