diff --git a/uppsrc/Core/CharSet.cpp b/uppsrc/Core/CharSet.cpp index 00d805389..a2d753a91 100644 --- a/uppsrc/Core/CharSet.cpp +++ b/uppsrc/Core/CharSet.cpp @@ -2458,6 +2458,35 @@ int lenAsUtf8(const wchar *s) return lenAsUtf8(s, wstrlen(s)); } +String ToUtf8(int code) +{ + char h[10]; + char *t = h; + if(code < 0x80) + *t++ = (char)code; + else + if(code < 0x800) { + *t++ = 0xc0 | (code >> 6); + *t++ = 0x80 | (code & 0x3f); + } + else + if((code & 0xFF00) == 0xEE00) + *t++ = (char) code; + else + if(code < 0xFFFF) { + *t++ = 0xe0 | (code >> 12); + *t++ = 0x80 | ((code >> 6) & 0x3f); + *t++ = 0x80 | (code & 0x3f); + } + else { + *t++ = 0xf0 | (code >> 18); + *t++ = 0x80 | ((code >> 12) & 0x3f); + *t++ = 0x80 | ((code >> 6) & 0x3f); + *t++ = 0x80 | (code & 0x3f); + } + return String(h, t); +} + String ToUtf8(wchar code) { return ToUtf8(&code, 1); diff --git a/uppsrc/Core/CharSet.h b/uppsrc/Core/CharSet.h index a37fbb9b3..1ae2a1521 100644 --- a/uppsrc/Core/CharSet.h +++ b/uppsrc/Core/CharSet.h @@ -50,7 +50,7 @@ inline bool IsUtf8Lead(int c) return (c & 0xc0) != 0x80; } -String ToUtf8(wchar code); +String ToUtf8(int code); String ToUtf8(const wchar *s, int len); String ToUtf8(const wchar *s); String ToUtf8(const WString& w); diff --git a/uppsrc/Core/Parser.h b/uppsrc/Core/Parser.h index fe438324d..0f9d76ec8 100644 --- a/uppsrc/Core/Parser.h +++ b/uppsrc/Core/Parser.h @@ -22,6 +22,7 @@ protected: const char *IsId0(const char *s) const; bool Id0(const char *id); void DoSpaces() { if(skipspaces) Spaces(); } + int ReadUTF16(); public: struct Error : public Exc { Error(const char *s) : Exc(s) {} }; diff --git a/uppsrc/Core/parser.cpp b/uppsrc/Core/parser.cpp index 2979ce05c..b0606b4f3 100644 --- a/uppsrc/Core/parser.cpp +++ b/uppsrc/Core/parser.cpp @@ -329,6 +329,25 @@ double CParser::ReadDouble() throw(Error) return n; } +int CParser::ReadUTF16() +{ + int hex = 0; + if(IsXDigit(*++term)) { + hex = ctoi(*term); + if(IsXDigit(*++term)) { + hex = 16 * hex + (*term >= 'A' ? ToUpper(*term) - 'A' + 10 : *term - '0'); + if(IsXDigit(*++term)) { + hex = 16 * hex + (*term >= 'A' ? ToUpper(*term) - 'A' + 10 : *term - '0'); + if(IsXDigit(*++term)) { + hex = 16 * hex + (*term >= 'A' ? ToUpper(*term) - 'A' + 10 : *term - '0'); + term++; + } + } + } + } + return hex; +} + String CParser::ReadOneString(int delim, bool chkend) throw(Error) { if(!IsChar(delim)) ThrowError("missing string"); @@ -364,21 +383,19 @@ String CParser::ReadOneString(int delim, bool chkend) throw(Error) { } case 'u': if(uescape) { - int hex = 0; - if(IsXDigit(*++term)) { - hex = ctoi(*term); - if(IsXDigit(*++term)) { - hex = 16 * hex + (*term >= 'A' ? ToUpper(*term) - 'A' + 10 : *term - '0'); - if(IsXDigit(*++term)) { - hex = 16 * hex + (*term >= 'A' ? ToUpper(*term) - 'A' + 10 : *term - '0'); - if(IsXDigit(*++term)) { - hex = 16 * hex + (*term >= 'A' ? ToUpper(*term) - 'A' + 10 : *term - '0'); - term++; - } - } + int hex = ReadUTF16(); + if(hex >= 0xD800 && hex < 0xDBFF) { + if(term[0] == '\\' && term[1] == 'u') { + term++; + int hex2 = ReadUTF16(); + if(hex2 >= 0xDC00 && hex2 <= 0xDFFF) + result.Cat(ToUtf8(((hex & 0x3ff) << 10) | (hex2 & 0x3ff) + 0x10000)); + break; } + ThrowError("Invalid UTF-16 surrogate pair"); } - result.Cat(WString(hex, 1).ToString()); + else + result.Cat(WString(hex, 1).ToString()); } else result.Cat(*term++); diff --git a/uppsrc/Core/src.tpp/CharSet$en-us.tpp b/uppsrc/Core/src.tpp/CharSet$en-us.tpp index 4c73016af..57f24f30c 100644 --- a/uppsrc/Core/src.tpp/CharSet$en-us.tpp +++ b/uppsrc/Core/src.tpp/CharSet$en-us.tpp @@ -175,8 +175,8 @@ or[%-*@3 scharset ]can be CHARSET`_UTF8.&] [s5;:IsUtf8Lead`(int`): [@(0.0.255) bool]_[* IsUtf8Lead]([@(0.0.255) int]_[*@3 c])&] [s2;%% Tests whether [%-*@3 c ]is lead UTF`-8 byte.&] [s3;%% &] -[s4;%% &] -[s5;:ToUtf8`(wchar`): [_^String^ String]_[* ToUtf8]([_^wchar^ wchar]_[*@3 code])&] +[s4; &] +[s5;:Upp`:`:ToUtf8`(int`): [_^Upp`:`:String^ String]_[* ToUtf8]([@(0.0.255) int]_[*@3 code])&] [s2;%% Converts single unicode character to Utf8. Bytes from private 0xEExx range are converted to xx bytes.&] [s3;%% &]