github-starred/ultimatepp
2
0
Fork 0
mirror of https://github.com/ultimatepp/ultimatepp.git synced 2026-08-24 22:02:44 -06:00
Code Issues 33 Projects Packages Wiki Activity Actions

Core: ToUtf8 for single codepoint extented to full UNICODE, CParser now supports \u surrogate pairs

Browse source 
git-svn-id: svn://ultimatepp.org/upp/trunk@11123 f0d560ea-af0d-0410-9eb7-867de7ffcac7
This commit is contained in:
cxl 2017-05-26 13:01:21 +00:00
parent d7f8c7ad7b
commit 272d416b91
5 changed files with 63 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

29
uppsrc/Core/CharSet.cpp
View file

@ -2458,6 +2458,35 @@ int lenAsUtf8(const wchar *s)
return lenAsUtf8(s, wstrlen(s));
}
String ToUtf8(int code)
{
char h[10];
char *t = h;
if(code < 0x80)
*t++ = (char)code;
else
if(code < 0x800) {
*t++ = 0xc0 | (code >> 6);
*t++ = 0x80 | (code & 0x3f);
}
else
if((code & 0xFF00) == 0xEE00)
*t++ = (char) code;
else
if(code < 0xFFFF) {
*t++ = 0xe0 | (code >> 12);
*t++ = 0x80 | ((code >> 6) & 0x3f);
*t++ = 0x80 | (code & 0x3f);
}
else {
*t++ = 0xf0 | (code >> 18);
*t++ = 0x80 | ((code >> 12) & 0x3f);
*t++ = 0x80 | ((code >> 6) & 0x3f);
*t++ = 0x80 | (code & 0x3f);
}
return String(h, t);
}
String ToUtf8(wchar code)
{
return ToUtf8(&code, 1);

2
uppsrc/Core/CharSet.h
View file

@ -50,7 +50,7 @@ inline bool IsUtf8Lead(int c)
return (c & 0xc0) != 0x80;
}
String ToUtf8(wchar code);
String ToUtf8(int code);
String ToUtf8(const wchar *s, int len);
String ToUtf8(const wchar *s);
String ToUtf8(const WString& w);

1
uppsrc/Core/Parser.h
View file

@ -22,6 +22,7 @@ protected:
const char *IsId0(const char *s) const;
bool Id0(const char *id);
void DoSpaces() { if(skipspaces) Spaces(); }
int ReadUTF16();
public:
struct Error : public Exc { Error(const char *s) : Exc(s) {} };

43
uppsrc/Core/parser.cpp
View file

@ -329,6 +329,25 @@ double CParser::ReadDouble() throw(Error)
return n;
}
int CParser::ReadUTF16()
{
int hex = 0;
if(IsXDigit(*++term)) {
hex = ctoi(*term);
if(IsXDigit(*++term)) {
hex = 16 * hex + (*term >= 'A' ? ToUpper(*term) - 'A' + 10 : *term - '0');
if(IsXDigit(*++term)) {
hex = 16 * hex + (*term >= 'A' ? ToUpper(*term) - 'A' + 10 : *term - '0');
if(IsXDigit(*++term)) {
hex = 16 * hex + (*term >= 'A' ? ToUpper(*term) - 'A' + 10 : *term - '0');
term++;
}
}
}
}
return hex;
}
String CParser::ReadOneString(int delim, bool chkend) throw(Error) {
if(!IsChar(delim))
ThrowError("missing string");
@ -364,21 +383,19 @@ String CParser::ReadOneString(int delim, bool chkend) throw(Error) {
}
case 'u':
if(uescape) {
int hex = 0;
if(IsXDigit(*++term)) {
hex = ctoi(*term);
if(IsXDigit(*++term)) {
hex = 16 * hex + (*term >= 'A' ? ToUpper(*term) - 'A' + 10 : *term - '0');
if(IsXDigit(*++term)) {
hex = 16 * hex + (*term >= 'A' ? ToUpper(*term) - 'A' + 10 : *term - '0');
if(IsXDigit(*++term)) {
hex = 16 * hex + (*term >= 'A' ? ToUpper(*term) - 'A' + 10 : *term - '0');
term++;
}
}
int hex = ReadUTF16();
if(hex >= 0xD800 && hex < 0xDBFF) {
if(term[0] == '\\' && term[1] == 'u') {
term++;
int hex2 = ReadUTF16();
if(hex2 >= 0xDC00 && hex2 <= 0xDFFF)
result.Cat(ToUtf8(((hex & 0x3ff) << 10) | (hex2 & 0x3ff) + 0x10000));
break;
}
ThrowError("Invalid UTF-16 surrogate pair");
}
result.Cat(WString(hex, 1).ToString());
else
result.Cat(WString(hex, 1).ToString());
}
else
result.Cat(*term++);

4
uppsrc/Core/src.tpp/CharSet$en-us.tpp
View file

@ -175,8 +175,8 @@ or[%-*@3 scharset ]can be CHARSET`_UTF8.&]
[s5;:IsUtf8Lead`(int`): [@(0.0.255) bool]_[* IsUtf8Lead]([@(0.0.255) int]_[*@3 c])&]
[s2;%% Tests whether [%-*@3 c ]is lead UTF`-8 byte.&]
[s3;%% &]
[s4;%% &]
[s5;:ToUtf8`(wchar`): [_^String^ String]_[* ToUtf8]([_^wchar^ wchar]_[*@3 code])&]
[s4; &]
[s5;:Upp`:`:ToUtf8`(int`): [_^Upp`:`:String^ String]_[* ToUtf8]([@(0.0.255) int]_[*@3 code])&]
[s2;%% Converts single unicode character to Utf8. Bytes from private
0xEExx range are converted to xx bytes.&]
[s3;%% &]