diff --git a/uppsrc/Core/CharSet.cpp b/uppsrc/Core/CharSet.cpp index 7b17c0f9c..fbbfdc76c 100644 --- a/uppsrc/Core/CharSet.cpp +++ b/uppsrc/Core/CharSet.cpp @@ -2185,295 +2185,6 @@ void ConvertCharset(char *t, byte tcharset, const char *s, byte scharset, int n) } } -bool utf8check(const char *_s, int len) -{ - const byte *s = (const byte *)_s; - const byte *lim = s + len; - int codePoint = 0; - while(s < lim) { - word code = (byte)*s++; - if(code >= 0x80) { - if(code < 0xC2) - return false; - else - if(code < 0xE0) { - if(s >= lim || *s < 0x80 || *s >= 0xc0) - return false; - codePoint = ((code - 0xC0) << 6) + *s - 0x80; - if(codePoint < 0x80 || codePoint > 0x07FF) - return false; - s++; - } - else - if(code < 0xF0) { - if(s + 1 >= lim || - s[0] < 0x80 || s[0] >= 0xc0 || - s[1] < 0x80 || s[1] >= 0xc0) - return false; - codePoint = ((code - 0xE0) << 12) + ((s[0] - 0x80) << 6) + s[1] - 0x80; - if(codePoint < 0x0800 || codePoint > 0xFFFF) - return false; - s += 2; - } - else - if(code < 0xF5) { - if(s + 2 >= lim || - s[0] < 0x80 || s[0] >= 0xc0 || - s[1] < 0x80 || s[1] >= 0xc0 || - s[2] < 0x80 || s[2] >= 0xc0) - return false; - codePoint = ((code - 0xf0) << 18) + ((s[0] - 0x80) << 12) + - ((s[1] - 0x80) << 6) + s[2] - 0x80; - if(codePoint < 0x010000 || codePoint > 0x10FFFF) - return false; - s += 3; - } - else - return false; - } - } - return true; -} - -int utf8len(const char *_s, int _len) -{ - const byte *s = (const byte *)_s; - const byte *lim = s + _len; - int len = 0; - if(_len > 4) - while(s < lim - 4) { - word code = (byte)*s++; - if(code < 0x80) - len++; - else - if(code < 0xC2) - len++; - else - if(code < 0xE0) { - word c = ((code - 0xC0) << 6) + s[0] - 0x80; - if(s[0] >= 0x80 && s[0] < 0xc0 && c >= 0x80 && c < 0x800) - len++; - else - len += 2; - s += 1; - } - else - if(code < 0xF0) { - word c = ((code - 0xE0) << 12) + ((s[0] - 0x80) << 6) + s[1] - 0x80; - if(s[0] >= 0x80 && s[0] < 0xc0 && s[1] >= 0x80 && s[1] < 0xc0 && c >= 0x800 - && !(c >= 0xEE00 && c <= 0xEEFF)) - len++; - else - len += 3; - s += 2; - } - else - len++; - } - while(s < lim) { - word code = (byte)*s++; - if(code < 0x80) - len++; - else - if(code < 0xC2) - len++; - else - if(code < 0xE0) { - if(s > lim - 1) { - len++; - break; - } - word c = ((code - 0xC0) << 6) + s[0] - 0x80; - if(s[0] >= 0x80 && s[0] < 0xc0 && c >= 0x80 && c < 0x800) - len++; - else - len += 2; - s += 1; - } - else - if(code < 0xF0) { - if(s > lim - 2) { - len += 1 + (int)(lim - s); - break; - } - word c = ((code - 0xE0) << 12) + ((s[0] - 0x80) << 6) + s[1] - 0x80; - if(s[0] >= 0x80 && s[0] < 0xc0 && s[1] >= 0x80 && s[1] < 0xc0 && c >= 0x800 - && !(c >= 0xEE00 && c <= 0xEEFF)) - len++; - else - len += 3; - s += 2; - } - else - len++; - } - return len; -} - -WString FromUtf8(const char *_s, int len) -{ - const byte *s = (const byte *)_s; - const byte *lim = s + len; - int tlen = utf8len(_s, len); - WStringBuffer result(tlen); - wchar *t = result; - if(len > 4) - while(s < lim - 4) { - word code = (byte)*s++; - if(code < 0x80) - *t++ = code; - else - if(code < 0xC2) - *t++ = 0xEE00 + code; - else - if(code < 0xE0) { - word c = ((code - 0xC0) << 6) + s[0] - 0x80; - if(s[0] >= 0x80 && s[0] < 0xc0 && c >= 0x80 && c < 0x800) - *t++ = c; - else { - *t++ = 0xEE00 + code; - *t++ = 0xEE00 + s[0]; - } - s += 1; - } - else - if(code < 0xF0) { - word c = ((code - 0xE0) << 12) + ((s[0] - 0x80) << 6) + s[1] - 0x80; - if(s[0] >= 0x80 && s[0] < 0xc0 && s[1] >= 0x80 && s[1] < 0xc0 && c >= 0x800 - && !(c >= 0xEE00 && c <= 0xEEFF)) - *t++ = c; - else { - *t++ = 0xEE00 + code; - *t++ = 0xEE00 + s[0]; - *t++ = 0xEE00 + s[1]; - } - s += 2; - } - else - *t++ = 0xEE00 + code; - } - while(s < lim) { - word code = (byte)*s++; - if(code < 0x80) - *t++ = code; - else - if(code < 0xC2) - *t++ = 0xEE00 + code; - else - if(code < 0xE0) { - if(s > lim - 1) { - *t++ = 0xEE00 + code; - break; - } - word c = ((code - 0xC0) << 6) + s[0] - 0x80; - if(s[0] >= 0x80 && s[0] < 0xc0 && c >= 0x80 && c < 0x800) - *t++ = c; - else { - *t++ = 0xEE00 + code; - *t++ = 0xEE00 + s[0]; - } - s += 1; - } - else - if(code < 0xF0) { - if(s > lim - 2) { - *t++ = 0xEE00 + code; - while(s < lim) - *t++ = 0xEE00 + *s++; - break; - } - word c = ((code - 0xE0) << 12) + ((s[0] - 0x80) << 6) + s[1] - 0x80; - if(s[0] >= 0x80 && s[0] < 0xc0 && s[1] >= 0x80 && s[1] < 0xc0 && c >= 0x800 - && !(c >= 0xEE00 && c <= 0xEEFF)) - *t++ = c; - else { - *t++ = 0xEE00 + code; - *t++ = 0xEE00 + s[0]; - *t++ = 0xEE00 + s[1]; - } - s += 2; - } - else - *t++ = 0xEE00 + code; - } - ASSERT(t - ~result == tlen); - return result; -} - -int lenAsUtf8(const wchar *s, int len) -{ - const wchar *lim = s + len; - len = 0; - while(s < lim) { - word code = *s++; - if(code < 0x80) - len++; - else - if(code < 0x800) - len += 2; - else - if((code & 0xFF00) == 0xEE00) - len++; - else - len += 3; - } - return len; -} - -int utf8len(const char *s) -{ - return utf8len(s, (int)strlen(s)); -} - -int lenAsUtf8(const wchar *s) -{ - return lenAsUtf8(s, wstrlen(s)); -} - -String ToUtf8(int code) -{ - char h[10]; - char *t = h; - if(code < 0x80) - *t++ = (char)code; - else - if(code < 0x800) { - *t++ = 0xc0 | (code >> 6); - *t++ = 0x80 | (code & 0x3f); - } - else - if((code & 0xFF00) == 0xEE00) - *t++ = (char) code; - else - if(code < 0xFFFF) { - *t++ = 0xe0 | (code >> 12); - *t++ = 0x80 | ((code >> 6) & 0x3f); - *t++ = 0x80 | (code & 0x3f); - } - else { - *t++ = 0xf0 | (code >> 18); - *t++ = 0x80 | ((code >> 12) & 0x3f); - *t++ = 0x80 | ((code >> 6) & 0x3f); - *t++ = 0x80 | (code & 0x3f); - } - return String(h, t); -} - -String ToUtf8(wchar code) -{ - return ToUtf8(&code, 1); -} - -WString FromUtf8(const char *s) -{ - return FromUtf8(s, (int)strlen(s)); -} - -WString FromUtf8(const String& s) -{ - return FromUtf8(s, s.GetLength()); -} - #ifdef flagSO bool IsLetter(int c) { return (dword)c < 2048 ? uni__info[c] & 0xc0000000 : 0; } bool IsUpper(int c) { return (dword)c < 2048 ? uni__info[c] & 0x40000000 : 0; } diff --git a/uppsrc/Core/CharSet.h b/uppsrc/Core/CharSet.h index a81d04908..b2a13f5c9 100644 --- a/uppsrc/Core/CharSet.h +++ b/uppsrc/Core/CharSet.h @@ -29,6 +29,11 @@ enum { int strlen32(const dword *s); +inline bool IsUtf8Lead(int c) +{ + return (c & 0xc0) != 0x80; +} + bool CheckUtf8(const char *s, int len); inline bool CheckUtf8(const char *s) { return CheckUtf8(s, (int)strlen(s)); } inline bool CheckUtf8(const String& s) { return CheckUtf8(~s, s.GetCount()); } @@ -38,16 +43,16 @@ inline int Utf8Len(const dword *s) { return Utf8Len(s, strlen inline int Utf8Len(const Vector& s) { return Utf8Len(s, s.GetCount()); } inline int Utf8Len(dword code) { return Utf8Len(&code, 1); } +int Utf8Len(const wchar *s, int len); +inline int Utf8Len(const wchar *s) { return Utf8Len(s, wstrlen(s)); } +inline int Utf8Len(const WString& s) { return Utf8Len(~s, s.GetCount()); } + void ToUtf8(char *t, const dword *s, int len); String ToUtf8(const dword *s, int len); inline String ToUtf8(const dword *s) { return ToUtf8(s, strlen32(s)); } inline String ToUtf8(const Vector& s) { return ToUtf8(s, s.GetCount()); } inline String ToUtf8(dword code) { return ToUtf8(&code, 1); } -int Utf8Len(const wchar *s, int len); -inline int Utf8Len(const wchar *s) { return Utf8Len(s, wstrlen(s)); } -inline int Utf8Len(const WString& s) { return Utf8Len(~s, s.GetCount()); } - void ToUtf8(char *t, const wchar *s, int len); String ToUtf8(const wchar *s, int len); inline String ToUtf8(const wchar *s) { return ToUtf8(s, wstrlen(s)); } @@ -58,16 +63,16 @@ inline int Utf16Len(const dword *s) { return Utf16Len(s, strle inline int Utf16Len(const Vector& s) { return Utf16Len(s, s.GetCount()); } inline int Utf16Len(dword code) { return Utf16Len(&code, 1); } +int Utf16Len(const char *s, int len); +inline int Utf16Len(const char *s) { return Utf16Len(s, (int)strlen(s)); } +inline int Utf16Len(const String& s) { return Utf16Len(~s, s.GetCount()); } + void ToUtf16(wchar *t, const dword *s, int len); WString ToUtf16(const dword *s, int len); inline WString ToUtf16(const dword *s) { return ToUtf16(s, strlen32(s)); } inline WString ToUtf16(const Vector& s) { return ToUtf16(s, s.GetCount()); } inline WString ToUtf16(dword code) { return ToUtf16(&code, 1); } -int Utf16Len(const char *s, int len); -inline int Utf16Len(const char *s) { return Utf16Len(s, (int)strlen(s)); } -inline int Utf16Len(const String& s) { return Utf16Len(~s, s.GetCount()); } - void ToUtf16(wchar *t, const char *s, int len); WString ToUtf16(const char *s, int len); inline WString ToUtf16(const char *s) { return ToUtf16(s, (int)strlen(s)); } @@ -77,15 +82,15 @@ int Utf32Len(const wchar *s, int len); inline int Utf32Len(const wchar *s) { return Utf32Len(s, wstrlen(s)); } inline int Utf32Len(const WString& s) { return Utf32Len(~s, s.GetCount()); } +int Utf32Len(const char *s, int len); +inline int Utf32Len(const char *s) { return Utf32Len(s, (int)strlen(s)); } +inline int Utf32Len(const String& s) { return Utf32Len(~s, s.GetCount()); } + void ToUtf32(dword *t, const wchar *s, int len); Vector ToUtf32(const wchar *s, int len); inline Vector ToUtf32(const wchar *s) { return ToUtf32(s, wstrlen(s)); } inline Vector ToUtf32(const WString& s) { return ToUtf32(~s, s.GetCount()); } -int Utf32Len(const char *s, int len); -inline int Utf32Len(const char *s) { return Utf32Len(s, (int)strlen(s)); } -inline int Utf32Len(const String& s) { return Utf32Len(~s, s.GetCount()); } - void ToUtf32(dword *t, const char *s, int len); Vector ToUtf32(const char *s, int len); inline Vector ToUtf32(const char *s) { return ToUtf32(s, (int)strlen(s)); } @@ -103,32 +108,14 @@ const char *CharsetName(byte charset); int CharsetCount(); int CharsetByName(const char *name); +void ConvertCharset(char *t, byte tcharset, const char *s, byte scharset, int n); + int ToUnicode(int chr, byte charset); int FromUnicode(wchar wchr, byte charset, int defchar = DEFAULTCHAR); void ToUnicode(wchar *ws, const char *s, int n, byte charset); void FromUnicode(char *s, const wchar *ws, int n, byte charset, int defchar = DEFAULTCHAR); -void ConvertCharset(char *t, byte tcharset, const char *s, byte scharset, int n); - -inline bool IsUtf8Lead(int c) -{ - return (c & 0xc0) != 0x80; -} - -WString FromUtf8(const char *_s, int len); -WString FromUtf8(const char *_s); -WString FromUtf8(const String& s); - -bool utf8check(const char *_s, int len); - -int utf8len(const char *s, int len); -int utf8len(const char *s); -int lenAsUtf8(const wchar *s, int len); -int lenAsUtf8(const wchar *s); - -bool CheckUtf8(const String& src); - WString ToUnicode(const String& src, byte charset); WString ToUnicode(const char *src, int n, byte charset); String FromUnicodeBuffer(const wchar *src, int len, byte charset = CHARSET_DEFAULT, int defchar = DEFAULTCHAR); @@ -227,13 +214,26 @@ String ToAscii(const char *s, byte charset = CHARSET_DEFAULT); WString LoadStreamBOMW(Stream& in, byte def_charset); WString LoadStreamBOMW(Stream& in); -String LoadStreamBOM(Stream& in, byte def_charset); -String LoadStreamBOM(Stream& in); +String LoadStreamBOM(Stream& in, byte def_charset); +String LoadStreamBOM(Stream& in); WString LoadFileBOMW(const char *path, byte def_charset); WString LoadFileBOMW(const char *path); -String LoadFileBOM(const char *path, byte def_charset); -String LoadFileBOM(const char *path); -bool SaveStreamBOM(Stream& out, const WString& data); -bool SaveFileBOM(const char *path, const WString& data); -bool SaveStreamBOMUtf8(Stream& out, const String& data); -bool SaveFileBOMUtf8(const char *path, const String& data); +String LoadFileBOM(const char *path, byte def_charset); +String LoadFileBOM(const char *path); +bool SaveStreamBOM(Stream& out, const WString& data); +bool SaveFileBOM(const char *path, const WString& data); +bool SaveStreamBOMUtf8(Stream& out, const String& data); +bool SaveFileBOMUtf8(const char *path, const String& data); + +// Deprecated names + +inline WString FromUtf8(const char *s, int len) { return ToUtf16(s, len); } +inline WString FromUtf8(const char *s) { return ToUtf16(s); } +inline WString FromUtf8(const String& s) { return ToUtf16(s); } + +inline bool utf8check(const char *s, int len) { return CheckUtf8(s, len); } + +inline int utf8len(const char *s, int len) { return Utf16Len(s, len); } +inline int utf8len(const char *s) { return Utf16Len(s); } +inline int lenAsUtf8(const wchar *s, int len) { return Utf8Len(s, len); } +inline int lenAsUtf8(const wchar *s) { return Utf8Len(s); } diff --git a/uppsrc/Core/src.tpp/CharSet$en-us.tpp b/uppsrc/Core/src.tpp/CharSet$en-us.tpp index 57f24f30c..6a4a11d5e 100644 --- a/uppsrc/Core/src.tpp/CharSet$en-us.tpp +++ b/uppsrc/Core/src.tpp/CharSet$en-us.tpp @@ -75,7 +75,8 @@ of characters that do not exist in target charset.&] [s4; &] [s5;:SetDefaultCharset`(byte`): [@(0.0.255) void]_[* SetDefaultCharset]([_^byte^ byte]_[*@3 c harset])&] -[s2;%% Sets the default [%-*@3 charset].&] +[s2;%% Sets the default [%-*@3 charset]. This is to support legacy +application; new applications should always use UTF8.&] [s3;%% &] [s4; &] [s5;:ResolveCharset`(byte`): [_^byte^ byte]_[* ResolveCharset]([_^byte^ byte]_[*@3 charset])&] @@ -172,86 +173,6 @@ Both arrays must have (at least) [%-*@3 n] elements. Neither [%-*@3 tcharset] or[%-*@3 scharset ]can be CHARSET`_UTF8.&] [s3;%% &] [s4;%% &] -[s5;:IsUtf8Lead`(int`): [@(0.0.255) bool]_[* IsUtf8Lead]([@(0.0.255) int]_[*@3 c])&] -[s2;%% Tests whether [%-*@3 c ]is lead UTF`-8 byte.&] -[s3;%% &] -[s4; &] -[s5;:Upp`:`:ToUtf8`(int`): [_^Upp`:`:String^ String]_[* ToUtf8]([@(0.0.255) int]_[*@3 code])&] -[s2;%% Converts single unicode character to Utf8. Bytes from private -0xEExx range are converted to xx bytes.&] -[s3;%% &] -[s4;%% &] -[s5;:ToUtf8`(const wchar`*`,int`): [_^String^ String]_[* ToUtf8]([@(0.0.255) const]_[_^wchar^ w -char]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] -[s2;%% Converts UNICODE array to UTF`-8. Bytes from private 0xEExx -range are converted to xx bytes.&] -[s3;%% &] -[s4;%% &] -[s5;:ToUtf8`(const wchar`*`): [_^String^ String]_[* ToUtf8]([@(0.0.255) const]_[_^wchar^ wcha -r]_`*[*@3 s])&] -[s2;%% Converts zero`-terminated string [%-*@3 s] to UTF`-8. Bytes -from private 0xEExx range are converted to xx bytes.&] -[s3;%% &] -[s4;%% &] -[s5;:ToUtf8`(const WString`&`): [_^String^ String]_[* ToUtf8]([@(0.0.255) const]_[_^WString^ W -String][@(0.0.255) `&]_[*@3 w])&] -[s2;%% Converts string to UTF`-8. Bytes from private 0xEExx range -are converted to xx bytes.&] -[s3;%% &] -[s4;%% &] -[s5;:FromUtf8`(const char`*`,int`): [_^WString^ WString]_[* FromUtf8]([@(0.0.255) const]_[@(0.0.255) c -har]_`*[*@3 `_s], [@(0.0.255) int]_[*@3 len])&] -[s2;%% Converts UTF`-8 to UNICODE string. Any wrong bytes and sequences -are converted to private 0xEExx range.&] -[s3;%% &] -[s4;%% &] -[s5;:FromUtf8`(const char`*`): [_^WString^ WString]_[* FromUtf8]([@(0.0.255) const]_[@(0.0.255) c -har]_`*[*@3 `_s])&] -[s2;%% Converts zero`-terminted UTF`-8 string to UNICODE. Any wrong -bytes and sequences are converted to private 0xEExx range.&] -[s3;%% &] -[s4;%% &] -[s5;:FromUtf8`(const String`&`): [_^WString^ WString]_[* FromUtf8]([@(0.0.255) const]_[_^String^ S -tring][@(0.0.255) `&]_[*@3 s])&] -[s2;%% Converts UTF`-8 string to UNICODE. Any wrong bytes and sequences -are converted to private 0xEExx range.&] -[s3;%% &] -[s4;%% &] -[s5;:utf8check`(const char`*`,int`): [@(0.0.255) bool]_[* utf8check]([@(0.0.255) const]_[@(0.0.255) c -har]_`*[*@3 `_s], [@(0.0.255) int]_[*@3 len])&] -[s2;%% Checks whether array contains a valid UTF`-8 sequence.&] -[s3;%% &] -[s4;%% &] -[s5;:utf8len`(const char`*`,int`): [@(0.0.255) int]_[* utf8len]([@(0.0.255) const]_[@(0.0.255) c -har]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] -[s2;%% Returns a number of UNICODE characters in UTF`-8 text. Error`-escaped -0xEExx characters for ill`-formed parts of UTF`-8 are correctly -accounted for.&] -[s3;%% &] -[s4;%% &] -[s5;:utf8len`(const char`*`): [@(0.0.255) int]_[* utf8len]([@(0.0.255) const]_[@(0.0.255) cha -r]_`*[*@3 s])&] -[s2;%% Returns a number of UNICODE characters in zero`-terminated -UTF`-8 text. Error`-escaped 0xEExx characters for ill`-formed -parts of UTF`-8 are correctly accounted for.&] -[s3;%% &] -[s4;%% &] -[s5;:lenAsUtf8`(const wchar`*`,int`): [@(0.0.255) int]_[* lenAsUtf8]([@(0.0.255) const]_[_^wchar^ w -char]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] -[s2;%% Returns number of bytes of UNICODE text when UTF`-8 encoded.&] -[s3;%% &] -[s4;%% &] -[s5;:lenAsUtf8`(const wchar`*`): [@(0.0.255) int]_[* lenAsUtf8]([@(0.0.255) const]_[_^wchar^ w -char]_`*[*@3 s])&] -[s2;%% Returns number of bytes of UNICODE zero`-terminated text when -UTF`-8 encoded..&] -[s3;%% &] -[s4;%% &] -[s5;:CheckUtf8`(const String`&`): [@(0.0.255) bool]_[* CheckUtf8]([@(0.0.255) const]_[_^String^ S -tring][@(0.0.255) `&]_[*@3 src])&] -[s2;%% Checks whether String contains a valid UTF`-8 sequence.&] -[s3;%% &] -[s4;%% &] [s5;:ToUnicode`(const String`&`,byte`): [_^WString^ WString]_[* ToUnicode]([@(0.0.255) cons t]_[_^String^ String][@(0.0.255) `&]_[*@3 src], [_^byte^ byte]_[*@3 charset])&] [s2;%% Converts [%-*@3 src] encoded in [%-*@3 charset] to UNICODE. [%-*@3 charset] @@ -715,4 +636,55 @@ true on success.&] [s2; Saves 8`-bit string in default encoding to the file. Returns true on success.&] [s3; &] +[s4; &] +[s5;:FromUtf8`(const char`*`,int`): [_^WString^ WString]_[* FromUtf8]([@(0.0.255) const]_[@(0.0.255) c +har]_`*[*@3 `_s], [@(0.0.255) int]_[*@3 len])&] +[s2;%% Converts UTF`-8 to UNICODE string. Any wrong bytes and sequences +are converted to private 0xEExx range. Deprecated, use ToUtf16.&] +[s3;%% &] +[s4;%% &] +[s5;:FromUtf8`(const char`*`): [_^WString^ WString]_[* FromUtf8]([@(0.0.255) const]_[@(0.0.255) c +har]_`*[*@3 `_s])&] +[s2;%% Converts zero`-terminted UTF`-8 string to UNICODE. Any wrong +bytes and sequences are converted to private 0xEExx range. Deprecated, +use ToUtf16.&] +[s3;%% &] +[s4;%% &] +[s5;:FromUtf8`(const String`&`): [_^WString^ WString]_[* FromUtf8]([@(0.0.255) const]_[_^String^ S +tring][@(0.0.255) `&]_[*@3 s])&] +[s2;%% Converts UTF`-8 string to UNICODE. Any wrong bytes and sequences +are converted to private 0xEExx range. Deprecated, use ToUtf16.&] +[s3;%% &] +[s4;%% &] +[s5;:utf8check`(const char`*`,int`): [@(0.0.255) bool]_[* utf8check]([@(0.0.255) const]_[@(0.0.255) c +har]_`*[*@3 `_s], [@(0.0.255) int]_[*@3 len])&] +[s2;%% Checks whether array contains a valid UTF`-8 sequence. Deprecated, +use CheckUtf8.&] +[s3;%% &] +[s4;%% &] +[s5;:utf8len`(const char`*`,int`): [@(0.0.255) int]_[* utf8len]([@(0.0.255) const]_[@(0.0.255) c +har]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] +[s2;%% Returns a number of UNICODE characters in UTF`-8 text. Error`-escaped +0xEExx characters for ill`-formed parts of UTF`-8 are correctly +accounted for. Deprecated, use Utf16Len.&] +[s3;%% &] +[s4;%% &] +[s5;:utf8len`(const char`*`): [@(0.0.255) int]_[* utf8len]([@(0.0.255) const]_[@(0.0.255) cha +r]_`*[*@3 s])&] +[s2;%% Returns a number of UNICODE characters in zero`-terminated +UTF`-8 text. Error`-escaped 0xEExx characters for ill`-formed +parts of UTF`-8 are correctly accounted for. Deprecated, use +Utf16Len.&] +[s3;%% &] +[s4;%% &] +[s5;:lenAsUtf8`(const wchar`*`,int`): [@(0.0.255) int]_[* lenAsUtf8]([@(0.0.255) const]_[_^wchar^ w +char]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] +[s2;%% Returns number of bytes of UNICODE text when UTF`-8 encoded. +Deprecated, use Utf8Len.&] +[s3;%% &] +[s4;%% &] +[s5;:lenAsUtf8`(const wchar`*`): [@(0.0.255) int]_[* lenAsUtf8]([@(0.0.255) const]_[_^wchar^ w +char]_`*[*@3 s])&] +[s2;%% Returns number of bytes of UNICODE zero`-terminated text when +UTF`-8 encoded. Deprecated, use Utf8Len.&] [s0; ]] \ No newline at end of file diff --git a/uppsrc/Core/src.tpp/Utf$en-us.tpp b/uppsrc/Core/src.tpp/Utf$en-us.tpp new file mode 100644 index 000000000..608a8b120 --- /dev/null +++ b/uppsrc/Core/src.tpp/Utf$en-us.tpp @@ -0,0 +1,207 @@ +topic ""; +[2 $$0,0#00000000000000000000000000000000:Default] +[i448;a25;kKO9;2 $$1,0#37138531426314131252341829483380:class] +[l288;2 $$2,2#27521748481378242620020725143825:desc] +[0 $$3,0#96390100711032703541132217272105:end] +[H6;0 $$4,0#05600065144404261032431302351956:begin] +[i448;a25;kKO9;2 $$5,0#37138531426314131252341829483370:item] +[l288;a4;*@5;1 $$6,6#70004532496200323422659154056402:requirement] +[l288;i1121;b17;O9;~~~.1408;2 $$7,0#10431211400427159095818037425705:param] +[i448;b42;O9;2 $$8,8#61672508125594000341940100500538:tparam] +[b42;2 $$9,9#13035079074754324216151401829390:normal] +[{_} +[ {{10000@(113.42.0) [s0;%% [*@7;4 Unicode UTF`[8,16,32`] support]]}}&] +[s0;%% &] +[s5;:IsUtf8Lead`(int`): [@(0.0.255) bool]_[* IsUtf8Lead]([@(0.0.255) int]_[*@3 c])&] +[s2;%% Tests whether [%-*@3 c ]is lead UTF`-8 byte.&] +[s3;%% &] +[s4; &] +[s5;:Upp`:`:CheckUtf8`(const char`*`,int`): [@(0.0.255) bool]_[* CheckUtf8]([@(0.0.255) con +st]_[@(0.0.255) char]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:CheckUtf8`(const char`*`): [@(0.0.255) bool]_[* CheckUtf8]([@(0.0.255) const]_[@(0.0.255) c +har]_`*[*@3 s])&] +[s5;:CheckUtf8`(const String`&`): [@(0.0.255) bool]_[* CheckUtf8]([@(0.0.255) const]_[_^String^ S +tring][@(0.0.255) `&]_[*@3 src])&] +[s2;%% Checks whether string contains a valid UTF`-8 sequence. If +source is specified as pointer [%-*@3 s] without [%-*@3 len], its +must be zero`-terminated.&] +[s3;%% &] +[s4; &] +[s5;:Upp`:`:Utf8Len`(const Upp`:`:dword`*`,int`): [@(0.0.255) int]_[* Utf8Len]([@(0.0.255) c +onst]_[_^Upp`:`:dword^ dword]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:Utf8Len`(const Upp`:`:dword`*`): [@(0.0.255) int]_[* Utf8Len]([@(0.0.255) const +]_[_^Upp`:`:dword^ dword]_`*[*@3 s])&] +[s5;:Upp`:`:Utf8Len`(const Upp`:`:Vector``&`): [@(0.0.255) int]_[* Utf8Len +]([@(0.0.255) const]_[_^Upp`:`:Vector^ Vector]<[_^Upp`:`:dword^ dword]>`&_[*@3 s])&] +[s2;%% Returns the size in bytes of UTF`-32 Unicode text in UTF`-8. +If source is specified as pointer [%-*@3 s] without [%-*@3 len], +its must be zero`-terminated.&] +[s3; &] +[s4; &] +[s5;:Upp`:`:Utf8Len`(const Upp`:`:wchar`*`,int`): [@(0.0.255) int]_[* Utf8Len]([@(0.0.255) c +onst]_[_^Upp`:`:wchar^ wchar]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:Utf8Len`(const Upp`:`:wchar`*`): [@(0.0.255) int]_[* Utf8Len]([@(0.0.255) const +]_[_^Upp`:`:wchar^ wchar]_`*[*@3 s])&] +[s5;:Upp`:`:Utf8Len`(const Upp`:`:WString`&`): [@(0.0.255) int]_[* Utf8Len]([@(0.0.255) con +st]_[_^Upp`:`:WString^ WString][@(0.0.255) `&]_[*@3 s])&] +[s2;%% Returns the size in bytes of UTF`-16 Unicode text in UTF`-8. +If source is specified as pointer [%-*@3 s] without [%-*@3 len], +its must be zero`-terminated.&] +[s3;%% &] +[s4; &] +[s5;:Upp`:`:Utf8Len`(Upp`:`:dword`): [@(0.0.255) int]_[* Utf8Len]([_^Upp`:`:dword^ dword]_[*@3 c +ode])&] +[s2;%% Returns the size in bytes of single codepoint in UTF`-8.&] +[s3;%% &] +[s4; &] +[s5;:Upp`:`:ToUtf8`(char`*`,const Upp`:`:wchar`*`,int`): [@(0.0.255) void]_[* ToUtf8]([@(0.0.255) c +har]_`*[*@3 t], [@(0.0.255) const]_[_^Upp`:`:wchar^ wchar]_`*[*@3 s], +[@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:ToUtf8`(const Upp`:`:wchar`*`,int`): [_^Upp`:`:String^ String]_[* ToUtf8]([@(0.0.255) c +onst]_[_^Upp`:`:wchar^ wchar]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:ToUtf8`(const Upp`:`:wchar`*`): [_^Upp`:`:String^ String]_[* ToUtf8]([@(0.0.255) c +onst]_[_^Upp`:`:wchar^ wchar]_`*[*@3 s])&] +[s5;:Upp`:`:ToUtf8`(const Upp`:`:WString`&`): [_^Upp`:`:String^ String]_[* ToUtf8]([@(0.0.255) c +onst]_[_^Upp`:`:WString^ WString][@(0.0.255) `&]_[*@3 s])&] +[s2;%% UTF`-16 to UTF`-8 conversion. If target is specified as pointer +to buffer [%-*@3 t], the buffer must contain enough space for the +output. If source is specified as pointer [%-*@3 s] without [%-*@3 len], +its must be zero`-terminated.&] +[s3;%% &] +[s4; &] +[s5;:Upp`:`:ToUtf8`(char`*`,const Upp`:`:dword`*`,int`): [@(0.0.255) void]_[* ToUtf8]([@(0.0.255) c +har]_`*[*@3 t], [@(0.0.255) const]_[_^Upp`:`:dword^ dword]_`*[*@3 s], +[@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:ToUtf8`(const Upp`:`:dword`*`,int`): [_^Upp`:`:String^ String]_[* ToUtf8]([@(0.0.255) c +onst]_[_^Upp`:`:dword^ dword]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:ToUtf8`(const Upp`:`:dword`*`): [_^Upp`:`:String^ String]_[* ToUtf8]([@(0.0.255) c +onst]_[_^Upp`:`:dword^ dword]_`*[*@3 s])&] +[s5;:Upp`:`:ToUtf8`(const Upp`:`:Vector``&`): [_^Upp`:`:String^ String]_ +[* ToUtf8]([@(0.0.255) const]_[_^Upp`:`:Vector^ Vector]<[_^Upp`:`:dword^ dword]>`&_[*@3 s]) +&] +[s2;%% UTF`-32 to UTF`-8 conversion. If target is specified as pointer +to buffer [%-*@3 t], the buffer must contain enough space for the +output. If source is specified as pointer [%-*@3 s] without [%-*@3 len], +its must be zero`-terminated.&] +[s3;%% &] +[s4; &] +[s5;:Upp`:`:ToUtf8`(Upp`:`:dword`): [_^Upp`:`:String^ String]_[* ToUtf8]([_^Upp`:`:dword^ d +word]_[*@3 code])&] +[s2;%% Converts single codepoint to UTF`-8.&] +[s3; &] +[s4; &] +[s5;:Upp`:`:Utf16Len`(const Upp`:`:dword`*`,int`): [@(0.0.255) int]_[* Utf16Len]([@(0.0.255) c +onst]_[_^Upp`:`:dword^ dword]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:Utf16Len`(const Upp`:`:dword`*`): [@(0.0.255) int]_[* Utf16Len]([@(0.0.255) con +st]_[_^Upp`:`:dword^ dword]_`*[*@3 s])&] +[s5;:Upp`:`:Utf16Len`(const Upp`:`:Vector``&`): [@(0.0.255) int]_[* Utf16L +en]([@(0.0.255) const]_[_^Upp`:`:Vector^ Vector]<[_^Upp`:`:dword^ dword]>`&_[*@3 s])&] +[s2;%% Returns the size in wchars of UTF`-32 Unicode text in UTF`-16. +If source is specified as pointer [%-*@3 s] without [%-*@3 len], +its must be zero`-terminated.&] +[s3; &] +[s4; &] +[s5;:Upp`:`:Utf16Len`(Upp`:`:dword`): [@(0.0.255) int]_[* Utf16Len]([_^Upp`:`:dword^ dword]_ +[*@3 code])&] +[s2;%% Returns the size in wchars of single codepoint in UTF`-16.&] +[s3;%% &] +[s4; &] +[s5;:Upp`:`:Utf16Len`(const char`*`,int`): [@(0.0.255) int]_[* Utf16Len]([@(0.0.255) const]_ +[@(0.0.255) char]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:Utf16Len`(const char`*`): [@(0.0.255) int]_[* Utf16Len]([@(0.0.255) const]_[@(0.0.255) c +har]_`*[*@3 s])&] +[s5;:Upp`:`:Utf16Len`(const Upp`:`:String`&`): [@(0.0.255) int]_[* Utf16Len]([@(0.0.255) co +nst]_[_^Upp`:`:String^ String][@(0.0.255) `&]_[*@3 s])&] +[s2;%% Returns the size in wchars of UTF`-8 Unicode text in UTF`-16. +If source is specified as pointer [%-*@3 s] without [%-*@3 len], +its must be zero`-terminated.&] +[s3;%% &] +[s4; &] +[s5;:Upp`:`:ToUtf16`(Upp`:`:wchar`*`,const Upp`:`:dword`*`,int`): [@(0.0.255) void]_[* To +Utf16]([_^Upp`:`:wchar^ wchar]_`*[*@3 t], [@(0.0.255) const]_[_^Upp`:`:dword^ dword]_`*[*@3 s +], [@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:ToUtf16`(const Upp`:`:dword`*`,int`): [_^Upp`:`:WString^ WString]_[* ToUtf16]( +[@(0.0.255) const]_[_^Upp`:`:dword^ dword]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:ToUtf16`(const Upp`:`:dword`*`): [_^Upp`:`:WString^ WString]_[* ToUtf16]([@(0.0.255) c +onst]_[_^Upp`:`:dword^ dword]_`*[*@3 s])&] +[s5;:Upp`:`:ToUtf16`(const Upp`:`:Vector``&`): [_^Upp`:`:WString^ WStrin +g]_[* ToUtf16]([@(0.0.255) const]_[_^Upp`:`:Vector^ Vector]<[_^Upp`:`:dword^ dword]>`&_[*@3 s +])&] +[s2;%% [%- UTF`-32 to UTF`-16 conversion.] If target is specified as +pointer to buffer [%-*@3 t], the buffer must contain enough space +for the output. If source is specified as pointer [%-*@3 s] without +[%-*@3 len], its must be zero`-terminated.&] +[s3; &] +[s4; &] +[s5;:Upp`:`:ToUtf16`(Upp`:`:dword`): [_^Upp`:`:WString^ WString]_[* ToUtf16]([_^Upp`:`:dword^ d +word]_[*@3 code])&] +[s2;%% Converts single codepoint to UTF`-16.&] +[s3;%% &] +[s4; &] +[s5;:Upp`:`:ToUtf16`(Upp`:`:wchar`*`,const char`*`,int`): [@(0.0.255) void]_[* ToUtf16]([_^Upp`:`:wchar^ w +char]_`*[*@3 t], [@(0.0.255) const]_[@(0.0.255) char]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:ToUtf16`(const char`*`,int`): [_^Upp`:`:WString^ WString]_[* ToUtf16]([@(0.0.255) c +onst]_[@(0.0.255) char]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:ToUtf16`(const char`*`): [_^Upp`:`:WString^ WString]_[* ToUtf16]([@(0.0.255) co +nst]_[@(0.0.255) char]_`*[*@3 s])&] +[s5;:Upp`:`:ToUtf16`(const Upp`:`:String`&`): [_^Upp`:`:WString^ WString]_[* ToUtf16]([@(0.0.255) c +onst]_[_^Upp`:`:String^ String][@(0.0.255) `&]_[*@3 s])&] +[s2;%% UTF`-8 to UTF`-16 conversion. If target is specified as pointer +to buffer [%-*@3 t], the buffer must contain enough space for the +output. If source is specified as pointer [%-*@3 s] without [%-*@3 len], +its must be zero`-terminated.&] +[s3;%% &] +[s4; &] +[s5;:Upp`:`:Utf32Len`(const Upp`:`:wchar`*`,int`): [@(0.0.255) int]_[* Utf32Len]([@(0.0.255) c +onst]_[_^Upp`:`:wchar^ wchar]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:Utf32Len`(const Upp`:`:wchar`*`): [@(0.0.255) int]_[* Utf32Len]([@(0.0.255) con +st]_[_^Upp`:`:wchar^ wchar]_`*[*@3 s])&] +[s5;:Upp`:`:Utf32Len`(const Upp`:`:WString`&`): [@(0.0.255) int]_[* Utf32Len]([@(0.0.255) c +onst]_[_^Upp`:`:WString^ WString][@(0.0.255) `&]_[*@3 s])&] +[s2;%% Returns the size in dwords of UTF`-16 Unicode text in UTF`-32. +Note that this is the same as the number of Unicode codepoints +in the text. If source is specified as pointer [%-*@3 s] without +[%-*@3 len], its must be zero`-terminated.&] +[s3;%% &] +[s4; &] +[s5;:Upp`:`:Utf32Len`(const char`*`,int`): [@(0.0.255) int]_[* Utf32Len]([@(0.0.255) const]_ +[@(0.0.255) char]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:Utf32Len`(const char`*`): [@(0.0.255) int]_[* Utf32Len]([@(0.0.255) const]_[@(0.0.255) c +har]_`*[*@3 s])&] +[s5;:Upp`:`:Utf32Len`(const Upp`:`:String`&`): [@(0.0.255) int]_[* Utf32Len]([@(0.0.255) co +nst]_[_^Upp`:`:String^ String][@(0.0.255) `&]_[*@3 s])&] +[s2;%% Returns the size in dwords of UTF`-8 Unicode text in UTF`-32. +Note that this is the same as the number of Unicode codepoints +in the text. If source is specified as pointer [%-*@3 s] without +[%-*@3 len], its must be zero`-terminated.&] +[s3;%% &] +[s4; &] +[s5;:Upp`:`:ToUtf32`(Upp`:`:dword`*`,const Upp`:`:wchar`*`,int`): [@(0.0.255) void]_[* To +Utf32]([_^Upp`:`:dword^ dword]_`*[*@3 t], [@(0.0.255) const]_[_^Upp`:`:wchar^ wchar]_`*[*@3 s +], [@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:ToUtf32`(const Upp`:`:wchar`*`,int`): [_^Upp`:`:Vector^ Vector]<[_^Upp`:`:dword^ d +word]>_[* ToUtf32]([@(0.0.255) const]_[_^Upp`:`:wchar^ wchar]_`*[*@3 s], +[@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:ToUtf32`(const Upp`:`:wchar`*`): [_^Upp`:`:Vector^ Vector]<[_^Upp`:`:dword^ d +word]>_[* ToUtf32]([@(0.0.255) const]_[_^Upp`:`:wchar^ wchar]_`*[*@3 s])&] +[s5;:Upp`:`:ToUtf32`(const Upp`:`:WString`&`): [_^Upp`:`:Vector^ Vector]<[_^Upp`:`:dword^ d +word]>_[* ToUtf32]([@(0.0.255) const]_[_^Upp`:`:WString^ WString][@(0.0.255) `&]_[*@3 s])&] +[s2;%% UTF`-16 to UTF`-32 conversion. If target is specified as pointer +to buffer [%-*@3 t], the buffer must contain enough space for the +output. If source is specified as pointer [%-*@3 s] without [%-*@3 len], +its must be zero`-terminated.&] +[s3;%% &] +[s4; &] +[s5;:Upp`:`:ToUtf32`(Upp`:`:dword`*`,const char`*`,int`): [@(0.0.255) void]_[* ToUtf32]([_^Upp`:`:dword^ d +word]_`*[*@3 t], [@(0.0.255) const]_[@(0.0.255) char]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:ToUtf32`(const char`*`,int`): [_^Upp`:`:Vector^ Vector]<[_^Upp`:`:dword^ dwor +d]>_[* ToUtf32]([@(0.0.255) const]_[@(0.0.255) char]_`*[*@3 s], [@(0.0.255) int]_[*@3 len])&] +[s5;:Upp`:`:ToUtf32`(const char`*`): [_^Upp`:`:Vector^ Vector]<[_^Upp`:`:dword^ dword]>_[* T +oUtf32]([@(0.0.255) const]_[@(0.0.255) char]_`*[*@3 s])&] +[s5;:Upp`:`:ToUtf32`(const Upp`:`:String`&`): [_^Upp`:`:Vector^ Vector]<[_^Upp`:`:dword^ d +word]>_[* ToUtf32]([@(0.0.255) const]_[_^Upp`:`:String^ String][@(0.0.255) `&]_[*@3 s])&] +[s2;%% UTF`-8 to UTF`-32 conversion. If target is specified as pointer +to buffer [%-*@3 t], the buffer must contain enough space for the +output. If source is specified as pointer [%-*@3 s] without [%-*@3 len], +its must be zero`-terminated.&] +[s3;%% ]] \ No newline at end of file