mirror of
https://github.com/ultimatepp/ultimatepp.git
synced 2026-08-24 14:22:40 -06:00
Core: Utf (8, 16, 32) support
git-svn-id: svn://ultimatepp.org/upp/trunk@11139 f0d560ea-af0d-0410-9eb7-867de7ffcac7
This commit is contained in:
cxl
parent
db8c8a5bb4
commit
a69fd4e112
6 changed files with 318 additions and 49 deletions
|
|
@ -2420,34 +2420,6 @@ int lenAsUtf8(const wchar *s, int len)
|
|||
return len;
|
||||
}
|
||||
|
||||
String ToUtf8(const wchar *s, int len)
|
||||
{
|
||||
const wchar *lim = s + len;
|
||||
int tlen = lenAsUtf8(s, len);
|
||||
StringBuffer result(tlen);
|
||||
char *t = result;
|
||||
while(s < lim) {
|
||||
word code = *s++;
|
||||
if(code < 0x80)
|
||||
*t++ = (char)code;
|
||||
else
|
||||
if(code < 0x800) {
|
||||
*t++ = 0xc0 | (code >> 6);
|
||||
*t++ = 0x80 | (code & 0x3f);
|
||||
}
|
||||
else
|
||||
if((code & 0xFF00) == 0xEE00)
|
||||
*t++ = (char) code;
|
||||
else {
|
||||
*t++ = 0xe0 | (code >> 12);
|
||||
*t++ = 0x80 | ((code >> 6) & 0x3f);
|
||||
*t++ = 0x80 | (code & 0x3f);
|
||||
}
|
||||
}
|
||||
ASSERT(t - ~result == tlen);
|
||||
return result;
|
||||
}
|
||||
|
||||
int utf8len(const char *s)
|
||||
{
|
||||
return utf8len(s, (int)strlen(s));
|
||||
|
|
@ -2492,21 +2464,6 @@ String ToUtf8(wchar code)
|
|||
return ToUtf8(&code, 1);
|
||||
}
|
||||
|
||||
String ToUtf8(const WString& w)
|
||||
{
|
||||
return ToUtf8(w, w.GetLength());
|
||||
}
|
||||
|
||||
String ToUtf8(const wchar *s)
|
||||
{
|
||||
return ToUtf8(s, wstrlen(s));
|
||||
}
|
||||
|
||||
bool CheckUtf8(const String& src)
|
||||
{
|
||||
return utf8check(~src, src.GetLength());
|
||||
}
|
||||
|
||||
WString FromUtf8(const char *s)
|
||||
{
|
||||
return FromUtf8(s, (int)strlen(s));
|
||||
|
|
|
|||
|
|
@ -25,6 +25,72 @@ enum {
|
|||
|
||||
#endif
|
||||
|
||||
#include "Utf.hpp"
|
||||
|
||||
int strlen32(const dword *s);
|
||||
|
||||
bool CheckUtf8(const char *s, int len);
|
||||
inline bool CheckUtf8(const char *s) { return CheckUtf8(s, strlen(s)); }
|
||||
inline bool CheckUtf8(const String& s) { return CheckUtf8(~s, s.GetCount()); }
|
||||
|
||||
int Utf8Len(const dword *s, int len);
|
||||
inline int Utf8Len(const dword *s) { return Utf8Len(s, strlen32(s)); }
|
||||
inline int Utf8Len(const Vector<dword>& s) { return Utf8Len(s, s.GetCount()); }
|
||||
inline int Utf8Len(dword code) { return Utf8Len(&code, 1); }
|
||||
|
||||
void ToUtf8(char *t, const dword *s, int len);
|
||||
String ToUtf8(const dword *s, int len);
|
||||
inline String ToUtf8(const dword *s) { return ToUtf8(s, strlen32(s)); }
|
||||
inline String ToUtf8(const Vector<dword>& s) { return ToUtf8(s, s.GetCount()); }
|
||||
inline String ToUtf8(dword code) { return ToUtf8(&code, 1); }
|
||||
|
||||
int Utf8Len(const wchar *s, int len);
|
||||
inline int Utf8Len(const wchar *s) { return Utf8Len(s, wstrlen(s)); }
|
||||
inline int Utf8Len(const WString& s) { return Utf8Len(~s, s.GetCount()); }
|
||||
|
||||
void ToUtf8(char *t, const wchar *s, int len);
|
||||
String ToUtf8(const wchar *s, int len);
|
||||
inline String ToUtf8(const wchar *s) { return ToUtf8(s, wstrlen(s)); }
|
||||
inline String ToUtf8(const WString& s) { return ToUtf8(~s, s.GetCount()); }
|
||||
|
||||
int Utf16Len(const dword *s, int len);
|
||||
inline int Utf16Len(const dword *s) { return Utf16Len(s, strlen32(s)); }
|
||||
inline int Utf16Len(const Vector<dword>& s) { return Utf16Len(s, s.GetCount()); }
|
||||
inline int Utf16Len(dword code) { return Utf16Len(&code, 1); }
|
||||
|
||||
void ToUtf16(wchar *t, const dword *s, int len);
|
||||
WString ToUtf16(const dword *s, int len);
|
||||
inline WString ToUtf16(const dword *s) { return ToUtf16(s, strlen32(s)); }
|
||||
inline WString ToUtf16(const Vector<dword>& s) { return ToUtf16(s, s.GetCount()); }
|
||||
inline WString ToUtf16(dword code) { return ToUtf16(&code, 1); }
|
||||
|
||||
int Utf16Len(const char *s, int len);
|
||||
inline int Utf16Len(const char *s) { return Utf16Len(s, strlen(s)); }
|
||||
inline int Utf16Len(const String& s) { return Utf16Len(~s, s.GetCount()); }
|
||||
|
||||
void ToUtf16(wchar *t, const char *s, int len);
|
||||
WString ToUtf16(const char *s, int len);
|
||||
inline WString ToUtf16(const char *s) { return ToUtf16(s, strlen(s)); }
|
||||
inline WString ToUtf16(const String& s) { return ToUtf16(~s, s.GetCount()); }
|
||||
|
||||
int Utf32Len(const wchar *s, int len);
|
||||
inline int Utf32Len(const wchar *s) { return Utf32Len(s, wstrlen(s)); }
|
||||
inline int Utf32Len(const WString& s) { return Utf32Len(~s, s.GetCount()); }
|
||||
|
||||
void ToUtf32(dword *t, const wchar *s, int len);
|
||||
Vector<dword> ToUtf32(const wchar *s, int len);
|
||||
inline Vector<dword> ToUtf32(const wchar *s) { return ToUtf32(s, wstrlen(s)); }
|
||||
inline Vector<dword> ToUtf32(const WString& s) { return ToUtf32(~s, s.GetCount()); }
|
||||
|
||||
int Utf32Len(const char *s, int len);
|
||||
inline int Utf32Len(const char *s) { return Utf32Len(s, strlen(s)); }
|
||||
inline int Utf32Len(const String& s) { return Utf32Len(~s, s.GetCount()); }
|
||||
|
||||
void ToUtf32(dword *t, const char *s, int len);
|
||||
Vector<dword> ToUtf32(const char *s, int len);
|
||||
inline Vector<dword> ToUtf32(const char *s) { return ToUtf32(s, strlen(s)); }
|
||||
inline Vector<dword> ToUtf32(const String& s) { return ToUtf32(~s, s.GetCount()); }
|
||||
|
||||
void SetDefaultCharset(byte charset);
|
||||
byte GetDefaultCharset();
|
||||
|
||||
|
|
@ -50,11 +116,6 @@ inline bool IsUtf8Lead(int c)
|
|||
return (c & 0xc0) != 0x80;
|
||||
}
|
||||
|
||||
String ToUtf8(int code);
|
||||
String ToUtf8(const wchar *s, int len);
|
||||
String ToUtf8(const wchar *s);
|
||||
String ToUtf8(const WString& w);
|
||||
|
||||
WString FromUtf8(const char *_s, int len);
|
||||
WString FromUtf8(const char *_s);
|
||||
WString FromUtf8(const String& s);
|
||||
|
|
|
|||
|
|
@ -270,7 +270,6 @@ class JsonIO;
|
|||
#include "Mt.h"
|
||||
#include "String.h"
|
||||
|
||||
#include "CharSet.h"
|
||||
#include "TimeDate.h"
|
||||
#include "Path.h"
|
||||
#include "Stream.h"
|
||||
|
|
@ -288,6 +287,8 @@ class JsonIO;
|
|||
#include "FixedMap.h"
|
||||
#include "InVector.h"
|
||||
|
||||
#include "CharSet.h"
|
||||
|
||||
#include "SplitMerge.h"
|
||||
|
||||
#include "Other.h"
|
||||
|
|
|
|||
|
|
@ -63,6 +63,8 @@ file
|
|||
SplitMerge.cpp,
|
||||
CharSet.i,
|
||||
CharSet.h,
|
||||
Utf.hpp,
|
||||
Utf.cpp,
|
||||
CharSet.cpp,
|
||||
Bom.cpp,
|
||||
Path.h,
|
||||
|
|
|
|||
139
uppsrc/Core/Utf.cpp
Normal file
139
uppsrc/Core/Utf.cpp
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
#include "Core.h"
|
||||
|
||||
namespace Upp {
|
||||
|
||||
int strlen32(const dword *s)
|
||||
{
|
||||
const dword *s0 = s;
|
||||
while(*s) s++;
|
||||
return s - s0;
|
||||
}
|
||||
|
||||
bool CheckUtf8(const char *s, int len)
|
||||
{
|
||||
return FromUtf8_([](const byte *, dword){}, s, len);
|
||||
}
|
||||
|
||||
int Utf8Len(const dword *s, int len)
|
||||
{
|
||||
int rlen = 0;
|
||||
for(const dword *lim = s + len; s < lim; s++)
|
||||
ToUtf8_([&](char) { rlen++; }, *s);
|
||||
return rlen;
|
||||
}
|
||||
|
||||
void ToUtf8(char *t, const dword *s, int len)
|
||||
{
|
||||
for(const dword *lim = s + len; s < lim; s++)
|
||||
ToUtf8_([&](char c) { *t++ = c; }, *s);
|
||||
}
|
||||
|
||||
String ToUtf8(const dword *s, int len)
|
||||
{
|
||||
String r;
|
||||
for(const dword *lim = s + len; s < lim; s++)
|
||||
ToUtf8_([&](char c) { r.Cat(c); }, *s);
|
||||
return r;
|
||||
}
|
||||
|
||||
int Utf8Len(const wchar *s, int len)
|
||||
{
|
||||
int rlen = 0;
|
||||
FromUtf16_([&](const wchar *, dword code) { ToUtf8_([&](char c) { rlen++; }, code); }, s, len);
|
||||
return rlen;
|
||||
}
|
||||
|
||||
void ToUtf8(char *t, const wchar *s, int len)
|
||||
{
|
||||
FromUtf16_([&](const wchar *, dword code) { ToUtf8_([&](char c) { *t++ = c; }, code); }, s, len);
|
||||
}
|
||||
|
||||
String ToUtf8(const wchar *s, int len)
|
||||
{
|
||||
StringBuffer r;
|
||||
r.Reserve(len);
|
||||
FromUtf16_([&](const wchar *, dword code) { ToUtf8_([&](char c) { r.Cat(c); }, code); }, s, len);
|
||||
return r;
|
||||
}
|
||||
|
||||
int Utf16Len(const dword *s, int len)
|
||||
{
|
||||
int rlen = 0;
|
||||
for(const dword *lim = s + len; s < lim; s++)
|
||||
ToUtf16_([&](wchar) { rlen++; }, *s);
|
||||
return rlen;
|
||||
}
|
||||
|
||||
void ToUtf16(wchar *t, const dword *s, int len)
|
||||
{
|
||||
for(const dword *lim = s + len; s < lim; s++)
|
||||
ToUtf16_([&](wchar c) { *t++ = c; }, *s);
|
||||
}
|
||||
|
||||
WString ToUtf16(const dword *s, int len)
|
||||
{
|
||||
WStringBuffer r;
|
||||
r.Reserve(len);
|
||||
for(const dword *lim = s + len; s < lim; s++)
|
||||
ToUtf16_([&](wchar c) { r.Cat(c); }, *s);
|
||||
return r;
|
||||
}
|
||||
|
||||
int Utf16Len(const char *s, int len)
|
||||
{
|
||||
int rlen = 0;
|
||||
FromUtf8_([&](const byte *, dword code) { ToUtf16_([&](wchar c) { rlen++; }, code); }, s, len);
|
||||
return rlen;
|
||||
}
|
||||
|
||||
void ToUtf16(wchar *t, const char *s, int len)
|
||||
{
|
||||
FromUtf8_([&](const byte *, dword code) { ToUtf16_([&](wchar c) { *t++ = c; }, code); }, s, len);
|
||||
}
|
||||
|
||||
WString ToUtf16(const char *s, int len)
|
||||
{
|
||||
WStringBuffer r;
|
||||
FromUtf8_([&](const byte *, dword code) { ToUtf16_([&](wchar c) { r.Cat(c); }, code); }, s, len);
|
||||
return r;
|
||||
}
|
||||
|
||||
int Utf32Len(const char *s, int len)
|
||||
{
|
||||
int rlen = 0;
|
||||
FromUtf8_([&](const byte *, dword) { rlen++; }, s, len);
|
||||
return rlen;
|
||||
}
|
||||
|
||||
void ToUtf32(dword *t, const char *s, int len)
|
||||
{
|
||||
FromUtf8_([&](const byte *, dword c) { *t++ = c; }, s, len);
|
||||
}
|
||||
|
||||
Vector<dword> ToUtf32(const char *s, int len)
|
||||
{
|
||||
Vector<dword> r;
|
||||
FromUtf8_([&](const byte *, dword c) { r.Add(c); }, s, len);
|
||||
return r;
|
||||
}
|
||||
|
||||
int Utf32Len(const wchar *s, int len)
|
||||
{
|
||||
int rlen = 0;
|
||||
FromUtf16_([&](const wchar *, dword) { rlen++; }, s, len);
|
||||
return rlen;
|
||||
}
|
||||
|
||||
void ToUtf32(dword *t, const wchar *s, int len)
|
||||
{
|
||||
FromUtf16_([&](const wchar *, dword c) { *t++ = c; }, s, len);
|
||||
}
|
||||
|
||||
Vector<dword> ToUtf32(const wchar *s, int len)
|
||||
{
|
||||
Vector<dword> r;
|
||||
FromUtf16_([&](const wchar *, dword c) { r.Add(c); }, s, len);
|
||||
return r;
|
||||
}
|
||||
|
||||
};
|
||||
109
uppsrc/Core/Utf.hpp
Normal file
109
uppsrc/Core/Utf.hpp
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
template <class Target>
|
||||
force_inline bool ToUtf8_(Target &t, dword codepoint)
|
||||
{
|
||||
if(codepoint < 0x80)
|
||||
t((char)codepoint);
|
||||
else
|
||||
if(codepoint < 0x800) {
|
||||
t(0xc0 | byte(codepoint >> 6));
|
||||
t(0x80 | byte(codepoint & 0x3f));
|
||||
}
|
||||
else
|
||||
if((codepoint & 0xFFFFFF00) == 0xEE00) // ERROR ESCAPE
|
||||
t((char) codepoint);
|
||||
else
|
||||
if(codepoint < 0x10000) {
|
||||
t(0xe0 | byte(codepoint >> 12));
|
||||
t(0x80 | byte((codepoint >> 6) & 0x3f));
|
||||
t(0x80 | byte(codepoint & 0x3f));
|
||||
}
|
||||
else
|
||||
if(codepoint < 0x110000) {
|
||||
t(0xf0 | byte(codepoint >> 18));
|
||||
t(0x80 | byte((codepoint >> 12) & 0x3f));
|
||||
t(0x80 | byte((codepoint >> 6) & 0x3f));
|
||||
t(0x80 | byte(codepoint & 0x3f));
|
||||
}
|
||||
else
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
template <class Target>
|
||||
force_inline bool FromUtf8_(Target& t, const char *_s, int len)
|
||||
{
|
||||
bool ok = true;
|
||||
const byte *s = (const byte *)_s;
|
||||
const byte *lim = s + len;
|
||||
while(s < lim) {
|
||||
dword code = *s;
|
||||
if(code < 0x80) {
|
||||
t(s++, code);
|
||||
continue;
|
||||
}
|
||||
else
|
||||
if(code >= 0xC2) {
|
||||
dword c;
|
||||
if(code < 0xE0 && s + 1 < lim &&
|
||||
s[1] >= 0x80 && s[1] < 0xc0 &&
|
||||
(c = ((code - 0xC0) << 6) + s[1] - 0x80) >= 0x80 && c < 0x800) {
|
||||
t(s, c);
|
||||
s += 2;
|
||||
continue;
|
||||
}
|
||||
else
|
||||
if(code < 0xF0 && s + 2 < lim &&
|
||||
s[1] >= 0x80 && s[1] < 0xc0 && s[2] >= 0x80 && s[2] < 0xc0 &&
|
||||
(c = ((code - 0xE0) << 12) + ((s[1] - 0x80) << 6) + s[2] - 0x80) >= 0x800 &&
|
||||
!(c >= 0xEE00 && c <= 0xEEFF)) {
|
||||
t(s, c);
|
||||
s += 3;
|
||||
continue;
|
||||
}
|
||||
else
|
||||
if(code < 0xF8 && s + 3 < lim &&
|
||||
s[1] >= 0x80 && s[1] < 0xc0 && s[2] >= 0x80 && s[2] < 0xc0 && s[3] >= 0x80 && s[3] < 0xc0 &&
|
||||
(c = ((code - 0xF0) << 18) + ((s[1] - 0x80) << 12) + ((s[2] - 0x80) << 6) + s[3] - 0x80) >= 0x10000 &&
|
||||
c < 0x110000) {
|
||||
t(s, c);
|
||||
s += 4;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
t(s++, 0xEE00 + code);
|
||||
ok = false;
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
template <class Target>
|
||||
force_inline bool ToUtf16_(Target &t, int codepoint)
|
||||
{
|
||||
if(codepoint < 0x10000)
|
||||
t((wchar)codepoint);
|
||||
else
|
||||
if(codepoint < 0x110000) {
|
||||
codepoint -= 0x10000;
|
||||
t(wchar(0xD800 + (0x3ff & (codepoint >> 10))));
|
||||
t(wchar(0xDC00 + (0x3ff & codepoint)));
|
||||
}
|
||||
else
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
template <class Target>
|
||||
force_inline void FromUtf16_(Target& t, const wchar *s, int len)
|
||||
{
|
||||
const wchar *lim = s + len;
|
||||
while(s < lim) {
|
||||
if((*s & 0XFC00) == 0xD800 && s + 1 < lim && (s[1] & 0xFC00) == 0xDC00) {
|
||||
t(s, ((dword(s[0] & 0x3ff) << 10) | (s[1] & 0x3ff)) + 0x10000);
|
||||
s += 2;
|
||||
}
|
||||
else {
|
||||
t((const wchar *)s, *s);
|
||||
s++;
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue