github-starred/ultimatepp
2
0
Fork 0
mirror of https://github.com/ultimatepp/ultimatepp.git synced 2026-07-11 22:03:01 -06:00
Code Issues 33 Projects Packages Wiki Activity Actions

ide: UTF16 support, UTF BOM autodetection #1049

Browse source 
git-svn-id: svn://ultimatepp.org/upp/trunk@8680 f0d560ea-af0d-0410-9eb7-867de7ffcac7
This commit is contained in:
cxl 2015-07-13 14:29:44 +00:00
parent 85ae3d1d80
commit 1f699baa8f
9 changed files with 146 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

2
uppsrc/Core/Bom.cpp
View file

@ -7,7 +7,7 @@ static void sLoadBom(Stream& in, String *t, WString *wt, byte def_charset)
if(in.IsOpen()) {
String s;
if(in.GetLeft() > 3) {
word header = in.Get16();
word header = in.Get16le();
if(header == 0xfffe || header == 0xfeff) {
int n = (int)in.GetLeft() / 2;
WStringBuffer ws(n);

24
uppsrc/Core/src.tpp/Stream$en-us.tpp
View file

@ -493,7 +493,7 @@ throw an exception) `- this is the difference from Get() method.&]
[s2; Reads 16`-bit value from the stream in platform specific format
(either little`-endian or big`-endian). If there is not enough
data in the stream, LoadError is invoked (that in turn might
throw an exception).&]
throw an exception) and `-1 returned.&]
[s7; [*/ Return value]-|16`-bit value.&]
[s3; &]
[s4;%- &]
@ -501,7 +501,8 @@ throw an exception).&]
[s2; Reads 32`-bit value from the stream in platform specific format
(either little`-endian or big`-endian). If there is not enough
data in the stream, LoadError is invoked (that in turn might
throw an exception).&]
throw an exception) and `-1 returned (but that is a valid return
value as well).&]
[s7; [*/ Return value]-|32`-bit value.&]
[s3; &]
[s4;%- &]
@ -509,49 +510,54 @@ throw an exception).&]
[s2; Reads 64`-bit value from the stream in platform specific format
(either little`-endian or big`-endian). If there is not enough
data in the stream, LoadError is invoked (that in turn might
throw an exception).&]
throw an exception) and `-1 returned (but that is a valid return
value as well).&]
[s7; [*/ Return value]-|64`-bit value.&]
[s3; &]
[s4;%- &]
[s5;:Stream`:`:Get16le`(`):%- [@(0.0.255) int]_[* Get16le]()&]
[s2; Reads 16`-bit value from the stream in the little`-endian mode.
If there is not enough data in the stream, LoadError is invoked
(that in turn might throw an exception).&]
(that in turn might throw an exception) and `-1 returned.&]
[s7; [*/ Return value]-|16`-bit value.&]
[s3; &]
[s4;%- &]
[s5;:Stream`:`:Get32le`(`):%- [@(0.0.255) int]_[* Get32le]()&]
[s2; Reads 32`-bit value from the stream in the little`-endian mode.
If there is not enough data in the stream, LoadError is invoked
(that in turn might throw an exception).&]
(that in turn might throw an exception) and `-1 returned (but
that is a valid return value as well).&]
[s7; [*/ Return value]-|32`-bit value.&]
[s3; &]
[s4;%- &]
[s5;:Stream`:`:Get64le`(`):%- [_^int64^ int64]_[* Get64le]()&]
[s2; Reads 64`-bit value from the stream in the little`-endian mode.
If there is not enough data in the stream, LoadError is invoked
(that in turn might throw an exception).&]
(that in turn might throw an exception) and `-1 returned (but
that is a valid value return as well).&]
[s7; [*/ Return value]-|64`-bit value.&]
[s3; &]
[s4;%- &]
[s5;:Stream`:`:Get16be`(`):%- [@(0.0.255) int]_[* Get16be]()&]
[s2; Reads 16`-bit value from the stream in the big`-endian mode.
If there is not enough data in the stream, LoadError is invoked
(that in turn might throw an exception).&]
(that in turn might throw an exception) and `-1 is returned.&]
[s7; [*/ Return value]-|16`-bit value.&]
[s3; &]
[s4;%- &]
[s5;:Stream`:`:Get32be`(`):%- [@(0.0.255) int]_[* Get32be]()&]
[s2; Reads 32`-bit value from the stream in the big`-endian mode.
If there is not enough data in the stream, LoadError is invoked
(that in turn might throw an exception).&]
(that in turn might throw an exception) and `-1 returned (but
that is a valid value return as well).&]
[s7; [*/ Return value]-|32`-bit value.&]
[s3; &]
[s4;%- &]
[s5;:Stream`:`:Get64be`(`):%- [_^int64^ int64]_[* Get64be]()&]
[s2; Reads 64`-bit value from the stream in the big`-endian mode.
If there is not enough data in the stream, LoadError is invoked
(that in turn might throw an exception).&]
(that in turn might throw an exception) and `-1 returned (but
that is a valid value return as well).&]
[s7; [*/ Return value]-|64`-bit value.&]
[s3; &]
[s4;%- &]

91
uppsrc/CtrlLib/Text.cpp
View file

@ -105,6 +105,12 @@ void TextCtrl::CacheLinePos(int linei)
}
}
bool TextCtrl::IsUnicodeCharset(byte charset)
{
return findarg(charset, CHARSET_UTF8, CHARSET_UTF8_BOM, CHARSET_UTF16_LE, CHARSET_UTF16_BE,
CHARSET_UTF16_LE_BOM, CHARSET_UTF16_BE_BOM) >= 0;
}
int TextCtrl::Load(Stream& in, byte charset) {
Clear();
line.Clear();
@ -120,9 +126,40 @@ int TextCtrl::Load(Stream& in, byte charset) {
in.Seek(pos);
charset = CHARSET_UTF8;
}
int be16 = findarg(charset, CHARSET_UTF16_LE_BOM, CHARSET_UTF16_BE_BOM);
if(be16 >= 0 && in.GetLeft() >= 2) {
int64 pos = in.GetPos();
dword h = in.Get16le();
if(h != (be16 ? 0xfffe : 0xfeff))
in.Seek(pos);
charset = be16 ? CHARSET_UTF16_BE : CHARSET_UTF16_LE;
}
bool cr = false;
byte b8 = 0;
truncated = true;
if(charset == CHARSET_UTF16_LE || charset == CHARSET_UTF16_BE) {
WStringBuffer wln;
for(;;) {
int c = charset == CHARSET_UTF16_LE ? in.Get16le() : in.Get16be();
if(c < 0) {
WString l = wln;
line.Add(l);
total += l.GetCount();
goto finish;
}
if(c == '\r')
cr = true;
else
if(c == '\n') {
WString l = wln;
line.Add(l);
total += l.GetCount() + 1;
wln.Clear();
}
else
wln.Cat(c);
}
}
else
if(charset == CHARSET_UTF8)
for(;;) {
byte h[200];
@ -233,19 +270,19 @@ int TextCtrl::Load(Stream& in, byte charset) {
}
}
truncated = false;
out_of_limit:
WString w = ToUnicode(~ln, ln.GetCount(), charset);
if(total + w.GetLength() <= max_total) {
line.Add(w);
total += w.GetLength();
{
WString w = ToUnicode(~ln, ln.GetCount(), charset);
if(total + w.GetLength() <= max_total) {
line.Add(w);
total += w.GetLength();
}
}
finish:
InsertLines(0, line.GetCount());
Update();
SetSb();
PlaceCaret(0);
if(truncated)
SetReadOnly();
return line.GetCount() > 1 ? cr ? LE_CRLF : LE_LF : LE_DEFAULT;
}
@ -257,6 +294,14 @@ void TextCtrl::Save(Stream& s, byte charset, int line_endings) const {
s.Put(bom, 3);
charset = CHARSET_UTF8;
}
if(charset == CHARSET_UTF16_LE_BOM) {
s.Put16le(0xfeff);
charset = CHARSET_UTF16_LE;
}
if(charset == CHARSET_UTF16_BE_BOM) {
s.Put16be(0xfeff);
charset = CHARSET_UTF16_BE;
}
charset = ResolveCharset(charset);
String le = "\n";
#ifdef PLATFORM_WIN32
@ -265,6 +310,30 @@ void TextCtrl::Save(Stream& s, byte charset, int line_endings) const {
#endif
if(line_endings == LE_CRLF)
le = "\r\n";
int be16 = findarg(charset, CHARSET_UTF16_LE, CHARSET_UTF16_BE);
if(be16 >= 0) {
String wle;
for(int i = 0; i < le.GetCount(); i++) {
if(be16)
wle.Cat(0);
wle.Cat(le[i]);
if(!be16)
wle.Cat(0);
}
for(int i = 0; i < line.GetCount(); i++) {
if(i)
s.Put(wle);
WString txt = line[i];
const wchar *e = txt.End();
if(be16)
for(const wchar *w = txt; w != e; w++)
s.Put16be(*w);
else
for(const wchar *w = txt; w != e; w++)
s.Put16le(*w);
}
return;
}
for(int i = 0; i < line.GetCount(); i++) {
if(i)
s.Put(le);
@ -294,7 +363,7 @@ String TextCtrl::Get(byte charset) const
int TextCtrl::GetInvalidCharPos(byte charset) const
{
int q = 0;
if(charset != CHARSET_UTF8 && charset != CHARSET_UTF8_BOM)
if(!IsUnicodeCharset(charset))
for(int i = 0; i < line.GetCount(); i++) {
WString txt = line[i];
WString ctxt = ToUnicode(FromUnicode(txt, charset), charset);
@ -349,7 +418,7 @@ Value TextCtrl::GetData() const
String TextCtrl::GetEncodedLine(int i, byte charset) const
{
charset = ResolveCharset(charset);
if(charset == CHARSET_UTF8 && charset != CHARSET_UTF8_BOM)
if(charset == CHARSET_UTF8)
return line[i].text;
return FromUnicode(FromUtf8(line[i].text), charset);
}
@ -648,7 +717,7 @@ int TextCtrl::Insert(int pos, const WString& _txt, bool typing) {
if(pos + _txt.GetCount() > max_total)
return 0;
WString txt = _txt;
if(charset != CHARSET_UNICODE && charset != CHARSET_UTF8_BOM)
if(!IsUnicodeCharset(charset))
for(int i = 0; i < txt.GetCount(); i++)
if(FromUnicode(txt[i], charset) == DEFAULTCHAR)
txt.Set(i, '?');

3
uppsrc/CtrlLib/TextEdit.h
View file

@ -98,6 +98,7 @@ protected:
void DoPaste() { Paste(); }
void DoRemoveSelection() { RemoveSelection(); }
void RefreshLines(int l1, int l2);
static bool IsUnicodeCharset(byte charset);
public:
virtual void RefreshLine(int i);
@ -109,7 +110,7 @@ public:
void CachePos(int pos);
void CacheLinePos(int linei);
enum { CHARSET_UTF8_BOM = 250 };
enum { CHARSET_UTF8_BOM = 250, CHARSET_UTF16_LE, CHARSET_UTF16_BE, CHARSET_UTF16_LE_BOM, CHARSET_UTF16_BE_BOM };
enum { LE_DEFAULT, LE_CRLF, LE_LF };
int Load(Stream& s, byte charset = CHARSET_DEFAULT);

10
uppsrc/ide/Core/Core.h
View file

@ -274,7 +274,13 @@ int GetType(const Vector<String>& conf, const char *flags, int def);
bool GetFlag(const Vector<String>& conf, const char *flag);
String RemoveType(Vector<String>& conf, const char *flags);
enum { CHARSET_UTF8_BOM = 250 }; // Same as TextCtrl::CHARSET_UTF8_BOM
enum {
CHARSET_UTF8_BOM = 250, // same as TextCtrl::CHARSET_UTF8_BOM; CtrlLib not included here
CHARSET_UTF16_LE,
CHARSET_UTF16_BE,
CHARSET_UTF16_LE_BOM,
CHARSET_UTF16_BE_BOM
};
class Package {
void Reset();
@ -338,6 +344,8 @@ public:
Package();
};
String IdeCharsetName(byte charset);
class Workspace {
void AddUses(Package& p, bool match, const Vector<String>& flag);
void AddLoad(const String& name, bool match, const Vector<String>& flag);

30
uppsrc/ide/Core/Package.cpp
View file

@ -187,7 +187,12 @@ void Package::SetPackageResolver(bool (*Resolve)(const String& error, const Stri
byte CharsetByNameX(const String& s)
{
return s == "UTF-8-BOM" ? CHARSET_UTF8_BOM : CharsetByName(s);
return decode(s, "UTF-8-BOM", CHARSET_UTF8_BOM,
"UTF-16-LE", CHARSET_UTF16_LE,
"UTF-16-BE", CHARSET_UTF16_BE,
"UTF-16-LE-BOM", CHARSET_UTF16_LE_BOM,
"UTF-16-BE-BOM", CHARSET_UTF16_BE_BOM,
CharsetByName(s));
}
void Package::Option(bool& option, const char *name)
@ -409,6 +414,15 @@ void putfopt(Stream& out, const char *key, const Array<OptItem>& m)
out << "\n\t\t" << key << AsStringWhen(m[i].when) << ' ' << WriteValue(m[i].text);
}
String IdeCharsetName(byte charset) {
return decode(charset, CHARSET_UTF8_BOM, "UTF-8-BOM",
CHARSET_UTF16_LE, "UTF-16-LE",
CHARSET_UTF16_BE, "UTF-16-BE",
CHARSET_UTF16_LE_BOM, "UTF-16-LE-BOM",
CHARSET_UTF16_BE_BOM, "UTF-16-BE-BOM",
CharsetName(charset));
}
bool Package::Save(const char *path) const {
StringStream out;
if(description.GetCount() || italic || bold || !IsNull(ink)) {
@ -422,11 +436,8 @@ bool Package::Save(const char *path) const {
d << (int)ink.GetR() << ',' << (int)ink.GetG() << ',' << (int)ink.GetB();
out << "description " << AsCString(d) << ";\n\n";
}
if(charset > 0 && charset < CharsetCount() || charset == CHARSET_UTF8)
out << "charset " << AsCString(CharsetName(charset)) << ";\n\n";
else
if(charset == CHARSET_UTF8_BOM)
out << "charset \"UTF-8-BOM\";\n\n";
if(charset > 0)
out << "charset " << AsCString(IdeCharsetName(charset)) << ";\n\n";
if(optimize_speed)
out << "optimize_speed;\n\n";
if(noblitz)
@ -465,11 +476,8 @@ bool Package::Save(const char *path) const {
out << " options(BUILDER_OPTION) NOPCH";
if(f.noblitz)
out << " options(BUILDER_OPTION) NOBLITZ";
if(f.charset > 0 && f.charset < CharsetCount() || f.charset == CHARSET_UTF8)
out << " charset " << AsCString(CharsetName(f.charset));
else
if(f.charset == CHARSET_UTF8_BOM)
out << " charset \"UTF-8-BOM\"";
if(f.charset > 0)
out << " charset " << AsCString(IdeCharsetName(f.charset));
if(!IsNull(f.highlight))
out << " highlight " << f.highlight;
putfopt(out, "options", f.option);

4
uppsrc/ide/Setup.cpp
View file

@ -11,6 +11,10 @@ void DlCharset(DropList& d)
{
d.Add(CHARSET_UTF8, "UTF8");
d.Add(TextCtrl::CHARSET_UTF8_BOM, "UTF8 BOM");
d.Add(TextCtrl::CHARSET_UTF16_LE, "UTF16 LE");
d.Add(TextCtrl::CHARSET_UTF16_BE, "UTF16 BE");
d.Add(TextCtrl::CHARSET_UTF16_LE_BOM, "UTF16 LE BOM");
d.Add(TextCtrl::CHARSET_UTF16_BE_BOM, "UTF16 BE BOM");
for(int i = 1; i < CharsetCount(); i++)
d.Add(i, CharsetName(i));
}

2
uppsrc/ide/ide.cpp
View file

@ -26,7 +26,7 @@ void Ide::MakeTitle()
if(!editfile.IsEmpty()) {
title << " - [" << editfile;
int chrset = editor.GetCharset();
title << " " << (chrset == TextCtrl::CHARSET_UTF8_BOM ? "UTF-8 BOM" : CharsetName(chrset))
title << " " << IdeCharsetName(chrset)
<< " " << (findarg(Nvl(editfile_line_endings, line_endings), LF, DETECT_LF) >= 0 ? "LF" : "CRLF");
if(editor.IsReadOnly())
title << " (Read Only)";

17
uppsrc/ide/idefile.cpp
View file

@ -29,7 +29,7 @@ void Ide::FileCursor()
{
WorkspaceWork::FileCursor();
if(IsActiveFile() && !filelist[filelist.GetCursor()].isdir) {
const Package::File& f = ActiveFile();
Package::File& f = ActiveFile();
editor.SetEditable(!f.readonly);
editor.TabSize(f.tabsize > 0 ? f.tabsize : editortabsize);
SetupEditor();
@ -47,6 +47,7 @@ void Ide::FileCursor()
String p = GetActiveFileName();
if(p != HELPNAME)
p = GetActiveFilePath();
EditFile0(p, f.charset ? f.charset : actual.charset ? actual.charset : default_charset,
false, headername);
}
@ -471,7 +472,8 @@ void Ide::EditFile0(const String& path, byte charset, bool astext, const String&
designer = GetIdeModule(i).CreateDesigner(this, path, charset);
}
if(!designer && editastext.Find(path) < 0 && (FileIsBinary(path) || editashex.Find(path) >= 0))
if(!designer && editastext.Find(path) < 0 &&
(charset == CHARSET_DEFAULT && FileIsBinary(path) || editashex.Find(path) >= 0))
designer.Create<FileHexView>().Open(path);
if(designer) {
@ -522,6 +524,17 @@ void Ide::EditFile0(const String& path, byte charset, bool astext, const String&
editor.SetCharset(CHARSET_UTF8);
}
else {
String s = in.Get(3);
DUMPHEX(s);
if(s.GetCount() >= 2) {
if((byte)s[0] == 0xff && (byte)s[1] == 0xfe)
charset = CHARSET_UTF16_LE_BOM;
if((byte)s[0] == 0xfe && (byte)s[1] == 0xff)
charset = CHARSET_UTF16_BE_BOM;
}
if(s.GetCount() >= 3 && (byte)s[0] == 0xef && (byte)s[1] == 0xbb && (byte)s[2] == 0xbf)
charset = CHARSET_UTF8_BOM;
in.Seek(0);
int le = editor.Load(in, charset);
editfile_line_endings = le == TextCtrl::LE_CRLF ? CRLF : le == TextCtrl::LE_LF ? LF : (int)Null;
}