From 1f699baa8f9e99fc978413c88ab5c7ec5468ff57 Mon Sep 17 00:00:00 2001 From: cxl Date: Mon, 13 Jul 2015 14:29:44 +0000 Subject: [PATCH] ide: UTF16 support, UTF BOM autodetection #1049 git-svn-id: svn://ultimatepp.org/upp/trunk@8680 f0d560ea-af0d-0410-9eb7-867de7ffcac7 --- uppsrc/Core/Bom.cpp | 2 +- uppsrc/Core/src.tpp/Stream$en-us.tpp | 24 +++++--- uppsrc/CtrlLib/Text.cpp | 91 ++++++++++++++++++++++++---- uppsrc/CtrlLib/TextEdit.h | 3 +- uppsrc/ide/Core/Core.h | 10 ++- uppsrc/ide/Core/Package.cpp | 30 +++++---- uppsrc/ide/Setup.cpp | 4 ++ uppsrc/ide/ide.cpp | 2 +- uppsrc/ide/idefile.cpp | 17 +++++- 9 files changed, 146 insertions(+), 37 deletions(-) diff --git a/uppsrc/Core/Bom.cpp b/uppsrc/Core/Bom.cpp index 13498a897..fff8e5423 100644 --- a/uppsrc/Core/Bom.cpp +++ b/uppsrc/Core/Bom.cpp @@ -7,7 +7,7 @@ static void sLoadBom(Stream& in, String *t, WString *wt, byte def_charset) if(in.IsOpen()) { String s; if(in.GetLeft() > 3) { - word header = in.Get16(); + word header = in.Get16le(); if(header == 0xfffe || header == 0xfeff) { int n = (int)in.GetLeft() / 2; WStringBuffer ws(n); diff --git a/uppsrc/Core/src.tpp/Stream$en-us.tpp b/uppsrc/Core/src.tpp/Stream$en-us.tpp index 95192bb04..a8ef799cf 100644 --- a/uppsrc/Core/src.tpp/Stream$en-us.tpp +++ b/uppsrc/Core/src.tpp/Stream$en-us.tpp @@ -493,7 +493,7 @@ throw an exception) `- this is the difference from Get() method.&] [s2; Reads 16`-bit value from the stream in platform specific format (either little`-endian or big`-endian). If there is not enough data in the stream, LoadError is invoked (that in turn might -throw an exception).&] +throw an exception) and `-1 returned.&] [s7; [*/ Return value]-|16`-bit value.&] [s3; &] [s4;%- &] @@ -501,7 +501,8 @@ throw an exception).&] [s2; Reads 32`-bit value from the stream in platform specific format (either little`-endian or big`-endian). If there is not enough data in the stream, LoadError is invoked (that in turn might -throw an exception).&] +throw an exception) and `-1 returned (but that is a valid return +value as well).&] [s7; [*/ Return value]-|32`-bit value.&] [s3; &] [s4;%- &] @@ -509,49 +510,54 @@ throw an exception).&] [s2; Reads 64`-bit value from the stream in platform specific format (either little`-endian or big`-endian). If there is not enough data in the stream, LoadError is invoked (that in turn might -throw an exception).&] +throw an exception) and `-1 returned (but that is a valid return +value as well).&] [s7; [*/ Return value]-|64`-bit value.&] [s3; &] [s4;%- &] [s5;:Stream`:`:Get16le`(`):%- [@(0.0.255) int]_[* Get16le]()&] [s2; Reads 16`-bit value from the stream in the little`-endian mode. If there is not enough data in the stream, LoadError is invoked -(that in turn might throw an exception).&] +(that in turn might throw an exception) and `-1 returned.&] [s7; [*/ Return value]-|16`-bit value.&] [s3; &] [s4;%- &] [s5;:Stream`:`:Get32le`(`):%- [@(0.0.255) int]_[* Get32le]()&] [s2; Reads 32`-bit value from the stream in the little`-endian mode. If there is not enough data in the stream, LoadError is invoked -(that in turn might throw an exception).&] +(that in turn might throw an exception) and `-1 returned (but +that is a valid return value as well).&] [s7; [*/ Return value]-|32`-bit value.&] [s3; &] [s4;%- &] [s5;:Stream`:`:Get64le`(`):%- [_^int64^ int64]_[* Get64le]()&] [s2; Reads 64`-bit value from the stream in the little`-endian mode. If there is not enough data in the stream, LoadError is invoked -(that in turn might throw an exception).&] +(that in turn might throw an exception) and `-1 returned (but +that is a valid value return as well).&] [s7; [*/ Return value]-|64`-bit value.&] [s3; &] [s4;%- &] [s5;:Stream`:`:Get16be`(`):%- [@(0.0.255) int]_[* Get16be]()&] [s2; Reads 16`-bit value from the stream in the big`-endian mode. If there is not enough data in the stream, LoadError is invoked -(that in turn might throw an exception).&] +(that in turn might throw an exception) and `-1 is returned.&] [s7; [*/ Return value]-|16`-bit value.&] [s3; &] [s4;%- &] [s5;:Stream`:`:Get32be`(`):%- [@(0.0.255) int]_[* Get32be]()&] [s2; Reads 32`-bit value from the stream in the big`-endian mode. If there is not enough data in the stream, LoadError is invoked -(that in turn might throw an exception).&] +(that in turn might throw an exception) and `-1 returned (but +that is a valid value return as well).&] [s7; [*/ Return value]-|32`-bit value.&] [s3; &] [s4;%- &] [s5;:Stream`:`:Get64be`(`):%- [_^int64^ int64]_[* Get64be]()&] [s2; Reads 64`-bit value from the stream in the big`-endian mode. If there is not enough data in the stream, LoadError is invoked -(that in turn might throw an exception).&] +(that in turn might throw an exception) and `-1 returned (but +that is a valid value return as well).&] [s7; [*/ Return value]-|64`-bit value.&] [s3; &] [s4;%- &] diff --git a/uppsrc/CtrlLib/Text.cpp b/uppsrc/CtrlLib/Text.cpp index a98c81c91..bed8c9af3 100644 --- a/uppsrc/CtrlLib/Text.cpp +++ b/uppsrc/CtrlLib/Text.cpp @@ -105,6 +105,12 @@ void TextCtrl::CacheLinePos(int linei) } } +bool TextCtrl::IsUnicodeCharset(byte charset) +{ + return findarg(charset, CHARSET_UTF8, CHARSET_UTF8_BOM, CHARSET_UTF16_LE, CHARSET_UTF16_BE, + CHARSET_UTF16_LE_BOM, CHARSET_UTF16_BE_BOM) >= 0; +} + int TextCtrl::Load(Stream& in, byte charset) { Clear(); line.Clear(); @@ -120,9 +126,40 @@ int TextCtrl::Load(Stream& in, byte charset) { in.Seek(pos); charset = CHARSET_UTF8; } + int be16 = findarg(charset, CHARSET_UTF16_LE_BOM, CHARSET_UTF16_BE_BOM); + if(be16 >= 0 && in.GetLeft() >= 2) { + int64 pos = in.GetPos(); + dword h = in.Get16le(); + if(h != (be16 ? 0xfffe : 0xfeff)) + in.Seek(pos); + charset = be16 ? CHARSET_UTF16_BE : CHARSET_UTF16_LE; + } bool cr = false; byte b8 = 0; - truncated = true; + if(charset == CHARSET_UTF16_LE || charset == CHARSET_UTF16_BE) { + WStringBuffer wln; + for(;;) { + int c = charset == CHARSET_UTF16_LE ? in.Get16le() : in.Get16be(); + if(c < 0) { + WString l = wln; + line.Add(l); + total += l.GetCount(); + goto finish; + } + if(c == '\r') + cr = true; + else + if(c == '\n') { + WString l = wln; + line.Add(l); + total += l.GetCount() + 1; + wln.Clear(); + } + else + wln.Cat(c); + } + } + else if(charset == CHARSET_UTF8) for(;;) { byte h[200]; @@ -233,19 +270,19 @@ int TextCtrl::Load(Stream& in, byte charset) { } } - truncated = false; out_of_limit: - WString w = ToUnicode(~ln, ln.GetCount(), charset); - if(total + w.GetLength() <= max_total) { - line.Add(w); - total += w.GetLength(); + { + WString w = ToUnicode(~ln, ln.GetCount(), charset); + if(total + w.GetLength() <= max_total) { + line.Add(w); + total += w.GetLength(); + } } +finish: InsertLines(0, line.GetCount()); Update(); SetSb(); PlaceCaret(0); - if(truncated) - SetReadOnly(); return line.GetCount() > 1 ? cr ? LE_CRLF : LE_LF : LE_DEFAULT; } @@ -257,6 +294,14 @@ void TextCtrl::Save(Stream& s, byte charset, int line_endings) const { s.Put(bom, 3); charset = CHARSET_UTF8; } + if(charset == CHARSET_UTF16_LE_BOM) { + s.Put16le(0xfeff); + charset = CHARSET_UTF16_LE; + } + if(charset == CHARSET_UTF16_BE_BOM) { + s.Put16be(0xfeff); + charset = CHARSET_UTF16_BE; + } charset = ResolveCharset(charset); String le = "\n"; #ifdef PLATFORM_WIN32 @@ -265,6 +310,30 @@ void TextCtrl::Save(Stream& s, byte charset, int line_endings) const { #endif if(line_endings == LE_CRLF) le = "\r\n"; + int be16 = findarg(charset, CHARSET_UTF16_LE, CHARSET_UTF16_BE); + if(be16 >= 0) { + String wle; + for(int i = 0; i < le.GetCount(); i++) { + if(be16) + wle.Cat(0); + wle.Cat(le[i]); + if(!be16) + wle.Cat(0); + } + for(int i = 0; i < line.GetCount(); i++) { + if(i) + s.Put(wle); + WString txt = line[i]; + const wchar *e = txt.End(); + if(be16) + for(const wchar *w = txt; w != e; w++) + s.Put16be(*w); + else + for(const wchar *w = txt; w != e; w++) + s.Put16le(*w); + } + return; + } for(int i = 0; i < line.GetCount(); i++) { if(i) s.Put(le); @@ -294,7 +363,7 @@ String TextCtrl::Get(byte charset) const int TextCtrl::GetInvalidCharPos(byte charset) const { int q = 0; - if(charset != CHARSET_UTF8 && charset != CHARSET_UTF8_BOM) + if(!IsUnicodeCharset(charset)) for(int i = 0; i < line.GetCount(); i++) { WString txt = line[i]; WString ctxt = ToUnicode(FromUnicode(txt, charset), charset); @@ -349,7 +418,7 @@ Value TextCtrl::GetData() const String TextCtrl::GetEncodedLine(int i, byte charset) const { charset = ResolveCharset(charset); - if(charset == CHARSET_UTF8 && charset != CHARSET_UTF8_BOM) + if(charset == CHARSET_UTF8) return line[i].text; return FromUnicode(FromUtf8(line[i].text), charset); } @@ -648,7 +717,7 @@ int TextCtrl::Insert(int pos, const WString& _txt, bool typing) { if(pos + _txt.GetCount() > max_total) return 0; WString txt = _txt; - if(charset != CHARSET_UNICODE && charset != CHARSET_UTF8_BOM) + if(!IsUnicodeCharset(charset)) for(int i = 0; i < txt.GetCount(); i++) if(FromUnicode(txt[i], charset) == DEFAULTCHAR) txt.Set(i, '?'); diff --git a/uppsrc/CtrlLib/TextEdit.h b/uppsrc/CtrlLib/TextEdit.h index e17374ba5..8cf966296 100644 --- a/uppsrc/CtrlLib/TextEdit.h +++ b/uppsrc/CtrlLib/TextEdit.h @@ -98,6 +98,7 @@ protected: void DoPaste() { Paste(); } void DoRemoveSelection() { RemoveSelection(); } void RefreshLines(int l1, int l2); + static bool IsUnicodeCharset(byte charset); public: virtual void RefreshLine(int i); @@ -109,7 +110,7 @@ public: void CachePos(int pos); void CacheLinePos(int linei); - enum { CHARSET_UTF8_BOM = 250 }; + enum { CHARSET_UTF8_BOM = 250, CHARSET_UTF16_LE, CHARSET_UTF16_BE, CHARSET_UTF16_LE_BOM, CHARSET_UTF16_BE_BOM }; enum { LE_DEFAULT, LE_CRLF, LE_LF }; int Load(Stream& s, byte charset = CHARSET_DEFAULT); diff --git a/uppsrc/ide/Core/Core.h b/uppsrc/ide/Core/Core.h index 07460d9d4..d66faef46 100644 --- a/uppsrc/ide/Core/Core.h +++ b/uppsrc/ide/Core/Core.h @@ -274,7 +274,13 @@ int GetType(const Vector& conf, const char *flags, int def); bool GetFlag(const Vector& conf, const char *flag); String RemoveType(Vector& conf, const char *flags); -enum { CHARSET_UTF8_BOM = 250 }; // Same as TextCtrl::CHARSET_UTF8_BOM +enum { + CHARSET_UTF8_BOM = 250, // same as TextCtrl::CHARSET_UTF8_BOM; CtrlLib not included here + CHARSET_UTF16_LE, + CHARSET_UTF16_BE, + CHARSET_UTF16_LE_BOM, + CHARSET_UTF16_BE_BOM +}; class Package { void Reset(); @@ -338,6 +344,8 @@ public: Package(); }; +String IdeCharsetName(byte charset); + class Workspace { void AddUses(Package& p, bool match, const Vector& flag); void AddLoad(const String& name, bool match, const Vector& flag); diff --git a/uppsrc/ide/Core/Package.cpp b/uppsrc/ide/Core/Package.cpp index 32bebbd4b..ac38bbf6b 100644 --- a/uppsrc/ide/Core/Package.cpp +++ b/uppsrc/ide/Core/Package.cpp @@ -187,7 +187,12 @@ void Package::SetPackageResolver(bool (*Resolve)(const String& error, const Stri byte CharsetByNameX(const String& s) { - return s == "UTF-8-BOM" ? CHARSET_UTF8_BOM : CharsetByName(s); + return decode(s, "UTF-8-BOM", CHARSET_UTF8_BOM, + "UTF-16-LE", CHARSET_UTF16_LE, + "UTF-16-BE", CHARSET_UTF16_BE, + "UTF-16-LE-BOM", CHARSET_UTF16_LE_BOM, + "UTF-16-BE-BOM", CHARSET_UTF16_BE_BOM, + CharsetByName(s)); } void Package::Option(bool& option, const char *name) @@ -409,6 +414,15 @@ void putfopt(Stream& out, const char *key, const Array& m) out << "\n\t\t" << key << AsStringWhen(m[i].when) << ' ' << WriteValue(m[i].text); } +String IdeCharsetName(byte charset) { + return decode(charset, CHARSET_UTF8_BOM, "UTF-8-BOM", + CHARSET_UTF16_LE, "UTF-16-LE", + CHARSET_UTF16_BE, "UTF-16-BE", + CHARSET_UTF16_LE_BOM, "UTF-16-LE-BOM", + CHARSET_UTF16_BE_BOM, "UTF-16-BE-BOM", + CharsetName(charset)); +} + bool Package::Save(const char *path) const { StringStream out; if(description.GetCount() || italic || bold || !IsNull(ink)) { @@ -422,11 +436,8 @@ bool Package::Save(const char *path) const { d << (int)ink.GetR() << ',' << (int)ink.GetG() << ',' << (int)ink.GetB(); out << "description " << AsCString(d) << ";\n\n"; } - if(charset > 0 && charset < CharsetCount() || charset == CHARSET_UTF8) - out << "charset " << AsCString(CharsetName(charset)) << ";\n\n"; - else - if(charset == CHARSET_UTF8_BOM) - out << "charset \"UTF-8-BOM\";\n\n"; + if(charset > 0) + out << "charset " << AsCString(IdeCharsetName(charset)) << ";\n\n"; if(optimize_speed) out << "optimize_speed;\n\n"; if(noblitz) @@ -465,11 +476,8 @@ bool Package::Save(const char *path) const { out << " options(BUILDER_OPTION) NOPCH"; if(f.noblitz) out << " options(BUILDER_OPTION) NOBLITZ"; - if(f.charset > 0 && f.charset < CharsetCount() || f.charset == CHARSET_UTF8) - out << " charset " << AsCString(CharsetName(f.charset)); - else - if(f.charset == CHARSET_UTF8_BOM) - out << " charset \"UTF-8-BOM\""; + if(f.charset > 0) + out << " charset " << AsCString(IdeCharsetName(f.charset)); if(!IsNull(f.highlight)) out << " highlight " << f.highlight; putfopt(out, "options", f.option); diff --git a/uppsrc/ide/Setup.cpp b/uppsrc/ide/Setup.cpp index be675ef68..a2dd0a85f 100644 --- a/uppsrc/ide/Setup.cpp +++ b/uppsrc/ide/Setup.cpp @@ -11,6 +11,10 @@ void DlCharset(DropList& d) { d.Add(CHARSET_UTF8, "UTF8"); d.Add(TextCtrl::CHARSET_UTF8_BOM, "UTF8 BOM"); + d.Add(TextCtrl::CHARSET_UTF16_LE, "UTF16 LE"); + d.Add(TextCtrl::CHARSET_UTF16_BE, "UTF16 BE"); + d.Add(TextCtrl::CHARSET_UTF16_LE_BOM, "UTF16 LE BOM"); + d.Add(TextCtrl::CHARSET_UTF16_BE_BOM, "UTF16 BE BOM"); for(int i = 1; i < CharsetCount(); i++) d.Add(i, CharsetName(i)); } diff --git a/uppsrc/ide/ide.cpp b/uppsrc/ide/ide.cpp index fb99a9351..bf97a9c1a 100644 --- a/uppsrc/ide/ide.cpp +++ b/uppsrc/ide/ide.cpp @@ -26,7 +26,7 @@ void Ide::MakeTitle() if(!editfile.IsEmpty()) { title << " - [" << editfile; int chrset = editor.GetCharset(); - title << " " << (chrset == TextCtrl::CHARSET_UTF8_BOM ? "UTF-8 BOM" : CharsetName(chrset)) + title << " " << IdeCharsetName(chrset) << " " << (findarg(Nvl(editfile_line_endings, line_endings), LF, DETECT_LF) >= 0 ? "LF" : "CRLF"); if(editor.IsReadOnly()) title << " (Read Only)"; diff --git a/uppsrc/ide/idefile.cpp b/uppsrc/ide/idefile.cpp index bac2c3465..284ff1834 100644 --- a/uppsrc/ide/idefile.cpp +++ b/uppsrc/ide/idefile.cpp @@ -29,7 +29,7 @@ void Ide::FileCursor() { WorkspaceWork::FileCursor(); if(IsActiveFile() && !filelist[filelist.GetCursor()].isdir) { - const Package::File& f = ActiveFile(); + Package::File& f = ActiveFile(); editor.SetEditable(!f.readonly); editor.TabSize(f.tabsize > 0 ? f.tabsize : editortabsize); SetupEditor(); @@ -47,6 +47,7 @@ void Ide::FileCursor() String p = GetActiveFileName(); if(p != HELPNAME) p = GetActiveFilePath(); + EditFile0(p, f.charset ? f.charset : actual.charset ? actual.charset : default_charset, false, headername); } @@ -471,7 +472,8 @@ void Ide::EditFile0(const String& path, byte charset, bool astext, const String& designer = GetIdeModule(i).CreateDesigner(this, path, charset); } - if(!designer && editastext.Find(path) < 0 && (FileIsBinary(path) || editashex.Find(path) >= 0)) + if(!designer && editastext.Find(path) < 0 && + (charset == CHARSET_DEFAULT && FileIsBinary(path) || editashex.Find(path) >= 0)) designer.Create().Open(path); if(designer) { @@ -522,6 +524,17 @@ void Ide::EditFile0(const String& path, byte charset, bool astext, const String& editor.SetCharset(CHARSET_UTF8); } else { + String s = in.Get(3); + DUMPHEX(s); + if(s.GetCount() >= 2) { + if((byte)s[0] == 0xff && (byte)s[1] == 0xfe) + charset = CHARSET_UTF16_LE_BOM; + if((byte)s[0] == 0xfe && (byte)s[1] == 0xff) + charset = CHARSET_UTF16_BE_BOM; + } + if(s.GetCount() >= 3 && (byte)s[0] == 0xef && (byte)s[1] == 0xbb && (byte)s[2] == 0xbf) + charset = CHARSET_UTF8_BOM; + in.Seek(0); int le = editor.Load(in, charset); editfile_line_endings = le == TextCtrl::LE_CRLF ? CRLF : le == TextCtrl::LE_LF ? LF : (int)Null; }