From 9fb35c2ca348749acfaf0efe638bf2f7612cc174 Mon Sep 17 00:00:00 2001 From: cxl Date: Mon, 2 Nov 2009 09:43:25 +0000 Subject: [PATCH] RichEdit related: MakeSpellScd utility to create .scd spelling checker files, also fixed RichEdit to support UTF-8 based format git-svn-id: svn://ultimatepp.org/upp/trunk@1673 f0d560ea-af0d-0410-9eb7-867de7ffcac7 --- uppsrc/Core/CharSet.cpp | 15 +- uppsrc/Core/CharSet.h | 1 + uppsrc/MakeSpellScd/MakeSpellScd.cpp | 234 +++++++++++++++++++++++++++ uppsrc/MakeSpellScd/MakeSpellScd.upp | 13 ++ uppsrc/MakeSpellScd/init | 4 + uppsrc/RichEdit/Speller.cpp | 60 ++++--- 6 files changed, 300 insertions(+), 27 deletions(-) create mode 100644 uppsrc/MakeSpellScd/MakeSpellScd.cpp create mode 100644 uppsrc/MakeSpellScd/MakeSpellScd.upp create mode 100644 uppsrc/MakeSpellScd/init diff --git a/uppsrc/Core/CharSet.cpp b/uppsrc/Core/CharSet.cpp index 564093086..84b89a1c3 100644 --- a/uppsrc/Core/CharSet.cpp +++ b/uppsrc/Core/CharSet.cpp @@ -2786,7 +2786,6 @@ void ToAscii(char *s, int len, byte charset) ToAscii(s, s, len, charset); } - WString ToUnicode(const char *src, int l, byte charset) { charset = ResolveCharset(charset); @@ -2802,17 +2801,21 @@ WString ToUnicode(const String& src, byte charset) return ToUnicode(src, src.GetLength(), charset); } -String FromUnicode(const WString& src, byte charset, int def) +String FromUnicodeBuffer(const wchar *src, int len, byte charset, int defchar) { charset = ResolveCharset(charset); if(charset == CHARSET_UTF8) - return ToUtf8(src); - int l = src.GetLength(); - StringBuffer result(l); - FromUnicode(result, src, l, charset, def); + return ToUtf8(src, len); + StringBuffer result(len); + FromUnicode(result, src, len, charset, defchar); return result; } +String FromUnicode(const WString& src, byte charset, int defchar) +{ + return FromUnicodeBuffer(~src, src.GetCount(), charset, defchar); +} + String ToCharset(byte charset, const String& src, byte scharset, int def) { charset = ResolveCharset(charset); diff --git a/uppsrc/Core/CharSet.h b/uppsrc/Core/CharSet.h index be9a29798..8c960c2ad 100644 --- a/uppsrc/Core/CharSet.h +++ b/uppsrc/Core/CharSet.h @@ -268,6 +268,7 @@ bool CheckUtf8(const String& src); WString ToUnicode(const String& src, byte charset); WString ToUnicode(const char *src, int n, byte charset); +String FromUnicodeBuffer(const wchar *src, int len, byte charset = CHARSET_DEFAULT, int defchar = DEFAULTCHAR); String FromUnicode(const WString& src, byte charset = CHARSET_DEFAULT, int defchar = DEFAULTCHAR); String ToCharset(byte charset, const String& s, byte scharset = CHARSET_DEFAULT, int defchar = DEFAULTCHAR); diff --git a/uppsrc/MakeSpellScd/MakeSpellScd.cpp b/uppsrc/MakeSpellScd/MakeSpellScd.cpp new file mode 100644 index 000000000..d1683b26e --- /dev/null +++ b/uppsrc/MakeSpellScd/MakeSpellScd.cpp @@ -0,0 +1,234 @@ +#include + +#define LLOG(x) // LOG(x) +#define LLOGHEXDUMP(a, b) // LOGHEXDUMP(a, b) + +using namespace Upp; + +byte charset; +int vocn[256]; +Vector voc; +VectorMap line; + +int LineCode(const String& us) +{ + WString s = FromUtf8(us); + return ToLower(ToAscii(s[0]), CHARSET_DEFAULT) + + (ToLower(ToAscii(s[1]), CHARSET_DEFAULT) << 8) + + (s.GetCount() < 3 ? 127 : (ToLower(ToAscii(s[2]), CHARSET_DEFAULT) << 16)); +} + +bool Contains(const String& a, const String& b) +{ + for(int i = 0; i + b.GetLength() <= a.GetLength(); i++) + if(memcmp(~a + i, ~b, b.GetLength()) == 0) return true; + return false; +} + +struct LengthOrder +{ + bool operator()(const String& a, const String& b) const + { + return a.GetLength() > b.GetLength(); + } +}; + +struct NoCaseOrder +{ + bool operator()(const String& a, const String& b) const + { + String la = ToAscii(a); + String lb = ToAscii(b); + int q = SgnCompare(ToLower(ToAscii(a)), ToLower(ToAscii(b))); + if(q) + return q < 0; + q = SgnCompare(ToLower(a), ToLower(b)); + if(q) + return q < 0; + return a < b; + } +}; + +void Make() +{ + FileIn in(CommandLine()[0]); + if(!in) { + Cout() << "Unable to open " << CommandLine()[0] << " for reading\n"; + SetExitCode(1); + return; + } + FileOut out(CommandLine()[1]); + if(!out) { + Cout() << "Unable to open " << CommandLine()[1] << " for writing\n"; + SetExitCode(1); + return; + } + SetDefaultCharset(CHARSET_UTF8); + Vector w; + Index alphabet; + String maxl; + int maxlen = 0; + while(!in.IsEof()) { + String l = in.GetLine(); + if(l.GetLength() > maxlen) { + maxlen = l.GetLength(); + maxl = l; + } + if(FromUtf8(l).GetLength() > 1) { + w.Add(l); + for(const char *s = l; s < l.End(); s++) + alphabet.FindAdd((byte)*s); + } + } + + if(alphabet.GetCount() > 250) { + Cout() << "Alphabet is too big - only 250 different codepoints allowed."; + SetExitCode(1); + return; + } + + Cout() << "Words loaded, now sorting\n"; + + ASSERT(maxlen < 64); + + LLOG("Maximal length:" << maxlen << " " << maxl); + + Sort(w, NoCaseOrder()); + + Cout() << "Sorted, now gathering voc candidates\n"; + +// ------------------ + + VectorMap part; + int dict = 0; + int i = 0; + while(i < w.GetCount()) { + int linecode = LineCode(w[i]); + String prevw; + LLOG("line " << ~ToLower(w[i].Mid(0, 3))); + while(i < w.GetCount() && LineCode(w[i]) == linecode) { + String ww = w[i]; + int j; + for(j = 0; j < prevw.GetLength(); j++) + if(ww[j] != prevw[j]) break; + if(j >= dict) + dict = j + 1; + for(int l = 2; l < ww.GetLength() - 1; l++) + for(int q = j; q + l <= ww.GetLength(); q++) + part.GetAdd(ww.Mid(q, l), 0)++; + prevw = ww; + i++; + } + } + + Cout() << "Creating voc\n"; + + int dcount = 256 - dict; + LLOG("dict: " << dict); + LLOG("dict size: " << dcount); + LLOG(" alphabet:" << alphabet.GetCount()); + LLOG(" combinations: " << dcount - alphabet.GetCount()); + + for(i = 0; i < alphabet.GetCount(); i++) + voc.Add(String(alphabet[i], 1)); + + Vector value; + + for(i = 0; i < part.GetCount(); i++) + value.Add() = part[i] * (part.GetKey(i).GetLength() - 1); + + while(voc.GetCount() + dict < 256) { + int m = 0; + int mi = 0; + int i; + for(i = 0; i < part.GetCount(); i++) + if(value[i] > m) { + m = value[i]; + mi = i; + } + if(m <= 0) break; + String v = part.GetKey(mi); + vocn[voc.GetCount()] = value[mi]; + voc.Add(v); + LLOG("Adding " << v << " [" << v.GetCount() << "] value:" << value[mi] << " count:" << part[mi]); + for(i = 0; i < part.GetCount(); i++) { + if(Contains(part.GetKey(i), v)) + value[i] -= v.GetLength() * part[i]; + if(Contains(v, part.GetKey(i))) + value[i] -= part.GetKey(i).GetLength() * part[i]; + } + value[mi] = 0; + } + + int sum = 0; + for(i = 0; i < voc.GetCount(); i++) { + sum += vocn[i]; + LLOG(vocn[i] << " " << voc[i]); + } + LLOG("Total " << sum); + +// ------------------ + Sort(voc, LengthOrder()); + + i = 0; + while(i < w.GetCount()) { + int linecode = LineCode(w[i]); + String& ln = line.GetAdd(linecode); + LLOG("---- Line " << ToLower(ToAscii(~w[i].Mid(0, 3)))); + String prevw; + bool next = false; + while(i < w.GetCount() && LineCode(w[i]) == linecode) { + String ww = w[i]; + int j; + for(j = 0; j < prevw.GetLength(); j++) + if(ww[j] != prevw[j]) break; + if(next) + ln.Cat(j); + LLOG(j << "\t" << w[i]); + next = true; + const char *s = ~ww + j; + while(*s) { + int i; + for(i = 0; i < voc.GetCount(); i++) { + if(memcmp(s, voc[i], voc[i].GetLength()) == 0) { + LLOG(" " << s << " " << voc[i]); + ln.Cat(i + dict); + s += voc[i].GetLength(); + break; + } + } + ASSERT(i < voc.GetCount()); + } + prevw = ww; + i++; + } + LLOGHEXDUMP(~ln, ln.GetCount()); + } + int l = 0; + for(i = 0; i < line.GetCount(); i++) { + line[i].Cat(0); + l += line[i].GetLength(); + } + out.Put(CHARSET_UTF8); + out.Put(0); + out.Put(dict); + for(i = 0; i < voc.GetCount(); i++) { + out.Put(voc[i]); + out.Put(0); + } + for(i = 0; i < line.GetCount(); i++) { + out.PutL(line.GetKey(i)); + out.PutL(line[i].GetLength()); + out.Put(line[i]); + } +} + +CONSOLE_APP_MAIN +{ + if(CommandLine().GetCount() != 2) { + Cout() << "Usage: MakeSpellScd \n"; + exit(1); + } + Make(); + Cout() << "* Finished\n"; +} diff --git a/uppsrc/MakeSpellScd/MakeSpellScd.upp b/uppsrc/MakeSpellScd/MakeSpellScd.upp new file mode 100644 index 000000000..7003c0559 --- /dev/null +++ b/uppsrc/MakeSpellScd/MakeSpellScd.upp @@ -0,0 +1,13 @@ +description "Utility for creation of RichEdit spelling dictionary files (.scd)\377"; + +optimize_speed; + +uses + Core; + +file + MakeSpellScd.cpp; + +mainconfig + "" = ""; + diff --git a/uppsrc/MakeSpellScd/init b/uppsrc/MakeSpellScd/init new file mode 100644 index 000000000..fd8113dac --- /dev/null +++ b/uppsrc/MakeSpellScd/init @@ -0,0 +1,4 @@ +#ifndef _MakeSpellScd_icpp_init_stub +#define _MakeSpellScd_icpp_init_stub +#include "Core/init" +#endif diff --git a/uppsrc/RichEdit/Speller.cpp b/uppsrc/RichEdit/Speller.cpp index ed8c47cae..7044f4e46 100644 --- a/uppsrc/RichEdit/Speller.cpp +++ b/uppsrc/RichEdit/Speller.cpp @@ -86,39 +86,57 @@ bool Speller::Check(const wchar *wrd, int wlen) const while(ws < we) *wt++ = ToLower(*ws++); - char w[64]; - char wl[64]; - FromUnicode(w, wrd, len, charset); - FromUnicode(wl, wh, len, charset); - if(len == 2) { - w[len] = 127; - wl[len++] = 127; + const char *w; + const char *wl; + int i; + int l2; + if(charset == CHARSET_UTF8) { + String wb = ToUtf8(wrd, len); + w = wb; + String wlb = ToUtf8(wh, len); + wl = wlb; + i = line.Find(ToLower(ToAscii(wrd[0]), charset) + + (ToLower(ToAscii(wrd[1]), charset) << 8) + + (wlen == 2 ? 127 : ToLower(ToAscii(wrd[2]), charset) << 16)); + len = wb.GetCount(); + l2 = wlb.GetCount(); + } + else { + char wb[64]; + FromUnicode(wb, wrd, len, charset); + w = wb; + char wlb[64]; + FromUnicode(wlb, wh, len, charset); + wl = wlb; + if(len == 2) { + wb[len] = 127; + wlb[len++] = 127; + } + i = line.Find(ToLower(wl[0], charset) + + (ToLower(wl[1], charset) << 8) + + (ToLower(wl[2], charset) << 16)); + l2 = len; } - int i = line.Find(ToLower(wl[0], charset) + - (ToLower(wl[1], charset) << 8) + - (ToLower(wl[2], charset) << 16)); if(i >= 0) { const byte *s = line[i].begin; const byte *e = line[i].end; - int capitals = 0; - char q[64]; - int pos = 0; + String q; while(s < e) if(*s < dict) { - if(pos == len && compare3(q, w, wl, len)) + if(q.GetLength() == len && memcmp(w, q, q.GetLength()) == 0) return true; - pos = *s++; - ASSERT(pos < 64); - capitals = 0; + if(q.GetLength() == l2 && memcmp(wl, q, q.GetLength()) == 0) + return true; + q.Trim(*s++); } else { ASSERT(*s >= dict); const char *x = voc[(int)*s++ - dict]; - while(*x) - q[pos++] = *x++; - ASSERT(pos < 64); + q.Cat(x); } - if(pos == len && compare3(q, w, wl, len)) + if(q.GetLength() == len && memcmp(w, q, q.GetLength()) == 0) + return true; + if(q.GetLength() == l2 && memcmp(wl, q, q.GetLength()) == 0) return true; } }