diff --git a/uppsrc/Core/Inet.h b/uppsrc/Core/Inet.h index 069579861..6b64d22e5 100644 --- a/uppsrc/Core/Inet.h +++ b/uppsrc/Core/Inet.h @@ -11,6 +11,8 @@ String UrlDecode(const char *s, const char *end); String UrlDecode(const char *s, int len); String UrlDecode(const String& s); +String QPDecode(const char *s); + String Base64Encode(const char *s, const char *end); String Base64Encode(const char *s, int len); String Base64Encode(const String& data); diff --git a/uppsrc/Core/InetUtil.cpp b/uppsrc/Core/InetUtil.cpp index 9f461a6e0..73a80f523 100644 --- a/uppsrc/Core/InetUtil.cpp +++ b/uppsrc/Core/InetUtil.cpp @@ -169,6 +169,26 @@ String UrlDecode(const String& s) return UrlDecode(~s, s.GetLength()); } +String QPDecode(const char *s) +{ + StringBuffer r; + while(*s) { + int c = *s++; + if(c == '=') { + int c1 = '0', c2 = '0'; + if(*s) + c1 = *s++; + if(*s) + c2 = *s++; + if(IsXDigit(c1) && IsXDigit(c2)) + r.Cat((ctoi(c1) << 4) | ctoi(c2)); + } + else + r.Cat(c); + } + return r; +} + String Base64Encode(const char *b, const char *e) { static const char encoder[] = diff --git a/uppsrc/Core/POP3/InetMessage.cpp b/uppsrc/Core/POP3/InetMessage.cpp index fe8c9a730..e6db5a1b8 100644 --- a/uppsrc/Core/POP3/InetMessage.cpp +++ b/uppsrc/Core/POP3/InetMessage.cpp @@ -133,7 +133,7 @@ bool InetMessage::Read(const String& s) String type = header.Get("content-type"); if(!type.StartsWith("multipart")) { - part.Add().body = LoadStream(ss); + part.Add().body = DecodeHeaderValue(LoadStream(ss)); return true; } @@ -175,7 +175,30 @@ bool InetMessage::Read(const String& s) break; if(ln == end_boundary) return true; - p.body << ln << "\r\n"; + p.body << DecodeHeaderValue(ln) << "\r\n"; } } } + +String InetMessage::GetHeader(int parti, const char *id) const +{ + return Nvl(GetPartHeader(parti, id), header.Get(id, Null)); +} + +String InetMessage::GetPartBody(int i) const +{ + String encoding = ToLower(Nvl(GetHeader(i, "Content-Transfer-Encoding"), "quoted-printable")); + String body = part[i].body; + body = decode(encoding, "quoted-printable", QPDecode(body), "base64", Base64Decode(body), body); + String content_type = ToLower(GetHeader(i, "content-type")); + int q = content_type.Find("charset="); + if(q >= 0) { + q += strlen("charset="); + int qq = content_type.Find(';', q); + String charset = qq >= 0 ? content_type.Mid(q, qq + 1) : content_type.Mid(q); + int cs = CharsetByName(charset); + if(cs >= 0) + body = ToCharset(CHARSET_DEFAULT, body, cs, '?'); + } + return body; +} diff --git a/uppsrc/Core/POP3/POP3.cpp b/uppsrc/Core/POP3/POP3.cpp index ddf18bf51..d218b3eda 100644 --- a/uppsrc/Core/POP3/POP3.cpp +++ b/uppsrc/Core/POP3/POP3.cpp @@ -24,12 +24,8 @@ String Pop3::GetTimeStamp() bool Pop3::GetListItems(ValueMap& list, dword type1, dword type2) { StringStream s(data); - for(;;) { + while(!s.IsEof()) { String line = s.GetLine(); - if(s.IsError()) - return false; - if(s.IsEof()) - break; Vector s = Split(line, ' '); if(s.GetCount() < 2) return false; @@ -64,8 +60,6 @@ String Pop3::GetMessageHeader(int index) bool Pop3::GetMessageList(ValueMap& list) { - VectorMap v; - if(!PutGet("LIST\r\n", true)) return false; return GetListItems(list, INT_V, INT_V); @@ -139,7 +133,7 @@ bool Pop3::PutGet(const String& s, bool multiline, bool nolog) LLOG("<< " << TrimRight(line)); if(line.StartsWith("+OK")) { if(!multiline) { - data << line; + data.Cat(line); return true; } else @@ -147,13 +141,11 @@ bool Pop3::PutGet(const String& s, bool multiline, bool nolog) line = GetDataLine(); if(line.IsEmpty()) break; - if(line.StartsWith("..")) - line.Remove(0); - data << line; - if(!data.EndsWith("\r\n.\r\n")) - continue; - LLOG("<< ..."); - return true; + if(line == ".\r\n") { + LLOG("<< ..."); + return true; + } + data.Cat(*line == '.' ? line.Mid(1) : line); } } else diff --git a/uppsrc/Core/POP3/POP3.h b/uppsrc/Core/POP3/POP3.h index d3bc1180f..6ff346e49 100644 --- a/uppsrc/Core/POP3/POP3.h +++ b/uppsrc/Core/POP3/POP3.h @@ -63,10 +63,11 @@ struct InetMessage { bool Read(const String& msg); bool ReadHeader(const String& msg); - String operator[](const char *id) const { return header.Get(id, Null); } - int GetPartCount() const { return part.GetCount(); } - String GetPartBody(int i) const { return part[i].body; } - String GetPartHeader(int i, const char *id) { return part[i].header.Get(id, Null); } + String operator[](const char *id) const { return header.Get(id, Null); } + int GetPartCount() const { return part.GetCount(); } + String GetPartBody(int i) const; + String GetPartHeader(int i, const char *id) const { return part[i].header.Get(id, Null); } + String GetHeader(int parti, const char *id) const; private: bool ReadHeader(VectorMap& hdr, StringStream& ss); diff --git a/uppsrc/Core/POP3/src.tpp/InetMessage$en-us.tpp b/uppsrc/Core/POP3/src.tpp/InetMessage$en-us.tpp index e911b7c4c..b68d0b2bb 100644 --- a/uppsrc/Core/POP3/src.tpp/InetMessage$en-us.tpp +++ b/uppsrc/Core/POP3/src.tpp/InetMessage$en-us.tpp @@ -59,10 +59,21 @@ message.&] [s4;%% &] [s5;:InetMessage`:`:GetPartBody`(int`)const: [_^String^ String]_[* GetPartBody]([@(0.0.255) i nt]_[*@3 i])_[@(0.0.255) const]&] -[s2;%% Returns the body of part [%-*@3 i].&] +[s2;%% Returns the body of part [%-*@3 i]. Bodies with transfer encoding +base64 or quoted`-printable are decoded. Also, if `'content`-type`', +has `'charset`' section, body is converted to application`'s +default encoding (usually utf`-8).&] [s3;%% &] -[s4;%% &] -[s5;:InetMessage`:`:GetPartHeader`(int`,const char`*`): [_^String^ String]_[* GetPartHead -er]([@(0.0.255) int]_[*@3 i], [@(0.0.255) const]_[@(0.0.255) char]_`*[*@3 id])&] +[s4; &] +[s5;:InetMessage`:`:GetPartHeader`(int`,const char`*`)const: [_^String^ String]_[* GetPar +tHeader]([@(0.0.255) int]_[*@3 i], [@(0.0.255) const]_[@(0.0.255) char]_`*[*@3 id])_[@(0.0.255) c +onst]&] [s2;%% Returns header [%-*@3 id] for part [%-*@3 i].&] +[s3;%% &] +[s4; &] +[s5;:InetMessage`:`:GetHeader`(int`,const char`*`)const: [_^String^ String]_[* GetHeader]( +[@(0.0.255) int]_[*@3 parti], [@(0.0.255) const]_[@(0.0.255) char]_`*[*@3 id])_[@(0.0.255) co +nst]&] +[s2;%% Returns non`-empty header [%-*@3 id] for part [%-*@3 parti]. If +header is empty, returns GetHeader([%-*@3 id]) `- main header.&] [s3;%% ]] \ No newline at end of file diff --git a/uppsrc/Core/XML.cpp b/uppsrc/Core/XML.cpp index 5fac51172..898dd7773 100644 --- a/uppsrc/Core/XML.cpp +++ b/uppsrc/Core/XML.cpp @@ -471,7 +471,8 @@ String XmlParser::ReadTag(bool next) LLOG("ReadTag " << tagtext); String h = tagtext; if(next) { - stack.Add(Nesting(h, npreserve)); + if(!raw) + stack.Add(Nesting(h, npreserve)); attr = pick(nattr); attr1 = nattr1; attrval1 = nattrval1; @@ -484,7 +485,8 @@ bool XmlParser::Tag(const char *tag) { if(IsTag() && tagtext == tag) { LLOG("Tag " << tagtext); - stack.Add(Nesting(tagtext, npreserve)); + if(!raw) + stack.Add(Nesting(tagtext, npreserve)); attr = pick(nattr); attr1 = nattr1; attrval1 = nattrval1; @@ -498,7 +500,8 @@ bool XmlParser::Tag(const String& tag) { if(IsTag() && tagtext == tag) { LLOG("Tag " << tagtext); - stack.Add(Nesting(tagtext, npreserve)); + if(!raw) + stack.Add(Nesting(tagtext, npreserve)); attr = pick(nattr); attr1 = nattr1; attrval1 = nattrval1; @@ -520,6 +523,16 @@ void XmlParser::PassTag(const String& tag) throw XmlError(String().Cat() << '\'' << tag << "'\' tag expected"); } +String XmlParser::ReadEnd(bool next) +{ + if(type != XML_END) + throw XmlError("Expected end-tag"); + String x = tagtext; + if(next) + Next(); + return x; +} + bool XmlParser::IsEnd() { return type == XML_END; @@ -531,13 +544,15 @@ bool XmlParser::End() throw XmlError("Unexpected end of file"); if(IsEnd()) { LLOG("EndTag " << tagtext); - if(stack.IsEmpty()) - throw XmlError(NFormat("Unexpected end-tag: ", tagtext)); - if(stack.Top().tag != tagtext && !relaxed) { - LLOG("Tag/end-tag mismatch: <" << stack.Top().tag << "> "); - throw XmlError(NFormat("Tag/end-tag mismatch: <%s> ", stack.Top().tag, tagtext)); + if(!raw) { + if(stack.IsEmpty()) + throw XmlError(NFormat("Unexpected end-tag: ", tagtext)); + if(stack.Top().tag != tagtext && !relaxed) { + LLOG("Tag/end-tag mismatch: <" << stack.Top().tag << "> "); + throw XmlError(NFormat("Tag/end-tag mismatch: <%s> ", stack.Top().tag, tagtext)); + } + stack.Drop(); } - stack.Drop(); npreserve = (!stack.IsEmpty() && stack.Top().preserve_blanks); Next(); return true; @@ -545,12 +560,36 @@ bool XmlParser::End() return false; } +bool XmlParser::End(const char *tag) +{ + if(IsEnd() && tagtext == tag) { + Next(); + return true; + } + return false; +} + +bool XmlParser::End(const String& tag) +{ + if(IsEnd() && tagtext == tag) { + Next(); + return true; + } + return false; +} + void XmlParser::PassEnd() { if(!End()) throw XmlError(String().Cat() << "Expected \'" << (stack.GetCount() ? stack.Top().tag : String()) << "\' end-tag"); } +void XmlParser::PassEnd(const char *tag) +{ + if(!End(tag)) + throw XmlError(String().Cat() << "Expected \'" << tag << "\' end-tag"); +} + bool XmlParser::TagE(const char *tag) { if(Tag(tag)) { @@ -693,7 +732,7 @@ void XmlParser::Skip() if(cdata.GetCount() && type != XML_TEXT) cdata.Clear(); else - if(IsTag()) { + if(IsTag() && !raw) { String n = ReadTag(); while(!End()) { if(IsEof()) @@ -737,6 +776,7 @@ void XmlParser::Init() begincolumn = 0; in = NULL; len = 0; + raw = false; } XmlParser::XmlParser(const char *s) diff --git a/uppsrc/Core/XML.h b/uppsrc/Core/XML.h index 17d133794..b5e5fe223 100644 --- a/uppsrc/Core/XML.h +++ b/uppsrc/Core/XML.h @@ -80,6 +80,7 @@ class XmlParser { bool empty_tag; bool npreserve, preserveall; bool relaxed; + bool raw; int line; @@ -92,6 +93,7 @@ class XmlParser { void Next(); void ReadAttr(StringBuffer& b, int c); String ReadTag(bool next); + String ReadEnd(bool next); String ReadDecl(bool next); String ReadPI(bool next); String ReadComment(bool next); @@ -113,8 +115,13 @@ public: void PassTag(const char *tag); void PassTag(const String& tag); bool IsEnd(); + String PeekEnd() { return ReadEnd(false); } + String ReadEnd() { return ReadEnd(true); } bool End(); + bool End(const char *tag); + bool End(const String& tag); void PassEnd(); + void PassEnd(const char *tag); bool TagE(const char *tag); void PassTagE(const char *tag); bool TagElseSkip(const char *tag); @@ -153,8 +160,9 @@ public: int GetLine() const { return line; } int GetColumn() const { return GetColumn0() + 1; } - void Relaxed(bool b) { relaxed = b; } + void Relaxed(bool b = true) { relaxed = b; } void PreserveAllWhiteSpaces(bool b = true) { preserveall = b; } + void Raw(bool b = true) { raw = b; } XmlParser(const char *s); XmlParser(Stream& in); diff --git a/uppsrc/Core/src.tpp/Inet$en-us.tpp b/uppsrc/Core/src.tpp/Inet$en-us.tpp index 57c624751..19ba32f3f 100644 --- a/uppsrc/Core/src.tpp/Inet$en-us.tpp +++ b/uppsrc/Core/src.tpp/Inet$en-us.tpp @@ -56,6 +56,12 @@ tring][@(0.0.255) `&]_[*@3 s])&] IME type application/x`-www`-form`-urlencoded] (see UrlEncode).&] [s3;%% &] [s4; &] +[s5;:QPDecode`(const char`*`): [_^String^ String]_[* QPDecode]([@(0.0.255) const]_[@(0.0.255) c +har]_`*[*@3 s])&] +[s2;%% Decodes string from [^http`:`/`/en`.wikipedia`.org`/wiki`/Quoted`-printable^ quo +ted`-printable] format .&] +[s3;%% &] +[s4; &] [s5;:Base64Encode`(const char`*`,const char`*`): [_^String^ String]_[* Base64Encode]([@(0.0.255) c onst]_[@(0.0.255) char]_`*[*@3 b], [@(0.0.255) const]_[@(0.0.255) char]_`*[*@3 e])&] [s5;:Base64Encode`(const char`*`,int`): [_^String^ String]_[* Base64Encode]([@(0.0.255) con diff --git a/uppsrc/Core/src.tpp/XmlParser$en-us.tpp b/uppsrc/Core/src.tpp/XmlParser$en-us.tpp index 046982031..0159e516c 100644 --- a/uppsrc/Core/src.tpp/XmlParser$en-us.tpp +++ b/uppsrc/Core/src.tpp/XmlParser$en-us.tpp @@ -230,10 +230,15 @@ the next start`-tag is processed.&] [s2;%% Activates the relaxed mode. In relaxed mode, XmlParser ignores mismatches of start`-tag and end`-tags. This mode was introduced to deal with broken XML files.&] -[s3; &] +[s3;%% &] +[s4; &] +[s5;:XmlParser`:`:Raw`(bool`): [@(0.0.255) void]_[* Raw]([@(0.0.255) bool]_[*@3 b])&] +[s2;%% Activtes the raw mode. I raw mode, all logic about start`-tags +and end`-tags is completely supressed. This mode was introduced +to deal with HTML.&] [s0; &] [ {{10000F(128)G(128)@1 [s0;%% [* Constructor detail]]}}&] -[s4;H0; &] +[s3;%% &] [s5;:XmlParser`:`:XmlParser`(const char`*`): [* XmlParser]([@(0.0.255) const]_[@(0.0.255) c har]_`*[*@3 s])&] [s2;%% Creates the parser for xml input text [%-*@3 s]. The pointed