diff --git a/uppsrc/Core/Http.cpp b/uppsrc/Core/Http.cpp index a02f1b16f..938e39240 100644 --- a/uppsrc/Core/Http.cpp +++ b/uppsrc/Core/Http.cpp @@ -1,595 +1,609 @@ -#include "Core.h" - -NAMESPACE_UPP - -bool HttpRequest_Trace__; - -#define LLOG(x) do { if(HttpRequest_Trace__) RLOG(x); } while(0) - -#ifdef _DEBUG -_DBG_ -// #define ENDZIP -#endif - -void HttpRequest::Trace(bool b) -{ - HttpRequest_Trace__ = b; -} - -void HttpRequest::Init() -{ - port = 0; - proxy_port = 0; - max_header_size = 1000000; - max_content_size = 10000000; - max_redirects = 5; - max_retries = 3; - force_digest = false; - std_headers = true; - hasurlvar = false; - method = METHOD_GET; - phase = START; - redirect_count = 0; - retry_count = 0; - gzip = false; - WhenContent = callback(this, &HttpRequest::ContentOut); - chunk = 4096; - timeout = 120000; - ssl = false; -} - -HttpRequest::HttpRequest() -{ - Init(); -} - -HttpRequest::HttpRequest(const char *url) -{ - Init(); - Url(url); -} - -HttpRequest& HttpRequest::Url(const char *u) -{ - ssl = memcmp(u, "https", 5) == 0; - const char *t = u; - while(*t && *t != '?') - if(*t++ == '/' && *t == '/') { - u = ++t; - break; - } - t = u; - while(*u && *u != ':' && *u != '/' && *u != '?') - u++; - if(*u == '?' && u[1]) - hasurlvar = true; - host = String(t, u); - port = 0; - if(*u == ':') - port = ScanInt(u + 1, &u); - path = u; - int q = path.Find('#'); - if(q >= 0) - path.Trim(q); - return *this; -} - -HttpRequest& HttpRequest::Proxy(const char *p) -{ - const char *t = p; - while(*p && *p != ':') - p++; - proxy_host = String(t, p); - proxy_port = 80; - if(*p++ == ':' && IsDigit(*p)) - proxy_port = ScanInt(p); - return *this; -} - -HttpRequest& HttpRequest::Post(const char *id, const String& data) -{ - POST(); - if(postdata.GetCount()) - postdata << '&'; - postdata << id << '=' << UrlEncode(data); - return *this; -} - -HttpRequest& HttpRequest::UrlVar(const char *id, const String& data) -{ - int c = *path.Last(); - if(hasurlvar && c != '&') - path << '&'; - if(!hasurlvar && c != '?') - path << '?'; - path << id << '=' << UrlEncode(data); - hasurlvar = true; - return *this; -} - -String HttpRequest::CalculateDigest(const String& authenticate) const -{ - const char *p = authenticate; - String realm, qop, nonce, opaque; - while(*p) { - if(!IsAlNum(*p)) { - p++; - continue; - } - else { - const char *b = p; - while(IsAlNum(*p)) - p++; - String var = ToLower(String(b, p)); - String value; - while(*p && (byte)*p <= ' ') - p++; - if(*p == '=') { - p++; - while(*p && (byte)*p <= ' ') - p++; - if(*p == '\"') { - p++; - while(*p && *p != '\"') - if(*p != '\\' || *++p) - value.Cat(*p++); - if(*p == '\"') - p++; - } - else { - b = p; - while(*p && *p != ',' && (byte)*p > ' ') - p++; - value = String(b, p); - } - } - if(var == "realm") - realm = value; - else if(var == "qop") - qop = value; - else if(var == "nonce") - nonce = value; - else if(var == "opaque") - opaque = value; - } - } - String hv1, hv2; - hv1 << username << ':' << realm << ':' << password; - String ha1 = MD5String(hv1); - hv2 << (method == METHOD_GET ? "GET" : method == METHOD_PUT ? "PUT" : method == METHOD_POST ? "POST" : "READ") - << ':' << path; - String ha2 = MD5String(hv2); - int nc = 1; - String cnonce = FormatIntHex(Random(), 8); - String hv; - hv << ha1 - << ':' << nonce - << ':' << FormatIntHex(nc, 8) - << ':' << cnonce - << ':' << qop << ':' << ha2; - String ha = MD5String(hv); - String auth; - auth << "username=" << AsCString(username) - << ", realm=" << AsCString(realm) - << ", nonce=" << AsCString(nonce) - << ", uri=" << AsCString(path) - << ", qop=" << AsCString(qop) - << ", nc=" << AsCString(FormatIntHex(nc, 8)) - << ", cnonce=" << cnonce - << ", response=" << AsCString(ha); - if(!IsNull(opaque)) - auth << ", opaque=" << AsCString(opaque); - return auth; -} - -HttpRequest& HttpRequest::Header(const char *id, const String& data) -{ - request_headers << id << ": " << data << "\r\n"; - return *this; -} - -void HttpRequest::HttpError(const char *s) -{ - if(IsError()) - return; - error = NFormat(t_("%s:%d: ") + String(s), host, port); - LLOG("HTTP ERROR: " << error); - Close(); -} - -void HttpRequest::StartPhase(int s) -{ - LLOG("Starting status " << s << ' ' << host); - phase = s; - data.Clear(); -} - -bool HttpRequest::Do() -{ - int c1, c2; - switch(phase) { - case START: - retry_count = 0; - redirect_count = 0; - start_time = msecs(); - Start(); - break; - case DNS: - Dns(); - break; - case REQUEST: - if(SendingData()) - break; - StartPhase(HEADER); - break; - case HEADER: - if(ReadingHeader()) - break; - StartBody(); - break; - case BODY: - if(ReadingBody()) - break; - Finish(); - break; - case CHUNK_HEADER: - ReadingChunkHeader(); - break; - case CHUNK_BODY: - if(ReadingBody()) - break; - c1 = Get(); - c2 = Get(); - if(c1 != '\r' || c2 != '\n') - HttpError("missing ending CRLF in chunked transfer"); - StartPhase(CHUNK_HEADER); - break; - case TRAILER: - if(ReadingHeader()) - break; - header.Parse(data); - Finish(); - break; - case FINISHED: - case FAILED: - return false; - default: - NEVER(); - } - - if(phase != FAILED) - if(IsSocketError() || IsError()) - phase = FAILED; - else - if(msecs() - start_time >= timeout) { - HttpError("connection timed out"); - phase = FAILED; - } - else - if(IsAbort()) { - HttpError("connection was aborted"); - phase = FAILED; - } - - if(phase == FAILED) { - if(retry_count++ < max_retries) { - LLOG("HTTP retry on error " << GetErrorDesc()); - StartRequest(); - } - } - return phase != FINISHED && phase != FAILED; -} - -void HttpRequest::Finish() -{ - if(gzip) { - #ifdef ENDZIP - body = GZDecompress(body); - if(body.IsVoid()) { - HttpError("gzip decompress at finish error"); - phase = FAILED; - return; - } - #else - z.End(); - if(z.IsError()) { - HttpError("gzip format error (finish)"); - phase = FAILED; - return; - } - #endif - } - Close(); - if(status_code == 401 && !IsNull(username)) { - String authenticate = header["www-authenticate"]; - if(authenticate.GetCount() && redirect_count++ < max_redirects) { - LLOG("HTTP auth digest"); - Digest(CalculateDigest(authenticate)); - StartRequest(); - return; - } - } - if(status_code >= 300 && status_code < 400) { - String url = GetRedirectUrl(); - if(url.GetCount() && redirect_count++ < max_redirects) { - LLOG("HTTP redirect " << url); - Url(url); - StartRequest(); - retry_count = 0; - return; - } - } - phase = FINISHED; - -// if(retry_count < 2) -// HttpError("Checking retry"); -} - -void HttpRequest::ReadingChunkHeader() -{ - for(;;) { - int c = Get(); - if(c < 0) - break; - else - if(c == '\n') { - int n = ScanInt(~data, NULL, 16); - LLOG("HTTP Chunk header: 0x" << data << " = " << n); - if(IsNull(n)) { - HttpError("invalid chunk header"); - break; - } - if(n == 0) { - StartPhase(TRAILER); - break; - } - count += n; - StartPhase(CHUNK_BODY); - break; - } - if(c != '\r') - data.Cat(c); - } -} - -String HttpRequest::GetRedirectUrl() -{ - String redirect_url = TrimLeft(header["location"]); - int q = redirect_url.Find('?'); - int p = path.Find('?'); - if(p >= 0 && q < 0) - redirect_url.Cat(path.Mid(p)); - return redirect_url; -} - -int HttpRequest::GetContentLength() -{ - return Nvl(ScanInt(header["content-length"]), -1); -} - -void HttpRequest::StartBody() -{ - LLOG("HTTP Header received: "); - LLOG(data); - header.Clear(); - if(!header.Parse(data)) { - HttpError("invalid HTTP header"); - return; - } - - if(!header.Response(protocol, status_code, reason_phrase)) { - HttpError("invalid HTTP response"); - return; - } - - LLOG("HTTP status code: " << status_code); - - count = GetContentLength(); - - if(count > 0) - body.Reserve(count); - - if(method == METHOD_HEAD) - phase = FINISHED; - else - if(header["transfer-encoding"] == "chunked") { - count = 0; - StartPhase(CHUNK_HEADER); - } - else - StartPhase(BODY); - body.Clear(); - bodylen = 0; - gzip = GetHeader("content-encoding") == "gzip"; - if(gzip) { - gzip = true; - z.WhenOut = callback(this, &HttpRequest::Out); - z.ChunkSize(chunk).GZip().Decompress(); - } -} - -void HttpRequest::ContentOut(const void *ptr, dword size) -{ - body.Cat((const char *)ptr, size); -} - -void HttpRequest::Out(const void *ptr, dword size) -{ - LLOG("HTTP Out " << size); - if(z.IsError()) - HttpError("gzip format error"); - int64 l = bodylen + size; - if(l > max_content_size) { - HttpError("content length exceeded " + AsString(max_content_size)); - phase = FAILED; - return; - } - WhenContent(ptr, size); - bodylen += size; -} - -bool HttpRequest::ReadingBody() -{ - LLOG("HTTP reading data " << count); - int n = chunk; - if(count >= 0) - n = min(n, count); - String s = Get(n); - if(s.GetCount() == 0) - return !IsEof() && count; -#ifndef ENDZIP - if(gzip) - z.Put(~s, s.GetCount()); - else -#endif - Out(~s, s.GetCount()); - if(count >= 0) { - count -= s.GetCount(); - return !IsEof() && count > 0; - } - return !IsEof(); -} - -void HttpRequest::Start() -{ - Close(); - ClearError(); - gzip = false; - z.Clear(); - - bool use_proxy = !IsNull(proxy_host); - - int p = use_proxy ? proxy_port : port; - if(!p) - p = DEFAULT_HTTP_PORT; - String h = use_proxy ? proxy_host : host; - if(IsNull(GetTimeout())) { - addrinfo.Execute(h, p); - StartRequest(); - } - else { - addrinfo.Start(h, p); - StartPhase(DNS); - } -} - -void HttpRequest::Dns() -{ - for(int i = 0; i <= Nvl(GetTimeout(), INT_MAX); i++) { - if(!addrinfo.InProgress()) { - StartRequest(); - return; - } - Sleep(1); - } -} - -void HttpRequest::StartRequest() -{ - if(!Connect(addrinfo)) - return; - - StartPhase(REQUEST); - count = 0; - String ctype = contenttype; - if((method == METHOD_POST || method == METHOD_PUT) && IsNull(ctype)) - ctype = "application/x-www-form-urlencoded"; - switch(method) { - case METHOD_GET: data << "GET "; break; - case METHOD_POST: data << "POST "; break; - case METHOD_PUT: data << "PUT "; break; - case METHOD_HEAD: data << "HEAD "; break; - default: NEVER(); // invalid method - } - String host_port = host; - if(port) - host_port << ':' << port; - String url; - url << "http://" << host_port << Nvl(path, "/"); - if(!IsNull(proxy_host)) - data << url; - else - data << Nvl(path, "/"); - data << " HTTP/1.1\r\n"; - if(std_headers) { - data << "URL: " << url << "\r\n" - << "Host: " << host_port << "\r\n" - << "Connection: close\r\n" - << "Accept: " << Nvl(accept, "*/*") << "\r\n" - << "Accept-Encoding: gzip\r\n" - << "Agent: " << Nvl(agent, "Ultimate++ HTTP client") << "\r\n"; - if(postdata.GetCount()) - data << "Content-Length: " << postdata.GetCount() << "\r\n"; - if(ctype.GetCount()) - data << "Content-Type: " << ctype << "\r\n"; - } - if(!IsNull(proxy_host) && !IsNull(proxy_username)) - data << "Proxy-Authorization: Basic " << Base64Encode(proxy_username + ':' + proxy_password) << "\r\n"; - if(!IsNull(digest)) - data << "Authorization: Digest " << digest << "\r\n"; - else - if(!force_digest && (!IsNull(username) || !IsNull(password))) - data << "Authorization: Basic " << Base64Encode(username + ":" + password) << "\r\n"; - data << request_headers << "\r\n" << postdata; // !!! POST PHASE !!! - LLOG("HTTP REQUEST " << host << ":" << port); - LLOG("HTTP request:\n" << data); - if(ssl) - StartSSL(); -} - -bool HttpRequest::SendingData() -{ - for(;;) { - int n = min(2048, data.GetLength() - count); - n = Put(~data + count, n); - if(n == 0) - break; - count += n; - } - return count < data.GetLength(); -} - -bool HttpRequest::ReadingHeader() -{ - for(;;) { - int c = Get(); - if(c < 0) - return !IsEof(); - else - data.Cat(c); - if(data.GetCount() > 3) { - const char *h = data.Last(); - if(h[0] == '\n' && (h[-1] == '\r' && h[-2] == '\n' || h[-1] == '\n')) - return false; - } - if(data.GetCount() > max_header_size) { - HttpError("HTTP header exceeded " + AsString(max_header_size)); - return true; - } - } -} - -String HttpRequest::Execute() -{ - while(Do()); - return IsSuccess() ? GetContent() : String::GetVoid(); -} - -String HttpRequest::GetPhaseName() const -{ - static const char *m[] = { - "Start", - "Resolving host name", - "Sending request", - "Receiving header", - "Receiving content", - "Receiving chunk header", - "Receiving content chunk", - "Receiving trailer", - "Finished", - "Failed", - }; - return phase >= 0 && phase <= FAILED ? m[phase] : ""; -} - -END_UPP_NAMESPACE +#include "Core.h" + +NAMESPACE_UPP + +bool HttpRequest_Trace__; + +#define LLOG(x) do { if(HttpRequest_Trace__) RLOG(x); } while(0) + +#ifdef _DEBUG +_DBG_ +// #define ENDZIP +#endif + +void HttpRequest::Trace(bool b) +{ + HttpRequest_Trace__ = b; +} + +void HttpRequest::Init() +{ + port = 0; + proxy_port = 0; + max_header_size = 1000000; + max_content_size = 10000000; + max_redirects = 5; + max_retries = 3; + force_digest = false; + std_headers = true; + hasurlvar = false; + method = METHOD_GET; + phase = START; + redirect_count = 0; + retry_count = 0; + gzip = false; + WhenContent = callback(this, &HttpRequest::ContentOut); + chunk = 4096; + timeout = 120000; + ssl = false; +} + +HttpRequest::HttpRequest() +{ + Init(); +} + +HttpRequest::HttpRequest(const char *url) +{ + Init(); + Url(url); +} + +HttpRequest& HttpRequest::Url(const char *u) +{ + ssl = memcmp(u, "https", 5) == 0; + const char *t = u; + while(*t && *t != '?') + if(*t++ == '/' && *t == '/') { + u = ++t; + break; + } + t = u; + while(*u && *u != ':' && *u != '/' && *u != '?') + u++; + if(*u == '?' && u[1]) + hasurlvar = true; + host = String(t, u); + port = 0; + if(*u == ':') + port = ScanInt(u + 1, &u); + path = u; + int q = path.Find('#'); + if(q >= 0) + path.Trim(q); + return *this; +} + +HttpRequest& HttpRequest::Proxy(const char *p) +{ + const char *t = p; + while(*p && *p != ':') + p++; + proxy_host = String(t, p); + proxy_port = 80; + if(*p++ == ':' && IsDigit(*p)) + proxy_port = ScanInt(p); + return *this; +} + +HttpRequest& HttpRequest::Post(const char *id, const String& data) +{ + POST(); + if(postdata.GetCount()) + postdata << '&'; + postdata << id << '=' << UrlEncode(data); + return *this; +} + +HttpRequest& HttpRequest::UrlVar(const char *id, const String& data) +{ + int c = *path.Last(); + if(hasurlvar && c != '&') + path << '&'; + if(!hasurlvar && c != '?') + path << '?'; + path << id << '=' << UrlEncode(data); + hasurlvar = true; + return *this; +} + +String HttpRequest::CalculateDigest(const String& authenticate) const +{ + const char *p = authenticate; + String realm, qop, nonce, opaque; + while(*p) { + if(!IsAlNum(*p)) { + p++; + continue; + } + else { + const char *b = p; + while(IsAlNum(*p)) + p++; + String var = ToLower(String(b, p)); + String value; + while(*p && (byte)*p <= ' ') + p++; + if(*p == '=') { + p++; + while(*p && (byte)*p <= ' ') + p++; + if(*p == '\"') { + p++; + while(*p && *p != '\"') + if(*p != '\\' || *++p) + value.Cat(*p++); + if(*p == '\"') + p++; + } + else { + b = p; + while(*p && *p != ',' && (byte)*p > ' ') + p++; + value = String(b, p); + } + } + if(var == "realm") + realm = value; + else if(var == "qop") + qop = value; + else if(var == "nonce") + nonce = value; + else if(var == "opaque") + opaque = value; + } + } + String hv1, hv2; + hv1 << username << ':' << realm << ':' << password; + String ha1 = MD5String(hv1); + hv2 << (method == METHOD_GET ? "GET" : method == METHOD_PUT ? "PUT" : method == METHOD_POST ? "POST" : "READ") + << ':' << path; + String ha2 = MD5String(hv2); + int nc = 1; + String cnonce = FormatIntHex(Random(), 8); + String hv; + hv << ha1 + << ':' << nonce + << ':' << FormatIntHex(nc, 8) + << ':' << cnonce + << ':' << qop << ':' << ha2; + String ha = MD5String(hv); + String auth; + auth << "username=" << AsCString(username) + << ", realm=" << AsCString(realm) + << ", nonce=" << AsCString(nonce) + << ", uri=" << AsCString(path) + << ", qop=" << AsCString(qop) + << ", nc=" << AsCString(FormatIntHex(nc, 8)) + << ", cnonce=" << cnonce + << ", response=" << AsCString(ha); + if(!IsNull(opaque)) + auth << ", opaque=" << AsCString(opaque); + return auth; +} + +HttpRequest& HttpRequest::Header(const char *id, const String& data) +{ + request_headers << id << ": " << data << "\r\n"; + return *this; +} + +void HttpRequest::HttpError(const char *s) +{ + if(IsError()) + return; + error = NFormat(t_("%s:%d: ") + String(s), host, port); + LLOG("HTTP ERROR: " << error); + Close(); +} + +void HttpRequest::StartPhase(int s) +{ + phase = s; + LLOG("Starting status " << s << " '" << GetPhaseName() << "' of " << host); + data.Clear(); +} + +bool HttpRequest::Do() +{ + int c1, c2; + switch(phase) { + case START: + retry_count = 0; + redirect_count = 0; + start_time = msecs(); + Start(); + break; + case DNS: + Dns(); + break; + case REQUEST: + if(SendingData()) + break; + StartPhase(HEADER); + break; + case HEADER: + if(ReadingHeader()) + break; + StartBody(); + break; + case BODY: + if(ReadingBody()) + break; + Finish(); + break; + case CHUNK_HEADER: + ReadingChunkHeader(); + break; + case CHUNK_BODY: + if(ReadingBody()) + break; + c1 = Get(); + c2 = Get(); + if(c1 != '\r' || c2 != '\n') + HttpError("missing ending CRLF in chunked transfer"); + StartPhase(CHUNK_HEADER); + break; + case TRAILER: + if(ReadingHeader()) + break; + header.Parse(data); + Finish(); + break; + case FINISHED: + case FAILED: + return false; + default: + NEVER(); + } + + if(phase != FAILED) + if(IsSocketError() || IsError()) + phase = FAILED; + else + if(msecs() - start_time >= timeout) { + HttpError("connection timed out"); + phase = FAILED; + } + else + if(IsAbort()) { + HttpError("connection was aborted"); + phase = FAILED; + } + + if(phase == FAILED) { + if(retry_count++ < max_retries) { + LLOG("HTTP retry on error " << GetErrorDesc()); + StartRequest(); + } + } + return phase != FINISHED && phase != FAILED; +} + +void HttpRequest::CopyCookies() +{ + int q = header.fields.Find("set-cookie"); + while(q >= 0) { + Cookie(header.fields[q]); + q = header.fields.FindNext(q); + } +} + +void HttpRequest::Finish() +{ + if(gzip) { + #ifdef ENDZIP + body = GZDecompress(body); + if(body.IsVoid()) { + HttpError("gzip decompress at finish error"); + phase = FAILED; + return; + } + #else + z.End(); + if(z.IsError()) { + HttpError("gzip format error (finish)"); + phase = FAILED; + return; + } + #endif + } + Close(); + if(status_code == 401 && !IsNull(username)) { + String authenticate = header["www-authenticate"]; + if(authenticate.GetCount() && redirect_count++ < max_redirects) { + LLOG("HTTP auth digest"); + CopyCookies(); + Digest(CalculateDigest(authenticate)); + Start(); + return; + } + } + if(status_code >= 300 && status_code < 400) { + String url = GetRedirectUrl(); + if(url.GetCount() && redirect_count++ < max_redirects) { + LLOG("HTTP redirect " << url); + Url(url); + CopyCookies(); + Start(); + retry_count = 0; + return; + } + } + phase = FINISHED; + +// if(retry_count < 2) +// HttpError("Checking retry"); +} + +void HttpRequest::ReadingChunkHeader() +{ + for(;;) { + int c = Get(); + if(c < 0) + break; + else + if(c == '\n') { + int n = ScanInt(~data, NULL, 16); + LLOG("HTTP Chunk header: 0x" << data << " = " << n); + if(IsNull(n)) { + HttpError("invalid chunk header"); + break; + } + if(n == 0) { + StartPhase(TRAILER); + break; + } + count += n; + StartPhase(CHUNK_BODY); + break; + } + if(c != '\r') + data.Cat(c); + } +} + +String HttpRequest::GetRedirectUrl() +{ + String redirect_url = TrimLeft(header["location"]); + int q = redirect_url.Find('?'); + int p = path.Find('?'); + if(p >= 0 && q < 0) + redirect_url.Cat(path.Mid(p)); + if(redirect_url.StartsWith("http://") || redirect_url.StartsWith("https://")) + return redirect_url; + return (ssl ? "https://" : "http://") + host + "/" + redirect_url; +} + +int HttpRequest::GetContentLength() +{ + return Nvl(ScanInt(header["content-length"]), -1); +} + +void HttpRequest::StartBody() +{ + LLOG("HTTP Header received: "); + LLOG(data); + header.Clear(); + if(!header.Parse(data)) { + HttpError("invalid HTTP header"); + return; + } + + if(!header.Response(protocol, status_code, reason_phrase)) { + HttpError("invalid HTTP response"); + return; + } + + LLOG("HTTP status code: " << status_code); + + count = GetContentLength(); + + if(count > 0) + body.Reserve(count); + + if(method == METHOD_HEAD) + phase = FINISHED; + else + if(header["transfer-encoding"] == "chunked") { + count = 0; + StartPhase(CHUNK_HEADER); + } + else + StartPhase(BODY); + body.Clear(); + bodylen = 0; + gzip = GetHeader("content-encoding") == "gzip"; + if(gzip) { + gzip = true; + z.WhenOut = callback(this, &HttpRequest::Out); + z.ChunkSize(chunk).GZip().Decompress(); + } +} + +void HttpRequest::ContentOut(const void *ptr, dword size) +{ + body.Cat((const char *)ptr, size); +} + +void HttpRequest::Out(const void *ptr, dword size) +{ + LLOG("HTTP Out " << size); + if(z.IsError()) + HttpError("gzip format error"); + int64 l = bodylen + size; + if(l > max_content_size) { + HttpError("content length exceeded " + AsString(max_content_size)); + phase = FAILED; + return; + } + WhenContent(ptr, size); + bodylen += size; +} + +bool HttpRequest::ReadingBody() +{ + LLOG("HTTP reading data " << count); + int n = chunk; + if(count >= 0) + n = min(n, count); + String s = Get(n); + if(s.GetCount() == 0) + return !IsEof() && count; +#ifndef ENDZIP + if(gzip) + z.Put(~s, s.GetCount()); + else +#endif + Out(~s, s.GetCount()); + if(count >= 0) { + count -= s.GetCount(); + return !IsEof() && count > 0; + } + return !IsEof(); +} + +void HttpRequest::Start() +{ + Close(); + ClearError(); + gzip = false; + z.Clear(); + header.Clear(); + + bool use_proxy = !IsNull(proxy_host); + + int p = use_proxy ? proxy_port : port; + if(!p) + p = DEFAULT_HTTP_PORT; + String h = use_proxy ? proxy_host : host; + if(IsNull(GetTimeout())) { + addrinfo.Execute(h, p); + StartRequest(); + } + else { + addrinfo.Start(h, p); + StartPhase(DNS); + } +} + +void HttpRequest::Dns() +{ + for(int i = 0; i <= Nvl(GetTimeout(), INT_MAX); i++) { + if(!addrinfo.InProgress()) { + StartRequest(); + return; + } + Sleep(1); + } +} + +void HttpRequest::StartRequest() +{ + if(!Connect(addrinfo)) + return; + + StartPhase(REQUEST); + count = 0; + String ctype = contenttype; + if((method == METHOD_POST || method == METHOD_PUT) && IsNull(ctype)) + ctype = "application/x-www-form-urlencoded"; + switch(method) { + case METHOD_GET: data << "GET "; break; + case METHOD_POST: data << "POST "; break; + case METHOD_PUT: data << "PUT "; break; + case METHOD_HEAD: data << "HEAD "; break; + default: NEVER(); // invalid method + } + String host_port = host; + if(port) + host_port << ':' << port; + String url; + url << "http://" << host_port << Nvl(path, "/"); + if(!IsNull(proxy_host)) + data << url; + else + data << Nvl(path, "/"); + data << " HTTP/1.1\r\n"; + if(std_headers) { + data << "URL: " << url << "\r\n" + << "Host: " << host_port << "\r\n" + << "Connection: close\r\n" + << "Accept: " << Nvl(accept, "*/*") << "\r\n" + << "Accept-Encoding: gzip\r\n" + << "Agent: " << Nvl(agent, "Ultimate++ HTTP client") << "\r\n"; + if(postdata.GetCount()) + data << "Content-Length: " << postdata.GetCount() << "\r\n"; + if(ctype.GetCount()) + data << "Content-Type: " << ctype << "\r\n"; + } + if(!IsNull(proxy_host) && !IsNull(proxy_username)) + data << "Proxy-Authorization: Basic " << Base64Encode(proxy_username + ':' + proxy_password) << "\r\n"; + if(!IsNull(digest)) + data << "Authorization: Digest " << digest << "\r\n"; + else + if(!force_digest && (!IsNull(username) || !IsNull(password))) + data << "Authorization: Basic " << Base64Encode(username + ":" + password) << "\r\n"; + data << request_headers << "\r\n" << postdata; // !!! POST PHASE !!! + LLOG("HTTP REQUEST " << host << ":" << port); + LLOG("HTTP request:\n" << data); + if(ssl) + StartSSL(); +} + +bool HttpRequest::SendingData() +{ + for(;;) { + int n = min(2048, data.GetLength() - count); + n = Put(~data + count, n); + if(n == 0) + break; + count += n; + } + return count < data.GetLength(); +} + +bool HttpRequest::ReadingHeader() +{ + for(;;) { + int c = Get(); + if(c < 0) + return !IsEof(); + else + data.Cat(c); + if(data.GetCount() > 3) { + const char *h = data.Last(); + if(h[0] == '\n' && (h[-1] == '\r' && h[-2] == '\n' || h[-1] == '\n')) + return false; + } + if(data.GetCount() > max_header_size) { + HttpError("HTTP header exceeded " + AsString(max_header_size)); + return true; + } + } +} + +String HttpRequest::Execute() +{ + while(Do()); + return IsSuccess() ? GetContent() : String::GetVoid(); +} + +String HttpRequest::GetPhaseName() const +{ + static const char *m[] = { + "Start", + "Resolving host name", + "Sending request", + "Receiving header", + "Receiving content", + "Receiving chunk header", + "Receiving content chunk", + "Receiving trailer", + "Finished", + "Failed", + }; + return phase >= 0 && phase <= FAILED ? m[phase] : ""; +} + +END_UPP_NAMESPACE diff --git a/uppsrc/Core/Socket.cpp b/uppsrc/Core/Socket.cpp index 9cc99b0ce..11b74f8e4 100644 --- a/uppsrc/Core/Socket.cpp +++ b/uppsrc/Core/Socket.cpp @@ -293,7 +293,9 @@ bool TcpSocket::Open(int family, int type, int protocol) bool TcpSocket::Listen(int port, int listen_count, bool ipv6_, bool reuse) { + Close(); Init(); + Reset(); ipv6 = ipv6_; if(!Open(ipv6 ? AF_INET6 : AF_INET, SOCK_STREAM, 0)) @@ -337,6 +339,9 @@ bool TcpSocket::Listen(int port, int listen_count, bool ipv6_, bool reuse) bool TcpSocket::Accept(TcpSocket& ls) { Close(); + Init(); + Reset(); + if(timeout && !ls.WaitRead()) return false; if(!Open(ls.ipv6 ? AF_INET6 : AF_INET, SOCK_STREAM, 0)) @@ -418,6 +423,7 @@ bool TcpSocket::Connect(IpAddrInfo& info) { LLOG("TCP Connect addrinfo"); Init(); + Reset(); addrinfo *result = info.GetResult(); return result && RawConnect(result); } @@ -427,6 +433,7 @@ bool TcpSocket::Connect(const char *host, int port) LLOG("TCP Connect(" << host << ':' << port << ')'); Init(); + Reset(); IpAddrInfo info; if(!info.Execute(host, port)) { SetSockError(Format("getaddrinfo(%s) failed", host)); @@ -456,7 +463,7 @@ void TcpSocket::RawClose() void TcpSocket::Close() { if(ssl) - ssl->Close(); + ssl->Close(); else RawClose(); ssl.Clear(); @@ -486,7 +493,7 @@ int TcpSocket::RawSend(const void *buf, int amount) int TcpSocket::Send(const void *buf, int amount) { - return ssl ? ssl->Send(buf, amount) : RawSend(buf, amount); + return ssl ? ssl->Send(buf, amount) : RawSend(buf, amount); } void TcpSocket::Shutdown() @@ -551,7 +558,7 @@ bool TcpSocket::RawWait(dword flags) bool TcpSocket::Wait(dword flags) { - return ssl ? ssl->Wait(flags) : RawWait(flags); + return ssl ? ssl->Wait(flags) : RawWait(flags); } int TcpSocket::Put(const char *s, int length) @@ -599,7 +606,7 @@ int TcpSocket::RawRecv(void *buf, int amount) int TcpSocket::Recv(void *buffer, int maxlen) { - return ssl ? ssl->Recv(buffer, maxlen) : RawRecv(buffer, maxlen); + return ssl ? ssl->Recv(buffer, maxlen) : RawRecv(buffer, maxlen); } void TcpSocket::ReadBuffer() @@ -685,31 +692,31 @@ String TcpSocket::GetLine(int maxlen) } } -void TcpSocket::SetSockError(const char *context, int code, const char *errdesc) +void TcpSocket::SetSockError(const char *context, int code, const char *errdesc) { - errorcode = code; - errordesc.Clear(); + errorcode = code; + errordesc.Clear(); if(socket != INVALID_SOCKET) - errordesc << "socket(" << (int)socket << ") / "; - errordesc << context << ": " << errdesc; + errordesc << "socket(" << (int)socket << ") / "; + errordesc << context << ": " << errdesc; is_error = true; } -void TcpSocket::SetSockError(const char *context, const char *errdesc) -{ - SetSockError(context, GetErrorCode(), errdesc); -} - +void TcpSocket::SetSockError(const char *context, const char *errdesc) +{ + SetSockError(context, GetErrorCode(), errdesc); +} + void TcpSocket::SetSockError(const char *context) { SetSockError(context, TcpSocketErrorDesc(GetErrorCode())); } -TcpSocket::SSL *(*TcpSocket::CreateSSL)(TcpSocket& socket); +TcpSocket::SSL *(*TcpSocket::CreateSSL)(TcpSocket& socket); bool TcpSocket::StartSSL() { - ASSERT(IsOpen()); + ASSERT(IsOpen()); if(!CreateSSL) { errorcode = -1; errordesc = "Missing SSL support (Core/SSL)"; @@ -720,8 +727,8 @@ bool TcpSocket::StartSSL() errordesc = "Socket not open or listening"; return false; } - ssl = (*CreateSSL)(*this); - if(!ssl->Start()) { + ssl = (*CreateSSL)(*this); + if(!ssl->Start()) { ssl.Clear(); return false; } diff --git a/uppsrc/Core/Web.h b/uppsrc/Core/Web.h index be31f6c17..96e930541 100644 --- a/uppsrc/Core/Web.h +++ b/uppsrc/Core/Web.h @@ -1,372 +1,375 @@ -String FormatIP(dword _ip); - -String UrlEncode(const String& s); -String UrlEncode(const String& s, const char *specials); -String UrlDecode(const char *b, const char *e); -inline String UrlDecode(const String& s) { return UrlDecode(s.Begin(), s.End() ); } - -String Base64Encode(const char *b, const char *e); -inline String Base64Encode(const String& data) { return Base64Encode(data.Begin(), data.End()); } -String Base64Decode(const char *b, const char *e); -inline String Base64Decode(const String& data) { return Base64Decode(data.Begin(), data.End()); } - -class IpAddrInfo { - enum { COUNT = 32 }; - struct Entry { - const char *host; - const char *port; - int status; - addrinfo *addr; - }; - static Entry pool[COUNT]; - - enum { - EMPTY = 0, WORKING, CANCELED, RESOLVED, FAILED - }; - - String host, port; - Entry *entry; - Entry exe[1]; - - static void EnterPool(); - static void LeavePool(); - static rawthread_t rawthread__ Thread(void *ptr); - - void Start(); - -public: - void Start(const String& host, int port); - bool InProgress(); - bool Execute(const String& host, int port); - addrinfo *GetResult(); - void Clear(); - - IpAddrInfo(); - ~IpAddrInfo() { Clear(); } -}; - -enum { WAIT_READ = 1, WAIT_WRITE = 2, WAIT_EXCEPTION = 4, WAIT_ALL = 7 }; - -class TcpSocket { - enum { BUFFERSIZE = 512 }; - enum { NONE, CONNECT, ACCEPT }; - SOCKET socket; - int mode; - char buffer[BUFFERSIZE]; - char *ptr; - char *end; - bool is_eof; - bool is_error; - bool is_abort; - bool ipv6; - - int timeout; - int waitstep; - int done; - - int errorcode; - String errordesc; - - struct SSL { - virtual bool Start() = 0; - virtual bool Wait(dword flags) = 0; - virtual int Send(const void *buffer, int maxlen) = 0; - virtual int Recv(void *buffer, int maxlen) = 0; - virtual void Close() = 0; - - virtual ~SSL() {} - }; - - One ssl; - - struct SSLImp; - friend struct SSLImp; - - static SSL *(*CreateSSL)(TcpSocket& socket); - static SSL *CreateSSLImp(TcpSocket& socket); - - friend void InitCreateSSL(); - - bool RawWait(dword flags); - SOCKET AcceptRaw(dword *ipaddr, int timeout_msec); - bool Open(int family, int type, int protocol); - int RawRecv(void *buffer, int maxlen); - int Recv(void *buffer, int maxlen); - int RawSend(const void *buffer, int maxlen); - int Send(const void *buffer, int maxlen); - bool RawConnect(addrinfo *info); - void RawClose(); - - void ReadBuffer(); - int Get_(); - int Peek_(); - - void Reset(); - - void SetSockError(const char *context, int code, const char *errdesc); - void SetSockError(const char *context, const char *errdesc); - void SetSockError(const char *context); - - static int GetErrorCode(); - static bool WouldBlock(); - -public: - Callback WhenWait; - - static String GetHostName(); - - int GetDone() const { return done; } - - static void Init(); - - bool IsOpen() const { return socket != INVALID_SOCKET; } - bool IsEof() const { return is_eof && ptr == end; } - - bool IsError() const { return is_error; } - void ClearError() { is_error = false; errorcode = 0; errordesc.Clear(); } - int GetError() const { return errorcode; } - String GetErrorDesc() const { return errordesc; } - - void Abort() { is_abort = true; } - bool IsAbort() const { return is_abort; } - void ClearAbort() { is_abort = false; } - - SOCKET GetSOCKET() const { return socket; } - String GetPeerAddr() const; - - void Attach(SOCKET socket); - bool Connect(const char *host, int port); - bool Connect(IpAddrInfo& info); - bool Listen(int port, int listen_count, bool ipv6 = false, bool reuse = true); - bool Accept(TcpSocket& listen_socket); - void Close(); - void Shutdown(); - - void NoDelay(); - void Linger(int msecs); - void NoLinger() { Linger(Null); } - void Reuse(bool reuse = true); - - bool Wait(dword events); - bool WaitRead() { return Wait(WAIT_READ); } - bool WaitWrite() { return Wait(WAIT_WRITE); } - - int Peek() { return ptr < end ? *ptr : Peek_(); } - int Term() { return Peek(); } - int Get() { return ptr < end ? *ptr++ : Get_(); } - int Get(void *buffer, int len); - String Get(int len); - int GetAll(void *buffer, int len) { return Get(buffer, len) == len; } - String GetAll(int len) { String s = Get(len); return s.GetCount() == len ? s : String::GetVoid(); } - String GetLine(int maxlen = 2000000); - - int Put(const char *s, int len); - int Put(const String& s) { return Put(s.Begin(), s.GetLength()); } - bool PutAll(const char *s, int len) { return Put(s, len) == len; } - bool PutAll(const String& s) { return Put(s) == s.GetCount(); } - - bool StartSSL(); - bool IsSSL() const { return ssl; } - - TcpSocket& Timeout(int ms) { timeout = ms; return *this; } - int GetTimeout() const { return timeout; } - TcpSocket& Blocking() { return Timeout(Null); } - - TcpSocket(); - ~TcpSocket() { Close(); } -}; - -class SocketWaitEvent { - Vector< Tuple2 > socket; - fd_set read[1], write[1], exception[1]; - -public: - void Clear() { socket.Clear(); } - void Add(SOCKET s, dword events = WAIT_ALL) { socket.Add(MakeTuple((int)s, events)); } - void Add(TcpSocket& s, dword events = WAIT_ALL) { Add(s.GetSOCKET(), events); } - int Wait(int timeout); - dword Get(int i) const; - dword operator[](int i) const { return Get(i); } - - SocketWaitEvent(); -}; - -struct HttpHeader { - String first_line; - VectorMap fields; - - String operator[](const char *id) { return fields.Get(id, Null); } - - bool Response(String& protocol, int& code, String& reason); - bool Request(String& method, String& uri, String& version); - - void Clear(); - bool Parse(const String& hdrs); -}; - -class HttpRequest : public TcpSocket { - int phase; - String data; - int count; - - HttpHeader header; - - String error; - String body; - - enum { - DEFAULT_HTTP_PORT = 80, - }; - - enum { - METHOD_GET, - METHOD_POST, - METHOD_HEAD, - METHOD_PUT, - }; - - int max_header_size; - int max_content_size; - int max_redirects; - int max_retries; - int timeout; - - String host; - int port; - String proxy_host; - int proxy_port; - String proxy_username; - String proxy_password; - String path; - bool ssl; - - int method; - String accept; - String agent; - bool force_digest; - bool is_post; - bool std_headers; - bool hasurlvar; - String contenttype; - String username; - String password; - String digest; - String request_headers; - String postdata; - - String protocol; - int status_code; - String reason_phrase; - - int start_time; - int retry_count; - int redirect_count; - - int chunk; - - IpAddrInfo addrinfo; - int bodylen; - bool gzip; - Zlib z; - - void Init(); - - void StartPhase(int s); - void Start(); - void Dns(); - void StartRequest(); - bool SendingData(); - bool ReadingHeader(); - void StartBody(); - bool ReadingBody(); - void ReadingChunkHeader(); - void Finish(); - - void HttpError(const char *s); - void ContentOut(const void *ptr, dword size); - void Out(const void *ptr, dword size); - - String CalculateDigest(const String& authenticate) const; - -public: - Callback2 WhenContent; - - HttpRequest& MaxHeaderSize(int m) { max_header_size = m; return *this; } - HttpRequest& MaxContentSize(int m) { max_content_size = m; return *this; } - HttpRequest& MaxRedirect(int n) { max_redirects = n; return *this; } - HttpRequest& MaxRetries(int n) { max_retries = n; return *this; } - HttpRequest& RequestTimeout(int ms) { timeout = ms; return *this; } - HttpRequest& ChunkSize(int n) { chunk = n; return *this; } - - HttpRequest& Method(int m) { method = m; return *this; } - HttpRequest& GET() { return Method(METHOD_GET); } - HttpRequest& POST() { return Method(METHOD_POST); } - HttpRequest& HEAD() { return Method(METHOD_HEAD); } - HttpRequest& PUT() { return Method(METHOD_PUT); } - - HttpRequest& Host(const String& h) { host = h; return *this; } - HttpRequest& Port(int p) { port = p; return *this; } - HttpRequest& SSL(bool b = true) { ssl = b; return *this; } - HttpRequest& Path(const String& p) { path = p; return *this; } - HttpRequest& User(const String& u, const String& p) { username = u; password = p; return *this; } - HttpRequest& Digest() { force_digest = true; return *this; } - HttpRequest& Digest(const String& d) { digest = d; return *this; } - HttpRequest& Url(const char *url); - HttpRequest& UrlVar(const char *id, const String& data); - HttpRequest& operator()(const char *id, const String& data) { return UrlVar(id, data); } - HttpRequest& PostData(const String& pd) { postdata = pd; return *this; } - HttpRequest& PostUData(const String& pd) { return PostData(UrlEncode(pd)); } - HttpRequest& Post(const String& data) { POST(); return PostData(data); } - HttpRequest& Post(const char *id, const String& data); - - HttpRequest& Headers(const String& h) { request_headers = h; return *this; } - HttpRequest& ClearHeaders() { return Headers(Null); } - HttpRequest& AddHeaders(const String& h) { request_headers.Cat(h); return *this; } - HttpRequest& Header(const char *id, const String& data); - - HttpRequest& StdHeaders(bool sh) { std_headers = sh; return *this; } - HttpRequest& NoStdHeaders() { return StdHeaders(false); } - HttpRequest& Accept(const String& a) { accept = a; return *this; } - HttpRequest& Agent(const String& a) { agent = a; return *this; } - HttpRequest& ContentType(const String& a) { contenttype = a; return *this; } - - HttpRequest& Proxy(const String& host, int port) { proxy_host = host; proxy_port = port; return *this; } - HttpRequest& Proxy(const char *url); - HttpRequest& ProxyAuth(const String& u, const String& p) { proxy_username = u; proxy_password = p; return *this; } - - bool IsSocketError() const { return TcpSocket::IsError(); } - bool IsHttpError() const { return !IsNull(error) ; } - bool IsError() const { return IsSocketError() || IsHttpError(); } - String GetErrorDesc() const { return IsSocketError() ? TcpSocket::GetErrorDesc() : error; } - void ClearError() { TcpSocket::ClearError(); error.Clear(); } - - String GetHeader(const char *s) { return header[s]; } - String operator[](const char *s) { return GetHeader(s); } - String GetRedirectUrl(); - int GetContentLength(); - int GetStatusCode() const { return status_code; } - String GetReasonPhrase() const { return reason_phrase; } - - String GetContent() const { return body; } - String operator~() const { return GetContent(); } - operator String() const { return GetContent(); } - void ClearContent() { body.Clear(); } - - enum Phase { - START, DNS, REQUEST, HEADER, BODY, CHUNK_HEADER, CHUNK_BODY, TRAILER, FINISHED, FAILED - }; - - bool Do(); - int GetPhase() const { return phase; } - String GetPhaseName() const; - bool InProgress() const { return phase != FAILED && phase != FINISHED; } - bool IsFailure() const { return phase == FAILED; } - bool IsSuccess() const { return phase == FINISHED && status_code >= 200 && status_code < 300; } - - String Execute(); - - HttpRequest(); - HttpRequest(const char *url); - - static void Trace(bool b = true); -}; +String FormatIP(dword _ip); + +String UrlEncode(const String& s); +String UrlEncode(const String& s, const char *specials); +String UrlDecode(const char *b, const char *e); +inline String UrlDecode(const String& s) { return UrlDecode(s.Begin(), s.End() ); } + +String Base64Encode(const char *b, const char *e); +inline String Base64Encode(const String& data) { return Base64Encode(data.Begin(), data.End()); } +String Base64Decode(const char *b, const char *e); +inline String Base64Decode(const String& data) { return Base64Decode(data.Begin(), data.End()); } + +class IpAddrInfo { + enum { COUNT = 32 }; + struct Entry { + const char *host; + const char *port; + int status; + addrinfo *addr; + }; + static Entry pool[COUNT]; + + enum { + EMPTY = 0, WORKING, CANCELED, RESOLVED, FAILED + }; + + String host, port; + Entry *entry; + Entry exe[1]; + + static void EnterPool(); + static void LeavePool(); + static rawthread_t rawthread__ Thread(void *ptr); + + void Start(); + +public: + void Start(const String& host, int port); + bool InProgress(); + bool Execute(const String& host, int port); + addrinfo *GetResult(); + void Clear(); + + IpAddrInfo(); + ~IpAddrInfo() { Clear(); } +}; + +enum { WAIT_READ = 1, WAIT_WRITE = 2, WAIT_EXCEPTION = 4, WAIT_ALL = 7 }; + +class TcpSocket { + enum { BUFFERSIZE = 512 }; + enum { NONE, CONNECT, ACCEPT }; + SOCKET socket; + int mode; + char buffer[BUFFERSIZE]; + char *ptr; + char *end; + bool is_eof; + bool is_error; + bool is_abort; + bool ipv6; + + int timeout; + int waitstep; + int done; + + int errorcode; + String errordesc; + + struct SSL { + virtual bool Start() = 0; + virtual bool Wait(dword flags) = 0; + virtual int Send(const void *buffer, int maxlen) = 0; + virtual int Recv(void *buffer, int maxlen) = 0; + virtual void Close() = 0; + + virtual ~SSL() {} + }; + + One ssl; + + struct SSLImp; + friend struct SSLImp; + + static SSL *(*CreateSSL)(TcpSocket& socket); + static SSL *CreateSSLImp(TcpSocket& socket); + + friend void InitCreateSSL(); + + bool RawWait(dword flags); + SOCKET AcceptRaw(dword *ipaddr, int timeout_msec); + bool Open(int family, int type, int protocol); + int RawRecv(void *buffer, int maxlen); + int Recv(void *buffer, int maxlen); + int RawSend(const void *buffer, int maxlen); + int Send(const void *buffer, int maxlen); + bool RawConnect(addrinfo *info); + void RawClose(); + + void ReadBuffer(); + int Get_(); + int Peek_(); + + void Reset(); + + void SetSockError(const char *context, int code, const char *errdesc); + void SetSockError(const char *context, const char *errdesc); + void SetSockError(const char *context); + + static int GetErrorCode(); + static bool WouldBlock(); + +public: + Callback WhenWait; + + static String GetHostName(); + + int GetDone() const { return done; } + + static void Init(); + + bool IsOpen() const { return socket != INVALID_SOCKET; } + bool IsEof() const { return is_eof && ptr == end; } + + bool IsError() const { return is_error; } + void ClearError() { is_error = false; errorcode = 0; errordesc.Clear(); } + int GetError() const { return errorcode; } + String GetErrorDesc() const { return errordesc; } + + void Abort() { is_abort = true; } + bool IsAbort() const { return is_abort; } + void ClearAbort() { is_abort = false; } + + SOCKET GetSOCKET() const { return socket; } + String GetPeerAddr() const; + + void Attach(SOCKET socket); + bool Connect(const char *host, int port); + bool Connect(IpAddrInfo& info); + bool Listen(int port, int listen_count, bool ipv6 = false, bool reuse = true); + bool Accept(TcpSocket& listen_socket); + void Close(); + void Shutdown(); + + void NoDelay(); + void Linger(int msecs); + void NoLinger() { Linger(Null); } + void Reuse(bool reuse = true); + + bool Wait(dword events); + bool WaitRead() { return Wait(WAIT_READ); } + bool WaitWrite() { return Wait(WAIT_WRITE); } + + int Peek() { return ptr < end ? *ptr : Peek_(); } + int Term() { return Peek(); } + int Get() { return ptr < end ? *ptr++ : Get_(); } + int Get(void *buffer, int len); + String Get(int len); + int GetAll(void *buffer, int len) { return Get(buffer, len) == len; } + String GetAll(int len) { String s = Get(len); return s.GetCount() == len ? s : String::GetVoid(); } + String GetLine(int maxlen = 2000000); + + int Put(const char *s, int len); + int Put(const String& s) { return Put(s.Begin(), s.GetLength()); } + bool PutAll(const char *s, int len) { return Put(s, len) == len; } + bool PutAll(const String& s) { return Put(s) == s.GetCount(); } + + bool StartSSL(); + bool IsSSL() const { return ssl; } + + TcpSocket& Timeout(int ms) { timeout = ms; return *this; } + int GetTimeout() const { return timeout; } + TcpSocket& Blocking() { return Timeout(Null); } + + TcpSocket(); + ~TcpSocket() { Close(); } +}; + +class SocketWaitEvent { + Vector< Tuple2 > socket; + fd_set read[1], write[1], exception[1]; + +public: + void Clear() { socket.Clear(); } + void Add(SOCKET s, dword events = WAIT_ALL) { socket.Add(MakeTuple((int)s, events)); } + void Add(TcpSocket& s, dword events = WAIT_ALL) { Add(s.GetSOCKET(), events); } + int Wait(int timeout); + dword Get(int i) const; + dword operator[](int i) const { return Get(i); } + + SocketWaitEvent(); +}; + +struct HttpHeader { + String first_line; + VectorMap fields; + + String operator[](const char *id) { return fields.Get(id, Null); } + + bool Response(String& protocol, int& code, String& reason); + bool Request(String& method, String& uri, String& version); + + void Clear(); + bool Parse(const String& hdrs); +}; + +class HttpRequest : public TcpSocket { + int phase; + String data; + int count; + + HttpHeader header; + + String error; + String body; + + enum { + DEFAULT_HTTP_PORT = 80, + }; + + enum { + METHOD_GET, + METHOD_POST, + METHOD_HEAD, + METHOD_PUT, + }; + + int max_header_size; + int max_content_size; + int max_redirects; + int max_retries; + int timeout; + + String host; + int port; + String proxy_host; + int proxy_port; + String proxy_username; + String proxy_password; + String path; + bool ssl; + + int method; + String accept; + String agent; + bool force_digest; + bool is_post; + bool std_headers; + bool hasurlvar; + String contenttype; + String username; + String password; + String digest; + String request_headers; + String postdata; + + String protocol; + int status_code; + String reason_phrase; + + int start_time; + int retry_count; + int redirect_count; + + int chunk; + + IpAddrInfo addrinfo; + int bodylen; + bool gzip; + Zlib z; + + void Init(); + + void StartPhase(int s); + void Start(); + void Dns(); + void StartRequest(); + bool SendingData(); + bool ReadingHeader(); + void StartBody(); + bool ReadingBody(); + void ReadingChunkHeader(); + void Finish(); + + void CopyCookies(); + + void HttpError(const char *s); + void ContentOut(const void *ptr, dword size); + void Out(const void *ptr, dword size); + + String CalculateDigest(const String& authenticate) const; + +public: + Callback2 WhenContent; + + HttpRequest& MaxHeaderSize(int m) { max_header_size = m; return *this; } + HttpRequest& MaxContentSize(int m) { max_content_size = m; return *this; } + HttpRequest& MaxRedirect(int n) { max_redirects = n; return *this; } + HttpRequest& MaxRetries(int n) { max_retries = n; return *this; } + HttpRequest& RequestTimeout(int ms) { timeout = ms; return *this; } + HttpRequest& ChunkSize(int n) { chunk = n; return *this; } + + HttpRequest& Method(int m) { method = m; return *this; } + HttpRequest& GET() { return Method(METHOD_GET); } + HttpRequest& POST() { return Method(METHOD_POST); } + HttpRequest& HEAD() { return Method(METHOD_HEAD); } + HttpRequest& PUT() { return Method(METHOD_PUT); } + + HttpRequest& Host(const String& h) { host = h; return *this; } + HttpRequest& Port(int p) { port = p; return *this; } + HttpRequest& SSL(bool b = true) { ssl = b; return *this; } + HttpRequest& Path(const String& p) { path = p; return *this; } + HttpRequest& User(const String& u, const String& p) { username = u; password = p; return *this; } + HttpRequest& Digest() { force_digest = true; return *this; } + HttpRequest& Digest(const String& d) { digest = d; return *this; } + HttpRequest& Url(const char *url); + HttpRequest& UrlVar(const char *id, const String& data); + HttpRequest& operator()(const char *id, const String& data) { return UrlVar(id, data); } + HttpRequest& PostData(const String& pd) { postdata = pd; return *this; } + HttpRequest& PostUData(const String& pd) { return PostData(UrlEncode(pd)); } + HttpRequest& Post(const String& data) { POST(); return PostData(data); } + HttpRequest& Post(const char *id, const String& data); + + HttpRequest& Headers(const String& h) { request_headers = h; return *this; } + HttpRequest& ClearHeaders() { return Headers(Null); } + HttpRequest& AddHeaders(const String& h) { request_headers.Cat(h); return *this; } + HttpRequest& Header(const char *id, const String& data); + HttpRequest& Cookie(const String& cookie) { return Header("Cookie", cookie); } + + HttpRequest& StdHeaders(bool sh) { std_headers = sh; return *this; } + HttpRequest& NoStdHeaders() { return StdHeaders(false); } + HttpRequest& Accept(const String& a) { accept = a; return *this; } + HttpRequest& Agent(const String& a) { agent = a; return *this; } + HttpRequest& ContentType(const String& a) { contenttype = a; return *this; } + + HttpRequest& Proxy(const String& host, int port) { proxy_host = host; proxy_port = port; return *this; } + HttpRequest& Proxy(const char *url); + HttpRequest& ProxyAuth(const String& u, const String& p) { proxy_username = u; proxy_password = p; return *this; } + + bool IsSocketError() const { return TcpSocket::IsError(); } + bool IsHttpError() const { return !IsNull(error) ; } + bool IsError() const { return IsSocketError() || IsHttpError(); } + String GetErrorDesc() const { return IsSocketError() ? TcpSocket::GetErrorDesc() : error; } + void ClearError() { TcpSocket::ClearError(); error.Clear(); } + + String GetHeader(const char *s) { return header[s]; } + String operator[](const char *s) { return GetHeader(s); } + String GetRedirectUrl(); + int GetContentLength(); + int GetStatusCode() const { return status_code; } + String GetReasonPhrase() const { return reason_phrase; } + + String GetContent() const { return body; } + String operator~() const { return GetContent(); } + operator String() const { return GetContent(); } + void ClearContent() { body.Clear(); } + + enum Phase { + START, DNS, REQUEST, HEADER, BODY, CHUNK_HEADER, CHUNK_BODY, TRAILER, FINISHED, FAILED + }; + + bool Do(); + int GetPhase() const { return phase; } + String GetPhaseName() const; + bool InProgress() const { return phase != FAILED && phase != FINISHED; } + bool IsFailure() const { return phase == FAILED; } + bool IsSuccess() const { return phase == FINISHED && status_code >= 200 && status_code < 300; } + + String Execute(); + + HttpRequest(); + HttpRequest(const char *url); + + static void Trace(bool b = true); +};