diff --git a/uppsrc/Core/Http.cpp b/uppsrc/Core/Http.cpp index 5ec83b1f0..ca5e65935 100644 --- a/uppsrc/Core/Http.cpp +++ b/uppsrc/Core/Http.cpp @@ -4,11 +4,7 @@ NAMESPACE_UPP bool HttpRequest_Trace__; -#ifdef _DEBUG -#define LLOG(x) LOG(x) -#else -#define LLOG(x) if(HttpRequest_Trace__) RLOG(x); else; -#endif +#define LLOG(x) do { if(HttpRequest_Trace__) RLOG(x); } while(0) void HttpRequest::Trace(bool b) { @@ -19,10 +15,10 @@ void HttpRequest::Init() { port = 0; proxy_port = 0; - max_header_size = DEFAULT_MAX_HEADER_SIZE; - max_content_size = DEFAULT_MAX_CONTENT_SIZE; - max_redirects = DEFAULT_MAX_REDIRECTS; - max_retries = DEFAULT_MAX_RETRIES; + max_header_size = 1000000; + max_content_size = 10000000; + max_redirects = 5; + max_retries = 3; force_digest = false; std_headers = true; hasurlvar = false; @@ -30,6 +26,9 @@ void HttpRequest::Init() phase = START; redirect_count = 0; retry_count = 0; + gzip = false; + WhenContent = callback(this, &HttpRequest::ContentOut); + chunk = 4096; } HttpRequest::HttpRequest() @@ -183,6 +182,8 @@ HttpRequest& HttpRequest::Header(const char *id, const String& data) void HttpRequest::HttpError(const char *s) { + if(IsError()) + return; error = NFormat(t_("%s:%d: ") + String(s), host, port); LLOG("HTTP ERROR: " << error); Close(); @@ -202,6 +203,7 @@ bool HttpRequest::Do() case START: retry_count = 0; redirect_count = 0; + gzip = false; StartRequest(); break; case REQUEST: @@ -286,9 +288,10 @@ void HttpRequest::Finish() return; } } - if(GetHeader("content-encoding") == "gzip") - body = GZDecompress(body); phase = FINISHED; + +// if(retry_count < 2) +// HttpError("Checking retry"); } void HttpRequest::ReadingChunkHeader() @@ -299,8 +302,8 @@ void HttpRequest::ReadingChunkHeader() break; else if(c == '\n') { - LLOG("Chunk header: " << data); int n = ScanInt(~data, NULL, 16); + LLOG("HTTP Chunk header: " << data << ' ' << n); if(IsNull(n)) { HttpError("invalid chunk header"); break; @@ -365,24 +368,51 @@ void HttpRequest::StartBody() else StartPhase(BODY); body.Clear(); + bodylen = 0; + if(GetHeader("content-encoding") == "gzip") { + gzip = true; + z.WhenOut = callback(this, &HttpRequest::Out); + z.ChunkSize(chunk).GZip().Decompress(); + } +} + +void HttpRequest::ContentOut(const void *ptr, dword size) +{ + body.Cat((const char *)ptr, size); +} + +void HttpRequest::Out(const void *ptr, dword size) +{ + LLOG("HTTP Out " << size); + if(z.IsError()) + HttpError("gzip format error"); + int64 l = bodylen + size; + if(l > max_content_size) { + HttpError("content length exceeded " + AsString(max_content_size)); + phase = FAILED; + return; + } + WhenContent(ptr, size); + bodylen += size; } bool HttpRequest::ReadingBody() { LLOG("HTTP reading data " << count); - for(;;) { - int n = 2048; - if(count >= 0) - n = min(n, count - body.GetLength()); - String s = Get(n); - if(s.GetCount() + body.GetCount() > max_content_size) { - HttpError("content length exceeded " + AsString(max_content_size)); - return true; - } - body.Cat(s); - if(count < 0 ? IsEof() : body.GetCount() >= count) - return false; + int n = chunk; + if(count >= 0) + n = min(n, count); + String s = Get(n); + if(gzip) { + z.Put(~s, s.GetCount()); } + else + Out(~s, s.GetCount()); + if(count >= 0) { + count -= n; + return !IsEof() && count > 0; + } + return !IsEof(); } void HttpRequest::StartRequest() @@ -398,20 +428,14 @@ void HttpRequest::StartRequest() StartPhase(REQUEST); count = 0; String ctype = contenttype; + if((method == METHOD_POST || method == METHOD_PUT) && IsNull(ctype)) + ctype = "application/x-www-form-urlencoded"; switch(method) { - case METHOD_GET: data << "GET "; break; - case METHOD_POST: - data << "POST "; - if(IsNull(ctype)) - ctype = "application/x-www-form-urlencoded"; - break; - case METHOD_PUT: - data << "PUT "; - if(IsNull(ctype)) - ctype = "application/x-www-form-urlencoded"; - break; - case METHOD_HEAD: data << "HEAD "; break; - default: NEVER(); // invalid method + case METHOD_GET: data << "GET "; break; + case METHOD_POST: data << "POST "; break; + case METHOD_PUT: data << "PUT "; break; + case METHOD_HEAD: data << "HEAD "; break; + default: NEVER(); // invalid method } String host_port = host; if(port) diff --git a/uppsrc/Core/Web.h b/uppsrc/Core/Web.h index b614a1859..17209a3c2 100644 --- a/uppsrc/Core/Web.h +++ b/uppsrc/Core/Web.h @@ -142,10 +142,6 @@ class HttpRequest : public TcpSocket { enum { DEFAULT_HTTP_PORT = 80, - DEFAULT_MAX_HEADER_SIZE = 1000000, - DEFAULT_MAX_CONTENT_SIZE = 10000000, - DEFAULT_MAX_REDIRECTS = 5, - DEFAULT_MAX_RETRIES = 3, }; enum { @@ -188,6 +184,12 @@ class HttpRequest : public TcpSocket { int retry_count; int redirect_count; + + int chunk; + + int bodylen; + bool gzip; + Zlib z; void Init(); @@ -201,15 +203,19 @@ class HttpRequest : public TcpSocket { void Finish(); void HttpError(const char *s); - String Execute0(); + void ContentOut(const void *ptr, dword size); + void Out(const void *ptr, dword size); String CalculateDigest(const String& authenticate) const; public: + Callback2 WhenContent; + HttpRequest& MaxHeaderSize(int m) { max_header_size = m; return *this; } HttpRequest& MaxContentSize(int m) { max_content_size = m; return *this; } HttpRequest& MaxRedirect(int n) { max_redirects = n; return *this; } HttpRequest& MaxRetries(int n) { max_retries = n; return *this; } + HttpRequest& ChunkSize(int n) { chunk = n; return *this; } HttpRequest& Method(int m) { method = m; return *this; } HttpRequest& GET() { return Method(METHOD_GET); } @@ -246,19 +252,23 @@ public: HttpRequest& Proxy(const char *url); HttpRequest& ProxyAuth(const String& u, const String& p) { proxy_username = u; proxy_password = p; return *this; } - bool IsSocketError() const { return TcpSocket::IsError(); } - bool IsHttpError() const { return !IsNull(error) ; } - bool IsError() const { return IsSocketError() || IsHttpError(); } - String GetErrorDesc() const { return IsSocketError() ? TcpSocket::GetErrorDesc() : error; } - void ClearError() { TcpSocket::ClearError(); error.Clear(); } + bool IsSocketError() const { return TcpSocket::IsError(); } + bool IsHttpError() const { return !IsNull(error) ; } + bool IsError() const { return IsSocketError() || IsHttpError(); } + String GetErrorDesc() const { return IsSocketError() ? TcpSocket::GetErrorDesc() : error; } + void ClearError() { TcpSocket::ClearError(); error.Clear(); } - String GetHeader(const char *s) { return header[s]; } - String operator[](const char *s) { return GetHeader(s); } + String GetHeader(const char *s) { return header[s]; } + String operator[](const char *s) { return GetHeader(s); } String GetRedirectUrl(); int GetContentLength(); - int GetStatusCode() const { return status_code; } - String GetResponsePhrase() const { return response_phrase; } - String GetContent() const { return body; } + int GetStatusCode() const { return status_code; } + String GetResponsePhrase() const { return response_phrase; } + + String GetContent() const { return body; } + String operator~() const { return GetContent(); } + operator String() const { return GetContent(); } + void ClearContent() { body.Clear(); } enum Phase { START, REQUEST, HEADER, BODY, CHUNK_HEADER, CHUNK_BODY, TRAILER, FINISHED, FAILED diff --git a/uppsrc/Core/z.h b/uppsrc/Core/z.h index eb5d20880..2c2a4f37b 100644 --- a/uppsrc/Core/z.h +++ b/uppsrc/Core/z.h @@ -30,6 +30,7 @@ class Zlib { bool gzip_footer; String gzip_hs; String footer; + String out; void PutOut(const void *ptr, dword size); void Pump(bool finish); @@ -40,22 +41,26 @@ class Zlib { public: Callback2 WhenOut; - String out; void Compress(); void Decompress(); void Put(const void *ptr, dword size); void Put(const String& s) { Put(~s, s.GetCount()); } void End(); + + const String& Get() const { return out; } + operator const String&() const { return out; } + const String& operator~() const { return out; } + void ClearOut() { out.Clear(); } - dword GetCrc() const { return crc; } + dword GetCRC() const { return crc; } bool IsError() const { return error; } Zlib& GZip(bool gzip_ = true) { gzip = gzip_; return *this; } Zlib& Header(bool hdr_ = true) { hdr = hdr_; return *this; } Zlib& NoHeader() { return Header(false); } - Zlib& Crc(bool b = true) { docrc = b; return *this; } - Zlib& NoCrc() { return Crc(false); } + Zlib& CRC(bool b = true) { docrc = b; return *this; } + Zlib& NoCRC() { return CRC(false); } Zlib& ChunkSize(int n); Zlib();