Core: FastCompress, xxHash, plugin/lz4: reworked as lz4f

git-svn-id: svn://ultimatepp.org/upp/trunk@9957 f0d560ea-af0d-0410-9eb7-867de7ffcac7

uppsrc/Core/Hash.h

@@ -17,6 +17,7 @@ public:
 	void   Finish(byte *hash16);
 	String FinishString();
 	String FinishStringS();
+	void   Reset();
 	
 	Md5Stream();
 	~Md5Stream();
@@ -44,7 +45,8 @@ public:
 	String FinishString();
 	String FinishStringS();
 
-	void   New();
+	void   Reset();
+	void   New()                         { Reset(); }
 	
 	Sha1Stream(); 
 	~Sha1Stream();
@@ -65,6 +67,8 @@ class xxHashStream : public OutStream {
 public:
 	int Finish();
 	
+	void Reset(dword seed = 0);
+	
 	xxHashStream(dword seed = 0);
 };
 

uppsrc/Core/MD5.cpp

@@ -339,11 +339,16 @@ String Md5Stream::FinishStringS()
 	return HexString(hash, 16, 4);
 }
 
-Md5Stream::Md5Stream()
+void Md5Stream::Reset()
 {
 	MD5Init (&context);
 }
 
+Md5Stream::Md5Stream()
+{
+	Reset();
+}
+
 Md5Stream::~Md5Stream()
 {
 	memset(&context, 0, sizeof(context));

uppsrc/Core/SHA1.cpp

@@ -158,7 +158,7 @@ String Sha1Stream::FinishStringS()
 	return HexString(hash, 20, 4);
 }
 
-void Sha1Stream::New() {
+void Sha1Stream::Reset() {
 	SHA1Init(state);
 	pos = 0;
 	size = 0;
@@ -166,7 +166,7 @@ void Sha1Stream::New() {
 
 Sha1Stream::Sha1Stream()
 {
-	New();
+	Reset();
 }
 
 Sha1Stream::~Sha1Stream()

uppsrc/Core/xxHsh.cpp

@@ -7,6 +7,11 @@ namespace Upp {
 xxHashStream::xxHashStream(dword seed)
 {
 	STATIC_ASSERT(sizeof(context) >= sizeof(XXH32_state_t));
+	Reset(seed);
+}
+
+void xxHashStream::Reset(dword seed)
+{
 	XXH32_reset((XXH32_state_t *)context, seed);
 }
 

uppsrc/plugin/lz4/lz4.h

@@ -8,19 +8,24 @@
 namespace Upp {
 
 class Lz4 {
-	enum { BLOCK_BYTES = 64 * 1024 };
-	char        *block[2];
 	Buffer<char> buffer;
+	Buffer<char> outbuf;
 	int8         compress;
 	bool         error;
 	
-	int          cbi;
+	enum { BLOCK_BYTES = 1024 * 1024 };
+	
+	xxHashStream xxh;
+	int          maxblock;
 	int          pos;
+	int          blockchksumsz;
+	bool         header;
+	byte         lz4hdr;
+	String       header_data;
 
-    LZ4_stream_t        lz4Stream[1];
-    LZ4_streamDecode_t  lz4StreamDecode[1];
-    
     String       out;
+    
+    void          TryHeader();
 
 	void          Init();
 	void          FlushOut();

uppsrc/plugin/lz4/lz4upp.cpp

@@ -4,23 +4,42 @@
 
 namespace Upp {
 
+enum {
+	LZ4F_MAGIC       = 0x184D2204,
+
+	LZ4F_VERSIONMASK = 0b11000000,
+	LZ4F_VERSION     = 0b01000000,
+	LZ4F_BLOCKINDEPENDENCE = (1 << 5),
+	LZ4F_BLOCKCHECKSUM     = (1 << 4),
+	LZ4F_CONTENTSIZE       = (1 << 3),
+	LZ4F_CONTENTCHECKSUM   = (1 << 2),
+	
+	LZ4F_MAXSIZEMASK      = 0x70,
+    LZ4F_MAXSIZE_64KB     = 0x40,
+    LZ4F_MAXSIZE_256KB    = 0x50,
+    LZ4F_MAXSIZE_1024KB   = 0x60,
+    LZ4F_MAXSIZE_4096KB   = 0x70,
+};
+
 void Lz4::Init()
 {
 	error = false;
-	cbi = pos = 0;
-	Zero(lz4Stream);
-	Zero(lz4StreamDecode);
+	pos = 0;
+	header = false;
+	xxh.Reset();
 }
 
 void Lz4::Compress()
 {
 	compress = 1;
+	buffer.Alloc(BLOCK_BYTES);
 	Init();
 }
 
 void Lz4::Decompress()
 {
 	compress = 0;
+	header_data.Clear();
 	Init();
 }
 
@@ -32,31 +51,93 @@ void Lz4::PutOut(const void *ptr, int size)
 
 void Lz4::FlushOut()
 {
+	if(!header) {
+		byte h[7];
+		Poke32le(h, LZ4F_MAGIC);
+		h[4] = LZ4F_VERSION | LZ4F_BLOCKINDEPENDENCE | LZ4F_CONTENTCHECKSUM;
+		h[5] = LZ4F_MAXSIZE_1024KB;
+		h[6] = xxHash(h + 4, 2) >> 8;
+		WhenOut(h, 7);
+		header = true;
+		maxblock = BLOCK_BYTES;
+		outbuf.Alloc(4 + LZ4_compressBound(maxblock));
+	}
+	
 	if(!pos)
 		return;
-	LLOG("To compress " << pos << ' ' << MD5String(block[cbi], pos) << ' ' << cbi);
-	int count = LZ4_compress_continue(lz4Stream, block[cbi], ~buffer + 4, pos);
-	if(count < 0) {
+
+	int clen = LZ4_compress(buffer, ~outbuf + 4, pos);
+	if(clen < 0) {
 		error = true;
 		return;
 	}
-	Poke32le(~buffer, count);
-	LLOG("WhenOut " << pos << ": " << count);
-	WhenOut(~buffer, count + 4);
-	cbi = !cbi;
+	xxh.Put(buffer, pos);
+	if(clen >= pos) {
+		Poke32le(~outbuf, 0x80000000 | pos);
+		memcpy(~outbuf + 4, buffer, pos);
+		WhenOut(~outbuf, pos + 4);
+	}
+	else {
+		Poke32le(~outbuf, clen);
+		WhenOut(~outbuf, clen + 4);
+	}
 	pos = 0;
 }
 
 void Lz4::End()
 {
 	ASSERT(compress >= 0);
-	if(compress)
+	if(compress) {
 		FlushOut();
+		byte h[8];
+		Poke32le(h, 0);
+		Poke32le(h + 4, xxh.Finish());
+		WhenOut(h, 8);
+	}
 	else
 		if(pos)
 			error = true;
 }
 
+void Lz4::TryHeader()
+{
+	if(header_data.GetCount() < 7)
+		return;
+
+	if(Peek32le(~header_data) != LZ4F_MAGIC) {
+		error = true;
+		return;
+	}
+	lz4hdr = header_data[4];
+	if((lz4hdr & LZ4F_VERSIONMASK) != LZ4F_VERSION) {
+		error = true;
+		return;
+	}
+	if(!(lz4hdr & LZ4F_BLOCKINDEPENDENCE)) {
+		error = true; // dependent blocks not supported
+		return;
+	}
+	maxblock = header_data[5];
+	maxblock = decode(maxblock & LZ4F_MAXSIZEMASK,
+	                  LZ4F_MAXSIZE_64KB, 1024 * 64,
+	                  LZ4F_MAXSIZE_256KB, 1024 * 256,
+	                  LZ4F_MAXSIZE_1024KB, 1024 * 1024,
+	                  LZ4F_MAXSIZE_4096KB, 1024 * 4096,
+	                  -1);
+	if(maxblock < 0) {
+		error = true;
+		return;
+	}
+	
+	if((lz4hdr & LZ4F_CONTENTSIZE) && header_data.GetCount() < 15)
+		return; // need to skip content length
+
+	header = true;
+	blockchksumsz = 4 * !!(lz4hdr & LZ4F_BLOCKCHECKSUM);
+	buffer.Alloc(maxblock + 4 + blockchksumsz);
+	outbuf.Alloc(maxblock);
+}
+
 void Lz4::Put(const void *ptr_, int size)
 {
 	LLOG("Put " << size);
@@ -74,21 +155,26 @@ void Lz4::Put(const void *ptr_, int size)
 				return;
 			int n = BLOCK_BYTES - pos;
 			if(size >= n) {
-				memcpy(block[cbi] + pos, ptr, n); // TODO: Avoid unnecessary memcpy
+				memcpy(~buffer + pos, ptr, n);
 				pos = BLOCK_BYTES;
 				FlushOut();
 				size -= n;
 				ptr += n;
 			}
 			else {
-				memcpy(block[cbi] + pos, ptr, size);
+				memcpy(buffer + pos, ptr, size);
 				pos += size;
 				break;
 			}
 		}
 	}
 	else {
-		while(size > 0) { // TODO might try to avoid memcpy
+		while(!header && size > 0 && !error) {
+			header_data.Cat(*ptr++);
+			size--;
+			TryHeader();
+		}
+		while(size > 0 && !error) { // TODO might try to avoid memcpy
 			if(pos < 4) { // Get length first
 				int n = min(size, 4 - pos);
 				memcpy(~buffer + pos, ptr, n);
@@ -96,27 +182,53 @@ void Lz4::Put(const void *ptr_, int size)
 				ptr += n;
 				size -= n;
 			}
-			if(pos < 4)
-				return;
-			int count = Peek32le(~buffer);
-			int need = count - (pos - 4);
-			if(size >= need) { // we have enough data to finish the block
-				memcpy(~buffer + pos, ptr, need);
-				int out_count = LZ4_decompress_safe_continue(lz4StreamDecode, ~buffer + 4, block[cbi], count, BLOCK_BYTES);
-				if(out_count <= 0) {
-					error = true;
-					return;
-				}
-				WhenOut(block[cbi], out_count);
-				cbi = !cbi;
-				ptr += need;
-				size -= need;
-				pos = 0;
-			}
 			else {
-				memcpy(~buffer + pos, ptr, size);
-				pos += size;
-				break;
+				int count = Peek32le(~buffer);
+				int len = count & 0x7fffffff;
+				if(len > maxblock)
+					error = true;
+				else
+				if(count == 0) {
+					int need = 4 - pos + 4;
+					if(size >= need) { // we have enough data for final checksum
+						memcpy(~buffer + pos, ptr, need);
+						if(Peek32le(~buffer + 4) != xxh.Finish()) {
+							error = true;
+							return;
+						}
+						size -= need;
+						ptr += need;
+						pos = 0;
+					}
+				}
+				else {
+					int need = len - (pos - 4 - blockchksumsz);
+					if(size >= need) { // we have enough data to finish the block
+						memcpy(~buffer + pos, ptr, need);
+						const char *src = ~buffer + 4;
+						if(count & 0x80000000) {
+							WhenOut(src, len);
+							xxh.Put(src, len);
+						}
+						else {
+							int sz = LZ4_decompress_safe(src, outbuf, len, maxblock);
+							if(sz < 0) {
+								error = true;
+								return;
+							}
+							WhenOut(outbuf, sz);
+							xxh.Put(outbuf, sz);
+						}
+						ptr += need;
+						size -= need;
+						pos = 0;
+					}
+					else {
+						memcpy(~buffer + pos, ptr, size);
+						pos += size;
+						break;
+					}
+				}
 			}
 		}
 	}
@@ -124,10 +236,6 @@ void Lz4::Put(const void *ptr_, int size)
 
 Lz4::Lz4()
 {
-	block[0] = new char[2 * BLOCK_BYTES];
-	block[1] = block[0] + BLOCK_BYTES;
-	buffer.Alloc(LZ4_COMPRESSBOUND(BLOCK_BYTES) + 4);
-	
 	compress = -1;
 	
 	WhenOut = callback(this, &Lz4::PutOut);
@@ -135,7 +243,6 @@ Lz4::Lz4()
 
 Lz4::~Lz4()
 {
-	delete[] block[0];
 }
 
 };

uppsrc/plugin/lz4/util.cpp

@@ -15,7 +15,7 @@ int64 lz4Press(Stream& out, Stream& in, int64 size, Gate2<int64, int64> progress
 			lz4.Decompress();
 		if(CopyStream(outs, in, size, progress) >= 0) {
 			outs.Close();
-			if(!out.IsError() && !outs.IsError())
+			if(!out.IsError() && !outs.IsError() && !lz4.IsError())
 				r = outs.GetCount();
 		}
 	}