From 0d5b95ea56d53ce3ff0cbf9ae2a95e9f7e9ebf98 Mon Sep 17 00:00:00 2001 From: cxl Date: Wed, 22 Jun 2016 17:24:47 +0000 Subject: [PATCH] Core: FastCompress, xxHash, plugin/lz4: reworked as lz4f git-svn-id: svn://ultimatepp.org/upp/trunk@9957 f0d560ea-af0d-0410-9eb7-867de7ffcac7 --- uppsrc/Core/Hash.h | 6 +- uppsrc/Core/MD5.cpp | 7 +- uppsrc/Core/SHA1.cpp | 4 +- uppsrc/Core/xxHsh.cpp | 5 + uppsrc/plugin/lz4/lz4.h | 17 ++-- uppsrc/plugin/lz4/lz4upp.cpp | 185 +++++++++++++++++++++++++++-------- uppsrc/plugin/lz4/util.cpp | 2 +- 7 files changed, 176 insertions(+), 50 deletions(-) diff --git a/uppsrc/Core/Hash.h b/uppsrc/Core/Hash.h index 52fdc0419..cf136e495 100644 --- a/uppsrc/Core/Hash.h +++ b/uppsrc/Core/Hash.h @@ -17,6 +17,7 @@ public: void Finish(byte *hash16); String FinishString(); String FinishStringS(); + void Reset(); Md5Stream(); ~Md5Stream(); @@ -44,7 +45,8 @@ public: String FinishString(); String FinishStringS(); - void New(); + void Reset(); + void New() { Reset(); } Sha1Stream(); ~Sha1Stream(); @@ -65,6 +67,8 @@ class xxHashStream : public OutStream { public: int Finish(); + void Reset(dword seed = 0); + xxHashStream(dword seed = 0); }; diff --git a/uppsrc/Core/MD5.cpp b/uppsrc/Core/MD5.cpp index a7183fb38..a63892f3f 100644 --- a/uppsrc/Core/MD5.cpp +++ b/uppsrc/Core/MD5.cpp @@ -339,11 +339,16 @@ String Md5Stream::FinishStringS() return HexString(hash, 16, 4); } -Md5Stream::Md5Stream() +void Md5Stream::Reset() { MD5Init (&context); } +Md5Stream::Md5Stream() +{ + Reset(); +} + Md5Stream::~Md5Stream() { memset(&context, 0, sizeof(context)); diff --git a/uppsrc/Core/SHA1.cpp b/uppsrc/Core/SHA1.cpp index 59567596c..b2e2fcbf0 100644 --- a/uppsrc/Core/SHA1.cpp +++ b/uppsrc/Core/SHA1.cpp @@ -158,7 +158,7 @@ String Sha1Stream::FinishStringS() return HexString(hash, 20, 4); } -void Sha1Stream::New() { +void Sha1Stream::Reset() { SHA1Init(state); pos = 0; size = 0; @@ -166,7 +166,7 @@ void Sha1Stream::New() { Sha1Stream::Sha1Stream() { - New(); + Reset(); } Sha1Stream::~Sha1Stream() diff --git a/uppsrc/Core/xxHsh.cpp b/uppsrc/Core/xxHsh.cpp index 9ccc2ec2a..831be4cf7 100644 --- a/uppsrc/Core/xxHsh.cpp +++ b/uppsrc/Core/xxHsh.cpp @@ -7,6 +7,11 @@ namespace Upp { xxHashStream::xxHashStream(dword seed) { STATIC_ASSERT(sizeof(context) >= sizeof(XXH32_state_t)); + Reset(seed); +} + +void xxHashStream::Reset(dword seed) +{ XXH32_reset((XXH32_state_t *)context, seed); } diff --git a/uppsrc/plugin/lz4/lz4.h b/uppsrc/plugin/lz4/lz4.h index 69be3bde0..f3e42631b 100644 --- a/uppsrc/plugin/lz4/lz4.h +++ b/uppsrc/plugin/lz4/lz4.h @@ -8,19 +8,24 @@ namespace Upp { class Lz4 { - enum { BLOCK_BYTES = 64 * 1024 }; - char *block[2]; Buffer buffer; + Buffer outbuf; int8 compress; bool error; - int cbi; + enum { BLOCK_BYTES = 1024 * 1024 }; + + xxHashStream xxh; + int maxblock; int pos; + int blockchksumsz; + bool header; + byte lz4hdr; + String header_data; - LZ4_stream_t lz4Stream[1]; - LZ4_streamDecode_t lz4StreamDecode[1]; - String out; + + void TryHeader(); void Init(); void FlushOut(); diff --git a/uppsrc/plugin/lz4/lz4upp.cpp b/uppsrc/plugin/lz4/lz4upp.cpp index ab65ab893..e6b638fbd 100644 --- a/uppsrc/plugin/lz4/lz4upp.cpp +++ b/uppsrc/plugin/lz4/lz4upp.cpp @@ -4,23 +4,42 @@ namespace Upp { +enum { + LZ4F_MAGIC = 0x184D2204, + + LZ4F_VERSIONMASK = 0b11000000, + LZ4F_VERSION = 0b01000000, + LZ4F_BLOCKINDEPENDENCE = (1 << 5), + LZ4F_BLOCKCHECKSUM = (1 << 4), + LZ4F_CONTENTSIZE = (1 << 3), + LZ4F_CONTENTCHECKSUM = (1 << 2), + + LZ4F_MAXSIZEMASK = 0x70, + LZ4F_MAXSIZE_64KB = 0x40, + LZ4F_MAXSIZE_256KB = 0x50, + LZ4F_MAXSIZE_1024KB = 0x60, + LZ4F_MAXSIZE_4096KB = 0x70, +}; + void Lz4::Init() { error = false; - cbi = pos = 0; - Zero(lz4Stream); - Zero(lz4StreamDecode); + pos = 0; + header = false; + xxh.Reset(); } void Lz4::Compress() { compress = 1; + buffer.Alloc(BLOCK_BYTES); Init(); } void Lz4::Decompress() { compress = 0; + header_data.Clear(); Init(); } @@ -32,31 +51,93 @@ void Lz4::PutOut(const void *ptr, int size) void Lz4::FlushOut() { + if(!header) { + byte h[7]; + Poke32le(h, LZ4F_MAGIC); + h[4] = LZ4F_VERSION | LZ4F_BLOCKINDEPENDENCE | LZ4F_CONTENTCHECKSUM; + h[5] = LZ4F_MAXSIZE_1024KB; + h[6] = xxHash(h + 4, 2) >> 8; + WhenOut(h, 7); + header = true; + maxblock = BLOCK_BYTES; + outbuf.Alloc(4 + LZ4_compressBound(maxblock)); + } + if(!pos) return; - LLOG("To compress " << pos << ' ' << MD5String(block[cbi], pos) << ' ' << cbi); - int count = LZ4_compress_continue(lz4Stream, block[cbi], ~buffer + 4, pos); - if(count < 0) { + + int clen = LZ4_compress(buffer, ~outbuf + 4, pos); + if(clen < 0) { error = true; return; } - Poke32le(~buffer, count); - LLOG("WhenOut " << pos << ": " << count); - WhenOut(~buffer, count + 4); - cbi = !cbi; + xxh.Put(buffer, pos); + if(clen >= pos) { + Poke32le(~outbuf, 0x80000000 | pos); + memcpy(~outbuf + 4, buffer, pos); + WhenOut(~outbuf, pos + 4); + } + else { + Poke32le(~outbuf, clen); + WhenOut(~outbuf, clen + 4); + } pos = 0; } void Lz4::End() { ASSERT(compress >= 0); - if(compress) + if(compress) { FlushOut(); + byte h[8]; + Poke32le(h, 0); + Poke32le(h + 4, xxh.Finish()); + WhenOut(h, 8); + } else if(pos) error = true; } +void Lz4::TryHeader() +{ + if(header_data.GetCount() < 7) + return; + + if(Peek32le(~header_data) != LZ4F_MAGIC) { + error = true; + return; + } + lz4hdr = header_data[4]; + if((lz4hdr & LZ4F_VERSIONMASK) != LZ4F_VERSION) { + error = true; + return; + } + if(!(lz4hdr & LZ4F_BLOCKINDEPENDENCE)) { + error = true; // dependent blocks not supported + return; + } + maxblock = header_data[5]; + maxblock = decode(maxblock & LZ4F_MAXSIZEMASK, + LZ4F_MAXSIZE_64KB, 1024 * 64, + LZ4F_MAXSIZE_256KB, 1024 * 256, + LZ4F_MAXSIZE_1024KB, 1024 * 1024, + LZ4F_MAXSIZE_4096KB, 1024 * 4096, + -1); + if(maxblock < 0) { + error = true; + return; + } + + if((lz4hdr & LZ4F_CONTENTSIZE) && header_data.GetCount() < 15) + return; // need to skip content length + + header = true; + blockchksumsz = 4 * !!(lz4hdr & LZ4F_BLOCKCHECKSUM); + buffer.Alloc(maxblock + 4 + blockchksumsz); + outbuf.Alloc(maxblock); +} + void Lz4::Put(const void *ptr_, int size) { LLOG("Put " << size); @@ -74,21 +155,26 @@ void Lz4::Put(const void *ptr_, int size) return; int n = BLOCK_BYTES - pos; if(size >= n) { - memcpy(block[cbi] + pos, ptr, n); // TODO: Avoid unnecessary memcpy + memcpy(~buffer + pos, ptr, n); pos = BLOCK_BYTES; FlushOut(); size -= n; ptr += n; } else { - memcpy(block[cbi] + pos, ptr, size); + memcpy(buffer + pos, ptr, size); pos += size; break; } } } else { - while(size > 0) { // TODO might try to avoid memcpy + while(!header && size > 0 && !error) { + header_data.Cat(*ptr++); + size--; + TryHeader(); + } + while(size > 0 && !error) { // TODO might try to avoid memcpy if(pos < 4) { // Get length first int n = min(size, 4 - pos); memcpy(~buffer + pos, ptr, n); @@ -96,27 +182,53 @@ void Lz4::Put(const void *ptr_, int size) ptr += n; size -= n; } - if(pos < 4) - return; - int count = Peek32le(~buffer); - int need = count - (pos - 4); - if(size >= need) { // we have enough data to finish the block - memcpy(~buffer + pos, ptr, need); - int out_count = LZ4_decompress_safe_continue(lz4StreamDecode, ~buffer + 4, block[cbi], count, BLOCK_BYTES); - if(out_count <= 0) { - error = true; - return; - } - WhenOut(block[cbi], out_count); - cbi = !cbi; - ptr += need; - size -= need; - pos = 0; - } else { - memcpy(~buffer + pos, ptr, size); - pos += size; - break; + int count = Peek32le(~buffer); + int len = count & 0x7fffffff; + if(len > maxblock) + error = true; + else + if(count == 0) { + int need = 4 - pos + 4; + if(size >= need) { // we have enough data for final checksum + memcpy(~buffer + pos, ptr, need); + if(Peek32le(~buffer + 4) != xxh.Finish()) { + error = true; + return; + } + size -= need; + ptr += need; + pos = 0; + } + } + else { + int need = len - (pos - 4 - blockchksumsz); + if(size >= need) { // we have enough data to finish the block + memcpy(~buffer + pos, ptr, need); + const char *src = ~buffer + 4; + if(count & 0x80000000) { + WhenOut(src, len); + xxh.Put(src, len); + } + else { + int sz = LZ4_decompress_safe(src, outbuf, len, maxblock); + if(sz < 0) { + error = true; + return; + } + WhenOut(outbuf, sz); + xxh.Put(outbuf, sz); + } + ptr += need; + size -= need; + pos = 0; + } + else { + memcpy(~buffer + pos, ptr, size); + pos += size; + break; + } + } } } } @@ -124,10 +236,6 @@ void Lz4::Put(const void *ptr_, int size) Lz4::Lz4() { - block[0] = new char[2 * BLOCK_BYTES]; - block[1] = block[0] + BLOCK_BYTES; - buffer.Alloc(LZ4_COMPRESSBOUND(BLOCK_BYTES) + 4); - compress = -1; WhenOut = callback(this, &Lz4::PutOut); @@ -135,7 +243,6 @@ Lz4::Lz4() Lz4::~Lz4() { - delete[] block[0]; } }; diff --git a/uppsrc/plugin/lz4/util.cpp b/uppsrc/plugin/lz4/util.cpp index 5f6f55091..390a687f7 100644 --- a/uppsrc/plugin/lz4/util.cpp +++ b/uppsrc/plugin/lz4/util.cpp @@ -15,7 +15,7 @@ int64 lz4Press(Stream& out, Stream& in, int64 size, Gate2 progress lz4.Decompress(); if(CopyStream(outs, in, size, progress) >= 0) { outs.Close(); - if(!out.IsError() && !outs.IsError()) + if(!out.IsError() && !outs.IsError() && !lz4.IsError()) r = outs.GetCount(); } }