diff --git a/uppdev/LZF/DEE.cpp b/uppdev/LZF/DEE.cpp index eac2a4e10..5de49ff9b 100644 --- a/uppdev/LZF/DEE.cpp +++ b/uppdev/LZF/DEE.cpp @@ -4,8 +4,8 @@ using namespace Upp; #ifndef COMPRESS2 -#define LLOG(x) -#define LDUMP(x) +#define LLOG(x) // LOG(x) +#define LDUMP(x) // DUMP(x) String FChar(int c) { diff --git a/uppdev/LZF/LZF.upp b/uppdev/LZF/LZF.upp index c3bf93203..b90007b56 100644 --- a/uppdev/LZF/LZF.upp +++ b/uppdev/LZF/LZF.upp @@ -8,6 +8,7 @@ file DEE.cpp, LZF.cpp, LZF3.cpp optimize_speed, + LZF4.cpp optimize_speed, LZP.cpp, main.cpp, info.txt; diff --git a/uppdev/LZF/LZF3.cpp b/uppdev/LZF/LZF3.cpp index aedb49489..985b5eba3 100644 --- a/uppdev/LZF/LZF3.cpp +++ b/uppdev/LZF/LZF3.cpp @@ -1,4 +1,4 @@ -#if 1 // Now broken after optimization +#if 0 #include @@ -8,10 +8,11 @@ using namespace Upp; // Test/Test FoooFooo -#define LLOG(x) LOG(x) +#define LLOG(x) // LOG(x) #define LTIMING(x) // RTIMING(x) +#if 0 force_inline int Hash(const char *s) { // return Peek32le(s) % 8191; @@ -20,12 +21,57 @@ force_inline int Hash(const char *s) // return (2654435761U * Peek32le(s)) & 8191; } -int Literals; -int LiteralsLen; +force_inline int Hash8(const char *s) +{ + return CombineHash(Hash(s), Hash(s + 4)); +} + +force_inline int Hash16(const char *s) +{ + return CombineHash(Hash8(s), Hash8(s + 8)); +} +#else +force_inline int Hash(const char *s) +{ +// return Peek32le(s) % 8191; +// return memhash(s, 4) & 8191; + dword h = Peek32le(s); return (((h >> 23) - (h >> 15) - (h >> 7) - h)); +// return (2654435761U * Peek32le(s)) & 8191; +} + +force_inline int Hash8(const char *s) +{ + dword h = Peek32le(s) - Peek32le(s + 4); + return (((h >> 23) - (h >> 15) - (h >> 7) - h)); +} + +force_inline int Hash16(const char *s) +{ + dword h = Peek32le(s) - Peek32le(s + 4) - Peek32le(s + 8) - Peek32le(s + 12); + return (((h >> 23) - (h >> 15) - (h >> 7) - h)); +} + +force_inline int Hash32(const char *s) +{ + dword h = Peek32le(s) - Peek32le(s + 4) - Peek32le(s + 8) - Peek32le(s + 12) - + Peek32le(s + 16) - Peek32le(s + 20) - Peek32le(s + 24) - Peek32le(s + 28); + return (((h >> 23) - (h >> 15) - (h >> 7) - h)); +} +#endif + +String Literals; +int LiteralsCount; int Matches; +int Match32; +int Match16; +int Match8; +int Match4; int BigMatches; int BigLiterals; int FarOffsets; +int HugeOffsets; +int Extended; +int LargeMatch; force_inline char *OutLen(char *out, int len, int maxn, int& stat) { @@ -45,17 +91,35 @@ force_inline char *OutLen(char *out, int len, int maxn, int& stat) int LZFCompress(char *out, const char *s, int length) { RTIMING("LZFCompress"); + Literals.Clear(); + LiteralsCount = 0; + Matches = 0; + Match16 = 0; + Match8 = 0; + Match4 = 0; + BigMatches = 0; + BigLiterals = 0; + FarOffsets = 0; + HugeOffsets = 0; + Extended = 0; + LargeMatch = 0; + const char *out0 = out; const char *end = s + length; - const char *hash_table[8192]; - const char *hash_table8[8192]; +// const int HASH_MASK = 2048 * 1024 - 1; + const int HASH_MASK = 32768 * 1024 - 1; - for(int i = 0; i < 8192; i++) - hash_table8[i] = hash_table[i] = s; + static const char *hash_table[HASH_MASK + 1]; + static const char *hash_table8[HASH_MASK + 1]; + static const char *hash_table16[HASH_MASK + 1]; + static const char *hash_table32[HASH_MASK + 1]; + + for(int i = 0; i < HASH_MASK + 1; i++) + hash_table32[i] = hash_table16[i] = hash_table8[i] = hash_table[i] = s; const char *literal_ptr = s; - int literal_len = min(length, 8); + int literal_len = min(length, 32); s += literal_len; for(;;) { @@ -64,21 +128,41 @@ int LZFCompress(char *out, const char *s, int length) int match_len; for(;;) { { LTIMING("Lookup"); - word hash4 = Hash(s); - hptr = hash_table8[CombineHash(hash4, Hash(s + 4)) & 8191]; +// word hash4 = Hash(s); +// hptr = hash_table8[CombineHash(hash4, Hash(s + 4)) & 8191]; // LLOG(AsCString(hptr, hptr + 4) << ' ' << AsCString(s, s + 4) << ' ' << hash); + + hptr = hash_table32[Hash32(s) & HASH_MASK]; + if(s - hptr < 0xfffff && Peek64le(hptr) == Peek64le(s) && + Peek64le(hptr + 8) == Peek64le(s + 8) && + Peek64le(hptr + 16) == Peek64le(s + 16) && + Peek64le(hptr + 24) == Peek64le(s + 24)) { + Match32++; + match_len = 32; + break; + } - if(s - hptr < 0xffff && Peek64le(hptr) == Peek64le(s)) { + hptr = hash_table16[Hash16(s) & HASH_MASK]; + if(s - hptr < 0xfffff && Peek64le(hptr) == Peek64le(s) && Peek64le(hptr + 8) == Peek64le(s + 8)) { + Match16++; + match_len = 16; + break; + } + + hptr = hash_table8[Hash8(s) & HASH_MASK]; + if(s - hptr < 0xfffff && Peek64le(hptr) == Peek64le(s)) { + Match8++; match_len = 8; break; } - hptr = hash_table[hash4 & 8181]; - +// hptr = hash_table[hash4 & 8181]; // LLOG(AsCString(hptr, hptr + 4) << ' ' << AsCString(s, s + 4) << ' ' << hash); - if(s - hptr < 0xffff && Peek32le(hptr) == Peek32le(s)) { + hptr = hash_table[Hash(s) & HASH_MASK]; + if(s - hptr < 0xfffff && Peek32le(hptr) == Peek32le(s)) { + Match4++; match_len = 4; break; } @@ -86,67 +170,82 @@ int LZFCompress(char *out, const char *s, int length) LTIMING("Hashing1"); literal_len++; - word hash4 = Hash(s - 3); - hash_table[hash4 & 8191] = s - 3; - hash_table8[CombineHash(Hash(s - 7), hash4) & 8191] = s - 7; + hash_table32[Hash32(s - 31) & HASH_MASK] = s - 31; + hash_table16[Hash16(s - 15) & HASH_MASK] = s - 15; + hash_table8[Hash8(s - 7) & HASH_MASK] = s - 7; + hash_table[Hash(s - 3) & HASH_MASK] = s - 3; + +// hash_table[hash4 & 8191] = s - 3; +// word hash4 = Hash(s - 3); +// hash_table8[CombineHash(Hash(s - 7), hash4) & 8191] = s - 7; s++; - if(s >= end - 8) { + if(s >= end - 32) { *out++ = 0xf0 | max(literal_len, 15); out = OutLen(out, literal_len, 15, BigLiterals); SVO_MEMCPY(out, literal_ptr, literal_len); + // Literals.Cat(literal_ptr, literal_len); return out + literal_len - out0; } } { LTIMING("MatchLen"); - while(hptr + match_len < s && s[match_len] == hptr[match_len]) - match_len++; + while(hptr + match_len < s && s[match_len] == hptr[match_len]) + match_len++; } LLOG("Literals (" << literal_len << "): " << AsCString(String(literal_ptr, literal_len))); LLOG("Match (" << match_len << "): " << AsCString(String(hptr, match_len))); int offset = s - hptr; - - if(literal_len) { - Literals++; - LiteralsLen += literal_len; - LTIMING("literal cat"); - *out++ = 0xf0 | max(literal_len, 15); - out = OutLen(out, literal_len, 15, BigLiterals); - SVO_MEMCPY(out, literal_ptr, literal_len); - out += literal_len; - } - + Matches++; int ml = match_len - 4; - { - LTIMING("Out match"); - word ctrl = (offset >= 0xf00 ? 0xf00 | (byte)offset : offset) - | (min(ml, 14) << 13); - - { - Poke16le(out, ctrl); - out += 2; - out = OutLen(out, ml, 14, BigMatches); + LiteralsCount += literal_len > 0; + + if(literal_len == 0 && offset <= 0xfff && ml < 8) { + *out++ = (ml << 4) & HIBYTE(offset); + *out++ = LOBYTE(offset); + } + else { + *out++ = 0x80 | (max(literal_len, 7) << 4) | max(ml, 15); + offset -= 0xfff; + if(offset < 0xfff0) { + *out++ = HIBYTE(offset); + *out++ = LOBYTE(offset); } - - if(offset >= 0xf00) { - FarOffsets++; + else { + HugeOffsets++; + // FIX THIS + *out++ = LOBYTE(offset); + *out++ = HIBYTE(offset); *out++ = HIBYTE(offset); } + Extended++; + if(offset > 0xfff) + FarOffsets++; + if(ml >= 8) + LargeMatch++; + out = OutLen(out, literal_len, 7, BigLiterals); + out = OutLen(out, ml, 15, BigMatches); + SVO_MEMCPY(out, literal_ptr, literal_len); + out += literal_len; +// Literals.Cat(literal_ptr, literal_len); } - + // literals.Cat(literal_ptr, literal_len); LTIMING("Hashing 2"); const char *e = s + match_len; while(s < e) { - word hash4 = Hash(s - 3); - hash_table[hash4 & 8191] = s - 3; - hash_table8[CombineHash(Hash(s - 7), hash4) & 8191] = s - 7; + hash_table[Hash(s - 3) & HASH_MASK] = s - 3; + hash_table8[Hash8(s - 7) & HASH_MASK] = s - 7; + hash_table16[Hash16(s - 15) & HASH_MASK] = s - 15; + hash_table32[Hash32(s - 31) & HASH_MASK] = s - 31; +// word hash4 = Hash(s - 3); +// hash_table[hash4 & 8191] = s - 3; +// hash_table8[CombineHash(Hash(s - 7), hash4) & 8191] = s - 7; s++; } diff --git a/uppdev/LZF/LZF4.cpp b/uppdev/LZF/LZF4.cpp new file mode 100644 index 000000000..85bad6e2b --- /dev/null +++ b/uppdev/LZF/LZF4.cpp @@ -0,0 +1,280 @@ +#if 1 + +#include + +#include + +using namespace Upp; + +// Test/Test FoooFooo + +#define LLOG(x) // LOG(x) +#define LTIMING(x) // RTIMING(x) + +//#define HASH4 +//#define HASH8 + +const int HASH_MASK = 128 * 1024 - 1; + +#if 0 +force_inline int Hash(const char *s) +{ +// return Peek32le(s) % 8191; +// return memhash(s, 4) & 8191; + dword h = Peek32le(s); return (((h >> 23) - (h >> 15) - (h >> 7) - h)); +// return (2654435761U * Peek32le(s)) & 8191; +} + +force_inline int Hash8(const char *s) +{ + return CombineHash(Hash(s), Hash(s + 4)); +} + +force_inline int Hash16(const char *s) +{ + return CombineHash(Hash8(s), Hash8(s + 8)); +} +#else + +force_inline int FoldHash(int h) +{ +// return 2654435761U * h; + return 31 * HIWORD(h) - LOWORD(h); +// return (((h >> 23) - (h >> 15) - (h >> 7) - h)); +} + +force_inline int Hash(const char *s) +{ +// return Peek32le(s) % 8191; +// return memhash(s, 4) & 8191; +// return (Peek16le(s + 2) * 17) ^ Peek16le(s); + return FoldHash(Peek32le(s)); +// return (2654435761U * Peek32le(s)) & 8191; +} + +force_inline int Hash8(const char *s) +{ + return FoldHash(CombineHash(Peek32le(s), Peek32le(s + 4))); +} + +force_inline int Hash16(const char *s) +{ + dword h = Peek32le(s) - Peek32le(s + 4) - Peek32le(s + 8) - Peek32le(s + 12); + return FoldHash(h); +} + +force_inline int Hash32(const char *s) +{ + dword h = Peek32le(s) - Peek32le(s + 4) - Peek32le(s + 8) - Peek32le(s + 12) - + Peek32le(s + 16) - Peek32le(s + 20) - Peek32le(s + 24) - Peek32le(s + 28); + return FoldHash(h); +} +#endif + +String Literals; +int LiteralsCount; +int Matches; +int Match32; +int Match16; +int Match8; +int Match4; +int BigMatches; +int BigLiterals; +int FarOffsets; +int HugeOffsets; +int Extended; +int LargeMatch; +int Simple; + +force_inline char *OutLen(char *out, int len, int maxn, int& stat) +{ + if(len >= maxn) { + len -= maxn; + while(len >= 255) { + *out++ = 255; + len -= 255; + stat++; + } + *out++ = len; + stat++; + } + return out; +} + +int LZFCompress(char *out, const char *s, int length) +{ + RTIMESTOP("LZFCompress"); + Literals.Clear(); + LiteralsCount = 0; + Matches = 0; + Match16 = 0; + Match8 = 0; + Match4 = 0; + BigMatches = 0; + BigLiterals = 0; + FarOffsets = 0; + HugeOffsets = 0; + Extended = 0; + LargeMatch = 0; + Simple = 0; + + const char *out0 = out; + const char *end = s + length; + +// const int HASH_MASK = 2048 * 1024 - 1; + + static const char *hash_table[HASH_MASK + 1]; + static const char *hash_table8[HASH_MASK + 1]; + static const char *hash_table16[HASH_MASK + 1]; + static const char *hash_table32[HASH_MASK + 1]; + + for(int i = 0; i < HASH_MASK + 1; i++) + hash_table32[i] = hash_table16[i] = hash_table8[i] = hash_table[i] = s; + + RTIMING("Part2"); + const char *literal_ptr = s; + int literal_len = min(length, 32); + s += literal_len; + + for(;;) { + const char *hptr; + + int match_len; + for(;;) { + { LTIMING("Lookup"); +// word hash4 = Hash(s); +// hptr = hash_table8[CombineHash(hash4, Hash(s + 4)) & 8191]; + +// LLOG(AsCString(hptr, hptr + 4) << ' ' << AsCString(s, s + 4) << ' ' << hash); + +#if 0 + hptr = hash_table32[Hash32(s) & HASH_MASK]; + if(0 && s - hptr < 0xfffff && Peek64le(hptr) == Peek64le(s) && + Peek64le(hptr + 8) == Peek64le(s + 8) && + Peek64le(hptr + 16) == Peek64le(s + 16) && + Peek64le(hptr + 24) == Peek64le(s + 24)) { + Match32++; + match_len = 32; + break; + } + + hptr = hash_table16[Hash16(s) & HASH_MASK]; + if(0 && s - hptr < 0xfffff && Peek64le(hptr) == Peek64le(s) && Peek64le(hptr + 8) == Peek64le(s + 8)) { + Match16++; + match_len = 16; + break; + } +#endif + +#ifdef HASH8 + hptr = hash_table8[Hash8(s) & HASH_MASK]; + if(s - hptr < 0xfffff && Peek64le(hptr) == Peek64le(s)) { + Match8++; + match_len = 8; + break; + } +#endif +// hptr = hash_table[hash4 & 8181]; +// LLOG(AsCString(hptr, hptr + 4) << ' ' << AsCString(s, s + 4) << ' ' << hash); +#ifdef HASH4 + hptr = hash_table[Hash(s) & HASH_MASK]; + if(s - hptr < 0xfffff && Peek32le(hptr) == Peek32le(s)) { + Match4++; + match_len = 4; + break; + } +#endif + } + + LTIMING("Hashing1"); + literal_len++; + // hash_table32[Hash32(s - 31) & HASH_MASK] = s - 31; + // hash_table16[Hash16(s - 15) & HASH_MASK] = s - 15; + hash_table[Hash(s - 3) & HASH_MASK] = s - 3; + #ifdef HASH8 + hash_table8[Hash8(s - 7) & HASH_MASK] = s - 7; + #endif + +// hash_table[hash4 & 8191] = s - 3; +// word hash4 = Hash(s - 3); +// hash_table8[CombineHash(Hash(s - 7), hash4) & 8191] = s - 7; + s++; + if(s >= end - 32) { + *out++ = 0xf0 | max(literal_len, 15); + out = OutLen(out, literal_len, 15, BigLiterals); + SVO_MEMCPY(out, literal_ptr, literal_len); + Literals.Cat(literal_ptr, literal_len); + return out + literal_len - out0; + } + } + + { LTIMING("MatchLen"); + while(hptr + match_len < s && s[match_len] == hptr[match_len]) + match_len++; + } + + LLOG("Literals (" << literal_len << "): " << AsCString(String(literal_ptr, literal_len))); + LLOG("Match (" << match_len << "): " << AsCString(String(hptr, match_len))); + + int offset = s - hptr; + + Matches++; + + int ml = match_len - 4; + + LiteralsCount += literal_len > 0; + + if(literal_len == 0 && offset <= 0xfff && ml < 8) { + LTIMING("Simple match"); + *out++ = (ml << 4) & HIBYTE(offset); + *out++ = LOBYTE(offset); + Simple++; + } + else { + LTIMING("Extended match"); + *out++ = 0x80 | (max(literal_len, 7) << 4) | max(ml, 15); + offset -= 0xfff; + if(offset < 0xfff0) { + *out++ = HIBYTE(offset); + *out++ = LOBYTE(offset); + } + else { + HugeOffsets++; + // FIX THIS + *out++ = LOBYTE(offset); + *out++ = HIBYTE(offset); + *out++ = HIBYTE(offset); + } + Extended++; + if(offset > 0xfff) + FarOffsets++; + if(ml >= 8) + LargeMatch++; + out = OutLen(out, literal_len, 7, BigLiterals); + out = OutLen(out, ml, 15, BigMatches); + SVO_MEMCPY(out, literal_ptr, literal_len); + out += literal_len; + Literals.Cat(literal_ptr, literal_len); + } + +// literals.Cat(literal_ptr, literal_len); + + LTIMING("Hashing 2"); + const char *e = s + match_len; + while(s < e) { + dword h1 = Peek32le(s - 3); + dword h2 = Peek32le(s - 7); + hash_table[Hash(s - 3) & HASH_MASK] = s - 3; +#ifdef HASH8 + hash_table8[Hash8(s - 7) & HASH_MASK] = s - 7; +#endif +// hash_table16[Hash16(s - 15) & HASH_MASK] = s - 15; +// hash_table32[Hash32(s - 31) & HASH_MASK] = s - 31; + s++; + } + literal_ptr = s; + literal_len = 0; + } +} + +#endif diff --git a/uppdev/LZF/info.txt b/uppdev/LZF/info.txt index a19bb688f..31765e3ce 100644 --- a/uppdev/LZF/info.txt +++ b/uppdev/LZF/info.txt @@ -48,3 +48,23 @@ FarOffsets = 2545912 BigLiterals + LiteralsLen + 2 * Matches + BigMatches + FarOffsets = 44983957 data.GetLength() = 100000000 ========== +enwik8 2048: +32 39910940 9.39 +16 39975701 +8 40532362 4.98 +4 45704862 + +with 8: +32768 40288716 7.3 +2048 40532362 5.078 +512 41037098 3.531 +256 41618628 2.5 +128 42502290 1.9 +64 43660608 1.766 +32 44616943 1.750 +8 48986896 1.594 +========== +viking.blb 2048 +32 111426059 23.39 +16 122522804 22.96 +8 146699593 22.57 diff --git a/uppdev/LZF/main.cpp b/uppdev/LZF/main.cpp index 60e5dfa2d..6071323cd 100644 --- a/uppdev/LZF/main.cpp +++ b/uppdev/LZF/main.cpp @@ -4,12 +4,20 @@ using namespace Upp; -extern int Literals; -extern int LiteralsLen; +extern String Literals; +extern int LiteralsCount; extern int Matches; extern int BigMatches; extern int BigLiterals; extern int FarOffsets; +extern int HugeOffsets; +extern int Simple; +extern int Extended; +extern int LargeMatch; +extern int Match32; +extern int Match16; +extern int Match8; +extern int Match4; int LZFCompress(char *out, const char *s, int length); @@ -19,28 +27,54 @@ String DEECompress(const char *data, const char *end); CONSOLE_APP_MAIN { // String data = LoadFile("C:/u/aws.data/Viking_Chanarambie_Fenton_With_SurroundingFarms_Backcast_EPEs_14Sep11_v1004_allTI_PCs.blb"); -// String data = LoadFile("C:/u/aws.data/Viking_Chanarambie_Fenton_With_SurroundingFarms_Backcast_EPEs_14Sep11_v1004_allTI_PCs.blb"); - String data = LoadFile("D:/enwik8"); -// String data = LoadFile("C:/u/upp.src/uppsrc/CtrlLib/ArrayCtrl.cpp"); +// String data = LoadFile("C:/u/aws.data/c12_OCOE_env.blb"); +// String data = LoadFile("D:/enwik8"); +// String data = LoadFile("D:/canterbury_last.tar"); + String data = LoadFile("C:/u/upp.src/uppsrc/CtrlLib/ArrayCtrl.cpp"); // String data = LoadFile(GetDataFile("LZF.cpp")); +// String data = LoadFile("C:/u/client/nos74.exe"); +// String data = LoadFile("C:/upp/theide.exe"); + String l; - Buffer out(11 * data.GetLength() / 10); - RDUMP(LZFCompress(out, ~data, data.GetLength())); + Buffer out(2 * data.GetLength()); + + { + RTIMESTOP("LZFCompress"); + int n = LZFCompress(out, ~data, data.GetLength()); + RDUMP(n); + RDUMP(data.GetLength()); + } // RDUMP(cdata.GetCount()); - RDUMP(Literals); - RDUMP(LiteralsLen); + RDUMP(Match32); + RDUMP(Match16); + RDUMP(Match8); + RDUMP(Match4); RDUMP(Matches); + RDUMP(Extended); + RDUMP(FarOffsets); + RDUMP(HugeOffsets); + RDUMP(LiteralsCount); + RDUMP(LargeMatch); RDUMP(BigMatches); RDUMP(BigLiterals); - RDUMP(FarOffsets); + RDUMP(Literals.GetCount()); - RDUMP(/*Literals + */BigLiterals + LiteralsLen + 2 * Matches + BigMatches + FarOffsets); +// RDUMP(Literals + BigLiterals + LiteralsLen + 2 * Matches + Extended); // RDUMP(AsCString(l, 100)); // RDUMP(l.GetCount()); -// RDUMP(DEECompress(l, l.End()).GetLength()); + int dee = DEECompress(Literals, Literals.End()).GetLength(); + + SaveFile("d:/literals", Literals); + + if(Literals.GetCount() < 10000000) + RDUMP(ZCompress(Literals).GetCount()); + + RDUMP(dee); + RDUMP(Literals.GetCount() - dee); + RDUMP(Literals.GetCount()); RDUMP(data.GetLength()); RLOG("=========="); @@ -49,6 +83,7 @@ CONSOLE_APP_MAIN RDUMP(LZ4Compress(data).GetLength()); #ifdef flagLZ4STATS RDUMP(lz4stat_LiteralLen); + RDUMP(lz4stat_Literals); RDUMP(lz4stat_Matches); RDUMP(lz4stat_BigMatch); RDUMP(lz4stat_BigLiteral); @@ -61,4 +96,11 @@ CONSOLE_APP_MAIN } if(data.GetLength() < 10000000) RDUMP(ZCompress(data).GetCount()); + + { + RTIMESTOP("LZ4 direct"); + Buffer out(2 * data.GetLength()); + RDUMP(LZ4_compress(data, out, data.GetLength())); + } + }