diff --git a/uppdev/LZF/LZF.cpp b/uppdev/LZF/LZF.cpp new file mode 100644 index 000000000..09aa77885 --- /dev/null +++ b/uppdev/LZF/LZF.cpp @@ -0,0 +1,74 @@ +#include + +using namespace Upp; + +// Test/Test FoooFooo + +int Hash(const char *s) +{ +// return Peek32le(s) % 8191; +// return memhash(s, 4) & 8191; + dword h = Peek32le(s); return (((h >> 23) - (h >> 15) - (h >> 7) - h)) & 8191; +// return (2654435761U * Peek32le(s)) & 8191; +} + +void LZFCompress(const char *s, int length, String& literals) +{ + const char *end = s + length; + + const char *hash_table[8192]; + + for(int i = 0; i < 8192; i++) + hash_table[i] = s; + + const char *lit = s; + int litlen = min(length, 4); + s += 4; + + for(;;) { + const char *hptr; + for(;;) { + int hash = Hash(s); + + hptr = hash_table[hash]; + + // LOG(AsCString(hptr, hptr + 4) << ' ' << AsCString(s, s + 4) << ' ' << hash); + + if(s - hptr < 65536 && Peek32le(hash_table[hash]) == Peek32le(s)) + break; + litlen++; + hash_table[Hash(s - 3)] = s - 3; + s++; + if(s >= end) + return; + } + + int matchlen = 4; + while(hptr + matchlen < s && s[matchlen] == hptr[matchlen]) + matchlen++; + + LOG("Literals (" << litlen << "): " << AsCString(String(lit, litlen))); + LOG("Match (" << matchlen << "): " << AsCString(String(hptr, matchlen))); + + literals.Cat(String(lit, litlen)); + + const char *e = s + matchlen; + while(s < e) { + hash_table[Hash(s - 3)] = s - 3; + s++; + } + + lit = s; + litlen = 0; + } +} + +CONSOLE_APP_MAIN +{ + String data = LoadFile(GetDataFile("LZF.cpp")); + String l; + LZFCompress(~data, data.GetLength(), l); + DDUMP(AsCString(l, 100)); + DDUMP(l.GetCount()); + DDUMP(data.GetLength()); +} diff --git a/uppdev/LZF/LZF.upp b/uppdev/LZF/LZF.upp new file mode 100644 index 000000000..191304e8a --- /dev/null +++ b/uppdev/LZF/LZF.upp @@ -0,0 +1,9 @@ +uses + Core; + +file + LZF.cpp; + +mainconfig + "" = "SSE2"; + diff --git a/uppdev/LZF/init b/uppdev/LZF/init new file mode 100644 index 000000000..6d0efe542 --- /dev/null +++ b/uppdev/LZF/init @@ -0,0 +1,4 @@ +#ifndef _LZF_icpp_init_stub +#define _LZF_icpp_init_stub +#include "Core/init" +#endif diff --git a/uppdev/NibblePtr/Compress.cpp b/uppdev/NibblePtr/Compress.cpp index 18eb6271e..e83ac5cb7 100644 --- a/uppdev/NibblePtr/Compress.cpp +++ b/uppdev/NibblePtr/Compress.cpp @@ -1,5 +1,8 @@ #include "Entropy.h" +#ifndef COMPRESS2 + + struct Recode { byte m; byte h1, h2, h3; @@ -11,7 +14,7 @@ String Compress(const char *data, const char *end) { int stat[256] = { 0 }; { - RTIMING("Stat"); + RTIMESTOP("Stat"); for(const char *s = data; s < end; s++) stat[(byte)*s]++; } @@ -33,6 +36,24 @@ String Compress(const char *data, const char *end) IndexSort(stat, stat + 256, ndx, StdGreater()); + for(int m = 1; m < 15; m++) { + int mini = 0; + int big = 0; + for(int i = 0; i < 256; i++) { + if(i < m) + mini += stat[i]; + if(i >= (15 - m) * 16) + big += stat[i]; +// LOG(FChar(ndx[i]) << " " << stat[i]); + } + + LOG("==== " << m << ":" << (15 - m) * 16 << " ==="); + DUMP(mini); + DUMP(big); + DUMP((mini - big) / 2); + LOG("--------------------"); + } + int mini = 0; int big = 0; for(int i = 0; i < 256; i++) { @@ -93,15 +114,13 @@ String Compress(const char *data, const char *end) } NibblePtr p; - + p.Set(t); { - RTIMING("Compress"); - int n = 0; - bool nibbler = false; - byte g = 0; + RTIMESTOP("Compress"); const char *s = data; + byte g = 0; for(;;) { nibble0: { @@ -140,3 +159,5 @@ String Compress(const char *data, const char *end) } return String(~output, t - ~output + 1); } + +#endif \ No newline at end of file diff --git a/uppdev/NibblePtr/Compress2.cpp b/uppdev/NibblePtr/Compress2.cpp new file mode 100644 index 000000000..001d5680c --- /dev/null +++ b/uppdev/NibblePtr/Compress2.cpp @@ -0,0 +1,144 @@ +#include "Entropy.h" + +#ifdef COMPRESS2 + +struct Recode { + dword data[2]; + byte len; + byte filler[7]; +}; + +String Compress(const char *data, const char *end) +{ + DUMP(sizeof(Recode)); + ASSERT(sizeof(Recode) == 16); + + int stat[256] = { 0 }; + { + RTIMESTOP("Stat"); + for(const char *s = data; s < end; s++) + stat[(byte)*s]++; + } + + + for(int i = 0; i < 256; i++) { + LOG(FChar(i) << " " << stat[i]); + } + + LOG("======================"); + + // 0000 - 1011 most frequent 12 + // 1100 0000 - 1110 1111 average 48 + // 1111 0000 0000 - 1111 1111 1111 rest + + byte ndx[256]; + for(int i = 0; i < 256; i++) + ndx[i] = i; + + IndexSort(stat, stat + 256, ndx, StdGreater()); + + for(int m = 1; m < 15; m++) { + int mini = 0; + int big = 0; + for(int i = 0; i < 256; i++) { + if(i < m) + mini += stat[i]; + if(i >= (15 - m) * 16) + big += stat[i]; +// LOG(FChar(ndx[i]) << " " << stat[i]); + } + + LOG("==== " << m << ":" << (15 - m) * 16 << " ==="); + DUMP(mini); + DUMP(big); + DUMP((mini - big) / 2); + LOG("--------------------"); + } + + int mini = 0; + int big = 0; + for(int i = 0; i < 256; i++) { + if(i < 12) + mini += stat[i]; + if(i >= 12 + 48) + big += stat[i]; + LOG(FChar(ndx[i]) << " " << stat[i]); + } + + DUMP(mini); + DUMP(big); + DUMP((mini - big) / 2); + LOG("--------------------"); + + Recode recode[256]; + + Buffer output(3 * (end - data) / 2 + 10000); + byte *t = ~output; + *t++ = 1; + Poke32le(t, (end - data)); + t += 4; + for(int i = 0; i < 256; i++) { + int c = ndx[i]; + LOG(i << ": " << FChar(c)); + Recode& r = recode[c]; + if(i < 12 + 48) { + *t++ = c; + if(i < 12) { + r.len = 1; + r.data[0] = i; + } + else { + r.len = 2; + r.data[0] = (i / 16 + 12) | (i << 4); + } + } + else { + r.len = 3; + r.data[0] = 0xf | (i << 4); + } + r.data[1] = r.data[0] << 4; + } + + NibblePtr p; + + p.Set(t); + + int pos = 0; + { + RTIMESTOP("Compress"); + const char *s = data; + while(s < end - 4) { + Recode *r; + byte *t0; + + r = &recode[(byte)*s++]; + t0 = t + (pos >> 1); + Poke32le(t0, Peek32le(t0) | r->data[pos & 1]); + pos += r->len; + + r = &recode[(byte)*s++]; + t0 = t + (pos >> 1); + Poke32le(t0, Peek32le(t0) | r->data[pos & 1]); + pos += r->len; + + r = &recode[(byte)*s++]; + t0 = t + (pos >> 1); + Poke32le(t0, Peek32le(t0) | r->data[pos & 1]); + pos += r->len; + + r = &recode[(byte)*s++]; + t0 = t + (pos >> 1); + Poke32le(t0, Peek32le(t0) | r->data[pos & 1]); + pos += r->len; + } + while(s < end) { + Recode& r = recode[(byte)*s++]; + byte *t0 = t + (pos >> 1); + Poke32le(t0, Peek32le(t0) | r.data[pos & 1]); + pos += r.len; + } + } + return String(~output, (pos + 1) / 2); +} + +#endif diff --git a/uppdev/NibblePtr/Entropy.h b/uppdev/NibblePtr/Entropy.h index dc2891192..ed78869b3 100644 --- a/uppdev/NibblePtr/Entropy.h +++ b/uppdev/NibblePtr/Entropy.h @@ -34,6 +34,14 @@ struct NibblePtr { String FChar(int c); +#define COMPRESS2 + String Compress(const char *data, const char *end); + +int LZVCompress(byte * in, byte * out, byte * heap, int len, int out_len); +int LZVDecompress (byte const *const in, byte * const out, int ilen, int len); + +String LZFCompress(const char *s, int length); + #endif diff --git a/uppdev/NibblePtr/NibblePtr.cpp b/uppdev/NibblePtr/NibblePtr.cpp index 96666d4aa..6a097c8d1 100644 --- a/uppdev/NibblePtr/NibblePtr.cpp +++ b/uppdev/NibblePtr/NibblePtr.cpp @@ -4,21 +4,65 @@ CONSOLE_APP_MAIN { - String data = LoadFile("D:/enwik8"); +// String data = LoadFile("C:/u/upp.src/uppsrc/CtrlLib/ArrayCtrl.cpp"); + String data = LoadFile("C:/xxx/enwik8.txt"); // String data = LoadFile("D:/20131117_ST_ZZSZ.xml"); -// String data = LoadFile("C:/xxx/Nos.exe"); +// String data = LoadFile("C:/u/client/Nos.exe"); +// String data = LoadFile("C:/u/Nos.version/90/Nos.exe"); // String data = LoadFile("C:/u/aws.data/Viking_Chanarambie_Fenton_With_SurroundingFarms_Backcast_EPEs_14Sep11_v1004_allTI_PCs.blb"); - + + String cdata = Compress(data.Begin(), data.End()); // String ddata = Decompress(cdata, cdata.GetCount()); RDUMP(cdata.GetCount()); - RLOG("Ratio " << cdata.GetCount() * 100.0 / data.GetCount() << "%"); + RLOG("DEE Ratio " << cdata.GetCount() * 100.0 / data.GetCount() << "%"); + { - RTIMING("LZ4"); - cdata = LZ4Compress(data); - RLOG("Ratio " << cdata.GetCount() * 100.0 / data.GetCount() << "%"); + RLOG("====================="); + String cdata; + { + RTIMESTOP("LZ4"); + cdata = LZ4Compress(data); + } + RLOG("LZ4 Ratio " << cdata.GetCount() * 100.0 / data.GetCount() << "%"); + LOG("-- OFFSETS -------"); + + int sm = 0; + for(int i = 0; i < 256; i++) + if(matchOffsetStat[i]) { + LOG(i * 256 << " - " << i * 256 + 255 << " " << matchOffsetStat[i] << " / " << sm); + sm += matchOffsetStat[i]; + } + + LOG("-- LENGTHS -------"); + sm = 0; + for(int i = 0; i < 1024; i++) + if(matchLengthStat[i]) { + LOG(i << " " << matchLengthStat[i] << " / " << sm); + sm += matchLengthStat[i]; + } + DUMP(matchLengthBig); + + LOG("-- LITERAL LENGTHS -------"); + sm = 0; + for(int i = 0; i < 1024; i++) + if(litLengthStat[i]) { + LOG(i << " " << litLengthStat[i] << " / " << sm); + sm += litLengthStat[i]; + } + DUMP(litLengthStatBig); + DUMP(literals); + } + + if(1) { + RLOG("====================="); + RTIMESTOP("LZV"); + Buffer out(2 * data.GetLength()); + byte h[16384 * 4]; + int q = LZVCompress((byte *)~data, out, h, data.GetLength(), 2 * data.GetLength()); + RLOG("LZV Ratio " << q * 100.0 / data.GetCount() << "%"); } } diff --git a/uppdev/NibblePtr/NibblePtr.upp b/uppdev/NibblePtr/NibblePtr.upp index 82c722865..60828cba7 100644 --- a/uppdev/NibblePtr/NibblePtr.upp +++ b/uppdev/NibblePtr/NibblePtr.upp @@ -4,12 +4,15 @@ uses file Entropy.h, + lzf.cpp, + lzv.cpp, Util.cpp, Decompress.cpp, help.txt, Compress.cpp optimize_speed, + Compress2.cpp optimize_speed, NibblePtr.cpp optimize_speed; mainconfig - "" = "SSE2"; + "" = "SSE2 LZ4STATS"; diff --git a/uppdev/NibblePtr/lzf.cpp b/uppdev/NibblePtr/lzf.cpp new file mode 100644 index 000000000..63ffae7be --- /dev/null +++ b/uppdev/NibblePtr/lzf.cpp @@ -0,0 +1,52 @@ +#include "Entropy.h" + +// Test/Test FooFoo + +String LZFCompress(const char *s, int length) +{ + String result; + + const char *end = s + length; + + const char *hash_table[8192]; + + for(int i = 0; i < 8192; i++) + hash_table[i] = s; + + char *lit = s; + int litlen = min(length, 4); + s += 4; + + for(;;) { + char *hptr; + for(;;) { + hash = (2654435761U * Peek32le(s)) & 8191; + + hptr = hash_table[hash]; + + if(s - hptr < 65536 && Peek32le(hash_table[hash]) == Peek32le(s)) + break; + litlen++; + hash_table[(2654435761U * Peek32le(s - 3)) & 8191] = s - 3; + s++; + if(s >= end) + return; + } + + int matchlen = 4; + while(hptr + matchlen < s && s[matchlen] == hptr[matchlen]) + matchlen++; + + LOGHEXDUMP(lit, litlen); + LOGHEXDUMP(hptr, matchlen); + + const char *e = s + matchlen; + while(s < e) { + hash_table[(2654435761U * Peek32le(s - 3)) & 8191] = s - 3; + s++; + } + + lit = s; + + } +} diff --git a/uppdev/NibblePtr/lzv.cpp b/uppdev/NibblePtr/lzv.cpp new file mode 100644 index 000000000..796446100 --- /dev/null +++ b/uppdev/NibblePtr/lzv.cpp @@ -0,0 +1,242 @@ +#include "Entropy.h" + + +/*********************************************************************** +** +** lzv.c -- an extremly fast compression/decompression-method +** +** Written by Hermann Vogt +** +** v 0.5 -- 00/04/10 fix unaligned access (Marc) +** v 0.4 -- 00/03/25 adapted for PApp by Marc Lehmann +** v 0.3 -- 94/03/08 aCembler version of rLZV built in. +** v 0.2 -- 94/03/04 Changes for usage with DouBle 0.2 built in. +** v 0.1 -- 94/03/01 Intensivly tested, removed all known bugs. +** v 0.0 -- 94/02/21 First Version. +** +** Copyright (c) 1994 Hermann Vogt. Redistribution of this file is +** permitted under the GNU Public Licence. +** +** The method presented here is faster and compresses better +** than lzrw1 and lzrw1-a. I named it lzv for "Lev-Zimpel-Vogt". +** It uses ideas introduced by Ross Williams in his algorithm lzrw1 +** [R. N. Williams (1991): "An Extremly Fast ZIV-Lempel Data +** Compression Algorithm", Proceedings IEEE Data Compression +** Conference, Snowbird, Utah, 362-371] and by Fiala and Green in their +** algorithm a1 [E. R. Fiala, D. H. Greene (1989): "Data Compression +** with Finite Windows", Communications of the ACM, 4, 490-505]. +** Because lzv differs strongly from both, I hope there will be no +** patent problems. The hashing-method has been stolen from Jean-loup +** Gailly's (patent free) gzip. +** +** KNOWN PROBLEMS: +** - My english is very bad. +** - Badly commented. (I hope this will be better in the next +** version.) +** - I'm not sure if lzv is free from patent problems. +** +***********************************************************************/ + +#define HSIZE 0x4000 +#define HMASK 0x3fff +#define HSHIFT 5 + +#define MLL 32 /* Maximum len of chain of literals */ +#define MML (8+256) /* Maximum len of match */ +#define MOFF 8191 /* Maximum offset */ +#define HSIZ 16384 /* Size of Hashtable */ + +/* ugly type names */ + +typedef byte uch; +typedef word ush; +typedef dword uit; + +#undef ONLY_64K /* 64k-max encoder is faster */ + /* but only veeeery slightly */ + +/* unconditionally aligning does not cost much much, so do it if unsure */ +#define align_ushort !defined(__i386) + +int LZVCompress(byte * in, byte * out, byte * heap, int len, int out_len) +{ + uit hval, op, ip, l_len, m_pos, m_off, m_len, maxlen; + ush *lzv1_htab = (word *)heap; + + maxlen = out_len; + hval = ((in[0] << 5) ^ in[1]) & (HSIZ - 1); + ip = op = l_len = 0; + + do + { + hval = ((hval << 5) ^ in[ip + 2]) & (HSIZ - 1); + m_pos = lzv1_htab[hval]; + lzv1_htab[hval] = ip; + +#ifndef ONLY_64K + /* + * If you want to compress more than 64K, uncomment + * the following lines. + */ + + m_pos = (ip & ~0xffff) + m_pos; + if (m_pos >= ip && m_pos >= 0x10000) + m_pos -= 0x10000; +#endif + + if (m_pos < ip + && in[m_pos ] == in[ip ] + && (m_off = ip - m_pos - 1) <= MOFF + && ip + 4 < len +#if align_ushort + && in[m_pos + 1] == in[ip + 1] + && in[m_pos + 2] == in[ip + 2] +#else + && *(ush *) (in + m_pos + 1) == *(ush *) (in + ip + 1) +#endif + ) + { + /* We have found a match */ + uit look = len - ip - 2; + if (look > MML) + look = MML; + m_len = 2; + + do + { + m_len++; + } + while (m_len != look && in[ip + m_len] == in[m_pos + m_len]); + + if (op + 2 + l_len + 3 >= maxlen) + return 0; + + if (l_len != 0) + { + out[op++] = (l_len - 1) << 3; + do + { + out[op++] = in[ip - l_len--]; + } + while (l_len != 0); + } + + m_len -= 2; + + if (m_len <= 6) + { + out[op++] = m_len | ((m_off >> 5) & 0xf8); + } + else + { + out[op++] = 0x07 | ((m_off >> 5) & 0xf8); + out[op++] = m_len - 7; + } + + out[op++] = m_off & 0xff; + ip++; + hval = ((hval << 5) ^ in[ip + 2]) & (HSIZ - 1); + lzv1_htab[hval] = ip; + ip++; + + do + { + hval = ((hval << 5) ^ in[ip + 2]) & (HSIZ - 1); + lzv1_htab[hval] = ip; + ip++; + m_len--; + } + while (0 != m_len); + } + else + { + /* No match found */ + ip++; + l_len++; + + if (MLL == l_len) + { + if (op + 2 + MLL >= maxlen) + return 0; + + out[op++] = 0xf8; + + do + { + out[op++] = in[ip - l_len--]; + } + while (l_len != 0); + } + } + } + while (ip < len); + + if (l_len != 0) + { + if (op + l_len + 3 >= maxlen) + return 0; + + out[op++] = (l_len - 1) << 3; + + do + { + out[op++] = in[ip - l_len--]; + } + while (l_len != 0); + } + return op; +} + +int LZVDecompress (uch const *const in, uch * const out, int ilen, int len) +{ + register uit tbuf, c_len; + uch *const out_end = out + len; + register uch *op = out; + uch const *const in_end = in + ilen; + register uch const *ip = in; + + do + { + tbuf = *ip++; + c_len = tbuf & 0x07; + + if (0 == c_len) + { + c_len = (tbuf >> 3) + 1; + + /*if (op + c_len > out_end) /* too many checks... */ + /* return 0;*/ + + do + *op++ = *ip++; + while (--c_len); /* effic: memcpy()? */ + } + else + { + register uch *m_pos; + + if (0x07 == c_len) + c_len = *ip++ + 7; + m_pos = op - 1 - (((uit) (tbuf & 0xf8) << 5) | *ip++); + + /* If we don't check this then we segfault (if in user + space) or leave process in uninteruptible state (if + in kernel) if the data is corrupt. */ + if (m_pos < out) + return 0; /* Compression error. */ + + /*if (op + c_len + 2 > out_end) /* too many checks */ + /* return 0;*/ + + *op++ = *m_pos++; + *op++ = *m_pos++; + + do + *op++ = *m_pos++; + while (--c_len); + } + } + while (op < out_end && ip < in_end); + + return op - out; +}