github-starred/ultimatepp
2
0
Fork 0
mirror of https://github.com/ultimatepp/ultimatepp.git synced 2026-07-11 22:03:01 -06:00
Code Issues 33 Projects Packages Wiki Activity Actions

.dev

Browse source 
git-svn-id: svn://ultimatepp.org/upp/trunk@7865 f0d560ea-af0d-0410-9eb7-867de7ffcac7
This commit is contained in:
cxl 2014-11-05 19:27:13 +00:00
parent e255977a84
commit 51412f842a
6 changed files with 504 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

4
uppdev/LZF/DEE.cpp
View file

@ -4,8 +4,8 @@ using namespace Upp;
#ifndef COMPRESS2
#define LLOG(x)
#define LDUMP(x)
#define LLOG(x) // LOG(x)
#define LDUMP(x) // DUMP(x)
String FChar(int c)
{

1
uppdev/LZF/LZF.upp
View file

@ -8,6 +8,7 @@ file
DEE.cpp,
LZF.cpp,
LZF3.cpp optimize_speed,
LZF4.cpp optimize_speed,
LZP.cpp,
main.cpp,
info.txt;

195
uppdev/LZF/LZF3.cpp
View file

@ -1,4 +1,4 @@
#if 1 // Now broken after optimization
#if 0
#include <Core/Core.h>
@ -8,10 +8,11 @@ using namespace Upp;
// Test/Test FoooFooo
#define LLOG(x) LOG(x)
#define LLOG(x) // LOG(x)
#define LTIMING(x) // RTIMING(x)
#if 0
force_inline int Hash(const char *s)
{
// return Peek32le(s) % 8191;
@ -20,12 +21,57 @@ force_inline int Hash(const char *s)
// return (2654435761U * Peek32le(s)) & 8191;
}
int Literals;
int LiteralsLen;
force_inline int Hash8(const char *s)
{
return CombineHash(Hash(s), Hash(s + 4));
}
force_inline int Hash16(const char *s)
{
return CombineHash(Hash8(s), Hash8(s + 8));
}
#else
force_inline int Hash(const char *s)
{
// return Peek32le(s) % 8191;
// return memhash(s, 4) & 8191;
dword h = Peek32le(s); return (((h >> 23) - (h >> 15) - (h >> 7) - h));
// return (2654435761U * Peek32le(s)) & 8191;
}
force_inline int Hash8(const char *s)
{
dword h = Peek32le(s) - Peek32le(s + 4);
return (((h >> 23) - (h >> 15) - (h >> 7) - h));
}
force_inline int Hash16(const char *s)
{
dword h = Peek32le(s) - Peek32le(s + 4) - Peek32le(s + 8) - Peek32le(s + 12);
return (((h >> 23) - (h >> 15) - (h >> 7) - h));
}
force_inline int Hash32(const char *s)
{
dword h = Peek32le(s) - Peek32le(s + 4) - Peek32le(s + 8) - Peek32le(s + 12) -
Peek32le(s + 16) - Peek32le(s + 20) - Peek32le(s + 24) - Peek32le(s + 28);
return (((h >> 23) - (h >> 15) - (h >> 7) - h));
}
#endif
String Literals;
int LiteralsCount;
int Matches;
int Match32;
int Match16;
int Match8;
int Match4;
int BigMatches;
int BigLiterals;
int FarOffsets;
int HugeOffsets;
int Extended;
int LargeMatch;
force_inline char *OutLen(char *out, int len, int maxn, int& stat)
{
@ -45,17 +91,35 @@ force_inline char *OutLen(char *out, int len, int maxn, int& stat)
int LZFCompress(char *out, const char *s, int length)
{
RTIMING("LZFCompress");
Literals.Clear();
LiteralsCount = 0;
Matches = 0;
Match16 = 0;
Match8 = 0;
Match4 = 0;
BigMatches = 0;
BigLiterals = 0;
FarOffsets = 0;
HugeOffsets = 0;
Extended = 0;
LargeMatch = 0;
const char *out0 = out;
const char *end = s + length;
const char *hash_table[8192];
const char *hash_table8[8192];
// const int HASH_MASK = 2048 * 1024 - 1;
const int HASH_MASK = 32768 * 1024 - 1;
for(int i = 0; i < 8192; i++)
hash_table8[i] = hash_table[i] = s;
static const char *hash_table[HASH_MASK + 1];
static const char *hash_table8[HASH_MASK + 1];
static const char *hash_table16[HASH_MASK + 1];
static const char *hash_table32[HASH_MASK + 1];
for(int i = 0; i < HASH_MASK + 1; i++)
hash_table32[i] = hash_table16[i] = hash_table8[i] = hash_table[i] = s;
const char *literal_ptr = s;
int literal_len = min(length, 8);
int literal_len = min(length, 32);
s += literal_len;
for(;;) {
@ -64,21 +128,41 @@ int LZFCompress(char *out, const char *s, int length)
int match_len;
for(;;) {
{ LTIMING("Lookup");
word hash4 = Hash(s);
hptr = hash_table8[CombineHash(hash4, Hash(s + 4)) & 8191];
// word hash4 = Hash(s);
// hptr = hash_table8[CombineHash(hash4, Hash(s + 4)) & 8191];
// LLOG(AsCString(hptr, hptr + 4) << ' ' << AsCString(s, s + 4) << ' ' << hash);
hptr = hash_table32[Hash32(s) & HASH_MASK];
if(s - hptr < 0xfffff && Peek64le(hptr) == Peek64le(s) &&
Peek64le(hptr + 8) == Peek64le(s + 8) &&
Peek64le(hptr + 16) == Peek64le(s + 16) &&
Peek64le(hptr + 24) == Peek64le(s + 24)) {
Match32++;
match_len = 32;
break;
}
if(s - hptr < 0xffff && Peek64le(hptr) == Peek64le(s)) {
hptr = hash_table16[Hash16(s) & HASH_MASK];
if(s - hptr < 0xfffff && Peek64le(hptr) == Peek64le(s) && Peek64le(hptr + 8) == Peek64le(s + 8)) {
Match16++;
match_len = 16;
break;
}
hptr = hash_table8[Hash8(s) & HASH_MASK];
if(s - hptr < 0xfffff && Peek64le(hptr) == Peek64le(s)) {
Match8++;
match_len = 8;
break;
}
hptr = hash_table[hash4 & 8181];
// hptr = hash_table[hash4 & 8181];
// LLOG(AsCString(hptr, hptr + 4) << ' ' << AsCString(s, s + 4) << ' ' << hash);
if(s - hptr < 0xffff && Peek32le(hptr) == Peek32le(s)) {
hptr = hash_table[Hash(s) & HASH_MASK];
if(s - hptr < 0xfffff && Peek32le(hptr) == Peek32le(s)) {
Match4++;
match_len = 4;
break;
}
@ -86,67 +170,82 @@ int LZFCompress(char *out, const char *s, int length)
LTIMING("Hashing1");
literal_len++;
word hash4 = Hash(s - 3);
hash_table[hash4 & 8191] = s - 3;
hash_table8[CombineHash(Hash(s - 7), hash4) & 8191] = s - 7;
hash_table32[Hash32(s - 31) & HASH_MASK] = s - 31;
hash_table16[Hash16(s - 15) & HASH_MASK] = s - 15;
hash_table8[Hash8(s - 7) & HASH_MASK] = s - 7;
hash_table[Hash(s - 3) & HASH_MASK] = s - 3;
// hash_table[hash4 & 8191] = s - 3;
// word hash4 = Hash(s - 3);
// hash_table8[CombineHash(Hash(s - 7), hash4) & 8191] = s - 7;
s++;
if(s >= end - 8) {
if(s >= end - 32) {
*out++ = 0xf0 | max(literal_len, 15);
out = OutLen(out, literal_len, 15, BigLiterals);
SVO_MEMCPY(out, literal_ptr, literal_len);
// Literals.Cat(literal_ptr, literal_len);
return out + literal_len - out0;
}
}
{ LTIMING("MatchLen");
while(hptr + match_len < s && s[match_len] == hptr[match_len])
match_len++;
while(hptr + match_len < s && s[match_len] == hptr[match_len])
match_len++;
}
LLOG("Literals (" << literal_len << "): " << AsCString(String(literal_ptr, literal_len)));
LLOG("Match (" << match_len << "): " << AsCString(String(hptr, match_len)));
int offset = s - hptr;
if(literal_len) {
Literals++;
LiteralsLen += literal_len;
LTIMING("literal cat");
*out++ = 0xf0 | max(literal_len, 15);
out = OutLen(out, literal_len, 15, BigLiterals);
SVO_MEMCPY(out, literal_ptr, literal_len);
out += literal_len;
}
Matches++;
int ml = match_len - 4;
{
LTIMING("Out match");
word ctrl = (offset >= 0xf00 ? 0xf00 | (byte)offset : offset)
| (min(ml, 14) << 13);
{
Poke16le(out, ctrl);
out += 2;
out = OutLen(out, ml, 14, BigMatches);
LiteralsCount += literal_len > 0;
if(literal_len == 0 && offset <= 0xfff && ml < 8) {
*out++ = (ml << 4) & HIBYTE(offset);
*out++ = LOBYTE(offset);
}
else {
*out++ = 0x80 | (max(literal_len, 7) << 4) | max(ml, 15);
offset -= 0xfff;
if(offset < 0xfff0) {
*out++ = HIBYTE(offset);
*out++ = LOBYTE(offset);
}
if(offset >= 0xf00) {
FarOffsets++;
else {
HugeOffsets++;
// FIX THIS
*out++ = LOBYTE(offset);
*out++ = HIBYTE(offset);
*out++ = HIBYTE(offset);
}
Extended++;
if(offset > 0xfff)
FarOffsets++;
if(ml >= 8)
LargeMatch++;
out = OutLen(out, literal_len, 7, BigLiterals);
out = OutLen(out, ml, 15, BigMatches);
SVO_MEMCPY(out, literal_ptr, literal_len);
out += literal_len;
// Literals.Cat(literal_ptr, literal_len);
}
// literals.Cat(literal_ptr, literal_len);
LTIMING("Hashing 2");
const char *e = s + match_len;
while(s < e) {
word hash4 = Hash(s - 3);
hash_table[hash4 & 8191] = s - 3;
hash_table8[CombineHash(Hash(s - 7), hash4) & 8191] = s - 7;
hash_table[Hash(s - 3) & HASH_MASK] = s - 3;
hash_table8[Hash8(s - 7) & HASH_MASK] = s - 7;
hash_table16[Hash16(s - 15) & HASH_MASK] = s - 15;
hash_table32[Hash32(s - 31) & HASH_MASK] = s - 31;
// word hash4 = Hash(s - 3);
// hash_table[hash4 & 8191] = s - 3;
// hash_table8[CombineHash(Hash(s - 7), hash4) & 8191] = s - 7;
s++;
}

280
uppdev/LZF/LZF4.cpp Normal file
View file

@ -0,0 +1,280 @@
#if 1
#include <Core/Core.h>
#include <plugin/lz4/lz4.h>
using namespace Upp;
// Test/Test FoooFooo
#define LLOG(x) // LOG(x)
#define LTIMING(x) // RTIMING(x)
//#define HASH4
//#define HASH8
const int HASH_MASK = 128 * 1024 - 1;
#if 0
force_inline int Hash(const char *s)
{
// return Peek32le(s) % 8191;
// return memhash(s, 4) & 8191;
dword h = Peek32le(s); return (((h >> 23) - (h >> 15) - (h >> 7) - h));
// return (2654435761U * Peek32le(s)) & 8191;
}
force_inline int Hash8(const char *s)
{
return CombineHash(Hash(s), Hash(s + 4));
}
force_inline int Hash16(const char *s)
{
return CombineHash(Hash8(s), Hash8(s + 8));
}
#else
force_inline int FoldHash(int h)
{
// return 2654435761U * h;
return 31 * HIWORD(h) - LOWORD(h);
// return (((h >> 23) - (h >> 15) - (h >> 7) - h));
}
force_inline int Hash(const char *s)
{
// return Peek32le(s) % 8191;
// return memhash(s, 4) & 8191;
// return (Peek16le(s + 2) * 17) ^ Peek16le(s);
return FoldHash(Peek32le(s));
// return (2654435761U * Peek32le(s)) & 8191;
}
force_inline int Hash8(const char *s)
{
return FoldHash(CombineHash(Peek32le(s), Peek32le(s + 4)));
}
force_inline int Hash16(const char *s)
{
dword h = Peek32le(s) - Peek32le(s + 4) - Peek32le(s + 8) - Peek32le(s + 12);
return FoldHash(h);
}
force_inline int Hash32(const char *s)
{
dword h = Peek32le(s) - Peek32le(s + 4) - Peek32le(s + 8) - Peek32le(s + 12) -
Peek32le(s + 16) - Peek32le(s + 20) - Peek32le(s + 24) - Peek32le(s + 28);
return FoldHash(h);
}
#endif
String Literals;
int LiteralsCount;
int Matches;
int Match32;
int Match16;
int Match8;
int Match4;
int BigMatches;
int BigLiterals;
int FarOffsets;
int HugeOffsets;
int Extended;
int LargeMatch;
int Simple;
force_inline char *OutLen(char *out, int len, int maxn, int& stat)
{
if(len >= maxn) {
len -= maxn;
while(len >= 255) {
*out++ = 255;
len -= 255;
stat++;
}
*out++ = len;
stat++;
}
return out;
}
int LZFCompress(char *out, const char *s, int length)
{
RTIMESTOP("LZFCompress");
Literals.Clear();
LiteralsCount = 0;
Matches = 0;
Match16 = 0;
Match8 = 0;
Match4 = 0;
BigMatches = 0;
BigLiterals = 0;
FarOffsets = 0;
HugeOffsets = 0;
Extended = 0;
LargeMatch = 0;
Simple = 0;
const char *out0 = out;
const char *end = s + length;
// const int HASH_MASK = 2048 * 1024 - 1;
static const char *hash_table[HASH_MASK + 1];
static const char *hash_table8[HASH_MASK + 1];
static const char *hash_table16[HASH_MASK + 1];
static const char *hash_table32[HASH_MASK + 1];
for(int i = 0; i < HASH_MASK + 1; i++)
hash_table32[i] = hash_table16[i] = hash_table8[i] = hash_table[i] = s;
RTIMING("Part2");
const char *literal_ptr = s;
int literal_len = min(length, 32);
s += literal_len;
for(;;) {
const char *hptr;
int match_len;
for(;;) {
{ LTIMING("Lookup");
// word hash4 = Hash(s);
// hptr = hash_table8[CombineHash(hash4, Hash(s + 4)) & 8191];
// LLOG(AsCString(hptr, hptr + 4) << ' ' << AsCString(s, s + 4) << ' ' << hash);
#if 0
hptr = hash_table32[Hash32(s) & HASH_MASK];
if(0 && s - hptr < 0xfffff && Peek64le(hptr) == Peek64le(s) &&
Peek64le(hptr + 8) == Peek64le(s + 8) &&
Peek64le(hptr + 16) == Peek64le(s + 16) &&
Peek64le(hptr + 24) == Peek64le(s + 24)) {
Match32++;
match_len = 32;
break;
}
hptr = hash_table16[Hash16(s) & HASH_MASK];
if(0 && s - hptr < 0xfffff && Peek64le(hptr) == Peek64le(s) && Peek64le(hptr + 8) == Peek64le(s + 8)) {
Match16++;
match_len = 16;
break;
}
#endif
#ifdef HASH8
hptr = hash_table8[Hash8(s) & HASH_MASK];
if(s - hptr < 0xfffff && Peek64le(hptr) == Peek64le(s)) {
Match8++;
match_len = 8;
break;
}
#endif
// hptr = hash_table[hash4 & 8181];
// LLOG(AsCString(hptr, hptr + 4) << ' ' << AsCString(s, s + 4) << ' ' << hash);
#ifdef HASH4
hptr = hash_table[Hash(s) & HASH_MASK];
if(s - hptr < 0xfffff && Peek32le(hptr) == Peek32le(s)) {
Match4++;
match_len = 4;
break;
}
#endif
}
LTIMING("Hashing1");
literal_len++;
// hash_table32[Hash32(s - 31) & HASH_MASK] = s - 31;
// hash_table16[Hash16(s - 15) & HASH_MASK] = s - 15;
hash_table[Hash(s - 3) & HASH_MASK] = s - 3;
#ifdef HASH8
hash_table8[Hash8(s - 7) & HASH_MASK] = s - 7;
#endif
// hash_table[hash4 & 8191] = s - 3;
// word hash4 = Hash(s - 3);
// hash_table8[CombineHash(Hash(s - 7), hash4) & 8191] = s - 7;
s++;
if(s >= end - 32) {
*out++ = 0xf0 | max(literal_len, 15);
out = OutLen(out, literal_len, 15, BigLiterals);
SVO_MEMCPY(out, literal_ptr, literal_len);
Literals.Cat(literal_ptr, literal_len);
return out + literal_len - out0;
}
}
{ LTIMING("MatchLen");
while(hptr + match_len < s && s[match_len] == hptr[match_len])
match_len++;
}
LLOG("Literals (" << literal_len << "): " << AsCString(String(literal_ptr, literal_len)));
LLOG("Match (" << match_len << "): " << AsCString(String(hptr, match_len)));
int offset = s - hptr;
Matches++;
int ml = match_len - 4;
LiteralsCount += literal_len > 0;
if(literal_len == 0 && offset <= 0xfff && ml < 8) {
LTIMING("Simple match");
*out++ = (ml << 4) & HIBYTE(offset);
*out++ = LOBYTE(offset);
Simple++;
}
else {
LTIMING("Extended match");
*out++ = 0x80 | (max(literal_len, 7) << 4) | max(ml, 15);
offset -= 0xfff;
if(offset < 0xfff0) {
*out++ = HIBYTE(offset);
*out++ = LOBYTE(offset);
}
else {
HugeOffsets++;
// FIX THIS
*out++ = LOBYTE(offset);
*out++ = HIBYTE(offset);
*out++ = HIBYTE(offset);
}
Extended++;
if(offset > 0xfff)
FarOffsets++;
if(ml >= 8)
LargeMatch++;
out = OutLen(out, literal_len, 7, BigLiterals);
out = OutLen(out, ml, 15, BigMatches);
SVO_MEMCPY(out, literal_ptr, literal_len);
out += literal_len;
Literals.Cat(literal_ptr, literal_len);
}
// literals.Cat(literal_ptr, literal_len);
LTIMING("Hashing 2");
const char *e = s + match_len;
while(s < e) {
dword h1 = Peek32le(s - 3);
dword h2 = Peek32le(s - 7);
hash_table[Hash(s - 3) & HASH_MASK] = s - 3;
#ifdef HASH8
hash_table8[Hash8(s - 7) & HASH_MASK] = s - 7;
#endif
// hash_table16[Hash16(s - 15) & HASH_MASK] = s - 15;
// hash_table32[Hash32(s - 31) & HASH_MASK] = s - 31;
s++;
}
literal_ptr = s;
literal_len = 0;
}
}
#endif

20
uppdev/LZF/info.txt
View file

@ -48,3 +48,23 @@ FarOffsets = 2545912
BigLiterals + LiteralsLen + 2 * Matches + BigMatches + FarOffsets = 44983957
data.GetLength() = 100000000
==========
enwik8 2048:
32 39910940 9.39
16 39975701
8 40532362 4.98
4 45704862
with 8:
32768 40288716 7.3
2048 40532362 5.078
512 41037098 3.531
256 41618628 2.5
128 42502290 1.9
64 43660608 1.766
32 44616943 1.750
8 48986896 1.594
==========
viking.blb 2048
32 111426059 23.39
16 122522804 22.96
8 146699593 22.57

66
uppdev/LZF/main.cpp
View file

@ -4,12 +4,20 @@
using namespace Upp;
extern int Literals;
extern int LiteralsLen;
extern String Literals;
extern int LiteralsCount;
extern int Matches;
extern int BigMatches;
extern int BigLiterals;
extern int FarOffsets;
extern int HugeOffsets;
extern int Simple;
extern int Extended;
extern int LargeMatch;
extern int Match32;
extern int Match16;
extern int Match8;
extern int Match4;
int LZFCompress(char *out, const char *s, int length);
@ -19,28 +27,54 @@ String DEECompress(const char *data, const char *end);
CONSOLE_APP_MAIN
{
// String data = LoadFile("C:/u/aws.data/Viking_Chanarambie_Fenton_With_SurroundingFarms_Backcast_EPEs_14Sep11_v1004_allTI_PCs.blb");
// String data = LoadFile("C:/u/aws.data/Viking_Chanarambie_Fenton_With_SurroundingFarms_Backcast_EPEs_14Sep11_v1004_allTI_PCs.blb");
String data = LoadFile("D:/enwik8");
// String data = LoadFile("C:/u/upp.src/uppsrc/CtrlLib/ArrayCtrl.cpp");
// String data = LoadFile("C:/u/aws.data/c12_OCOE_env.blb");
// String data = LoadFile("D:/enwik8");
// String data = LoadFile("D:/canterbury_last.tar");
String data = LoadFile("C:/u/upp.src/uppsrc/CtrlLib/ArrayCtrl.cpp");
// String data = LoadFile(GetDataFile("LZF.cpp"));
// String data = LoadFile("C:/u/client/nos74.exe");
// String data = LoadFile("C:/upp/theide.exe");
String l;
Buffer<char> out(11 * data.GetLength() / 10);
RDUMP(LZFCompress(out, ~data, data.GetLength()));
Buffer<char> out(2 * data.GetLength());
{
RTIMESTOP("LZFCompress");
int n = LZFCompress(out, ~data, data.GetLength());
RDUMP(n);
RDUMP(data.GetLength());
}
// RDUMP(cdata.GetCount());
RDUMP(Literals);
RDUMP(LiteralsLen);
RDUMP(Match32);
RDUMP(Match16);
RDUMP(Match8);
RDUMP(Match4);
RDUMP(Matches);
RDUMP(Extended);
RDUMP(FarOffsets);
RDUMP(HugeOffsets);
RDUMP(LiteralsCount);
RDUMP(LargeMatch);
RDUMP(BigMatches);
RDUMP(BigLiterals);
RDUMP(FarOffsets);
RDUMP(Literals.GetCount());
RDUMP(/*Literals + */BigLiterals + LiteralsLen + 2 * Matches + BigMatches + FarOffsets);
// RDUMP(Literals + BigLiterals + LiteralsLen + 2 * Matches + Extended);
// RDUMP(AsCString(l, 100));
// RDUMP(l.GetCount());
// RDUMP(DEECompress(l, l.End()).GetLength());
int dee = DEECompress(Literals, Literals.End()).GetLength();
SaveFile("d:/literals", Literals);
if(Literals.GetCount() < 10000000)
RDUMP(ZCompress(Literals).GetCount());
RDUMP(dee);
RDUMP(Literals.GetCount() - dee);
RDUMP(Literals.GetCount());
RDUMP(data.GetLength());
RLOG("==========");
@ -49,6 +83,7 @@ CONSOLE_APP_MAIN
RDUMP(LZ4Compress(data).GetLength());
#ifdef flagLZ4STATS
RDUMP(lz4stat_LiteralLen);
RDUMP(lz4stat_Literals);
RDUMP(lz4stat_Matches);
RDUMP(lz4stat_BigMatch);
RDUMP(lz4stat_BigLiteral);
@ -61,4 +96,11 @@ CONSOLE_APP_MAIN
}
if(data.GetLength() < 10000000)
RDUMP(ZCompress(data).GetCount());
{
RTIMESTOP("LZ4 direct");
Buffer<char> out(2 * data.GetLength());
RDUMP(LZ4_compress(data, out, data.GetLength()));
}
}