diff --git a/uppsrc/Core/Copying b/uppsrc/Core/Copying index 78862af24..1f269146e 100644 --- a/uppsrc/Core/Copying +++ b/uppsrc/Core/Copying @@ -19,4 +19,72 @@ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file +POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------------------------------ + +LZ4 - Fast LZ compression algorithm +Copyright (C) 2011-2015, Yann Collet. + +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- LZ4 source repository : https://github.com/Cyan4973/lz4 +- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c + + +------------------------------------------------------------------------------------------------ + +xxHash - Fast Hash algorithm +Copyright (C) 2012-2015, Yann Collet + +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- xxHash source repository : https://github.com/Cyan4973/xxHash diff --git a/uppsrc/plugin/lz4/lz4.h b/uppsrc/plugin/lz4/lz4.h index 6b660896a..df2acbbd2 100644 --- a/uppsrc/plugin/lz4/lz4.h +++ b/uppsrc/plugin/lz4/lz4.h @@ -111,20 +111,20 @@ public: }; -int64 LZ4Compress(Stream& out, Stream& in, Gate2 progress = false); -int64 LZ4Decompress(Stream& out, Stream& in, Gate2 progress = false); -String LZ4Compress(const void *data, int64 len, Gate2 progress = false); -String LZ4Compress(const String& s, Gate2 progress = false); -String LZ4Decompress(const void *data, int64 len, Gate2 progress = false); -String LZ4Decompress(const String& s, Gate2 progress = false); +int64 LZ4Compress(Stream& out, Stream& in, Function progress = false); +int64 LZ4Decompress(Stream& out, Stream& in, Function progress = false); +String LZ4Compress(const void *data, int64 len, Function progress = false); +String LZ4Compress(const String& s, Function progress = false); +String LZ4Decompress(const void *data, int64 len, Function progress = false); +String LZ4Decompress(const String& s, Function progress = false); #ifdef _MULTITHREADED -int64 CoLZ4Compress(Stream& out, Stream& in, Gate2 progress = false); -int64 CoLZ4Decompress(Stream& out, Stream& in, Gate2 progress = false); -String CoLZ4Compress(const void *data, int64 len, Gate2 progress = false); -String CoLZ4Compress(const String& s, Gate2 progress = false); -String CoLZ4Decompress(const void *data, int64 len, Gate2 progress = false); -String CoLZ4Decompress(const String& s, Gate2 progress = false); +int64 CoLZ4Compress(Stream& out, Stream& in, Function progress = false); +int64 CoLZ4Decompress(Stream& out, Stream& in, Function progress = false); +String CoLZ4Compress(const void *data, int64 len, Function progress = false); +String CoLZ4Compress(const String& s, Function progress = false); +String CoLZ4Decompress(const void *data, int64 len, Function progress = false); +String CoLZ4Decompress(const String& s, Function progress = false); #endif bool IsLZ4(Stream& s); diff --git a/uppsrc/plugin/lz4/util.cpp b/uppsrc/plugin/lz4/util.cpp index f7c332deb..59567b310 100644 --- a/uppsrc/plugin/lz4/util.cpp +++ b/uppsrc/plugin/lz4/util.cpp @@ -2,110 +2,113 @@ namespace Upp { -static void sCopy(Stream& out, Stream& in, Gate2 progress) +void sCompressStreamCopy_(Stream& out, Stream& in, Function progress, Stream& orig_in, int64 insz) { - const int CHUNK = 16384; + const int CHUNK = 32678; Buffer b(CHUNK); - while(!in.IsEof()) { // TODO: progress!!! + while(!in.IsEof()) { + if(progress(orig_in.GetPos(), insz)) + break; int n = in.Get(b, CHUNK); out.Put(b, n); } + progress(orig_in.GetPos(), insz); } -static int64 sLZ4Compress(Stream& out, Stream& in, int64 size, Gate2 progress, bool co) +static int64 sLZ4Compress(Stream& out, Stream& in, int64 size, Function progress, bool co) { LZ4CompressStream outs(out); #ifdef _MULTITHREADED if(co) outs.Concurrent(); #endif - sCopy(outs, in, progress); + sCompressStreamCopy_(outs, in, progress, in, size); outs.Close(); if(!out.IsError() && !outs.IsError()) return out.GetSize(); return -1; } -static int64 sLZ4Decompress(Stream& out, Stream& in, int64 size, Gate2 progress, bool co) +static int64 sLZ4Decompress(Stream& out, Stream& in, int64 size, Function progress, bool co) { LZ4DecompressStream ins(in); #ifdef _MULTITHREADED if(co) ins.Concurrent(); #endif - sCopy(out, ins, progress); + sCompressStreamCopy_(out, ins, progress, in, size); ins.Close(); if(!out.IsError() && !ins.IsError()) return out.GetSize(); return -1; } -int64 LZ4Compress(Stream& out, Stream& in, Gate2 progress) +int64 LZ4Compress(Stream& out, Stream& in, Function progress) { return sLZ4Compress(out, in, in.GetLeft(), progress, false); } -int64 LZ4Decompress(Stream& out, Stream& in, Gate2 progress) +int64 LZ4Decompress(Stream& out, Stream& in, Function progress) { return sLZ4Decompress(out, in, in.GetLeft(), progress, false); } -String LZ4Compress(const void *data, int64 len, Gate2 progress) +String LZ4Compress(const void *data, int64 len, Function progress) { StringStream out; MemReadStream in(data, len); return LZ4Compress(out, in, progress) < 0 ? String::GetVoid() : out.GetResult(); } -String LZ4Compress(const String& s, Gate2 progress) +String LZ4Compress(const String& s, Function progress) { return LZ4Compress(~s, s.GetLength(), progress); } -String LZ4Decompress(const void *data, int64 len, Gate2 progress) +String LZ4Decompress(const void *data, int64 len, Function progress) { StringStream out; MemReadStream in(data, len); return LZ4Decompress(out, in, progress) < 0 ? String::GetVoid() : out.GetResult(); } -String LZ4Decompress(const String& s, Gate2 progress) +String LZ4Decompress(const String& s, Function progress) { return LZ4Decompress(~s, s.GetLength(), progress); } #ifdef _MULTITHREADED -int64 CoLZ4Compress(Stream& out, Stream& in, Gate2 progress) +int64 CoLZ4Compress(Stream& out, Stream& in, Function progress) { return sLZ4Compress(out, in, in.GetLeft(), progress, true); } -int64 CoLZ4Decompress(Stream& out, Stream& in, Gate2 progress) +int64 CoLZ4Decompress(Stream& out, Stream& in, Function progress) { return sLZ4Decompress(out, in, in.GetLeft(), progress, true); } -String CoLZ4Compress(const void *data, int64 len, Gate2 progress) +String CoLZ4Compress(const void *data, int64 len, Function progress) { StringStream out; MemReadStream in(data, len); return CoLZ4Compress(out, in, progress) < 0 ? String::GetVoid() : out.GetResult(); } -String CoLZ4Compress(const String& s, Gate2 progress) +String CoLZ4Compress(const String& s, Function progress) { return CoLZ4Compress(~s, s.GetLength(), progress); } -String CoLZ4Decompress(const void *data, int64 len, Gate2 progress) +String CoLZ4Decompress(const void *data, int64 len, Function progress) { StringStream out; MemReadStream in(data, len); return CoLZ4Decompress(out, in, progress) < 0 ? String::GetVoid() : out.GetResult(); } -String CoLZ4Decompress(const String& s, Gate2 progress) +String CoLZ4Decompress(const String& s, Function progress) { return CoLZ4Decompress(~s, s.GetLength(), progress); } diff --git a/uppsrc/plugin/zstd/Util.cpp b/uppsrc/plugin/zstd/Util.cpp new file mode 100644 index 000000000..5b32f3dd7 --- /dev/null +++ b/uppsrc/plugin/zstd/Util.cpp @@ -0,0 +1,107 @@ +#include "zstd.h" + +namespace Upp { + +void sCompressStreamCopy_(Stream& out, Stream& in, Function progress, Stream& orig_in, int64 insz); + +static int64 sZstdCompress(Stream& out, Stream& in, int64 size, Function progress, bool co) +{ + ZstdCompressStream outs(out); +#ifdef _MULTITHREADED + if(co) + outs.Concurrent(); +#endif + sCompressStreamCopy_(outs, in, progress, in, size); + outs.Close(); + if(!out.IsError() && !outs.IsError()) + return out.GetSize(); + return -1; +} + +static int64 sZstdDecompress(Stream& out, Stream& in, int64 size, Function progress, bool co) +{ + ZstdDecompressStream ins(in); +#ifdef _MULTITHREADED + if(co) + ins.Concurrent(); +#endif + sCompressStreamCopy_(out, ins, progress, in, size); + ins.Close(); + if(!out.IsError() && !ins.IsError()) + return out.GetSize(); + return -1; +} + +int64 ZstdCompress(Stream& out, Stream& in, Function progress) +{ + return sZstdCompress(out, in, in.GetLeft(), progress, false); +} + +int64 ZstdDecompress(Stream& out, Stream& in, Function progress) +{ + return sZstdDecompress(out, in, in.GetLeft(), progress, false); +} + +String ZstdCompress(const void *data, int64 len, Function progress) +{ + StringStream out; + MemReadStream in(data, len); + return ZstdCompress(out, in, progress) < 0 ? String::GetVoid() : out.GetResult(); +} + +String ZstdCompress(const String& s, Function progress) +{ + return ZstdCompress(~s, s.GetLength(), progress); +} + +String ZstdDecompress(const void *data, int64 len, Function progress) +{ + StringStream out; + MemReadStream in(data, len); + return ZstdDecompress(out, in, progress) < 0 ? String::GetVoid() : out.GetResult(); +} + +String ZstdDecompress(const String& s, Function progress) +{ + return ZstdDecompress(~s, s.GetLength(), progress); +} + +#ifdef _MULTITHREADED + +int64 CoZstdCompress(Stream& out, Stream& in, Function progress) +{ + return sZstdCompress(out, in, in.GetLeft(), progress, true); +} + +int64 CoZstdDecompress(Stream& out, Stream& in, Function progress) +{ + return sZstdDecompress(out, in, in.GetLeft(), progress, true); +} + +String CoZstdCompress(const void *data, int64 len, Function progress) +{ + StringStream out; + MemReadStream in(data, len); + return CoZstdCompress(out, in, progress) < 0 ? String::GetVoid() : out.GetResult(); +} + +String CoZstdCompress(const String& s, Function progress) +{ + return CoZstdCompress(~s, s.GetLength(), progress); +} + +String CoZstdDecompress(const void *data, int64 len, Function progress) +{ + StringStream out; + MemReadStream in(data, len); + return CoZstdDecompress(out, in, progress) < 0 ? String::GetVoid() : out.GetResult(); +} + +String CoZstdDecompress(const String& s, Function progress) +{ + return CoZstdDecompress(~s, s.GetLength(), progress); +} + +#endif + +}; \ No newline at end of file diff --git a/uppsrc/plugin/zstd/zstd.h b/uppsrc/plugin/zstd/zstd.h index efa19cc61..42d0f6d91 100644 --- a/uppsrc/plugin/zstd/zstd.h +++ b/uppsrc/plugin/zstd/zstd.h @@ -95,21 +95,20 @@ public: ~ZstdDecompressStream(); }; - -int64 ZstdCompress(Stream& out, Stream& in, Gate2 progress = false); -int64 ZstdDecompress(Stream& out, Stream& in, Gate2 progress = false); -String ZstdCompress(const void *data, int64 len, Gate2 progress = false); -String ZstdCompress(const String& s, Gate2 progress = false); -String ZstdDecompress(const void *data, int64 len, Gate2 progress = false); -String ZstdDecompress(const String& s, Gate2 progress = false); +int64 ZstdCompress(Stream& out, Stream& in, Function progress); +int64 ZstdDecompress(Stream& out, Stream& in, Function progress); +String ZstdCompress(const void *data, int64 len, Function progress); +String ZstdCompress(const String& s, Function progress); +String ZstdDecompress(const void *data, int64 len, Function progress); +String ZstdDecompress(const String& s, Function progress); #ifdef _MULTITHREADED -int64 CoZstdCompress(Stream& out, Stream& in, Gate2 progress = false); -int64 CoZstdDecompress(Stream& out, Stream& in, Gate2 progress = false); -String CoZstdCompress(const void *data, int64 len, Gate2 progress = false); -String CoZstdCompress(const String& s, Gate2 progress = false); -String CoZstdDecompress(const void *data, int64 len, Gate2 progress = false); -String CoZstdDecompress(const String& s, Gate2 progress = false); +int64 CoZstdCompress(Stream& out, Stream& in, Function progress); +int64 CoZstdDecompress(Stream& out, Stream& in, Function progress); +String CoZstdCompress(const void *data, int64 len, Function progress); +String CoZstdCompress(const String& s, Function progress); +String CoZstdDecompress(const void *data, int64 len, Function progress); +String CoZstdDecompress(const String& s, Function progress); #endif bool IsZstd(Stream& s); diff --git a/uppsrc/plugin/zstd/zstd.upp b/uppsrc/plugin/zstd/zstd.upp index 69c1f4f22..646e45b4b 100644 --- a/uppsrc/plugin/zstd/zstd.upp +++ b/uppsrc/plugin/zstd/zstd.upp @@ -5,5 +5,6 @@ file zstd.h, Compress.cpp, Decompress.cpp, + Util.cpp, common\zstd.h;