From f7233ea69bc3fe8d0a9611aaeb337753d88fe89d Mon Sep 17 00:00:00 2001 From: User0755 <159659696+User0755@users.noreply.github.com> Date: Sun, 24 Mar 2024 15:16:39 +0200 Subject: [PATCH] plugin/zip: Zip64 support for UnZip and fixes for Zip (#194) plugin/zip: zip64 fixes and unzip64 (thanks Tom) --- uppsrc/plugin/zip/UnZip.cpp | 55 +++++++++++++++++++++++++++++-------- uppsrc/plugin/zip/Zip.cpp | 36 +++++++++++++++--------- uppsrc/plugin/zip/zip.h | 8 +++--- 3 files changed, 71 insertions(+), 28 deletions(-) diff --git a/uppsrc/plugin/zip/UnZip.cpp b/uppsrc/plugin/zip/UnZip.cpp index 8f5fc3197..2b97bb41e 100644 --- a/uppsrc/plugin/zip/UnZip.cpp +++ b/uppsrc/plugin/zip/UnZip.cpp @@ -9,25 +9,29 @@ void UnZip::ReadDir() file.Clear(); current = 0; - int entries = -1; - int offset; + int64 entries = -1; + int64 offset; int64 zipsize = zip->GetSize(); int64 pos = zipsize - 1; //22; zip->Seek(max((int64)0, zip->GetSize() - 4000)); // Precache end of zip zip->Get(); + + int64 zip64eocdl = 0; while(pos >= max((int64)0, zip->GetSize() - 65536)) { zip->ClearError(); zip->Seek(pos); + entries = -1; // ensure error return when header fails if(zip->Get32le() == 0x06054b50) { zip->Get16le(); // number of this disk zip->Get16le(); // number of the disk with the start of the central directory - int h = zip->Get16le(); // total number of entries in the central directory on this disk - entries = zip->Get16le(); // total number of entries in the central directory + int h = (word)zip->Get16le(); // total number of entries in the central directory on this disk + entries = (word)zip->Get16le(); // total number of entries in the central directory if(h != entries) // Multiple disks not supported return; zip->Get32le(); // size of the central directory - offset = zip->Get32le(); //offset of start of central directory with respect to the starting disk number + offset = (dword)zip->Get32le(); //offset of start of central directory with respect to the starting disk number + zip64eocdl = pos - 20; // offset of zip64 end of central directory locator int commentlen = zip->Get16le(); if(zip->GetPos() + commentlen == zipsize) break; @@ -37,6 +41,27 @@ void UnZip::ReadDir() if(entries < 0) return; + zip->Seek(zip64eocdl); + if(zip->Get32le() == 0x07064b50) { + zip->Get32le(); // ZIP64 end of central directory locator : number of the disk with the start of the zip64 end of central directory + int64 zip64eocdr = zip->Get64le(); // ZIP64 end of central directory locator : relative offset of the zip64 end of central directory record + zip->Seek(zip64eocdr); + if(zip->Get32le() == 0x06064b50) { + zip->Get64le(); // ZIP64 end of central directory record : the rest of the record after this field + zip->Get16le(); // ZIP64 end of central directory record : version made by + zip->Get16le(); // ZIP64 end of central directory record : version required to extract + zip->Get32le(); // ZIP64 end of central directory record : disk number + zip->Get32le(); // ZIP64 end of central directory record : number of disk with the start of central directory + int64 de = zip->Get64le(); // ZIP64 end of central directory record : number of directory entries on this disk + int64 te = zip->Get64le(); // ZIP64 end of central directory record : number of directory entries (total) + if(de != te) // Multiple disks not supported + return; + if(entries == 0xffff) entries = te; // Use ZIP64 entry counter + zip->Get64le(); // size of the central directory + if(offset == 0xffffffff) offset = zip->Get64le(); // ZIP64 offset of the central directory + } + } + zip->Seek(offset); for(int i = 0; i < entries; i++) { if(zip->Get32le() != 0x02014b50 && zip->IsEof()) @@ -48,8 +73,8 @@ void UnZip::ReadDir() f.method = zip->Get16le(); f.time = zip->Get32le(); f.crc = zip->Get32le(); - f.csize = zip->Get32le(); - f.usize = zip->Get32le(); + f.csize = (dword)zip->Get32le(); + f.usize = (dword)zip->Get32le(); int fnlen = zip->Get16le(); int extralen = zip->Get16le(); // extra field length 2 bytes int commentlen = zip->Get16le(); // file comment length 2 bytes @@ -59,7 +84,15 @@ void UnZip::ReadDir() zip->Get32le(); // external file attributes f.offset = zip->Get32le(); f.path = zip->Get(fnlen); - zip->SeekCur(extralen + commentlen); + int64 skipto = zip->GetPos() + extralen + commentlen; + if(extralen>=4 && zip->Get16le()==1){ // ZIP64 extra field : header ID + int bytes = zip->Get16le(); // ZIP64 extra field : bytes to follow + if(bytes>=8) f.usize = zip->Get64le(); // ZIP64 extra field : uncomp size + if(bytes>=16) f.csize = zip->Get64le(); // ZIP64 extra field : comp size + if(bytes>=24) f.offset = zip->Get64le(); // ZIP64 extra field : relative offset of local header + } + + zip->Seek(skipto); if(zip->IsEof() || zip->IsError()) return; } @@ -108,11 +141,11 @@ bool UnZip::ReadFile(Stream& out, Gate progress) dword extralen = zip->Get16le(); zip->SeekCur(filelen + extralen); dword crc; - dword l; + qword l; if(f.method == 0) { Buffer temp(65536); int loaded; - int count = f.csize; + int64 count = f.csize; Crc32Stream crc32; while(count > 0 && (loaded = zip->Get(temp, (int)min(count, 65536))) > 0) { out.Put(temp, loaded); @@ -126,7 +159,7 @@ bool UnZip::ReadFile(Stream& out, Gate progress) } else if(f.method == 8) - l = (int)zPress(out, *zip, f.csize, AsGate64(progress), false, false, &crc, false); + l = zPress(out, *zip, f.csize, AsGate64(progress), false, false, &crc, false); else return false; if(crc != f.crc || l != f.usize) diff --git a/uppsrc/plugin/zip/Zip.cpp b/uppsrc/plugin/zip/Zip.cpp index b8cb53f33..a4c444fd3 100644 --- a/uppsrc/plugin/zip/Zip.cpp +++ b/uppsrc/plugin/zip/Zip.cpp @@ -7,7 +7,7 @@ void Zip::WriteFolder(const char *path, Time tm) String p = UnixPath(path); if(*p.Last() != '/') p.Cat('/'); - WriteFile(~p, 0, p, Null, tm); + WriteFile(~p, 0, p, Null, tm, false); } int64 zPress(Stream& out, Stream& in, int64 size, Gate progress, bool gzip, @@ -43,6 +43,11 @@ void Zip::FileHeader(const char *path, Time tm) done += 5*2 + 5*4 + f.path.GetCount() + (f.zip64 ? 28 : 0); } +static bool IsPlainASCII(const String &s){ + for(int i=0;i=127)) return false; + return true; +} + void Zip::BeginFile(const char *path, Time tm, bool deflate, bool zip64) { ASSERT(!IsFileOpened()); @@ -55,9 +60,12 @@ void Zip::BeginFile(const char *path, Time tm, bool deflate, bool zip64) crc32.Clear(); uncompressed = true; } + + if(done>=0xffffffffULL) zip64 = true; // must switch to Zip64 due to large archive size + File& f = file.Add(); f.version = zip64 ? 45 : 20; - f.gpflag = 0x8 | 1<<11; // Added UTF-8 marker, i.e.: " | 1<<11"; + f.gpflag = IsPlainASCII(path) ? 0x8 : 0x8 | 1<<11; // Added UTF-8 marker, i.e.: " | 1<<11"; only for files with non-ASCII characters f.method = deflate ? 8 : 0; f.zip64 = zip64; f.crc = 0; @@ -189,15 +197,15 @@ void Zip::Finish() qword off = done; qword rof = 0; + int version = ((file.GetCount() >= 0xffff) || (done >= 0xffffffffULL)) ? 45 : 20; - bool zip64 = (file.GetCount() >= 0xffff) || (done > 19LL*INT_MAX/20); // Enable zip64 if too many files for old zip or very large data - for(int i = 0; i < file.GetCount(); i++) if(file[i].zip64) zip64 = true; // Pre-enable zip64 if any of the files require it + // Update version info for Zip64 where required: + for(int i = 0; i < file.GetCount(); i++) if(file[i].zip64 || (file[i].csize>=0xffffffffULL) || (file[i].usize>=0xffffffffULL)) file[i].version = version = 45; - int version = zip64 ? 45 : 20; - for(int i = 0; i < file.GetCount(); i++) { File& f = file[i]; - + bool zip64record = f.zip64 || (rof>=0xffffffffULL) || (f.csize>=0xffffffffULL) || (f.usize>=0xffffffffULL); + zip->Put32le(0x02014b50); zip->Put16le(version); // version made by zip->Put16le(f.version); // version required to extract @@ -205,28 +213,30 @@ void Zip::Finish() zip->Put16le(f.method); zip->Put32le(f.time); zip->Put32le(f.crc); - zip->Put32le((dword)(zip64 ? 0xffffffff : f.csize)); - zip->Put32le((dword)(zip64 ? 0xffffffff : f.usize)); + zip->Put32le((dword)(zip64record ? 0xffffffff : f.csize)); + zip->Put32le((dword)(zip64record ? 0xffffffff : f.usize)); zip->Put16le(f.path.GetCount()); - zip->Put16le(zip64 ? 28 : 0); // extra field length 2 bytes + zip->Put16le(zip64record ? 28 : 0); // extra field length 2 bytes zip->Put16le(0); // file comment length 2 bytes zip->Put16le(0); // disk number start 2 bytes zip->Put16le(0); // internal file attributes 2 bytes zip->Put32le(0); // external file attributes 4 bytes - zip->Put32le((dword)(zip64 ? 0xffffffff : rof)); // relative offset of local header 4 bytes + zip->Put32le((dword)(zip64record ? 0xffffffff : rof)); // relative offset of local header 4 bytes zip->Put(f.path); // ZIP64 additions: - if(zip64){ + if(zip64record){ zip->Put16le(1); // ZIP64 extra field : header ID zip->Put16le(24); // ZIP64 extra field : bytes to follow zip->Put64le(f.usize); // ZIP64 extra field : uncomp size zip->Put64le(f.csize); // ZIP64 extra field : comp size zip->Put64le(rof); // ZIP64 extra field : relative offset of local header } - done = done + 7 * 4 + 9 * 2 + f.path.GetCount() + (zip64 ? 28 : 0); + done = done + 7 * 4 + 9 * 2 + f.path.GetCount() + (zip64record ? 28 : 0); rof = rof + 5 * 2 + 5 * 4 + f.csize + f.path.GetCount() + (f.gpflag & 0x8 ? (f.zip64 ? 20 : 12) : 0) + (f.zip64 ? 28 : 0); } + bool zip64 = version==45; + if(zip64){ zip->Put32le(0x06064b50); // ZIP64 end of central directory record zip->Put64le(44); // ZIP64 end of central directory record : the rest of the record after this field diff --git a/uppsrc/plugin/zip/zip.h b/uppsrc/plugin/zip/zip.h index e314c2f2e..82e328180 100644 --- a/uppsrc/plugin/zip/zip.h +++ b/uppsrc/plugin/zip/zip.h @@ -12,8 +12,8 @@ class UnZip { dword time; int method; dword crc; - dword csize; - dword usize; + qword csize; + qword usize; int64 offset; }; @@ -37,7 +37,7 @@ public: String GetPath(int i) const { return file[i].path; } bool IsFolder(int i) const { return *file[i].path.Last() == '/'; } bool IsFile(int i) const { return !IsFolder(i); } - int GetLength(int i) const { return file[i].usize; } + int64 GetLength(int i) const { return file[i].usize; } Time GetTime(int i) const { return GetZipTime(file[i].time); } void Seek(int i) { ASSERT(i >= 0 && i < file.GetCount()); current = i; } @@ -45,7 +45,7 @@ public: bool IsFolder() const { return IsFolder(current); } bool IsFile() const { return !IsFolder(); } String GetPath() const { return GetPath(current); } - int GetLength() const { return GetLength(current); } + int64 GetLength() const { return GetLength(current); } Time GetTime() const { return GetTime(current); } void Skip() { current++; }