From ff2cec90e61094777eea816a851560fca4a3ad64 Mon Sep 17 00:00:00 2001 From: Paul Knopf Date: Fri, 12 Apr 2019 21:02:30 -0400 Subject: [PATCH] Added our own tar extractor. --- src/net/Qml.Net.Tests/RuntimeManagerTests.cs | 78 ++++----- src/net/Qml.Net.sln.DotSettings | 1 + src/net/Qml.Net/QmlNetConfig.cs | 2 +- src/net/Qml.Net/Runtimes/Chmod.cs | 19 +++ src/net/Qml.Net/Runtimes/RuntimeManager.cs | 52 ++---- src/net/Qml.Net/Runtimes/Tar.cs | 159 +++++++++++++++++++ 6 files changed, 233 insertions(+), 78 deletions(-) create mode 100644 src/net/Qml.Net/Runtimes/Chmod.cs create mode 100644 src/net/Qml.Net/Runtimes/Tar.cs diff --git a/src/net/Qml.Net.Tests/RuntimeManagerTests.cs b/src/net/Qml.Net.Tests/RuntimeManagerTests.cs index 4187359b..9b90fdb5 100644 --- a/src/net/Qml.Net.Tests/RuntimeManagerTests.cs +++ b/src/net/Qml.Net.Tests/RuntimeManagerTests.cs @@ -2,9 +2,8 @@ using System; using System.IO; using System.Runtime.InteropServices; using FluentAssertions; +using Mono.Unix; using Qml.Net.Runtimes; -using SharpCompress.Common; -using SharpCompress.Readers; using Xunit; namespace Qml.Net.Tests @@ -12,53 +11,56 @@ namespace Qml.Net.Tests public class RuntimeManagerTests : IDisposable { private readonly string _tempDirectory; - + public RuntimeManagerTests() { _tempDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString().Replace("-", "")); Directory.CreateDirectory(_tempDirectory); - - RuntimeManager.ExtractTarGZStream = (stream, directory) => - { - using (var reader = ReaderFactory.Open(stream, new ReaderOptions())) - { - while (reader.MoveToNextEntry()) - { - if (!reader.Entry.IsDirectory) - { - reader.WriteEntryToDirectory(directory, new ExtractionOptions() - { - ExtractFullPath = true, - Overwrite = true, - WriteSymbolicLink = (sourcePath, targetPath) => - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - throw new Exception("File links aren't supported."); - } - - var link = new Mono.Unix.UnixSymbolicLinkInfo(sourcePath); - if (File.Exists(sourcePath)) - { - link.Delete(); // equivalent to ln -s -f - } - - link.CreateSymbolicLinkTo(targetPath); - } - }); - } - } - } - }; } [Fact] - public void Can_download_runtime() + public void Can_download_windows_untime() { RuntimeManager.DownloadRuntimeToDirectory(QmlNetConfig.QtBuildVersion, RuntimeTarget.Windows64, _tempDirectory); File.ReadAllText(Path.Combine(_tempDirectory, "version.txt")).Should().Be($"{QmlNetConfig.QtBuildVersion}-win-x64"); } - + + [Fact] + public void Can_download_linux_runtime() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + RuntimeManager.DownloadRuntimeToDirectory(QmlNetConfig.QtBuildVersion, RuntimeTarget.LinuxX64, _tempDirectory); + File.ReadAllText(Path.Combine(_tempDirectory, "version.txt")).Should().Be($"{QmlNetConfig.QtBuildVersion}-linux-x64"); + + // Make sure the permissions are set correctly. + var permissions = UnixFileSystemInfo + .GetFileSystemEntry(Path.Combine(_tempDirectory, "qt", "lib", "libQt5Xml.so.5.12.2")) + .FileAccessPermissions; + permissions.Should().Be(FileAccessPermissions.UserReadWriteExecute + | FileAccessPermissions.GroupRead | FileAccessPermissions.GroupExecute + | FileAccessPermissions.OtherRead | FileAccessPermissions.OtherExecute); + } + } + + [Fact] + public void Can_download_osx_runtime() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + RuntimeManager.DownloadRuntimeToDirectory(QmlNetConfig.QtBuildVersion, RuntimeTarget.OSX64, _tempDirectory); + File.ReadAllText(Path.Combine(_tempDirectory, "version.txt")).Should().Be($"{QmlNetConfig.QtBuildVersion}-osx-x64"); + + var permissions = UnixFileInfo + .GetFileSystemEntry(Path.Combine(_tempDirectory, "qt", "lib", "QtXml.framework", "Versions", "5", "QtXml")) + .FileAccessPermissions; + + permissions.Should().Be(FileAccessPermissions.UserReadWriteExecute + | FileAccessPermissions.GroupRead | FileAccessPermissions.GroupExecute + | FileAccessPermissions.OtherRead | FileAccessPermissions.OtherExecute); + } + } + public void Dispose() { if (Directory.Exists(_tempDirectory)) diff --git a/src/net/Qml.Net.sln.DotSettings b/src/net/Qml.Net.sln.DotSettings index 72ea2e23..57e31192 100644 --- a/src/net/Qml.Net.sln.DotSettings +++ b/src/net/Qml.Net.sln.DotSettings @@ -1,4 +1,5 @@  GC OSX + True True \ No newline at end of file diff --git a/src/net/Qml.Net/QmlNetConfig.cs b/src/net/Qml.Net/QmlNetConfig.cs index 344a621a..41a27adc 100644 --- a/src/net/Qml.Net/QmlNetConfig.cs +++ b/src/net/Qml.Net/QmlNetConfig.cs @@ -5,7 +5,7 @@ namespace Qml.Net public class QmlNetConfig { public static string QtBuildVersion => "qt-5.12.2-877b810"; - + public static bool ListenForExceptionsWhenInvokingTasks { get; set; } public static event Action UnhandledTaskException; diff --git a/src/net/Qml.Net/Runtimes/Chmod.cs b/src/net/Qml.Net/Runtimes/Chmod.cs new file mode 100644 index 00000000..c053321e --- /dev/null +++ b/src/net/Qml.Net/Runtimes/Chmod.cs @@ -0,0 +1,19 @@ +using System; +using System.Runtime.InteropServices; + +namespace Qml.Net.Runtimes +{ + internal class Chmod + { + [DllImport("libc", SetLastError = true)] + private static extern int chmod(string pathname, int mode); + + public static void Set(string pathName, int mode) + { + if (chmod(pathName, mode) != 0) + { + throw new Exception($"Unable to set mode: {pathName}"); + } + } + } +} \ No newline at end of file diff --git a/src/net/Qml.Net/Runtimes/RuntimeManager.cs b/src/net/Qml.Net/Runtimes/RuntimeManager.cs index a485a268..afe34bcb 100644 --- a/src/net/Qml.Net/Runtimes/RuntimeManager.cs +++ b/src/net/Qml.Net/Runtimes/RuntimeManager.cs @@ -22,16 +22,16 @@ namespace Qml.Net.Runtimes case RuntimeTarget.LinuxX64: return url.Replace("{target}", "linux-x64"); case RuntimeTarget.OSX64: - return url.Replace("{target}", "osx-64"); + return url.Replace("{target}", "osx-x64"); default: throw new Exception($"Unknown target {target}"); } }; - + public delegate void ExtractTarGZStreamDelegate(Stream stream, string destinationDirectory); - public static ExtractTarGZStreamDelegate ExtractTarGZStream; - + public static ExtractTarGZStreamDelegate ExtractTarGZStream = Tar.ExtractTarFromGzipStream; + public static RuntimeTarget GetCurrentRuntimeTarget() { if (IntPtr.Size != 8) @@ -43,21 +43,22 @@ namespace Qml.Net.Runtimes { return RuntimeTarget.Windows64; } - + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { return RuntimeTarget.LinuxX64; } - - if(RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { return RuntimeTarget.OSX64; } - + throw new Exception("Unknown OS platform"); } - public static void DownloadRuntimeToDirectory(string qtVersion, + public static void DownloadRuntimeToDirectory( + string qtVersion, RuntimeTarget runtimeTarget, string destinationDirectory) { @@ -83,7 +84,7 @@ namespace Qml.Net.Runtimes } var url = BuildRuntimeUrl(qtVersion, runtimeTarget); - + GetUrlStream(url, stream => { extractTarGZStreamDel(stream, destinationDirectory); @@ -108,40 +109,13 @@ namespace Qml.Net.Runtimes } } -// -// private static string GetRuntimeContainerDirectory() -// { -// var homeDirectory = (Environment.OSVersion.Platform == PlatformID.Unix || -// Environment.OSVersion.Platform == PlatformID.MacOSX) -// ? Environment.GetEnvironmentVariable("HOME") -// : Environment.ExpandEnvironmentVariables("%HOMEDRIVE%%HOMEPATH%"); -// var runtimeDirector = Path.Combine(homeDirectory, ".qmlnet-runtimes"); -// if (!Directory.Exists(runtimeDirector)) -// { -// -// } -// } -// -// public static string GetRuntimeDirectory() -// { -// var runtimePath = Environment.GetEnvironmentVariable("QMLNET_QT_RUNTIME_DIR"); -// if (!string.IsNullOrEmpty(runtimePath)) -// { -// // There is already one ready for us to start using! -// return runtimePath; -// } -// -// // We must now detect the proper version, download it, and return it's path. -// var url = $"https://github.com/qmlnet/qt-runtimes/releases/download/releases/{QmlNetConfig.QtBuildVersion}-{GetPlatformIdentifier()}-runtime.tar.gz"; -// } - public static void ConfigureRuntimeDirectory(string directory) { if (string.IsNullOrEmpty(directory)) { throw new ArgumentNullException(nameof(directory)); } - + if (!Directory.Exists(directory)) { throw new Exception("The directory doesn't exist."); @@ -168,7 +142,7 @@ namespace Qml.Net.Runtimes throw new Exception($"Plugins directory didn't exist: {pluginsDirectory}"); } Environment.SetEnvironmentVariable("QT_PLUGIN_PATH", pluginsDirectory); - + var qmlDirectory = Path.Combine(directory, "qml"); if (!Directory.Exists(qmlDirectory)) { diff --git a/src/net/Qml.Net/Runtimes/Tar.cs b/src/net/Qml.Net/Runtimes/Tar.cs new file mode 100644 index 00000000..ba8f679f --- /dev/null +++ b/src/net/Qml.Net/Runtimes/Tar.cs @@ -0,0 +1,159 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text; +using System.Text.RegularExpressions; +using Newtonsoft.Json.Schema; + +namespace Qml.Net.Runtimes +{ + internal class Tar + { + internal enum EntryType : byte + { + // ReSharper disable UnusedMember.Global + File = 0, + OldFile = (byte)'0', + HardLink = (byte)'1', + SymLink = (byte)'2', + CharDevice = (byte)'3', + BlockDevice = (byte)'4', + Directory = (byte)'5', + Fifo = (byte)'6', + LongLink = (byte)'K', + LongName = (byte)'L', + SparseFile = (byte)'S', + VolumeHeader = (byte)'V', + GlobalExtendedHeader = (byte)'g' + // ReSharper restore UnusedMember.Global + } + + public static Header ReadHeader(Stream stream) + { + var buffer = new byte[512]; + var bytesRead = stream.Read(buffer, 0, buffer.Length); + + if (bytesRead != 512) + { + throw new Exception("Couldn't ready block"); + } + + if (buffer.All(singleByte => singleByte == 0)) + { + // end of archive + return null; + } + + var header = new Header(); + header.EntryType = (EntryType)buffer[156]; + + switch (header.EntryType) + { + case EntryType.File: + case EntryType.OldFile: + case EntryType.Directory: + break; + case EntryType.SymLink: + header.LinkName = Encoding.ASCII.GetString(buffer, 157, 100).Trim('\0', ' '); + break; + default: + throw new Exception($"Unsupported type: {header.EntryType}"); + } + + header.Name = Encoding.ASCII.GetString(buffer, 0, 100).Trim('\0', ' '); + header.Size = Convert.ToUInt64(Encoding.ASCII.GetString(buffer, 124, 12).Trim('\0', ' '), 8); + header.Mode = Convert.ToInt32(Encoding.ASCII.GetString(buffer, 100, 8).Trim('\0', ' '), 8); + + return header; + } + + public class Header + { + public EntryType EntryType { get; set; } + + public string Name { get; set; } + + public string LinkName { get; set; } + + public ulong Size { get; set; } + + public int Mode { get; set; } + } + + public static void ExtractTarFromGzipStream(Stream stream, string destinationDirectory) + { + using (var gstream = new GZipStream(stream, CompressionMode.Decompress)) + { + var fileBuffer = new byte[1024 * 4]; + + while (true) + { + var header = ReadHeader(gstream); + + if (header == null) + { + break; + } + + if (header.Size > 0) + { + var output = Path.Combine(destinationDirectory, header.Name); + + var parentDirectory = Path.GetDirectoryName(output); + if (!Directory.Exists(parentDirectory)) + { + Directory.CreateDirectory(parentDirectory); + } + + using (var fs = File.Open(output, FileMode.Create, FileAccess.Write)) + { + var byteToRead = header.Size; + while (byteToRead > 0) + { + var bytesRead = gstream.Read( + fileBuffer, + 0, + (int)Math.Min(byteToRead, (ulong)fileBuffer.Length)); + if (bytesRead == 0) + { + throw new Exception("Couldn't read bytes."); + } + + fs.Write(fileBuffer, 0, bytesRead); + + byteToRead -= (ulong)bytesRead; + } + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + // This OS supports file modes. + // Let's set them. + if (Path.GetFileName(output) == "libQt5Xml.so.5.12.2") + { + Debugger.Break(); + } + Chmod.Set(output, header.Mode); + } + + var trailing = 512 - (int)(header.Size % 512); + if (trailing == 512) + { + trailing = 0; + } + if (trailing > 0) + { + if (gstream.Read(fileBuffer, 0, trailing) != trailing) + { + throw new Exception("Couldn't read bytes"); + } + } + } + } + } + } + } +} \ No newline at end of file