From afcccb09488ac2cd0ed701b52b17a5959dee5a41 Mon Sep 17 00:00:00 2001 From: Richard Webb Date: Mon, 4 Mar 2019 16:40:41 +0000 Subject: [PATCH 01/19] Merge PR #323: Fix ZipOutputStream.CloseEntry for AES encrypted Stored entries * Fix ZipOutputStream.CloseEntry() to work for Stored AES encrypted entries * Add unit tests for AES encrypted zips whose contents are Stored rather than Deflated --- .../Streams/DeflaterOutputStream.cs | 5 +- .../Zip/ZipOutputStream.cs | 6 ++ .../Zip/ZipEncryptionHandling.cs | 64 ++++++++++++++++++- 3 files changed, 71 insertions(+), 4 deletions(-) diff --git a/src/ICSharpCode.SharpZipLib/Zip/Compression/Streams/DeflaterOutputStream.cs b/src/ICSharpCode.SharpZipLib/Zip/Compression/Streams/DeflaterOutputStream.cs index f08947d0e..58bbc4c2d 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/Compression/Streams/DeflaterOutputStream.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/Compression/Streams/DeflaterOutputStream.cs @@ -414,7 +414,10 @@ protected override void Dispose(bool disposing) } } - private void GetAuthCodeIfAES() + /// + /// Get the Auth code for AES encrypted entries + /// + protected void GetAuthCodeIfAES() { if (cryptoTransform_ is ZipAESTransform) { diff --git a/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs b/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs index 88c810393..ab711d19b 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs @@ -486,6 +486,12 @@ public void CloseEntry() deflater_.Reset(); } } + else if (curMethod == CompressionMethod.Stored) + { + // This is done by Finsh() for Deflated entries, but we need to do it + // ourselves for Stored ones + base.GetAuthCodeIfAES(); + } // Write the AES Authentication Code (a hash of the compressed and encrypted data) if (curEntry.AESKeySize > 0) diff --git a/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipEncryptionHandling.cs b/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipEncryptionHandling.cs index c805fa9b4..865965e0d 100644 --- a/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipEncryptionHandling.cs +++ b/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipEncryptionHandling.cs @@ -19,6 +19,14 @@ public void Aes128Encryption() CreateZipWithEncryptedEntries("foo", 128); } + [Test] + [Category("Encryption")] + [Category("Zip")] + public void Aes128EncryptionStored() + { + CreateZipWithEncryptedEntries("foo", 128, CompressionMethod.Stored); + } + [Test] [Category("Encryption")] [Category("Zip")] @@ -27,6 +35,14 @@ public void Aes256Encryption() CreateZipWithEncryptedEntries("foo", 256); } + [Test] + [Category("Encryption")] + [Category("Zip")] + public void Aes256EncryptionStored() + { + CreateZipWithEncryptedEntries("foo", 256, CompressionMethod.Stored); + } + [Test] [Category("Encryption")] [Category("Zip")] @@ -88,6 +104,47 @@ public void ZipFileAesRead() } } + /// + /// Test using AES encryption on a file whose contents are Stored rather than deflated + /// + [Test] + [Category("Encryption")] + [Category("Zip")] + public void ZipFileStoreAes() + { + string password = "password"; + + using (var memoryStream = new MemoryStream()) + { + // Try to create a zip stream + WriteEncryptedZipToStream(memoryStream, password, 256, CompressionMethod.Stored); + + // reset + memoryStream.Seek(0, SeekOrigin.Begin); + + // try to read it + var zipFile = new ZipFile(memoryStream, leaveOpen: true) + { + Password = password + }; + + foreach (ZipEntry entry in zipFile) + { + if (!entry.IsFile) continue; + + // Should be stored rather than deflated + Assert.That(entry.CompressionMethod, Is.EqualTo(CompressionMethod.Stored), "Entry should be stored"); + + using (var zis = zipFile.GetInputStream(entry)) + using (var sr = new StreamReader(zis, Encoding.UTF8)) + { + var content = sr.ReadToEnd(); + Assert.That(content, Is.EqualTo(DummyDataString), "Decompressed content does not match input data"); + } + } + } + } + private static readonly string[] possible7zPaths = new[] { // Check in PATH "7z", "7za", @@ -135,7 +192,7 @@ public static bool TryGet7zBinPath(out string path7z) return false; } - public void WriteEncryptedZipToStream(Stream stream, string password, int keySize) + public void WriteEncryptedZipToStream(Stream stream, string password, int keySize, CompressionMethod compressionMethod = CompressionMethod.Deflated) { using (var zs = new ZipOutputStream(stream)) { @@ -146,6 +203,7 @@ public void WriteEncryptedZipToStream(Stream stream, string password, int keySiz ZipEntry zipEntry = new ZipEntry("test"); zipEntry.AESKeySize = keySize; zipEntry.DateTime = DateTime.Now; + zipEntry.CompressionMethod = compressionMethod; zs.PutNextEntry(zipEntry); @@ -160,11 +218,11 @@ public void WriteEncryptedZipToStream(Stream stream, string password, int keySiz } } - public void CreateZipWithEncryptedEntries(string password, int keySize) + public void CreateZipWithEncryptedEntries(string password, int keySize, CompressionMethod compressionMethod = CompressionMethod.Deflated) { using (var ms = new MemoryStream()) { - WriteEncryptedZipToStream(ms, password, keySize); + WriteEncryptedZipToStream(ms, password, keySize, compressionMethod); if (TryGet7zBinPath(out string path7z)) { From 43fd81eaf8e3a4d2ec935580ae8539a816bb289b Mon Sep 17 00:00:00 2001 From: Steve Bjorg Date: Sat, 15 Jun 2019 09:24:53 -0700 Subject: [PATCH 02/19] Merge PR #325: Ability to mimic a zip file created by a Linux file-system * respect the VersionMadeBy property of ZipEntry * correctly set HostSystem without clearing version --- src/ICSharpCode.SharpZipLib/Zip/ZipEntry.cs | 2 +- src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs | 6 +++--- src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ICSharpCode.SharpZipLib/Zip/ZipEntry.cs b/src/ICSharpCode.SharpZipLib/Zip/ZipEntry.cs index 572108f0f..6986c7ec3 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/ZipEntry.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/ZipEntry.cs @@ -526,7 +526,7 @@ public int HostSystem set { - versionMadeBy &= 0xff; + versionMadeBy &= 0x00ff; versionMadeBy |= (ushort)((value & 0xff) << 8); } } diff --git a/src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs b/src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs index 0db9cc7f0..e82c6ce2f 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs @@ -428,7 +428,7 @@ public ZipFile(string name) public ZipFile(FileStream file) : this(file, false) { - + } /// @@ -489,7 +489,7 @@ public ZipFile(FileStream file, bool leaveOpen) public ZipFile(Stream stream) : this(stream, false) { - + } /// @@ -2157,7 +2157,7 @@ private int WriteCentralDirectoryHeader(ZipEntry entry) WriteLEInt(ZipConstants.CentralHeaderSignature); // Version made by - WriteLEShort(ZipConstants.VersionMadeBy); + WriteLEShort((entry.HostSystem << 8) | entry.VersionMadeBy); // Version required to extract WriteLEShort(entry.Version); diff --git a/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs b/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs index ab711d19b..c3dd31af2 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs @@ -750,7 +750,7 @@ public override void Finish() foreach (ZipEntry entry in entries) { WriteLeInt(ZipConstants.CentralHeaderSignature); - WriteLeShort(ZipConstants.VersionMadeBy); + WriteLeShort((entry.HostSystem << 8) | entry.VersionMadeBy); WriteLeShort(entry.Version); WriteLeShort(entry.Flags); WriteLeShort((short)entry.CompressionMethodForHeader); From 1c3f459581cb347ec5056b2c21faa1828eab1708 Mon Sep 17 00:00:00 2001 From: Sean Date: Sat, 15 Jun 2019 09:56:10 -0700 Subject: [PATCH 03/19] Merge PR #336: Treat empty string as no RootPath in TarArchive Fixes #334 --- src/ICSharpCode.SharpZipLib/Tar/TarArchive.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ICSharpCode.SharpZipLib/Tar/TarArchive.cs b/src/ICSharpCode.SharpZipLib/Tar/TarArchive.cs index 34aaf65c4..2161ab12a 100644 --- a/src/ICSharpCode.SharpZipLib/Tar/TarArchive.cs +++ b/src/ICSharpCode.SharpZipLib/Tar/TarArchive.cs @@ -784,7 +784,7 @@ private void WriteEntryCore(TarEntry sourceEntry, bool recurse) string newName = null; - if (rootPath != null) + if (!String.IsNullOrEmpty(rootPath)) { if (entry.Name.StartsWith(rootPath, StringComparison.OrdinalIgnoreCase)) { From 98bbddaa0b354981dad7de773ecc511e66917396 Mon Sep 17 00:00:00 2001 From: Richard Webb Date: Sat, 15 Jun 2019 18:41:45 +0100 Subject: [PATCH 04/19] Merge PR #331: Change ZipAESStream to handle reads of less data than the AES block size * Change ZipAESStream to better handle reads of less than the crypto block size * Add unit test for reading AES Encrypted/Stored zip entries * Fixes #324 --- .../Encryption/ZipAESStream.cs | 122 ++++++++++++++---- .../Zip/ZipEncryptionHandling.cs | 63 ++++++++- 2 files changed, 160 insertions(+), 25 deletions(-) diff --git a/src/ICSharpCode.SharpZipLib/Encryption/ZipAESStream.cs b/src/ICSharpCode.SharpZipLib/Encryption/ZipAESStream.cs index cf0792c47..ffafee5df 100644 --- a/src/ICSharpCode.SharpZipLib/Encryption/ZipAESStream.cs +++ b/src/ICSharpCode.SharpZipLib/Encryption/ZipAESStream.cs @@ -27,8 +27,6 @@ public ZipAESStream(Stream stream, ZipAESTransform transform, CryptoStreamMode m _transform = transform; _slideBuffer = new byte[1024]; - _blockAndAuth = CRYPTO_BLOCK_SIZE + AUTH_CODE_LENGTH; - // mode: // CryptoStreamMode.Read means we read from "stream" and pass decrypted to our Read() method. // Write bypasses this stream and uses the Transform directly. @@ -41,33 +39,72 @@ public ZipAESStream(Stream stream, ZipAESTransform transform, CryptoStreamMode m // The final n bytes of the AES stream contain the Auth Code. private const int AUTH_CODE_LENGTH = 10; + // Blocksize is always 16 here, even for AES-256 which has transform.InputBlockSize of 32. + private const int CRYPTO_BLOCK_SIZE = 16; + + // total length of block + auth code + private const int BLOCK_AND_AUTH = CRYPTO_BLOCK_SIZE + AUTH_CODE_LENGTH; + private Stream _stream; private ZipAESTransform _transform; private byte[] _slideBuffer; private int _slideBufStartPos; private int _slideBufFreePos; - // Blocksize is always 16 here, even for AES-256 which has transform.InputBlockSize of 32. - private const int CRYPTO_BLOCK_SIZE = 16; + // Buffer block transforms to enable partial reads + private byte[] _transformBuffer = null;// new byte[CRYPTO_BLOCK_SIZE]; + private int _transformBufferFreePos; + private int _transformBufferStartPos; - private int _blockAndAuth; + // Do we have some buffered data available? + private bool HasBufferedData =>_transformBuffer != null && _transformBufferStartPos < _transformBufferFreePos; /// /// Reads a sequence of bytes from the current CryptoStream into buffer, /// and advances the position within the stream by the number of bytes read. /// public override int Read(byte[] buffer, int offset, int count) + { + // Nothing to do + if (count == 0) + return 0; + + // If we have buffered data, read that first + int nBytes = 0; + if (HasBufferedData) + { + nBytes = ReadBufferedData(buffer, offset, count); + + // Read all requested data from the buffer + if (nBytes == count) + return nBytes; + + offset += nBytes; + count -= nBytes; + } + + // Read more data from the input, if available + if (_slideBuffer != null) + nBytes += ReadAndTransform(buffer, offset, count); + + return nBytes; + } + + // Read data from the underlying stream and decrypt it + private int ReadAndTransform(byte[] buffer, int offset, int count) { int nBytes = 0; while (nBytes < count) { + int bytesLeftToRead = count - nBytes; + // Calculate buffer quantities vs read-ahead size, and check for sufficient free space int byteCount = _slideBufFreePos - _slideBufStartPos; // Need to handle final block and Auth Code specially, but don't know total data length. // Maintain a read-ahead equal to the length of (crypto block + Auth Code). // When that runs out we can detect these final sections. - int lengthToRead = _blockAndAuth - byteCount; + int lengthToRead = BLOCK_AND_AUTH - byteCount; if (_slideBuffer.Length - _slideBufFreePos < lengthToRead) { // Shift the data to the beginning of the buffer @@ -84,17 +121,11 @@ public override int Read(byte[] buffer, int offset, int count) // Recalculate how much data we now have byteCount = _slideBufFreePos - _slideBufStartPos; - if (byteCount >= _blockAndAuth) + if (byteCount >= BLOCK_AND_AUTH) { - // At least a 16 byte block and an auth code remains. - _transform.TransformBlock(_slideBuffer, - _slideBufStartPos, - CRYPTO_BLOCK_SIZE, - buffer, - offset); - nBytes += CRYPTO_BLOCK_SIZE; - offset += CRYPTO_BLOCK_SIZE; - _slideBufStartPos += CRYPTO_BLOCK_SIZE; + var read = TransformAndBufferBlock(buffer, offset, bytesLeftToRead, CRYPTO_BLOCK_SIZE); + nBytes += read; + offset += read; } else { @@ -103,14 +134,7 @@ public override int Read(byte[] buffer, int offset, int count) { // At least one byte of data plus auth code int finalBlock = byteCount - AUTH_CODE_LENGTH; - _transform.TransformBlock(_slideBuffer, - _slideBufStartPos, - finalBlock, - buffer, - offset); - - nBytes += finalBlock; - _slideBufStartPos += finalBlock; + nBytes += TransformAndBufferBlock(buffer, offset, bytesLeftToRead, finalBlock); } else if (byteCount < AUTH_CODE_LENGTH) throw new Exception("Internal error missed auth code"); // Coding bug @@ -125,12 +149,62 @@ public override int Read(byte[] buffer, int offset, int count) } } + // don't need this any more, so use it as a 'complete' flag + _slideBuffer = null; + break; // Reached the auth code } } return nBytes; } + // read some buffered data + private int ReadBufferedData(byte[] buffer, int offset, int count) + { + int copyCount = Math.Min(count, _transformBufferFreePos - _transformBufferStartPos); + + Array.Copy(_transformBuffer, _transformBufferStartPos, buffer, offset, count); + _transformBufferStartPos += copyCount; + + return copyCount; + } + + // Perform the crypto transform, and buffer the data if less than one block has been requested. + private int TransformAndBufferBlock(byte[] buffer, int offset, int count, int blockSize) + { + // If the requested data is greater than one block, transform it directly into the output + // If it's smaller, do it into a temporary buffer and copy the requested part + bool bufferRequired = (blockSize > count); + + if (bufferRequired && _transformBuffer == null) + _transformBuffer = new byte[CRYPTO_BLOCK_SIZE]; + + var targetBuffer = bufferRequired ? _transformBuffer : buffer; + var targetOffset = bufferRequired ? 0 : offset; + + // Transform the data + _transform.TransformBlock(_slideBuffer, + _slideBufStartPos, + blockSize, + targetBuffer, + targetOffset); + + _slideBufStartPos += blockSize; + + if (!bufferRequired) + { + return blockSize; + } + else + { + Array.Copy(_transformBuffer, 0, buffer, offset, count); + _transformBufferStartPos = count; + _transformBufferFreePos = blockSize; + + return count; + } + } + /// /// Writes a sequence of bytes to the current stream and advances the current position within this stream by the number of bytes written. /// diff --git a/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipEncryptionHandling.cs b/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipEncryptionHandling.cs index 865965e0d..3f8f64427 100644 --- a/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipEncryptionHandling.cs +++ b/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipEncryptionHandling.cs @@ -1,4 +1,5 @@ -using ICSharpCode.SharpZipLib.Zip; +using ICSharpCode.SharpZipLib.Core; +using ICSharpCode.SharpZipLib.Zip; using NUnit.Framework; using System; using System.Diagnostics; @@ -145,6 +146,66 @@ public void ZipFileStoreAes() } } + /// + /// Test using AES encryption on a file whose contents are Stored rather than deflated + /// + [Test] + [Category("Encryption")] + [Category("Zip")] + public void ZipFileStoreAesPartialRead() + { + string password = "password"; + + using (var memoryStream = new MemoryStream()) + { + // Try to create a zip stream + WriteEncryptedZipToStream(memoryStream, password, 256, CompressionMethod.Stored); + + // reset + memoryStream.Seek(0, SeekOrigin.Begin); + + // try to read it + var zipFile = new ZipFile(memoryStream, leaveOpen: true) + { + Password = password + }; + + foreach (ZipEntry entry in zipFile) + { + if (!entry.IsFile) continue; + + // Should be stored rather than deflated + Assert.That(entry.CompressionMethod, Is.EqualTo(CompressionMethod.Stored), "Entry should be stored"); + + using (var ms = new MemoryStream()) + { + using (var zis = zipFile.GetInputStream(entry)) + { + byte[] buffer = new byte[1]; + + while (true) + { + int b = zis.ReadByte(); + + if (b == -1) + break; + + ms.WriteByte((byte)b); + } + } + + ms.Seek(0, SeekOrigin.Begin); + + using (var sr = new StreamReader(ms, Encoding.UTF8)) + { + var content = sr.ReadToEnd(); + Assert.That(content, Is.EqualTo(DummyDataString), "Decompressed content does not match input data"); + } + } + } + } + } + private static readonly string[] possible7zPaths = new[] { // Check in PATH "7z", "7za", From 1a12d9c114d03c3c21e6c011e3591ed31c401c2a Mon Sep 17 00:00:00 2001 From: Richard Webb Date: Sat, 15 Jun 2019 18:51:03 +0100 Subject: [PATCH 05/19] Merge PR #330: Add a benchmarks project using BenchmarkDotNet --- ICSharpCode.SharpZipLib.sln | 15 ++++- .../Checksum/Adler32.cs | 44 +++++++++++++++ .../Checksum/BZip2Crc.cs | 44 +++++++++++++++ .../Checksum/Crc32.cs | 44 +++++++++++++++ .../ICSharpCode.SharpZipLib.Benchmark.csproj | 18 ++++++ .../Program.cs | 27 +++++++++ .../Zip/ZipInputStream.cs | 56 +++++++++++++++++++ .../Zip/ZipOutputStream.cs | 40 +++++++++++++ 8 files changed, 285 insertions(+), 3 deletions(-) create mode 100644 benchmark/ICSharpCode.SharpZipLib.Benchmark/Checksum/Adler32.cs create mode 100644 benchmark/ICSharpCode.SharpZipLib.Benchmark/Checksum/BZip2Crc.cs create mode 100644 benchmark/ICSharpCode.SharpZipLib.Benchmark/Checksum/Crc32.cs create mode 100644 benchmark/ICSharpCode.SharpZipLib.Benchmark/ICSharpCode.SharpZipLib.Benchmark.csproj create mode 100644 benchmark/ICSharpCode.SharpZipLib.Benchmark/Program.cs create mode 100644 benchmark/ICSharpCode.SharpZipLib.Benchmark/Zip/ZipInputStream.cs create mode 100644 benchmark/ICSharpCode.SharpZipLib.Benchmark/Zip/ZipOutputStream.cs diff --git a/ICSharpCode.SharpZipLib.sln b/ICSharpCode.SharpZipLib.sln index 8d183664a..cab9675b5 100644 --- a/ICSharpCode.SharpZipLib.sln +++ b/ICSharpCode.SharpZipLib.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.26228.9 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.28705.295 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Configuration", "Solution Configuration", "{F1097E98-4DEB-4A0A-81EE-5CEC667EBDF0}" ProjectSection(SolutionItems) = preProject @@ -15,7 +15,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ICSharpCode.SharpZipLib", " EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ICSharpCode.SharpZipLib.Tests", "test\ICSharpCode.SharpZipLib.Tests\ICSharpCode.SharpZipLib.Tests.csproj", "{82211166-9C45-4603-8E3A-2CA2EFFCBC26}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ICSharpCode.SharpZipLib.TestBootstrapper", "test\ICSharpCode.SharpZipLib.TestBootstrapper\ICSharpCode.SharpZipLib.TestBootstrapper.csproj", "{535D7365-C5B1-4253-9233-D72D972CA851}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ICSharpCode.SharpZipLib.TestBootstrapper", "test\ICSharpCode.SharpZipLib.TestBootstrapper\ICSharpCode.SharpZipLib.TestBootstrapper.csproj", "{535D7365-C5B1-4253-9233-D72D972CA851}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ICSharpCode.SharpZipLib.Benchmark", "benchmark\ICSharpCode.SharpZipLib.Benchmark\ICSharpCode.SharpZipLib.Benchmark.csproj", "{C51E638B-DDD0-48B6-A6BD-EBC4E6A104C7}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -35,8 +37,15 @@ Global {535D7365-C5B1-4253-9233-D72D972CA851}.Debug|Any CPU.Build.0 = Debug|Any CPU {535D7365-C5B1-4253-9233-D72D972CA851}.Release|Any CPU.ActiveCfg = Release|Any CPU {535D7365-C5B1-4253-9233-D72D972CA851}.Release|Any CPU.Build.0 = Release|Any CPU + {C51E638B-DDD0-48B6-A6BD-EBC4E6A104C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C51E638B-DDD0-48B6-A6BD-EBC4E6A104C7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C51E638B-DDD0-48B6-A6BD-EBC4E6A104C7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C51E638B-DDD0-48B6-A6BD-EBC4E6A104C7}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {0A049193-65F8-49AF-82CB-75D42563DA16} + EndGlobalSection EndGlobal diff --git a/benchmark/ICSharpCode.SharpZipLib.Benchmark/Checksum/Adler32.cs b/benchmark/ICSharpCode.SharpZipLib.Benchmark/Checksum/Adler32.cs new file mode 100644 index 000000000..0b3b906fb --- /dev/null +++ b/benchmark/ICSharpCode.SharpZipLib.Benchmark/Checksum/Adler32.cs @@ -0,0 +1,44 @@ +using System; +using BenchmarkDotNet.Attributes; + +namespace ICSharpCode.SharpZipLib.Benchmark.Checksum +{ + [Config(typeof(MultipleRuntimes))] + public class Adler32 + { + private const int ChunkCount = 256; + private const int ChunkSize = 1024 * 1024; + private const int N = ChunkCount * ChunkSize; + private readonly byte[] data; + + public Adler32() + { + data = new byte[N]; + new Random(1).NextBytes(data); + } + + [Benchmark] + public long Adler32LargeUpdate() + { + var adler32 = new ICSharpCode.SharpZipLib.Checksum.Adler32(); + adler32.Update(data); + return adler32.Value; + } + + /* + [Benchmark] + public long Adler32ChunkedUpdate() + { + var adler32 = new ICSharpCode.SharpZipLib.Checksum.Adler32(); + + for (int i = 0; i < ChunkCount; i++) + { + var segment = new ArraySegment(data, ChunkSize * i, ChunkSize); + adler32.Update(segment); + } + + return adler32.Value; + } + */ + } +} diff --git a/benchmark/ICSharpCode.SharpZipLib.Benchmark/Checksum/BZip2Crc.cs b/benchmark/ICSharpCode.SharpZipLib.Benchmark/Checksum/BZip2Crc.cs new file mode 100644 index 000000000..c7c233691 --- /dev/null +++ b/benchmark/ICSharpCode.SharpZipLib.Benchmark/Checksum/BZip2Crc.cs @@ -0,0 +1,44 @@ +using System; +using BenchmarkDotNet.Attributes; + +namespace ICSharpCode.SharpZipLib.Benchmark.Checksum +{ + [Config(typeof(MultipleRuntimes))] + public class BZip2Crc + { + private const int ChunkCount = 256; + private const int ChunkSize = 1024 * 1024; + private const int N = ChunkCount * ChunkSize; + private readonly byte[] data; + + public BZip2Crc() + { + data = new byte[N]; + new Random(1).NextBytes(data); + } + + [Benchmark] + public long BZip2CrcLargeUpdate() + { + var bzipCrc = new ICSharpCode.SharpZipLib.Checksum.BZip2Crc(); + bzipCrc.Update(data); + return bzipCrc.Value; + } + + /* + [Benchmark] + public long BZip2CrcChunkedUpdate() + { + var bzipCrc = new ICSharpCode.SharpZipLib.Checksum.BZip2Crc(); + + for (int i = 0; i < ChunkCount; i++) + { + var segment = new ArraySegment(data, ChunkSize * i, ChunkSize); + bzipCrc.Update(segment); + } + + return bzipCrc.Value; + } + */ + } +} diff --git a/benchmark/ICSharpCode.SharpZipLib.Benchmark/Checksum/Crc32.cs b/benchmark/ICSharpCode.SharpZipLib.Benchmark/Checksum/Crc32.cs new file mode 100644 index 000000000..1e355552b --- /dev/null +++ b/benchmark/ICSharpCode.SharpZipLib.Benchmark/Checksum/Crc32.cs @@ -0,0 +1,44 @@ +using System; +using BenchmarkDotNet.Attributes; + +namespace ICSharpCode.SharpZipLib.Benchmark.Checksum +{ + [Config(typeof(MultipleRuntimes))] + public class Crc32 + { + private const int ChunkCount = 256; + private const int ChunkSize = 1024 * 1024; + private const int N = ChunkCount * ChunkSize; + private readonly byte[] data; + + public Crc32() + { + data = new byte[N]; + new Random(1).NextBytes(data); + } + + [Benchmark] + public long Crc32LargeUpdate() + { + var crc32 = new ICSharpCode.SharpZipLib.Checksum.Crc32(); + crc32.Update(data); + return crc32.Value; + } + + /* + [Benchmark] + public long Crc32ChunkedUpdate() + { + var crc32 = new ICSharpCode.SharpZipLib.Checksum.Crc32(); + + for (int i = 0; i < ChunkCount; i++) + { + var segment = new ArraySegment(data, ChunkSize * i, ChunkSize); + crc32.Update(segment); + } + + return crc32.Value; + } + */ + } +} diff --git a/benchmark/ICSharpCode.SharpZipLib.Benchmark/ICSharpCode.SharpZipLib.Benchmark.csproj b/benchmark/ICSharpCode.SharpZipLib.Benchmark/ICSharpCode.SharpZipLib.Benchmark.csproj new file mode 100644 index 000000000..8067edf35 --- /dev/null +++ b/benchmark/ICSharpCode.SharpZipLib.Benchmark/ICSharpCode.SharpZipLib.Benchmark.csproj @@ -0,0 +1,18 @@ + + + + Exe + netcoreapp2.1;net461 + + + + + 0.11.4 + + + + + + + + diff --git a/benchmark/ICSharpCode.SharpZipLib.Benchmark/Program.cs b/benchmark/ICSharpCode.SharpZipLib.Benchmark/Program.cs new file mode 100644 index 000000000..6e37bcf34 --- /dev/null +++ b/benchmark/ICSharpCode.SharpZipLib.Benchmark/Program.cs @@ -0,0 +1,27 @@ +using System; +using BenchmarkDotNet; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.CsProj; + +namespace ICSharpCode.SharpZipLib.Benchmark +{ + public class MultipleRuntimes : ManualConfig + { + public MultipleRuntimes() + { + Add(Job.Default.With(CsProjClassicNetToolchain.Net461).AsBaseline()); // NET 4.6.1 + Add(Job.Default.With(CsProjCoreToolchain.NetCoreApp21)); // .NET Core 2.1 + //Add(Job.Default.With(CsProjCoreToolchain.NetCoreApp30)); // .NET Core 3.0 + } + } + + class Program + { + static void Main(string[] args) + { + BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); + } + } +} diff --git a/benchmark/ICSharpCode.SharpZipLib.Benchmark/Zip/ZipInputStream.cs b/benchmark/ICSharpCode.SharpZipLib.Benchmark/Zip/ZipInputStream.cs new file mode 100644 index 000000000..d112e22e3 --- /dev/null +++ b/benchmark/ICSharpCode.SharpZipLib.Benchmark/Zip/ZipInputStream.cs @@ -0,0 +1,56 @@ +using System; +using System.IO; +using BenchmarkDotNet.Attributes; + +namespace ICSharpCode.SharpZipLib.Benchmark.Zip +{ + [Config(typeof(MultipleRuntimes))] + public class ZipInputStream + { + private const int ChunkCount = 64; + private const int ChunkSize = 1024 * 1024; + private const int N = ChunkCount * ChunkSize; + + byte[] zippedData; + + public ZipInputStream() + { + using (var memoryStream = new MemoryStream()) + { + using (var zipOutputStream = new SharpZipLib.Zip.ZipOutputStream(memoryStream)) + { + zipOutputStream.PutNextEntry(new SharpZipLib.Zip.ZipEntry("0")); + + var inputBuffer = new byte[ChunkSize]; + + for (int i = 0; i < ChunkCount; i++) + { + zipOutputStream.Write(inputBuffer, 0, inputBuffer.Length); + } + } + + zippedData = memoryStream.ToArray(); + } + } + + [Benchmark] + public long ReadZipInputStream() + { + using (var memoryStream = new MemoryStream(zippedData)) + { + using (var zipInputStream = new SharpZipLib.Zip.ZipInputStream(memoryStream)) + { + var buffer = new byte[4096]; + var entry = zipInputStream.GetNextEntry(); + + while (zipInputStream.Read(buffer, 0, buffer.Length) > 0) + { + + } + + return entry.Size; + } + } + } + } +} diff --git a/benchmark/ICSharpCode.SharpZipLib.Benchmark/Zip/ZipOutputStream.cs b/benchmark/ICSharpCode.SharpZipLib.Benchmark/Zip/ZipOutputStream.cs new file mode 100644 index 000000000..0f7b5c7c4 --- /dev/null +++ b/benchmark/ICSharpCode.SharpZipLib.Benchmark/Zip/ZipOutputStream.cs @@ -0,0 +1,40 @@ +using System; +using System.IO; +using BenchmarkDotNet.Attributes; + +namespace ICSharpCode.SharpZipLib.Benchmark.Zip +{ + [Config(typeof(MultipleRuntimes))] + public class ZipOutputStream + { + private const int ChunkCount = 64; + private const int ChunkSize = 1024 * 1024; + private const int N = ChunkCount * ChunkSize; + + byte[] outputBuffer; + byte[] inputBuffer; + + public ZipOutputStream() + { + inputBuffer = new byte[ChunkSize]; + outputBuffer = new byte[N]; + } + + [Benchmark] + public long WriteZipOutputStream() + { + using (var memoryStream = new MemoryStream(outputBuffer)) + { + var zipOutputStream = new SharpZipLib.Zip.ZipOutputStream(memoryStream); + zipOutputStream.PutNextEntry(new SharpZipLib.Zip.ZipEntry("0")); + + for (int i = 0; i < ChunkCount; i++) + { + zipOutputStream.Write(inputBuffer, 0, inputBuffer.Length); + } + + return memoryStream.Position; + } + } + } +} From 88cfaa627d4d0f876bb29bca86555155f4a0539a Mon Sep 17 00:00:00 2001 From: Adam Reeve Date: Sat, 15 Jun 2019 19:19:34 +0100 Subject: [PATCH 06/19] Merge PR #225: Fix flushing of GZipOutputStream * Fix flushing a GZipOutputStream When flushing a GZipOutputStream, ensure we deflate all data in the input buffer and write it to the underlying stream before we flush the underlying stream. * Fix infinite loop when flushing DeflaterOutputStream with no compression DeflaterEngine.DeflateStored would always write more output even if there was no more input data to write, resulting in an infinite loop. --- .../GZip/GzipInputStream.cs | 3 + .../Zip/Compression/DeflaterEngine.cs | 2 +- .../Streams/DeflaterOutputStream.cs | 9 ++- .../GZip/GZipTests.cs | 55 +++++++++++++++++++ 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/src/ICSharpCode.SharpZipLib/GZip/GzipInputStream.cs b/src/ICSharpCode.SharpZipLib/GZip/GzipInputStream.cs index be10e85c9..6ecaafcea 100644 --- a/src/ICSharpCode.SharpZipLib/GZip/GzipInputStream.cs +++ b/src/ICSharpCode.SharpZipLib/GZip/GzipInputStream.cs @@ -139,6 +139,9 @@ public override int Read(byte[] buffer, int offset, int count) if (inf.IsFinished) { ReadFooter(); + } else if (inf.RemainingInput == 0) { + // If the stream is not finished but we have no more data to read, don't keep looping forever + throw new GZipException("Unexpected EOF"); } if (bytesRead > 0) diff --git a/src/ICSharpCode.SharpZipLib/Zip/Compression/DeflaterEngine.cs b/src/ICSharpCode.SharpZipLib/Zip/Compression/DeflaterEngine.cs index 973257f19..2d9b5559a 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/Compression/DeflaterEngine.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/Compression/DeflaterEngine.cs @@ -643,7 +643,7 @@ private bool DeflateStored(bool flush, bool finish) huffman.FlushStoredBlock(window, blockStart, storedLength, lastBlock); blockStart += storedLength; - return !lastBlock; + return !(lastBlock || storedLength == 0); } return true; } diff --git a/src/ICSharpCode.SharpZipLib/Zip/Compression/Streams/DeflaterOutputStream.cs b/src/ICSharpCode.SharpZipLib/Zip/Compression/Streams/DeflaterOutputStream.cs index 58bbc4c2d..b655bca19 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/Compression/Streams/DeflaterOutputStream.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/Compression/Streams/DeflaterOutputStream.cs @@ -241,7 +241,12 @@ protected void InitializeAESPassword(ZipEntry entry, string rawPassword, /// protected void Deflate() { - while (!deflater_.IsNeedingInput) + Deflate(false); + } + + private void Deflate(bool flushing) + { + while (flushing || !deflater_.IsNeedingInput) { int deflateCount = deflater_.Deflate(buffer_, 0, buffer_.Length); @@ -380,7 +385,7 @@ public override int Read(byte[] buffer, int offset, int count) public override void Flush() { deflater_.Flush(); - Deflate(); + Deflate(true); baseOutputStream_.Flush(); } diff --git a/test/ICSharpCode.SharpZipLib.Tests/GZip/GZipTests.cs b/test/ICSharpCode.SharpZipLib.Tests/GZip/GZipTests.cs index 1dbe7f9e9..ea3399ac4 100644 --- a/test/ICSharpCode.SharpZipLib.Tests/GZip/GZipTests.cs +++ b/test/ICSharpCode.SharpZipLib.Tests/GZip/GZipTests.cs @@ -289,6 +289,61 @@ public void TrailingGarbage() } } + /// + /// Test that if we flush a GZip output stream then all data that has been written + /// is flushed through to the underlying stream and can be successfully read back + /// even if the stream is not yet finished. + /// + [Test] + [Category("GZip")] + public void FlushToUnderlyingStream() + { + var ms = new MemoryStream(); + var outStream = new GZipOutputStream(ms); + + byte[] buf = new byte[100000]; + var rnd = new Random(); + rnd.NextBytes(buf); + + outStream.Write(buf, 0, buf.Length); + // Flush output stream but don't finish it yet + outStream.Flush(); + + ms.Seek(0, SeekOrigin.Begin); + + var inStream = new GZipInputStream(ms); + byte[] buf2 = new byte[buf.Length]; + int currentIndex = 0; + int count = buf2.Length; + + while (true) + { + try + { + int numRead = inStream.Read(buf2, currentIndex, count); + if (numRead <= 0) + { + break; + } + currentIndex += numRead; + count -= numRead; + } + catch (GZipException) + { + // We should get an unexpected EOF exception once we've read all + // data as the stream isn't yet finished. + break; + } + } + + Assert.AreEqual(0, count); + + for (int i = 0; i < buf.Length; ++i) + { + Assert.AreEqual(buf2[i], buf[i]); + } + } + [Test] [Category("GZip")] [Category("Performance")] From 5d020516f235bdec71fa1fea403503114ca0b097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nils=20m=C3=A5s=C3=A9n?= Date: Sat, 15 Jun 2019 22:54:32 +0200 Subject: [PATCH 07/19] Merge PR #350: Add tests and overload for ZipEntry.HostSystem --- src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs | 23 +++++++ .../Zip/ZipFileHandling.cs | 63 +++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs b/src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs index e82c6ce2f..f05e3c563 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs @@ -1839,6 +1839,29 @@ public void Add(ZipEntry entry) AddUpdate(new ZipUpdate(UpdateCommand.Add, entry)); } + /// + /// Add a with data. + /// + /// The source of the data for this entry. + /// The entry to add. + /// This can be used to add file entries with a custom data source. + public void Add(IStaticDataSource dataSource, ZipEntry entry) + { + if (entry == null) + { + throw new ArgumentNullException(nameof(entry)); + } + + if (dataSource == null) + { + throw new ArgumentNullException(nameof(dataSource)); + } + + CheckUpdating(); + + AddUpdate(new ZipUpdate(dataSource, entry)); + } + /// /// Add a directory entry to the archive. /// diff --git a/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipFileHandling.cs b/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipFileHandling.cs index e4b068c14..6d9a1ea9c 100644 --- a/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipFileHandling.cs +++ b/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipFileHandling.cs @@ -1371,6 +1371,7 @@ public void FileStreamNotClosedWhenNotOwner() /// Check that input stream is closed when construction fails and leaveOpen is false /// [Test] + [Category("Zip")] public void StreamClosedOnError() { var ms = new TrackedMemoryStream(new byte[32]); @@ -1397,6 +1398,7 @@ public void StreamClosedOnError() /// Check that input stream is not closed when construction fails and leaveOpen is true /// [Test] + [Category("Zip")] public void StreamNotClosedOnError() { var ms = new TrackedMemoryStream(new byte[32]); @@ -1418,5 +1420,66 @@ public void StreamNotClosedOnError() Assert.IsTrue(blewUp, "Should have failed to load the file"); Assert.IsFalse(ms.IsClosed, "Underlying stream should NOT be closed"); } + + [Test] + [Category("Zip")] + public void HostSystemPersistedFromOutputStream() + { + using (var ms = new MemoryStream()) + { + var fileName = "testfile"; + + using (var zos = new ZipOutputStream(ms) { IsStreamOwner = false }) + { + var source = new StringMemoryDataSource("foo"); + zos.PutNextEntry(new ZipEntry(fileName) { HostSystem = (int)HostSystemID.Unix }); + source.GetSource().CopyTo(zos); + zos.CloseEntry(); + zos.Finish(); + } + + ms.Seek(0, SeekOrigin.Begin); + + using (var zis = new ZipFile(ms)) + { + var ze = zis.GetEntry(fileName); + Assert.NotNull(ze); + + Assert.AreEqual((int)HostSystemID.Unix, ze.HostSystem); + Assert.AreEqual(ZipConstants.VersionMadeBy, ze.VersionMadeBy); + } + } + } + + [Test] + [Category("Zip")] + public void HostSystemPersistedFromZipFile() + { + using (var ms = new MemoryStream()) + { + var fileName = "testfile"; + + using (var zof = new ZipFile(ms, true)) + { + var ze = zof.EntryFactory.MakeFileEntry(fileName, false); + ze.HostSystem = (int)HostSystemID.Unix; + + zof.BeginUpdate(); + zof.Add(new StringMemoryDataSource("foo"), ze); + zof.CommitUpdate(); + } + + ms.Seek(0, SeekOrigin.Begin); + + using (var zis = new ZipFile(ms)) + { + var ze = zis.GetEntry(fileName); + Assert.NotNull(ze); + + Assert.AreEqual((int)HostSystemID.Unix, ze.HostSystem); + Assert.AreEqual(ZipConstants.VersionMadeBy, ze.VersionMadeBy); + } + } + } } } From 5cca93d1d5731021689ab02df774b350bf8e63b9 Mon Sep 17 00:00:00 2001 From: Richard Webb Date: Sat, 15 Jun 2019 22:25:21 +0100 Subject: [PATCH 08/19] Merge PR #329: Do not set the StrongEncryption flag for WinZipAes encrypted entries Change ZipEntry.ProcessAESExtraData to not set the StrongEncryption flag for WinZipAes encrypted entries. --- src/ICSharpCode.SharpZipLib/Zip/ZipEntry.cs | 5 ++- src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs | 40 ++++++++++++--------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/ICSharpCode.SharpZipLib/Zip/ZipEntry.cs b/src/ICSharpCode.SharpZipLib/Zip/ZipEntry.cs index 6986c7ec3..730d8e48c 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/ZipEntry.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/ZipEntry.cs @@ -1131,10 +1131,9 @@ private void ProcessAESExtraData(ZipExtraData extraData) { if (extraData.Find(0x9901)) { - // Set version and flag for Zipfile.CreateAndInitDecryptionStream + // Set version for Zipfile.CreateAndInitDecryptionStream versionToExtract = ZipConstants.VERSION_AES; // Ver 5.1 = AES see "Version" getter - // Set StrongEncryption flag for ZipFile.CreateAndInitDecryptionStream - Flags = Flags | (int)GeneralBitFlags.StrongEncryption; + // // Unpack AES extra data field see http://www.winzip.com/aes_info.htm int length = extraData.ValueLength; // Data size currently 7 diff --git a/src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs b/src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs index f05e3c563..3583d0ccc 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs @@ -3565,23 +3565,9 @@ private Stream CreateAndInitDecryptionStream(Stream baseStream, ZipEntry entry) { CryptoStream result = null; - if ((entry.Version < ZipConstants.VersionStrongEncryption) - || (entry.Flags & (int)GeneralBitFlags.StrongEncryption) == 0) - { - var classicManaged = new PkzipClassicManaged(); - - OnKeysRequired(entry.Name); - if (HaveKeys == false) - { - throw new ZipException("No password available for encrypted stream"); - } - - result = new CryptoStream(baseStream, classicManaged.CreateDecryptor(key, null), CryptoStreamMode.Read); - CheckClassicPassword(result, entry); - } - else + if (entry.CompressionMethodForHeader == CompressionMethod.WinZipAES) { - if (entry.Version == ZipConstants.VERSION_AES) + if (entry.Version >= ZipConstants.VERSION_AES) { // OnKeysRequired(entry.Name); @@ -3610,6 +3596,28 @@ private Stream CreateAndInitDecryptionStream(Stream baseStream, ZipEntry entry) throw new ZipException("Decryption method not supported"); } } + else + { + if ((entry.Version < ZipConstants.VersionStrongEncryption) + || (entry.Flags & (int)GeneralBitFlags.StrongEncryption) == 0) + { + var classicManaged = new PkzipClassicManaged(); + + OnKeysRequired(entry.Name); + if (HaveKeys == false) + { + throw new ZipException("No password available for encrypted stream"); + } + + result = new CryptoStream(baseStream, classicManaged.CreateDecryptor(key, null), CryptoStreamMode.Read); + CheckClassicPassword(result, entry); + } + else + { + // We don't support PKWare strong encryption + throw new ZipException("Decryption method not supported"); + } + } return result; } From 27ac139f213a8a2f723b43418130b0098b7e4080 Mon Sep 17 00:00:00 2001 From: Richard Webb Date: Sat, 15 Jun 2019 22:37:02 +0100 Subject: [PATCH 09/19] Merge PR #352: Change ZipInputStream.GetNextEntry to set the entry compression method via its constructor * Add a unit test for ZipInputStream reading an entry with an unsupported compression type * Change ZipInputStream.GetNextEntry to pass the compression method to the ZipEntry constructor. --- .../Zip/ZipInputStream.cs | 3 +- .../Zip/StreamHandling.cs | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/ICSharpCode.SharpZipLib/Zip/ZipInputStream.cs b/src/ICSharpCode.SharpZipLib/Zip/ZipInputStream.cs index fc783caa0..829cbc9b1 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/ZipInputStream.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/ZipInputStream.cs @@ -206,10 +206,9 @@ public ZipEntry GetNextEntry() string name = ZipStrings.ConvertToStringExt(flags, buffer); - entry = new ZipEntry(name, versionRequiredToExtract) + entry = new ZipEntry(name, versionRequiredToExtract, ZipConstants.VersionMadeBy, (CompressionMethod)method) { Flags = flags, - CompressionMethod = (CompressionMethod)method }; if ((flags & 8) == 0) diff --git a/test/ICSharpCode.SharpZipLib.Tests/Zip/StreamHandling.cs b/test/ICSharpCode.SharpZipLib.Tests/Zip/StreamHandling.cs index 43676f77b..a2a9f635b 100644 --- a/test/ICSharpCode.SharpZipLib.Tests/Zip/StreamHandling.cs +++ b/test/ICSharpCode.SharpZipLib.Tests/Zip/StreamHandling.cs @@ -324,5 +324,33 @@ public void SingleLargeEntry() } ); } + + const string BZip2CompressedZip = + "UEsDBC4AAAAMAEyxgU5p3ou9JwAAAAcAAAAFAAAAYS5kYXRCWmg5MUFZJlNZ0buMcAAAAkgACABA" + + "ACAAIQCCCxdyRThQkNG7jHBQSwECMwAuAAAADABMsYFOad6LvScAAAAHAAAABQAAAAAAAAAAAAAA" + + "AAAAAAAAYS5kYXRQSwUGAAAAAAEAAQAzAAAASgAAAAAA"; + + /// + /// Should fail to read a zip with BZip2 compression + /// + [Test] + [Category("Zip")] + public void ShouldReadBZip2EntryButNotDecompress() + { + var fileBytes = System.Convert.FromBase64String(BZip2CompressedZip); + + using (var input = new MemoryStream(fileBytes, false)) + { + var zis = new ZipInputStream(input); + var entry = zis.GetNextEntry(); + + Assert.That(entry.Name, Is.EqualTo("a.dat"), "Should be able to get entry name"); + Assert.That(entry.CompressionMethod, Is.EqualTo(CompressionMethod.BZip2), "Entry should be BZip2 compressed"); + Assert.That(zis.CanDecompressEntry, Is.False, "Should not be able to decompress BZip2 entry"); + + var buffer = new byte[1]; + Assert.Throws(() => zis.Read(buffer, 0, 1), "Trying to read the stream should throw"); + } + } } } From f03a2ef068977ebfaad376335a2db524278d8813 Mon Sep 17 00:00:00 2001 From: Richard Webb Date: Sun, 16 Jun 2019 10:39:54 +0100 Subject: [PATCH 10/19] Merge PR #354: Add LZMA and PPMd to the CompressionMethod enum --- src/ICSharpCode.SharpZipLib/Zip/ZipConstants.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/ICSharpCode.SharpZipLib/Zip/ZipConstants.cs b/src/ICSharpCode.SharpZipLib/Zip/ZipConstants.cs index ba9ecc010..3e34294d8 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/ZipConstants.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/ZipConstants.cs @@ -52,6 +52,16 @@ public enum CompressionMethod /// BZip2 = 12, + /// + /// LZMA compression. Not supported by #Zip. + /// + LZMA = 14, + + /// + /// PPMd compression. Not supported by #Zip. + /// + PPMd = 98, + /// /// WinZip special for AES encryption, Now supported by #Zip. /// From 6f245b357a10588f6aa0289310e39f5669483bdf Mon Sep 17 00:00:00 2001 From: Richard Webb Date: Tue, 18 Jun 2019 21:39:26 +0100 Subject: [PATCH 11/19] Merge PR #326: Dispose TarArchive output streams in case of an exception * Refactor TarArchive.ExtractEntry() to use using blocks for stream disposal * Unit test to check that TarArchive.ExtractContents() doesn't leak file handles when an exception occurs --- src/ICSharpCode.SharpZipLib/Tar/TarArchive.cs | 63 ++++++++++--------- .../Tar/TarTests.cs | 47 ++++++++++++++ 2 files changed, 79 insertions(+), 31 deletions(-) diff --git a/src/ICSharpCode.SharpZipLib/Tar/TarArchive.cs b/src/ICSharpCode.SharpZipLib/Tar/TarArchive.cs index 2161ab12a..133b33081 100644 --- a/src/ICSharpCode.SharpZipLib/Tar/TarArchive.cs +++ b/src/ICSharpCode.SharpZipLib/Tar/TarArchive.cs @@ -623,20 +623,32 @@ private void ExtractEntry(string destDir, TarEntry entry) if (process) { - bool asciiTrans = false; - - Stream outputStream = File.Create(destFile); - if (this.asciiTranslate) + using (var outputStream = File.Create(destFile)) { - asciiTrans = !IsBinary(destFile); + if (this.asciiTranslate) + { + // May need to translate the file. + ExtractAndTranslateEntry(destFile, outputStream); + } + else + { + // If translation is disabled, just copy the entry across directly. + tarIn.CopyEntryContents(outputStream); + } } + } + } + } - StreamWriter outw = null; - if (asciiTrans) - { - outw = new StreamWriter(outputStream); - } + // Extract a TAR entry, and perform an ASCII translation if required. + private void ExtractAndTranslateEntry(string destFile, Stream outputStream) + { + bool asciiTrans = !IsBinary(destFile); + if (asciiTrans) + { + using (var outw = new StreamWriter(outputStream, new UTF8Encoding(false), 1024, true)) + { byte[] rdbuf = new byte[32 * 1024]; while (true) @@ -648,34 +660,23 @@ private void ExtractEntry(string destDir, TarEntry entry) break; } - if (asciiTrans) + for (int off = 0, b = 0; b < numRead; ++b) { - for (int off = 0, b = 0; b < numRead; ++b) + if (rdbuf[b] == 10) { - if (rdbuf[b] == 10) - { - string s = Encoding.ASCII.GetString(rdbuf, off, (b - off)); - outw.WriteLine(s); - off = b + 1; - } + string s = Encoding.ASCII.GetString(rdbuf, off, (b - off)); + outw.WriteLine(s); + off = b + 1; } } - else - { - outputStream.Write(rdbuf, 0, numRead); - } - } - - if (asciiTrans) - { - outw.Dispose(); - } - else - { - outputStream.Dispose(); } } } + else + { + // No translation required. + tarIn.CopyEntryContents(outputStream); + } } /// diff --git a/test/ICSharpCode.SharpZipLib.Tests/Tar/TarTests.cs b/test/ICSharpCode.SharpZipLib.Tests/Tar/TarTests.cs index 33f9ae4df..c7945f142 100644 --- a/test/ICSharpCode.SharpZipLib.Tests/Tar/TarTests.cs +++ b/test/ICSharpCode.SharpZipLib.Tests/Tar/TarTests.cs @@ -1,4 +1,5 @@ using ICSharpCode.SharpZipLib.Tar; +using ICSharpCode.SharpZipLib.GZip; using ICSharpCode.SharpZipLib.Tests.TestSupport; using NUnit.Framework; using System; @@ -787,5 +788,51 @@ public void SingleLargeEntry() } ); } + + // Test for corruption issue described @ https://github.com/icsharpcode/SharpZipLib/issues/321 + [Test] + [Category("Tar")] + public void ExtractingCorruptTarShouldntLeakFiles() + { + using (var memoryStream = new MemoryStream()) + { + //Create a tar.gz in the output stream + using (var gzipStream = new GZipOutputStream(memoryStream)) + { + gzipStream.IsStreamOwner = false; + + using (var tarOut = TarArchive.CreateOutputTarArchive(gzipStream)) + using (var dummyFile = Utils.GetDummyFile(32000)) + { + tarOut.IsStreamOwner = false; + tarOut.WriteEntry(TarEntry.CreateEntryFromFile(dummyFile.Filename), false); + } + } + + // corrupt archive - make sure the file still has more than one block + memoryStream.SetLength(16000); + memoryStream.Seek(0, SeekOrigin.Begin); + + // try to extract + using (var gzipStream = new GZipInputStream(memoryStream)) + { + string tempDirName; + gzipStream.IsStreamOwner = false; + + using (var tempDir = new Utils.TempDir()) + { + tempDirName = tempDir.Fullpath; + + using (var tarIn = TarArchive.CreateInputTarArchive(gzipStream)) + { + tarIn.IsStreamOwner = false; + Assert.Throws(() => tarIn.ExtractContents(tempDir.Fullpath)); + } + } + + Assert.That(Directory.Exists(tempDirName), Is.False, "Temporary folder should have been removed"); + } + } + } } } From 1435683cb5bec26b8129ca462af69ec3811112b4 Mon Sep 17 00:00:00 2001 From: Richard Webb Date: Thu, 20 Jun 2019 14:39:30 +0100 Subject: [PATCH 12/19] Merge PR #359: Update test sdk and nunit packages to the curent versions --- .../ICSharpCode.SharpZipLib.TestBootstrapper.csproj | 6 +++--- .../ICSharpCode.SharpZipLib.Tests.csproj | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/ICSharpCode.SharpZipLib.TestBootstrapper/ICSharpCode.SharpZipLib.TestBootstrapper.csproj b/test/ICSharpCode.SharpZipLib.TestBootstrapper/ICSharpCode.SharpZipLib.TestBootstrapper.csproj index 44325d267..3f599186e 100644 --- a/test/ICSharpCode.SharpZipLib.TestBootstrapper/ICSharpCode.SharpZipLib.TestBootstrapper.csproj +++ b/test/ICSharpCode.SharpZipLib.TestBootstrapper/ICSharpCode.SharpZipLib.TestBootstrapper.csproj @@ -8,10 +8,10 @@ - + - - + + diff --git a/test/ICSharpCode.SharpZipLib.Tests/ICSharpCode.SharpZipLib.Tests.csproj b/test/ICSharpCode.SharpZipLib.Tests/ICSharpCode.SharpZipLib.Tests.csproj index 6a25b0ca5..324f04721 100644 --- a/test/ICSharpCode.SharpZipLib.Tests/ICSharpCode.SharpZipLib.Tests.csproj +++ b/test/ICSharpCode.SharpZipLib.Tests/ICSharpCode.SharpZipLib.Tests.csproj @@ -8,12 +8,12 @@ - + - - - - + + + + From 34d7472c4379770ec27140e902ed25b87d61c72d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nils=20m=C3=A5s=C3=A9n?= Date: Tue, 6 Aug 2019 22:47:35 +0200 Subject: [PATCH 13/19] Merge PR #371: Add test for GZip with small buffer Add test reproducing issue #360 --- .../GZip/GZipTests.cs | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/test/ICSharpCode.SharpZipLib.Tests/GZip/GZipTests.cs b/test/ICSharpCode.SharpZipLib.Tests/GZip/GZipTests.cs index ea3399ac4..a8138aa6a 100644 --- a/test/ICSharpCode.SharpZipLib.Tests/GZip/GZipTests.cs +++ b/test/ICSharpCode.SharpZipLib.Tests/GZip/GZipTests.cs @@ -344,6 +344,56 @@ public void FlushToUnderlyingStream() } } + [Test] + [Category("GZip")] + public void SmallBufferDecompression() + { + var outputBufferSize = 100000; + var inputBufferSize = outputBufferSize * 4; + + var outputBuffer = new byte[outputBufferSize]; + var inputBuffer = new byte[inputBufferSize]; + + using (var msGzip = new MemoryStream()) + { + using (var gzos = new GZipOutputStream(msGzip)) + { + gzos.IsStreamOwner = false; + + var rnd = new Random(0); + rnd.NextBytes(inputBuffer); + gzos.Write(inputBuffer, 0, inputBuffer.Length); + + gzos.Flush(); + gzos.Finish(); + } + + msGzip.Seek(0, SeekOrigin.Begin); + + + using (var gzis = new GZipInputStream(msGzip)) + using (var msRaw = new MemoryStream()) + { + + int readOut; + while ((readOut = gzis.Read(outputBuffer, 0, outputBuffer.Length)) > 0) + { + msRaw.Write(outputBuffer, 0, readOut); + } + + var resultBuffer = msRaw.ToArray(); + + for (var i = 0; i < resultBuffer.Length; i++) + { + Assert.AreEqual(inputBuffer[i], resultBuffer[i]); + } + + + } + } + + } + [Test] [Category("GZip")] [Category("Performance")] From ffe51150b130dacc4d8685b0b0e1d9e8e377515a Mon Sep 17 00:00:00 2001 From: Richard Webb Date: Thu, 8 Aug 2019 13:44:37 +0100 Subject: [PATCH 14/19] Merge PR #363: Change ZipFile.ReadEntries to always look for the Zip64 central directory * Change ZipFile.ReadEntries to always look for the Zip64 central directory * Add test for Zip64 preference --- src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs | 16 ++++++++++--- .../Zip/ZipCorruptionHandling.cs | 24 +++++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs b/src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs index 3583d0ccc..003881988 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs @@ -3408,6 +3408,7 @@ private void ReadEntries() } bool isZip64 = false; + bool requireZip64 = false; // Check if zip64 header information is required. if ((thisDiskNumber == 0xffff) || @@ -3417,13 +3418,22 @@ private void ReadEntries() (centralDirSize == 0xffffffff) || (offsetOfCentralDir == 0xffffffff)) { - isZip64 = true; + requireZip64 = true; + } - long offset = LocateBlockWithSignature(ZipConstants.Zip64CentralDirLocatorSignature, locatedEndOfCentralDir, 0, 0x1000); - if (offset < 0) + // #357 - always check for the existance of the Zip64 central directory. + long locatedZip64EndOfCentralDir = LocateBlockWithSignature(ZipConstants.Zip64CentralDirLocatorSignature, locatedEndOfCentralDir, 0, 0x1000); + if (locatedZip64EndOfCentralDir < 0) + { + if (requireZip64) { + // This is only an error in cases where the Zip64 directory is required. throw new ZipException("Cannot find Zip64 locator"); } + } + else + { + isZip64 = true; // number of the disk with the start of the zip64 end of central directory 4 bytes // relative offset of the zip64 end of central directory record 8 bytes diff --git a/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipCorruptionHandling.cs b/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipCorruptionHandling.cs index f8dccfcbb..07d192fb0 100644 --- a/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipCorruptionHandling.cs +++ b/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipCorruptionHandling.cs @@ -1,15 +1,12 @@ using System; using System.IO; -using System.Runtime.InteropServices; -using System.Text; using System.Threading; -using ICSharpCode.SharpZipLib; using ICSharpCode.SharpZipLib.Zip; using NUnit.Framework; namespace ICSharpCode.SharpZipLib.Tests.Zip { - public class ZipCorruptionHandling + public class ZipCorruptionHandling { const string TestFileZeroCodeLength = "UEsDBBQA+AAIANwyZ0U5T8HwjQAAAL8AAAAIABUAbGltZXJpY2t" + @@ -52,6 +49,23 @@ public void ZeroCodeLengthZipFile() }); } - } + const string TestFileBadCDGoodCD64 = @"UEsDBC0AAAAIANhy+k4cj+r8//////////8IABQAdGVzdGZpbGUBABAAAAA + AAAAAAAAUAAAAAAAAACtJLS5Jy8xJVUjOzytJzSsp5gIAUEsBAjMALQAAAAgA2HL6ThyP6vz//////////wgAFAAAAAAAA + AAAAAAAAAAAAHRlc3RmaWxlAQAQABIAAAAAAAAAFAAAAAAAAABQSwUGAAAAAAEAAQBKAAAATgAAAAAA"; + + [Test] + [Category("Zip")] + public void CorruptCentralDirWithCorrectZip64CD() + { + var fileBytes = Convert.FromBase64String(TestFileBadCDGoodCD64); + using (var ms = new MemoryStream(fileBytes)) + using (var zip = new ZipFile(ms)) + { + Assert.AreEqual(1, zip.Count); + Assert.AreNotEqual(0, zip[0].Size, "Uncompressed file size read from corrupt CentralDir instead of CD64"); + } + } + + } } \ No newline at end of file From 36ece7a3933ec996d23d4aa0d6ae2d965e07ec8d Mon Sep 17 00:00:00 2001 From: Richard Webb Date: Thu, 8 Aug 2019 14:30:23 +0100 Subject: [PATCH 15/19] Merge PR #369: Make the custom exception types serializable * Make all the custom exception classes serializable. * Add unit tests for exception serialization. --- .../BZip2/BZip2Exception.cs | 18 +++++ .../Core/Exceptions/SharpZipBaseException.cs | 18 +++++ .../Exceptions/StreamDecodingException.cs | 18 +++++ .../Exceptions/StreamUnsupportedException.cs | 18 +++++ .../UnexpectedEndOfStreamException.cs | 18 +++++ .../Exceptions/ValueOutOfRangeException.cs | 18 +++++ .../Core/InvalidNameException.cs | 18 +++++ .../GZip/GZipException.cs | 18 +++++ .../Lzw/LzwException.cs | 18 +++++ .../Tar/TarException.cs | 18 +++++ .../Zip/ZipException.cs | 18 +++++ .../Serialization/SerializationTests.cs | 78 +++++++++++++++++++ 12 files changed, 276 insertions(+) create mode 100644 test/ICSharpCode.SharpZipLib.Tests/Serialization/SerializationTests.cs diff --git a/src/ICSharpCode.SharpZipLib/BZip2/BZip2Exception.cs b/src/ICSharpCode.SharpZipLib/BZip2/BZip2Exception.cs index a77404dde..111d21cdc 100644 --- a/src/ICSharpCode.SharpZipLib/BZip2/BZip2Exception.cs +++ b/src/ICSharpCode.SharpZipLib/BZip2/BZip2Exception.cs @@ -1,10 +1,12 @@ using System; +using System.Runtime.Serialization; namespace ICSharpCode.SharpZipLib.BZip2 { /// /// BZip2Exception represents exceptions specific to BZip2 classes and code. /// + [Serializable] public class BZip2Exception : SharpZipBaseException { /// @@ -32,5 +34,21 @@ public BZip2Exception(string message, Exception innerException) : base(message, innerException) { } + + /// + /// Initializes a new instance of the BZip2Exception class with serialized data. + /// + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized + /// object data about the exception being thrown. + /// + /// + /// The System.Runtime.Serialization.StreamingContext that contains contextual information + /// about the source or destination. + /// + protected BZip2Exception(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } } } diff --git a/src/ICSharpCode.SharpZipLib/Core/Exceptions/SharpZipBaseException.cs b/src/ICSharpCode.SharpZipLib/Core/Exceptions/SharpZipBaseException.cs index 8ce046d9e..eb14e2d49 100644 --- a/src/ICSharpCode.SharpZipLib/Core/Exceptions/SharpZipBaseException.cs +++ b/src/ICSharpCode.SharpZipLib/Core/Exceptions/SharpZipBaseException.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.Serialization; namespace ICSharpCode.SharpZipLib { @@ -8,6 +9,7 @@ namespace ICSharpCode.SharpZipLib /// /// NOTE: Not all exceptions thrown will be derived from this class. /// A variety of other exceptions are possible for example + [Serializable] public class SharpZipBaseException : Exception { /// @@ -36,5 +38,21 @@ public SharpZipBaseException(string message, Exception innerException) : base(message, innerException) { } + + /// + /// Initializes a new instance of the SharpZipBaseException class with serialized data. + /// + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized + /// object data about the exception being thrown. + /// + /// + /// The System.Runtime.Serialization.StreamingContext that contains contextual information + /// about the source or destination. + /// + protected SharpZipBaseException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } } } diff --git a/src/ICSharpCode.SharpZipLib/Core/Exceptions/StreamDecodingException.cs b/src/ICSharpCode.SharpZipLib/Core/Exceptions/StreamDecodingException.cs index df247a6bd..389b7d065 100644 --- a/src/ICSharpCode.SharpZipLib/Core/Exceptions/StreamDecodingException.cs +++ b/src/ICSharpCode.SharpZipLib/Core/Exceptions/StreamDecodingException.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.Serialization; namespace ICSharpCode.SharpZipLib { @@ -6,6 +7,7 @@ namespace ICSharpCode.SharpZipLib /// Indicates that an error occured during decoding of a input stream due to corrupt /// data or (unintentional) library incompability. /// + [Serializable] public class StreamDecodingException : SharpZipBaseException { private const string GenericMessage = "Input stream could not be decoded"; @@ -28,5 +30,21 @@ public StreamDecodingException(string message) : base(message) { } /// A message describing the exception. /// The inner exception public StreamDecodingException(string message, Exception innerException) : base(message, innerException) { } + + /// + /// Initializes a new instance of the StreamDecodingException class with serialized data. + /// + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized + /// object data about the exception being thrown. + /// + /// + /// The System.Runtime.Serialization.StreamingContext that contains contextual information + /// about the source or destination. + /// + protected StreamDecodingException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } } } diff --git a/src/ICSharpCode.SharpZipLib/Core/Exceptions/StreamUnsupportedException.cs b/src/ICSharpCode.SharpZipLib/Core/Exceptions/StreamUnsupportedException.cs index 7fdc7a4ce..5827e559d 100644 --- a/src/ICSharpCode.SharpZipLib/Core/Exceptions/StreamUnsupportedException.cs +++ b/src/ICSharpCode.SharpZipLib/Core/Exceptions/StreamUnsupportedException.cs @@ -1,10 +1,12 @@ using System; +using System.Runtime.Serialization; namespace ICSharpCode.SharpZipLib { /// /// Indicates that the input stream could not decoded due to known library incompability or missing features /// + [Serializable] public class StreamUnsupportedException : StreamDecodingException { private const string GenericMessage = "Input stream is in a unsupported format"; @@ -27,5 +29,21 @@ public StreamUnsupportedException(string message) : base(message) { } /// A message describing the exception. /// The inner exception public StreamUnsupportedException(string message, Exception innerException) : base(message, innerException) { } + + /// + /// Initializes a new instance of the StreamUnsupportedException class with serialized data. + /// + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized + /// object data about the exception being thrown. + /// + /// + /// The System.Runtime.Serialization.StreamingContext that contains contextual information + /// about the source or destination. + /// + protected StreamUnsupportedException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } } } diff --git a/src/ICSharpCode.SharpZipLib/Core/Exceptions/UnexpectedEndOfStreamException.cs b/src/ICSharpCode.SharpZipLib/Core/Exceptions/UnexpectedEndOfStreamException.cs index fc8391851..a35c49f03 100644 --- a/src/ICSharpCode.SharpZipLib/Core/Exceptions/UnexpectedEndOfStreamException.cs +++ b/src/ICSharpCode.SharpZipLib/Core/Exceptions/UnexpectedEndOfStreamException.cs @@ -1,10 +1,12 @@ using System; +using System.Runtime.Serialization; namespace ICSharpCode.SharpZipLib { /// /// Indicates that the input stream could not decoded due to the stream ending before enough data had been provided /// + [Serializable] public class UnexpectedEndOfStreamException : StreamDecodingException { private const string GenericMessage = "Input stream ended unexpectedly"; @@ -27,5 +29,21 @@ public UnexpectedEndOfStreamException(string message) : base(message) { } /// A message describing the exception. /// The inner exception public UnexpectedEndOfStreamException(string message, Exception innerException) : base(message, innerException) { } + + /// + /// Initializes a new instance of the UnexpectedEndOfStreamException class with serialized data. + /// + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized + /// object data about the exception being thrown. + /// + /// + /// The System.Runtime.Serialization.StreamingContext that contains contextual information + /// about the source or destination. + /// + protected UnexpectedEndOfStreamException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } } } diff --git a/src/ICSharpCode.SharpZipLib/Core/Exceptions/ValueOutOfRangeException.cs b/src/ICSharpCode.SharpZipLib/Core/Exceptions/ValueOutOfRangeException.cs index 2af5d6e12..aefefb61e 100644 --- a/src/ICSharpCode.SharpZipLib/Core/Exceptions/ValueOutOfRangeException.cs +++ b/src/ICSharpCode.SharpZipLib/Core/Exceptions/ValueOutOfRangeException.cs @@ -1,10 +1,12 @@ using System; +using System.Runtime.Serialization; namespace ICSharpCode.SharpZipLib { /// /// Indicates that a value was outside of the expected range when decoding an input stream /// + [Serializable] public class ValueOutOfRangeException : StreamDecodingException { /// @@ -44,5 +46,21 @@ private ValueOutOfRangeException() private ValueOutOfRangeException(string message, Exception innerException) : base(message, innerException) { } + + /// + /// Initializes a new instance of the ValueOutOfRangeException class with serialized data. + /// + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized + /// object data about the exception being thrown. + /// + /// + /// The System.Runtime.Serialization.StreamingContext that contains contextual information + /// about the source or destination. + /// + protected ValueOutOfRangeException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } } } diff --git a/src/ICSharpCode.SharpZipLib/Core/InvalidNameException.cs b/src/ICSharpCode.SharpZipLib/Core/InvalidNameException.cs index 13abfd2f2..6647631bd 100644 --- a/src/ICSharpCode.SharpZipLib/Core/InvalidNameException.cs +++ b/src/ICSharpCode.SharpZipLib/Core/InvalidNameException.cs @@ -1,10 +1,12 @@ using System; +using System.Runtime.Serialization; namespace ICSharpCode.SharpZipLib.Core { /// /// InvalidNameException is thrown for invalid names such as directory traversal paths and names with invalid characters /// + [Serializable] public class InvalidNameException : SharpZipBaseException { /// @@ -31,5 +33,21 @@ public InvalidNameException(string message) : base(message) public InvalidNameException(string message, Exception innerException) : base(message, innerException) { } + + /// + /// Initializes a new instance of the InvalidNameException class with serialized data. + /// + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized + /// object data about the exception being thrown. + /// + /// + /// The System.Runtime.Serialization.StreamingContext that contains contextual information + /// about the source or destination. + /// + protected InvalidNameException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } } } diff --git a/src/ICSharpCode.SharpZipLib/GZip/GZipException.cs b/src/ICSharpCode.SharpZipLib/GZip/GZipException.cs index 1a5952991..a0ec6bb51 100644 --- a/src/ICSharpCode.SharpZipLib/GZip/GZipException.cs +++ b/src/ICSharpCode.SharpZipLib/GZip/GZipException.cs @@ -1,10 +1,12 @@ using System; +using System.Runtime.Serialization; namespace ICSharpCode.SharpZipLib.GZip { /// /// GZipException represents exceptions specific to GZip classes and code. /// + [Serializable] public class GZipException : SharpZipBaseException { /// @@ -32,5 +34,21 @@ public GZipException(string message, Exception innerException) : base(message, innerException) { } + + /// + /// Initializes a new instance of the GZipException class with serialized data. + /// + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized + /// object data about the exception being thrown. + /// + /// + /// The System.Runtime.Serialization.StreamingContext that contains contextual information + /// about the source or destination. + /// + protected GZipException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } } } diff --git a/src/ICSharpCode.SharpZipLib/Lzw/LzwException.cs b/src/ICSharpCode.SharpZipLib/Lzw/LzwException.cs index 3bc0cb212..1d5c44c3c 100644 --- a/src/ICSharpCode.SharpZipLib/Lzw/LzwException.cs +++ b/src/ICSharpCode.SharpZipLib/Lzw/LzwException.cs @@ -1,10 +1,12 @@ using System; +using System.Runtime.Serialization; namespace ICSharpCode.SharpZipLib.Lzw { /// /// LzwException represents exceptions specific to LZW classes and code. /// + [Serializable] public class LzwException : SharpZipBaseException { /// @@ -32,5 +34,21 @@ public LzwException(string message, Exception innerException) : base(message, innerException) { } + + /// + /// Initializes a new instance of the LzwException class with serialized data. + /// + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized + /// object data about the exception being thrown. + /// + /// + /// The System.Runtime.Serialization.StreamingContext that contains contextual information + /// about the source or destination. + /// + protected LzwException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } } } diff --git a/src/ICSharpCode.SharpZipLib/Tar/TarException.cs b/src/ICSharpCode.SharpZipLib/Tar/TarException.cs index c24011c79..9d448ca7d 100644 --- a/src/ICSharpCode.SharpZipLib/Tar/TarException.cs +++ b/src/ICSharpCode.SharpZipLib/Tar/TarException.cs @@ -1,10 +1,12 @@ using System; +using System.Runtime.Serialization; namespace ICSharpCode.SharpZipLib.Tar { /// /// TarException represents exceptions specific to Tar classes and code. /// + [Serializable] public class TarException : SharpZipBaseException { /// @@ -32,5 +34,21 @@ public TarException(string message, Exception innerException) : base(message, innerException) { } + + /// + /// Initializes a new instance of the TarException class with serialized data. + /// + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized + /// object data about the exception being thrown. + /// + /// + /// The System.Runtime.Serialization.StreamingContext that contains contextual information + /// about the source or destination. + /// + protected TarException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } } } diff --git a/src/ICSharpCode.SharpZipLib/Zip/ZipException.cs b/src/ICSharpCode.SharpZipLib/Zip/ZipException.cs index 28843883e..ef8142b65 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/ZipException.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/ZipException.cs @@ -1,10 +1,12 @@ using System; +using System.Runtime.Serialization; namespace ICSharpCode.SharpZipLib.Zip { /// /// ZipException represents exceptions specific to Zip classes and code. /// + [Serializable] public class ZipException : SharpZipBaseException { /// @@ -32,5 +34,21 @@ public ZipException(string message, Exception innerException) : base(message, innerException) { } + + /// + /// Initializes a new instance of the ZipException class with serialized data. + /// + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized + /// object data about the exception being thrown. + /// + /// + /// The System.Runtime.Serialization.StreamingContext that contains contextual information + /// about the source or destination. + /// + protected ZipException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } } } diff --git a/test/ICSharpCode.SharpZipLib.Tests/Serialization/SerializationTests.cs b/test/ICSharpCode.SharpZipLib.Tests/Serialization/SerializationTests.cs new file mode 100644 index 000000000..6118022e4 --- /dev/null +++ b/test/ICSharpCode.SharpZipLib.Tests/Serialization/SerializationTests.cs @@ -0,0 +1,78 @@ +using System; +using System.IO; +using System.Runtime.Serialization; + +using NUnit.Framework; +using ICSharpCode.SharpZipLib.BZip2; +using ICSharpCode.SharpZipLib.Core; +using ICSharpCode.SharpZipLib.GZip; +using ICSharpCode.SharpZipLib.Lzw; +using ICSharpCode.SharpZipLib.Tar; +using ICSharpCode.SharpZipLib.Zip; + +namespace ICSharpCode.SharpZipLib.Tests.Serialization +{ + [TestFixture] + public class SerializationTests + { + /// + /// Test that SharpZipLib Custom Exceptions can be serialized. + /// + [Test] + [Category("Core")] + [Category("Serialization")] + [TestCase(typeof(BZip2Exception))] + [TestCase(typeof(GZipException))] + [TestCase(typeof(InvalidNameException))] + [TestCase(typeof(LzwException))] + [TestCase(typeof(SharpZipBaseException))] + [TestCase(typeof(StreamDecodingException))] + [TestCase(typeof(StreamUnsupportedException))] + [TestCase(typeof(TarException))] + [TestCase(typeof(UnexpectedEndOfStreamException))] + [TestCase(typeof(ZipException))] + public void SerializeException(Type exceptionType) + { + string message = $"Serialized {exceptionType.Name}"; + var exception = Activator.CreateInstance(exceptionType, message); + + var deserializedException = ExceptionSerialiseHelper(exception, exceptionType) as Exception; + Assert.That(deserializedException, Is.InstanceOf(exceptionType), "deserialized object should have the correct type"); + Assert.That(deserializedException.Message, Is.EqualTo(message), "deserialized message should match original message"); + } + + /// + /// Test that ValueOutOfRangeException can be serialized. + /// + [Test] + [Category("Core")] + [Category("Serialization")] + public void SerializeValueOutOfRangeException() + { + string message = "Serialized ValueOutOfRangeException"; + var exception = new ValueOutOfRangeException(message); + + var deserializedException = ExceptionSerialiseHelper(exception, typeof(ValueOutOfRangeException)) as ValueOutOfRangeException; + + // ValueOutOfRangeException appends 'out of range' to the end of the message + Assert.That(deserializedException.Message, Is.EqualTo($"{message} out of range"), "should have expected message"); + } + + // Shared serialization helper + // round trips the specified exception using DataContractSerializer + private static object ExceptionSerialiseHelper(object exception, Type exceptionType) + { + DataContractSerializer ser = new DataContractSerializer(exceptionType); + + using (var memoryStream = new MemoryStream()) + { + ser.WriteObject(memoryStream, exception); + + memoryStream.Seek(0, loc: SeekOrigin.Begin); + + return ser.ReadObject(memoryStream); + } + } + } +} + From 805dd79551fd657cea287fdbb2ab2905d58af195 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nils=20m=C3=A5s=C3=A9n?= Date: Fri, 9 Aug 2019 10:31:06 +0200 Subject: [PATCH 16/19] Merge PR #372: Remove invalid exception and handle 0 reads --- src/ICSharpCode.SharpZipLib/GZip/GzipInputStream.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ICSharpCode.SharpZipLib/GZip/GzipInputStream.cs b/src/ICSharpCode.SharpZipLib/GZip/GzipInputStream.cs index 6ecaafcea..a924a7ffc 100644 --- a/src/ICSharpCode.SharpZipLib/GZip/GzipInputStream.cs +++ b/src/ICSharpCode.SharpZipLib/GZip/GzipInputStream.cs @@ -139,12 +139,10 @@ public override int Read(byte[] buffer, int offset, int count) if (inf.IsFinished) { ReadFooter(); - } else if (inf.RemainingInput == 0) { - // If the stream is not finished but we have no more data to read, don't keep looping forever - throw new GZipException("Unexpected EOF"); } - if (bytesRead > 0) + // Attempting to read 0 bytes will never yield any bytesRead, so we return instead of looping forever + if (bytesRead > 0 || count == 0) { return bytesRead; } From 56cbe991dfbc629a155c6739501a658cf2f75cff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nils=20m=C3=A5s=C3=A9n?= Date: Fri, 9 Aug 2019 10:32:42 +0200 Subject: [PATCH 17/19] Update csproj for v1.2 --- .../ICSharpCode.SharpZipLib.csproj | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ICSharpCode.SharpZipLib/ICSharpCode.SharpZipLib.csproj b/src/ICSharpCode.SharpZipLib/ICSharpCode.SharpZipLib.csproj index 1114826b4..ae6959d55 100644 --- a/src/ICSharpCode.SharpZipLib/ICSharpCode.SharpZipLib.csproj +++ b/src/ICSharpCode.SharpZipLib/ICSharpCode.SharpZipLib.csproj @@ -10,9 +10,9 @@ - 1.1.0.6 - 1.1.0.6 - 1.1.0 + 1.2.0.7 + 1.2.0.7 + 1.2.0 SharpZipLib ICSharpCode ICSharpCode @@ -21,11 +21,11 @@ http://icsharpcode.github.io/SharpZipLib/ http://icsharpcode.github.io/SharpZipLib/assets/sharpziplib-nuget-256x256.png https://github.com/icsharpcode/SharpZipLib - Copyright © 2000-2018 SharpZipLib Contributors + Copyright © 2000-2019 SharpZipLib Contributors Compression Library Zip GZip BZip2 LZW Tar en-US -Please see https://github.com/icsharpcode/SharpZipLib/wiki/Release-1.1 for more information. +Please see https://github.com/icsharpcode/SharpZipLib/wiki/Release-1.2 for more information. https://github.com/icsharpcode/SharpZipLib @@ -46,7 +46,7 @@ Please see https://github.com/icsharpcode/SharpZipLib/wiki/Release-1.1 for more - + From 414b8c15f5700b22b9a716fa30480c570ad60ab0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nils=20m=C3=A5s=C3=A9n?= Date: Sun, 11 Aug 2019 13:09:17 +0200 Subject: [PATCH 18/19] Merge PR #374: Update Docs generation * Fix docfx metadata generation * Move CI scripts to separate files and add docfx generation * Fix typos and add more output * Fix directory traversal for docfx and change output colors * Fix swapped output logic * Add basic docfx template * Update docfx.json to use template * Update year and repo URL --- .gitignore | 1 + appveyor.yml | 9 ++++- docs/help/docfx.json | 16 ++++++--- docs/help/template/logo.svg | 34 +++++++++++++++++++ .../template/partials/navbar.tmpl.partial | 20 +++++++++++ docs/help/template/styles/main.css | 17 ++++++++++ tools/appveyor-docfx-build.ps1 | 24 +++++++++++++ tools/appveyor-docfx-init.ps1 | 5 +++ 8 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 docs/help/template/logo.svg create mode 100644 docs/help/template/partials/navbar.tmpl.partial create mode 100644 docs/help/template/styles/main.css create mode 100644 tools/appveyor-docfx-build.ps1 create mode 100644 tools/appveyor-docfx-init.ps1 diff --git a/.gitignore b/.gitignore index 045735d39..7bc034335 100644 --- a/.gitignore +++ b/.gitignore @@ -252,3 +252,4 @@ paket-files/ *.sln.iml /test/ICSharpCode.SharpZipLib.TestBootstrapper/Properties/launchSettings.json _testRunner/ +docs/help/api/.manifest diff --git a/appveyor.yml b/appveyor.yml index d4cc936be..a6197ff29 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -18,10 +18,17 @@ nuget: disable_publish_on_pr: true before_build: - ps: nuget restore ICSharpCode.SharpZipLib.sln +- ps: .\tools\appveyor-docfx-init.ps1 build: project: ICSharpCode.SharpZipLib.sln publish_nuget: true publish_nuget_symbols: true verbosity: normal +after_build: +- ps: .\tools\appveyor-docfx-build.ps1 test_script: -- ps: tools/appveyor-test.ps1 +- ps: tools\appveyor-test.ps1 +artifacts: +- path: docs\help\_site + type: zip + name: Documentation \ No newline at end of file diff --git a/docs/help/docfx.json b/docs/help/docfx.json index 173bb4360..b2123cfa2 100644 --- a/docs/help/docfx.json +++ b/docs/help/docfx.json @@ -14,7 +14,10 @@ ] } ], - "dest": "api" + "dest": "api", + "properties": { + "TargetFramework": "NETSTANDARD2" + } } ], "build": { @@ -62,17 +65,22 @@ ], "globalMetadata": { "_appTitle": "SharpZipLib Help", - "_appFooter": "Copyright © 2000-2017 SharpZipLib Contributors" + "_appFooter": "Copyright © 2000-2019 SharpZipLib Contributors", + "_gitContribute": { + "repo": "https://github.com/icsharpcode/SharpZipLib", + "branch": "master" + } }, "dest": "_site", "globalMetadataFiles": [], "fileMetadataFiles": [], "template": [ - "default" + "default", + "template" ], "postProcessors": [], "noLangKeyword": false, "keepFileLink": false, "cleanupCacheHistory": false } -} \ No newline at end of file +} diff --git a/docs/help/template/logo.svg b/docs/help/template/logo.svg new file mode 100644 index 000000000..f7d5d1069 --- /dev/null +++ b/docs/help/template/logo.svg @@ -0,0 +1,34 @@ + + + + + diff --git a/docs/help/template/partials/navbar.tmpl.partial b/docs/help/template/partials/navbar.tmpl.partial new file mode 100644 index 000000000..5f9bb9242 --- /dev/null +++ b/docs/help/template/partials/navbar.tmpl.partial @@ -0,0 +1,20 @@ + \ No newline at end of file diff --git a/docs/help/template/styles/main.css b/docs/help/template/styles/main.css new file mode 100644 index 000000000..9002a4323 --- /dev/null +++ b/docs/help/template/styles/main.css @@ -0,0 +1,17 @@ +button, a { + color: #516d86; + cursor: pointer; +} + +.navbar-default .navbar-nav>.active>a, .navbar-default .navbar-nav>.active>a:focus, .navbar-default .navbar-nav>.active>a:hover { + color: #181c20; + background-color: #c4cdd4; +} + +svg:hover path { + fill: currentColor; +} + +.navbar-default .navbar-brand:focus, .navbar-default .navbar-brand:hover { + color: #3d4852; +} \ No newline at end of file diff --git a/tools/appveyor-docfx-build.ps1 b/tools/appveyor-docfx-build.ps1 new file mode 100644 index 000000000..d5db36dfa --- /dev/null +++ b/tools/appveyor-docfx-build.ps1 @@ -0,0 +1,24 @@ +if(-Not $env:APPVEYOR_PULL_REQUEST_TITLE -and $env:CONFIGURATION -eq "Release") +{ + pushd docs\help + # & docfx metadata + & docfx docfx.json + if ($lastexitcode -ne 0){ + throw [System.Exception] "docfx build failed with exit code $lastexitcode." + } + popd +<# + ## Useful for automatically updating gh pages: + + git config --global credential.helper store + Add-Content "$env:USERPROFILE\.git-credentials" "https://$($env:access_token):x-oauth-basic@github.com`n" + git config --global user.email $env:op_build_user_email + git config --global user.name $env:op_build_user + git clone https://github.com/ICSharpCode/SharpZipLib.git -b gh-pages origin_site -q + Copy-Item origin_site/.git _site -recurse + CD _site + git add -A 2>&1 + git commit -m "CI Updates" -q + git push origin gh-pages -q +#> +} \ No newline at end of file diff --git a/tools/appveyor-docfx-init.ps1 b/tools/appveyor-docfx-init.ps1 new file mode 100644 index 000000000..a5a40d6fa --- /dev/null +++ b/tools/appveyor-docfx-init.ps1 @@ -0,0 +1,5 @@ +if(-Not $env:APPVEYOR_PULL_REQUEST_TITLE -and $env:CONFIGURATION -eq "Release") +{ + git checkout $env:APPVEYOR_REPO_BRANCH -q + choco install docfx -y +} \ No newline at end of file From ab7f8c5b24c12e0656a9b8e9be55b29aeecc5593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nils=20ma=CC=8Ase=CC=81n?= Date: Sun, 11 Aug 2019 13:35:57 +0200 Subject: [PATCH 19/19] Fix API docs styling --- docs/help/api/toc.yml | 14 ++++++++++++++ docs/help/template/logo.svg | 8 ++++---- docs/help/template/styles/main.css | 20 +++++++++++++++++++- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/docs/help/api/toc.yml b/docs/help/api/toc.yml index 2cc204bde..a56fa8ddf 100644 --- a/docs/help/api/toc.yml +++ b/docs/help/api/toc.yml @@ -4,6 +4,14 @@ items: - uid: ICSharpCode.SharpZipLib.SharpZipBaseException name: SharpZipBaseException + - uid: ICSharpCode.SharpZipLib.StreamDecodingException + name: StreamDecodingException + - uid: ICSharpCode.SharpZipLib.StreamUnsupportedException + name: StreamUnsupportedException + - uid: ICSharpCode.SharpZipLib.UnexpectedEndOfStreamException + name: UnexpectedEndOfStreamException + - uid: ICSharpCode.SharpZipLib.ValueOutOfRangeException + name: ValueOutOfRangeException - uid: ICSharpCode.SharpZipLib.BZip2 name: ICSharpCode.SharpZipLib.BZip2 items: @@ -43,6 +51,8 @@ name: FileSystemScanner - uid: ICSharpCode.SharpZipLib.Core.INameTransform name: INameTransform + - uid: ICSharpCode.SharpZipLib.Core.InvalidNameException + name: InvalidNameException - uid: ICSharpCode.SharpZipLib.Core.IScanFilter name: IScanFilter - uid: ICSharpCode.SharpZipLib.Core.NameAndSizeFilter @@ -109,6 +119,8 @@ name: TarEntry - uid: ICSharpCode.SharpZipLib.Tar.TarException name: TarException + - uid: ICSharpCode.SharpZipLib.Tar.TarExtendedHeaderReader + name: TarExtendedHeaderReader - uid: ICSharpCode.SharpZipLib.Tar.TarHeader name: TarHeader - uid: ICSharpCode.SharpZipLib.Tar.TarInputStream @@ -204,6 +216,8 @@ name: ZipNameTransform - uid: ICSharpCode.SharpZipLib.Zip.ZipOutputStream name: ZipOutputStream + - uid: ICSharpCode.SharpZipLib.Zip.ZipStrings + name: ZipStrings - uid: ICSharpCode.SharpZipLib.Zip.ZipTestResultHandler name: ZipTestResultHandler - uid: ICSharpCode.SharpZipLib.Zip.Compression diff --git a/docs/help/template/logo.svg b/docs/help/template/logo.svg index f7d5d1069..25535c631 100644 --- a/docs/help/template/logo.svg +++ b/docs/help/template/logo.svg @@ -3,11 +3,11 @@ diff --git a/docs/help/template/styles/main.css b/docs/help/template/styles/main.css index 9002a4323..db19bb18a 100644 --- a/docs/help/template/styles/main.css +++ b/docs/help/template/styles/main.css @@ -3,6 +3,15 @@ button, a { cursor: pointer; } +.affix > ul > li.active > a, .affix > ul > li.active > a:before, +.affix ul > li.active > a, .affix ul > li.active > a:before { + color: #516d86; +} + +.toc .nav > li.active > a { + color: #7391ab; +} + .navbar-default .navbar-nav>.active>a, .navbar-default .navbar-nav>.active>a:focus, .navbar-default .navbar-nav>.active>a:hover { color: #181c20; background-color: #c4cdd4; @@ -14,4 +23,13 @@ svg:hover path { .navbar-default .navbar-brand:focus, .navbar-default .navbar-brand:hover { color: #3d4852; -} \ No newline at end of file +} + +#toc .level1 > li { + font-weight: normal; + font-size: 14px; +} + +#sidetoc .sidetoc.shiftup { + bottom: 50px; +}