From e7a038a6c33ba1667b41dab0afbd85af1bc0a396 Mon Sep 17 00:00:00 2001 From: tanyarybina Date: Wed, 18 Oct 2023 22:59:02 +0300 Subject: [PATCH 1/6] add stream tests classes --- .../Storages/Azure/AzureBlobStreamTests.cs | 6 ++++++ .../Storages/StreamTests.cs | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs create mode 100644 ManagedCode.Storage.Tests/Storages/StreamTests.cs diff --git a/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs b/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs new file mode 100644 index 00000000..6bfeac0d --- /dev/null +++ b/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs @@ -0,0 +1,6 @@ +namespace ManagedCode.Storage.Tests.Storages.Azure; + +public class AzureBlobStreamTests +{ + +} \ No newline at end of file diff --git a/ManagedCode.Storage.Tests/Storages/StreamTests.cs b/ManagedCode.Storage.Tests/Storages/StreamTests.cs new file mode 100644 index 00000000..1e74e683 --- /dev/null +++ b/ManagedCode.Storage.Tests/Storages/StreamTests.cs @@ -0,0 +1,16 @@ +using System.Threading.Tasks; +using DotNet.Testcontainers.Containers; +using ManagedCode.Storage.Tests.Common; +using Xunit; + +namespace ManagedCode.Storage.Tests.Storages; + +public abstract class StreamTests : BaseContainer + where T : IContainer +{ + [Fact] + public async Task ReadStream_WhenFileExists_ReturnData() + { + + } +} \ No newline at end of file From 8ffc6cc06a8a036bca52325c18de7ad8792ac8ac Mon Sep 17 00:00:00 2001 From: TRybina132 Date: Thu, 19 Oct 2023 13:29:52 +0300 Subject: [PATCH 2/6] fix azure blob stream lenght --- ManagedCode.Storage.Azure/BlobStream.cs | 6 ++- .../Common/FileHelper.cs | 28 ++++++++++ .../Storages/Azure/AzureBlobStreamTests.cs | 54 ++++++++++++++++++- .../Storages/StreamTests.cs | 6 +-- 4 files changed, 86 insertions(+), 8 deletions(-) diff --git a/ManagedCode.Storage.Azure/BlobStream.cs b/ManagedCode.Storage.Azure/BlobStream.cs index f7d46b9c..d50a5c95 100644 --- a/ManagedCode.Storage.Azure/BlobStream.cs +++ b/ManagedCode.Storage.Azure/BlobStream.cs @@ -8,6 +8,7 @@ namespace ManagedCode.Storage.Azure; public class BlobStream : Stream { + // TODO: add same thing to google and aws private const string MetadataLengthKey = "STREAM_LENGTH"; private const int PageSizeInBytes = 512; public const int DefaultBufferSize = 1024 * 1024 * 4; @@ -39,6 +40,7 @@ public override long Length { var realLength = 0L; var metadata = _pageBlob.GetProperties().Value.Metadata; + var contentLenght = _pageBlob.GetProperties().Value.ContentLength; if (metadata.TryGetValue(MetadataLengthKey, out var length)) { if (long.TryParse(length, out realLength)) @@ -47,8 +49,8 @@ public override long Length } } - SetLengthInternal(realLength); - return realLength; + SetLengthInternal(contentLenght); + return contentLenght; } } diff --git a/ManagedCode.Storage.Tests/Common/FileHelper.cs b/ManagedCode.Storage.Tests/Common/FileHelper.cs index 4d5c8c7f..34df22be 100644 --- a/ManagedCode.Storage.Tests/Common/FileHelper.cs +++ b/ManagedCode.Storage.Tests/Common/FileHelper.cs @@ -25,6 +25,34 @@ public static LocalFile GenerateLocalFile(string fileName, int byteSize) return localFile; } + public static LocalFile GenerateLocalFileWithData(LocalFile file, int sizeInBytes) + { + using (var fileStream = file.FileStream) + { + Random random = new Random(); + byte[] buffer = new byte[1024]; // Buffer for writing in chunks + + while (sizeInBytes > 0) + { + int bytesToWrite = (int) Math.Min(sizeInBytes, buffer.Length); + + for (int i = 0; i < bytesToWrite; i++) + { + buffer[i] = (byte) random.Next(65, 91); // 'A' to 'Z' + if (random.Next(2) == 0) + { + buffer[i] = (byte) random.Next(97, 123); // 'a' to 'z' + } + } + + fileStream.Write(buffer, 0, bytesToWrite); + sizeInBytes -= bytesToWrite; + } + } + + return file; + } + public static IFormFile GenerateFormFile(string fileName, int byteSize) { var localFile = GenerateLocalFile(fileName, byteSize); diff --git a/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs b/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs index 6bfeac0d..344f0b07 100644 --- a/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs +++ b/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs @@ -1,6 +1,58 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using ManagedCode.Storage.Azure; +using ManagedCode.Storage.Core.Models; +using ManagedCode.Storage.Tests.Common; +using Microsoft.Extensions.DependencyInjection; +using Testcontainers.Azurite; +using Xunit; + namespace ManagedCode.Storage.Tests.Storages.Azure; -public class AzureBlobStreamTests +public class AzureBlobStreamTests : StreamTests { + protected override AzuriteContainer Build() + { + return new AzuriteBuilder() + .WithImage("mcr.microsoft.com/azure-storage/azurite:3.26.0") + .Build(); + } + + protected override ServiceProvider ConfigureServices() + { + return AzureConfigurator.ConfigureServices(Container.GetConnectionString()); + } + [Fact] + public async Task ReadStream_WhenFileExists_ReturnData() + { + // Arrange + var directory = "test-directory"; + var fileName = Guid.NewGuid().ToString(); + await using var localFile = LocalFile.FromRandomNameWithExtension(".txt"); + FileHelper.GenerateLocalFileWithData(localFile, 10); + var storage = (IAzureStorage) Storage; + + UploadOptions options = new() { FileName = localFile.Name, Directory = directory }; + var result = await storage.UploadAsync(localFile.FileInfo.OpenRead(), options); + + await using var blobStream = storage.GetBlobStream(result.Value.FullName); + + var chunkSize = localFile.FileInfo.Length; + byte[] fileChunk1 = new byte[chunkSize]; + byte[] fileChunk2 = new byte[chunkSize]; + + // Act + using var streamReader = new StreamReader(blobStream); + var content = await streamReader.ReadToEndAsync(); + + // Assert + content.Should().NotBeNullOrEmpty(); + using var fileReader = new StreamReader(localFile.FileStream); + var fileContent = await fileReader.ReadToEndAsync(); + fileContent.Should().NotBeNullOrEmpty(); + content.Should().Be(fileContent); + } } \ No newline at end of file diff --git a/ManagedCode.Storage.Tests/Storages/StreamTests.cs b/ManagedCode.Storage.Tests/Storages/StreamTests.cs index 1e74e683..ae4f8de1 100644 --- a/ManagedCode.Storage.Tests/Storages/StreamTests.cs +++ b/ManagedCode.Storage.Tests/Storages/StreamTests.cs @@ -8,9 +8,5 @@ namespace ManagedCode.Storage.Tests.Storages; public abstract class StreamTests : BaseContainer where T : IContainer { - [Fact] - public async Task ReadStream_WhenFileExists_ReturnData() - { - - } + } \ No newline at end of file From 714a842c4a495ff9f50ced66a08a079d9d507c0c Mon Sep 17 00:00:00 2001 From: TRybina132 Date: Thu, 19 Oct 2023 14:41:07 +0300 Subject: [PATCH 3/6] fix tests --- ManagedCode.Storage.Azure/BlobStream.cs | 8 ++-- .../Storages/Azure/AzureBlobStreamTests.cs | 45 +++++++++++++++---- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/ManagedCode.Storage.Azure/BlobStream.cs b/ManagedCode.Storage.Azure/BlobStream.cs index d50a5c95..7cfe6b09 100644 --- a/ManagedCode.Storage.Azure/BlobStream.cs +++ b/ManagedCode.Storage.Azure/BlobStream.cs @@ -8,7 +8,6 @@ namespace ManagedCode.Storage.Azure; public class BlobStream : Stream { - // TODO: add same thing to google and aws private const string MetadataLengthKey = "STREAM_LENGTH"; private const int PageSizeInBytes = 512; public const int DefaultBufferSize = 1024 * 1024 * 4; @@ -38,17 +37,16 @@ public override long Length { get { - var realLength = 0L; var metadata = _pageBlob.GetProperties().Value.Metadata; - var contentLenght = _pageBlob.GetProperties().Value.ContentLength; if (metadata.TryGetValue(MetadataLengthKey, out var length)) { - if (long.TryParse(length, out realLength)) + if (long.TryParse(length, out var realLength)) { return realLength; } } - + + var contentLenght = _pageBlob.GetProperties().Value.ContentLength; SetLengthInternal(contentLenght); return contentLenght; } diff --git a/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs b/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs index 344f0b07..a8d704d9 100644 --- a/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs +++ b/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs @@ -26,33 +26,60 @@ protected override ServiceProvider ConfigureServices() } [Fact] - public async Task ReadStream_WhenFileExists_ReturnData() + public async Task ReadStreamWithStreamReader_WhenFileExists_ReturnData() { // Arrange var directory = "test-directory"; - var fileName = Guid.NewGuid().ToString(); await using var localFile = LocalFile.FromRandomNameWithExtension(".txt"); FileHelper.GenerateLocalFileWithData(localFile, 10); var storage = (IAzureStorage) Storage; UploadOptions options = new() { FileName = localFile.Name, Directory = directory }; - var result = await storage.UploadAsync(localFile.FileInfo.OpenRead(), options); + await using var localFileStream = localFile.FileInfo.OpenRead(); + var result = await storage.UploadAsync(localFileStream, options); await using var blobStream = storage.GetBlobStream(result.Value.FullName); - - var chunkSize = localFile.FileInfo.Length; - byte[] fileChunk1 = new byte[chunkSize]; - byte[] fileChunk2 = new byte[chunkSize]; // Act using var streamReader = new StreamReader(blobStream); var content = await streamReader.ReadToEndAsync(); // Assert - content.Should().NotBeNullOrEmpty(); - using var fileReader = new StreamReader(localFile.FileStream); + await using var fileStream = localFile.FileInfo.OpenRead(); + using var fileReader = new StreamReader(fileStream); var fileContent = await fileReader.ReadToEndAsync(); + content.Should().NotBeNullOrEmpty(); fileContent.Should().NotBeNullOrEmpty(); content.Should().Be(fileContent); } + + [Fact] + public async Task ReadStream_WhenFileExists_ReturnData() + { + // Arrange + var directory = "test-directory"; + var localFile = LocalFile.FromRandomNameWithExtension(".txt"); + FileHelper.GenerateLocalFileWithData(localFile, 10); + var storage = (IAzureStorage) Storage; + + UploadOptions options = new() { FileName = localFile.Name, Directory = directory }; + await using var fileStream = localFile.FileInfo.OpenRead(); + var result = await storage.UploadAsync(fileStream, options); + + await using var blobStream = storage.GetBlobStream(result.Value.FullName); + + var chunkSize = (int) blobStream.Length / 2; + var chunk1 = new byte[chunkSize]; + var chunk2 = new byte[chunkSize]; + + // Act + var bytesReadForChunk1 = await blobStream.ReadAsync(chunk1, 0, chunkSize); + var bytesReadForChunk2 = await blobStream.ReadAsync(chunk2, 0, chunkSize); + + // Assert + bytesReadForChunk1.Should().Be(chunkSize); + bytesReadForChunk2.Should().Be(chunkSize); + chunk1.Should().NotBeNullOrEmpty().And.HaveCount(chunkSize); + chunk2.Should().NotBeNullOrEmpty().And.HaveCount(chunkSize); + } } \ No newline at end of file From a7523bd6854b44e2f4890b5a5ecf662b7c5f8e5e Mon Sep 17 00:00:00 2001 From: TRybina132 Date: Thu, 19 Oct 2023 17:32:15 +0300 Subject: [PATCH 4/6] add write test for azure blob --- .../Storages/Azure/AzureBlobStreamTests.cs | 56 ++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs b/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs index a8d704d9..4e560e0d 100644 --- a/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs +++ b/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs @@ -4,6 +4,7 @@ using FluentAssertions; using ManagedCode.Storage.Azure; using ManagedCode.Storage.Core.Models; +using ManagedCode.Storage.Server; using ManagedCode.Storage.Tests.Common; using Microsoft.Extensions.DependencyInjection; using Testcontainers.Azurite; @@ -58,7 +59,7 @@ public async Task ReadStream_WhenFileExists_ReturnData() { // Arrange var directory = "test-directory"; - var localFile = LocalFile.FromRandomNameWithExtension(".txt"); + await using var localFile = LocalFile.FromRandomNameWithExtension(".txt"); FileHelper.GenerateLocalFileWithData(localFile, 10); var storage = (IAzureStorage) Storage; @@ -82,4 +83,57 @@ public async Task ReadStream_WhenFileExists_ReturnData() chunk1.Should().NotBeNullOrEmpty().And.HaveCount(chunkSize); chunk2.Should().NotBeNullOrEmpty().And.HaveCount(chunkSize); } + + [Fact] + public async Task ReadStream_WhenFileDoesNotExists_ReturnNoData() + { + // Arrange + var directory = "test-directory"; + var storage = (IAzureStorage) Storage; + await storage.CreateContainerAsync(); + var fullFileName = $"{directory}/{Guid.NewGuid()}.txt"; + + await using var blobStream = storage.GetBlobStream(fullFileName); + var chunk = new byte[4]; + + // Act + var bytesRead = await blobStream.ReadAsync(chunk, 0, 4); + + // Assert + bytesRead.Should().Be(0); + chunk.Should().NotBeNullOrEmpty(); + chunk.Should().AllBeEquivalentTo(0); + } + + [Fact] + public async Task WriteStreamWithStreamWriter_WhenFileExists_SaveData() + { + // Arrange + var directory = "test-directory"; + await using var localFile = LocalFile.FromRandomNameWithExtension(".txt"); + FileHelper.GenerateLocalFileWithData(localFile, 512); + var fullFileName = $"{directory}/{localFile.FileInfo.FullName}"; + + var storage = (IAzureStorage) Storage; + + await storage.CreateContainerAsync(); + + // Act + await using (var blobStream = storage.GetBlobStream(fullFileName)) + { + await using (var localFileStream = localFile.FileStream) + { + await localFileStream.CopyToAsync(blobStream); + } + } + + // Assert + var fileResult = await storage.DownloadAsync(fullFileName); + fileResult.IsSuccess.Should().BeTrue(); + fileResult.Value.Should().NotBeNull(); + await using var fileStream = fileResult.Value.FileStream; + using var streamReader = new StreamReader(fileStream); + var fileContent = await streamReader.ReadLineAsync(); + fileContent.Should().NotBeNullOrEmpty(); + } } \ No newline at end of file From afccd5319f979c825767df4cb1dd80b276ecca8d Mon Sep 17 00:00:00 2001 From: TRybina132 Date: Thu, 19 Oct 2023 17:49:42 +0300 Subject: [PATCH 5/6] add seek test --- .../Storages/Azure/AzureBlobStreamTests.cs | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs b/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs index 4e560e0d..98614081 100644 --- a/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs +++ b/ManagedCode.Storage.Tests/Storages/Azure/AzureBlobStreamTests.cs @@ -1,10 +1,10 @@ using System; using System.IO; +using System.Text; using System.Threading.Tasks; using FluentAssertions; using ManagedCode.Storage.Azure; using ManagedCode.Storage.Core.Models; -using ManagedCode.Storage.Server; using ManagedCode.Storage.Tests.Common; using Microsoft.Extensions.DependencyInjection; using Testcontainers.Azurite; @@ -31,8 +31,9 @@ public async Task ReadStreamWithStreamReader_WhenFileExists_ReturnData() { // Arrange var directory = "test-directory"; + var fileSizeInBytes = 10; await using var localFile = LocalFile.FromRandomNameWithExtension(".txt"); - FileHelper.GenerateLocalFileWithData(localFile, 10); + FileHelper.GenerateLocalFileWithData(localFile, fileSizeInBytes); var storage = (IAzureStorage) Storage; UploadOptions options = new() { FileName = localFile.Name, Directory = directory }; @@ -59,8 +60,9 @@ public async Task ReadStream_WhenFileExists_ReturnData() { // Arrange var directory = "test-directory"; + var fileSizeInBytes = 10; await using var localFile = LocalFile.FromRandomNameWithExtension(".txt"); - FileHelper.GenerateLocalFileWithData(localFile, 10); + FileHelper.GenerateLocalFileWithData(localFile, fileSizeInBytes); var storage = (IAzureStorage) Storage; UploadOptions options = new() { FileName = localFile.Name, Directory = directory }; @@ -106,12 +108,13 @@ public async Task ReadStream_WhenFileDoesNotExists_ReturnNoData() } [Fact] - public async Task WriteStreamWithStreamWriter_WhenFileExists_SaveData() + public async Task WriteStreamWithStreamWriter_SaveData() { // Arrange var directory = "test-directory"; await using var localFile = LocalFile.FromRandomNameWithExtension(".txt"); - FileHelper.GenerateLocalFileWithData(localFile, 512); + var fileSizeInBytes = 10; + FileHelper.GenerateLocalFileWithData(localFile, fileSizeInBytes); var fullFileName = $"{directory}/{localFile.FileInfo.FullName}"; var storage = (IAzureStorage) Storage; @@ -136,4 +139,37 @@ public async Task WriteStreamWithStreamWriter_WhenFileExists_SaveData() var fileContent = await streamReader.ReadLineAsync(); fileContent.Should().NotBeNullOrEmpty(); } + + [Fact] + public async Task Seek_WhenFileExists_ReturnData() + { + // Arrange + var directory = "test-directory"; + var fileSizeInBytes = 10; + await using var localFile = LocalFile.FromRandomNameWithExtension(".txt"); + FileHelper.GenerateLocalFileWithData(localFile, fileSizeInBytes); + var storage = (IAzureStorage) Storage; + + UploadOptions options = new() { FileName = localFile.Name, Directory = directory }; + await using var localFileStream = localFile.FileInfo.OpenRead(); + var result = await storage.UploadAsync(localFileStream, options); + + await using var blobStream = storage.GetBlobStream(result.Value.FullName); + + // Act + var seekInPosition = fileSizeInBytes / 2; + blobStream.Seek(seekInPosition, SeekOrigin.Current); + var buffer = new byte[seekInPosition]; + var bytesRead = await blobStream.ReadAsync(buffer); + + // Assert + bytesRead.Should().Be(seekInPosition); + await using var fileStream = localFile.FileInfo.OpenRead(); + using var fileReader = new StreamReader(fileStream); + var fileContent = await fileReader.ReadToEndAsync(); + var content = Encoding.UTF8.GetString(buffer); + content.Should().NotBeNullOrEmpty(); + var trimmedFileContent = fileContent.Remove(0, seekInPosition); + content.Should().Be(trimmedFileContent); + } } \ No newline at end of file From 292df328fa7506b5e9bde1bbe9523e6ada0f4e60 Mon Sep 17 00:00:00 2001 From: TRybina132 Date: Fri, 20 Oct 2023 11:30:26 +0300 Subject: [PATCH 6/6] add properties variable --- ManagedCode.Storage.Azure/BlobStream.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ManagedCode.Storage.Azure/BlobStream.cs b/ManagedCode.Storage.Azure/BlobStream.cs index 7cfe6b09..af68ee20 100644 --- a/ManagedCode.Storage.Azure/BlobStream.cs +++ b/ManagedCode.Storage.Azure/BlobStream.cs @@ -37,7 +37,8 @@ public override long Length { get { - var metadata = _pageBlob.GetProperties().Value.Metadata; + var properties = _pageBlob.GetProperties(); + var metadata = properties.Value.Metadata; if (metadata.TryGetValue(MetadataLengthKey, out var length)) { if (long.TryParse(length, out var realLength)) @@ -46,7 +47,7 @@ public override long Length } } - var contentLenght = _pageBlob.GetProperties().Value.ContentLength; + var contentLenght = properties.Value.ContentLength; SetLengthInternal(contentLenght); return contentLenght; }