From 5de994ea5fdaf9558f000d4a0464591406c7cc6e Mon Sep 17 00:00:00 2001 From: Sam Bosell Date: Tue, 19 Sep 2017 10:47:25 -0500 Subject: [PATCH 1/2] #8 Addition of an AzureAppendBlobProvider. Minor changes to the existing AzureBlob tests to allow the full tests to be run against storage instead of the emulator. AppendBlobs don't support the emulator so you must provide a connection string to run them. --- .../AzureAppendBlobVirtualDirectory.cs | 111 ++++++ .../Storage/AzureAppendBlobVirtualFile.cs | 61 +++ .../Storage/AzureAppendBlobVirtualFiles.cs | 157 ++++++++ .../Storage/CloudBlobContainerExtension.cs | 25 +- ...AzureAppendBlobVirtualPathProviderTests.cs | 360 ++++++++++++++++++ .../AzureBlobVirtualPathProviderTests.cs | 11 +- 6 files changed, 719 insertions(+), 6 deletions(-) create mode 100644 src/ServiceStack.Azure/Storage/AzureAppendBlobVirtualDirectory.cs create mode 100644 src/ServiceStack.Azure/Storage/AzureAppendBlobVirtualFile.cs create mode 100644 src/ServiceStack.Azure/Storage/AzureAppendBlobVirtualFiles.cs create mode 100644 tests/ServiceStack.Azure.Tests/Storage/AzureAppendBlobVirtualPathProviderTests.cs diff --git a/src/ServiceStack.Azure/Storage/AzureAppendBlobVirtualDirectory.cs b/src/ServiceStack.Azure/Storage/AzureAppendBlobVirtualDirectory.cs new file mode 100644 index 0000000..cb4a9e4 --- /dev/null +++ b/src/ServiceStack.Azure/Storage/AzureAppendBlobVirtualDirectory.cs @@ -0,0 +1,111 @@ +using ServiceStack.IO; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Collections; +using Microsoft.WindowsAzure.Storage.Blob; +using ServiceStack.VirtualPath; + +namespace ServiceStack.Azure.Storage +{ + public class AzureAppendBlobVirtualDirectory : AbstractVirtualDirectoryBase + { + private readonly AzureAppendBlobVirtualFiles pathProvider; + + public AzureAppendBlobVirtualDirectory(AzureAppendBlobVirtualFiles pathProvider, string directoryPath) + : base(pathProvider) + { + this.pathProvider = pathProvider; + this.DirectoryPath = directoryPath; + + if (directoryPath == "/" || directoryPath.IsNullOrEmpty()) + return; + + var separatorIndex = directoryPath.LastIndexOf(pathProvider.RealPathSeparator, StringComparison.Ordinal); + + ParentDirectory = new AzureAppendBlobVirtualDirectory(pathProvider, + separatorIndex == -1 ? string.Empty : directoryPath.Substring(0, separatorIndex)); + } + + public string DirectoryPath { get; set; } + + public override IEnumerable Directories + { + get + { + var blobs = pathProvider.Container.ListBlobs(DirectoryPath == null + ? null + : DirectoryPath + pathProvider.RealPathSeparator); + + return blobs.Where(q => q.GetType() == typeof(CloudBlobDirectory)) + .Select(q => + { + var blobDir = (CloudBlobDirectory)q; + return new AzureAppendBlobVirtualDirectory(pathProvider, blobDir.Prefix.Trim(pathProvider.RealPathSeparator[0])); + }); + } + } + + public override DateTime LastModified + { + get + { + throw new NotImplementedException(); + } + } + + public override IEnumerable Files => pathProvider.GetImmediateFiles(this.DirectoryPath); + + // Azure Blob storage directories only exist if there are contents beneath them + public bool Exists() + { + var ret = pathProvider.Container.ListBlobs(this.DirectoryPath, false) + .Where(q => q.GetType() == typeof(CloudBlobDirectory)) + .Any(); + return ret; + + } + + public override string Name => DirectoryPath?.SplitOnLast(pathProvider.RealPathSeparator).Last(); + + public override string VirtualPath => DirectoryPath; + + public override IEnumerator GetEnumerator() + { + throw new NotImplementedException(); + } + + protected override IVirtualFile GetFileFromBackingDirectoryOrDefault(string fileName) + { + fileName = pathProvider.CombineVirtualPath(this.DirectoryPath, pathProvider.SanitizePath(fileName)); + return pathProvider.GetFile(fileName); + } + + protected override IEnumerable GetMatchingFilesInDir(string globPattern) + { + var dir = (this.DirectoryPath == null) ? null : this.DirectoryPath + pathProvider.RealPathSeparator; + + var ret = pathProvider.Container.ListBlobs(dir) + .Where(q => q.GetType() == typeof(CloudAppendBlob)) + .Where(q => + { + var x = ((CloudAppendBlob)q).Name.Glob(globPattern); + return x; + }) + .Select(q => + { + return new AzureAppendBlobVirtualFile(pathProvider, this).Init(q as CloudAppendBlob); + }); + return ret; + } + + protected override IVirtualDirectory GetDirectoryFromBackingDirectoryOrDefault(string directoryName) + { + return new AzureAppendBlobVirtualDirectory(this.pathProvider, pathProvider.SanitizePath(DirectoryPath.CombineWith(directoryName))); + } + + + } +} \ No newline at end of file diff --git a/src/ServiceStack.Azure/Storage/AzureAppendBlobVirtualFile.cs b/src/ServiceStack.Azure/Storage/AzureAppendBlobVirtualFile.cs new file mode 100644 index 0000000..20b83f5 --- /dev/null +++ b/src/ServiceStack.Azure/Storage/AzureAppendBlobVirtualFile.cs @@ -0,0 +1,61 @@ +using ServiceStack.IO; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.IO; +using ServiceStack.VirtualPath; +using Microsoft.WindowsAzure.Storage.Blob; + +namespace ServiceStack.Azure.Storage +{ + public class AzureAppendBlobVirtualFile : AbstractVirtualFileBase + { + + private readonly AzureAppendBlobVirtualFiles pathProvider; + private readonly CloudBlobContainer container; + + public CloudAppendBlob Blob { get; private set; } + + public AzureAppendBlobVirtualFile(AzureAppendBlobVirtualFiles owningProvider, IVirtualDirectory directory) + : base(owningProvider, directory) + { + this.pathProvider = owningProvider; + this.container = pathProvider.Container; + } + + public AzureAppendBlobVirtualFile Init(CloudAppendBlob blob) + { + this.Blob = blob; + return this; + } + + public override DateTime LastModified => Blob.Properties.LastModified?.UtcDateTime ?? DateTime.MinValue; + + public override long Length => Blob.Properties.Length; + + public override string Name => Blob.Name.Contains(pathProvider.VirtualPathSeparator) + ? Blob.Name.SplitOnLast(pathProvider.VirtualPathSeparator)[1] + : Blob.Name; + + public string FilePath => Blob.Name; + + public string ContentType => Blob.Properties.ContentType; + + public override string VirtualPath => FilePath; + + public override Stream OpenRead() + { + return Blob.OpenRead(); + } + + public override void Refresh() + { + CloudAppendBlob blob = pathProvider.Container.GetAppendBlobReference(Blob.Name); + if (!blob.Exists()) return; + + Init(blob); + } + } +} diff --git a/src/ServiceStack.Azure/Storage/AzureAppendBlobVirtualFiles.cs b/src/ServiceStack.Azure/Storage/AzureAppendBlobVirtualFiles.cs new file mode 100644 index 0000000..706d2ac --- /dev/null +++ b/src/ServiceStack.Azure/Storage/AzureAppendBlobVirtualFiles.cs @@ -0,0 +1,157 @@ +using ServiceStack.IO; +using System; +using System.Collections.Generic; +using System.Linq; +using System.IO; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using ServiceStack.VirtualPath; +using Microsoft.WindowsAzure.Storage.RetryPolicies; + +namespace ServiceStack.Azure.Storage +{ + public class AzureAppendBlobVirtualFiles : AbstractVirtualPathProviderBase, IVirtualFiles + { + public CloudBlobContainer Container { get; } + + private readonly AzureAppendBlobVirtualDirectory rootDirectory; + + public override IVirtualDirectory RootDirectory => rootDirectory; + + public override string VirtualPathSeparator => "/"; + + public override string RealPathSeparator => "/"; + + public AzureAppendBlobVirtualFiles(string connectionString, string containerName) + { + var storageAccount = CloudStorageAccount.Parse(connectionString); + + //containerName is the name of Azure Storage Blob container + Container = storageAccount.CreateCloudBlobClient().GetContainerReference(containerName); + Container.CreateIfNotExists(); + rootDirectory = new AzureAppendBlobVirtualDirectory(this, null); + } + + + public AzureAppendBlobVirtualFiles(CloudBlobContainer container) + { + Container = container; + Container.CreateIfNotExists(); + rootDirectory = new AzureAppendBlobVirtualDirectory(this, null); + } + + protected override void Initialize() + { + } + + public void WriteFile(string filePath, string textContents) + { + var blob = Container.GetAppendBlobReference(SanitizePath(filePath)); + blob.CreateOrReplace(null,null,null); + blob.Properties.ContentType = MimeTypes.GetMimeType(filePath); + blob.AppendText(textContents); + } + + public void WriteFile(string filePath, Stream stream) + { + var blob = Container.GetAppendBlobReference(SanitizePath(filePath)); + blob.CreateOrReplace(AccessCondition.GenerateEmptyCondition(), new BlobRequestOptions() { RetryPolicy = new LinearRetry(TimeSpan.FromSeconds(1), 10) }, null); + blob.Properties.ContentType = MimeTypes.GetMimeType(filePath); + blob.AppendFromStream(stream); + } + + public void WriteFiles(IEnumerable files, Func toPath = null) + { + this.CopyFrom(files, toPath); + } + + public void AppendFile(string filePath, string textContents) + { + var blob = Container.GetAppendBlobReference(SanitizePath(filePath)); + blob.AppendText(textContents); + + } + + public void AppendFile(string filePath, Stream stream) + { + var blob = Container.GetAppendBlobReference(SanitizePath(filePath)); + blob.AppendFromStream(stream); + } + + public void DeleteFile(string filePath) + { + var blob = Container.GetAppendBlobReference(SanitizePath(filePath)); + blob.Delete(); + } + + public void DeleteFiles(IEnumerable filePaths) + { + filePaths.Each(DeleteFile); + } + + public void DeleteFolder(string dirPath) + { + dirPath = SanitizePath(dirPath); + // Delete based on a wildcard search of the directory + if (!dirPath.EndsWith("/")) dirPath += "/"; + //directoryPath += "*"; + foreach (var blob in Container.ListBlobs(dirPath, true)) + { + Container.GetAppendBlobReference(((CloudAppendBlob)blob).Name).DeleteIfExists(); + } + } + + public override IVirtualFile GetFile(string virtualPath) + { + var filePath = SanitizePath(virtualPath); + + CloudAppendBlob blob = Container.GetAppendBlobReference(filePath); + if (!blob.Exists()) return null; + + return new AzureAppendBlobVirtualFile(this, GetDirectory(GetDirPath(virtualPath))).Init(blob); + } + + public override IVirtualDirectory GetDirectory(string virtualPath) + { + return new AzureAppendBlobVirtualDirectory(this, virtualPath); + } + + public override bool DirectoryExists(string virtualPath) + { + var ret = ((AzureAppendBlobVirtualDirectory)GetDirectory(virtualPath)).Exists(); + return ret; + } + + public string GetDirPath(string filePath) + { + if (string.IsNullOrEmpty(filePath)) + return null; + + var lastDirPos = filePath.LastIndexOf(VirtualPathSeparator[0]); + return lastDirPos >= 0 + ? filePath.Substring(0, lastDirPos) + : null; + } + + public IEnumerable GetImmediateFiles(string fromDirPath) + { + var dir = new AzureAppendBlobVirtualDirectory(this, fromDirPath); + + return Container.ListBlobs((fromDirPath == null) ? null : fromDirPath + this.RealPathSeparator) + .Where(q => q.GetType() == typeof(CloudAppendBlob)) + .Select(q => new AzureAppendBlobVirtualFile(this, dir).Init(q as CloudAppendBlob)); + + } + + public string SanitizePath(string filePath) + { + var sanitizedPath = string.IsNullOrEmpty(filePath) + ? null + : (filePath[0] == VirtualPathSeparator[0] ? filePath.Substring(1) : filePath); + + return sanitizedPath != null + ? sanitizedPath.Replace('\\', VirtualPathSeparator[0]) + : null; + } + } +} \ No newline at end of file diff --git a/src/ServiceStack.Azure/Storage/CloudBlobContainerExtension.cs b/src/ServiceStack.Azure/Storage/CloudBlobContainerExtension.cs index 4c9f732..4e0839c 100644 --- a/src/ServiceStack.Azure/Storage/CloudBlobContainerExtension.cs +++ b/src/ServiceStack.Azure/Storage/CloudBlobContainerExtension.cs @@ -4,6 +4,7 @@ using System.Text; using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.Blob.Protocol; +using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Table; namespace ServiceStack.Azure.Storage @@ -35,17 +36,23 @@ public static void CreateIfNotExists(this CloudBlobContainer container) container.CreateIfNotExistsAsync().Wait(); } + public static void DeleteIfExists(this CloudBlobContainer container) { container.DeleteIfExistsAsync().Wait(); } - public static void Delete(this CloudBlockBlob blob) + public static void CreateOrReplace(this CloudAppendBlob blob, AccessCondition condition, BlobRequestOptions options, OperationContext operationContext) { + blob.CreateOrReplaceAsync(condition, options, operationContext).Wait(); + } + + + public static void Delete(this ICloudBlob blob) { blob.DeleteAsync().Wait(); } - public static void DeleteIfExists(this CloudBlockBlob blob) + public static void DeleteIfExists(this ICloudBlob blob) { blob.DeleteIfExistsAsync().Wait(); } @@ -60,12 +67,22 @@ public static void UploadFromStream(this CloudBlockBlob blob, Stream stream) blob.UploadFromStreamAsync(stream).Wait(); } - public static Stream OpenRead(this CloudBlockBlob blob) + public static void AppendText(this CloudAppendBlob blob, string content) + { + ((CloudAppendBlob) blob).AppendTextAsync(content).Wait(); + } + + public static void AppendFromStream(this CloudAppendBlob blob, Stream stream) + { + ((CloudAppendBlob) blob).AppendFromStreamAsync(stream).Wait(); + } + + public static Stream OpenRead(this CloudBlob blob) { return blob.OpenReadAsync().Result; } - public static bool Exists(this CloudBlockBlob blob) + public static bool Exists(this CloudBlob blob) { return blob.ExistsAsync().Result; } diff --git a/tests/ServiceStack.Azure.Tests/Storage/AzureAppendBlobVirtualPathProviderTests.cs b/tests/ServiceStack.Azure.Tests/Storage/AzureAppendBlobVirtualPathProviderTests.cs new file mode 100644 index 0000000..77d0f61 --- /dev/null +++ b/tests/ServiceStack.Azure.Tests/Storage/AzureAppendBlobVirtualPathProviderTests.cs @@ -0,0 +1,360 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading; +using NUnit.Framework; +using ServiceStack.IO; +using ServiceStack.Testing; +using ServiceStack.Text; +using ServiceStack.VirtualPath; +using Microsoft.WindowsAzure.Storage; +using ServiceStack.Azure.Storage; + +namespace ServiceStack.Azure.Tests.Storage +{ + + [TestFixture] + public class AzureAppendBlobVirtualPathProviderTests : VirtualAppendPathProviderTests + { + public const string ContainerName = "ss-ci-test-append"; + + // you must provide an azure account to run these tests + private readonly CloudStorageAccount storageAccount = CloudStorageAccount.Parse(""); + + public override IVirtualPathProvider GetPathProvider() + { + var client = storageAccount.CreateCloudBlobClient(); + var container = client.GetContainerReference(ContainerName); + return new AzureAppendBlobVirtualFiles(container); + } + + [OneTimeSetUp] + public void Setup() + { + storageAccount.CreateCloudBlobClient().GetContainerReference(ContainerName).CreateIfNotExists(); + } + + [OneTimeTearDown] + public void Teardown() + { + storageAccount.CreateCloudBlobClient().GetContainerReference(ContainerName).DeleteIfExists(); + } + + [Test] + public void Can_have_many_items() + { + var pathProvider = GetPathProvider(); + + int count = 20; + count.Times(i => + { + var filePath = "file-{0}.txt".Fmt(i); + pathProvider.WriteFile(filePath, "data"); + }); + + Assert.That(pathProvider.RootDirectory.Files.Count, Is.EqualTo(count)); + count.Times(i => + { + var filePath = "file-{0}.txt".Fmt(i); + pathProvider.DeleteFile(filePath); + }); + + + + } + + } + + + + public abstract class VirtualAppendPathProviderTests + { + public abstract IVirtualPathProvider GetPathProvider(); + + protected ServiceStackHost appHost; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + appHost = new BasicAppHost() + .Init(); + } + + [OneTimeTearDown] + public virtual void OneTimeTearDown() + { + appHost.Dispose(); + } + + [Test] + public void Can_create_file() + { + var pathProvider = GetPathProvider(); + + var filePath = "dir/file.txt"; + pathProvider.WriteFile(filePath, "file"); + + var file = pathProvider.GetFile(filePath); + + Assert.That(file.ReadAllText(), Is.EqualTo("file")); + Assert.That(file.ReadAllText(), Is.EqualTo("file")); //can read twice + + Assert.That(file.VirtualPath, Is.EqualTo(filePath)); + Assert.That(file.Name, Is.EqualTo("file.txt")); + Assert.That(file.Directory.Name, Is.EqualTo("dir")); + Assert.That(file.Directory.VirtualPath, Is.EqualTo("dir")); + Assert.That(file.Extension, Is.EqualTo("txt")); + + Assert.That(file.Directory.Name, Is.EqualTo("dir")); + + pathProvider.DeleteFolder("dir"); + } + + [Test] + + public void Does_refresh_LastModified() + { + var pathProvider = GetPathProvider(); + + var filePath = "dir/file.txt"; + pathProvider.WriteFile(filePath, "file1"); + + var file = pathProvider.GetFile(filePath); + var prevLastModified = file.LastModified; + + file.Refresh(); + Assert.That(file.LastModified, Is.EqualTo(prevLastModified)); + + pathProvider.WriteFile(filePath, "file2"); + file.Refresh(); + + if (file.GetType() == typeof(AzureBlobVirtualFile) && file.LastModified == prevLastModified) + { + Thread.Sleep(1000); + pathProvider.WriteFile(filePath, "file3"); + file.Refresh(); + } + + Assert.That(file.LastModified, Is.Not.EqualTo(prevLastModified)); + + pathProvider.DeleteFolder("dir"); + } + + [Test] + public void Can_create_file_from_root() + { + var pathProvider = GetPathProvider(); + + var filePath = "file.txt"; + pathProvider.WriteFile(filePath, "file"); + + var file = pathProvider.GetFile(filePath); + + Assert.That(file.ReadAllText(), Is.EqualTo("file")); + Assert.That(file.Name, Is.EqualTo(filePath)); + Assert.That(file.Extension, Is.EqualTo("txt")); + + Assert.That(file.Directory.VirtualPath, Is.Null); + Assert.That(file.Directory.Name, Is.Null.Or.EqualTo("App_Data")); + + pathProvider.DeleteFiles(new[] { "file.txt" }); + } + + [Test] + public void Does_override_existing_file() + { + var pathProvider = GetPathProvider(); + + pathProvider.WriteFile("file.txt", "original"); + pathProvider.WriteFile("file.txt", "updated"); + Assert.That(pathProvider.GetFile("file.txt").ReadAllText(), Is.EqualTo("updated")); + + pathProvider.WriteFile("/a/file.txt", "original"); + pathProvider.WriteFile("/a/file.txt", "updated"); + Assert.That(pathProvider.GetFile("/a/file.txt").ReadAllText(), Is.EqualTo("updated")); + + pathProvider.DeleteFiles(new[] { "file.txt", "/a/file.txt" }); + pathProvider.DeleteFolder("a"); + } + + [Test] + public void Can_view_files_in_Directory() + { + var pathProvider = GetPathProvider(); + + var testdirFileNames = new[] + { + "testdir/a.txt", + "testdir/b.txt", + "testdir/c.txt", + }; + + testdirFileNames.Each(x => pathProvider.WriteFile(x, "textfile")); + + var testdir = pathProvider.GetDirectory("testdir"); + var filePaths = testdir.Files.Map(x => x.VirtualPath); + + Assert.That(filePaths, Is.EquivalentTo(testdirFileNames)); + + var fileNames = testdir.Files.Map(x => x.Name); + Assert.That(fileNames, Is.EquivalentTo(testdirFileNames.Map(x => + x.SplitOnLast('/').Last()))); + + pathProvider.DeleteFolder("testdir"); + } + + [Test] + public void Does_resolve_nested_files_and_folders() + { + var pathProvider = GetPathProvider(); + + var allFilePaths = new[] { + "testfile.txt", + "a/testfile-a1.txt", + "a/testfile-a2.txt", + "a/b/testfile-ab1.txt", + "a/b/testfile-ab2.txt", + "a/b/c/testfile-abc1.txt", + "a/b/c/testfile-abc2.txt", + "a/d/testfile-ad1.txt", + "e/testfile-e1.txt", + }; + + allFilePaths.Each(x => pathProvider.WriteFile(x, x.SplitOnLast('.').First().SplitOnLast('/').Last())); + + Assert.That(allFilePaths.All(x => pathProvider.IsFile(x))); + Assert.That(new[] { "a", "a/b", "a/b/c", "a/d", "e" }.All(x => pathProvider.IsDirectory(x))); + + Assert.That(!pathProvider.IsFile("notfound.txt")); + Assert.That(!pathProvider.IsFile("a/notfound.txt")); + Assert.That(!pathProvider.IsDirectory("f")); + Assert.That(!pathProvider.IsDirectory("a/f")); + Assert.That(!pathProvider.IsDirectory("testfile.txt")); + Assert.That(!pathProvider.IsDirectory("a/testfile-a1.txt")); + + AssertContents(pathProvider.RootDirectory, new[] { + "testfile.txt", + }, new[] { + "a", + "e" + }); + + AssertContents(pathProvider.GetDirectory("a"), new[] { + "a/testfile-a1.txt", + "a/testfile-a2.txt", + }, new[] { + "a/b", + "a/d" + }); + + AssertContents(pathProvider.GetDirectory("a/b"), new[] { + "a/b/testfile-ab1.txt", + "a/b/testfile-ab2.txt", + }, new[] { + "a/b/c" + }); + + AssertContents(pathProvider.GetDirectory("a").GetDirectory("b"), new[] { + "a/b/testfile-ab1.txt", + "a/b/testfile-ab2.txt", + }, new[] { + "a/b/c" + }); + + AssertContents(pathProvider.GetDirectory("a/b/c"), new[] { + "a/b/c/testfile-abc1.txt", + "a/b/c/testfile-abc2.txt", + }, new string[0]); + + AssertContents(pathProvider.GetDirectory("a/d"), new[] { + "a/d/testfile-ad1.txt", + }, new string[0]); + + AssertContents(pathProvider.GetDirectory("e"), new[] { + "e/testfile-e1.txt", + }, new string[0]); + + Assert.That(pathProvider.GetFile("a/b/c/testfile-abc1.txt").ReadAllText(), Is.EqualTo("testfile-abc1")); + Assert.That(pathProvider.GetDirectory("a").GetFile("b/c/testfile-abc1.txt").ReadAllText(), Is.EqualTo("testfile-abc1")); + Assert.That(pathProvider.GetDirectory("a/b").GetFile("c/testfile-abc1.txt").ReadAllText(), Is.EqualTo("testfile-abc1")); + Assert.That(pathProvider.GetDirectory("a").GetDirectory("b").GetDirectory("c").GetFile("testfile-abc1.txt").ReadAllText(), Is.EqualTo("testfile-abc1")); + + var dirs = pathProvider.RootDirectory.Directories.Map(x => x.VirtualPath); + Assert.That(dirs, Is.EquivalentTo(new[] { "a", "e" })); + + var rootDirFiles = pathProvider.RootDirectory.GetAllMatchingFiles("*", 1).Map(x => x.VirtualPath); + Assert.That(rootDirFiles, Is.EquivalentTo(new[] { "testfile.txt" })); + + var allFiles = pathProvider.GetAllMatchingFiles("*").Map(x => x.VirtualPath); + Assert.That(allFiles, Is.EquivalentTo(allFilePaths)); + + allFiles = pathProvider.GetAllFiles().Map(x => x.VirtualPath); + Assert.That(allFiles, Is.EquivalentTo(allFilePaths)); + + Assert.That(pathProvider.DirectoryExists("a")); + Assert.That(!pathProvider.DirectoryExists("f")); + Assert.That(!pathProvider.GetDirectory("a/b/c").IsRoot); + Assert.That(!pathProvider.GetDirectory("a/b").IsRoot); + Assert.That(!pathProvider.GetDirectory("a").IsRoot); + Assert.That(pathProvider.GetDirectory("").IsRoot); + + pathProvider.DeleteFile("testfile.txt"); + pathProvider.DeleteFolder("a"); + pathProvider.DeleteFolder("e"); + + Assert.That(pathProvider.GetAllFiles().ToList().Count, Is.EqualTo(0)); + } + + [Test] + + public void Does_append_to_file() + { + var pathProvider = GetPathProvider(); + + pathProvider.WriteFile("original.txt", "original\n"); + + pathProvider.AppendFile("original.txt", "New Line1\n"); + pathProvider.AppendFile("original.txt", "New Line2\n"); + + var contents = pathProvider.GetFile("original.txt").ReadAllText(); + Assert.That(contents, Is.EqualTo("original\nNew Line1\nNew Line2\n")); + + pathProvider.DeleteFile("original.txt"); + } + + [Test] + + public void Does_append_to_file_bytes() + { + var pathProvider = GetPathProvider(); + + pathProvider.WriteFile("original.bin", "original\n".ToUtf8Bytes()); + + pathProvider.AppendFile("original.bin", "New Line1\n".ToUtf8Bytes()); + pathProvider.AppendFile("original.bin", "New Line2\n".ToUtf8Bytes()); + + var contents = pathProvider.GetFile("original.bin").ReadAllBytes(); + Assert.That(contents, Is.EquivalentTo("original\nNew Line1\nNew Line2\n".ToUtf8Bytes())); + + pathProvider.DeleteFile("original.bin"); + } + + public void AssertContents(IVirtualDirectory dir, + string[] expectedFilePaths, string[] expectedDirPaths) + { + var filePaths = dir.Files.Map(x => x.VirtualPath); + Assert.That(filePaths, Is.EquivalentTo(expectedFilePaths)); + + var fileNames = dir.Files.Map(x => x.Name); + Assert.That(fileNames, Is.EquivalentTo(expectedFilePaths.Map(x => + x.SplitOnLast('/').Last()))); + + var dirPaths = dir.Directories.Map(x => x.VirtualPath); + Assert.That(dirPaths, Is.EquivalentTo(expectedDirPaths)); + + var dirNames = dir.Directories.Map(x => x.Name); + Assert.That(dirNames, Is.EquivalentTo(expectedDirPaths.Map(x => + x.SplitOnLast('/').Last()))); + } + } +} diff --git a/tests/ServiceStack.Azure.Tests/Storage/AzureBlobVirtualPathProviderTests.cs b/tests/ServiceStack.Azure.Tests/Storage/AzureBlobVirtualPathProviderTests.cs index a486e65..6e83d00 100644 --- a/tests/ServiceStack.Azure.Tests/Storage/AzureBlobVirtualPathProviderTests.cs +++ b/tests/ServiceStack.Azure.Tests/Storage/AzureBlobVirtualPathProviderTests.cs @@ -27,13 +27,13 @@ public override IVirtualPathProvider GetPathProvider() return new AzureBlobVirtualFiles(container); } - [SetUp] + [OneTimeSetUp] public void Setup() { storageAccount.CreateCloudBlobClient().GetContainerReference(ContainerName).CreateIfNotExists(); } - [TearDown] + [OneTimeTearDown] public void Teardown() { storageAccount.CreateCloudBlobClient().GetContainerReference(ContainerName).DeleteIfExists(); @@ -53,6 +53,13 @@ public void Can_have_many_items() Assert.That(pathProvider.RootDirectory.Files.Count, Is.EqualTo(count)); + // clean them up or future tests might fail + count.Times(i => + { + var filePath = "file-{0}.txt".Fmt(i); + pathProvider.DeleteFile(filePath); + }); + } } From a909f69c010fdb87ae127fb269691cb93c69c17c Mon Sep 17 00:00:00 2001 From: lucuma Date: Tue, 19 Sep 2017 11:24:49 -0500 Subject: [PATCH 2/2] Updates to readme for Append scenarios --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index 3fc2ba4..78f3bd9 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ ServiceStack.Azure includes implementation of the following ServiceStack provide - [ServiceBusMqServer](#ServiceBusMqServer) - [MQ Server](http://docs.servicestack.net/messaging) for invoking ServiceStack Services via Azure ServiceBus - [AzureBlobVirtualFiles](#virtual-filesystem-backed-by-azure-blob-storage) - Virtual file system based on Azure Blob Storage +- [AzureAppendBlobVirtualFiles](#virtual-filesystem-backed-by-azure-blob-storage) - Virtual file system based on Azure Blob Storage for appending scenarios - [AzureTableCacheClient](#caching-support-with-azure-table-storage) - Cache client over Azure Table Storage @@ -50,6 +51,26 @@ public class AppHost : AppHostBase } ``` +In addition you can use **AzureAppendBlobVirtualFiles** in scenarios that require appending such as logging. + +```csharp +public class AppHost : AppHostBase +{ + public override void Configure(Container container) + { + Plugins.Add(new RequestLogsFeature + { + RequestLogger = new CsvRequestLogger( + files: new AzureAppendBlobVirtualFiles(AppSettings.Get("storageConnection"), "logfiles"), + requestLogsPattern: "requestlogs/{year}-{month}/{year}-{month}-{day}.csv", + errorLogsPattern: "requestlogs/{year}-{month}/{year}-{month}-{day}-errors.csv", + appendEvery: TimeSpan.FromSeconds(30)) + + }); + } +} +``` + ## Caching support with Azure Table Storage The AzureTableCacheClient implements [ICacheClientExteded](https://github.com/ServiceStack/ServiceStack/blob/master/src/ServiceStack.Interfaces/Caching/ICacheClientExtended.cs) and [IRemoveByPattern](https://github.com/ServiceStack/ServiceStack/blob/master/src/ServiceStack.Interfaces/Caching/IRemoveByPattern.cs) using Azure Table Storage.