Mesh weaver

.config/dotnet-tools.json

@@ -0,0 +1,12 @@
+{
+  "version": 1,
+  "isRoot": true,
+  "tools": {
+    "meshweaver.azure.publish": {
+      "version": "1.0.0",
+      "commands": [
+        "MeshWeaver.Azure.Publish"
+      ]
+    }
+  }
+}

Directory.Packages.props

@@ -16,6 +16,8 @@
     <PackageVersion Include="Azure.Provisioning.Storage" Version="0.2.0" />
     <PackageVersion Include="Azure.Search.Documents" Version="11.6.0" />
     <PackageVersion Include="Azure.Storage.Blobs" Version="12.22.0" />
+    <PackageVersion Include="Azure.Storage.Files.Shares" Version="12.22.0" />
+    <PackageVersion Include="Azure.Storage.Queues" Version="12.22.0" />
     <PackageVersion Include="Castle.Core" Version="5.1.1" />
     <PackageVersion Include="CSharpVitamins.ShortGuid" Version="2.0.0" />
     <PackageVersion Include="CsvHelper" Version="15.0.5" />
@@ -32,11 +34,16 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageVersion>
+    <PackageVersion Include="Microsoft.Extensions.Azure" Version="1.7.3" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.UserSecrets" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Orleans.Core.Abstractions" Version="8.2.0" />
     <PackageVersion Include="Microsoft.Orleans.Runtime" Version="8.2.0" />
     <PackageVersion Include="Microsoft.Orleans.Serialization.Abstractions" Version="8.2.0" />
     <PackageVersion Include="OpenTelemetry" Version="1.9.0" />
     <PackageVersion Include="System.Collections.Immutable" Version="8.0.0" />
+    <PackageVersion Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
+    <PackageVersion Include="System.CommandLine.NamingConventionBinder" Version="2.0.0-beta4.22272.1" />
     <PackageVersion Include="System.Text.Encodings.Web" Version="8.0.0" />
     <PackageVersion Include="System.Text.Json" Version="8.0.4" />
     <PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="3.11.0-beta1.24219.2" />

MeshWeaver.sln

@@ -207,8 +207,8 @@ EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "monolith", "monolith", "{4E1A12BB-5348-4DA4-B05F-F831DC16F7DE}"
 	ProjectSection(SolutionItems) = preProject
 		monolith\Directory.Build.props = monolith\Directory.Build.props
-		Readme.md = Readme.md
 		MIT License.md = MIT License.md
+		Readme.md = Readme.md
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MeshWeaver.Portal", "monolith\MeshWeaver.Portal\MeshWeaver.Portal.csproj", "{1471FB82-912E-411A-9141-096E0AECAAAE}"
@@ -221,6 +221,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MeshWeaver.Search", "src\Me
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MeshWeaver.Search.Test", "test\MeshWeaver.Search.Test\MeshWeaver.Search.Test.csproj", "{BDEC6D93-F212-456F-BEEE-DE2BADAE2A56}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MeshWeaver.Azure.Publish", "src\MeshWeaver.Azure.Publish\MeshWeaver.Azure.Publish.csproj", "{844B4566-81A4-4CB2-9761-9B174FB7BCCB}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -587,6 +589,10 @@ Global
 		{BDEC6D93-F212-456F-BEEE-DE2BADAE2A56}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{BDEC6D93-F212-456F-BEEE-DE2BADAE2A56}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{BDEC6D93-F212-456F-BEEE-DE2BADAE2A56}.Release|Any CPU.Build.0 = Release|Any CPU
+		{844B4566-81A4-4CB2-9761-9B174FB7BCCB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{844B4566-81A4-4CB2-9761-9B174FB7BCCB}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{844B4566-81A4-4CB2-9761-9B174FB7BCCB}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{844B4566-81A4-4CB2-9761-9B174FB7BCCB}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

modules/Northwind/MeshWeaver.Northwind.ViewModel/AnnualReportExtensions.cs

@@ -1,5 +1,6 @@
 using MeshWeaver.Layout;
 using MeshWeaver.Layout.Composition;
+using System;
 
 namespace MeshWeaver.Northwind.ViewModel
 {

modules/Northwind/MeshWeaver.Northwind.ViewModel/MeshWeaver.Northwind.ViewModel.csproj

@@ -1,38 +1,53 @@
 <Project Sdk="Microsoft.NET.Sdk">
     <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
         <HasMeshNodes>true</HasMeshNodes>
         <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
     </PropertyGroup>
     <ItemGroup>
-    <ProjectReference Include="..\..\..\src\MeshWeaver.DataCubes\MeshWeaver.DataCubes.csproj" />
-    <ProjectReference Include="..\..\..\src\MeshWeaver.Documentation\MeshWeaver.Domain.Layout.csproj" />
-    <ProjectReference Include="..\..\..\src\MeshWeaver.Mesh.Contract\MeshWeaver.Mesh.Contract.csproj" />
-    <ProjectReference Include="..\..\..\src\MeshWeaver.Messaging.Hub\MeshWeaver.Messaging.Hub.csproj" />
-    <ProjectReference Include="..\..\..\src\MeshWeaver.Data\MeshWeaver.Data.csproj" />
-    <ProjectReference Include="..\..\..\src\MeshWeaver.Import\MeshWeaver.Import.csproj" />
-    <ProjectReference Include="..\..\..\src\MeshWeaver.Layout\MeshWeaver.Layout.csproj" />
-    <ProjectReference Include="..\..\..\src\MeshWeaver.Pivot\MeshWeaver.Pivot.csproj" />
-    <ProjectReference Include="..\..\..\src\MeshWeaver.Charting.Pivot\MeshWeaver.Charting.Pivot.csproj" />
-    <ProjectReference Include="..\..\..\src\MeshWeaver.Reporting\MeshWeaver.Reporting.csproj" />
-    <ProjectReference Include="..\MeshWeaver.Northwind.Model\MeshWeaver.Northwind.Model.csproj" />
-  </ItemGroup>
-  <ItemGroup>
-      <!-- Include all Markdown files in the Markdown folder as embedded resources -->
-      <EmbeddedResource Include="Markdown\**\*.md" />
-  </ItemGroup>
+        <ProjectReference Include="..\..\..\src\MeshWeaver.DataCubes\MeshWeaver.DataCubes.csproj" />
+        <ProjectReference Include="..\..\..\src\MeshWeaver.Documentation\MeshWeaver.Domain.Layout.csproj" />
+        <ProjectReference Include="..\..\..\src\MeshWeaver.Mesh.Contract\MeshWeaver.Mesh.Contract.csproj" />
+        <ProjectReference Include="..\..\..\src\MeshWeaver.Messaging.Hub\MeshWeaver.Messaging.Hub.csproj" />
+        <ProjectReference Include="..\..\..\src\MeshWeaver.Data\MeshWeaver.Data.csproj" />
+        <ProjectReference Include="..\..\..\src\MeshWeaver.Import\MeshWeaver.Import.csproj" />
+        <ProjectReference Include="..\..\..\src\MeshWeaver.Layout\MeshWeaver.Layout.csproj" />
+        <ProjectReference Include="..\..\..\src\MeshWeaver.Pivot\MeshWeaver.Pivot.csproj" />
+        <ProjectReference Include="..\..\..\src\MeshWeaver.Charting.Pivot\MeshWeaver.Charting.Pivot.csproj" />
+        <ProjectReference Include="..\..\..\src\MeshWeaver.Reporting\MeshWeaver.Reporting.csproj" />
+        <ProjectReference Include="..\MeshWeaver.Northwind.Model\MeshWeaver.Northwind.Model.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <!-- Include all Markdown files in the Markdown folder as embedded resources -->
+        <EmbeddedResource Include="Markdown\**\*.md" />
+    </ItemGroup>
+
+    <UsingTask TaskName="MeshWeaver.BuildTasks.ProcessFilesTask" AssemblyFile="$(TargetDir)MeshWeaver.BuildTasks.dll" />
+
+    <Target Name="ProcessFiles" DependsOnTargets="Build">
+        <!-- Define the input files -->
+        <ItemGroup>
+            <InputFiles Include="Markdown\**\*.md" />
+        </ItemGroup>
+
+        <!-- Execute the custom task -->
+        <ProcessFilesTask InputFiles="@(InputFiles)" BlobStorageConnectionString="YourConnectionString" BlobContainerName="YourContainerName" Address="YourAddress" />
+    </Target>
+
     <PropertyGroup>
-        <!--generate xml comment files -->
+        <!-- Generate XML comment files -->
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <DocumentationFile>bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml</DocumentationFile>
         <!-- Embed source files in the PDB -->
         <EmbedAllSources>true</EmbedAllSources>
     </PropertyGroup>
+
     <ItemGroup>
         <EmbeddedResource Include="bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml">
             <LogicalName>$(AssemblyName).xml</LogicalName>
             <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
             <Embed>true</Embed>
         </EmbeddedResource>
     </ItemGroup>
-
-</Project>
+</Project>

modules/Northwind/MeshWeaver.Northwind.ViewModel/OrdersSummaryArea.cs

@@ -1,7 +1,6 @@
 using System.Reactive.Linq;
 using MeshWeaver.Application.Styles;
 using MeshWeaver.Data;
-using MeshWeaver.Domain.Layout;
 using MeshWeaver.Domain.Layout.Documentation;
 using MeshWeaver.Layout;
 using MeshWeaver.Layout.Composition;
@@ -48,7 +47,8 @@ RenderingContext ctx
     )
     {
         var years = layoutArea
-            .Workspace.GetObservable<Order>()
+            .Workspace
+            .GetObservable<Order>()
             .DistinctUntilChanged()
             .Select(x =>
                 x.Select(y => y.OrderDate.Year)

modules/Northwind/MeshWeaver.Northwind.ViewModel/ProductOverviewItem.cs

@@ -44,5 +44,5 @@ public record ProductOverviewItem
     /// Gets the total amount.
     /// </summary>
     [DisplayFormat(DataFormatString = "N2")]
-    public double TotalAmount { get; init; }
+       public double TotalAmount { get; init; }
 }

src/MeshWeaver.Azure.Publish/MeshWeaver.Azure.Publish.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+      <OutputType>Exe</OutputType>
+      <PackAsTool>true</PackAsTool>
+  </PropertyGroup>
+  <ItemGroup>
+      <PackageReference Include="Azure.Storage.Blobs" />
+      <PackageReference Include="Microsoft.Extensions.Logging.Console" />
+      <PackageReference Include="System.CommandLine" />
+      <PackageReference Include="System.CommandLine.NamingConventionBinder" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\MeshWeaver.Search\MeshWeaver.Search.csproj" />
+  </ItemGroup>
+  <!--<ItemGroup>
+      <None Update="MeshWeaver.Azure.Publish.runtimeconfig.json">
+          <Pack>true</Pack>
+          <PackagePath>tools/net8.0/any/</PackagePath>
+      </None>
+  </ItemGroup>-->
+</Project>

src/MeshWeaver.Azure.Publish/MeshWeaver.Azure.Publish.runtimeconfig.json

@@ -0,0 +1,9 @@
+{
+  "runtimeOptions": {
+    "tfm": "net8.0",
+    "framework": {
+      "name": "Microsoft.NETCore.App",
+      "version": "8.0.0"
+    }
+  }
+}

src/MeshWeaver.Azure.Publish/Program.cs

@@ -0,0 +1,159 @@
+using System.CommandLine;
+using System.CommandLine.NamingConventionBinder;
+using System.Text.Json;
+using Azure.Storage.Blobs;
+using MeshWeaver.Search;
+using Microsoft.Extensions.Logging;
+
+namespace MeshWeaver.Azure.Publish
+{
+    public static class Program
+    {
+        public static async Task Main(string[] args)
+        {
+            using var loggerFactory = LoggerFactory.Create(builder =>
+            {
+                builder.AddConsole();
+            });
+            var logger = loggerFactory.CreateLogger(typeof(Program));
+
+            var rootCommand = new RootCommand
+            {
+                new Option<string>(
+                    "--path",
+                    "The input files to process"),
+                new Option<string>(
+                    "--connection-string",
+                    "The connection string for the blob storage"),
+                new Option<string>(
+                    "--container",
+                    "The name of the blob container"),
+                new Option<string>(
+                    "--html-container",
+                    "The name of the blob container containing pre-rendered html"),
+                new Option<string>(
+                    "--address",
+                    "The address to use")
+            };
+
+            rootCommand.Description = "MeshWeaver Build Tasks";
+
+            rootCommand.Handler = CommandHandler.Create<string, string, string, string, string>(async (path, connectionString, container, htmlContainer, address) =>
+            {
+                if (string.IsNullOrEmpty(path) || string.IsNullOrEmpty(connectionString) || string.IsNullOrEmpty(container) || string.IsNullOrEmpty(address))
+                {
+                    logger.LogError("Error: All options (--path, --connection-string, --container, --address) must be provided and cannot be null or empty.");
+                    return;
+                }
+
+                // Call your custom task logic here
+                logger.LogInformation("Processing files from\npath: {path}\nstorage: {connectionString}\ncontainer: {container}\nhtmlContainer: {htmlContainer}\naddress: {address}", path, connectionString, container, htmlContainer, address);
+                await ExecuteAsync(connectionString, container, htmlContainer, path, address, logger);
+            });
+
+            await rootCommand.InvokeAsync(args);
+        }
+
+        public static async Task<bool> ExecuteAsync(string connectionString, string container, string htmlContainer, string path, string address, ILogger logger)
+        {
+            try
+            {
+                var blobServiceClient = new BlobServiceClient(connectionString);
+                var blobContainer = blobServiceClient.GetBlobContainerClient(container);
+                var htmlBlobContainer = htmlContainer == null ? null : blobServiceClient.GetBlobContainerClient(container);
+
+                // Create the container if it doesn't exist
+                await blobContainer.CreateIfNotExistsAsync();
+                if(htmlContainer != null)
+                    await htmlBlobContainer.CreateIfNotExistsAsync();
+
+
+                var localFiles = Directory.GetFiles(path, "*", SearchOption.AllDirectories).Select(f => Path.Combine(address, Path.GetRelativePath(path, f))).ToHashSet();
+                var blobs = blobContainer.GetBlobs(prefix:address).ToDictionary(b => b.Name);
+
+                // Delete blobs that no longer exist in the local directory
+                foreach (var blob in blobs)
+                {
+                    if (!localFiles.Contains(blob.Key))
+                    {
+                        logger.LogInformation($"Deleting blob {blob.Key} as it no longer exists in the local directory.");
+                        await blobContainer.DeleteBlobAsync(blob.Key);
+                    }
+                }
+
+                foreach (var inputFile in Directory.GetFiles(path))
+                {
+                    try
+                    {
+                        var filePath = Path.Combine(address, Path.GetRelativePath(path, inputFile)).Replace('\\', '/');
+                        var fileContent = await File.ReadAllTextAsync(inputFile);
+                        var fileLastModified = File.GetLastWriteTimeUtc(inputFile);
+
+                        // Check if the blob exists and if the local file is newer
+                        if (blobs.TryGetValue(filePath, out var blobItem) && blobItem.Properties.LastModified >= fileLastModified)
+                        {
+                            logger.LogInformation($"Skipping upload for {filePath} as the blob is up-to-date.");
+                            continue;
+                        }
+
+                        var extension = Path.GetExtension(filePath);
+                        logger.LogInformation($"Uploading file {inputFile} to {filePath}. Parsing as {extension}");
+
+                        var metadata = extension switch
+                        {
+                            ".md" => await ParseMarkdown(address, logger, filePath, fileContent, htmlBlobContainer),
+                            _ => DefaultMetadata(filePath, extension.Trim('.'))
+                        };
+
+                        logger.LogInformation("Uploading {filePath} with {metadata}", filePath, JsonSerializer.Serialize(metadata));
+                        var blob = blobContainer.GetBlobClient(filePath);
+                        await blob.UploadAsync(new BinaryData(fileContent), overwrite: true);
+                        await blob.SetMetadataAsync(metadata);
+                    }
+                    catch (Exception ex)
+                    {
+                        logger.LogError(ex, "An error occurred during file upload.");
+                    }
+                }
+
+                return true;
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "An error occurred during execution.");
+                return false;
+            }
+        }
+
+        private static Dictionary<string, string> DefaultMetadata(string filePath, string type)
+        {
+            return new Dictionary<string, string>
+            {
+                { "type", type },
+                { "path", filePath }
+            };
+}
+
+        private static async Task<IDictionary<string, string>> ParseMarkdown(string address, ILogger logger, string filePath, string fileContent, BlobContainerClient htmlContainer)
+        {
+            // Parse metadata and HTML
+            var (article, html) = MarkdownIndexer.ParseArticle(filePath, fileContent, address);
+
+            // Set metadata on the HTML blob
+            var metadata = article?.ToMetadata(filePath, "md") ?? DefaultMetadata(filePath, "md");
+
+            if (htmlContainer != null)
+            {
+                // Store metadata and HTML in blob storage
+                var htmlFilePath = Path.ChangeExtension(filePath, ".html");
+                logger.LogInformation("Uploading pre-rendered html {path}", htmlFilePath);
+                var htmlBlobClient = htmlContainer.GetBlobClient(htmlFilePath);
+                await htmlBlobClient.UploadAsync(new BinaryData(html), overwrite: true);
+                await htmlBlobClient.SetMetadataAsync(metadata);
+
+            }
+
+            return metadata;
+        }
+    }
+}

src/MeshWeaver.Blazor/BlazorClientRegistry.cs

@@ -1,6 +1,7 @@
 using System.Collections.Immutable;
 using System.Runtime.CompilerServices;
 using System.Text.Json;
+using MeshWeaver.Application;
 using MeshWeaver.Data;
 using Microsoft.DotNet.Interactive.Formatting;
 using MeshWeaver.Data.Serialization;
@@ -23,7 +24,7 @@ internal static MessageHubConfiguration AddBlazor(
     ) => config
         .AddData()
         .AddLayoutClient(c => (configuration ?? (x => x)).Invoke(c.WithView(DefaultFormatting)))
-
+        .WithTypes(typeof(ApplicationAddress))
     ;
     #region Standard Formatting
     private static ViewDescriptor DefaultFormatting(