From a482b2dac480fe43630749454baba490b12017c5 Mon Sep 17 00:00:00 2001 From: Anna Kuleshova Date: Mon, 30 Sep 2024 11:54:56 +0300 Subject: [PATCH 01/25] heading are added to the reports --- .../Markdown/AnnualReport/ClientsOverview.md | 20 +++++++++++++++ .../AnnualReport/EmployeesOverview.md | 20 +++++++++++++++ .../Markdown/AnnualReport/OrdersOverview.md | 25 +++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/AnnualReport/ClientsOverview.md b/modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/AnnualReport/ClientsOverview.md index 4b7f6e423..cefc7e830 100644 --- a/modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/AnnualReport/ClientsOverview.md +++ b/modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/AnnualReport/ClientsOverview.md @@ -1,3 +1,23 @@ +--- +Name: "Top 5 Clients by Amount per Month" +Description: "The report analyzes the top 5 clients based on their total purchase amounts, highlighting key clients such as QUICK-Stop and Ernst Handel. It suggests targeted reward strategies to enhance customer satisfaction and retention, including discounts, exclusive access, and personalized services. Additionally, the report provides general recommendations for creating effective reward programs to foster overall customer loyalty." +Thumbnail: "images/thumbnail.jpg" +Published: "2024-09-24" +Authors: + - "Roland B�rgi" + - "Anna Kuleshova" +Tags: + - "Top Clients" + - "Purchase Analysis" + - "Customer Loyalty" + - "Reward Programs" + - "Client Retention" + - "VIP Clients" + - "Discount Strategies" + - "Customer Satisfaction" + - "Loyalty Programs" + - "Business Development" +--- # Top 5 Clients by Amount per Month This report provides a detailed comparison of sales across various product categories between the years 2022 and 2023. The analysis highlights the growth in sales for each category, showcasing the overall performance and trends within the market. The data reveals significant increases in sales across all categories, with some experiencing remarkable growth rates. +Click the year above the chart to select which information to display. +
@("SalesByCategoryWithPrevYear"){ Id = "?Year=2023" } diff --git a/modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/AnnualReport/ClientsOverview.md b/modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/AnnualReport/TopClientsOverview.md similarity index 100% rename from modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/AnnualReport/ClientsOverview.md rename to modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/AnnualReport/TopClientsOverview.md diff --git a/modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/AnnualReport/ProductOverview.md b/modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/AnnualReport/TopProductsOverview.md similarity index 100% rename from modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/AnnualReport/ProductOverview.md rename to modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/AnnualReport/TopProductsOverview.md diff --git a/modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/AnnualReport/EmployeesOverview.md b/modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/AnnualReport/TopSalesRepresentatives.md similarity index 100% rename from modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/AnnualReport/EmployeesOverview.md rename to modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/AnnualReport/TopSalesRepresentatives.md diff --git a/modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/Northwind.md b/modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/Northwind.md deleted file mode 100644 index 16b281d8d..000000000 --- a/modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/Northwind.md +++ /dev/null @@ -1,21 +0,0 @@ -# Overview of the Project - -The Northwind.ViewModel project is designed to provide a structured and efficient way to represent and manipulate data from the Northwind Database within an application. This project focuses on creating view models that encapsulate the data and business logic required for the presentation layer, ensuring a clear separation of concerns and enhancing maintainability. - -## Key Features - -- **Data Representation**: Provides view models that represent the data entities from the Northwind Database, such as products, orders, customers, and employees. -- **Business Logic**: Encapsulates business logic within the view models to ensure that the presentation layer remains clean and focused on UI concerns. -- **Data Binding**: Facilitates data binding in UI frameworks, making it easier to display and interact with data in a user-friendly manner. -- **Validation**: Includes validation logic within the view models to ensure data integrity and provide immediate feedback to users. -- **Interactivity**: Supports interactive features, such as filtering and sorting, to enhance the user experience in reporting and data visualization. - -## Components - -- **ProductViewModel**: Represents product data, including details such as name, category, price, and stock levels. -- **OrderViewModel**: Encapsulates order information, including customer details, order items, and order status. -- **CustomerViewModel**: Manages customer data, including contact information, order history, and preferences. -- **EmployeeViewModel**: Handles employee information, including roles, contact details, and performance metrics. - -The Northwind.ViewModel project aims to provide a robust foundation for building interactive and data-driven applications, leveraging the rich dataset of the Northwind Database. - diff --git a/modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/Overview.md b/modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/Overview.md deleted file mode 100644 index 28d8f8ad3..000000000 --- a/modules/Northwind/MeshWeaver.Northwind.ViewModel/Markdown/Overview.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -Name: "Northwind Overview" -Description: "This is a sample description of the article." -Thumbnail: "images/thumbnail.jpg" -Published: "2024-09-24" -Authors: - - "Roland Bürgi" - - "Anna Kuleshova" -Tags: - - "Northwind" - - "Conceptual" ---- - -# Northwind - -This is a model for the [Northwind Database](https://github.com/microsoft/sql-server-samples/blob/master/samples/databases/northwind-pubs/readme.md). -It is a small data domain modelling an e-commerce store. The complexity is moderate, it is more realistic than, -e.g. a TODO application or a blog application. Nevertheless, the complexity is not too big, so that we can -describe the basic principles of data modeling. - -## Counter - -The counter doesn't really fit here, we should move it to a spearate project. - -@("ProductSummary"){ Layout = "Documentation" } - -## Interactive Reporting - -Make your reports interactive by using the [Interactive Reporting](InteractiveReporting.md) feature. Here an example: - -@("OrderSummary"){ Layout = "Documentation" } - - From 2f71a834a4105ed17fc5563012f19239b8bb7fec Mon Sep 17 00:00:00 2001 From: Dmitry Kalabin Date: Thu, 3 Oct 2024 13:13:32 +0200 Subject: [PATCH 17/25] fix after merge --- .../Documentation/DocumentationLayout.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MeshWeaver.Documentation/Documentation/DocumentationLayout.cs b/src/MeshWeaver.Documentation/Documentation/DocumentationLayout.cs index f5b2cb5bd..1716a735a 100644 --- a/src/MeshWeaver.Documentation/Documentation/DocumentationLayout.cs +++ b/src/MeshWeaver.Documentation/Documentation/DocumentationLayout.cs @@ -20,7 +20,7 @@ public static LayoutDefinition AddDocumentation(this LayoutDefinition layout) _ => true, (tabs, host, ctx) => tabs.WithView(NamedArea(host.Stream.Reference.Area), ctx.DisplayName) ) - .WithView(nameof(Doc), Doc); + .WithView(nameof(Doc), (Func>)Doc); From 89d5ec06f3c7e8700e8b440e826bbdfcdec4c46a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Thu, 3 Oct 2024 15:31:04 +0200 Subject: [PATCH 18/25] bug fixes # Conflicts: # src/MeshWeaver.Layout/LayoutDefinitionExtensions.cs --- .../OrdersSummaryArea.cs | 3 +- src/MeshWeaver.Data.Contract/EntityStore.cs | 8 ++- .../IDomainLayoutService.cs | 30 ++------ .../Composition/LayoutAreaHost.cs | 3 +- src/MeshWeaver.Layout/ContainerControl.cs | 2 +- .../LayoutDefinitionExtensions.cs | 2 +- src/MeshWeaver.Layout/Template.cs | 71 ++++++++++++++++--- src/MeshWeaver.Layout/UiControl.cs | 11 ++- test/MeshWeaver.Layout.Test/LayoutTest.cs | 5 +- 9 files changed, 85 insertions(+), 50 deletions(-) diff --git a/modules/Northwind/MeshWeaver.Northwind.ViewModel/OrdersSummaryArea.cs b/modules/Northwind/MeshWeaver.Northwind.ViewModel/OrdersSummaryArea.cs index 6a79741fd..deb73910a 100644 --- a/modules/Northwind/MeshWeaver.Northwind.ViewModel/OrdersSummaryArea.cs +++ b/modules/Northwind/MeshWeaver.Northwind.ViewModel/OrdersSummaryArea.cs @@ -47,7 +47,8 @@ RenderingContext ctx ) { var years = layoutArea - .Workspace.GetObservable() + .Workspace + .GetObservable() .DistinctUntilChanged() .Select(x => x.Select(y => y.OrderDate.Year) diff --git a/src/MeshWeaver.Data.Contract/EntityStore.cs b/src/MeshWeaver.Data.Contract/EntityStore.cs index eac9926e3..f39b855d0 100644 --- a/src/MeshWeaver.Data.Contract/EntityStore.cs +++ b/src/MeshWeaver.Data.Contract/EntityStore.cs @@ -149,7 +149,13 @@ public override int GetHashCode() .Aggregate((x, y) => x ^ y); } -public record EntityStoreAndUpdates(IEnumerable Changes, EntityStore Store); + +public record EntityStoreAndUpdates(IEnumerable Changes, EntityStore Store) +{ + public EntityStoreAndUpdates(EntityStore Store) : this([], Store) + { + } +} public record EntityStoreUpdate(string Collection, object Id, object Value) { diff --git a/src/MeshWeaver.Documentation/IDomainLayoutService.cs b/src/MeshWeaver.Documentation/IDomainLayoutService.cs index eb0d9b829..a46220515 100644 --- a/src/MeshWeaver.Documentation/IDomainLayoutService.cs +++ b/src/MeshWeaver.Documentation/IDomainLayoutService.cs @@ -97,11 +97,13 @@ public DomainViewConfiguration WithPropertyView(Func context.TypeDefinition.Type.GetProperties() .Aggregate(Controls.EditForm, (grid, property) => @@ -111,31 +113,7 @@ public object DetailsLayout(LayoutAreaHost host, RenderingContext ctx, EntityRen ) ); - var stream = host.Workspace - .GetStreamFor(new EntityReference(context.TypeDefinition.CollectionName, context.Id), host.Stream.Subscriber); - object current = null; - var forwardSubscription = stream.Subscribe(changeItem => - { - if (Equals(changeItem.Value, current)) - return; - current = changeItem.Value; - host.Stream.SetData(context.IdString, changeItem.SetValue(current)); - }); - //var subscription = host.Stream.Subscribe(changeItem => - //{ - // if(changeItem.Patch?.Value is null) - // return; - // if (changeItem.ChangedBy.Equals(host.Stream.Subscriber) &&changeItem.Patch.Value.Operations.Any(x => x.Path.ToString().StartsWith(ret.DataContext))) - // { - // var instance = changeItem.Value.GetCollection(LayoutAreaReference.Data)?.Instances - // .GetValueOrDefault(context.IdString); - // if(instance is not null) - // stream.Update(i => changeItem.SetValue(instance)); - // } - //}); - - host.AddDisposable(context.RenderingContext.Area, forwardSubscription); return ret; } diff --git a/src/MeshWeaver.Layout/Composition/LayoutAreaHost.cs b/src/MeshWeaver.Layout/Composition/LayoutAreaHost.cs index 75a600741..8b2cad450 100644 --- a/src/MeshWeaver.Layout/Composition/LayoutAreaHost.cs +++ b/src/MeshWeaver.Layout/Composition/LayoutAreaHost.cs @@ -254,8 +254,7 @@ private EntityStoreAndUpdates DisposeExistingAreas(EntityStore store, RenderingC } - internal EntityStoreAndUpdates - RenderArea(RenderingContext context, ViewStream generator, EntityStore store) + internal EntityStoreAndUpdates RenderArea(RenderingContext context, ViewStream generator, EntityStore store) { var ret = DisposeExistingAreas(store, context); AddDisposable(context.Parent?.Area ?? context.Area, diff --git a/src/MeshWeaver.Layout/ContainerControl.cs b/src/MeshWeaver.Layout/ContainerControl.cs index 1936553e9..8aa3d96f1 100644 --- a/src/MeshWeaver.Layout/ContainerControl.cs +++ b/src/MeshWeaver.Layout/ContainerControl.cs @@ -134,7 +134,7 @@ public TControl WithView(Func viewDefini public TControl WithView(Func> viewDefinition) => WithView((h, c, _) => viewDefinition.Invoke(h, c), x => x); public TControl WithView(Func> viewDefinition) - => WithView((h,c,store) => h.RenderArea(c,(hh,cc,_)=>viewDefinition.Invoke(hh,cc), store), x => x); + => WithView((h,c,_) => viewDefinition.Invoke(h,c), x => x); public TControl WithView(ViewStream viewDefinition) => WithView(viewDefinition, x => x); public TControl WithView(Func> viewDefinition, Func options) diff --git a/src/MeshWeaver.Layout/LayoutDefinitionExtensions.cs b/src/MeshWeaver.Layout/LayoutDefinitionExtensions.cs index c21182a29..65cf745c8 100644 --- a/src/MeshWeaver.Layout/LayoutDefinitionExtensions.cs +++ b/src/MeshWeaver.Layout/LayoutDefinitionExtensions.cs @@ -148,7 +148,7 @@ Func view public static EntityStoreAndUpdates UpdateControl(this EntityStore store, string id, UiControl control) => new ([new EntityStoreUpdate(LayoutAreaReference.Areas, id, control)], store.Update(LayoutAreaReference.Areas, i => i.Update(id, control))); public static EntityStoreAndUpdates UpdateData(this EntityStore store, string id, object control) - => new ([new EntityStoreUpdate(LayoutAreaReference.Data, id, control)], store.Update(LayoutAreaReference.Data, i => i.Update(id, control))); + => new([new EntityStoreUpdate(LayoutAreaReference.Data, id, control)], store.Update(LayoutAreaReference.Data, i => i.Update(id, control))); public static LayoutDefinition WithRenderer( this LayoutDefinition layout, diff --git a/src/MeshWeaver.Layout/Template.cs b/src/MeshWeaver.Layout/Template.cs index f0602db47..5dd00499b 100644 --- a/src/MeshWeaver.Layout/Template.cs +++ b/src/MeshWeaver.Layout/Template.cs @@ -1,6 +1,7 @@ using System.Linq.Expressions; using System.Reflection; using MeshWeaver.Data; +using MeshWeaver.Data.Serialization; using MeshWeaver.Domain.Layout; using MeshWeaver.Layout.Composition; using MeshWeaver.Layout.DataBinding; @@ -9,16 +10,34 @@ namespace MeshWeaver.Layout; public static class Template{ - /// /// This is a generic template method which can be used if streams are connected to synchronize with Workspace. /// - /// + /// Type of the view + /// The type of the entity to be data bound /// Id to be referenced in the data binding /// View Template. - /// - public static TView Bind(string id, Expression> dataTemplate) - where TView : UiControl => BindObject(null, id, dataTemplate); + /// The view template + public static TView Bind(this ISynchronizationStream stream, string id, Expression> dataTemplate) + where TView : UiControl + { + object current = null; + + return (TView)GetTemplateControl(id, dataTemplate) + .WithBuildup((host, context, store) => + { + var forwardSubscription = stream.Subscribe(changeItem => + { + if (Equals(changeItem.Value, current)) + return; + current = changeItem.Value; + host.Stream.SetData(id, changeItem.SetValue(current)); + }); + host.AddDisposable(context.Area, forwardSubscription); + return new(store); + }); + + } /// @@ -52,6 +71,33 @@ Expression> dataTemplate return new ItemTemplateControl(view, data); } + /// + /// Takes expression tree of data template and replaces all property getters by binding instances and sets data context property + /// + [ReplaceBindMethod] + public static TView BindObservable( + this IObservable stream, + string id, + Expression> dataTemplate + ) + where TView : UiControl + { + object current = null; + return (TView)GetTemplateControl(id, dataTemplate) + .WithBuildup((host, context, store) => + { + var forwardSubscription = stream.Subscribe(val => + { + if (Equals(val, current)) + return; + current = val; + host.Stream.SetData(id, new ChangeItem(host.Stream.Owner, host.Stream.Reference, val, host.Stream.Owner, null, host.Hub.Version)); + }); + host.AddDisposable(context.Area, forwardSubscription); + return new(store); + }); + } + private static readonly MethodInfo ItemTemplateMethodNonGeneric = ReflectionHelper.GetStaticMethodGeneric( () => ItemTemplateNonGeneric(default(IEnumerable), null) ); @@ -99,12 +145,19 @@ Expression> dataTemplate ) where TView : UiControl { - var topLevel = UpdateData(data, id); + var view = GetTemplateControl(id, dataTemplate); + if(data != null) + view = (TView)view.WithBuildup((_,_,store) => store.UpdateData(id, data)); + return view; + } + + private static TView GetTemplateControl(string id, Expression> dataTemplate) + where TView : UiControl + { + var topLevel = LayoutAreaReference.GetDataPointer(id); var view = dataTemplate.Build(topLevel, out var _); if (view == null) throw new ArgumentException("Data template was not specified."); - if(data != null) - view = (TView)view.WithUpdates(store => store.UpdateData(id, data)); return view; } @@ -170,7 +223,7 @@ Expression> dataTemplate { DataContext = dataContext } - .WithUpdates(store => store.UpdateData(id, data)) + .WithBuildup((_,_,store) => store.UpdateData(id, data)) ; } diff --git a/src/MeshWeaver.Layout/UiControl.cs b/src/MeshWeaver.Layout/UiControl.cs index 91a2721bc..d19ac4d17 100644 --- a/src/MeshWeaver.Layout/UiControl.cs +++ b/src/MeshWeaver.Layout/UiControl.cs @@ -69,11 +69,10 @@ public UiControl AddSkin(Skin skin) => this with { Skins = (Skins ?? ImmutableList.Empty).Add(skin) }; - protected ImmutableList> Updates { get; init; } = - ImmutableList>.Empty; - public UiControl WithUpdates(Func update) + protected ImmutableList> Buildup { get; init; } = []; + public UiControl WithBuildup(Func buildup) { - return this with { Updates = Updates.Add(update) }; + return this with { Buildup = Buildup.Add(buildup) }; } @@ -181,10 +180,10 @@ protected override void Dispose() protected override EntityStoreAndUpdates Render (LayoutAreaHost host, RenderingContext context, EntityStore store) => - Updates + Buildup .Aggregate(RenderSelf(host, context, store), (r, u) => { - var updated = u.Invoke(r.Store); + var updated = u.Invoke(host, context, r.Store); return new(r.Changes.Concat(updated.Changes), updated.Store); }); protected virtual EntityStoreAndUpdates RenderSelf(LayoutAreaHost host, RenderingContext context, EntityStore store) diff --git a/test/MeshWeaver.Layout.Test/LayoutTest.cs b/test/MeshWeaver.Layout.Test/LayoutTest.cs index d2d56e908..52adcbd0f 100644 --- a/test/MeshWeaver.Layout.Test/LayoutTest.cs +++ b/test/MeshWeaver.Layout.Test/LayoutTest.cs @@ -146,8 +146,7 @@ private static object UpdatingView() return Controls .Stack - .WithView( (_, _) => - Template.Bind(toolbar, nameof(toolbar), tb => Controls.Text(tb.Year)), "Toolbar") + .WithView(Template.Bind(toolbar, nameof(toolbar), tb => Controls.Text(tb.Year)), "Toolbar") .WithView((area, _) => area.GetDataStream(nameof(toolbar)) .Select(tb => Controls.Html($"Report for year {tb.Year}")), "Content"); @@ -166,7 +165,7 @@ public async Task TestUpdatingView() ); var reportArea = $"{reference.Area}/Content"; var content = await stream.GetControlStream(reportArea) - .Timeout(3.Seconds()) + // .Timeout(3.Seconds()) .FirstAsync(x => x is not null); content.Should().BeOfType().Which.Data.ToString().Should().Contain("2024"); From e6d721a032ef0c6d4b4dc6ef5f6ebce51ceaf51e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Fri, 4 Oct 2024 10:26:50 +0200 Subject: [PATCH 19/25] fixing string id for details view --- src/MeshWeaver.Documentation/DomainViews.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/MeshWeaver.Documentation/DomainViews.cs b/src/MeshWeaver.Documentation/DomainViews.cs index 87a7c4251..8c7991428 100644 --- a/src/MeshWeaver.Documentation/DomainViews.cs +++ b/src/MeshWeaver.Documentation/DomainViews.cs @@ -40,7 +40,8 @@ public static object Details(LayoutAreaHost area, RenderingContext ctx) { var typeDefinition = typeSource.TypeDefinition; var idString = parts[1]; - var id = JsonSerializer.Deserialize(idString, typeDefinition.GetKeyType()); + var keyType = typeDefinition.GetKeyType(); + var id = keyType == typeof(string) ? idString : JsonSerializer.Deserialize(idString, keyType); return area.Hub.ServiceProvider.GetRequiredService().Render(new(area, typeDefinition, idString, id, ctx)); } catch (Exception e) From 86e6de0d313fd1c9ed25d6878ea3e227f77c6273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Fri, 4 Oct 2024 23:05:14 +0200 Subject: [PATCH 20/25] added MeshWeaver.Azure.Publish --- .config/dotnet-tools.json | 12 +++ Directory.Packages.props | 2 + MeshWeaver.sln | 8 +- .../MeshWeaver.Northwind.ViewModel.csproj | 53 ++++++++----- .../MeshWeaver.Azure.Publish.csproj | 23 ++++++ ...eshWeaver.Azure.Publish.runtimeconfig.json | 9 +++ src/MeshWeaver.Azure.Publish/Program.cs | 77 +++++++++++++++++++ src/MeshWeaver.Search/MarkdownIndexer.cs | 3 +- src/MeshWeaver.Search/MeshArticleIndex.cs | 19 +++++ .../MeshWeaver.Search.csproj | 2 +- .../ArticleParsingTest.cs | 1 - 11 files changed, 185 insertions(+), 24 deletions(-) create mode 100644 .config/dotnet-tools.json create mode 100644 src/MeshWeaver.Azure.Publish/MeshWeaver.Azure.Publish.csproj create mode 100644 src/MeshWeaver.Azure.Publish/MeshWeaver.Azure.Publish.runtimeconfig.json create mode 100644 src/MeshWeaver.Azure.Publish/Program.cs diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json new file mode 100644 index 000000000..49c65db34 --- /dev/null +++ b/.config/dotnet-tools.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "meshweaver.azure.publish": { + "version": "1.0.0", + "commands": [ + "meshweaver-azure-publish" + ] + } + } +} \ No newline at end of file diff --git a/Directory.Packages.props b/Directory.Packages.props index f4423de37..17c2af4d6 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -37,6 +37,8 @@ + + diff --git a/MeshWeaver.sln b/MeshWeaver.sln index 43cbe1b2e..d2738355d 100644 --- a/MeshWeaver.sln +++ b/MeshWeaver.sln @@ -207,8 +207,8 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "monolith", "monolith", "{4E1A12BB-5348-4DA4-B05F-F831DC16F7DE}" ProjectSection(SolutionItems) = preProject monolith\Directory.Build.props = monolith\Directory.Build.props - Readme.md = Readme.md MIT License.md = MIT License.md + Readme.md = Readme.md EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MeshWeaver.Portal", "monolith\MeshWeaver.Portal\MeshWeaver.Portal.csproj", "{1471FB82-912E-411A-9141-096E0AECAAAE}" @@ -221,6 +221,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MeshWeaver.Search", "src\Me EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MeshWeaver.Search.Test", "test\MeshWeaver.Search.Test\MeshWeaver.Search.Test.csproj", "{BDEC6D93-F212-456F-BEEE-DE2BADAE2A56}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MeshWeaver.Azure.Publish", "src\MeshWeaver.Azure.Publish\MeshWeaver.Azure.Publish.csproj", "{844B4566-81A4-4CB2-9761-9B174FB7BCCB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -587,6 +589,10 @@ Global {BDEC6D93-F212-456F-BEEE-DE2BADAE2A56}.Debug|Any CPU.Build.0 = Debug|Any CPU {BDEC6D93-F212-456F-BEEE-DE2BADAE2A56}.Release|Any CPU.ActiveCfg = Release|Any CPU {BDEC6D93-F212-456F-BEEE-DE2BADAE2A56}.Release|Any CPU.Build.0 = Release|Any CPU + {844B4566-81A4-4CB2-9761-9B174FB7BCCB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {844B4566-81A4-4CB2-9761-9B174FB7BCCB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {844B4566-81A4-4CB2-9761-9B174FB7BCCB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {844B4566-81A4-4CB2-9761-9B174FB7BCCB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/modules/Northwind/MeshWeaver.Northwind.ViewModel/MeshWeaver.Northwind.ViewModel.csproj b/modules/Northwind/MeshWeaver.Northwind.ViewModel/MeshWeaver.Northwind.ViewModel.csproj index 1c2e39df5..928174865 100644 --- a/modules/Northwind/MeshWeaver.Northwind.ViewModel/MeshWeaver.Northwind.ViewModel.csproj +++ b/modules/Northwind/MeshWeaver.Northwind.ViewModel/MeshWeaver.Northwind.ViewModel.csproj @@ -1,32 +1,48 @@  + net8.0 true true - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + true bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml true + $(AssemblyName).xml @@ -34,5 +50,4 @@ true - - + \ No newline at end of file diff --git a/src/MeshWeaver.Azure.Publish/MeshWeaver.Azure.Publish.csproj b/src/MeshWeaver.Azure.Publish/MeshWeaver.Azure.Publish.csproj new file mode 100644 index 000000000..f2f285cfd --- /dev/null +++ b/src/MeshWeaver.Azure.Publish/MeshWeaver.Azure.Publish.csproj @@ -0,0 +1,23 @@ + + + Exe + true + meshweaver-azure-publish + 1.0.0 + + + + + + + + + + + + diff --git a/src/MeshWeaver.Azure.Publish/MeshWeaver.Azure.Publish.runtimeconfig.json b/src/MeshWeaver.Azure.Publish/MeshWeaver.Azure.Publish.runtimeconfig.json new file mode 100644 index 000000000..170449ccb --- /dev/null +++ b/src/MeshWeaver.Azure.Publish/MeshWeaver.Azure.Publish.runtimeconfig.json @@ -0,0 +1,9 @@ +{ + "runtimeOptions": { + "tfm": "net8.0", + "framework": { + "name": "Microsoft.NETCore.App", + "version": "8.0.0" + } + } +} diff --git a/src/MeshWeaver.Azure.Publish/Program.cs b/src/MeshWeaver.Azure.Publish/Program.cs new file mode 100644 index 000000000..a394bfe06 --- /dev/null +++ b/src/MeshWeaver.Azure.Publish/Program.cs @@ -0,0 +1,77 @@ +using System.CommandLine; +using System.CommandLine.NamingConventionBinder; +using Azure.Storage.Blobs; +using MeshWeaver.Search; + +namespace MeshWeaver.Azure.Publish +{ + public static class Program + { + public static async Task Main(string[] args) + { + var rootCommand = new RootCommand + { + new Option( + "--path", + "The input files to process"), + new Option( + "--connection-string", + "The connection string for the blob storage"), + new Option( + "--container", + "The name of the blob container"), + new Option( + "--address", + "The address to use") + }; + + rootCommand.Description = "MeshWeaver Build Tasks"; + + rootCommand.Handler = CommandHandler.Create((filePath, blobStorageConnectionString, blobContainerName, address) => + { + // Call your custom task logic here + Console.WriteLine($"Processing files from path: {filePath}"); + Console.WriteLine($"Blob Storage Connection String: {blobStorageConnectionString}"); + Console.WriteLine($"Blob Container Name: {blobContainerName}"); + Console.WriteLine($"Address: {address}"); + }); + + await rootCommand.InvokeAsync(args); + } + + public static bool Execute(string connectionString, string container, string path, string address) + { + try + { + var blobServiceClient = new BlobServiceClient(connectionString); + var blobContainerClient = blobServiceClient.GetBlobContainerClient(container); + + foreach (var inputFile in Directory.GetFiles(path)) + { + var filePath = Path.GetRelativePath(path, inputFile); + var fileContent = File.ReadAllText(filePath); + + // Parse metadata and HTML + var (article, html) = MarkdownIndexer.ParseArticle(filePath, fileContent, address); + + // Store metadata and HTML in blob storage + var htmlBlobClient = blobContainerClient.GetBlobClient(Path.GetFileName(filePath)); + + htmlBlobClient.Upload(new BinaryData(html)); + + // Set metadata on the HTML blob + htmlBlobClient.SetMetadata(article.ToMetadata()); + } + + return true; + } + catch (Exception ex) + { + Console.WriteLine(ex); + return false; + } + } + + + } +} diff --git a/src/MeshWeaver.Search/MarkdownIndexer.cs b/src/MeshWeaver.Search/MarkdownIndexer.cs index eed0723b4..7d9e7cd5a 100644 --- a/src/MeshWeaver.Search/MarkdownIndexer.cs +++ b/src/MeshWeaver.Search/MarkdownIndexer.cs @@ -3,10 +3,9 @@ using Markdig; using Markdig.Extensions.Yaml; using Markdig.Syntax; -using MeshWeaver.Search; using Markdown = Markdig.Markdown; -namespace MeshWeaver.Ai.Index; +namespace MeshWeaver.Search; public static class MarkdownIndexer { diff --git a/src/MeshWeaver.Search/MeshArticleIndex.cs b/src/MeshWeaver.Search/MeshArticleIndex.cs index c4c5198d8..4e9e36d3c 100644 --- a/src/MeshWeaver.Search/MeshArticleIndex.cs +++ b/src/MeshWeaver.Search/MeshArticleIndex.cs @@ -1,4 +1,6 @@ using System.ComponentModel.DataAnnotations; +using System.Globalization; +using System.Text.Json; using Azure.Search.Documents.Indexes; using Azure.Search.Documents.Indexes.Models; @@ -25,4 +27,21 @@ public class MeshArticleIndex [VectorSearchField(VectorSearchDimensions = 1536, VectorSearchProfileName = "vector-profile")] public float[] VectorRepresentation { get; set; } + + public IDictionary ToMetadata() => + new Dictionary() + { + { nameof(Url), Url }, + { nameof(Name), Name }, + { nameof(Description), Description }, + { nameof(Thumbnail), Thumbnail }, + { nameof(Published), Published.ToString(CultureInfo.InvariantCulture) }, + { nameof(Authors), JsonSerializer.Serialize(Authors) }, + { nameof(Tags), JsonSerializer.Serialize(Tags) }, + { nameof(Path), Path }, + }; + + + } + diff --git a/src/MeshWeaver.Search/MeshWeaver.Search.csproj b/src/MeshWeaver.Search/MeshWeaver.Search.csproj index 2c848d917..eb6b99c65 100644 --- a/src/MeshWeaver.Search/MeshWeaver.Search.csproj +++ b/src/MeshWeaver.Search/MeshWeaver.Search.csproj @@ -1,6 +1,6 @@  - + diff --git a/test/MeshWeaver.Search.Test/ArticleParsingTest.cs b/test/MeshWeaver.Search.Test/ArticleParsingTest.cs index 0e0f5f65f..4c78dad0f 100644 --- a/test/MeshWeaver.Search.Test/ArticleParsingTest.cs +++ b/test/MeshWeaver.Search.Test/ArticleParsingTest.cs @@ -1,7 +1,6 @@ using System.Text; using Azure.Provisioning.Storage; using Azure.Storage.Blobs; -using MeshWeaver.Ai.Index; using MeshWeaver.Hub.Fixture; using Xunit.Abstractions; From 756d7357e165f37ad86d6b13826fa0b48b277d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sat, 5 Oct 2024 16:01:44 +0200 Subject: [PATCH 21/25] iterating on publish tool --- .config/dotnet-tools.json | 2 +- Directory.Packages.props | 5 ++ .../MeshWeaver.Azure.Publish.csproj | 3 +- src/MeshWeaver.Azure.Publish/Program.cs | 86 +++++++++++++++---- 4 files changed, 77 insertions(+), 19 deletions(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 49c65db34..8e3aae52b 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -5,7 +5,7 @@ "meshweaver.azure.publish": { "version": "1.0.0", "commands": [ - "meshweaver-azure-publish" + "MeshWeaver.Azure.Publish" ] } } diff --git a/Directory.Packages.props b/Directory.Packages.props index 17c2af4d6..6a70d02f2 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -16,6 +16,8 @@ + + @@ -32,6 +34,9 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all + + + diff --git a/src/MeshWeaver.Azure.Publish/MeshWeaver.Azure.Publish.csproj b/src/MeshWeaver.Azure.Publish/MeshWeaver.Azure.Publish.csproj index f2f285cfd..b1ae36a0f 100644 --- a/src/MeshWeaver.Azure.Publish/MeshWeaver.Azure.Publish.csproj +++ b/src/MeshWeaver.Azure.Publish/MeshWeaver.Azure.Publish.csproj @@ -2,11 +2,10 @@ Exe true - meshweaver-azure-publish - 1.0.0 + diff --git a/src/MeshWeaver.Azure.Publish/Program.cs b/src/MeshWeaver.Azure.Publish/Program.cs index a394bfe06..d38fdfea7 100644 --- a/src/MeshWeaver.Azure.Publish/Program.cs +++ b/src/MeshWeaver.Azure.Publish/Program.cs @@ -1,7 +1,9 @@ using System.CommandLine; using System.CommandLine.NamingConventionBinder; +using System.Text.Json; using Azure.Storage.Blobs; using MeshWeaver.Search; +using Microsoft.Extensions.Logging; namespace MeshWeaver.Azure.Publish { @@ -9,6 +11,12 @@ public static class Program { public static async Task Main(string[] args) { + using var loggerFactory = LoggerFactory.Create(builder => + { + builder.AddConsole(); + }); + var logger = loggerFactory.CreateLogger(typeof(Program)); + var rootCommand = new RootCommand { new Option( @@ -27,51 +35,97 @@ public static async Task Main(string[] args) rootCommand.Description = "MeshWeaver Build Tasks"; - rootCommand.Handler = CommandHandler.Create((filePath, blobStorageConnectionString, blobContainerName, address) => + rootCommand.Handler = CommandHandler.Create(async (path, connectionString, container, address) => { + if (string.IsNullOrEmpty(path) || string.IsNullOrEmpty(connectionString) || string.IsNullOrEmpty(container) || string.IsNullOrEmpty(address)) + { + logger.LogError("Error: All options (--path, --connection-string, --container, --address) must be provided and cannot be null or empty."); + return; + } + // Call your custom task logic here - Console.WriteLine($"Processing files from path: {filePath}"); - Console.WriteLine($"Blob Storage Connection String: {blobStorageConnectionString}"); - Console.WriteLine($"Blob Container Name: {blobContainerName}"); - Console.WriteLine($"Address: {address}"); + logger.LogInformation($"Processing files from path: {path}"); + logger.LogInformation($"Blob Storage Connection String: {connectionString}"); + logger.LogInformation($"Blob Container Name: {container}"); + logger.LogInformation($"Address: {address}"); + + await ExecuteAsync(connectionString, container, path, address, logger); }); await rootCommand.InvokeAsync(args); } - public static bool Execute(string connectionString, string container, string path, string address) + public static async Task ExecuteAsync(string connectionString, string container, string path, string address, ILogger logger) { try { var blobServiceClient = new BlobServiceClient(connectionString); var blobContainerClient = blobServiceClient.GetBlobContainerClient(container); + // Create the container if it doesn't exist + await blobContainerClient.CreateIfNotExistsAsync(); + + var localFiles = Directory.GetFiles(path).Select(f => Path.Combine(address, Path.GetRelativePath(path, f))).ToHashSet(); + var blobs = blobContainerClient.GetBlobs().ToDictionary(b => b.Name); + + // Delete blobs that no longer exist in the local directory + foreach (var blob in blobs) + { + if (!localFiles.Contains(blob.Key)) + { + logger.LogInformation($"Deleting blob {blob.Key} as it no longer exists in the local directory."); + await blobContainerClient.DeleteBlobAsync(blob.Key); + } + } + foreach (var inputFile in Directory.GetFiles(path)) { - var filePath = Path.GetRelativePath(path, inputFile); - var fileContent = File.ReadAllText(filePath); + var filePath = Path.Combine(address, Path.GetRelativePath(path, inputFile)); + var fileContent = await File.ReadAllTextAsync(inputFile); + var fileLastModified = File.GetLastWriteTimeUtc(inputFile); - // Parse metadata and HTML - var (article, html) = MarkdownIndexer.ParseArticle(filePath, fileContent, address); + // Check if the blob exists and if the local file is newer + if (blobs.TryGetValue(filePath, out var blobItem) && blobItem.Properties.LastModified >= fileLastModified) + { + logger.LogInformation($"Skipping upload for {filePath} as the blob is up-to-date."); + continue; + } - // Store metadata and HTML in blob storage - var htmlBlobClient = blobContainerClient.GetBlobClient(Path.GetFileName(filePath)); + logger.LogInformation($"Uploading file {inputFile} to {filePath}"); - htmlBlobClient.Upload(new BinaryData(html)); + if (Path.GetExtension(filePath) == "md") + await ParseAndUploadHtml(address, logger, filePath, fileContent, blobContainerClient); + + // upload the file + var markdownBlobClient = blobContainerClient.GetBlobClient(filePath); + await markdownBlobClient.UploadAsync(new BinaryData(fileContent), overwrite: true); - // Set metadata on the HTML blob - htmlBlobClient.SetMetadata(article.ToMetadata()); } return true; } catch (Exception ex) { - Console.WriteLine(ex); + logger.LogError(ex, "An error occurred during execution."); return false; } } + private static async Task ParseAndUploadHtml(string address, ILogger logger, string filePath, string fileContent, + BlobContainerClient blobContainerClient) + { + // Parse metadata and HTML + var (article, html) = MarkdownIndexer.ParseArticle(filePath, fileContent, address); + + // Store metadata and HTML in blob storage + var htmlFilePath = Path.ChangeExtension(filePath, ".html"); + var htmlBlobClient = blobContainerClient.GetBlobClient(htmlFilePath); + await htmlBlobClient.UploadAsync(new BinaryData(html), overwrite: true); + // Set metadata on the HTML blob + var metadata = article.ToMetadata(); + logger.LogInformation($"{JsonSerializer.Serialize(metadata)}"); + await htmlBlobClient.SetMetadataAsync(metadata); + } } } From 7e2ba51fc736a9e90561576be9bb6dba9cbcfb61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sun, 6 Oct 2024 16:16:02 +0200 Subject: [PATCH 22/25] Iterating on tool --- src/MeshWeaver.Azure.Publish/Program.cs | 107 ++++++++++++++-------- src/MeshWeaver.Search/MeshArticleIndex.cs | 5 +- 2 files changed, 70 insertions(+), 42 deletions(-) diff --git a/src/MeshWeaver.Azure.Publish/Program.cs b/src/MeshWeaver.Azure.Publish/Program.cs index d38fdfea7..486ea0970 100644 --- a/src/MeshWeaver.Azure.Publish/Program.cs +++ b/src/MeshWeaver.Azure.Publish/Program.cs @@ -28,6 +28,9 @@ public static async Task Main(string[] args) new Option( "--container", "The name of the blob container"), + new Option( + "--html-container", + "The name of the blob container containing pre-rendered html"), new Option( "--address", "The address to use") @@ -35,7 +38,7 @@ public static async Task Main(string[] args) rootCommand.Description = "MeshWeaver Build Tasks"; - rootCommand.Handler = CommandHandler.Create(async (path, connectionString, container, address) => + rootCommand.Handler = CommandHandler.Create(async (path, connectionString, container, htmlContainer, address) => { if (string.IsNullOrEmpty(path) || string.IsNullOrEmpty(connectionString) || string.IsNullOrEmpty(container) || string.IsNullOrEmpty(address)) { @@ -44,29 +47,28 @@ public static async Task Main(string[] args) } // Call your custom task logic here - logger.LogInformation($"Processing files from path: {path}"); - logger.LogInformation($"Blob Storage Connection String: {connectionString}"); - logger.LogInformation($"Blob Container Name: {container}"); - logger.LogInformation($"Address: {address}"); - - await ExecuteAsync(connectionString, container, path, address, logger); + logger.LogInformation("Processing files from\npath: {path}\nstorage: {connectionString}\ncontainer: {container}\nhtmlContainer: {htmlContainer}\naddress: {address}", path, connectionString, container, htmlContainer, address); + await ExecuteAsync(connectionString, container, htmlContainer, path, address, logger); }); await rootCommand.InvokeAsync(args); } - public static async Task ExecuteAsync(string connectionString, string container, string path, string address, ILogger logger) + public static async Task ExecuteAsync(string connectionString, string container, string htmlContainer, string path, string address, ILogger logger) { try { var blobServiceClient = new BlobServiceClient(connectionString); - var blobContainerClient = blobServiceClient.GetBlobContainerClient(container); + var blobContainer = blobServiceClient.GetBlobContainerClient(container); + var htmlBlobContainer = htmlContainer == null ? null : blobServiceClient.GetBlobContainerClient(container); // Create the container if it doesn't exist - await blobContainerClient.CreateIfNotExistsAsync(); + await blobContainer.CreateIfNotExistsAsync(); + if(htmlContainer != null) + await htmlBlobContainer.CreateIfNotExistsAsync(); var localFiles = Directory.GetFiles(path).Select(f => Path.Combine(address, Path.GetRelativePath(path, f))).ToHashSet(); - var blobs = blobContainerClient.GetBlobs().ToDictionary(b => b.Name); + var blobs = blobContainer.GetBlobs().ToDictionary(b => b.Name); // Delete blobs that no longer exist in the local directory foreach (var blob in blobs) @@ -74,32 +76,43 @@ public static async Task ExecuteAsync(string connectionString, string cont if (!localFiles.Contains(blob.Key)) { logger.LogInformation($"Deleting blob {blob.Key} as it no longer exists in the local directory."); - await blobContainerClient.DeleteBlobAsync(blob.Key); + await blobContainer.DeleteBlobAsync(blob.Key); } } foreach (var inputFile in Directory.GetFiles(path)) { - var filePath = Path.Combine(address, Path.GetRelativePath(path, inputFile)); - var fileContent = await File.ReadAllTextAsync(inputFile); - var fileLastModified = File.GetLastWriteTimeUtc(inputFile); - - // Check if the blob exists and if the local file is newer - if (blobs.TryGetValue(filePath, out var blobItem) && blobItem.Properties.LastModified >= fileLastModified) + try { - logger.LogInformation($"Skipping upload for {filePath} as the blob is up-to-date."); - continue; + var filePath = Path.Combine(address, Path.GetRelativePath(path, inputFile)).Replace('\\', '/'); + var fileContent = await File.ReadAllTextAsync(inputFile); + var fileLastModified = File.GetLastWriteTimeUtc(inputFile); + + // Check if the blob exists and if the local file is newer + if (blobs.TryGetValue(filePath, out var blobItem) && blobItem.Properties.LastModified >= fileLastModified) + { + logger.LogInformation($"Skipping upload for {filePath} as the blob is up-to-date."); + continue; + } + + var extension = Path.GetExtension(filePath); + logger.LogInformation($"Uploading file {inputFile} to {filePath}. Parsing as {extension}"); + + var metadata = extension switch + { + ".md" => await ParseMarkdown(address, logger, filePath, fileContent, htmlBlobContainer), + _ => DefaultMetadata(filePath, extension.Trim('.')) + }; + + logger.LogInformation("Uploading {filePath} with {metadata}", filePath, JsonSerializer.Serialize(metadata)); + var blob = blobContainer.GetBlobClient(filePath); + await blob.UploadAsync(new BinaryData(fileContent), overwrite: true); + await blob.SetMetadataAsync(metadata); + } + catch (Exception ex) + { + logger.LogError(ex, "An error occurred during file upload."); } - - logger.LogInformation($"Uploading file {inputFile} to {filePath}"); - - if (Path.GetExtension(filePath) == "md") - await ParseAndUploadHtml(address, logger, filePath, fileContent, blobContainerClient); - - // upload the file - var markdownBlobClient = blobContainerClient.GetBlobClient(filePath); - await markdownBlobClient.UploadAsync(new BinaryData(fileContent), overwrite: true); - } return true; @@ -111,21 +124,35 @@ public static async Task ExecuteAsync(string connectionString, string cont } } - private static async Task ParseAndUploadHtml(string address, ILogger logger, string filePath, string fileContent, - BlobContainerClient blobContainerClient) + private static Dictionary DefaultMetadata(string filePath, string type) + { + return new Dictionary + { + { "type", type }, + { "path", filePath } + }; +} + + private static async Task> ParseMarkdown(string address, ILogger logger, string filePath, string fileContent, BlobContainerClient htmlContainer) { // Parse metadata and HTML var (article, html) = MarkdownIndexer.ParseArticle(filePath, fileContent, address); - // Store metadata and HTML in blob storage - var htmlFilePath = Path.ChangeExtension(filePath, ".html"); - var htmlBlobClient = blobContainerClient.GetBlobClient(htmlFilePath); - await htmlBlobClient.UploadAsync(new BinaryData(html), overwrite: true); - // Set metadata on the HTML blob - var metadata = article.ToMetadata(); - logger.LogInformation($"{JsonSerializer.Serialize(metadata)}"); - await htmlBlobClient.SetMetadataAsync(metadata); + var metadata = article?.ToMetadata(filePath, "md") ?? DefaultMetadata(filePath, "md"); + + if (htmlContainer != null) + { + // Store metadata and HTML in blob storage + var htmlFilePath = Path.ChangeExtension(filePath, ".html"); + logger.LogInformation("Uploading pre-rendered html {path}", htmlFilePath); + var htmlBlobClient = htmlContainer.GetBlobClient(htmlFilePath); + await htmlBlobClient.UploadAsync(new BinaryData(html), overwrite: true); + await htmlBlobClient.SetMetadataAsync(metadata); + + } + + return metadata; } } } diff --git a/src/MeshWeaver.Search/MeshArticleIndex.cs b/src/MeshWeaver.Search/MeshArticleIndex.cs index 4e9e36d3c..4f5cca7d9 100644 --- a/src/MeshWeaver.Search/MeshArticleIndex.cs +++ b/src/MeshWeaver.Search/MeshArticleIndex.cs @@ -28,7 +28,7 @@ public class MeshArticleIndex public float[] VectorRepresentation { get; set; } - public IDictionary ToMetadata() => + public IDictionary ToMetadata(string path, string type) => new Dictionary() { { nameof(Url), Url }, @@ -38,7 +38,8 @@ public IDictionary ToMetadata() => { nameof(Published), Published.ToString(CultureInfo.InvariantCulture) }, { nameof(Authors), JsonSerializer.Serialize(Authors) }, { nameof(Tags), JsonSerializer.Serialize(Tags) }, - { nameof(Path), Path }, + { nameof(Path), path }, + { nameof(Type), type }, }; From 194bea150d26514c9c6ca6f3e5f5d0a6af5d8d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sun, 6 Oct 2024 20:20:43 +0200 Subject: [PATCH 23/25] fixes --- src/MeshWeaver.Azure.Publish/Program.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/MeshWeaver.Azure.Publish/Program.cs b/src/MeshWeaver.Azure.Publish/Program.cs index 486ea0970..620551468 100644 --- a/src/MeshWeaver.Azure.Publish/Program.cs +++ b/src/MeshWeaver.Azure.Publish/Program.cs @@ -67,8 +67,9 @@ public static async Task ExecuteAsync(string connectionString, string cont if(htmlContainer != null) await htmlBlobContainer.CreateIfNotExistsAsync(); - var localFiles = Directory.GetFiles(path).Select(f => Path.Combine(address, Path.GetRelativePath(path, f))).ToHashSet(); - var blobs = blobContainer.GetBlobs().ToDictionary(b => b.Name); + + var localFiles = Directory.GetFiles(path, "*", SearchOption.AllDirectories).Select(f => Path.Combine(address, Path.GetRelativePath(path, f))).ToHashSet(); + var blobs = blobContainer.GetBlobs(prefix:address).ToDictionary(b => b.Name); // Delete blobs that no longer exist in the local directory foreach (var blob in blobs) From 003a85eb0a2eae87ec9f8f43cedc9f53a2ba7fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Mon, 7 Oct 2024 18:33:05 +0200 Subject: [PATCH 24/25] Fix test --- .../SynchronizationStreamTest.cs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/test/MeshWeaver.Data.Test/SynchronizationStreamTest.cs b/test/MeshWeaver.Data.Test/SynchronizationStreamTest.cs index 74a6b5aa4..fecd07ae9 100644 --- a/test/MeshWeaver.Data.Test/SynchronizationStreamTest.cs +++ b/test/MeshWeaver.Data.Test/SynchronizationStreamTest.cs @@ -40,14 +40,19 @@ public async Task ParallelUpdate() .Subscribe(tracker.Add); var count = 0; - Enumerable.Range(0, 10).AsParallel().ForEach(_ => stream.Update(state => + Enumerable.Range(0, 10).AsParallel().Select(_ => { - var instance = new MyData(Instance, (++count).ToString()); - var existingInstance = state.Collections.GetValueOrDefault(collectionName)?.Instances.GetValueOrDefault(Instance); - return stream.ApplyChanges( - (state ?? new()).Update(collectionName, i => i.Update(Instance, instance)), - [new(collectionName, Instance, instance){OldValue = existingInstance }]); - })); + stream.Update(state => + { + var instance = new MyData(Instance, (++count).ToString()); + var existingInstance = state.Collections.GetValueOrDefault(collectionName)?.Instances + .GetValueOrDefault(Instance); + return stream.ApplyChanges( + (state ?? new()).Update(collectionName, i => i.Update(Instance, instance)), + [new(collectionName, Instance, instance) { OldValue = existingInstance }]); + }); + return true; + }).ToArray(); await DisposeAsync(); tracker.Should().HaveCount(10) From 8f9fa0f3f81e0610cb8075492b03113c7dead791 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Mon, 7 Oct 2024 18:49:43 +0200 Subject: [PATCH 25/25] fixing stream initialization --- .../Serialization/SynchronizationStream.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs index d1c0e73f3..f18cbad6f 100644 --- a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs +++ b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs @@ -146,6 +146,7 @@ public virtual void Initialize(ChangeItem initial) current = initial ?? throw new ArgumentNullException(nameof(initial)); Store.OnNext(initial); initialized.SetResult(current.Value); + deferral.Dispose(); } public void OnCompleted() @@ -181,8 +182,13 @@ IMessageDelivery delivery public void Update(Func> update) => InvokeAsync(() => SetCurrent(update.Invoke(Current is null ? default : Current.Value))); - public void OnNext(ChangeItem value) => - InvokeAsync(() => SetCurrent(value)); + public void OnNext(ChangeItem value) + { + if(!IsInitialized) + Initialize(value); + else + InvokeAsync(() => SetCurrent(value)); + } public virtual DataChangeResponse RequestChange(Func> update) { @@ -207,6 +213,7 @@ public SynchronizationStream(object Owner, this.Reference = Reference; this.InitializationMode = InitializationMode; synchronizationStreamHub = Hub.GetHostedHub(new SynchronizationStreamAddress(Hub.Address)); + deferral = synchronizationStreamHub.Defer(_ => true); } @@ -226,6 +233,7 @@ private record SynchronizationStreamAddress(object Host) : IHostedAddress } private readonly IMessageHub synchronizationStreamHub; + private readonly IDisposable deferral; //private void InvokeAsync(Func task) // => synchronizationStreamHub.InvokeAsync(task);