From 62e2bb8f761659a04811c8a8373e073ab99ac074 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Mon, 13 Mar 2023 10:24:15 -0700 Subject: [PATCH 1/8] Implement support for entities. --- .../TestEntityClient.cs | 116 ++ .../MessageSorterTests.cs | 340 ++++ .../AzureStorageOrchestrationService.cs | 13 +- ...zureStorageOrchestrationServiceSettings.cs | 19 + src/DurableTask.Core/Common/Entities.cs | 12 +- src/DurableTask.Core/Common/Utils.cs | 29 +- .../Entities/ClientEntityContext.cs | 88 + .../Entities/EntityBackendInformation.cs | 93 + .../Entities/EntityExecutionOptions.cs | 33 + src/DurableTask.Core/Entities/EntityId.cs | 125 ++ .../Entities/EntitySchedulerException.cs | 52 + .../Entities/EntityWorkItemProcessor.cs | 638 +++++++ .../EventFormat/EntityMessageEventNames.cs | 37 + .../Entities/EventFormat/ReleaseMessage.cs | 30 + .../Entities/EventFormat/RequestMessage.cs | 113 ++ .../Entities/EventFormat/ResponseMessage.cs | 45 + .../Entities/LocalSDK/EntityContext.cs | 149 ++ .../Entities/LocalSDK/TaskEntityContext.cs | 440 +++++ .../Entities/LocalSDK/TaskHubEntityClient.cs | 537 ++++++ .../OperationFormat/OperationAction.cs | 29 + .../OperationActionConverter.cs | 41 + .../OperationFormat/OperationActionType.cs | 31 + .../OperationFormat/OperationBatchRequest.cs | 44 + .../OperationFormat/OperationBatchResult.cs | 48 + .../OperationFormat/OperationRequest.cs | 43 + .../OperationFormat/OperationResult.cs | 46 + .../SendSignalOperationAction.cs | 51 + .../StartNewOrchestrationOperationAction.cs | 55 + .../Entities/OrchestrationEntityContext.cs | 463 +++++ src/DurableTask.Core/Entities/Serializer.cs | 34 + .../Entities/StateFormat/EntityStatus.cs | 42 + .../Entities/StateFormat/MessageSorter.cs | 271 +++ .../Entities/StateFormat/SchedulerState.cs | 112 ++ src/DurableTask.Core/Entities/TaskEntity.cs | 122 ++ .../EntityLockingRulesViolationException.cs | 62 + .../Exceptions/OperationFailedException.cs | 108 ++ src/DurableTask.Core/FailureDetails.cs | 13 +- src/DurableTask.Core/Logging/EventIds.cs | 4 + src/DurableTask.Core/Logging/LogEvents.cs | 189 ++ src/DurableTask.Core/Logging/LogHelper.cs | 55 + .../Logging/StructuredEventSource.cs | 96 + src/DurableTask.Core/OrchestrationContext.cs | 72 + .../OrchestrationWorkItemProcessor.cs | 604 ++++++ src/DurableTask.Core/TaskHubClient.cs | 4 + src/DurableTask.Core/TaskHubWorker.cs | 67 +- .../TaskOrchestrationContext.cs | 177 +- .../TaskOrchestrationDispatcher.cs | 674 +------ .../TaskOrchestrationExecutor.cs | 24 +- .../AzureStorageScenarioTests.cs | 1612 ++++++++++++++++- .../TestHelpers.cs | 4 +- .../TestOrchestrationClient.cs | 2 + .../TestOrchestrationHost.cs | 27 +- 52 files changed, 7474 insertions(+), 661 deletions(-) create mode 100644 Test/DurableTask.AzureStorage.Tests/TestEntityClient.cs create mode 100644 Test/DurableTask.Core.Tests/MessageSorterTests.cs create mode 100644 src/DurableTask.Core/Entities/ClientEntityContext.cs create mode 100644 src/DurableTask.Core/Entities/EntityBackendInformation.cs create mode 100644 src/DurableTask.Core/Entities/EntityExecutionOptions.cs create mode 100644 src/DurableTask.Core/Entities/EntityId.cs create mode 100644 src/DurableTask.Core/Entities/EntitySchedulerException.cs create mode 100644 src/DurableTask.Core/Entities/EntityWorkItemProcessor.cs create mode 100644 src/DurableTask.Core/Entities/EventFormat/EntityMessageEventNames.cs create mode 100644 src/DurableTask.Core/Entities/EventFormat/ReleaseMessage.cs create mode 100644 src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs create mode 100644 src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs create mode 100644 src/DurableTask.Core/Entities/LocalSDK/EntityContext.cs create mode 100644 src/DurableTask.Core/Entities/LocalSDK/TaskEntityContext.cs create mode 100644 src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs create mode 100644 src/DurableTask.Core/Entities/OperationFormat/OperationAction.cs create mode 100644 src/DurableTask.Core/Entities/OperationFormat/OperationActionConverter.cs create mode 100644 src/DurableTask.Core/Entities/OperationFormat/OperationActionType.cs create mode 100644 src/DurableTask.Core/Entities/OperationFormat/OperationBatchRequest.cs create mode 100644 src/DurableTask.Core/Entities/OperationFormat/OperationBatchResult.cs create mode 100644 src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs create mode 100644 src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs create mode 100644 src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs create mode 100644 src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs create mode 100644 src/DurableTask.Core/Entities/OrchestrationEntityContext.cs create mode 100644 src/DurableTask.Core/Entities/Serializer.cs create mode 100644 src/DurableTask.Core/Entities/StateFormat/EntityStatus.cs create mode 100644 src/DurableTask.Core/Entities/StateFormat/MessageSorter.cs create mode 100644 src/DurableTask.Core/Entities/StateFormat/SchedulerState.cs create mode 100644 src/DurableTask.Core/Entities/TaskEntity.cs create mode 100644 src/DurableTask.Core/Exceptions/EntityLockingRulesViolationException.cs create mode 100644 src/DurableTask.Core/Exceptions/OperationFailedException.cs create mode 100644 src/DurableTask.Core/OrchestrationWorkItemProcessor.cs diff --git a/Test/DurableTask.AzureStorage.Tests/TestEntityClient.cs b/Test/DurableTask.AzureStorage.Tests/TestEntityClient.cs new file mode 100644 index 000000000..46627a792 --- /dev/null +++ b/Test/DurableTask.AzureStorage.Tests/TestEntityClient.cs @@ -0,0 +1,116 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.AzureStorage.Tests +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Threading; + using System.Threading.Tasks; + using DurableTask.AzureStorage.Tracking; + using DurableTask.Core; + using DurableTask.Core.Entities; + using DurableTask.Core.History; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json; + + class TestEntityClient + { + readonly EntityId entityId; + + public TestEntityClient( + TaskHubEntityClient client, + EntityId entityId) + { + this.InnerClient = client; + this.entityId = entityId; + } + + public TaskHubEntityClient InnerClient { get; } + + public async Task SignalEntity( + string operationName, + object operationContent = null) + { + Trace.TraceInformation($"Signaling entity {this.entityId} with operation named {operationName}."); + await this.InnerClient.SignalEntityAsync(this.entityId, operationName, operationContent); + } + + public async Task SignalEntity( + DateTime startTimeUtc, + string operationName, + object operationContent = null) + { + Trace.TraceInformation($"Signaling entity {this.entityId} with operation named {operationName}."); + await this.InnerClient.SignalEntityAsync(this.entityId, operationName, operationContent, startTimeUtc); + } + + public async Task WaitForEntityState( + TimeSpan? timeout = null, + Func describeWhatWeAreWaitingFor = null) + { + if (timeout == null) + { + timeout = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(30); + } + + Stopwatch sw = Stopwatch.StartNew(); + + TaskHubEntityClient.StateResponse response; + + do + { + response = await this.InnerClient.ReadEntityStateAsync(this.entityId); + if (response.EntityExists) + { + if (describeWhatWeAreWaitingFor == null) + { + break; + } + else + { + var waitForResult = describeWhatWeAreWaitingFor(response.EntityState); + + if (string.IsNullOrEmpty(waitForResult)) + { + break; + } + else + { + Trace.TraceInformation($"Waiting for {this.entityId} : {waitForResult}"); + } + } + } + else + { + Trace.TraceInformation($"Waiting for {this.entityId} to have state."); + } + + await Task.Delay(TimeSpan.FromMilliseconds(100)); + } + while (sw.Elapsed < timeout); + + if (response.EntityExists) + { + string serializedState = JsonConvert.SerializeObject(response.EntityState); + Trace.TraceInformation($"Found state: {serializedState}"); + return response.EntityState; + } + else + { + throw new TimeoutException($"Durable entity '{this.entityId}' still doesn't have any state!"); + } + } + } +} diff --git a/Test/DurableTask.Core.Tests/MessageSorterTests.cs b/Test/DurableTask.Core.Tests/MessageSorterTests.cs new file mode 100644 index 000000000..f68268bb5 --- /dev/null +++ b/Test/DurableTask.Core.Tests/MessageSorterTests.cs @@ -0,0 +1,340 @@ +// --------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// --------------------------------------------------------------- + +namespace DurableTask.Core.Tests +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Threading.Tasks; + using DurableTask.Core.Entities.EventFormat; + using DurableTask.Core.Entities.StateFormat; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class MessageSorterTests + { + private static readonly TimeSpan ReorderWindow = TimeSpan.FromMinutes(30); + + [TestMethod] + public void SimpleInOrder() + { + var senderId = "A"; + var receiverId = "B"; + + var senderSorter = new MessageSorter(); + + var message1 = Send(senderId, receiverId, "1", senderSorter, DateTime.UtcNow); + var message2 = Send(senderId, receiverId, "2", senderSorter, DateTime.UtcNow); + var message3 = Send(senderId, receiverId, "3", senderSorter, DateTime.UtcNow); + + List batch; + MessageSorter receiverSorter = new MessageSorter(); + + // delivering the sequence in order produces 1 message each time + batch = receiverSorter.ReceiveInOrder(message1, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("1"); + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("2"); + batch = receiverSorter.ReceiveInOrder(message3, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("3"); + + Assert.AreEqual(0, receiverSorter.NumberBufferedRequests); + } + + [TestMethod] + public void WackySystemClock() + { + var senderId = "A"; + var receiverId = "B"; + + var senderSorter = new MessageSorter(); + + // simulate system clock that goes backwards - mechanism should still guarantee monotonicitty + var message1 = Send(senderId, receiverId, "1", senderSorter, DateTime.UtcNow); + var message2 = Send(senderId, receiverId, "2", senderSorter, DateTime.UtcNow - TimeSpan.FromSeconds(1)); + var message3 = Send(senderId, receiverId, "3", senderSorter, DateTime.UtcNow - TimeSpan.FromSeconds(2)); + + List batch; + MessageSorter receiverSorter = new MessageSorter(); + + // delivering the sequence in order produces 1 message each time + batch = receiverSorter.ReceiveInOrder(message1, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("1"); + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("2"); + batch = receiverSorter.ReceiveInOrder(message3, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("3"); + + Assert.AreEqual(0, receiverSorter.NumberBufferedRequests); + } + + [TestMethod] + public void DelayedElement() + { + var senderId = "A"; + var receiverId = "B"; + + var senderSorter = new MessageSorter(); + + var message1 = Send(senderId, receiverId, "1", senderSorter, DateTime.UtcNow); + var message2 = Send(senderId, receiverId, "2", senderSorter, DateTime.UtcNow); + var message3 = Send(senderId, receiverId, "3", senderSorter, DateTime.UtcNow); + + List batch; + MessageSorter receiverSorter; + + // delivering first message last delays all messages until getting the first one + receiverSorter = new MessageSorter(); + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Empty(batch); + batch = receiverSorter.ReceiveInOrder(message3, ReorderWindow).ToList(); + Assert.That.Empty(batch); + batch = receiverSorter.ReceiveInOrder(message1, ReorderWindow).ToList(); + Assert.That.Collection( + batch, + first => first.Input.Equals("1"), + second => second.Input.Equals("2"), + third => third.Input.Equals("3")); + + Assert.AreEqual(0, receiverSorter.NumberBufferedRequests); + } + + [TestMethod] + public void NoFilteringOrSortingPastReorderWindow() + { + var senderId = "A"; + var receiverId = "B"; + + var senderSorter = new MessageSorter(); + var now = DateTime.UtcNow; + + // last message is sent after an interval exceeding the reorder window + var message1 = Send(senderId, receiverId, "1", senderSorter, now); + var message2 = Send(senderId, receiverId, "2", senderSorter, now + TimeSpan.FromTicks(1)); + var message3 = Send(senderId, receiverId, "3", senderSorter, now + TimeSpan.FromTicks(2) + ReorderWindow); + + List batch; + MessageSorter receiverSorter = new MessageSorter(); + + // delivering the sequence in order produces 1 message each time + receiverSorter = new MessageSorter(); + batch = receiverSorter.ReceiveInOrder(message1, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("1"); + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("2"); + batch = receiverSorter.ReceiveInOrder(message3, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("3"); + + // duplicates are not filtered or sorted, but simply passed through, because we are past the reorder window + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("2"); + batch = receiverSorter.ReceiveInOrder(message1, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("1"); + + Assert.AreEqual(0, receiverSorter.NumberBufferedRequests); + } + + [TestMethod] + public void DuplicatedElements() + { + var senderId = "A"; + var receiverId = "B"; + + var senderSorter = new MessageSorter(); + + var message1 = Send(senderId, receiverId, "1", senderSorter, DateTime.UtcNow); + var message2 = Send(senderId, receiverId, "2", senderSorter, DateTime.UtcNow); + var message3 = Send(senderId, receiverId, "3", senderSorter, DateTime.UtcNow); + + List batch; + MessageSorter receiverSorter; + + // delivering first message last delays all messages until getting the first one + receiverSorter = new MessageSorter(); + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Empty(batch); + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Empty(batch); + batch = receiverSorter.ReceiveInOrder(message1, ReorderWindow).ToList(); + Assert.That.Collection( + batch, + first => first.Input.Equals("1"), + second => second.Input.Equals("2")); + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Empty(batch); + batch = receiverSorter.ReceiveInOrder(message1, ReorderWindow).ToList(); + Assert.That.Empty(batch); + batch = receiverSorter.ReceiveInOrder(message3, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("3"); + batch = receiverSorter.ReceiveInOrder(message3, ReorderWindow).ToList(); + Assert.That.Empty(batch); + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Empty(batch); + batch = receiverSorter.ReceiveInOrder(message1, ReorderWindow).ToList(); + Assert.That.Empty(batch); + + Assert.AreEqual(0, receiverSorter.NumberBufferedRequests); + } + + [TestMethod] + public void RandomShuffleAndDuplication() + { + var senderId = "A"; + var receiverId = "B"; + + var senderSorter = new MessageSorter(); + var receiverSorter = new MessageSorter(); + + var messageCount = 100; + var duplicateCount = 100; + + // create a ordered sequence of messages + var messages = new List(); + for (int i = 0; i < messageCount; i++) + { + messages.Add(Send(senderId, receiverId, i.ToString(), senderSorter, DateTime.UtcNow)); + } + + // add some random duplicates + var random = new Random(0); + for (int i = 0; i < duplicateCount; i++) + { + messages.Add(messages[random.Next(messageCount)]); + } + + // shuffle the messages + Shuffle(messages, random); + + // deliver all the messages + var deliveredMessages = new List(); + + foreach (var msg in messages) + { + foreach (var deliveredMessage in receiverSorter.ReceiveInOrder(msg, ReorderWindow)) + { + deliveredMessages.Add(deliveredMessage); + } + } + + // check that the delivered messages are the original sequence + Assert.AreEqual(messageCount, deliveredMessages.Count()); + for (int i = 0; i < messageCount; i++) + { + Assert.AreEqual(i.ToString(), deliveredMessages[i].Input); + } + + Assert.AreEqual(0, receiverSorter.NumberBufferedRequests); + } + + /// + /// Tests that if messages get reordered beyond the supported reorder window, + /// we still deliver them all but they may now be out of order. + /// + [TestMethod] + public void RandomCollection() + { + var senderId = "A"; + var receiverId = "B"; + + var senderSorter = new MessageSorter(); + var receiverSorter = new MessageSorter(); + + var messageCount = 100; + + var random = new Random(0); + var now = DateTime.UtcNow; + + // create a ordered sequence of messages + var messages = new List(); + for (int i = 0; i < messageCount; i++) + { + messages.Add(Send(senderId, receiverId, i.ToString(), senderSorter, now + TimeSpan.FromSeconds(random.Next(5)), TimeSpan.FromSeconds(10))); + } + + // shuffle the messages + Shuffle(messages, random); + + // add a final message + messages.Add(Send(senderId, receiverId, (messageCount + 1).ToString(), senderSorter, now + TimeSpan.FromSeconds(1000), TimeSpan.FromSeconds(10))); + + // deliver all the messages + var deliveredMessages = new List(); + + for (int i = 0; i < messageCount; i++) + { + foreach (var deliveredMessage in receiverSorter.ReceiveInOrder(messages[i], TimeSpan.FromSeconds(10))) + { + deliveredMessages.Add(deliveredMessage); + } + + Assert.AreEqual(i + 1, deliveredMessages.Count + receiverSorter.NumberBufferedRequests); + } + + // receive the final messages + foreach (var deliveredMessage in receiverSorter.ReceiveInOrder(messages[messageCount], TimeSpan.FromSeconds(10))) + { + deliveredMessages.Add(deliveredMessage); + } + + // check that all messages were delivered + Assert.AreEqual(messageCount + 1, deliveredMessages.Count()); + + Assert.AreEqual(0, receiverSorter.NumberBufferedRequests); + } + + private static RequestMessage Send(string senderId, string receiverId, string input, MessageSorter sorter, DateTime now, TimeSpan? reorderWindow = null) + { + var msg = new RequestMessage() + { + Id = Guid.NewGuid(), + ParentInstanceId = senderId, + Input = input, + }; + sorter.LabelOutgoingMessage(msg, receiverId, now, reorderWindow.HasValue ? reorderWindow.Value : ReorderWindow); + return msg; + } + + private static void Shuffle(IList list, Random random) + { + int n = list.Count; + while (n > 1) + { + n--; + int k = random.Next(n + 1); + T value = list[k]; + list[k] = list[n]; + list[n] = value; + } + } + } + + internal static class AssertExtensions + { + public static void Empty(this Assert assert, IEnumerable collection) + { + Assert.AreEqual(0, collection.Count()); + } + + public static T Single(this Assert assert, IEnumerable collection) + { + var e = collection.GetEnumerator(); + Assert.IsTrue(e.MoveNext()); + T element = e.Current; + Assert.IsFalse(e.MoveNext()); + return element; + } + + public static void Collection(this Assert assert, IEnumerable collection, params Action[] elementInspectors) + { + var list = collection.ToList(); + Assert.AreEqual(elementInspectors.Length, list.Count); + for(int i = 0; i < elementInspectors.Length; i++) + { + elementInspectors[i](list[i]); + } + } + } +} \ No newline at end of file diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs index 3b1993d4c..5b8ed18c4 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs @@ -28,6 +28,7 @@ namespace DurableTask.AzureStorage using DurableTask.AzureStorage.Storage; using DurableTask.AzureStorage.Tracking; using DurableTask.Core; + using DurableTask.Core.Entities; using DurableTask.Core.Exceptions; using DurableTask.Core.History; using DurableTask.Core.Query; @@ -42,7 +43,8 @@ public sealed class AzureStorageOrchestrationService : IOrchestrationServiceClient, IDisposable, IOrchestrationServiceQueryClient, - IOrchestrationServicePurgeClient + IOrchestrationServicePurgeClient, + EntityBackendInformation.IInformationProvider { static readonly HistoryEvent[] EmptyHistoryEventList = new HistoryEvent[0]; @@ -268,6 +270,15 @@ public BehaviorOnContinueAsNew EventBehaviourForContinueAsNew /// public int TaskOrchestrationDispatcherCount { get; } = 1; + EntityBackendInformation EntityBackendInformation.IInformationProvider.GetEntityBackendInformation() + => new EntityBackendInformation() + { + EntityMessageReorderWindow = TimeSpan.FromMinutes(this.settings.EntityMessageReorderWindowInMinutes), + MaxEntityOperationBatchSize = this.settings.MaxEntityOperationBatchSize, + SupportsImplicitEntityDeletion = false, // not supported by this backend + MaximumSignalDelayTime = TimeSpan.FromDays(6), + }; + #region Management Operations (Create/Delete/Start/Stop) /// /// Deletes and creates the neccesary Azure Storage resources for the orchestration service. diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs index c84621951..18b43c329 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs @@ -22,6 +22,7 @@ namespace DurableTask.AzureStorage using Microsoft.WindowsAzure.Storage.Table; using System.Runtime.Serialization; using System.Threading.Tasks; + using DurableTask.Core.Entities; /// /// Settings that impact the runtime behavior of the . @@ -274,5 +275,23 @@ internal LogHelper Logger return this.logHelper; } } + + /// + /// Gets or sets the maximum number of entity operations that are processed as a single batch. + /// + /// + /// Reducing this number can help to avoid timeouts in execution environments that impose time limitations on work items. + /// If set to 1, batching is disabled, and each operation executes as a separate work item. + /// + /// + /// A positive integer configured by the host. + /// + public int? MaxEntityOperationBatchSize { get; set; } = null; + + /// + /// Gets or sets the time window within which entity messages get deduplicated and reordered. + /// If set to zero, there is no sorting or deduplication, and all messages are just passed through. + /// + public int EntityMessageReorderWindowInMinutes { get; set; } = 30; } } diff --git a/src/DurableTask.Core/Common/Entities.cs b/src/DurableTask.Core/Common/Entities.cs index 2484153de..64fce9486 100644 --- a/src/DurableTask.Core/Common/Entities.cs +++ b/src/DurableTask.Core/Common/Entities.cs @@ -10,14 +10,14 @@ // See the License for the specific language governing permissions and // limitations under the License. // ---------------------------------------------------------------------------------- - -using DurableTask.Core.History; -using System; -using System.Collections.Generic; -using System.Text; - +#nullable enable namespace DurableTask.Core.Common { + using DurableTask.Core.History; + using System; + using System.Collections.Generic; + using System.Text; + /// /// Helpers for dealing with special naming conventions around auto-started orchestrations (entities) /// diff --git a/src/DurableTask.Core/Common/Utils.cs b/src/DurableTask.Core/Common/Utils.cs index 2d63eb895..ea03a1cd3 100644 --- a/src/DurableTask.Core/Common/Utils.cs +++ b/src/DurableTask.Core/Common/Utils.cs @@ -21,6 +21,7 @@ namespace DurableTask.Core.Common using System.Linq; using System.Reflection; using System.Runtime.ExceptionServices; + using System.Security.Cryptography; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -152,7 +153,7 @@ public static object DeserializeFromJson(JsonSerializer serializer, string jsonS /// The default value comes from the WEBSITE_SITE_NAME environment variable, which is defined /// in Azure App Service. Other environments can use DTFX_APP_NAME to set this value. /// - public static string AppName { get; set; } = + public static string AppName { get; set; } = Environment.GetEnvironmentVariable("WEBSITE_SITE_NAME") ?? Environment.GetEnvironmentVariable("DTFX_APP_NAME") ?? string.Empty; @@ -624,6 +625,32 @@ public static bool TryGetTaskScheduledId(HistoryEvent historyEvent, out int task } } + /// + /// Creates a determinstic Guid from a string using a hash function. This is a simple hash + /// meant to produce pseudo-random Guids, it is not meant to be cryptographically secure, + /// and does not follow any formatting conventions for UUIDs (such as RFC 4122). + /// + /// The string to hash. + /// A Guid constructed from the hash. + /// + internal static Guid CreateGuidFromHash(string stringToHash) + { + if (string.IsNullOrEmpty(stringToHash)) + { + throw new ArgumentException("string to hash must not be null or empty", nameof(stringToHash)); + } + + byte[] hashByteArray; + using (HashAlgorithm hashAlgorithm = (HashAlgorithm)SHA1.Create()) + { + hashByteArray = hashAlgorithm.ComputeHash(Encoding.UTF8.GetBytes(stringToHash)); + } + + byte[] newGuidByteArray = new byte[16]; + Array.Copy(hashByteArray, 0, newGuidByteArray, 0, 16); + return new Guid(newGuidByteArray); + } + /// /// Gets the generic return type for a specific . /// diff --git a/src/DurableTask.Core/Entities/ClientEntityContext.cs b/src/DurableTask.Core/Entities/ClientEntityContext.cs new file mode 100644 index 000000000..c92647c37 --- /dev/null +++ b/src/DurableTask.Core/Entities/ClientEntityContext.cs @@ -0,0 +1,88 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +namespace DurableTask.Core.Entities +{ + using DurableTask.Core.Entities.EventFormat; + using DurableTask.Core.Entities.OperationFormat; + using DurableTask.Core.Entities.StateFormat; + using Newtonsoft.Json.Linq; + using Newtonsoft.Json; + using System; + + /// + /// Utility functions for clients that interact with entities, either by sending events or by accessing the entity state directly in storage + /// + public static class ClientEntityContext + { + /// + /// Create an event to represent an entity signal. + /// + /// A unique identifier for the request. + /// The name of the operation. + /// The serialized input for the operation. + /// The time to schedule this signal, or null if not a scheduled signal + /// + public static (string eventName, object eventContent) EmitOperationSignal(Guid requestId, string operationName, string input, (DateTime original, DateTime capped)? scheduledTimeUtc) + { + var request = new RequestMessage() + { + ParentInstanceId = null, // means this was sent by a client + ParentExecutionId = null, + Id = requestId, + IsSignal = true, + Operation = operationName, + ScheduledTime = scheduledTimeUtc?.original, + Input = input, + }; + + var jrequest = JToken.FromObject(request, Serializer.InternalSerializer); + + var eventName = scheduledTimeUtc.HasValue + ? EntityMessageEventNames.ScheduledRequestMessageEventName(scheduledTimeUtc.Value.capped) + : EntityMessageEventNames.RequestMessageEventName; + + return (eventName, jrequest); + } + + /// + /// Create an event to represent an entity unlock, which is called by clients to fix orphaned locks. + /// + /// The instance id of the entity to be unlocked. + /// + public static (string eventName, object eventContent) EmitUnlockForOrphanedLock(string lockOwnerInstanceId) + { + var message = new ReleaseMessage() + { + ParentInstanceId = lockOwnerInstanceId, + Id = "fix-orphaned-lock", // we don't know the original id but it does not matter + }; + + var jmessage = JToken.FromObject(message, Serializer.InternalSerializer); + + return (EntityMessageEventNames.ReleaseMessageEventName, jmessage); + } + + /// + /// Extracts the user-defined entity state (as a serialized string) from the scheduler state (also a serialized string). + /// + /// The state of the scheduler, as a serialized string. + /// The entity state + /// True if the entity exists, or false otherwise + public static bool TryGetEntityStateFromSerializedSchedulerState(string serializedSchedulerState, out string entityState) + { + var schedulerState = JsonConvert.DeserializeObject(serializedSchedulerState, Serializer.InternalSerializerSettings); + entityState = schedulerState.EntityState; + return schedulerState.EntityExists; + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/EntityBackendInformation.cs b/src/DurableTask.Core/Entities/EntityBackendInformation.cs new file mode 100644 index 000000000..2d1ea25d9 --- /dev/null +++ b/src/DurableTask.Core/Entities/EntityBackendInformation.cs @@ -0,0 +1,93 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +using System; + +namespace DurableTask.Core.Entities +{ + /// + /// Entity processing characteristics that are controlled by the backend provider, i.e. the orchestration service. + /// + public class EntityBackendInformation + { + /// + /// Get the entity options specified by the orchestration service, or the default options if the service does not specify options. + /// + /// The orchestration service. + /// The options that the provider specifies. + /// The entity options + public static bool BackendSupportsEntities(IOrchestrationService orchestrationService, out EntityBackendInformation entityBackendInformation) + { + if (orchestrationService is IInformationProvider optionsProvider) + { + entityBackendInformation = optionsProvider.GetEntityBackendInformation(); + return true; + } + else + { + entityBackendInformation = null; + return false; + } + } + + /// + /// Interface for objects that provide entity backend information. + /// + public interface IInformationProvider + { + /// + /// The entity backend info. + /// + /// The entity backend information object. + EntityBackendInformation GetEntityBackendInformation(); + } + + /// + /// The time window within which entity messages should be deduplicated and reordered. + /// This is zero for providers that already guarantee exactly-once and ordered delivery. + /// + public TimeSpan EntityMessageReorderWindow { get; set; } + + /// + /// The maximum number of entity operations that should be processed as a single batch. + /// + public int? MaxEntityOperationBatchSize { get; set; } + + /// + /// Whether the backend supports implicit deletion, i.e. setting the entity scheduler state to null implicitly deletes the storage record. + /// + public bool SupportsImplicitEntityDeletion { get; set; } + + /// + /// Value of maximum durable timer delay. Used for delayed signals. + /// + public TimeSpan MaximumSignalDelayTime { get; set; } + + /// + /// Computes a cap on the scheduled time of an entity signal, based on the maximum signal delay time + /// + /// + /// + /// + public (DateTime original, DateTime capped) GetCappedScheduledTime(DateTime nowUtc, DateTime scheduledUtcTime) + { + if ((scheduledUtcTime - nowUtc) <= this.MaximumSignalDelayTime) + { + return (scheduledUtcTime, scheduledUtcTime); + } + else + { + return (scheduledUtcTime, nowUtc + this.MaximumSignalDelayTime); + } + } + } +} diff --git a/src/DurableTask.Core/Entities/EntityExecutionOptions.cs b/src/DurableTask.Core/Entities/EntityExecutionOptions.cs new file mode 100644 index 000000000..18ddea780 --- /dev/null +++ b/src/DurableTask.Core/Entities/EntityExecutionOptions.cs @@ -0,0 +1,33 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; + +namespace DurableTask.Core.Entities +{ + /// + /// Options that are used for configuring how a TaskEntity executes entity operations. + /// + public class EntityExecutionOptions + { + /// + /// Information about backend entity support. + /// + internal EntityBackendInformation EntityBackendInformation { get; set; } + + /// + /// The mode that is used for propagating errors, as specified in the . + /// + internal ErrorPropagationMode ErrorPropagationMode { get; set; } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/EntityId.cs b/src/DurableTask.Core/Entities/EntityId.cs new file mode 100644 index 000000000..4b7258278 --- /dev/null +++ b/src/DurableTask.Core/Entities/EntityId.cs @@ -0,0 +1,125 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +namespace DurableTask.Core.Entities +{ + using System; + using Newtonsoft.Json; + + /// + /// A unique identifier for an entity, consisting of entity name and entity key. + /// + public struct EntityId : IEquatable, IComparable + { + [JsonIgnore] + private string schedulerId; + + /// + /// Create an entity id for an entity. + /// + /// The name of this class of entities. + /// The entity key. + public EntityId(string entityName, string entityKey) + { + if (string.IsNullOrEmpty(entityName)) + { + throw new ArgumentNullException(nameof(entityName), "Invalid entity id: entity name must not be a null or empty string."); + } + + this.EntityName = entityName.ToLowerInvariant(); + this.EntityKey = entityKey ?? throw new ArgumentNullException(nameof(entityKey), "Invalid entity id: entity key must not be null."); + this.schedulerId = GetSchedulerId(this.EntityName, this.EntityKey); + } + + /// + /// The name for this class of entities. + /// + [JsonProperty(PropertyName = "name", Required = Required.Always)] + public string EntityName { get; private set; } // do not remove set, is needed by Json Deserializer + + /// + /// The entity key. Uniquely identifies an entity among all entities of the same name. + /// + [JsonProperty(PropertyName = "key", Required = Required.Always)] + public string EntityKey { get; private set; } // do not remove set, is needed by Json Deserializer + + /// + /// Returns the instance ID for a given entity ID. + /// + /// The entity ID. + /// The corresponding instance ID. + public static string GetInstanceIdFromEntityId(EntityId entityId) + { + return GetSchedulerId(entityId.EntityName, entityId.EntityKey); + } + + private static string GetSchedulerId(string entityName, string entityKey) + { + return $"@{entityName}@{entityKey}"; + } + + internal static string GetSchedulerIdPrefixFromEntityName(string entityName) + { + return $"@{entityName.ToLowerInvariant()}@"; + } + + /// + /// Returns the entity ID for a given instance ID. + /// + /// The instance ID. + /// the corresponding entity ID. + public static EntityId GetEntityIdFromInstanceId(string instanceId) + { + var pos = instanceId.IndexOf('@', 1); + var entityName = instanceId.Substring(1, pos - 1); + var entityKey = instanceId.Substring(pos + 1); + return new EntityId(entityName, entityKey); + } + + /// + public override string ToString() + { + // The scheduler id could be null if the object was deserialized. + if (this.schedulerId == null) + { + this.schedulerId = GetInstanceIdFromEntityId(this); + } + + return this.schedulerId; + } + + /// + public override bool Equals(object obj) + { + return (obj is EntityId other) && this.Equals(other); + } + + /// + public bool Equals(EntityId other) + { + return this.ToString().Equals(other.ToString()); + } + + /// + public override int GetHashCode() + { + return this.ToString().GetHashCode(); + } + + /// + public int CompareTo(object obj) + { + var other = (EntityId)obj; + return this.ToString().CompareTo(other.ToString()); + } + } +} diff --git a/src/DurableTask.Core/Entities/EntitySchedulerException.cs b/src/DurableTask.Core/Entities/EntitySchedulerException.cs new file mode 100644 index 000000000..520e6d1b2 --- /dev/null +++ b/src/DurableTask.Core/Entities/EntitySchedulerException.cs @@ -0,0 +1,52 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities +{ + using System; + using System.Runtime.Serialization; + + /// + /// Exception used to describe various issues encountered by the entity scheduler. + /// + [Serializable] + public class EntitySchedulerException : Exception + { + /// + /// Initializes a new instance of the class. + /// + public EntitySchedulerException() + { + } + + /// + /// Initializes an new instance of the class. + /// + /// The message that describes the error. + /// The exception that was caught. + public EntitySchedulerException(string errorMessage, Exception innerException) + : base(errorMessage, innerException) + { + } + + /// + /// Initializes a new instance of the class with serialized data. + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized object data about the exception being thrown. + /// The System.Runtime.Serialization.StreamingContext that contains contextual information about the source or destination. + protected EntitySchedulerException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/EntityWorkItemProcessor.cs b/src/DurableTask.Core/Entities/EntityWorkItemProcessor.cs new file mode 100644 index 000000000..f1d698f5a --- /dev/null +++ b/src/DurableTask.Core/Entities/EntityWorkItemProcessor.cs @@ -0,0 +1,638 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +namespace DurableTask.Core.Entities +{ + using DurableTask.Core.Common; + using DurableTask.Core.Entities.EventFormat; + using DurableTask.Core.Entities.OperationFormat; + using DurableTask.Core.Entities.StateFormat; + using DurableTask.Core.Exceptions; + using DurableTask.Core.History; + using DurableTask.Core.Logging; + using DurableTask.Core.Middleware; + using DurableTask.Core.Tracing; + using Newtonsoft.Json; + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Linq; + using System.Threading.Tasks; + + internal class EntityWorkItemProcessor : TaskOrchestrationDispatcher.WorkItemProcessor + { + readonly LogHelper logHelper; + readonly INameVersionObjectManager objectManager; + readonly DispatchMiddlewarePipeline dispatchPipeline; + readonly EntityBackendInformation entityBackendInformation; + readonly string instanceId; + readonly ErrorPropagationMode errorPropagationMode; + + SchedulerState schedulerState; + int idCounter; + + public EntityWorkItemProcessor( + TaskOrchestrationDispatcher dispatcher, + TaskOrchestrationWorkItem workItem, + LogHelper logHelper, + INameVersionObjectManager objectManager, + DispatchMiddlewarePipeline dispatchPipeline, + EntityBackendInformation entityBackendInformation, + ErrorPropagationMode errorPropagationMode) : base(dispatcher, workItem) + { + this.logHelper = logHelper; + this.objectManager = objectManager; + this.dispatchPipeline = dispatchPipeline; + this.entityBackendInformation = entityBackendInformation; + this.instanceId = workItem.InstanceId; + this.errorPropagationMode = errorPropagationMode; + } + + public override async Task ProcessWorkItemAsync() + { + // we start with processing all the requests and figuring out which ones to execute + // results can depend on whether the entity is locked, what the maximum batch size is, + // and whether the messages arrived out of order + var workToDoNow = this.DetermineWork(); + + if (workToDoNow.OperationCount > 0) + { + // execute the user-defined operations on this entity, via the middleware + var result = await this.ExecuteViaMiddlewareAsync(workToDoNow); + + // go through all results + // for each operation that is not a signal, send a result message back to the calling orchestrator + for (int i = 0; i < result.Results.Count; i++) + { + var req = workToDoNow.Operations[i]; + if (!req.IsSignal) + { + this.SendResultMessage(req, result.Results[i]); + } + } + + if (result.Results.Count < workToDoNow.OperationCount) + { + // some operations were not processed + var deferred = workToDoNow.RemoveDeferredWork(result.Results.Count); + this.schedulerState.PutBack(deferred); + workToDoNow.ToBeContinued(this.schedulerState); + } + + // update the entity state based on the result + this.schedulerState.EntityState = result.EntityState; + this.schedulerState.EntityExists = result.EntityState != null; + + // perform the actions + foreach (var action in result.Actions) + { + switch (action) + { + case (SendSignalOperationAction sendSignalAction): + this.SendSignalMessage(sendSignalAction); + break; + case (StartNewOrchestrationOperationAction startAction): + this.ProcessSendStartMessage(startAction); + break; + } + } + } + + // process the lock request, if any + if (workToDoNow.LockRequest != null) + { + this.ProcessLockRequest(workToDoNow.LockRequest); + } + + if (workToDoNow.ToBeRescheduled != null) + { + foreach (var request in workToDoNow.ToBeRescheduled) + { + // Reschedule all signals that were received before their time + this.SendScheduledSelfMessage(request); + } + } + + if (workToDoNow.SuspendAndContinue) + { + this.SendContinueSelfMessage(); + } + + // this batch is complete. Since this is an entity, we now + // (always) start a new execution, as in continue-as-new + + var serializedSchedulerState = this.SerializeSchedulerStateForNextExecution(); + var nextExecutionStartedEvent = new ExecutionStartedEvent(-1, serializedSchedulerState) + { + OrchestrationInstance = new OrchestrationInstance + { + InstanceId = this.instanceId, + ExecutionId = Guid.NewGuid().ToString("N") + }, + Tags = runtimeState.Tags, + ParentInstance = runtimeState.ParentInstance, + Name = runtimeState.Name, + Version = runtimeState.Version + }; + var entityStatus = new EntityStatus() + { + EntityExists = this.schedulerState.EntityExists, + QueueSize = this.schedulerState.Queue?.Count ?? 0, + LockedBy = this.schedulerState.LockedBy, + }; + var serializedEntityStatus = JsonConvert.SerializeObject(entityStatus, Serializer.InternalSerializerSettings); + + // create the runtime state for the next execution + this.runtimeState = new OrchestrationRuntimeState(); + this.runtimeState.Status = serializedEntityStatus; + this.runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); + this.runtimeState.AddEvent(nextExecutionStartedEvent); + this.runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); + + this.workItem.OrchestrationRuntimeState = this.runtimeState; + this.instanceState = Utils.BuildOrchestrationState(this.runtimeState); + } + + void ProcessLockRequest(RequestMessage request) + { + this.logHelper.EntityLockAcquired(this.instanceId, request); + + // mark the entity state as locked + this.schedulerState.LockedBy = request.ParentInstanceId; + + request.Position++; + + if (request.Position < request.LockSet.Length) + { + // send lock request to next entity in the lock set + var target = new OrchestrationInstance() { InstanceId = EntityId.GetInstanceIdFromEntityId(request.LockSet[request.Position]) }; + this.SendLockRequestMessage(target, request); + } + else + { + // send lock acquisition completed response back to originating orchestration instance + var target = new OrchestrationInstance() { InstanceId = request.ParentInstanceId, ExecutionId = request.ParentExecutionId }; + this.SendLockResponseMessage(target, request.Id); + } + } + + string SerializeSchedulerStateForNextExecution() + { + if (this.entityBackendInformation.SupportsImplicitEntityDeletion && this.schedulerState.IsEmpty && !this.schedulerState.Suspended) + { + // this entity scheduler is idle and the entity is deleted, so the instance and history can be removed from storage + // we convey this to the durability provider by issuing a continue-as-new with null input + return null; + } + else + { + // we persist the state of the entity scheduler and entity + return JsonConvert.SerializeObject(this.schedulerState, typeof(SchedulerState), Serializer.InternalSerializerSettings); + } + } + + #region Preprocess to determine work + + Work DetermineWork() + { + Queue lockHolderMessages = null; + Work batch = new Work(); + + foreach (HistoryEvent e in this.runtimeState.Events) + { + switch (e.EventType) + { + case EventType.ExecutionStarted: + + this.schedulerState = new SchedulerState(); + + if (runtimeState.Input != null) + { + try + { + // restore the scheduler state from the input + JsonConvert.PopulateObject(runtimeState.Input, this.schedulerState, Serializer.InternalSerializerSettings); + } + catch (Exception exception) + { + throw new EntitySchedulerException("Failed to deserialize entity scheduler state - may be corrupted or wrong version.", exception); + } + } + break; + + case EventType.EventRaised: + EventRaisedEvent eventRaisedEvent = (EventRaisedEvent)e; + + if (EntityMessageEventNames.IsRequestMessage(eventRaisedEvent.Name)) + { + // we are receiving an operation request or a lock request + var requestMessage = new RequestMessage(); + + try + { + JsonConvert.PopulateObject(eventRaisedEvent.Input, requestMessage, Serializer.InternalSerializerSettings); + } + catch (Exception exception) + { + throw new EntitySchedulerException("Failed to deserialize incoming request message - may be corrupted or wrong version.", exception); + } + + IEnumerable deliverNow; + + if (requestMessage.ScheduledTime.HasValue) + { + if ((requestMessage.ScheduledTime.Value - DateTime.UtcNow) > TimeSpan.FromMilliseconds(100)) + { + // message was delivered too early. This can happen e.g. if the orchestration service has limits on the delay times for messages. + // We handle this by rescheduling the message instead of processing it. + deliverNow = Array.Empty(); + batch.AddMessageToBeRescheduled(requestMessage); + } + else + { + // the message is scheduled to be delivered immediately. + // There are no FIFO guarantees for scheduled messages, so we skip the message sorter. + deliverNow = new RequestMessage[] { requestMessage }; + } + } + else + { + // run this through the message sorter to help with reordering and duplicate filtering + deliverNow = this.schedulerState.MessageSorter.ReceiveInOrder(requestMessage, this.entityBackendInformation.EntityMessageReorderWindow); + } + + foreach (var message in deliverNow) + { + if (this.schedulerState.LockedBy != null && this.schedulerState.LockedBy == message.ParentInstanceId) + { + if (lockHolderMessages == null) + { + lockHolderMessages = new Queue(); + } + + lockHolderMessages.Enqueue(message); + } + else + { + this.schedulerState.Enqueue(message); + } + } + } + else if (EntityMessageEventNames.IsReleaseMessage(eventRaisedEvent.Name)) + { + // we are receiving a lock release + var message = new ReleaseMessage(); + try + { + // restore the scheduler state from the input + JsonConvert.PopulateObject(eventRaisedEvent.Input, message, Serializer.InternalSerializerSettings); + } + catch (Exception exception) + { + throw new EntitySchedulerException("Failed to deserialize lock release message - may be corrupted or wrong version.", exception); + } + + if (this.schedulerState.LockedBy == message.ParentInstanceId) + { + this.logHelper.EntityLockReleased(this.instanceId, message); + this.schedulerState.LockedBy = null; + } + } + else + { + // this is a continue message. + // Resumes processing of previously queued operations, if any. + this.schedulerState.Suspended = false; + } + + break; + } + } + + // lock holder messages go to the front of the queue + if (lockHolderMessages != null) + { + this.schedulerState.PutBack(lockHolderMessages); + } + + if (!this.schedulerState.Suspended) + { + // 2. We add as many requests from the queue to the batch as possible, + // stopping at lock requests or when the maximum batch size is reached + while (this.schedulerState.MayDequeue()) + { + if (batch.OperationCount == this.entityBackendInformation.MaxEntityOperationBatchSize) + { + // we have reached the maximum batch size already + // insert a delay after this batch to ensure write back + batch.ToBeContinued(this.schedulerState); + break; + } + + var request = this.schedulerState.Dequeue(); + + if (request.IsLockRequest) + { + batch.AddLockRequest(request); + break; + } + else + { + batch.AddOperation(request); + } + } + } + + return batch; + } + + class Work + { + List operationBatch; // a (possibly empty) sequence of operations to be executed on the entity + RequestMessage lockRequest = null; // zero or one lock request to be executed after all the operations + List toBeRescheduled; // a (possibly empty) list of timed messages that were delivered too early and should be rescheduled + bool suspendAndContinue; // a flag telling as to send ourselves a continue signal + + public int OperationCount => this.operationBatch?.Count ?? 0; + public IReadOnlyList Operations => this.operationBatch; + public IReadOnlyList ToBeRescheduled => this.toBeRescheduled; + public RequestMessage LockRequest => this.lockRequest; + public bool SuspendAndContinue => this.suspendAndContinue; + + public void AddOperation(RequestMessage operationMessage) + { + if (this.operationBatch == null) + { + this.operationBatch = new List(); + } + this.operationBatch.Add(operationMessage); + } + + public void AddLockRequest(RequestMessage lockRequest) + { + Debug.Assert(this.lockRequest == null); + this.lockRequest = lockRequest; + } + + public void AddMessageToBeRescheduled(RequestMessage requestMessage) + { + if (this.toBeRescheduled == null) + { + this.toBeRescheduled = new List(); + } + this.toBeRescheduled.Add(requestMessage); + } + + public void ToBeContinued(SchedulerState schedulerState) + { + if (!schedulerState.Suspended) + { + this.suspendAndContinue = true; + } + } + + public List GetOperationRequests() + { + var operations = new List(this.operationBatch.Count); + for (int i = 0; i < this.operationBatch.Count; i++) + { + var request = this.operationBatch[i]; + operations.Add(new OperationRequest() + { + Operation = request.Operation, + Id = request.Id, + Input = request.Input, + }); + } + return operations; + } + + public Queue RemoveDeferredWork(int index) + { + var deferred = new Queue(); + for (int i = index; i < this.operationBatch.Count; i++) + { + deferred.Enqueue(this.operationBatch[i]); + } + this.operationBatch.RemoveRange(index, this.operationBatch.Count - index); + if (this.lockRequest != null) + { + deferred.Enqueue(this.lockRequest); + this.lockRequest = null; + } + return deferred; + } + } + + #endregion + + #region Send Messages + + void SendResultMessage(RequestMessage request, OperationResult result) + { + var destination = new OrchestrationInstance() + { + InstanceId = request.ParentInstanceId, + ExecutionId = request.ParentExecutionId, + }; + var responseMessage = new ResponseMessage() + { + Result = result.Result, + ErrorMessage = result.ErrorMessage, + FailureDetails = result.FailureDetails, + }; + this.ProcessSendEventMessage(destination, EntityMessageEventNames.ResponseMessageEventName(request.Id), responseMessage); + } + + void SendSignalMessage(SendSignalOperationAction action) + { + OrchestrationInstance destination = new OrchestrationInstance() + { + InstanceId = action.InstanceId + }; + RequestMessage message = new RequestMessage() + { + ParentInstanceId = this.instanceId, + ParentExecutionId = null, // for entities, message sorter persists across executions + Id = Guid.NewGuid(), + IsSignal = true, + Operation = action.Name, + ScheduledTime = action.ScheduledTime, + }; + string eventName; + if (action.ScheduledTime.HasValue) + { + (DateTime original, DateTime capped) = this.entityBackendInformation.GetCappedScheduledTime(DateTime.UtcNow, action.ScheduledTime.Value); + eventName = EntityMessageEventNames.ScheduledRequestMessageEventName(capped); + } + else + { + eventName = EntityMessageEventNames.RequestMessageEventName; + this.schedulerState.MessageSorter.LabelOutgoingMessage(message, action.InstanceId, DateTime.UtcNow, this.entityBackendInformation.EntityMessageReorderWindow); + } + this.ProcessSendEventMessage(destination, eventName, message); + } + + internal void SendLockRequestMessage(OrchestrationInstance target, RequestMessage message) + { + this.schedulerState.MessageSorter.LabelOutgoingMessage(message, target.InstanceId, DateTime.UtcNow, this.entityBackendInformation.EntityMessageReorderWindow); + this.ProcessSendEventMessage(target, EntityMessageEventNames.RequestMessageEventName, message); + } + + internal void SendLockResponseMessage(OrchestrationInstance target, Guid requestId) + { + var message = new ResponseMessage() + { + Result = "Lock Acquisition Completed", // ignored by receiver but shows up in traces + }; + this.ProcessSendEventMessage(target, EntityMessageEventNames.ResponseMessageEventName(requestId), message); + } + + void SendScheduledSelfMessage(RequestMessage request) + { + var self = new OrchestrationInstance() + { + InstanceId = this.instanceId, + }; + this.ProcessSendEventMessage(self, EntityMessageEventNames.ScheduledRequestMessageEventName(request.ScheduledTime.Value), request); + } + + void SendContinueSelfMessage() + { + var self = new OrchestrationInstance() + { + InstanceId = this.instanceId, + }; + this.ProcessSendEventMessage(self, EntityMessageEventNames.ContinueMessageEventName, null); + } + + void ProcessSendEventMessage(OrchestrationInstance destination, string eventName, object eventContent) + { + string serializedContent = null; + if (eventContent != null) + { + serializedContent = JsonConvert.SerializeObject(eventContent, Serializer.InternalSerializerSettings); + } + + var eventSentEvent = new EventSentEvent(this.idCounter++) + { + InstanceId = destination.InstanceId, + Name = eventName, + Input = serializedContent, + }; + this.logHelper.RaisingEvent(runtimeState.OrchestrationInstance!, eventSentEvent); + + this.orchestratorMessages.Add(new TaskMessage + { + OrchestrationInstance = destination, + Event = new EventRaisedEvent(-1, serializedContent) + { + Name = eventName, + Input = serializedContent, + }, + }); + } + + void ProcessSendStartMessage(StartNewOrchestrationOperationAction action) + { + OrchestrationInstance destination = new OrchestrationInstance() + { + InstanceId = action.InstanceId, + ExecutionId = Guid.NewGuid().ToString("N"), + }; + var executionStartedEvent = new ExecutionStartedEvent(-1, action.Input) + { + Tags = OrchestrationTags.MergeTags(action.Tags, runtimeState.Tags), + OrchestrationInstance = destination, + ParentInstance = new ParentInstance + { + OrchestrationInstance = runtimeState.OrchestrationInstance, + Name = runtimeState.Name, + Version = runtimeState.Version, + TaskScheduleId = idCounter++, + }, + Name = action.Name, + Version = action.Version, + }; + this.logHelper.SchedulingOrchestration(executionStartedEvent); + + this.orchestratorMessages.Add(new TaskMessage + { + OrchestrationInstance = destination, + Event = executionStartedEvent, + }); + } + + #endregion + + async Task ExecuteViaMiddlewareAsync(Work workToDoNow) + { + // the request object that will be passed to the worker + var request = new OperationBatchRequest() + { + InstanceId = this.instanceId, + EntityState = this.schedulerState.EntityState, + Operations = workToDoNow.GetOperationRequests(), + }; + + this.logHelper.EntityBatchExecuting(request); + + string entityName = EntityId.GetEntityIdFromInstanceId(this.instanceId).EntityName; + string entityVersion = string.Empty; // TODO consider whether we should support explicit versions + + // Get the TaskOrchestration implementation. If it's not found, it either means that the developer never + // registered it (which is an error, and we'll throw for this further down) or it could be that some custom + // middleware (e.g. out-of-process execution middleware) is intended to implement the orchestration logic. + TaskEntity taskEntity = this.objectManager.GetObject(entityName, entityVersion); + + var dispatchContext = new DispatchMiddlewareContext(); + dispatchContext.SetProperty(request); + + await this.dispatchPipeline.RunAsync(dispatchContext, async _ => + { + // Check to see if the custom middleware intercepted and substituted the orchestration execution + // with its own execution behavior, providing us with the end results. If so, we can terminate + // the dispatch pipeline here. + var resultFromMiddleware = dispatchContext.GetProperty(); + if (resultFromMiddleware != null) + { + return; + } + + if (taskEntity == null) + { + throw TraceHelper.TraceExceptionInstance( + TraceEventType.Error, + "TaskOrchestrationDispatcher-EntityTypeMissing", + runtimeState.OrchestrationInstance, + new TypeMissingException($"Entity not found: {entityName}")); + } + + var options = new EntityExecutionOptions() + { + EntityBackendInformation = this.entityBackendInformation, + ErrorPropagationMode = this.errorPropagationMode, + }; + + var resultFromTaskEntityObject = await taskEntity.ExecuteOperationBatchAsync(request, options); + + dispatchContext.SetProperty(resultFromTaskEntityObject); + }); + + var result = dispatchContext.GetProperty(); + + this.logHelper.EntityBatchExecuted(request, result); + + return result; + } + } +} diff --git a/src/DurableTask.Core/Entities/EventFormat/EntityMessageEventNames.cs b/src/DurableTask.Core/Entities/EventFormat/EntityMessageEventNames.cs new file mode 100644 index 000000000..adecea1dc --- /dev/null +++ b/src/DurableTask.Core/Entities/EventFormat/EntityMessageEventNames.cs @@ -0,0 +1,37 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities.EventFormat +{ + using System; + + /// + /// Determines event names to use for messages sent to and from entities. + /// + internal static class EntityMessageEventNames + { + public static string RequestMessageEventName => "op"; + + public static string ReleaseMessageEventName => "release"; + + public static string ContinueMessageEventName => "continue"; + + public static string ScheduledRequestMessageEventName(DateTime scheduledUtc) => $"op@{scheduledUtc:o}"; + + public static string ResponseMessageEventName(Guid requestId) => requestId.ToString(); + + public static bool IsRequestMessage(string eventName) => eventName.StartsWith("op"); + + public static bool IsReleaseMessage(string eventName) => eventName == "release"; + } +} diff --git a/src/DurableTask.Core/Entities/EventFormat/ReleaseMessage.cs b/src/DurableTask.Core/Entities/EventFormat/ReleaseMessage.cs new file mode 100644 index 000000000..db54b6a98 --- /dev/null +++ b/src/DurableTask.Core/Entities/EventFormat/ReleaseMessage.cs @@ -0,0 +1,30 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +namespace DurableTask.Core.Entities.EventFormat +{ + using Newtonsoft.Json; + + internal class ReleaseMessage + { + [JsonProperty(PropertyName = "parent")] + public string ParentInstanceId { get; set; } + + [JsonProperty(PropertyName = "id")] + public string Id { get; set; } + + public override string ToString() + { + return $"[Release lock {Id} by {ParentInstanceId}]"; + } + } +} diff --git a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs new file mode 100644 index 000000000..23d297e9b --- /dev/null +++ b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs @@ -0,0 +1,113 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +namespace DurableTask.Core.Entities.EventFormat +{ + using System; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + /// + /// A message sent to an entity, such as operation, signal, lock, or continue messages. + /// + internal class RequestMessage + { + /// + /// The name of the operation being called (if this is an operation message) or null + /// (if this is a lock request). + /// + [JsonProperty(PropertyName = "op")] + public string Operation { get; set; } + + /// + /// Whether or not this is a one-way message. + /// + [JsonProperty(PropertyName = "signal", DefaultValueHandling = DefaultValueHandling.Ignore)] + public bool IsSignal { get; set; } + + /// + /// The operation input. + /// + [JsonProperty(PropertyName = "input", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string Input { get; set; } + + /// + /// A unique identifier for this operation. + /// + [JsonProperty(PropertyName = "id", Required = Required.Always)] + public Guid Id { get; set; } + + /// + /// The parent instance that called this operation. + /// + [JsonProperty(PropertyName = "parent", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string ParentInstanceId { get; set; } + + /// + /// The parent instance that called this operation. + /// + [JsonProperty(PropertyName = "parentExecution", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string ParentExecutionId { get; set; } + + /// + /// Optionally, a scheduled time at which to start the operation. + /// + [JsonProperty(PropertyName = "due", DefaultValueHandling = DefaultValueHandling.Ignore)] + public DateTime? ScheduledTime { get; set; } + + /// + /// A timestamp for this request. + /// Used for duplicate filtering and in-order delivery. + /// + [JsonProperty] + public DateTime Timestamp { get; set; } + + /// + /// A timestamp for the predecessor request in the stream, or DateTime.MinValue if none. + /// Used for duplicate filtering and in-order delivery. + /// + [JsonProperty] + public DateTime Predecessor { get; set; } + + /// + /// For lock requests, the set of locks being acquired. Is sorted, + /// contains at least one element, and has no repetitions. + /// + [JsonProperty(PropertyName = "lockset", DefaultValueHandling = DefaultValueHandling.Ignore, TypeNameHandling = TypeNameHandling.None)] + public EntityId[] LockSet { get; set; } + + /// + /// For lock requests involving multiple locks, the message number. + /// + [JsonProperty(PropertyName = "pos", DefaultValueHandling = DefaultValueHandling.Ignore)] + public int Position { get; set; } + + /// + /// whether this message is a lock request + /// + [JsonIgnore] + public bool IsLockRequest => LockSet != null; + + /// + public override string ToString() + { + if (IsLockRequest) + { + return $"[Request lock {Id} by {ParentInstanceId} {ParentExecutionId}, position {Position}]"; + } + else + { + return $"[{(IsSignal ? "Signal" : "Call")} '{Operation}' operation {Id} by {ParentInstanceId} {ParentExecutionId}]"; + } + } + } +} diff --git a/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs new file mode 100644 index 000000000..1588bba60 --- /dev/null +++ b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs @@ -0,0 +1,45 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +namespace DurableTask.Core.Entities.EventFormat +{ + using System; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + internal class ResponseMessage + { + [JsonProperty(PropertyName = "result")] + public string Result { get; set; } + + [JsonProperty(PropertyName = "exceptionType", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string ErrorMessage { get; set; } + + [JsonProperty(PropertyName = "failureDetails", DefaultValueHandling = DefaultValueHandling.Ignore)] + public FailureDetails FailureDetails { get; set; } + + [JsonIgnore] + public bool IsErrorResult => this.ErrorMessage != null; + + public override string ToString() + { + if (this.IsErrorResult) + { + return $"[ErrorResponse {this.Result}]"; + } + else + { + return $"[Response {this.Result}]"; + } + } + } +} diff --git a/src/DurableTask.Core/Entities/LocalSDK/EntityContext.cs b/src/DurableTask.Core/Entities/LocalSDK/EntityContext.cs new file mode 100644 index 000000000..cf872309c --- /dev/null +++ b/src/DurableTask.Core/Entities/LocalSDK/EntityContext.cs @@ -0,0 +1,149 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core +{ + using DurableTask.Core.Entities; + using System; + using System.Reflection; + using System.Threading.Tasks; + + /// + /// Context for an entity, which is available to the application code while it is executing entity operations. + /// + /// The JSON-serializable type of the entity state. + public abstract class EntityContext + { + /// + /// Gets the name of the currently executing entity. + /// + public abstract string EntityName { get; } + + /// + /// Gets the key of the currently executing entity. + /// + public abstract string EntityKey { get; } + + /// + /// Gets the id of the currently executing entity. + /// + public abstract EntityId EntityId { get; } + + /// + /// Gets the name of the operation that was called. + /// + /// + /// An operation invocation on an entity includes an operation name, which states what + /// operation to perform, and optionally an operation input. + /// + public abstract string OperationName { get; } + + /// + /// Whether this entity has state. + /// + /// The value of changes as entity operations access or delete the state during their execution. It is set to true after is accessed from within an operation, + /// and it is set to false after is called. This can happen repeatedly; neither creation nor deletion are permanent. + public abstract bool HasState { get; } + + /// + /// The size of the current batch of operations. + /// + public abstract int BatchSize { get; } + + /// + /// The position of the currently executing operation within the current batch of operations. + /// + public abstract int BatchPosition { get; } + + /// + /// Gets or sets the current state of the entity. Implicitly creates the state if the entity does not have state yet. + /// + /// If the entity does not have a state yet (first time access), + /// or does not have a state anymore (was deleted), accessing this property implicitly creates state for the entity. For the 'get' accessor, this means + /// that is called to create the state. + /// For the 'set' accessor, the given value is used. + public abstract TState State { get; set; } + + /// + /// Deletes the state of this entity. + /// + public abstract void DeleteState(); + + /// + /// Gets the input for this operation, as a deserialized value. + /// + /// The JSON-serializable type used for the operation input. + /// The operation input, or default() if none. + /// + /// An operation invocation on an entity includes an operation name, which states what + /// operation to perform, and optionally an operation input. + /// + public abstract TInput GetInput(); + + /// + /// Gets the input for this operation, as a deserialized value. + /// + /// The JSON-serializable type used for the operation input. + /// The operation input, or default() if none. + /// + /// An operation invocation on an entity includes an operation name, which states what + /// operation to perform, and optionally an operation input. + /// + public abstract object GetInput(Type inputType); + + /// + /// Returns the given result to the caller of this operation. + /// + /// the result to return. + public abstract void Return(object result); + + /// + /// Signals an entity to perform an operation, without waiting for a response. Any result or exception is ignored (fire and forget). + /// + /// The target entity. + /// The name of the operation. + /// The operation input. + public abstract void SignalEntity(EntityId entity, string operationName, object operationInput = null); + + /// + /// Signals an entity to perform an operation, at a specified time. Any result or exception is ignored (fire and forget). + /// + /// The target entity. + /// The time at which to start the operation. + /// The name of the operation. + /// The input for the operation. + public abstract void SignalEntity(EntityId entity, DateTime scheduledTimeUtc, string operationName, object operationInput = null); + + /// + /// Schedules an orchestration function with the given name and version for execution./>. + /// Any result or exception is ignored (fire and forget). + /// + /// The name of the orchestrator, as specified by the ObjectCreator. + /// The version of the orchestrator. + /// the input to pass to the orchestrator function. + /// optionally, an instance id for the orchestration. By default, a random GUID is used. + /// The instance id of the new orchestration. + public abstract string StartNewOrchestration(string name, string version, object input, string instanceId = null); + + /// + /// Schedules an orchestration function with the given type for execution./>. + /// Any result or exception is ignored (fire and forget). + /// + /// Type of the TaskOrchestration derived class to instantiate + /// the input to pass to the orchestrator function. + /// optionally, an instance id for the orchestration. By default, a random GUID is used. + /// The instance id of the new orchestration. + public virtual string StartNewOrchestration(Type orchestrationType, object input, string instanceId = null) + => this.StartNewOrchestration(NameVersionHelper.GetDefaultName(orchestrationType), NameVersionHelper.GetDefaultVersion(orchestrationType), input, instanceId); + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/LocalSDK/TaskEntityContext.cs b/src/DurableTask.Core/Entities/LocalSDK/TaskEntityContext.cs new file mode 100644 index 000000000..68780d4f8 --- /dev/null +++ b/src/DurableTask.Core/Entities/LocalSDK/TaskEntityContext.cs @@ -0,0 +1,440 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core +{ + using DurableTask.Core.Common; + using DurableTask.Core.Entities; + using DurableTask.Core.Entities.OperationFormat; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + using System; + using System.Collections.Generic; + using System.Reflection; + using System.Runtime.Serialization; + using System.Threading.Tasks; + + internal class TaskEntityContext : EntityContext + { + readonly TaskEntity taskEntity; + readonly EntityExecutionOptions executionOptions; + readonly OperationBatchRequest batchRequest; + readonly OperationBatchResult batchResult; + + int batchPosition; + OperationResult currentOperationResult; + StateAccess currentStateAccess; + TState currentState; + + public TaskEntityContext(TaskEntity taskEntity, EntityId entityId, EntityExecutionOptions options, OperationBatchRequest batchRequest, OperationBatchResult batchResult) + { + this.taskEntity = taskEntity; + this.EntityId = entityId; + this.executionOptions = options; + this.batchRequest = batchRequest; + this.batchResult = batchResult; + } + + public override string EntityName => this.EntityId.EntityName; + public override string EntityKey => this.EntityId.EntityKey; + public override EntityId EntityId { get; } + + public EntityExecutionOptions ExecutionOptions => this.executionOptions; + + public override string OperationName => this.batchRequest.Operations[this.batchPosition].Operation; + public override int BatchSize => this.batchRequest.Operations.Count; + public override int BatchPosition => this.batchPosition; + + OperationRequest CurrentOperation => this.batchRequest.Operations[batchPosition]; + + // The last serialized checkpoint of the entity state is always stored in + // this.LastSerializedState. The current state is determined by this.CurrentStateAccess and this.CurrentState. + internal enum StateAccess + { + NotAccessed, // current state is stored in this.LastSerializedState + Accessed, // current state is stored in this.currentState + Clean, // current state is stored in both this.currentState (deserialized) and in this.LastSerializedState + Deleted, // current state is deleted + } + + internal string LastSerializedState + { + get { return this.batchResult.EntityState; } + set { this.batchResult.EntityState = value; } + } + + public override bool HasState + { + get + { + switch (this.currentStateAccess) + { + case StateAccess.Accessed: + case StateAccess.Clean: + return true; + + case StateAccess.Deleted: + return false; + + default: return this.LastSerializedState != null; + } + } + } + + public override TState State + { + get + { + if (this.currentStateAccess == StateAccess.Accessed) + { + return this.currentState; + } + else if (this.currentStateAccess == StateAccess.Clean) + { + this.currentStateAccess = StateAccess.Accessed; + return this.currentState; + } + + TState result; + + if (this.LastSerializedState != null && this.currentStateAccess != StateAccess.Deleted) + { + try + { + result = taskEntity.StateDataConverter.Deserialize(this.LastSerializedState); + } + catch (Exception e) + { + throw new EntitySchedulerException($"Failed to deserialize entity state: {e.Message}", e); + } + } + else + { + try + { + result = this.taskEntity.CreateInitialState(this); + } + catch (Exception e) + { + throw new EntitySchedulerException($"Failed to initialize entity state: {e.Message}", e); + } + } + + this.currentStateAccess = StateAccess.Accessed; + this.currentState = result; + return result; + } + set + { + this.currentState = value; + this.currentStateAccess = StateAccess.Accessed; + } + } + + public override void DeleteState() + { + this.currentStateAccess = StateAccess.Deleted; + this.currentState = default; + } + + public void Rollback(int positionBeforeCurrentOperation) + { + // We discard the current state, which means we go back to the last serialized one + this.currentStateAccess = StateAccess.NotAccessed; + this.currentState = default; + + // we also roll back the list of outgoing messages, + // so any signals sent by this operation are discarded. + this.batchResult.Actions.RemoveRange(positionBeforeCurrentOperation, this.batchResult.Actions.Count - positionBeforeCurrentOperation); + } + + private bool TryWriteback(out OperationResult serializationErrorResult, OperationRequest operationRequest = null) + { + if (this.currentStateAccess == StateAccess.Deleted) + { + this.LastSerializedState = null; + this.currentStateAccess = StateAccess.NotAccessed; + } + else if (this.currentStateAccess == StateAccess.Accessed) + { + try + { + string serializedState = this.taskEntity.StateDataConverter.Serialize(this.currentState); + this.LastSerializedState = serializedState; + this.currentStateAccess = StateAccess.Clean; + } + catch (Exception serializationException) + { + // we cannot serialize the entity state - this is an application error. To help users diagnose this, + // we wrap it into a descriptive exception, and propagate this error result to any calling orchestrations. + + serializationException = new EntitySchedulerException( + $"Operation was rolled back because state for entity '{this.EntityId}' could not be serialized: {serializationException.Message}", serializationException); + serializationErrorResult = new OperationResult(); + this.CaptureExceptionInOperationResult(serializationErrorResult, serializationException); + + // we have no choice but to roll back to the state prior to this operation + this.currentStateAccess = StateAccess.NotAccessed; + this.currentState = default; + + return false; + } + } + else + { + // the state was not accessed, or is clean, so we don't need to write anything back + } + + serializationErrorResult = null; + return true; + } + + public override TInput GetInput() + { + try + { + return this.taskEntity.MessageDataConverter.Deserialize(this.CurrentOperation.Input); + } + catch(Exception e) + { + throw new EntitySchedulerException($"Failed to deserialize input for operation '{this.CurrentOperation.Operation}': {e.Message}", e); + } + } + + public override object GetInput(Type inputType) + { + try + { + return this.taskEntity.MessageDataConverter.Deserialize(this.CurrentOperation.Input, inputType); + } + catch (Exception e) + { + throw new EntitySchedulerException($"Failed to deserialize input for operation '{this.CurrentOperation.Operation}': {e.Message}", e); + } + } + + public override void Return(object result) + { + try + { + this.currentOperationResult.Result = this.taskEntity.MessageDataConverter.Serialize(result); + } + catch (Exception e) + { + throw new EntitySchedulerException($"Failed to serialize output for operation '{this.CurrentOperation.Operation}': {e.Message}", e); + } + } + + + public override void SignalEntity(EntityId entity, string operationName, object operationInput = null) + { + this.SignalEntityInternal(entity, null, operationName, operationInput); + } + + + public override void SignalEntity(EntityId entity, DateTime scheduledTimeUtc, string operationName, object operationInput = null) + { + this.SignalEntityInternal(entity, scheduledTimeUtc, operationName, operationInput); + } + + private void SignalEntityInternal(EntityId entity, DateTime? scheduledTimeUtc, string operationName, object operationInput) + { + if (operationName == null) + { + throw new ArgumentNullException(nameof(operationName)); + } + + string functionName = entity.EntityName; + + var action = new SendSignalOperationAction() + { + InstanceId = EntityId.GetInstanceIdFromEntityId(entity), + Name = operationName, + ScheduledTime = scheduledTimeUtc, + Input = null, + }; + + if (operationInput != null) + { + try + { + action.Input = taskEntity.MessageDataConverter.Serialize(operationInput); + } + catch (Exception e) + { + throw new EntitySchedulerException($"Failed to serialize input for operation '{operationName}': {e.Message}", e); + } + } + + // add the action to the results, under a lock since user code may be concurrent + lock (this.batchResult.Actions) + { + this.batchResult.Actions.Add(action); + } + } + + public override string StartNewOrchestration(string name, string version, object input, string instanceId = null) + { + if (string.IsNullOrEmpty(instanceId)) + { + instanceId = Guid.NewGuid().ToString(); // this is an entity, so we don't need to be deterministic + } + else if (DurableTask.Core.Common.Entities.IsEntityInstance(instanceId)) + { + throw new ArgumentException(nameof(instanceId), "Invalid orchestration instance ID, must not be an entity ID"); + } + + var action = new StartNewOrchestrationOperationAction() + { + InstanceId = instanceId, + Name = name, + Version = version, + Tags = new Dictionary() { { OrchestrationTags.FireAndForget, "" } }, + }; + + if (input != null) + { + try + { + action.Input = taskEntity.MessageDataConverter.Serialize(input); + } + catch (Exception e) + { + throw new EntitySchedulerException($"Failed to serialize input for orchestration '{name}': {e.Message}", e); + } + } + + // add the action to the results, under a lock since user code may be concurrent + lock (this.batchResult.Actions) + { + this.batchResult.Actions.Add(action); + } + + return instanceId; + } + + public async Task ExecuteBatchAsync() + { + // execute all the operations in a loop and record the results. + for (int i = 0; i < this.batchRequest.Operations.Count; i++) + { + await this.ProcessOperationRequestAsync(i); + } + + if (this.taskEntity.RollbackOnExceptions) + { + // the state has already been written back, since it is + // done right after each operation. + } + else + { + // we are writing back the state only now, after the whole batch is complete. + + var writeBackSuccessful = this.TryWriteback(out OperationResult serializationErrorMessage); + + if (!writeBackSuccessful) + { + // failed to write state, we now consider all operations in the batch as failed. + // We thus record the results (which are now fail results) and commit the batch + // with the original state. + + // we clear the actions (if we cannot update the state, we should not take other actions either) + this.batchResult.Actions.Clear(); + + // we replace all response messages with the serialization error message, + // so that callers get to know that this operation failed, and why + for (int i = 0; i < this.batchResult.Results.Count; i++) + { + this.batchResult.Results[i] = serializationErrorMessage; + } + } + } + } + + async ValueTask ProcessOperationRequestAsync(int index) + { + // set context for operation + var operation = this.batchRequest.Operations[index]; + this.batchPosition = index; + this.currentOperationResult = new OperationResult(); + + var actionPositionCheckpoint = this.batchResult.Actions.Count; + + try + { + await taskEntity.ExecuteOperationAsync(this); + } + catch (Exception e) when (!Utils.IsFatal(e) && !Utils.IsExecutionAborting(e)) + { + this.CaptureExceptionInOperationResult(this.currentOperationResult, e); + } + + if (this.taskEntity.RollbackOnExceptions) + { + // we write back the entity state after each successful operation + if (this.currentOperationResult.ErrorMessage == null) + { + if (!this.TryWriteback(out OperationResult errorResult, operation)) + { + // state serialization failed; create error response and roll back. + this.currentOperationResult = errorResult; + } + } + + if (this.currentOperationResult.ErrorMessage != null) + { + // we must also roll back any actions that this operation has issued + this.Rollback(actionPositionCheckpoint); + } + } + + // write the result to the list of results for the batch + this.batchResult.Results.Add(this.currentOperationResult); + } + + public void CaptureExceptionInOperationResult(OperationResult result, Exception originalException) + { + // a non-null ErrorMessage field is how we track internally that this result represents a failed operation + result.ErrorMessage = originalException.Message ?? originalException.GetType().FullName; + + // record additional information, based on the error propagation mode + switch (this.executionOptions.ErrorPropagationMode) + { + case ErrorPropagationMode.SerializeExceptions: + try + { + result.Result = this.taskEntity.ErrorDataConverter.Serialize(originalException); + } + catch (Exception serializationException) when (!Utils.IsFatal(serializationException)) + { + // we can't serialize the original exception. We can't throw it here. + // So let us try to at least serialize the serialization exception + // because this information may help users that are trying to troubleshoot their application. + try + { + result.Result = this.taskEntity.ErrorDataConverter.Serialize(serializationException); + } + catch (Exception serializationExceptionSerializationException) when (!Utils.IsFatal(serializationExceptionSerializationException)) + { + // there seems to be nothing we can do. + } + } + break; + + case ErrorPropagationMode.UseFailureDetails: + result.FailureDetails = new FailureDetails(originalException); + break; + } + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs b/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs new file mode 100644 index 000000000..01da29f08 --- /dev/null +++ b/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs @@ -0,0 +1,537 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core +{ + using DurableTask.Core.Logging; + using DurableTask.Core.Serializing; + using Microsoft.Extensions.Logging; + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + using System.Threading; + using DurableTask.Core.Entities; + using DurableTask.Core.History; + using System.Linq; + using System.Diagnostics; + using DurableTask.Core.Query; + using Newtonsoft.Json; + + /// + /// Client used to manage and query entity instances + /// + public sealed class TaskHubEntityClient + { + readonly DataConverter messageDataConverter; + readonly DataConverter stateDataConverter; + readonly LogHelper logHelper; + readonly EntityBackendInformation backendInformation; + readonly IOrchestrationServiceQueryClient queryClient; + readonly IOrchestrationServicePurgeClient purgeClient; + + /// + /// The orchestration service client for this task hub client + /// + public IOrchestrationServiceClient ServiceClient { get; } + + private void CheckEntitySupport(string name) + { + if (this.backendInformation == null) + { + throw new InvalidOperationException($"{nameof(TaskHubEntityClient)}.{name} is not supported because the chosen backend does not support entities."); + } + } + private void CheckQuerySupport(string name) + { + if (this.queryClient == null) + { + throw new InvalidOperationException($"{nameof(TaskHubEntityClient)}.{name} is not supported because the chosen backend does not implement {nameof(IOrchestrationServiceQueryClient)}."); + } + } + private void CheckPurgeSupport(string name) + { + if (this.purgeClient == null) + { + throw new InvalidOperationException($"{nameof(TaskHubEntityClient)}.{name} is not supported because the chosen backend does not implement {nameof(IOrchestrationServicePurgeClient)}."); + } + } + + /// + /// Create a new TaskHubEntityClient from a given TaskHubClient. + /// + /// The taskhub client. + /// The to use for entity state deserialization, or null if the default converter should be used for that purpose. + public TaskHubEntityClient(TaskHubClient client, DataConverter stateDataConverter = null) + { + this.ServiceClient = client.ServiceClient; + this.messageDataConverter = client.DefaultConverter; + this.stateDataConverter = stateDataConverter ?? client.DefaultConverter; + this.logHelper = client.LogHelper; + this.backendInformation = (client.ServiceClient as EntityBackendInformation.IInformationProvider)?.GetEntityBackendInformation(); + this.queryClient = client.ServiceClient as IOrchestrationServiceQueryClient; + this.purgeClient = client.ServiceClient as IOrchestrationServicePurgeClient; + } + + /// + /// Signals an entity to perform an operation. + /// + /// The target entity. + /// The name of the operation. + /// The input for the operation. + /// A future time for which to schedule the start of this operation, or null if is should start as soon as possible. + /// A task that completes when the message has been reliably enqueued. + public async Task SignalEntityAsync(EntityId entityId, string operationName, object operationInput = null, DateTime? scheduledTimeUtc = null) + { + this.CheckEntitySupport(nameof(SignalEntityAsync)); + + (DateTime original, DateTime capped)? scheduledTime = null; + if (scheduledTimeUtc.HasValue) + { + scheduledTime = this.backendInformation.GetCappedScheduledTime(DateTime.UtcNow, scheduledTimeUtc.Value.ToUniversalTime()); + } + + var guid = Guid.NewGuid(); // unique id for this request + var instanceId = EntityId.GetInstanceIdFromEntityId(entityId); + var instance = new OrchestrationInstance() { InstanceId = instanceId }; + + string serializedInput = null; + if (operationInput != null) + { + serializedInput = this.messageDataConverter.Serialize(operationInput); + } + + (string name, object content) eventToSend = ClientEntityContext.EmitOperationSignal( + guid, + operationName, + serializedInput, + scheduledTime); + + string serializedEventContent = this.messageDataConverter.Serialize(eventToSend.content); + + var eventRaisedEvent = new EventRaisedEvent(-1, serializedEventContent) + { + Name = eventToSend.name + }; + + var taskMessage = new TaskMessage + { + OrchestrationInstance = instance, + Event = eventRaisedEvent, + }; + + this.logHelper.RaisingEvent(instance, eventRaisedEvent); + + await this.ServiceClient.SendTaskOrchestrationMessageAsync(taskMessage); + } + + /// + /// Tries to read the current state of an entity. + /// + /// The JSON-serializable type of the entity. + /// The target entity. + /// a response containing the current state of the entity. + public async Task> ReadEntityStateAsync(EntityId entityId) + { + var instanceId = EntityId.GetInstanceIdFromEntityId(entityId); + + this.logHelper.FetchingInstanceState(instanceId); + IList stateList = await this.ServiceClient.GetOrchestrationStateAsync(instanceId, allExecutions:false); + + OrchestrationState state = stateList?.FirstOrDefault(); + if (state != null + && state.OrchestrationInstance != null + && state.Input != null) + { + if (ClientEntityContext.TryGetEntityStateFromSerializedSchedulerState(state.Input, out string serializedEntityState)) + { + return new StateResponse() + { + EntityExists = true, + EntityState = this.messageDataConverter.Deserialize(serializedEntityState), + }; + } + } + + return new StateResponse() + { + EntityExists = false, + EntityState = default, + }; + } + + /// + /// The response returned by . + /// + /// The JSON-serializable type of the entity. + public struct StateResponse + { + /// + /// Whether this entity has a state or not. + /// + /// An entity initially has no state, but a state is created and persisted in storage once operations access it. + public bool EntityExists { get; set; } + + /// + /// The current state of the entity, if it exists, or default() otherwise. + /// + public T EntityState { get; set; } + } + + /// + /// Gets the status of all entity instances that match the specified query conditions. + /// + /// Return entity instances that match the specified query conditions. + /// Cancellation token that can be used to cancel the query operation. + /// Returns a page of entity instances and a continuation token for fetching the next page. + public async Task ListEntitiesAsync(Query query, CancellationToken cancellationToken) + { + this.CheckEntitySupport(nameof(ListEntitiesAsync)); + this.CheckQuerySupport(nameof(ListEntitiesAsync)); + + OrchestrationQuery innerQuery = new OrchestrationQuery() + { + FetchInputsAndOutputs = query.FetchState, + ContinuationToken = query.ContinuationToken, + CreatedTimeFrom = query.LastOperationFrom, + CreatedTimeTo = query.LastOperationTo, + InstanceIdPrefix = "@", + PageSize = query.PageSize, + RuntimeStatus = null, + TaskHubNames = query.TaskHubNames, + }; + + bool unsatisfiable = false; + + void ApplyPrefixConjunction(string prefix) + { + if (innerQuery.InstanceIdPrefix.Length >= prefix.Length) + { + unsatisfiable = unsatisfiable || !innerQuery.InstanceIdPrefix.StartsWith(prefix); + } + else + { + unsatisfiable = unsatisfiable || !prefix.StartsWith(innerQuery.InstanceIdPrefix); + innerQuery.InstanceIdPrefix = prefix; + } + } + + if (query.InstanceIdPrefix != null) + { + ApplyPrefixConjunction(query.InstanceIdPrefix); + } + if (query.EntityName != null) + { + ApplyPrefixConjunction(EntityId.GetSchedulerIdPrefixFromEntityName(query.EntityName)); + } + + if (unsatisfiable) + { + return new QueryResult() + { + Entities = new List(), + ContinuationToken = null, + }; + } + + Stopwatch stopwatch = new Stopwatch(); + stopwatch.Start(); + QueryResult entityResult = new QueryResult() + { + Entities = new List(), + ContinuationToken = innerQuery.ContinuationToken, + }; + + do + { + var result = await queryClient.GetOrchestrationWithQueryAsync(innerQuery, cancellationToken).ConfigureAwait(false); + entityResult.Entities.AddRange(result.OrchestrationState + .Select(ConvertStatusResult) + .Where(status => status != null)); + entityResult.ContinuationToken = innerQuery.ContinuationToken = result.ContinuationToken; + } + while ( // run multiple queries if no records are found, but never in excess of 100ms + entityResult.ContinuationToken != null + && !entityResult.Entities.Any() + && stopwatch.ElapsedMilliseconds <= 100 + && !cancellationToken.IsCancellationRequested); + + return entityResult; + + EntityStatus ConvertStatusResult(OrchestrationState orchestrationState) + { + string state = null; + bool hasState = false; + + if (query.FetchState && orchestrationState.Input != null) + { + hasState = ClientEntityContext.TryGetEntityStateFromSerializedSchedulerState(orchestrationState.Input, out state); + } + else if (orchestrationState.Status != null && orchestrationState.Status != "null") + { + var entityStatus = new DurableTask.Core.Entities.StateFormat.EntityStatus(); + JsonConvert.PopulateObject(orchestrationState.Status, entityStatus, Serializer.InternalSerializerSettings); + hasState = entityStatus.EntityExists; + } + + if (hasState || query.IncludeDeleted) + { + return new EntityStatus() + { + EntityId = EntityId.GetEntityIdFromInstanceId(orchestrationState.OrchestrationInstance.InstanceId), + LastOperationTime = orchestrationState.CreatedTime, + State = state, + }; + } + else + { + return null; + } + } + } + + /// + /// Query condition for searching the status of entity instances. + /// + public class Query + { + /// + /// If not null, return only entities whose name matches this name. + /// + public string EntityName { get; set; } + + /// + /// If not null, return only entities whose instance id starts with this prefix. + /// + public string InstanceIdPrefix { get; set; } + + /// + /// If not null, return only entity instances which had operations after this DateTime. + /// + public DateTime? LastOperationFrom { get; set; } + + /// + /// If not null, return only entity instances which had operations before this DateTime. + /// + public DateTime? LastOperationTo { get; set; } + + /// + /// If not null, return only entity instances from task hubs whose name is in this list. + /// + public ICollection TaskHubNames { get; set; } + + /// + /// Number of records per one request. The default value is 100. + /// + /// + /// Requests may return fewer records than the specified page size, even if there are more records. + /// Always check the continuation token to determine whether there are more records. + /// + public int PageSize { get; set; } = 100; + + /// + /// ContinuationToken of the pager. + /// + public string ContinuationToken { get; set; } + + /// + /// Determines whether the query results include the state of the entity. + /// + public bool FetchState { get; set; } = false; + + /// + /// Determines whether the results may include entities that currently have no state (such as deleted entities). + /// + /// The effects of this vary by backend. Some backends do not retain deleted entities, so this parameter is irrelevant in that situation. + public bool IncludeDeleted { get; set; } = false; + } + + /// + /// A partial result of an entity status query. + /// + public class QueryResult + { + /// + /// Gets or sets a collection of statuses of entity instances matching the query description. + /// + /// A collection of entity instance status values. + public List Entities { get; set; } + + /// + /// Gets or sets a token that can be used to resume the query with data not already returned by this query. + /// + /// A server-generated continuation token or null if there are no further continuations. + public string ContinuationToken { get; set; } + } + + /// + /// The status of an entity, as returned by entity queries. + /// + public class EntityStatus + { + /// + /// The EntityId of the queried entity instance. + /// + /// + /// The unique EntityId of the instance. + /// + public EntityId EntityId { get; set; } + + /// + /// The time of the last operation processed by the entity instance. + /// + /// + /// The last operation time in UTC. + /// + public DateTime LastOperationTime { get; set; } + + /// + /// The current state of the entity instance, or null if states were not fetched or the entity has no state. + /// + public string State { get; set; } + } + + /// + /// Removes empty entities from storage and releases orphaned locks. + /// + /// An entity is considered empty, and is removed, if it has no state, is not locked, and has + /// been idle for more than minutes. + /// Locks are considered orphaned, and are released, if the orchestration that holds them is not in state . This + /// should not happen under normal circumstances, but can occur if the orchestration instance holding the lock + /// exhibits replay nondeterminism failures, or if it is explicitly purged. + /// Whether to remove empty entities. + /// Whether to release orphaned locks. + /// Cancellation token that can be used to cancel the operation. + /// A task that completes when the operation is finished. + public async Task CleanEntityStorageAsync(bool removeEmptyEntities, bool releaseOrphanedLocks, CancellationToken cancellationToken) + { + this.CheckEntitySupport(nameof(CleanEntityStorageAsync)); + this.CheckQuerySupport(nameof(CleanEntityStorageAsync)); + + if (removeEmptyEntities) + { + this.CheckPurgeSupport(nameof(CleanEntityStorageAsync)); + } + + DateTime now = DateTime.UtcNow; + CleanEntityStorageResult finalResult = default; + + var query = new OrchestrationQuery() + { + InstanceIdPrefix = "@", + FetchInputsAndOutputs = false, + }; + + // list all entities (without fetching the input) and for each one that requires action, + // perform that action. Waits for all actions to finish after each page. + do + { + var page = await this.queryClient.GetOrchestrationWithQueryAsync(query, cancellationToken); + + List tasks = new List(); + foreach (var state in page.OrchestrationState) + { + var status = new DurableTask.Core.Entities.StateFormat.EntityStatus(); + JsonConvert.PopulateObject(state.Status, status, Serializer.InternalSerializerSettings); + + if (releaseOrphanedLocks && status.LockedBy != null) + { + tasks.Add(CheckForOrphanedLockAndFixIt(state.OrchestrationInstance.InstanceId, status.LockedBy)); + } + + if (removeEmptyEntities) + { + bool isEmptyEntity = !status.EntityExists && status.LockedBy == null && status.QueueSize == 0; + bool safeToRemoveWithoutBreakingMessageSorterLogic = (this.backendInformation.EntityMessageReorderWindow == TimeSpan.Zero) ? + true : (now - state.LastUpdatedTime > this.backendInformation.EntityMessageReorderWindow); + if (isEmptyEntity && safeToRemoveWithoutBreakingMessageSorterLogic) + { + tasks.Add(DeleteIdleOrchestrationEntity(state)); + } + } + } + + async Task DeleteIdleOrchestrationEntity(OrchestrationState state) + { + await this.purgeClient.PurgeInstanceStateAsync(state.OrchestrationInstance.InstanceId); + Interlocked.Increment(ref finalResult.NumberOfEmptyEntitiesRemoved); + } + + async Task CheckForOrphanedLockAndFixIt(string instanceId, string lockOwner) + { + bool lockOwnerIsStillRunning = false; + + IList stateList = await this.ServiceClient.GetOrchestrationStateAsync(lockOwner, allExecutions: false); + OrchestrationState state = stateList?.FirstOrDefault(); + if (state != null) + { + lockOwnerIsStillRunning = + (state.OrchestrationStatus == OrchestrationStatus.Running + || state.OrchestrationStatus == OrchestrationStatus.Suspended); + } + + if (!lockOwnerIsStillRunning) + { + // the owner is not a running orchestration. Send a lock release. + (string name, object content) eventToSend = ClientEntityContext.EmitUnlockForOrphanedLock(lockOwner); + + string serializedEventContent = this.messageDataConverter.Serialize(eventToSend.content); + + var eventRaisedEvent = new EventRaisedEvent(-1, serializedEventContent) + { + Name = eventToSend.name + }; + + OrchestrationInstance targetInstance = new OrchestrationInstance() + { + InstanceId = instanceId, + }; + + var taskMessage = new TaskMessage + { + OrchestrationInstance = targetInstance, + Event = eventRaisedEvent, + }; + + this.logHelper.RaisingEvent(targetInstance, eventRaisedEvent); + + await this.ServiceClient.SendTaskOrchestrationMessageAsync(taskMessage); + + Interlocked.Increment(ref finalResult.NumberOfOrphanedLocksRemoved); + } + } + + await Task.WhenAll(tasks); + query.ContinuationToken = page.ContinuationToken; + } + while (query.ContinuationToken != null); + + return finalResult; + } + + /// + /// The result of a clean entity storage operation. + /// + public struct CleanEntityStorageResult + { + /// + /// The number of orphaned locks that were removed. + /// + public int NumberOfOrphanedLocksRemoved; + + /// + /// The number of entities whose metadata was removed from storage. + /// + public int NumberOfEmptyEntitiesRemoved; + } + } +} diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationAction.cs new file mode 100644 index 000000000..467cecda4 --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationAction.cs @@ -0,0 +1,29 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities.OperationFormat +{ + using Newtonsoft.Json; + + /// + /// Defines a set of base properties for an operator action. + /// + [JsonConverter(typeof(OperationActionConverter))] + public abstract class OperationAction + { + /// + /// The type of the orchestrator action. + /// + public abstract OperationActionType OperationActionType { get; } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationActionConverter.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationActionConverter.cs new file mode 100644 index 000000000..e8081ad06 --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationActionConverter.cs @@ -0,0 +1,41 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Entities.OperationFormat +{ + using System; + using Newtonsoft.Json.Linq; + using DurableTask.Core.Serializing; + + internal class OperationActionConverter : JsonCreationConverter + { + protected override OperationAction CreateObject(Type objectType, JObject jObject) + { + if (jObject.TryGetValue("OperationActionType", StringComparison.OrdinalIgnoreCase, out JToken actionType)) + { + var type = (OperationActionType)int.Parse((string)actionType); + switch (type) + { + case OperationActionType.SendSignal: + return new SendSignalOperationAction(); + case OperationActionType.StartNewOrchestration: + return new StartNewOrchestrationOperationAction(); + default: + throw new NotSupportedException("Unrecognized action type."); + } + } + + throw new NotSupportedException("Action Type not provided."); + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationActionType.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationActionType.cs new file mode 100644 index 000000000..d0c2dc177 --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationActionType.cs @@ -0,0 +1,31 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Entities.OperationFormat +{ + /// + /// Enumeration of entity operation actions. + /// + public enum OperationActionType + { + /// + /// A signal was sent to an entity + /// + SendSignal, + + /// + /// A new fire-and-forget orchestration was started + /// + StartNewOrchestration, + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationBatchRequest.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationBatchRequest.cs new file mode 100644 index 000000000..69515b184 --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationBatchRequest.cs @@ -0,0 +1,44 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities.OperationFormat +{ + using Newtonsoft.Json; + using System; + using System.Collections.Generic; + using System.Text; + + /// + /// A request for execution of a batch of operations on an entity. + /// + public class OperationBatchRequest + { + // NOTE: Actions must be serializable by a variety of different serializer types to support out-of-process execution. + // To ensure maximum compatibility, all properties should be public and settable by default. + + /// + /// The instance id for this entity. + /// + public string? InstanceId { get; set; } + + /// + /// The current state of the entity, or null if the entity does not exist. + /// + public string? EntityState { get; set; } + + /// + /// The list of operations to be performed on the entity. + /// + public List? Operations { get; set; } + } +} diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationBatchResult.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationBatchResult.cs new file mode 100644 index 000000000..9a8e9deee --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationBatchResult.cs @@ -0,0 +1,48 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities.OperationFormat +{ + using Newtonsoft.Json; + using System; + using System.Collections.Generic; + using System.Text; + + /// + /// The results of executing a batch of operations on the entity out of process. + /// + public class OperationBatchResult + { + // NOTE: Actions must be serializable by a variety of different serializer types to support out-of-process execution. + // To ensure maximum compatibility, all properties should be public and settable by default. + + /// + /// The results of executing the operations in the batch. The length of this list must match + /// the size of the batch if all messages were processed; In particular, all execution errors must be reported as a result. + /// However, this list of results can be shorter than the list of operations if + /// some suffix of the operation list was skipped, e.g. due to shutdown, send throttling, or timeouts. + /// + public List? Results { get; set; } + + /// + /// The list of actions (outgoing messages) performed while executing the operations in the batch. Can be empty. + /// + public List? Actions { get; set; } + + /// + /// The state of the entity after executing the batch, + /// or null if the entity has no state (e.g. if it has been deleted). + /// + public string? EntityState { get; set; } + } +} diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs new file mode 100644 index 000000000..09405cb41 --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs @@ -0,0 +1,43 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities.OperationFormat +{ + using System; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + /// + /// A request message sent to an entity when calling or signaling the entity. + /// + public class OperationRequest + { + // NOTE: Actions must be serializable by a variety of different serializer types to support out-of-process execution. + // To ensure maximum compatibility, all properties should be public and settable by default. + + /// + /// The name of the operation. + /// + public string? Operation { get; set; } + + /// + /// The unique GUID of the operation. + /// + public Guid Id { get; set; } + + /// + /// The input for the operation. Can be null if no input was given. + /// + public string? Input { get; set; } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs new file mode 100644 index 000000000..2fdee8733 --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs @@ -0,0 +1,46 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities.OperationFormat +{ + using System; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + /// + /// A response message sent by an entity to a caller after it executes an operation. + /// + public class OperationResult + { + // NOTE: Actions must be serializable by a variety of different serializer types to support out-of-process execution. + // To ensure maximum compatibility, all properties should be public and settable by default. + + /// + /// The serialized result returned by the operation. Can be null, if the operation returned no result. + /// May contain error details, such as a serialized exception, if is not null. + /// + public string? Result { get; set; } + + /// + /// If non-null, this string indicates that this operation did not successfully complete. + /// The actual content and its interpretation varies depending on the SDK used. + /// + public string? ErrorMessage { get; set; } + + /// + /// A structured language-independent representation of the error. Whether this field is present + /// depends on which SDK is used, and on configuration settings. + /// + public FailureDetails? FailureDetails { get; set; } + } +} diff --git a/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs new file mode 100644 index 000000000..800613cad --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs @@ -0,0 +1,51 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities.OperationFormat +{ + using Newtonsoft.Json; + using System; + using System.Collections.Generic; + + /// + /// Operation action for sending a signal. + /// + public class SendSignalOperationAction : OperationAction + { + /// + public override OperationActionType OperationActionType => OperationActionType.SendSignal; + + // NOTE: Actions must be serializable by a variety of different serializer types to support out-of-process execution. + // To ensure maximum compatibility, all properties should be public and settable by default. + + /// + /// The destination entity for the signal. + /// + public string? InstanceId { get; set; } + + /// + /// The name of the operation being signaled. + /// + public string? Name { get; set; } + + /// + /// The input of the operation being signaled. + /// + public string? Input { get; set; } + + /// + /// Optionally, a scheduled delivery time for the signal. + /// + public DateTime? ScheduledTime { get; set; } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs new file mode 100644 index 000000000..27b486d08 --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs @@ -0,0 +1,55 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities.OperationFormat +{ + using DurableTask.Core.Entities; + using System.Collections.Generic; + + /// + /// Orchestrator action for creating sub-orchestrations. + /// + public class StartNewOrchestrationOperationAction : OperationAction + { + /// + public override OperationActionType OperationActionType => OperationActionType.StartNewOrchestration; + + // NOTE: Actions must be serializable by a variety of different serializer types to support out-of-process execution. + // To ensure maximum compatibility, all properties should be public and settable by default. + + /// + /// The name of the sub-orchestrator to start. + /// + public string? Name { get; set; } + + /// + /// The version of the sub-orchestrator to start. + /// + public string? Version { get; set; } + + /// + /// The instance ID of the created sub-orchestration. + /// + public string? InstanceId { get; set; } + + /// + /// The input of the sub-orchestration. + /// + public string? Input { get; set; } + + /// + /// Tags to be applied to the sub-orchestration. + /// + public IDictionary? Tags { get; set; } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs new file mode 100644 index 000000000..11a2447a5 --- /dev/null +++ b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs @@ -0,0 +1,463 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +namespace DurableTask.Core.Entities +{ + using DurableTask.Core.Common; + using DurableTask.Core.Entities.EventFormat; + using DurableTask.Core.Entities.OperationFormat; + using DurableTask.Core.Entities.StateFormat; + using DurableTask.Core.History; + using DurableTask.Core.Serializing; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + + /// + /// Tracks the entity-related state of an orchestration. + /// Tracks and validates the synchronization state. + /// + public class OrchestrationEntityContext + { + private readonly string instanceId; + private readonly string executionId; + private readonly OrchestrationContext innerContext; + private readonly Dictionary pending; + private readonly MessageSorter messageSorter; + + private Guid? criticalSectionId; + private EntityId[] criticalSectionLocks; + private bool lockAcquisitionPending; + private HashSet availableLocks; + + /// + /// Constructs an OrchestrationEntityContext. + /// + /// The instance id. + /// The execution id. + /// The inner context. + public OrchestrationEntityContext( + string instanceId, + string executionId, + OrchestrationContext innerContext) + { + this.instanceId = instanceId; + this.executionId = executionId; + this.innerContext = innerContext; + this.pending = new Dictionary(); + this.messageSorter = new MessageSorter(); + } + + /// + /// Whether this orchestration is currently inside a critical section. + /// + public bool IsInsideCriticalSection => this.criticalSectionId != null; + + /// + /// Enumerate all the entities that are available for calling from within a critical section. + /// This set contains all the entities that were locked prior to entering the critical section, + /// and for which there is not currently an operation call pending. + /// + /// An enumeration of all the currently available entities. + public IEnumerable<(string entityName, string entityKey)> GetAvailableEntities() + { + if (this.IsInsideCriticalSection) + { + foreach(var e in this.availableLocks) + { + yield return (e.EntityName, e.EntityKey); + } + } + } + + /// + /// Check that a suborchestration is a valid transition in the current state. + /// + /// The error message, if it is not valid, or null otherwise + /// whether the transition is valid + public bool ValidateSuborchestrationTransition(out string errorMessage) + { + if (this.IsInsideCriticalSection) + { + errorMessage = "While holding locks, cannot call suborchestrators."; + return false; + } + + errorMessage = null; + return true; + } + + /// + /// Check that acquire is a valid transition in the current state. + /// + /// Whether this is a signal or a call. + /// The target instance id. + /// The error message, if it is not valid, or null otherwise + /// whether the transition is valid + public bool ValidateOperationTransition(string targetInstanceId, bool oneWay, out string errorMessage) + { + if (this.IsInsideCriticalSection) + { + var lockToUse = EntityId.GetEntityIdFromInstanceId(targetInstanceId); + if (oneWay) + { + if (this.criticalSectionLocks.Contains(lockToUse)) + { + errorMessage = "Must not signal a locked entity from a critical section."; + return false; + } + } + else + { + if (!this.availableLocks.Remove(lockToUse)) + { + if (this.lockAcquisitionPending) + { + errorMessage = "Must await the completion of the lock request prior to calling any entity."; + return false; + } + if (this.criticalSectionLocks.Contains(lockToUse)) + { + errorMessage = "Must not call an entity from a critical section while a prior call to the same entity is still pending."; + return false; + } + else + { + errorMessage = "Must not call an entity from a critical section if it is not one of the locked entities."; + return false; + } + } + } + } + + errorMessage = null; + return true; + } + + /// + /// Check that acquire is a valid transition in the current state. + /// + /// The error message, if it is not valid, or null otherwise + /// whether the transition is valid + public bool ValidateAcquireTransition(out string errorMessage) + { + if (this.IsInsideCriticalSection) + { + errorMessage = "Must not enter another critical section from within a critical section."; + return false; + } + + errorMessage = null; + return true; + } + + /// + /// Called after an operation call within a critical section completes. + /// + /// + public void RecoverLockAfterCall(string targetInstanceId) + { + if (this.IsInsideCriticalSection) + { + var lockToUse = EntityId.GetEntityIdFromInstanceId(targetInstanceId); + this.availableLocks.Add(lockToUse); + } + } + + /// + /// Get release messages for all locks in the critical section, and release them + /// + public IEnumerable<(OrchestrationInstance target, string eventName, object eventContent)> EmitLockReleaseMessages() + { + if (this.IsInsideCriticalSection) + { + var message = new ReleaseMessage() + { + ParentInstanceId = instanceId, + Id = this.criticalSectionId.Value.ToString(), + }; + + foreach (var entityId in this.criticalSectionLocks) + { + var instance = new OrchestrationInstance() { InstanceId = EntityId.GetInstanceIdFromEntityId(entityId) }; + var jmessage = JObject.FromObject(message, Serializer.InternalSerializer); + yield return (instance, EntityMessageEventNames.ReleaseMessageEventName, jmessage); + } + + this.criticalSectionLocks = null; + this.availableLocks = null; + this.criticalSectionId = null; + } + } + + /// + /// Creates a request message to be sent to an entity. + /// + /// The target entity. + /// The name of the operation. + /// If true, this is a signal, otherwise it is a call. + /// A unique identifier for this request. + /// A time for which to schedule the delivery, or null if this is not a scheduled message + /// The operation input + /// + public (string eventName, object eventContent) EmitRequestMessage( + OrchestrationInstance target, + string operationName, + bool oneWay, + Guid operationId, + (DateTime original, DateTime capped)? scheduledTimeUtc, + string input) + { + var request = new RequestMessage() + { + ParentInstanceId = this.instanceId, + ParentExecutionId = this.executionId, + Id = operationId, + IsSignal = oneWay, + Operation = operationName, + ScheduledTime = scheduledTimeUtc?.original, + Input = input, + }; + + this.AdjustOutgoingMessage(target.InstanceId, request, scheduledTimeUtc?.capped, out string eventName); + + // we pre-serialize to JObject so we can avoid exposure to application-specific serialization settings + var jrequest = JObject.FromObject(request, Serializer.InternalSerializer); + + return (eventName, jrequest); + } + + /// + /// Creates an acquire message to be sent to an entity. + /// + /// A unique request id. + /// All the entities that are to be acquired. + /// + public (OrchestrationInstance target, string eventName, object eventContent) EmitAcquireMessage(Guid lockRequestId, EntityId[] entities) + { + // All the entities in entity[] need to be locked, but to avoid deadlock, the locks have to be acquired + // sequentially, in order. So, we send the lock request to the first entity; when the first lock + // is granted by the first entity, the first entity will forward the lock request to the second entity, + // and so on; after the last entity grants the last lock, a response is sent back here. + + // acquire the locks in a globally fixed order to avoid deadlocks + Array.Sort(entities); + + // remove duplicates if necessary. Probably quite rare, so no need to optimize more. + for (int i = 0; i < entities.Length - 1; i++) + { + if (entities[i].Equals(entities[i + 1])) + { + entities = entities.Distinct().ToArray(); + break; + } + } + + // send lock request to first entity in the lock set + var target = new OrchestrationInstance() { InstanceId = EntityId.GetInstanceIdFromEntityId(entities[0]) }; + var request = new RequestMessage() + { + Id = lockRequestId, + ParentInstanceId = this.instanceId, + ParentExecutionId = this.executionId, + LockSet = entities, + Position = 0, + }; + + this.criticalSectionId = lockRequestId; + this.criticalSectionLocks = entities; + this.lockAcquisitionPending = true; + + this.AdjustOutgoingMessage(target.InstanceId, request, null, out string eventName); + + // we pre-serialize to JObject so we can avoid exposure to application-specific serialization settings + var jrequest = JObject.FromObject(request, Serializer.InternalSerializer); + + return (target, eventName, jrequest); + } + + /// + /// Called when a response to the acquire message is received from the last entity. + /// + /// The result returned. + /// The guid for the lock operation + public void CompleteAcquire(OperationResult result, Guid criticalSectionId) + { + this.availableLocks = new HashSet(this.criticalSectionLocks); + this.lockAcquisitionPending = false; + } + + internal void AdjustOutgoingMessage(string instanceId, RequestMessage requestMessage, DateTime? cappedTime, out string eventName) + { + if (requestMessage.ScheduledTime.HasValue) + { + eventName = EntityMessageEventNames.ScheduledRequestMessageEventName(cappedTime.Value); + } + else + { + this.messageSorter.LabelOutgoingMessage( + requestMessage, + instanceId, + this.innerContext.CurrentUtcDateTime, + this.innerContext.EntityBackendInformation.EntityMessageReorderWindow); + + eventName = EntityMessageEventNames.RequestMessageEventName; + } + } + + /// + /// Extracts the operation result from an event that represents an entity response. + /// + /// + /// + public OperationResult DeserializeEntityResponseEvent(string eventContent) + { + var responseMessage = new ResponseMessage(); + + // for compatibility, we deserialize in a way that is resilient to any typename presence/absence/mismatch + try + { + // restore the scheduler state from the input + JsonConvert.PopulateObject(eventContent, responseMessage, Serializer.InternalSerializerSettings); + } + catch (Exception exception) + { + throw new EntitySchedulerException("Failed to deserialize entity response.", exception); + } + + return new OperationResult() + { + Result = responseMessage.Result, + ErrorMessage = responseMessage.ErrorMessage, + FailureDetails = responseMessage.FailureDetails, + }; + } + + + private interface IEntityResponseContinuation + { + void DeliverResult(OperationResult operationResult, EventRaisedEvent eventRaisedEvent, TaskOrchestrationContext taskOrchestrationContext); + } + + private class OperationContinuation : TaskCompletionSource, IEntityResponseContinuation + { + readonly private int taskId; + readonly private string instanceId; + + public OperationContinuation(int taskId, string instanceId) + { + this.taskId = taskId; + this.instanceId = instanceId; + } + + public void DeliverResult(OperationResult operationResult, EventRaisedEvent eventRaisedEvent, TaskOrchestrationContext taskOrchestrationContext) + { + if (taskOrchestrationContext.EntityContext.IsInsideCriticalSection) + { + // the lock is available again now that the entity call returned + taskOrchestrationContext.EntityContext.RecoverLockAfterCall(this.instanceId); + } + + if (operationResult.ErrorMessage == null) + { + taskOrchestrationContext.HandleEntityOperationCompletedEvent(eventRaisedEvent, this.taskId, this.instanceId, operationResult, this); + } + else + { + taskOrchestrationContext.HandleEntityOperationFailedEvent(eventRaisedEvent, this.taskId, this.instanceId, operationResult, this); + } + } + } + + private class LockAcquisitionContinuation : TaskCompletionSource, IEntityResponseContinuation + { + readonly private Guid criticalSectionId; + readonly private int taskId; + + public LockAcquisitionContinuation(Guid lockRequestId, int taskId) + { + this.criticalSectionId = lockRequestId; + this.taskId = taskId; + } + + public void DeliverResult(OperationResult operationResult, EventRaisedEvent eventRaisedEvent, TaskOrchestrationContext taskOrchestrationContext) + { + taskOrchestrationContext.EntityContext.CompleteAcquire(operationResult, this.criticalSectionId); + this.SetResult(new LockReleaser(taskOrchestrationContext, criticalSectionId)); + } + } + + private class LockReleaser : IDisposable + { + private readonly TaskOrchestrationContext context; + private readonly Guid criticalSectionId; + + public LockReleaser(TaskOrchestrationContext context, Guid criticalSectionId) + { + this.context = context; + this.criticalSectionId = criticalSectionId; + } + + public void Dispose() + { + this.context.ExitCriticalSection(); + } + } + + internal Task WaitForOperationResponseAsync(Guid operationId, int taskId, string instanceId) + { + var promise = new OperationContinuation(taskId, instanceId); + lock (this.pending) + { + this.pending.Add(operationId, promise); + } + return promise.Task; + } + + internal Task WaitForLockResponseAsync(Guid criticalSectionId, int taskId) + { + var promise = new LockAcquisitionContinuation(criticalSectionId, taskId); + lock (this.pending) + { + this.pending.Add(criticalSectionId, promise); + } + return promise.Task; + } + + internal bool HandledAsEntityResponse(EventRaisedEvent evt, TaskOrchestrationContext taskOrchestrationContext) + { + Guid operationId; + if (!Guid.TryParse(evt.Name, out operationId)) + { + return false; + } + + IEntityResponseContinuation completionSource; + lock (this.pending) + { + if (!this.pending.TryGetValue(operationId, out completionSource)) + { + return false; + } + this.pending.Remove(operationId); + } + + var operationResult = this.DeserializeEntityResponseEvent(evt.Input); + completionSource.DeliverResult(operationResult, evt, taskOrchestrationContext); + + return true; + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/Serializer.cs b/src/DurableTask.Core/Entities/Serializer.cs new file mode 100644 index 000000000..19a662fa4 --- /dev/null +++ b/src/DurableTask.Core/Entities/Serializer.cs @@ -0,0 +1,34 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using DurableTask.Core.Serializing; +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Text; + +namespace DurableTask.Core.Entities +{ + internal static class Serializer + { + /// + /// This serializer is used exclusively for internally defined data structures and cannot be customized by user. + /// This is intentional, to avoid problems caused by our unability to control the exact format. + /// For example, including typenames can cause compatibility problems if the type name is later changed. + /// + public static JsonSerializer InternalSerializer = JsonSerializer.Create(InternalSerializerSettings); + + public static JsonSerializerSettings InternalSerializerSettings + = new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.None }; + } +} diff --git a/src/DurableTask.Core/Entities/StateFormat/EntityStatus.cs b/src/DurableTask.Core/Entities/StateFormat/EntityStatus.cs new file mode 100644 index 000000000..9118c9d4f --- /dev/null +++ b/src/DurableTask.Core/Entities/StateFormat/EntityStatus.cs @@ -0,0 +1,42 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Entities.StateFormat +{ + using Newtonsoft.Json; + + /// + /// Information about the current status of an entity. Excludes potentially large data + /// (such as the entity state, or the contents of the queue) so it can always be read with low latency. + /// + public class EntityStatus + { + /// + /// Whether this entity exists or not. + /// + [JsonProperty(PropertyName = "entityExists", DefaultValueHandling = DefaultValueHandling.Ignore)] + public bool EntityExists { get; set; } + + /// + /// The size of the queue, i.e. the number of operations that are waiting for the current operation to complete. + /// + [JsonProperty(PropertyName = "queueSize", DefaultValueHandling = DefaultValueHandling.Ignore)] + public int QueueSize { get; set; } + + /// + /// The instance id of the orchestration that currently holds the lock of this entity. + /// + [JsonProperty(PropertyName = "lockedBy", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string LockedBy { get; set; } + } +} diff --git a/src/DurableTask.Core/Entities/StateFormat/MessageSorter.cs b/src/DurableTask.Core/Entities/StateFormat/MessageSorter.cs new file mode 100644 index 000000000..517773ce3 --- /dev/null +++ b/src/DurableTask.Core/Entities/StateFormat/MessageSorter.cs @@ -0,0 +1,271 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Entities.StateFormat +{ + using System; + using System.Collections.Generic; + using System.Linq; + using DurableTask.Core.Entities.EventFormat; + using Newtonsoft.Json; + + /// + /// provides message ordering and deduplication of request messages (operations or lock requests) + /// that are sent to entities, from other entities, or from orchestrations. + /// + internal class MessageSorter + { + // don't update the reorder window too often since the garbage collection incurs some overhead. + private static readonly TimeSpan MinIntervalBetweenCollections = TimeSpan.FromSeconds(10); + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public Dictionary LastSentToInstance { get; set; } + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public Dictionary ReceivedFromInstance { get; set; } + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public DateTime ReceiveHorizon { get; set; } + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public DateTime SendHorizon { get; set; } + + /// + /// Used for testing purposes. + /// + [JsonIgnore] + internal int NumberBufferedRequests => + ReceivedFromInstance?.Select(kvp => kvp.Value.Buffered?.Count ?? 0).Sum() ?? 0; + + /// + /// Called on the sending side, to fill in timestamp and predecessor fields. + /// + public void LabelOutgoingMessage(RequestMessage message, string destination, DateTime now, TimeSpan reorderWindow) + { + if (reorderWindow.Ticks == 0) + { + return; // we are not doing any message sorting. + } + + DateTime timestamp = now; + + if (SendHorizon + reorderWindow + MinIntervalBetweenCollections < now) + { + SendHorizon = now - reorderWindow; + + // clean out send clocks that are past the reorder window + + if (LastSentToInstance != null) + { + List expired = new List(); + + foreach (var kvp in LastSentToInstance) + { + if (kvp.Value < SendHorizon) + { + expired.Add(kvp.Key); + } + } + + foreach (var t in expired) + { + LastSentToInstance.Remove(t); + } + } + } + + if (LastSentToInstance == null) + { + LastSentToInstance = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + else if (LastSentToInstance.TryGetValue(destination, out var last)) + { + message.Predecessor = last; + + // ensure timestamps are monotonic even if system clock is not + if (timestamp <= last) + { + timestamp = new DateTime(last.Ticks + 1); + } + } + + message.Timestamp = timestamp; + LastSentToInstance[destination] = timestamp; + } + + /// + /// Called on the receiving side, to reorder and deduplicate within the window. + /// + public IEnumerable ReceiveInOrder(RequestMessage message, TimeSpan reorderWindow) + { + // messages sent from clients and forwarded lock messages are not participating in the sorting. + if (reorderWindow.Ticks == 0 || message.ParentInstanceId == null || message.Position > 0) + { + // Just pass the message through. + yield return message; + yield break; + } + + // advance the horizon based on the latest timestamp + if (ReceiveHorizon + reorderWindow + MinIntervalBetweenCollections < message.Timestamp) + { + ReceiveHorizon = message.Timestamp - reorderWindow; + + // deliver any messages that were held in the receive buffers + // but are now past the reorder window + + List emptyReceiveBuffers = new List(); + + if (ReceivedFromInstance != null) + { + foreach (var kvp in ReceivedFromInstance) + { + if (kvp.Value.Last < ReceiveHorizon) + { + kvp.Value.Last = DateTime.MinValue; + } + + while (TryDeliverNextMessage(kvp.Value, out var next)) + { + yield return next; + } + + if (kvp.Value.Last == DateTime.MinValue + && (kvp.Value.Buffered == null || kvp.Value.Buffered.Count == 0)) + { + emptyReceiveBuffers.Add(kvp.Key); + } + } + + foreach (var t in emptyReceiveBuffers) + { + ReceivedFromInstance.Remove(t); + } + + if (ReceivedFromInstance.Count == 0) + { + ReceivedFromInstance = null; + } + } + } + + // Messages older than the reorder window are not participating. + if (message.Timestamp < ReceiveHorizon) + { + // Just pass the message through. + yield return message; + yield break; + } + + ReceiveBuffer receiveBuffer; + + if (ReceivedFromInstance == null) + { + ReceivedFromInstance = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + if (!ReceivedFromInstance.TryGetValue(message.ParentInstanceId, out receiveBuffer)) + { + ReceivedFromInstance[message.ParentInstanceId] = receiveBuffer = new ReceiveBuffer() + { + ExecutionId = message.ParentExecutionId, + }; + } + else if (receiveBuffer.ExecutionId != message.ParentExecutionId) + { + // this message is from a new execution; release all buffered messages and start over + if (receiveBuffer.Buffered != null) + { + foreach (var kvp in receiveBuffer.Buffered) + { + yield return kvp.Value; + } + + receiveBuffer.Buffered.Clear(); + } + + receiveBuffer.Last = DateTime.MinValue; + receiveBuffer.ExecutionId = message.ParentExecutionId; + } + + if (message.Timestamp <= receiveBuffer.Last) + { + // This message was already delivered, it's a duplicate + yield break; + } + + if (message.Predecessor > receiveBuffer.Last + && message.Predecessor >= ReceiveHorizon) + { + // this message is waiting for a non-delivered predecessor in the window, buffer it + if (receiveBuffer.Buffered == null) + { + receiveBuffer.Buffered = new SortedDictionary(); + } + + receiveBuffer.Buffered[message.Timestamp] = message; + } + else + { + yield return message; + + receiveBuffer.Last = message.Timestamp >= ReceiveHorizon ? message.Timestamp : DateTime.MinValue; + + while (TryDeliverNextMessage(receiveBuffer, out var next)) + { + yield return next; + } + } + } + + private bool TryDeliverNextMessage(ReceiveBuffer buffer, out RequestMessage message) + { + if (buffer.Buffered != null) + { + using (var e = buffer.Buffered.GetEnumerator()) + { + if (e.MoveNext()) + { + var pred = e.Current.Value.Predecessor; + + if (pred <= buffer.Last || pred < ReceiveHorizon) + { + message = e.Current.Value; + + buffer.Last = message.Timestamp >= ReceiveHorizon ? message.Timestamp : DateTime.MinValue; + + buffer.Buffered.Remove(message.Timestamp); + + return true; + } + } + } + } + + message = null; + return false; + } + + public class ReceiveBuffer + { + [JsonProperty] + public DateTime Last { get; set; }// last message delivered, or DateTime.Min if none + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public string ExecutionId { get; set; } // execution id of last message, if any + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public SortedDictionary Buffered { get; set; } + } + } +} diff --git a/src/DurableTask.Core/Entities/StateFormat/SchedulerState.cs b/src/DurableTask.Core/Entities/StateFormat/SchedulerState.cs new file mode 100644 index 000000000..86d81b84b --- /dev/null +++ b/src/DurableTask.Core/Entities/StateFormat/SchedulerState.cs @@ -0,0 +1,112 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Entities.StateFormat +{ + using System.Collections.Generic; + using DurableTask.Core.Entities.EventFormat; + using Newtonsoft.Json; + + /// + /// The persisted state of an entity scheduler, as handed forward between ContinueAsNew instances. + /// + [JsonObject(MemberSerialization.OptIn)] + internal class SchedulerState + { + /// + /// Whether this entity exists or not. + /// + [JsonProperty(PropertyName = "exists", DefaultValueHandling = DefaultValueHandling.Ignore)] + public bool EntityExists { get; set; } + + /// + /// The last serialized entity state. + /// + [JsonProperty(PropertyName = "state", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string EntityState { get; set; } + + /// + /// The queue of waiting operations, or null if none. + /// + [JsonProperty(PropertyName = "queue", DefaultValueHandling = DefaultValueHandling.Ignore)] + public Queue Queue { get; private set; } + + /// + /// The instance id of the orchestration that currently holds the lock of this entity. + /// + [JsonProperty(PropertyName = "lockedBy", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string LockedBy { get; set; } + + /// + /// Whether processing on this entity is currently suspended. + /// + [JsonProperty(PropertyName = "suspended", DefaultValueHandling = DefaultValueHandling.Ignore)] + public bool Suspended { get; set; } + + /// + /// The metadata used for reordering and deduplication of messages sent to entities. + /// + [JsonProperty(PropertyName = "sorter", DefaultValueHandling = DefaultValueHandling.Ignore)] + public MessageSorter MessageSorter { get; set; } = new MessageSorter(); + + [JsonIgnore] + public bool IsEmpty => !EntityExists && (Queue == null || Queue.Count == 0) && LockedBy == null; + + internal void Enqueue(RequestMessage operationMessage) + { + if (Queue == null) + { + Queue = new Queue(); + } + + Queue.Enqueue(operationMessage); + } + + internal void PutBack(Queue messages) + { + if (Queue != null) + { + foreach (var message in Queue) + { + messages.Enqueue(message); + } + } + + Queue = messages; + } + + internal bool MayDequeue() + { + return Queue != null + && Queue.Count > 0 + && (LockedBy == null || LockedBy == Queue.Peek().ParentInstanceId); + } + + internal RequestMessage Dequeue() + { + var result = Queue.Dequeue(); + + if (Queue.Count == 0) + { + Queue = null; + } + + return result; + } + + public override string ToString() + { + return $"exists={EntityExists} queue.count={(Queue != null ? Queue.Count : 0)}"; + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/TaskEntity.cs b/src/DurableTask.Core/Entities/TaskEntity.cs new file mode 100644 index 000000000..f81877a98 --- /dev/null +++ b/src/DurableTask.Core/Entities/TaskEntity.cs @@ -0,0 +1,122 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Entities +{ + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + using DurableTask.Core.Common; + using DurableTask.Core.Entities.EventFormat; + using DurableTask.Core.Entities.OperationFormat; + using DurableTask.Core.Exceptions; + using DurableTask.Core.Serializing; + using Newtonsoft.Json; + + /// + /// Base class for TaskEntity. + /// + /// For in-process user code, we recommend using the specialized + /// class which has a per-operation interface, + /// provides type-safe access to the state, and handles state management, including initialization, + /// serialization, and deserialization. + public abstract class TaskEntity + { + /// + /// Executes the given operation batch and return the results. + /// + /// The batch of operations to execute. + /// Options to control entity execution. + /// + public abstract Task ExecuteOperationBatchAsync(OperationBatchRequest operations, EntityExecutionOptions options); + } + + /// + /// TaskEntity representing an entity with typed state and options for controlling serialization and batch execution. + /// + /// The type of the entity state. + public abstract class TaskEntity : TaskEntity + { + /// + /// Defines how this entity processes operations. Implementations must override this. + /// + /// The context for the operation. + /// a task that indicates completion of the operation + public abstract ValueTask ExecuteOperationAsync(EntityContext context); + + /// + /// A function for creating the initial state of the entity. This is + /// automatically called when the entity is first accessed, or + /// when it is accessed after having been deleted. + /// Implementations may override this if they want to perform a different initialization. + /// + public virtual TState CreateInitialState(EntityContext context) + { + return default(TState); + } + + /// + /// The data converter used for converting inputs and outputs for operations. + /// Implementations may override this setting. + /// + public virtual DataConverter MessageDataConverter => JsonDataConverter.Default; + + /// + /// The data converter used for the entity state. + /// Implementations may override this setting. + /// + public virtual DataConverter StateDataConverter => this.MessageDataConverter; + + /// + /// The data converter used for exceptions. + /// Implementations may override this setting. + /// + public virtual DataConverter ErrorDataConverter => this.MessageDataConverter; + + /// + /// If true, all effects of an entity operation (all state changes and all actions) are rolled back + /// if the entity operation completes with an exception. + /// Implementations may override this setting. + /// + public virtual bool RollbackOnExceptions => true; + + /// + /// Options for executing entities. + /// + public EntityExecutionOptions EntityExecutionOptions { get; set; } + + /// + public override async Task ExecuteOperationBatchAsync(OperationBatchRequest operations, EntityExecutionOptions options) + { + this.EntityExecutionOptions = options; + + var result = new OperationBatchResult() + { + Results = new List(), + Actions = new List(), + EntityState = operations.EntityState, + }; + + var entityContext = new TaskEntityContext( + this, + EntityId.GetEntityIdFromInstanceId(operations.InstanceId), + options, + operations, + result); + + await entityContext.ExecuteBatchAsync(); + + return result; + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Exceptions/EntityLockingRulesViolationException.cs b/src/DurableTask.Core/Exceptions/EntityLockingRulesViolationException.cs new file mode 100644 index 000000000..35da8a437 --- /dev/null +++ b/src/DurableTask.Core/Exceptions/EntityLockingRulesViolationException.cs @@ -0,0 +1,62 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Exceptions +{ + using System; + using System.Runtime.Serialization; + + /// + /// Indicates an invalid use of entities in critical sections. + /// + [Serializable] + public class EntityLockingRulesViolationException : InvalidOperationException + { + /// + /// Initializes a new instance. + /// + public EntityLockingRulesViolationException() + { + } + + /// + /// Initializes an new instance with a specified error message. + /// + /// The message that describes the error. + public EntityLockingRulesViolationException(string message) + : base(message) + { + } + + /// + /// Initializes an new instance with a specified error message. + /// and a reference to the inner exception that is the cause of this exception. + /// + /// The message that describes the error. + /// The exception that is the cause of the current exception, or a null reference if no inner exception is specified. + public EntityLockingRulesViolationException(string message, Exception innerException) + : base(message, innerException) + { + } + + /// + /// Initializes a new instance with serialized data. + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized object data about the exception being thrown. + /// The System.Runtime.Serialization.StreamingContext that contains contextual information about the source or destination. + protected EntityLockingRulesViolationException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Exceptions/OperationFailedException.cs b/src/DurableTask.Core/Exceptions/OperationFailedException.cs new file mode 100644 index 000000000..b529c020d --- /dev/null +++ b/src/DurableTask.Core/Exceptions/OperationFailedException.cs @@ -0,0 +1,108 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Exceptions +{ + using System; + using System.Runtime.Serialization; + + /// + /// Represents errors experienced while executing an operation on an entity. + /// + [Serializable] + public class OperationFailedException : OrchestrationException + { + /// + /// Initializes an new instance of the OperationFailedException class + /// + public OperationFailedException() + { + } + + /// + /// Initializes an new instance of the OperationFailedException class with a specified error message + /// + /// The message that describes the error. + public OperationFailedException(string reason) + : base(reason) + { + } + + /// + /// Initializes an new instance of the OperationFailedException class with a specified error message + /// and a reference to the inner exception that is the cause of this exception. + /// + /// The message that describes the error. + /// The exception that is the cause of the current exception, or a null reference if no inner exception is specified. + public OperationFailedException(string reason, Exception innerException) + : base(reason, innerException) + { + } + + /// + /// Initializes an new instance of the OperationFailedException class with a specified event id, schedule id, name, version and error message + /// and a reference to the inner exception that is the cause of this exception. + /// + /// EventId of the error. + /// ScheduleId of the error. + /// The instance id of the entity. + /// The operation id of the operation. + /// The message that describes the error. + /// The exception that is the cause of the current exception, or a null reference if no cause is specified. + public OperationFailedException(int eventId, int scheduleId, string instanceId, string operationId, string reason, + Exception cause) + : base(eventId, reason, cause) + { + ScheduleId = scheduleId; + InstanceId = instanceId; + OperationId = operationId; + } + + /// + /// Initializes a new instance of the OperationFailedException class with serialized data. + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized object data about the exception being thrown. + /// The System.Runtime.Serialization.StreamingContext that contains contextual information about the source or destination. + protected OperationFailedException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + ScheduleId = info.GetInt32(nameof(ScheduleId)); + InstanceId = info.GetString(nameof(InstanceId)); + OperationId = info.GetString(nameof(OperationId)); + } + + /// + public override void GetObjectData(SerializationInfo info, StreamingContext context) + { + base.GetObjectData(info, context); + info.AddValue(nameof(ScheduleId), ScheduleId); + info.AddValue(nameof(InstanceId), InstanceId); + info.AddValue(nameof(OperationId), OperationId); + } + + /// + /// Schedule Id of the exception + /// + public int ScheduleId { get; set; } + + /// + /// The instance id of the entity that experienced the failure when executing the operation. + /// + public string InstanceId { get; set; } + + /// + /// The operation id of the operation that experienced the failure. + /// + public string OperationId { get; set; } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/FailureDetails.cs b/src/DurableTask.Core/FailureDetails.cs index b99584833..6e929afec 100644 --- a/src/DurableTask.Core/FailureDetails.cs +++ b/src/DurableTask.Core/FailureDetails.cs @@ -19,7 +19,7 @@ namespace DurableTask.Core using Newtonsoft.Json; /// - /// Details of an activity or orchestration failure. + /// Details of an activity, orchestration, or entity operation failure. /// [Serializable] public class FailureDetails : IEquatable @@ -51,6 +51,17 @@ public FailureDetails(Exception e) { } + /// + /// Initializes a new instance of the class from an exception object and + /// an explicitly specified inner exception. + /// + /// The exception used to generate the failure details. + /// The inner exception to use. + public FailureDetails(Exception e, Exception InnerException) + : this(e.GetType().FullName, GetErrorMessage(e), e.StackTrace, FromException(InnerException), false) + { + } + /// /// For testing purposes only: Initializes a new, empty instance of the class. /// diff --git a/src/DurableTask.Core/Logging/EventIds.cs b/src/DurableTask.Core/Logging/EventIds.cs index 963de5f71..2998e5f8d 100644 --- a/src/DurableTask.Core/Logging/EventIds.cs +++ b/src/DurableTask.Core/Logging/EventIds.cs @@ -47,6 +47,10 @@ static class EventIds public const int OrchestrationExecuted = 52; public const int OrchestrationAborted = 53; public const int DiscardingMessage = 54; + public const int EntityBatchExecuting = 55; + public const int EntityBatchExecuted = 56; + public const int EntityLockAcquired = 57; + public const int EntityLockReleased = 58; public const int TaskActivityStarting = 60; public const int TaskActivityCompleted = 61; diff --git a/src/DurableTask.Core/Logging/LogEvents.cs b/src/DurableTask.Core/Logging/LogEvents.cs index a914154a2..258172173 100644 --- a/src/DurableTask.Core/Logging/LogEvents.cs +++ b/src/DurableTask.Core/Logging/LogEvents.cs @@ -14,9 +14,12 @@ namespace DurableTask.Core.Logging { using System; + using System.Linq; using System.Text; using DurableTask.Core.Command; using DurableTask.Core.Common; + using DurableTask.Core.Entities; + using DurableTask.Core.Entities.OperationFormat; using DurableTask.Core.History; using Microsoft.Extensions.Logging; @@ -1177,6 +1180,192 @@ void IEventSourceEvent.WriteEventSource() => Utils.PackageVersion); } + /// + /// Log event representing a task hub worker executing a batch of entity operations. + /// + internal class EntityBatchExecuting : StructuredLogEvent, IEventSourceEvent + { + public EntityBatchExecuting(OperationBatchRequest request) + { + this.InstanceId = request.InstanceId; + this.OperationCount = request.Operations.Count; + this.EntityStateLength = request.EntityState?.Length ?? 0; + } + + [StructuredLogField] + public string InstanceId { get; } + + [StructuredLogField] + public int OperationCount { get; } + + [StructuredLogField] + public int EntityStateLength { get; } + + public override EventId EventId => new EventId( + EventIds.EntityBatchExecuting, + nameof(EventIds.EntityBatchExecuting)); + + public override LogLevel Level => LogLevel.Debug; + + protected override string CreateLogMessage() => + $"{this.InstanceId}: executing batch of {this.OperationCount} operations on entity state of length {this.EntityStateLength}."; + + void IEventSourceEvent.WriteEventSource() => + StructuredEventSource.Log.EntityBatchExecuting( + this.InstanceId, + this.OperationCount, + this.EntityStateLength, + Utils.AppName, + Utils.PackageVersion); + } + + /// + /// Log event representing a task hub worker executed a batch of entity operations. + /// + internal class EntityBatchExecuted : StructuredLogEvent, IEventSourceEvent + { + public EntityBatchExecuted(OperationBatchRequest request, OperationBatchResult result) + { + this.InstanceId = request.InstanceId; + this.OperationCount = request.Operations.Count; + this.ResultCount = result.Results.Count; + this.ErrorCount = result.Results.Count(x => x.ErrorMessage != null); + this.ActionCount = result.Actions.Count; + this.EntityStateLength = request.EntityState?.Length ?? 0; + } + + [StructuredLogField] + public string InstanceId { get; } + + [StructuredLogField] + public int OperationCount { get; } + + [StructuredLogField] + public int ResultCount { get; } + + [StructuredLogField] + public int ErrorCount { get; } + + [StructuredLogField] + public int ActionCount { get; } + + [StructuredLogField] + public int EntityStateLength { get; } + + public override EventId EventId => new EventId( + EventIds.EntityBatchExecuting, + nameof(EventIds.EntityBatchExecuting)); + + public override LogLevel Level => LogLevel.Information; + + protected override string CreateLogMessage() => + $"{this.InstanceId}: completed {this.ResultCount} of {this.OperationCount} entity operations, resulting in {this.ErrorCount} errors, {this.ActionCount} actions, and entity state of length {this.EntityStateLength}."; + + void IEventSourceEvent.WriteEventSource() => + StructuredEventSource.Log.EntityBatchExecuted( + this.InstanceId, + this.OperationCount, + this.ResultCount, + this.ErrorCount, + this.ActionCount, + this.EntityStateLength, + Utils.AppName, + Utils.PackageVersion); + } + + /// + /// Logs that an entity processed a lock acquire message. + /// + internal class EntityLockAcquired : StructuredLogEvent, IEventSourceEvent + { + public EntityLockAcquired(string entityId, Core.Entities.EventFormat.RequestMessage message) + { + this.EntityId = entityId; + this.InstanceId = message.ParentInstanceId; + this.ExecutionId = message.ParentExecutionId; + this.CriticalSectionId = message.Id; + this.LockSet = message.LockSet; + this.Position = message.Position; + } + + [StructuredLogField] + public string EntityId { get; } + + [StructuredLogField] + public string InstanceId { get; set; } + + [StructuredLogField] + public string ExecutionId { get; set; } + + [StructuredLogField] + public Guid CriticalSectionId { get; set; } + + [StructuredLogField] + public EntityId[] LockSet { get; set; } + + [StructuredLogField] + public int Position { get; set; } + + public override EventId EventId => new EventId( + EventIds.EntityLockAcquired, + nameof(EventIds.EntityLockAcquired)); + + public override LogLevel Level => LogLevel.Information; + + protected override string CreateLogMessage() => + $"{this.EntityId}: acquired lock {this.Position+1}/{this.LockSet.Length} for orchestration instanceId={this.InstanceId} executionId={this.ExecutionId} criticalSectionId={this.CriticalSectionId}"; + + void IEventSourceEvent.WriteEventSource() => + StructuredEventSource.Log.EntityLockAcquired( + this.EntityId, + this.InstanceId ?? string.Empty, + this.ExecutionId ?? string.Empty, + this.CriticalSectionId, + this.LockSet.Length, + this.Position, + Utils.AppName, + Utils.PackageVersion); + } + + /// + /// Logs that an entity processed a lock release message. + /// + internal class EntityLockReleased : StructuredLogEvent, IEventSourceEvent + { + public EntityLockReleased(string entityId, Core.Entities.EventFormat.ReleaseMessage message) + { + this.EntityId = entityId; + this.InstanceId = message.ParentInstanceId; + this.Id = message.Id; + } + + [StructuredLogField] + public string EntityId { get; } + + [StructuredLogField] + public string InstanceId { get; set; } + + [StructuredLogField] + public string Id { get; set; } + + public override EventId EventId => new EventId( + EventIds.EntityLockReleased, + nameof(EventIds.EntityLockReleased)); + + public override LogLevel Level => LogLevel.Information; + + protected override string CreateLogMessage() => + $"{this.EntityId}: released lock for orchestration instanceId={this.InstanceId} id={this.Id}"; + + void IEventSourceEvent.WriteEventSource() => + StructuredEventSource.Log.EntityLockReleased( + this.EntityId, + this.InstanceId ?? string.Empty, + this.Id ?? string.Empty, + Utils.AppName, + Utils.PackageVersion); + } + /// /// Log event indicating that an activity execution is starting. /// diff --git a/src/DurableTask.Core/Logging/LogHelper.cs b/src/DurableTask.Core/Logging/LogHelper.cs index f4d1ffe34..b6ff3534f 100644 --- a/src/DurableTask.Core/Logging/LogHelper.cs +++ b/src/DurableTask.Core/Logging/LogHelper.cs @@ -16,8 +16,11 @@ namespace DurableTask.Core.Logging using System; using System.Collections.Generic; using System.Text; + using System.Web; using DurableTask.Core.Command; using DurableTask.Core.Common; + using DurableTask.Core.Entities; + using DurableTask.Core.Entities.OperationFormat; using DurableTask.Core.History; using Microsoft.Extensions.Logging; @@ -561,6 +564,58 @@ internal void RenewOrchestrationWorkItemFailed(TaskOrchestrationWorkItem workIte } } + + /// + /// Logs that an entity operation batch is about to start executing. + /// + /// The batch request. + internal void EntityBatchExecuting(OperationBatchRequest request) + { + if (this.IsStructuredLoggingEnabled) + { + this.WriteStructuredLog(new LogEvents.EntityBatchExecuting(request)); + } + } + + /// + /// Logs that an entity operation batch completed its execution. + /// + /// The batch request. + /// The batch result. + internal void EntityBatchExecuted(OperationBatchRequest request, OperationBatchResult result) + { + if (this.IsStructuredLoggingEnabled) + { + this.WriteStructuredLog(new LogEvents.EntityBatchExecuted(request, result)); + } + } + + /// + /// Logs that an entity processed a lock acquire message. + /// + /// The entity id. + /// The message. + internal void EntityLockAcquired(string entityId, Core.Entities.EventFormat.RequestMessage message) + { + if (this.IsStructuredLoggingEnabled) + { + this.WriteStructuredLog(new LogEvents.EntityLockAcquired(entityId, message)); + } + } + + /// + /// Logs that an entity processed a lock release message. + /// + /// The entity id. + /// The message. + internal void EntityLockReleased(string entityId, Core.Entities.EventFormat.ReleaseMessage message) + { + if (this.IsStructuredLoggingEnabled) + { + this.WriteStructuredLog(new LogEvents.EntityLockReleased(entityId, message)); + } + } + #endregion #region Activity dispatcher diff --git a/src/DurableTask.Core/Logging/StructuredEventSource.cs b/src/DurableTask.Core/Logging/StructuredEventSource.cs index b9edc0a46..a738425df 100644 --- a/src/DurableTask.Core/Logging/StructuredEventSource.cs +++ b/src/DurableTask.Core/Logging/StructuredEventSource.cs @@ -624,6 +624,102 @@ internal void DiscardingMessage( } } + [Event(EventIds.EntityBatchExecuting, Level = EventLevel.Informational, Version = 1)] + internal void EntityBatchExecuting( + string InstanceId, + int OperationCount, + int EntityStateLength, + string AppName, + string ExtensionVersion) + { + if (this.IsEnabled(EventLevel.Informational)) + { + // TODO: Use WriteEventCore for better performance + this.WriteEvent( + EventIds.EntityBatchExecuting, + InstanceId, + OperationCount, + EntityStateLength, + AppName, + ExtensionVersion); + } + } + + [Event(EventIds.EntityBatchExecuted, Level = EventLevel.Informational, Version = 1)] + internal void EntityBatchExecuted( + string InstanceId, + int OperationCount, + int ResultCount, + int ErrorCount, + int ActionCount, + int EntityStateLength, + string AppName, + string ExtensionVersion) + { + if (this.IsEnabled(EventLevel.Informational)) + { + // TODO: Use WriteEventCore for better performance + this.WriteEvent( + EventIds.EntityBatchExecuted, + InstanceId, + OperationCount, + ResultCount, + ErrorCount, + ActionCount, + EntityStateLength, + AppName, + ExtensionVersion); + } + } + + [Event(EventIds.EntityLockAcquired, Level = EventLevel.Informational, Version = 1)] + internal void EntityLockAcquired( + string EntityId, + string InstanceId, + string ExecutionId, + Guid CriticalSectionId, + int LockSetSize, + int Position, + string AppName, + string ExtensionVersion) + { + if (this.IsEnabled(EventLevel.Informational)) + { + // TODO: Use WriteEventCore for better performance + this.WriteEvent( + EventIds.EntityBatchExecuted, + EntityId, + InstanceId, + ExecutionId, + CriticalSectionId, + LockSetSize, + Position, + AppName, + ExtensionVersion); + } + } + + [Event(EventIds.EntityLockReleased, Level = EventLevel.Informational, Version = 1)] + internal void EntityLockReleased( + string EntityId, + string InstanceId, + string Id, + string AppName, + string ExtensionVersion) + { + if (this.IsEnabled(EventLevel.Informational)) + { + // TODO: Use WriteEventCore for better performance + this.WriteEvent( + EventIds.EntityBatchExecuted, + EntityId, + InstanceId, + Id, + AppName, + ExtensionVersion); + } + } + [Event(EventIds.TaskActivityStarting, Level = EventLevel.Informational, Version = 1)] internal void TaskActivityStarting( string InstanceId, diff --git a/src/DurableTask.Core/OrchestrationContext.cs b/src/DurableTask.Core/OrchestrationContext.cs index 52238bbc2..bb057b4a5 100644 --- a/src/DurableTask.Core/OrchestrationContext.cs +++ b/src/DurableTask.Core/OrchestrationContext.cs @@ -18,6 +18,9 @@ namespace DurableTask.Core using System.Threading; using System.Threading.Tasks; using Castle.DynamicProxy; + using DurableTask.Core.Entities; + using DurableTask.Core.Entities.OperationFormat; + using DurableTask.Core.Exceptions; using DurableTask.Core.Serializing; /// @@ -67,6 +70,11 @@ public abstract class OrchestrationContext /// internal ErrorPropagationMode ErrorPropagationMode { get; set; } + /// + /// Information about backend entity support, or null if the configured backend does not support entities. + /// + internal EntityBackendInformation EntityBackendInformation { get; set; } + /// /// Create a proxy client class to schedule remote TaskActivities via a strongly typed interface. /// @@ -368,6 +376,70 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve public abstract Task CreateSubOrchestrationInstance(string name, string version, string instanceId, object input, IDictionary tags); + /// + /// Calls an operation on an entity and returns the result asynchronously. + /// + /// The result type of the operation. + /// The target entity. + /// The name of the operation. + /// The input for the operation. + /// if the orchestration is inside a critical section and the lock for this entity is not available. + /// A task representing the result of the operation. + public abstract Task CallEntityAsync(Entities.EntityId entityId, string operationName, object operationInput = null); + + /// + /// Calls an operation on an entity and waits for it to complete. + /// + /// The target entity. + /// The name of the operation. + /// The input for the operation. + /// A task representing the completion of the operation on the entity. + /// if the orchestration is inside a critical section and the lock for this entity is not available. + public abstract Task CallEntityAsync(Entities.EntityId entityId, string operationName, object operationInput = null); + + /// + /// Signals an entity operation on the specified entity. + /// + /// The entity ID of the target. + /// The name of the operation. + /// The input for the operation. + /// if the orchestration is inside a critical section that locked this entity. + public abstract void SignalEntity(Entities.EntityId entityId, string operationName, object operationInput = null); + + /// + /// Signals an entity to perform an operation, at a specified time. Any result or exception is ignored (fire and forget). + /// + /// The entity ID of the target. + /// The time at which to start the operation. + /// The name of the operation. + /// if the orchestration is inside a critical section that locked this entity. + /// The input for the operation. + public abstract void SignalEntity(Entities.EntityId entityId, DateTime scheduledTimeUtc, string operationName, object operationInput = null); + + /// + /// Locks one or more entities for the duration of the critical section. + /// + /// + /// Locks can only be acquired if the current context is not already in a critical section. + /// + /// The entities whose locks should be acquired. + /// A task that must be awaited prior to entering the critical section, and which returns a IDisposable that must be disposed after exiting the critical section. + /// if the context is already in a critical section. + public abstract Task LockEntitiesAsync(params EntityId[] entities); + + /// + /// Whether this orchestration is currently inside a critical section. Critical sections are entered when calling + /// , and are exited when disposing the returned IDisposable. + /// + public abstract bool IsInsideCriticalSection { get; } + + /// + /// Enumerates all the entities that can be called from within the current critical section. + /// This set contains all the entities that were locked prior to entering the critical section, + /// and for which there is not currently an operation call pending. + /// + /// An enumeration of all the currently available entities. + public abstract IEnumerable GetAvailableEntities(); /// /// Raises an event for the specified orchestration instance, which eventually causes the OnEvent() method in the diff --git a/src/DurableTask.Core/OrchestrationWorkItemProcessor.cs b/src/DurableTask.Core/OrchestrationWorkItemProcessor.cs new file mode 100644 index 000000000..9e133b229 --- /dev/null +++ b/src/DurableTask.Core/OrchestrationWorkItemProcessor.cs @@ -0,0 +1,604 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +using DurableTask.Core.Command; +using DurableTask.Core.Common; +using DurableTask.Core.Exceptions; +using DurableTask.Core.History; +using DurableTask.Core.Logging; +using DurableTask.Core.Middleware; +using DurableTask.Core.Serializing; +using DurableTask.Core.Tracing; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace DurableTask.Core +{ + /// + /// Handles orchestration work items. + /// + class OrchestrationWorkItemProcessor : TaskOrchestrationDispatcher.WorkItemProcessor + { + static readonly Task CompletedTask = Task.FromResult(0); + + readonly IOrchestrationService orchestrationService; + readonly LogHelper logHelper; + ErrorPropagationMode errorPropagationMode; + readonly INameVersionObjectManager objectManager; + readonly DispatchMiddlewarePipeline dispatchPipeline; + + public OrchestrationWorkItemProcessor( + TaskOrchestrationDispatcher dispatcher, + TaskOrchestrationWorkItem workItem, + IOrchestrationService orchestrationService, + ErrorPropagationMode errorPropagationMode, + LogHelper logHelper, + INameVersionObjectManager orchestrationObjectManager, + DispatchMiddlewarePipeline dispatchPipeline) : base(dispatcher, workItem) + { + this.orchestrationService = orchestrationService; + this.errorPropagationMode = errorPropagationMode; + this.logHelper = logHelper; + this.objectManager = orchestrationObjectManager; + this.dispatchPipeline = dispatchPipeline; + } + + public override async Task ProcessWorkItemAsync() + { + ExecutionStartedEvent? continueAsNewExecutionStarted = null; + IList? carryOverEvents = null; + string? carryOverStatus = null; + + do + { + this.continuedAsNew = false; + this.continuedAsNewMessage = null; + + this.logHelper.OrchestrationExecuting(this.runtimeState.OrchestrationInstance!, this.runtimeState.Name); + TraceHelper.TraceInstance( + TraceEventType.Verbose, + "TaskOrchestrationDispatcher-ExecuteUserOrchestration-Begin", + this.runtimeState.OrchestrationInstance, + "Executing user orchestration: {0}", + JsonDataConverter.Default.Serialize(this.runtimeState.GetOrchestrationRuntimeStateDump(), true)); + + if (this.workItem.Cursor == null) + { + this.workItem.Cursor = await this.ExecuteOrchestrationAsync(this.runtimeState, this.workItem); + } + else + { + await this.ResumeOrchestrationAsync(this.workItem); + } + + IReadOnlyList decisions = this.workItem.Cursor.LatestDecisions.ToList(); + + this.logHelper.OrchestrationExecuted( + this.runtimeState.OrchestrationInstance!, + this.runtimeState.Name, + decisions); + TraceHelper.TraceInstance( + TraceEventType.Information, + "TaskOrchestrationDispatcher-ExecuteUserOrchestration-End", + this.runtimeState.OrchestrationInstance, + "Executed user orchestration. Received {0} orchestrator actions: {1}", + decisions.Count, + string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType))); + + // TODO: Exception handling for invalid decisions, which is increasingly likely + // when custom middleware is involved (e.g. out-of-process scenarios). + foreach (OrchestratorAction decision in decisions) + { + TraceHelper.TraceInstance( + TraceEventType.Information, + "TaskOrchestrationDispatcher-ProcessOrchestratorAction", + this.runtimeState.OrchestrationInstance, + "Processing orchestrator action of type {0}", + decision.OrchestratorActionType); + switch (decision.OrchestratorActionType) + { + case OrchestratorActionType.ScheduleOrchestrator: + var scheduleTaskAction = (ScheduleTaskOrchestratorAction)decision; + var message = this.ProcessScheduleTaskDecision( + scheduleTaskAction, + this.runtimeState, + this.dispatcher.IncludeParameters); + this.messagesToSend.Add(message); + break; + case OrchestratorActionType.CreateTimer: + var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision; + this.timerMessages.Add(this.ProcessCreateTimerDecision( + timerOrchestratorAction, + this.runtimeState, + isInternal: false)); + break; + case OrchestratorActionType.CreateSubOrchestration: + var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; + this.orchestratorMessages.Add( + this.ProcessCreateSubOrchestrationInstanceDecision( + createSubOrchestrationAction, + this.runtimeState, + this.dispatcher.IncludeParameters)); + break; + case OrchestratorActionType.SendEvent: + var sendEventAction = (SendEventOrchestratorAction)decision; + this.orchestratorMessages.Add( + this.ProcessSendEventDecision(sendEventAction, this.runtimeState)); + break; + case OrchestratorActionType.OrchestrationComplete: + OrchestrationCompleteOrchestratorAction completeDecision = (OrchestrationCompleteOrchestratorAction)decision; + TaskMessage? workflowInstanceCompletedMessage = + this.ProcessWorkflowCompletedTaskDecision(completeDecision, this.runtimeState, this.dispatcher.IncludeDetails, out this.continuedAsNew); + if (workflowInstanceCompletedMessage != null) + { + // Send complete message to parent workflow or to itself to start a new execution + // Store the event so we can rebuild the state + carryOverEvents = null; + if (this.continuedAsNew) + { + this.continuedAsNewMessage = workflowInstanceCompletedMessage; + continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent; + if (completeDecision.CarryoverEvents.Any()) + { + carryOverEvents = completeDecision.CarryoverEvents.ToList(); + completeDecision.CarryoverEvents.Clear(); + } + } + else + { + this.orchestratorMessages.Add(workflowInstanceCompletedMessage); + } + } + + this.isCompleted = !this.continuedAsNew; + break; + default: + throw TraceHelper.TraceExceptionInstance( + TraceEventType.Error, + "TaskOrchestrationDispatcher-UnsupportedDecisionType", + this.runtimeState.OrchestrationInstance, + new NotSupportedException($"Decision type '{decision.OrchestratorActionType}' not supported")); + } + + // Underlying orchestration service provider may have a limit of messages per call, to avoid the situation + // we keep on asking the provider if message count is ok and stop processing new decisions if not. + // + // We also put in a fake timer to force next orchestration task for remaining messages + int totalMessages = this.messagesToSend.Count + this.orchestratorMessages.Count + this.timerMessages.Count; + if (this.orchestrationService.IsMaxMessageCountExceeded(totalMessages, this.runtimeState)) + { + TraceHelper.TraceInstance( + TraceEventType.Information, + "TaskOrchestrationDispatcher-MaxMessageCountReached", + this.runtimeState.OrchestrationInstance, + "MaxMessageCount reached. Adding timer to process remaining events in next attempt."); + + if (this.isCompleted || this.continuedAsNew) + { + TraceHelper.TraceInstance( + TraceEventType.Information, + "TaskOrchestrationDispatcher-OrchestrationAlreadyCompleted", + this.runtimeState.OrchestrationInstance, + "Orchestration already completed. Skip adding timer for splitting messages."); + break; + } + + var dummyTimer = new CreateTimerOrchestratorAction + { + Id = FrameworkConstants.FakeTimerIdToSplitDecision, + FireAt = DateTime.UtcNow + }; + + this.timerMessages.Add(this.ProcessCreateTimerDecision( + dummyTimer, + this.runtimeState, + isInternal: true)); + this.isInterrupted = true; + break; + } + } + + // correlation + CorrelationTraceClient.Propagate(() => { + if (this.runtimeState.ExecutionStartedEvent != null) + this.runtimeState.ExecutionStartedEvent.Correlation = CorrelationTraceContext.Current.SerializableTraceContext; + }); + + // finish up processing of the work item + if (!this.continuedAsNew && this.runtimeState.Events.Last().EventType != EventType.OrchestratorCompleted) + { + this.runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); + } + + if (this.isCompleted) + { + TraceHelper.TraceSession(TraceEventType.Information, "TaskOrchestrationDispatcher-DeletingSessionState", this.workItem.InstanceId, "Deleting session state"); + if (this.runtimeState.ExecutionStartedEvent != null) + { + this.instanceState = Utils.BuildOrchestrationState(this.runtimeState); + } + } + else + { + if (this.continuedAsNew) + { + TraceHelper.TraceSession( + TraceEventType.Information, + "TaskOrchestrationDispatcher-UpdatingStateForContinuation", + this.workItem.InstanceId, + "Updating state for continuation"); + + // correlation + CorrelationTraceClient.Propagate(() => + { + continueAsNewExecutionStarted!.Correlation = CorrelationTraceContext.Current.SerializableTraceContext; + }); + + this.runtimeState = new OrchestrationRuntimeState(); + this.runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); + this.runtimeState.AddEvent(continueAsNewExecutionStarted!); + this.runtimeState.Status = this.workItem.OrchestrationRuntimeState.Status ?? carryOverStatus; + carryOverStatus = this.workItem.OrchestrationRuntimeState.Status; + + if (carryOverEvents != null) + { + foreach (var historyEvent in carryOverEvents) + { + this.runtimeState.AddEvent(historyEvent); + } + } + + this.runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); + this.workItem.OrchestrationRuntimeState = this.runtimeState; + + this.workItem.Cursor = null; + } + + this.instanceState = Utils.BuildOrchestrationState(this.runtimeState); + } + } while (this.continuedAsNew); + + this.runtimeState.Status = this.runtimeState.Status ?? carryOverStatus; + + if (this.instanceState != null) + { + this.instanceState.Status = this.runtimeState.Status; + } + } + + static OrchestrationExecutionContext GetOrchestrationExecutionContext(OrchestrationRuntimeState runtimeState) + { + return new OrchestrationExecutionContext { OrchestrationTags = runtimeState.Tags ?? new Dictionary(capacity: 0) }; + } + + async Task ExecuteOrchestrationAsync(OrchestrationRuntimeState runtimeState, TaskOrchestrationWorkItem workItem) + { + // Get the TaskOrchestration implementation. If it's not found, it either means that the developer never + // registered it (which is an error, and we'll throw for this further down) or it could be that some custom + // middleware (e.g. out-of-process execution middleware) is intended to implement the orchestration logic. + TaskOrchestration? taskOrchestration = this.objectManager.GetObject(runtimeState.Name, runtimeState.Version!); + + var dispatchContext = new DispatchMiddlewareContext(); + dispatchContext.SetProperty(runtimeState.OrchestrationInstance); + dispatchContext.SetProperty(taskOrchestration); + dispatchContext.SetProperty(runtimeState); + dispatchContext.SetProperty(workItem); + dispatchContext.SetProperty(GetOrchestrationExecutionContext(runtimeState)); + + TaskOrchestrationExecutor? executor = null; + + await this.dispatchPipeline.RunAsync(dispatchContext, _ => + { + // Check to see if the custom middleware intercepted and substituted the orchestration execution + // with its own execution behavior, providing us with the end results. If so, we can terminate + // the dispatch pipeline here. + var resultFromMiddleware = dispatchContext.GetProperty(); + if (resultFromMiddleware != null) + { + return CompletedTask; + } + + if (taskOrchestration == null) + { + throw TraceHelper.TraceExceptionInstance( + TraceEventType.Error, + "TaskOrchestrationDispatcher-TypeMissing", + runtimeState.OrchestrationInstance, + new TypeMissingException($"Orchestration not found: ({runtimeState.Name}, {runtimeState.Version})")); + } + + executor = new TaskOrchestrationExecutor( + runtimeState, + taskOrchestration, + this.orchestrationService.EventBehaviourForContinueAsNew, + this.dispatcher.EntityBackendInformation, + this.errorPropagationMode); + OrchestratorExecutionResult resultFromOrchestrator = executor.Execute(); + dispatchContext.SetProperty(resultFromOrchestrator); + return CompletedTask; + }); + + var result = dispatchContext.GetProperty(); + IEnumerable decisions = result?.Actions ?? Enumerable.Empty(); + runtimeState.Status = result?.CustomStatus; + + return new OrchestrationExecutionCursor(runtimeState, taskOrchestration, executor, decisions); + } + + async Task ResumeOrchestrationAsync(TaskOrchestrationWorkItem workItem) + { + OrchestrationExecutionCursor cursor = workItem.Cursor; + + var dispatchContext = new DispatchMiddlewareContext(); + dispatchContext.SetProperty(cursor.RuntimeState.OrchestrationInstance); + dispatchContext.SetProperty(cursor.TaskOrchestration); + dispatchContext.SetProperty(cursor.RuntimeState); + dispatchContext.SetProperty(workItem); + + cursor.LatestDecisions = Enumerable.Empty(); + await this.dispatchPipeline.RunAsync(dispatchContext, _ => + { + OrchestratorExecutionResult result = cursor.OrchestrationExecutor.ExecuteNewEvents(); + dispatchContext.SetProperty(result); + return CompletedTask; + }); + + var result = dispatchContext.GetProperty(); + cursor.LatestDecisions = result?.Actions ?? Enumerable.Empty(); + cursor.RuntimeState.Status = result?.CustomStatus; + } + + + TaskMessage? ProcessWorkflowCompletedTaskDecision( + OrchestrationCompleteOrchestratorAction completeOrchestratorAction, + OrchestrationRuntimeState runtimeState, + bool includeDetails, + out bool continuedAsNew) + { + ExecutionCompletedEvent executionCompletedEvent; + continuedAsNew = (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew); + if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew) + { + executionCompletedEvent = new ContinueAsNewEvent(completeOrchestratorAction.Id, + completeOrchestratorAction.Result); + } + else + { + executionCompletedEvent = new ExecutionCompletedEvent(completeOrchestratorAction.Id, + completeOrchestratorAction.Result, + completeOrchestratorAction.OrchestrationStatus, + completeOrchestratorAction.FailureDetails); + } + + runtimeState.AddEvent(executionCompletedEvent); + + this.logHelper.OrchestrationCompleted(runtimeState, completeOrchestratorAction); + TraceHelper.TraceInstance( + runtimeState.OrchestrationStatus == OrchestrationStatus.Failed ? TraceEventType.Warning : TraceEventType.Information, + "TaskOrchestrationDispatcher-InstanceCompleted", + runtimeState.OrchestrationInstance, + "Instance Id '{0}' completed in state {1} with result: {2}", + runtimeState.OrchestrationInstance, + runtimeState.OrchestrationStatus, + completeOrchestratorAction.Result); + TraceHelper.TraceInstance( + TraceEventType.Information, + "TaskOrchestrationDispatcher-InstanceCompletionEvents", + runtimeState.OrchestrationInstance, + () => Utils.EscapeJson(JsonDataConverter.Default.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true))); + + // Check to see if we need to start a new execution + if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew) + { + var taskMessage = new TaskMessage(); + var startedEvent = new ExecutionStartedEvent(-1, completeOrchestratorAction.Result) + { + OrchestrationInstance = new OrchestrationInstance + { + InstanceId = runtimeState.OrchestrationInstance!.InstanceId, + ExecutionId = Guid.NewGuid().ToString("N") + }, + Tags = runtimeState.Tags, + ParentInstance = runtimeState.ParentInstance, + Name = runtimeState.Name, + Version = completeOrchestratorAction.NewVersion ?? runtimeState.Version + }; + + taskMessage.OrchestrationInstance = startedEvent.OrchestrationInstance; + taskMessage.Event = startedEvent; + + return taskMessage; + } + + // If this is a Sub Orchestration, and not tagged as fire-and-forget, + // then notify the parent by sending a complete message + if (runtimeState.ParentInstance != null + && !OrchestrationTags.IsTaggedAsFireAndForget(runtimeState.Tags)) + { + var taskMessage = new TaskMessage(); + if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Completed) + { + var subOrchestrationCompletedEvent = + new SubOrchestrationInstanceCompletedEvent(-1, runtimeState.ParentInstance.TaskScheduleId, + completeOrchestratorAction.Result); + + taskMessage.Event = subOrchestrationCompletedEvent; + } + else if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Failed || + completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Terminated) + { + var subOrchestrationFailedEvent = + new SubOrchestrationInstanceFailedEvent(-1, runtimeState.ParentInstance.TaskScheduleId, + completeOrchestratorAction.Result, + includeDetails ? completeOrchestratorAction.Details : null); + subOrchestrationFailedEvent.FailureDetails = completeOrchestratorAction.FailureDetails; + + taskMessage.Event = subOrchestrationFailedEvent; + } + + if (taskMessage.Event != null) + { + taskMessage.OrchestrationInstance = runtimeState.ParentInstance.OrchestrationInstance; + return taskMessage; + } + } + + return null; + } + + TaskMessage ProcessScheduleTaskDecision( + ScheduleTaskOrchestratorAction scheduleTaskOrchestratorAction, + OrchestrationRuntimeState runtimeState, + bool includeParameters) + { + if (scheduleTaskOrchestratorAction.Name == null) + { + throw new ArgumentException("No name was given for the task activity to schedule!", nameof(scheduleTaskOrchestratorAction)); + } + + var taskMessage = new TaskMessage(); + + var scheduledEvent = new TaskScheduledEvent( + eventId: scheduleTaskOrchestratorAction.Id, + name: scheduleTaskOrchestratorAction.Name, + version: scheduleTaskOrchestratorAction.Version, + input: scheduleTaskOrchestratorAction.Input); + + taskMessage.Event = scheduledEvent; + taskMessage.OrchestrationInstance = runtimeState.OrchestrationInstance; + taskMessage.OrchestrationExecutionContext = GetOrchestrationExecutionContext(runtimeState); + + if (!includeParameters) + { + scheduledEvent = new TaskScheduledEvent( + eventId: scheduleTaskOrchestratorAction.Id, + name: scheduleTaskOrchestratorAction.Name, + version: scheduleTaskOrchestratorAction.Version); + } + + this.logHelper.SchedulingActivity( + runtimeState.OrchestrationInstance!, + scheduledEvent); + + runtimeState.AddEvent(scheduledEvent); + return taskMessage; + } + + TaskMessage ProcessCreateTimerDecision( + CreateTimerOrchestratorAction createTimerOrchestratorAction, + OrchestrationRuntimeState runtimeState, + bool isInternal) + { + var taskMessage = new TaskMessage(); + + var timerCreatedEvent = new TimerCreatedEvent(createTimerOrchestratorAction.Id) + { + FireAt = createTimerOrchestratorAction.FireAt + }; + + runtimeState.AddEvent(timerCreatedEvent); + + taskMessage.Event = new TimerFiredEvent(-1) + { + TimerId = createTimerOrchestratorAction.Id, + FireAt = createTimerOrchestratorAction.FireAt + }; + + this.logHelper.CreatingTimer( + runtimeState.OrchestrationInstance!, + timerCreatedEvent, + isInternal); + + taskMessage.OrchestrationInstance = runtimeState.OrchestrationInstance; + + return taskMessage; + } + + TaskMessage ProcessCreateSubOrchestrationInstanceDecision( + CreateSubOrchestrationAction createSubOrchestrationAction, + OrchestrationRuntimeState runtimeState, + bool includeParameters) + { + var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id) + { + Name = createSubOrchestrationAction.Name, + Version = createSubOrchestrationAction.Version, + InstanceId = createSubOrchestrationAction.InstanceId + }; + if (includeParameters) + { + historyEvent.Input = createSubOrchestrationAction.Input; + } + + runtimeState.AddEvent(historyEvent); + + var taskMessage = new TaskMessage(); + + var startedEvent = new ExecutionStartedEvent(-1, createSubOrchestrationAction.Input) + { + Tags = OrchestrationTags.MergeTags(createSubOrchestrationAction.Tags, runtimeState.Tags), + OrchestrationInstance = new OrchestrationInstance + { + InstanceId = createSubOrchestrationAction.InstanceId, + ExecutionId = Guid.NewGuid().ToString("N") + }, + ParentInstance = new ParentInstance + { + OrchestrationInstance = runtimeState.OrchestrationInstance, + Name = runtimeState.Name, + Version = runtimeState.Version, + TaskScheduleId = createSubOrchestrationAction.Id + }, + Name = createSubOrchestrationAction.Name, + Version = createSubOrchestrationAction.Version + }; + + this.logHelper.SchedulingOrchestration(startedEvent); + + taskMessage.OrchestrationInstance = startedEvent.OrchestrationInstance; + taskMessage.Event = startedEvent; + taskMessage.OrchestrationExecutionContext = GetOrchestrationExecutionContext(runtimeState); + + return taskMessage; + } + + TaskMessage ProcessSendEventDecision( + SendEventOrchestratorAction sendEventAction, + OrchestrationRuntimeState runtimeState) + { + var historyEvent = new EventSentEvent(sendEventAction.Id) + { + InstanceId = sendEventAction.Instance?.InstanceId, + Name = sendEventAction.EventName, + Input = sendEventAction.EventData + }; + + runtimeState.AddEvent(historyEvent); + + this.logHelper.RaisingEvent(runtimeState.OrchestrationInstance!, historyEvent); + + return new TaskMessage + { + OrchestrationInstance = sendEventAction.Instance, + Event = new EventRaisedEvent(-1, sendEventAction.EventData) + { + Name = sendEventAction.EventName + } + }; + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/TaskHubClient.cs b/src/DurableTask.Core/TaskHubClient.cs index fdd27592e..c48736b63 100644 --- a/src/DurableTask.Core/TaskHubClient.cs +++ b/src/DurableTask.Core/TaskHubClient.cs @@ -19,6 +19,7 @@ namespace DurableTask.Core using System.Linq; using System.Threading; using System.Threading.Tasks; + using DurableTask.Core.Entities; using DurableTask.Core.History; using DurableTask.Core.Logging; using DurableTask.Core.Serializing; @@ -32,6 +33,9 @@ public sealed class TaskHubClient readonly DataConverter defaultConverter; readonly LogHelper logHelper; + internal LogHelper LogHelper => this.logHelper; + internal DataConverter DefaultConverter => this.defaultConverter; + /// /// The orchestration service client for this task hub client /// diff --git a/src/DurableTask.Core/TaskHubWorker.cs b/src/DurableTask.Core/TaskHubWorker.cs index 1b93bd62e..1ea4f393f 100644 --- a/src/DurableTask.Core/TaskHubWorker.cs +++ b/src/DurableTask.Core/TaskHubWorker.cs @@ -21,6 +21,7 @@ namespace DurableTask.Core using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; + using DurableTask.Core.Entities; using DurableTask.Core.Exceptions; using DurableTask.Core.Logging; using DurableTask.Core.Middleware; @@ -34,8 +35,10 @@ public sealed class TaskHubWorker : IDisposable { readonly INameVersionObjectManager activityManager; readonly INameVersionObjectManager orchestrationManager; + readonly INameVersionObjectManager entityManager; readonly DispatchMiddlewarePipeline orchestrationDispatchPipeline = new DispatchMiddlewarePipeline(); + readonly DispatchMiddlewarePipeline entityDispatchPipeline = new DispatchMiddlewarePipeline(); readonly DispatchMiddlewarePipeline activityDispatchPipeline = new DispatchMiddlewarePipeline(); readonly SemaphoreSlim slimLock = new SemaphoreSlim(1, 1); @@ -50,7 +53,7 @@ public sealed class TaskHubWorker : IDisposable volatile bool isStarted; TaskActivityDispatcher activityDispatcher; - TaskOrchestrationDispatcher orchestrationDispatcher; + TaskOrchestrationDispatcher orchestrationDispatcher; // used for both orchestrations and entities /// /// Create a new TaskHubWorker with given OrchestrationService @@ -60,7 +63,8 @@ public TaskHubWorker(IOrchestrationService orchestrationService) : this( orchestrationService, new NameVersionObjectManager(), - new NameVersionObjectManager()) + new NameVersionObjectManager(), + new NameVersionObjectManager()) { } @@ -75,6 +79,7 @@ public TaskHubWorker(IOrchestrationService orchestrationService, ILoggerFactory orchestrationService, new NameVersionObjectManager(), new NameVersionObjectManager(), + new NameVersionObjectManager(), loggerFactory) { } @@ -85,14 +90,17 @@ public TaskHubWorker(IOrchestrationService orchestrationService, ILoggerFactory /// Reference the orchestration service implementation /// NameVersionObjectManager for Orchestrations /// NameVersionObjectManager for Activities + /// The for entities public TaskHubWorker( IOrchestrationService orchestrationService, INameVersionObjectManager orchestrationObjectManager, - INameVersionObjectManager activityObjectManager) + INameVersionObjectManager activityObjectManager, + INameVersionObjectManager entityObjectManager) : this( orchestrationService, orchestrationObjectManager, activityObjectManager, + entityObjectManager, loggerFactory: null) { } @@ -104,21 +112,24 @@ public TaskHubWorker( /// The orchestration service implementation /// The for orchestrations /// The for activities + /// The for entities /// The to use for logging public TaskHubWorker( IOrchestrationService orchestrationService, INameVersionObjectManager orchestrationObjectManager, INameVersionObjectManager activityObjectManager, + INameVersionObjectManager entityObjectManager, ILoggerFactory loggerFactory = null) { this.orchestrationManager = orchestrationObjectManager ?? throw new ArgumentException("orchestrationObjectManager"); this.activityManager = activityObjectManager ?? throw new ArgumentException("activityObjectManager"); + this.entityManager = entityObjectManager ?? throw new ArgumentException("entityObjectManager"); this.orchestrationService = orchestrationService ?? throw new ArgumentException("orchestrationService"); this.logHelper = new LogHelper(loggerFactory?.CreateLogger("DurableTask.Core")); } /// - /// Gets the orchestration dispatcher + /// Gets the orchestration and entity dispatcher /// public TaskOrchestrationDispatcher TaskOrchestrationDispatcher => this.orchestrationDispatcher; @@ -153,6 +164,15 @@ public void AddOrchestrationDispatcherMiddleware(Func + /// Adds a middleware delegate to the entity dispatch pipeline. + /// + /// Delegate to invoke whenever a message is dispatched to an entity. + public void AddEntityDispatcherMiddleware(Func, Task> middleware) + { + this.entityDispatchPipeline.Add(middleware ?? throw new ArgumentNullException(nameof(middleware))); + } + /// /// Adds a middleware delegate to the activity dispatch pipeline. /// @@ -182,7 +202,9 @@ public async Task StartAsync() this.orchestrationDispatcher = new TaskOrchestrationDispatcher( this.orchestrationService, this.orchestrationManager, + this.entityManager, this.orchestrationDispatchPipeline, + this.entityDispatchPipeline, this.logHelper, this.ErrorPropagationMode); this.activityDispatcher = new TaskActivityDispatcher( @@ -282,6 +304,43 @@ public TaskHubWorker AddTaskOrchestrations(params ObjectCreator + /// Loads user defined TaskEntity classes in the TaskHubWorker + /// + /// Types deriving from TaskEntity class + /// + public TaskHubWorker AddTaskEntities(params Type[] taskEntityTypes) + { + foreach (Type type in taskEntityTypes) + { + ObjectCreator creator = new NameValueObjectCreator( + type.Name.ToLowerInvariant(), // entity names are always case-normalized + string.Empty, + type); + + this.entityManager.Add(creator); + } + + return this; + } + + /// + /// Loads user defined TaskEntity classes in the TaskHubWorker + /// + /// + /// User specified ObjectCreators that will + /// create classes deriving TaskEntities with specific names and versions + /// + public TaskHubWorker AddTaskEntities(params ObjectCreator[] taskEntityCreators) + { + foreach (ObjectCreator creator in taskEntityCreators) + { + this.entityManager.Add(creator); + } + + return this; + } + /// /// Loads user defined TaskActivity objects in the TaskHubWorker /// diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index a9831ff47..6b3acfaac 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -21,6 +21,8 @@ namespace DurableTask.Core using System.Threading.Tasks; using DurableTask.Core.Command; using DurableTask.Core.Common; + using DurableTask.Core.Entities; + using DurableTask.Core.Entities.OperationFormat; using DurableTask.Core.Exceptions; using DurableTask.Core.History; using DurableTask.Core.Serializing; @@ -31,6 +33,7 @@ internal class TaskOrchestrationContext : OrchestrationContext private readonly IDictionary openTasks; private readonly IDictionary orchestratorActionsMap; private OrchestrationCompleteOrchestratorAction continueAsNew; + private OrchestrationEntityContext entityContext; private bool executionCompletedOrTerminated; private int idCounter; private readonly Queue eventsWhileSuspended; @@ -44,9 +47,16 @@ public void AddEventToNextIteration(HistoryEvent he) continueAsNew.CarryoverEvents.Add(he); } + internal OrchestrationEntityContext EntityContext + => this.entityContext ?? (this.entityContext = new OrchestrationEntityContext( + this.OrchestrationInstance.InstanceId, + this.OrchestrationInstance.ExecutionId, + this)); + public TaskOrchestrationContext( OrchestrationInstance orchestrationInstance, TaskScheduler taskScheduler, + EntityBackendInformation entityBackendInformation = null, ErrorPropagationMode errorPropagationMode = ErrorPropagationMode.SerializeExceptions) { Utils.UnusedParameter(taskScheduler); @@ -58,6 +68,7 @@ public TaskOrchestrationContext( this.ErrorDataConverter = JsonDataConverter.Default; OrchestrationInstance = orchestrationInstance; IsReplaying = false; + this.EntityBackendInformation = entityBackendInformation; ErrorPropagationMode = errorPropagationMode; this.eventsWhileSuspended = new Queue(); } @@ -66,6 +77,28 @@ public TaskOrchestrationContext( public bool HasOpenTasks => this.openTasks.Count > 0; + public override bool IsInsideCriticalSection => this.entityContext?.IsInsideCriticalSection ?? false; + + private void CheckEntitySupport() + { + if (this.EntityBackendInformation == null) + { + throw new InvalidOperationException("Entities are not supported by the configured persistence backend."); + } + } + + public override IEnumerable GetAvailableEntities() + { + this.CheckEntitySupport(); + if (this.entityContext != null) + { + foreach(var (name, key) in this.entityContext.GetAvailableEntities()) + { + yield return new EntityId(name, key); + } + } + } + internal void ClearPendingActions() { this.orchestratorActionsMap.Clear(); @@ -188,6 +221,93 @@ async Task CreateSubOrchestrationInstanceCore( } } + public override Task CallEntityAsync(EntityId entityId, string operationName, object input = null) + { + this.CheckEntitySupport(); + (Guid operationId, string targetInstanceId, int taskId) = this.EntityOperationCore(entityId, operationName, input, false, null); + return this.entityContext.WaitForOperationResponseAsync(operationId, taskId, targetInstanceId); + } + + public override Task CallEntityAsync(EntityId entityId, string operationName, object input = null) + { + return this.CallEntityAsync(entityId, operationName, input); + } + + public override void SignalEntity(EntityId entityId, string operationName, object input = null) + { + this.CheckEntitySupport(); + this.EntityOperationCore(entityId, operationName, input, true, null); + } + + public override void SignalEntity(EntityId entityId, DateTime scheduledTimeUtc, string operationName, object input = null) + { + this.CheckEntitySupport(); + this.EntityOperationCore(entityId, operationName, input, true, EntityBackendInformation.GetCappedScheduledTime(this.CurrentUtcDateTime, scheduledTimeUtc)); + } + + (Guid operationId, string targetInstanceId, int taskId) EntityOperationCore(EntityId entityId, string operationName, object input, bool oneWay, (DateTime original, DateTime capped)? scheduledTimeUtc = null) + { + OrchestrationInstance target = new OrchestrationInstance() + { + InstanceId = EntityId.GetInstanceIdFromEntityId(entityId), + }; + + if (!this.EntityContext.ValidateOperationTransition(target.InstanceId, oneWay, out string errorMessage)) + { + throw new EntityLockingRulesViolationException(errorMessage); + } + + int taskId = this.idCounter; // we are not incrementing the counter here because it will be incremented when calling this.SendEvent below + Guid operationId = Utils.CreateGuidFromHash(string.Concat(OrchestrationInstance.ExecutionId, ":", taskId)); + + string serializedInput = this.MessageDataConverter.Serialize(input); + + (string name, object content) eventToSend + = this.entityContext.EmitRequestMessage(target, operationName, oneWay, operationId, scheduledTimeUtc, serializedInput); + + this.SendEvent(target, eventToSend.name, eventToSend.content); + + return (operationId, target.InstanceId, taskId); + } + + public override Task LockEntitiesAsync(params EntityId[] entities) + { + this.CheckEntitySupport(); + + if (!this.EntityContext.ValidateAcquireTransition(out string errormsg)) + { + throw new EntityLockingRulesViolationException(errormsg); + } + + if (entities == null || entities.Length == 0) + { + throw new ArgumentException("The list of entities to lock must not be null or empty.", nameof(entities)); + } + + int taskId = this.idCounter; // we are not incrementing the counter here because it will be incremented when calling this.SendEvent below + Guid criticalSectionId = Utils.CreateGuidFromHash(string.Concat(OrchestrationInstance.ExecutionId, ":", taskId)); + + // send a message to the first entity to be acquired + (OrchestrationInstance target, string name, object content) eventToSend = + this.entityContext.EmitAcquireMessage(criticalSectionId, entities); + + this.SendEvent(eventToSend.target, eventToSend.name, eventToSend.content); + + return this.entityContext.WaitForLockResponseAsync(criticalSectionId, taskId); + } + + internal void ExitCriticalSection() + { + if (this.entityContext != null) + { + foreach (var releaseMessage in this.entityContext.EmitLockReleaseMessages()) + { + this.SendEvent(releaseMessage.target, releaseMessage.eventName, releaseMessage.eventContent); + } + } + } + + public override void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData) { if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) @@ -406,6 +526,12 @@ public void HandleEventSentEvent(EventSentEvent eventSentEvent) public void HandleEventRaisedEvent(EventRaisedEvent eventRaisedEvent, bool skipCarryOverEvents, TaskOrchestration taskOrchestration) { + if (this.entityContext?.HandledAsEntityResponse(eventRaisedEvent, this) == true) + { + // this EventRaisedEvent was not an application-defined event, but an entity response + return; + } + if (skipCarryOverEvents || !this.HasContinueAsNew) { taskOrchestration.RaiseEvent(this, eventRaisedEvent.Name, eventRaisedEvent.Input); @@ -416,6 +542,47 @@ public void HandleEventRaisedEvent(EventRaisedEvent eventRaisedEvent, bool skipC } } + public void HandleEntityOperationCompletedEvent(EventRaisedEvent eventRaisedEvent, int taskId, string instanceId, OperationResult operationResult, TaskCompletionSource tcs) + { + T result; + if (operationResult.Result != null) + { + result = this.MessageDataConverter.Deserialize(operationResult.Result); + } + else + { + result = default; + } + tcs.SetResult(result); + } + + public void HandleEntityOperationFailedEvent(EventRaisedEvent eventRaisedEvent, int taskId, string instanceId, OperationResult operationResult, TaskCompletionSource tcs) + { + Exception cause = null; + + // When using ErrorPropagationMode.SerializeExceptions the "cause" is deserialized from history. + // This isn't fully reliable because not all exception types can be serialized/deserialized. + if (this.ErrorPropagationMode == ErrorPropagationMode.SerializeExceptions && !string.IsNullOrEmpty(operationResult.Result)) + { + try + { + cause = this.ErrorDataConverter.Deserialize(operationResult.Result); + } + catch (Exception converterException) when (!Utils.IsFatal(converterException)) + { + cause = new TaskFailedExceptionDeserializationException(operationResult.Result, converterException); + } + } + + var exceptionForCallingOrchestration = new OperationFailedException(eventRaisedEvent.EventId, taskId, instanceId, eventRaisedEvent.Name, operationResult.ErrorMessage, cause); + + if (this.ErrorPropagationMode == ErrorPropagationMode.UseFailureDetails) + { + exceptionForCallingOrchestration.FailureDetails = operationResult.FailureDetails; + } + + tcs.SetException(exceptionForCallingOrchestration); + } public void HandleTaskCompletedEvent(TaskCompletedEvent completedEvent) { @@ -497,8 +664,8 @@ public void HandleSubOrchestrationInstanceFailedEvent(SubOrchestrationInstanceFa // When using ErrorPropagationMode.UseFailureDetails we instead use FailureDetails to convey // error information, which doesn't involve any serialization at all. Exception cause = this.ErrorPropagationMode == ErrorPropagationMode.SerializeExceptions ? - Utils.RetrieveCause(failedEvent.Details, this.ErrorDataConverter) : - null; + Utils.RetrieveCause(failedEvent.Details, this.ErrorDataConverter) : + null; var failedException = new SubOrchestrationFailedException(failedEvent.EventId, taskId, info.Name, info.Version, @@ -608,7 +775,7 @@ public void FailOrchestration(Exception failure) details = orchestrationFailureException.Details; } } - else + else { if (this.ErrorPropagationMode == ErrorPropagationMode.UseFailureDetails) { @@ -625,6 +792,10 @@ public void FailOrchestration(Exception failure) public void CompleteOrchestration(string result, string details, OrchestrationStatus orchestrationStatus, FailureDetails failureDetails = null) { + // if we are still inside a critical section, we exit it before completing the orchestration + // so that the lock release messages are sent. + this.ExitCriticalSection(); + int id = this.idCounter++; OrchestrationCompleteOrchestratorAction completedOrchestratorAction; if (orchestrationStatus == OrchestrationStatus.Completed && this.continueAsNew != null) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index d7e3dcc98..6ef1a2bf8 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -15,47 +15,54 @@ namespace DurableTask.Core { using System; using System.Collections.Generic; - using System.Collections.ObjectModel; using System.Diagnostics; - using System.Linq; using System.Threading; using System.Threading.Tasks; - using DurableTask.Core.Command; using DurableTask.Core.Common; + using DurableTask.Core.Entities; using DurableTask.Core.Exceptions; using DurableTask.Core.History; using DurableTask.Core.Logging; using DurableTask.Core.Middleware; - using DurableTask.Core.Serializing; using DurableTask.Core.Tracing; /// - /// Dispatcher for orchestrations to handle processing and renewing, completion of orchestration events + /// Dispatcher for orchestrations and entities to handle processing and renewing, completion of orchestration events. /// public class TaskOrchestrationDispatcher { - static readonly Task CompletedTask = Task.FromResult(0); - - readonly INameVersionObjectManager objectManager; + readonly INameVersionObjectManager orchestrationObjectManager; + readonly INameVersionObjectManager entityObjectManager; readonly IOrchestrationService orchestrationService; readonly WorkItemDispatcher dispatcher; - readonly DispatchMiddlewarePipeline dispatchPipeline; + readonly DispatchMiddlewarePipeline orchestrationDispatchPipeline; + readonly DispatchMiddlewarePipeline entityDispatchPipeline; + readonly EntityBackendInformation? entityBackendInformation; readonly LogHelper logHelper; ErrorPropagationMode errorPropagationMode; readonly NonBlockingCountdownLock concurrentSessionLock; internal TaskOrchestrationDispatcher( IOrchestrationService orchestrationService, - INameVersionObjectManager objectManager, - DispatchMiddlewarePipeline dispatchPipeline, + INameVersionObjectManager orchestrationObjectManager, + INameVersionObjectManager entityObjectManager, + DispatchMiddlewarePipeline orchestrationDispatchPipeline, + DispatchMiddlewarePipeline entityDispatchPipeline, LogHelper logHelper, ErrorPropagationMode errorPropagationMode) { - this.objectManager = objectManager ?? throw new ArgumentNullException(nameof(objectManager)); + this.orchestrationObjectManager = orchestrationObjectManager ?? throw new ArgumentNullException(nameof(orchestrationObjectManager)); + this.entityObjectManager = entityObjectManager ?? throw new ArgumentNullException(nameof(entityObjectManager)); this.orchestrationService = orchestrationService ?? throw new ArgumentNullException(nameof(orchestrationService)); - this.dispatchPipeline = dispatchPipeline ?? throw new ArgumentNullException(nameof(dispatchPipeline)); + this.orchestrationDispatchPipeline = orchestrationDispatchPipeline ?? throw new ArgumentNullException(nameof(orchestrationDispatchPipeline)); + this.entityDispatchPipeline = entityDispatchPipeline ?? throw new ArgumentNullException(nameof(entityDispatchPipeline)); this.logHelper = logHelper ?? throw new ArgumentNullException(nameof(logHelper)); this.errorPropagationMode = errorPropagationMode; + + if (EntityBackendInformation.BackendSupportsEntities(orchestrationService, out var options)) + { + this.entityBackendInformation = options; + } this.dispatcher = new WorkItemDispatcher( "TaskOrchestrationDispatcher", @@ -78,6 +85,11 @@ internal TaskOrchestrationDispatcher( this.concurrentSessionLock = new NonBlockingCountdownLock(maxConcurrentSessions); } + /// + /// The entity options configured, or null if the backend does not support entities. + /// + public EntityBackendInformation? EntityBackendInformation => this.entityBackendInformation; + /// /// Starts the dispatcher to start getting and processing orchestration events /// @@ -116,7 +128,6 @@ protected Task OnFetchWorkItemAsync(TimeSpan receiveT return this.orchestrationService.LockNextTaskOrchestrationWorkItemAsync(receiveTimeout, cancellationToken); } - /// /// Ensures the first ExecutionStarted event in the batch (if any) appears at the beginning /// of its executionID history. @@ -279,29 +290,35 @@ async Task OnProcessWorkItemSessionAsync(TaskOrchestrationWorkItem workItem) /// /// The work item to process protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem workItem) - { - var messagesToSend = new List(); - var timerMessages = new List(); - var orchestratorMessages = new List(); - var isCompleted = false; - var continuedAsNew = false; - var isInterrupted = false; - + { // correlation CorrelationTraceClient.Propagate(() => CorrelationTraceContext.Current = workItem.TraceContext); - ExecutionStartedEvent? continueAsNewExecutionStarted = null; - TaskMessage? continuedAsNewMessage = null; - IList? carryOverEvents = null; - string? carryOverStatus = null; - - OrchestrationRuntimeState runtimeState = workItem.OrchestrationRuntimeState; - - runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); - - OrchestrationRuntimeState originalOrchestrationRuntimeState = runtimeState; - - OrchestrationState? instanceState = null; + OrchestrationRuntimeState originalOrchestrationRuntimeState = workItem.OrchestrationRuntimeState; + + bool isEntity = Common.Entities.IsEntityInstance(workItem.InstanceId); + + WorkItemProcessor specializedDispatcher = isEntity + ? new EntityWorkItemProcessor( + this, + workItem, + this.logHelper, + this.entityObjectManager, + this.entityDispatchPipeline, + this.entityBackendInformation, + this.errorPropagationMode) + : new OrchestrationWorkItemProcessor( + this, + workItem, + this.orchestrationService, + this.errorPropagationMode, + this.logHelper, + this.orchestrationObjectManager, + this.orchestrationDispatchPipeline); + + specializedDispatcher.workItem = workItem; + specializedDispatcher.runtimeState = workItem.OrchestrationRuntimeState; + specializedDispatcher.runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); Task? renewTask = null; using var renewCancellationTokenSource = new CancellationTokenSource(); @@ -323,223 +340,17 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work TraceHelper.TraceSession( TraceEventType.Error, "TaskOrchestrationDispatcher-DeletedOrchestration", - runtimeState.OrchestrationInstance?.InstanceId, + specializedDispatcher.runtimeState.OrchestrationInstance?.InstanceId, "Received work-item for an invalid orchestration"); - isCompleted = true; + + specializedDispatcher.isCompleted = true; } else { - do - { - continuedAsNew = false; - continuedAsNewMessage = null; - - this.logHelper.OrchestrationExecuting(runtimeState.OrchestrationInstance!, runtimeState.Name); - TraceHelper.TraceInstance( - TraceEventType.Verbose, - "TaskOrchestrationDispatcher-ExecuteUserOrchestration-Begin", - runtimeState.OrchestrationInstance, - "Executing user orchestration: {0}", - JsonDataConverter.Default.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true)); - - if (workItem.Cursor == null) - { - workItem.Cursor = await this.ExecuteOrchestrationAsync(runtimeState, workItem); - } - else - { - await this.ResumeOrchestrationAsync(workItem); - } - - IReadOnlyList decisions = workItem.Cursor.LatestDecisions.ToList(); - - this.logHelper.OrchestrationExecuted( - runtimeState.OrchestrationInstance!, - runtimeState.Name, - decisions); - TraceHelper.TraceInstance( - TraceEventType.Information, - "TaskOrchestrationDispatcher-ExecuteUserOrchestration-End", - runtimeState.OrchestrationInstance, - "Executed user orchestration. Received {0} orchestrator actions: {1}", - decisions.Count, - string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType))); - - // TODO: Exception handling for invalid decisions, which is increasingly likely - // when custom middleware is involved (e.g. out-of-process scenarios). - foreach (OrchestratorAction decision in decisions) - { - TraceHelper.TraceInstance( - TraceEventType.Information, - "TaskOrchestrationDispatcher-ProcessOrchestratorAction", - runtimeState.OrchestrationInstance, - "Processing orchestrator action of type {0}", - decision.OrchestratorActionType); - switch (decision.OrchestratorActionType) - { - case OrchestratorActionType.ScheduleOrchestrator: - var scheduleTaskAction = (ScheduleTaskOrchestratorAction)decision; - var message = this.ProcessScheduleTaskDecision( - scheduleTaskAction, - runtimeState, - this.IncludeParameters); - messagesToSend.Add(message); - break; - case OrchestratorActionType.CreateTimer: - var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision; - timerMessages.Add(this.ProcessCreateTimerDecision( - timerOrchestratorAction, - runtimeState, - isInternal: false)); - break; - case OrchestratorActionType.CreateSubOrchestration: - var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; - orchestratorMessages.Add( - this.ProcessCreateSubOrchestrationInstanceDecision( - createSubOrchestrationAction, - runtimeState, - this.IncludeParameters)); - break; - case OrchestratorActionType.SendEvent: - var sendEventAction = (SendEventOrchestratorAction)decision; - orchestratorMessages.Add( - this.ProcessSendEventDecision(sendEventAction, runtimeState)); - break; - case OrchestratorActionType.OrchestrationComplete: - OrchestrationCompleteOrchestratorAction completeDecision = (OrchestrationCompleteOrchestratorAction)decision; - TaskMessage? workflowInstanceCompletedMessage = - this.ProcessWorkflowCompletedTaskDecision(completeDecision, runtimeState, this.IncludeDetails, out continuedAsNew); - if (workflowInstanceCompletedMessage != null) - { - // Send complete message to parent workflow or to itself to start a new execution - // Store the event so we can rebuild the state - carryOverEvents = null; - if (continuedAsNew) - { - continuedAsNewMessage = workflowInstanceCompletedMessage; - continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent; - if (completeDecision.CarryoverEvents.Any()) - { - carryOverEvents = completeDecision.CarryoverEvents.ToList(); - completeDecision.CarryoverEvents.Clear(); - } - } - else - { - orchestratorMessages.Add(workflowInstanceCompletedMessage); - } - } - - isCompleted = !continuedAsNew; - break; - default: - throw TraceHelper.TraceExceptionInstance( - TraceEventType.Error, - "TaskOrchestrationDispatcher-UnsupportedDecisionType", - runtimeState.OrchestrationInstance, - new NotSupportedException($"Decision type '{decision.OrchestratorActionType}' not supported")); - } - - // Underlying orchestration service provider may have a limit of messages per call, to avoid the situation - // we keep on asking the provider if message count is ok and stop processing new decisions if not. - // - // We also put in a fake timer to force next orchestration task for remaining messages - int totalMessages = messagesToSend.Count + orchestratorMessages.Count + timerMessages.Count; - if (this.orchestrationService.IsMaxMessageCountExceeded(totalMessages, runtimeState)) - { - TraceHelper.TraceInstance( - TraceEventType.Information, - "TaskOrchestrationDispatcher-MaxMessageCountReached", - runtimeState.OrchestrationInstance, - "MaxMessageCount reached. Adding timer to process remaining events in next attempt."); - - if (isCompleted || continuedAsNew) - { - TraceHelper.TraceInstance( - TraceEventType.Information, - "TaskOrchestrationDispatcher-OrchestrationAlreadyCompleted", - runtimeState.OrchestrationInstance, - "Orchestration already completed. Skip adding timer for splitting messages."); - break; - } - - var dummyTimer = new CreateTimerOrchestratorAction - { - Id = FrameworkConstants.FakeTimerIdToSplitDecision, - FireAt = DateTime.UtcNow - }; - - timerMessages.Add(this.ProcessCreateTimerDecision( - dummyTimer, - runtimeState, - isInternal: true)); - isInterrupted = true; - break; - } - } - - // correlation - CorrelationTraceClient.Propagate(() => - { - if (runtimeState.ExecutionStartedEvent != null) - runtimeState.ExecutionStartedEvent.Correlation = CorrelationTraceContext.Current.SerializableTraceContext; - }); - - - // finish up processing of the work item - if (!continuedAsNew && runtimeState.Events.Last().EventType != EventType.OrchestratorCompleted) - { - runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); - } - - if (isCompleted) - { - TraceHelper.TraceSession(TraceEventType.Information, "TaskOrchestrationDispatcher-DeletingSessionState", workItem.InstanceId, "Deleting session state"); - if (runtimeState.ExecutionStartedEvent != null) - { - instanceState = Utils.BuildOrchestrationState(runtimeState); - } - } - else - { - if (continuedAsNew) - { - TraceHelper.TraceSession( - TraceEventType.Information, - "TaskOrchestrationDispatcher-UpdatingStateForContinuation", - workItem.InstanceId, - "Updating state for continuation"); - - // correlation - CorrelationTraceClient.Propagate(() => - { - continueAsNewExecutionStarted!.Correlation = CorrelationTraceContext.Current.SerializableTraceContext; - }); - - runtimeState = new OrchestrationRuntimeState(); - runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); - runtimeState.AddEvent(continueAsNewExecutionStarted!); - runtimeState.Status = workItem.OrchestrationRuntimeState.Status ?? carryOverStatus; - carryOverStatus = workItem.OrchestrationRuntimeState.Status; - - if (carryOverEvents != null) - { - foreach (var historyEvent in carryOverEvents) - { - runtimeState.AddEvent(historyEvent); - } - } - - runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); - workItem.OrchestrationRuntimeState = runtimeState; - - workItem.Cursor = null; - } - - instanceState = Utils.BuildOrchestrationState(runtimeState); - } - } while (continuedAsNew); + // now, do the actual processing of the work item + await specializedDispatcher.ProcessWorkItemAsync(); } + } finally { @@ -567,34 +378,21 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work workItem.OrchestrationRuntimeState = originalOrchestrationRuntimeState; } - runtimeState.Status = runtimeState.Status ?? carryOverStatus; - - if (instanceState != null) - { - instanceState.Status = runtimeState.Status; - } - - await this.orchestrationService.CompleteTaskOrchestrationWorkItemAsync( workItem, - runtimeState, - continuedAsNew ? null : messagesToSend, - orchestratorMessages, - continuedAsNew ? null : timerMessages, - continuedAsNewMessage, - instanceState); + specializedDispatcher.runtimeState, + specializedDispatcher.continuedAsNew ? null : specializedDispatcher.messagesToSend, + specializedDispatcher.orchestratorMessages, + specializedDispatcher.continuedAsNew ? null : specializedDispatcher.timerMessages, + specializedDispatcher.continuedAsNewMessage, + specializedDispatcher.instanceState); if (workItem.RestoreOriginalRuntimeStateDuringCompletion) { - workItem.OrchestrationRuntimeState = runtimeState; + workItem.OrchestrationRuntimeState = specializedDispatcher.runtimeState; } - return isCompleted || continuedAsNew || isInterrupted; - } - - static OrchestrationExecutionContext GetOrchestrationExecutionContext(OrchestrationRuntimeState runtimeState) - { - return new OrchestrationExecutionContext { OrchestrationTags = runtimeState.Tags ?? new Dictionary(capacity: 0) }; + return specializedDispatcher.isCompleted || specializedDispatcher.continuedAsNew || specializedDispatcher.isInterrupted; } TimeSpan MinRenewalInterval = TimeSpan.FromSeconds(5); // prevents excessive retries if clocks are off @@ -637,82 +435,6 @@ async Task RenewUntil(TaskOrchestrationWorkItem workItem, CancellationToken canc } } - async Task ExecuteOrchestrationAsync(OrchestrationRuntimeState runtimeState, TaskOrchestrationWorkItem workItem) - { - // Get the TaskOrchestration implementation. If it's not found, it either means that the developer never - // registered it (which is an error, and we'll throw for this further down) or it could be that some custom - // middleware (e.g. out-of-process execution middleware) is intended to implement the orchestration logic. - TaskOrchestration? taskOrchestration = this.objectManager.GetObject(runtimeState.Name, runtimeState.Version!); - - var dispatchContext = new DispatchMiddlewareContext(); - dispatchContext.SetProperty(runtimeState.OrchestrationInstance); - dispatchContext.SetProperty(taskOrchestration); - dispatchContext.SetProperty(runtimeState); - dispatchContext.SetProperty(workItem); - dispatchContext.SetProperty(GetOrchestrationExecutionContext(runtimeState)); - - TaskOrchestrationExecutor? executor = null; - - await this.dispatchPipeline.RunAsync(dispatchContext, _ => - { - // Check to see if the custom middleware intercepted and substituted the orchestration execution - // with its own execution behavior, providing us with the end results. If so, we can terminate - // the dispatch pipeline here. - var resultFromMiddleware = dispatchContext.GetProperty(); - if (resultFromMiddleware != null) - { - return CompletedTask; - } - - if (taskOrchestration == null) - { - throw TraceHelper.TraceExceptionInstance( - TraceEventType.Error, - "TaskOrchestrationDispatcher-TypeMissing", - runtimeState.OrchestrationInstance, - new TypeMissingException($"Orchestration not found: ({runtimeState.Name}, {runtimeState.Version})")); - } - - executor = new TaskOrchestrationExecutor( - runtimeState, - taskOrchestration, - this.orchestrationService.EventBehaviourForContinueAsNew, - this.errorPropagationMode); - OrchestratorExecutionResult resultFromOrchestrator = executor.Execute(); - dispatchContext.SetProperty(resultFromOrchestrator); - return CompletedTask; - }); - - var result = dispatchContext.GetProperty(); - IEnumerable decisions = result?.Actions ?? Enumerable.Empty(); - runtimeState.Status = result?.CustomStatus; - - return new OrchestrationExecutionCursor(runtimeState, taskOrchestration, executor, decisions); - } - - async Task ResumeOrchestrationAsync(TaskOrchestrationWorkItem workItem) - { - OrchestrationExecutionCursor cursor = workItem.Cursor; - - var dispatchContext = new DispatchMiddlewareContext(); - dispatchContext.SetProperty(cursor.RuntimeState.OrchestrationInstance); - dispatchContext.SetProperty(cursor.TaskOrchestration); - dispatchContext.SetProperty(cursor.RuntimeState); - dispatchContext.SetProperty(workItem); - - cursor.LatestDecisions = Enumerable.Empty(); - await this.dispatchPipeline.RunAsync(dispatchContext, _ => - { - OrchestratorExecutionResult result = cursor.OrchestrationExecutor.ExecuteNewEvents(); - dispatchContext.SetProperty(result); - return CompletedTask; - }); - - var result = dispatchContext.GetProperty(); - cursor.LatestDecisions = result?.Actions ?? Enumerable.Empty(); - cursor.RuntimeState.Status = result?.CustomStatus; - } - /// /// Converts new messages into history events that get appended to the existing orchestration state. /// Returns False if the workItem should be discarded. True if it should be processed further. @@ -798,244 +520,6 @@ bool ReconcileMessagesWithState(TaskOrchestrationWorkItem workItem) return true; } - TaskMessage? ProcessWorkflowCompletedTaskDecision( - OrchestrationCompleteOrchestratorAction completeOrchestratorAction, - OrchestrationRuntimeState runtimeState, - bool includeDetails, - out bool continuedAsNew) - { - ExecutionCompletedEvent executionCompletedEvent; - continuedAsNew = (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew); - if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew) - { - executionCompletedEvent = new ContinueAsNewEvent(completeOrchestratorAction.Id, - completeOrchestratorAction.Result); - } - else - { - executionCompletedEvent = new ExecutionCompletedEvent(completeOrchestratorAction.Id, - completeOrchestratorAction.Result, - completeOrchestratorAction.OrchestrationStatus, - completeOrchestratorAction.FailureDetails); - } - - runtimeState.AddEvent(executionCompletedEvent); - - this.logHelper.OrchestrationCompleted(runtimeState, completeOrchestratorAction); - TraceHelper.TraceInstance( - runtimeState.OrchestrationStatus == OrchestrationStatus.Failed ? TraceEventType.Warning : TraceEventType.Information, - "TaskOrchestrationDispatcher-InstanceCompleted", - runtimeState.OrchestrationInstance, - "Instance Id '{0}' completed in state {1} with result: {2}", - runtimeState.OrchestrationInstance, - runtimeState.OrchestrationStatus, - completeOrchestratorAction.Result); - TraceHelper.TraceInstance( - TraceEventType.Information, - "TaskOrchestrationDispatcher-InstanceCompletionEvents", - runtimeState.OrchestrationInstance, - () => Utils.EscapeJson(JsonDataConverter.Default.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true))); - - // Check to see if we need to start a new execution - if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew) - { - var taskMessage = new TaskMessage(); - var startedEvent = new ExecutionStartedEvent(-1, completeOrchestratorAction.Result) - { - OrchestrationInstance = new OrchestrationInstance - { - InstanceId = runtimeState.OrchestrationInstance!.InstanceId, - ExecutionId = Guid.NewGuid().ToString("N") - }, - Tags = runtimeState.Tags, - ParentInstance = runtimeState.ParentInstance, - Name = runtimeState.Name, - Version = completeOrchestratorAction.NewVersion ?? runtimeState.Version - }; - - taskMessage.OrchestrationInstance = startedEvent.OrchestrationInstance; - taskMessage.Event = startedEvent; - - return taskMessage; - } - - // If this is a Sub Orchestration, and not tagged as fire-and-forget, - // then notify the parent by sending a complete message - if (runtimeState.ParentInstance != null - && !OrchestrationTags.IsTaggedAsFireAndForget(runtimeState.Tags)) - { - var taskMessage = new TaskMessage(); - if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Completed) - { - var subOrchestrationCompletedEvent = - new SubOrchestrationInstanceCompletedEvent(-1, runtimeState.ParentInstance.TaskScheduleId, - completeOrchestratorAction.Result); - - taskMessage.Event = subOrchestrationCompletedEvent; - } - else if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Failed || - completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Terminated) - { - var subOrchestrationFailedEvent = - new SubOrchestrationInstanceFailedEvent(-1, runtimeState.ParentInstance.TaskScheduleId, - completeOrchestratorAction.Result, - includeDetails ? completeOrchestratorAction.Details : null); - subOrchestrationFailedEvent.FailureDetails = completeOrchestratorAction.FailureDetails; - - taskMessage.Event = subOrchestrationFailedEvent; - } - - if (taskMessage.Event != null) - { - taskMessage.OrchestrationInstance = runtimeState.ParentInstance.OrchestrationInstance; - return taskMessage; - } - } - - return null; - } - - TaskMessage ProcessScheduleTaskDecision( - ScheduleTaskOrchestratorAction scheduleTaskOrchestratorAction, - OrchestrationRuntimeState runtimeState, - bool includeParameters) - { - if (scheduleTaskOrchestratorAction.Name == null) - { - throw new ArgumentException("No name was given for the task activity to schedule!", nameof(scheduleTaskOrchestratorAction)); - } - - var taskMessage = new TaskMessage(); - - var scheduledEvent = new TaskScheduledEvent( - eventId: scheduleTaskOrchestratorAction.Id, - name: scheduleTaskOrchestratorAction.Name, - version: scheduleTaskOrchestratorAction.Version, - input: scheduleTaskOrchestratorAction.Input); - - taskMessage.Event = scheduledEvent; - taskMessage.OrchestrationInstance = runtimeState.OrchestrationInstance; - taskMessage.OrchestrationExecutionContext = GetOrchestrationExecutionContext(runtimeState); - - if (!includeParameters) - { - scheduledEvent = new TaskScheduledEvent( - eventId: scheduleTaskOrchestratorAction.Id, - name: scheduleTaskOrchestratorAction.Name, - version: scheduleTaskOrchestratorAction.Version); - } - - this.logHelper.SchedulingActivity( - runtimeState.OrchestrationInstance!, - scheduledEvent); - - runtimeState.AddEvent(scheduledEvent); - return taskMessage; - } - - TaskMessage ProcessCreateTimerDecision( - CreateTimerOrchestratorAction createTimerOrchestratorAction, - OrchestrationRuntimeState runtimeState, - bool isInternal) - { - var taskMessage = new TaskMessage(); - - var timerCreatedEvent = new TimerCreatedEvent(createTimerOrchestratorAction.Id) - { - FireAt = createTimerOrchestratorAction.FireAt - }; - - runtimeState.AddEvent(timerCreatedEvent); - - taskMessage.Event = new TimerFiredEvent(-1) - { - TimerId = createTimerOrchestratorAction.Id, - FireAt = createTimerOrchestratorAction.FireAt - }; - - this.logHelper.CreatingTimer( - runtimeState.OrchestrationInstance!, - timerCreatedEvent, - isInternal); - - taskMessage.OrchestrationInstance = runtimeState.OrchestrationInstance; - - return taskMessage; - } - - TaskMessage ProcessCreateSubOrchestrationInstanceDecision( - CreateSubOrchestrationAction createSubOrchestrationAction, - OrchestrationRuntimeState runtimeState, - bool includeParameters) - { - var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id) - { - Name = createSubOrchestrationAction.Name, - Version = createSubOrchestrationAction.Version, - InstanceId = createSubOrchestrationAction.InstanceId - }; - if (includeParameters) - { - historyEvent.Input = createSubOrchestrationAction.Input; - } - - runtimeState.AddEvent(historyEvent); - - var taskMessage = new TaskMessage(); - - var startedEvent = new ExecutionStartedEvent(-1, createSubOrchestrationAction.Input) - { - Tags = OrchestrationTags.MergeTags(createSubOrchestrationAction.Tags, runtimeState.Tags), - OrchestrationInstance = new OrchestrationInstance - { - InstanceId = createSubOrchestrationAction.InstanceId, - ExecutionId = Guid.NewGuid().ToString("N") - }, - ParentInstance = new ParentInstance - { - OrchestrationInstance = runtimeState.OrchestrationInstance, - Name = runtimeState.Name, - Version = runtimeState.Version, - TaskScheduleId = createSubOrchestrationAction.Id - }, - Name = createSubOrchestrationAction.Name, - Version = createSubOrchestrationAction.Version - }; - - this.logHelper.SchedulingOrchestration(startedEvent); - - taskMessage.OrchestrationInstance = startedEvent.OrchestrationInstance; - taskMessage.Event = startedEvent; - taskMessage.OrchestrationExecutionContext = GetOrchestrationExecutionContext(runtimeState); - - return taskMessage; - } - - TaskMessage ProcessSendEventDecision( - SendEventOrchestratorAction sendEventAction, - OrchestrationRuntimeState runtimeState) - { - var historyEvent = new EventSentEvent(sendEventAction.Id) - { - InstanceId = sendEventAction.Instance?.InstanceId, - Name = sendEventAction.EventName, - Input = sendEventAction.EventData - }; - - runtimeState.AddEvent(historyEvent); - - this.logHelper.RaisingEvent(runtimeState.OrchestrationInstance!, historyEvent); - - return new TaskMessage - { - OrchestrationInstance = sendEventAction.Instance, - Event = new EventRaisedEvent(-1, sendEventAction.EventData) - { - Name = sendEventAction.EventName - } - }; - } - class NonBlockingCountdownLock { int available; @@ -1075,5 +559,33 @@ public void Release() Interlocked.Increment(ref this.available); } } + + /// + /// Base class for the specialized work item processing for entity and orchestration work items, respectively + /// + internal abstract class WorkItemProcessor + { + public TaskOrchestrationDispatcher dispatcher; + public TaskOrchestrationWorkItem workItem; + public OrchestrationRuntimeState runtimeState; + + public List messagesToSend = new List(); + public List timerMessages = new List(); + public List orchestratorMessages = new List(); + public bool isCompleted; + public bool continuedAsNew; + public bool isInterrupted; + public OrchestrationState? instanceState; + public TaskMessage? continuedAsNewMessage; + + public WorkItemProcessor(TaskOrchestrationDispatcher dispatcher, TaskOrchestrationWorkItem workItem) + { + this.dispatcher = dispatcher; + this.workItem = workItem; + this.runtimeState = workItem.OrchestrationRuntimeState; + } + + public abstract Task ProcessWorkItemAsync(); + } } } \ No newline at end of file diff --git a/src/DurableTask.Core/TaskOrchestrationExecutor.cs b/src/DurableTask.Core/TaskOrchestrationExecutor.cs index b0ca99976..2036cb105 100644 --- a/src/DurableTask.Core/TaskOrchestrationExecutor.cs +++ b/src/DurableTask.Core/TaskOrchestrationExecutor.cs @@ -22,6 +22,7 @@ namespace DurableTask.Core using System.Threading; using System.Threading.Tasks; using DurableTask.Core.Common; + using DurableTask.Core.Entities; using DurableTask.Core.Exceptions; using DurableTask.Core.History; @@ -43,23 +44,44 @@ public class TaskOrchestrationExecutor /// /// /// + /// /// public TaskOrchestrationExecutor( OrchestrationRuntimeState orchestrationRuntimeState, TaskOrchestration taskOrchestration, BehaviorOnContinueAsNew eventBehaviourForContinueAsNew, + EntityBackendInformation? entityBackendInformation, ErrorPropagationMode errorPropagationMode = ErrorPropagationMode.SerializeExceptions) { this.decisionScheduler = new SynchronousTaskScheduler(); this.context = new TaskOrchestrationContext( orchestrationRuntimeState.OrchestrationInstance, this.decisionScheduler, - errorPropagationMode); + entityBackendInformation, + errorPropagationMode + ); this.orchestrationRuntimeState = orchestrationRuntimeState; this.taskOrchestration = taskOrchestration; this.skipCarryOverEvents = eventBehaviourForContinueAsNew == BehaviorOnContinueAsNew.Ignore; } + /// + /// Initializes a new instance of the class. + /// This overload is needed only to avoid breaking changes because this is a public constructor. + /// + /// + /// + /// + /// + public TaskOrchestrationExecutor( + OrchestrationRuntimeState orchestrationRuntimeState, + TaskOrchestration taskOrchestration, + BehaviorOnContinueAsNew eventBehaviourForContinueAsNew, + ErrorPropagationMode errorPropagationMode = ErrorPropagationMode.SerializeExceptions) + : this(orchestrationRuntimeState, taskOrchestration, eventBehaviourForContinueAsNew, new EntityBackendInformation(), errorPropagationMode) + { + } + internal bool IsCompleted => this.result != null && (this.result.IsCompleted || this.result.IsFaulted); /// diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index 5d9ca75b9..d82db750e 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -19,14 +19,18 @@ namespace DurableTask.AzureStorage.Tests using System.Diagnostics; using System.IO; using System.Linq; + using System.Reflection; using System.Runtime.Serialization; using System.Text; using System.Threading; using System.Threading.Tasks; using DurableTask.AzureStorage.Tracking; using DurableTask.Core; + using DurableTask.Core.Entities; + using DurableTask.Core.Entities.OperationFormat; using DurableTask.Core.Exceptions; using DurableTask.Core.History; + using DurableTask.Core.Serializing; using Microsoft.Practices.EnterpriseLibrary.SemanticLogging.Utility; using Microsoft.VisualStudio.TestTools.UnitTesting; using Microsoft.WindowsAzure.Storage; @@ -348,6 +352,661 @@ public async Task ContinueAsNewThenTimer(bool enableExtendedSessions) } } + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task CallCounterEntityFromOrchestration(bool enableExtendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + { + await host.StartAsync(); + + var entityId = new EntityId(nameof(Entities.Counter), Guid.NewGuid().ToString()); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.CallCounterEntity), entityId); + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual("OK", JToken.Parse(status?.Output)); + + await host.StopAsync(); + } + } + + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task BatchedEntitySignals(bool enableExtendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + { + await host.StartAsync(); + + int numIterations = 100; + var entityId = new EntityId(nameof(Entities.BatchEntity), Guid.NewGuid().ToString()); + TestEntityClient client = await host.GetEntityClientAsync(typeof(Entities.BatchEntity), entityId); + + // send a number of signals immediately after each other + List tasks = new List(); + for (int i = 0; i < numIterations; i++) + { + tasks.Add(client.SignalEntity(i.ToString())); + } + + await Task.WhenAll(tasks); + + var result = await client.WaitForEntityState>( + timeout: Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(20), + list => list.Count == numIterations ? null : $"waiting for {numIterations - list.Count} signals"); + + // validate the batching positions and sizes + int? cursize = null; + int curpos = 0; + int numBatches = 0; + foreach (var (position, size) in result) + { + if (cursize == null) + { + cursize = size; + curpos = 0; + numBatches++; + } + + Assert.AreEqual(curpos, position); + + if (++curpos == cursize) + { + cursize = null; + } + } + + // there should always be some batching going on + Assert.IsTrue(numBatches < numIterations); + + await host.StopAsync(); + } + } + + [DataTestMethod] + public async Task CleanEntityStorage_OrphanedLock() + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost( + enableExtendedSessions: false, + modifySettingsAction: (settings) => settings.EntityMessageReorderWindowInMinutes = 0)) + { + await host.StartAsync(); + + // construct unique names for this test + string prefix = Guid.NewGuid().ToString("N").Substring(0, 6); + var orphanedEntityId = new EntityId(nameof(Entities.Counter), $"{prefix}-orphaned"); + var orchestrationA = $"{prefix}-A"; + var orchestrationB = $"{prefix}-B"; + + // run an orchestration A that leaves an orphaned lock + var clientA = await host.StartOrchestrationAsync(typeof(Orchestrations.LockThenFailReplay), (orphanedEntityId, true), orchestrationA); + var status = await clientA.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + // run an orchestration B that queues behind A for the lock (and thus gets stuck) + var clientB = await host.StartOrchestrationAsync(typeof(Orchestrations.LockThenFailReplay), (orphanedEntityId, false), orchestrationB); + + // remove empty entity and release orphaned lock + var entityClient = new TaskHubEntityClient(clientA.InnerClient); + var response = await entityClient.CleanEntityStorageAsync(true, true, CancellationToken.None); + Assert.AreEqual(1, response.NumberOfOrphanedLocksRemoved); + Assert.AreEqual(0, response.NumberOfEmptyEntitiesRemoved); + + // wait for orchestration B to complete, now that the lock has been released + status = await clientB.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + Assert.IsTrue(status.OrchestrationStatus == OrchestrationStatus.Completed); + + // give the orchestration status time to be updated + await Task.Delay(TimeSpan.FromSeconds(20)); + + // clean again to remove the orphaned entity which is now empty also + response = await entityClient.CleanEntityStorageAsync(true, true, CancellationToken.None); + Assert.AreEqual(0, response.NumberOfOrphanedLocksRemoved); + Assert.AreEqual(1, response.NumberOfEmptyEntitiesRemoved); + + await host.StopAsync(); + } + } + + [DataTestMethod] + [DataRow(1)] + [DataRow(120)] + public async Task CleanEntityStorage_EmptyEntities(int numReps) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost( + enableExtendedSessions: false, + modifySettingsAction: (settings) => settings.EntityMessageReorderWindowInMinutes = 0)) + { + await host.StartAsync(); + + // construct unique names for this test + string prefix = Guid.NewGuid().ToString("N").Substring(0, 6); + EntityId[] entityIds = new EntityId[numReps]; + for (int i = 0; i < entityIds.Length; i++) + { + entityIds[i] = new EntityId("Counter", $"{prefix}-{i:D3}"); + } + + // create the empty entities + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.CreateEmptyEntities), entityIds); + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + // account for delay in updating instance tables + await Task.Delay(TimeSpan.FromSeconds(20)); + + // remove all empty entities + var entityClient = new TaskHubEntityClient(client.InnerClient); + var response = await entityClient.CleanEntityStorageAsync(true, true, CancellationToken.None); + Assert.AreEqual(0, response.NumberOfOrphanedLocksRemoved); + Assert.AreEqual(numReps, response.NumberOfEmptyEntitiesRemoved); + + await host.StopAsync(); + } + } + + [DataTestMethod] + public async Task EntityQueries() + { + var yesterday = DateTime.UtcNow.Subtract(TimeSpan.FromDays(1)); + var tomorrow = DateTime.UtcNow.Add(TimeSpan.FromDays(1)); + + List entityIds = new List() + { + new EntityId("StringStore", "foo"), + new EntityId("StringStore", "bar"), + new EntityId("StringStore", "baz"), + new EntityId("StringStore2", "foo"), + }; + + var queries = new (TaskHubEntityClient.Query,Action>)[] + { + (new TaskHubEntityClient.Query + { + }, + result => + { + Assert.AreEqual(4, result.Count); + }), + + (new TaskHubEntityClient.Query + { + EntityName = "StringStore", + LastOperationFrom = yesterday, + LastOperationTo = tomorrow, + FetchState = false, + }, + result => + { + Assert.AreEqual(3, result.Count); + Assert.IsTrue(result[0].State == null); + }), + + (new TaskHubEntityClient.Query + { + EntityName = "StringStore", + LastOperationFrom = yesterday, + LastOperationTo = tomorrow, + FetchState = true, + }, + result => + { + Assert.AreEqual(3, result.Count); + Assert.IsTrue(result[0].State != null); + }), + + (new TaskHubEntityClient.Query + { + EntityName = "StringStore", + PageSize = 1, + }, + result => + { + Assert.AreEqual(3, result.Count()); + }), + + (new TaskHubEntityClient.Query + { + EntityName = "StringStore", + PageSize = 2, + }, + result => + { + Assert.AreEqual(3, result.Count()); + }), + + (new TaskHubEntityClient.Query + { + EntityName = "noResult", + }, + result => + { + Assert.AreEqual(0, result.Count()); + }), + + (new TaskHubEntityClient.Query + { + EntityName = "noResult", + LastOperationFrom = yesterday, + LastOperationTo = tomorrow, + }, + result => + { + Assert.AreEqual(0, result.Count()); + }), + + (new TaskHubEntityClient.Query + { + LastOperationFrom = tomorrow, + }, + result => + { + Assert.AreEqual(0, result.Count()); + }), + + (new TaskHubEntityClient.Query + { + LastOperationTo = yesterday, + }, + result => + { + Assert.AreEqual(0, result.Count()); + }), + + (new TaskHubEntityClient.Query + { + InstanceIdPrefix = "StringStore", + }, + result => + { + Assert.AreEqual(0, result.Count()); + }), + + (new TaskHubEntityClient.Query + { + InstanceIdPrefix = "@StringStore", + }, + result => + { + Assert.AreEqual(0, result.Count()); + }), + + (new TaskHubEntityClient.Query + { + InstanceIdPrefix = "@stringstore", + }, + result => + { + Assert.AreEqual(4, result.Count()); + }), + + (new TaskHubEntityClient.Query + { + InstanceIdPrefix = "@stringstore@", + }, + result => + { + Assert.AreEqual(3, result.Count()); + }), + + (new TaskHubEntityClient.Query + { + InstanceIdPrefix = "@stringstore", + EntityName = "stringstore", + }, + result => + { + Assert.AreEqual(3, result.Count()); + }), + + (new TaskHubEntityClient.Query + { + InstanceIdPrefix = "@stringstore@", + EntityName = "StringStore", + }, + result => + { + Assert.AreEqual(3, result.Count()); + }), + + (new TaskHubEntityClient.Query + { + InstanceIdPrefix = "@stringstore@b", + EntityName = "StringStore", + }, + result => + { + Assert.AreEqual(2, result.Count()); + }), + }; + + await this.RunEntityQueries(queries, entityIds); + } + + [DataTestMethod] + public async Task EntityQueries_Deleted() + { + var queries = new (TaskHubEntityClient.Query, Action>)[] + { + (new TaskHubEntityClient.Query() + { + IncludeDeleted = false, + }, + result => + { + Assert.AreEqual(4, result.Count); + }), + + (new TaskHubEntityClient.Query() + { + IncludeDeleted = true, + }, + result => + { + Assert.AreEqual(8, result.Count); + }), + + (new TaskHubEntityClient.Query() + { + IncludeDeleted = false, + PageSize = 3, + }, + result => + { + Assert.AreEqual(4, result.Count); + }), + + (new TaskHubEntityClient.Query() + { + IncludeDeleted = true, + PageSize = 3, + }, + result => + { + Assert.AreEqual(8, result.Count); + }), + }; + + List entityIds = new List() + { + new EntityId("StringStore", "foo"), + new EntityId("StringStore2", "bar"), + new EntityId("StringStore2", "baz"), + new EntityId("StringStore2", "foo"), + new EntityId("StringStore2", "ffo"), + new EntityId("StringStore2", "zzz"), + new EntityId("StringStore2", "aaa"), + new EntityId("StringStore2", "bbb"), + }; + + List orchestrations = new List() + { + typeof(Orchestrations.SignalAndCallStringStore), + typeof(Orchestrations.CallAndDeleteStringStore), + typeof(Orchestrations.SignalAndCallStringStore), + typeof(Orchestrations.CallAndDeleteStringStore), + typeof(Orchestrations.SignalAndCallStringStore), + typeof(Orchestrations.CallAndDeleteStringStore), + typeof(Orchestrations.SignalAndCallStringStore), + typeof(Orchestrations.CallAndDeleteStringStore), + }; + + await this.RunEntityQueries(queries, entityIds, orchestrations); + } + + private async Task RunEntityQueries( + (TaskHubEntityClient.Query, Action>)[] queries, + IList entitiyIds, + IList orchestrations = null) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false)) + { + await host.StartAsync(); + + TestOrchestrationClient client = null; + + for (int i = 0; i < entitiyIds.Count; i++) + { + EntityId id = entitiyIds[i]; + Type orchestration = orchestrations == null ? typeof(Orchestrations.SignalAndCallStringStore) : orchestrations[i]; + client = await host.StartOrchestrationAsync(orchestration, id); + await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + } + + // account for delay in updating instance tables + await Task.Delay(TimeSpan.FromSeconds(10)); + + for (int i = 0; i < queries.Count(); i++) + { + var query = queries[i].Item1; + var test = queries[i].Item2; + var results = new List(); + var entityClient = new TaskHubEntityClient(client.InnerClient); + + do + { + var result = await entityClient.ListEntitiesAsync(query, CancellationToken.None); + + // The result may return fewer records than the page size, but never more + Assert.IsTrue(result.Entities.Count() <= query.PageSize); + + foreach (var element in result.Entities) + { + results.Add(element); + } + + query.ContinuationToken = result.ContinuationToken; + } + while (query.ContinuationToken != null); + + test(results); + } + + await host.StopAsync(); + } + } + + /// + /// End-to-end test which validates launching orchestrations from entities. + /// + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task EntityFireAndForget(bool extendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: extendedSessions)) + { + await host.StartAsync(); + + var entityId = new EntityId(nameof(Entities.Launcher), Guid.NewGuid().ToString()); + + var client = await host.StartOrchestrationAsync( + typeof(Orchestrations.LaunchOrchestrationFromEntity), + entityId); + + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(90)); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + + var instanceId = (string) JToken.Parse(status?.Output); + Assert.IsTrue(instanceId != null); + var orchestrationState = (await client.InnerClient.GetOrchestrationStateAsync(instanceId, false)).FirstOrDefault(); + Assert.AreEqual(OrchestrationStatus.Completed, orchestrationState.OrchestrationStatus); + + await host.StopAsync(); + } + } + + /// + /// End-to-end test which validates a simple entity scenario which sends a signal + /// to a relay which forwards it to counter, and polls until the signal is delivered. + /// + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task DurableEntity_SignalThenPoll(bool extendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: extendedSessions)) + { + await host.StartAsync(); + + var relayEntityId = new EntityId("Relay", ""); + var counterEntityId = new EntityId("Counter", Guid.NewGuid().ToString()); + + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.PollCounterEntity), counterEntityId); + + var entityClient = new TaskHubEntityClient(client.InnerClient); + await entityClient.SignalEntityAsync(relayEntityId, "", (counterEntityId, "increment")); + + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual("ok", (string)JToken.Parse(status?.Output)); + + await host.StopAsync(); + } + } + + /// + /// End-to-end test which validates an entity scenario where a "LockedTransfer" orchestration locks + /// two "Counter" entities, and then in parallel increments/decrements them, respectively, using + /// a read-modify-write pattern. + /// + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task DurableEntity_SingleLockedTransfer(bool extendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: extendedSessions)) + { + await host.StartAsync(); + + var counter1 = new EntityId("Counter", Guid.NewGuid().ToString()); + var counter2 = new EntityId("Counter", Guid.NewGuid().ToString()); + + var client = await host.StartOrchestrationAsync( + typeof(Orchestrations.LockedTransfer), + (counter1, counter2)); + + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + + // validate the state of the counters + var entityClient = new TaskHubEntityClient(client.InnerClient); + var response1 = await entityClient.ReadEntityStateAsync(counter1); + var response2 = await entityClient.ReadEntityStateAsync(counter2); + Assert.IsTrue(response1.EntityExists); + Assert.IsTrue(response2.EntityExists); + Assert.AreEqual(-1, response1.EntityState); + Assert.AreEqual(1, response2.EntityState); + + await host.StopAsync(); + } + } + + /// + /// End-to-end test which validates an entity scenario where a a number of "LockedTransfer" orchestrations + /// concurrently operate on a number of entities, in a classical dining-philosophers configuration. + /// This showcases the deadlock prevention mechanism achieved by the sequential, ordered lock acquisition. + /// + [DataTestMethod] + [DataRow(false, 5)] + [DataRow(true, 5)] + public async Task DurableEntity_MultipleLockedTransfers(bool extendedSessions, int numberEntities) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: extendedSessions)) + { + await host.StartAsync(); + + // create specified number of entities + var counters = new EntityId[numberEntities]; + for (int i = 0; i < numberEntities; i++) + { + counters[i] = new EntityId("Counter", Guid.NewGuid().ToString()); + } + + // in parallel, start one transfer per counter, each decrementing a counter and incrementing + // its successor (where the last one wraps around to the first) + // This is a pattern that would deadlock if we didn't order the lock acquisition. + var clients = new Task[numberEntities]; + for (int i = 0; i < numberEntities; i++) + { + clients[i] = host.StartOrchestrationAsync( + typeof(Orchestrations.LockedTransfer), + (counters[i], counters[(i + 1) % numberEntities])); + } + + await Task.WhenAll(clients); + + // in parallel, wait for all transfers to complete + var stati = new Task[numberEntities]; + for (int i = 0; i < numberEntities; i++) + { + stati[i] = clients[i].Result.WaitForCompletionAsync(TimeSpan.FromSeconds(60)); + } + + await Task.WhenAll(stati); + + // check that they all completed + for (int i = 0; i < numberEntities; i++) + { + Assert.AreEqual(OrchestrationStatus.Completed, stati[i].Result.OrchestrationStatus); + } + + // in parallel, read all the entity states + var entityClient = new TaskHubEntityClient(clients[0].Result.InnerClient); + var entityStates = new Task>[numberEntities]; + for (int i = 0; i < numberEntities; i++) + { + entityStates[i] = entityClient.ReadEntityStateAsync(counters[i]); + } + await Task.WhenAll(entityStates); + + // check that the counter states are all back to 0 + // (since each participated in 2 transfers, one incrementing and one decrementing) + for (int i = 0; i < numberEntities; i++) + { + Assert.IsTrue(entityStates[i].Result.EntityExists); + Assert.AreEqual(0, entityStates[i].Result.EntityState); + } + + await host.StopAsync(); + } + } + + /// + /// End-to-end test which validates exception handling in entity operations. + /// + [DataTestMethod] + [DataRow(false, false, false)] + [DataRow(false, true, false)] + [DataRow(true, false, false)] + [DataRow(true, true, false)] + [DataRow(false, false, true)] + public async Task CallFaultyEntity(bool extendedSessions, bool rollbackOnExceptions, bool useFailureDetails) + { + var errorPropagationMode = useFailureDetails ? ErrorPropagationMode.UseFailureDetails : ErrorPropagationMode.SerializeExceptions; + + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost( + enableExtendedSessions: extendedSessions, + errorPropagationMode: errorPropagationMode)) + { + await host.StartAsync(); + + var entityName = rollbackOnExceptions ? "FaultyEntityWithRollback" : "FaultyEntityWithoutRollback"; + var entityId = new EntityId(entityName, Guid.NewGuid().ToString()); + + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.CallFaultyEntity), (entityId, rollbackOnExceptions, errorPropagationMode)); + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(90)); + + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual("\"ok\"", status?.Output); + + await host.StopAsync(); + } + } + [TestMethod] public async Task PurgeInstanceHistoryForSingleInstanceWithoutLargeMessageBlobs() { @@ -2594,85 +3253,582 @@ public override async Task RunTask(OrchestrationContext context, bool wa { await r; } - results[i] = r; + results[i] = r; + } + + string[] data = await Task.WhenAll(results); + Result = string.Concat(data); + return Result; + } + } + + [KnownType(typeof(Orchestrations.ParentWorkflowNestedActivityFail))] + [KnownType(typeof(Orchestrations.ChildWorkflowNestedActivityFail))] + [KnownType(typeof(Activities.HelloFailNestedSuborchestration))] + public class GrandparentWorkflowNestedActivityFail : TaskOrchestration + { + public static string Result; + public override async Task RunTask(OrchestrationContext context, bool waitForCompletion) + { + var results = new Task[2]; + for (int i = 0; i < 2; i++) + { + Task r = context.CreateSubOrchestrationInstance(typeof(Orchestrations.ParentWorkflowNestedActivityFail), i); + if (waitForCompletion) + { + await r; + } + results[i] = r; + } + + string[] data = await Task.WhenAll(results); + Result = string.Concat(data); + return Result; + } + } + + internal class Counter : TaskOrchestration + { + TaskCompletionSource waitForOperationHandle; + + public override async Task RunTask(OrchestrationContext context, int currentValue) + { + string operation = await this.WaitForOperation(); + + bool done = false; + switch (operation?.ToLowerInvariant()) + { + case "incr": + currentValue++; + break; + case "decr": + currentValue--; + break; + case "end": + done = true; + break; + } + + if (!done) + { + context.ContinueAsNew(currentValue); + } + + return currentValue; + + } + + async Task WaitForOperation() + { + this.waitForOperationHandle = new TaskCompletionSource(); + string operation = await this.waitForOperationHandle.Task; + this.waitForOperationHandle = null; + return operation; + } + + public override void OnEvent(OrchestrationContext context, string name, string input) + { + Assert.AreEqual("operation", name, true, "Unknown signal recieved..."); + if (this.waitForOperationHandle != null) + { + this.waitForOperationHandle.SetResult(input); + } + } + } + + [KnownType(typeof(Entities.Counter))] + public sealed class CallCounterEntity : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext context, EntityId entityId) + { + await context.CallEntityAsync(entityId, "set", 33); + int result = await context.CallEntityAsync(entityId, "get"); + + if (result == 33) + { + return "OK"; + } + else + { + return $"wrong result: {result} instead of 33"; + } + } + } + + [KnownType(typeof(Entities.Counter))] + [KnownType(typeof(Entities.Relay))] + public sealed class PollCounterEntity : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext ctx, EntityId entityId) + { + while (true) + { + var result = await ctx.CallEntityAsync(entityId, "get"); + + if (result != 0) + { + if (result == 1) + { + return "ok"; + } + else + { + return $"fail: wrong entity state: expected 1, got {result}"; + } + } + + await ctx.CreateTimer(DateTime.UtcNow + TimeSpan.FromSeconds(1), CancellationToken.None); + } + } + } + + [KnownType(typeof(Entities.Counter))] + public sealed class CreateEmptyEntities : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext context, EntityId[] entityIds) + { + var tasks = new List(); + for (int i = 0; i < entityIds.Length; i++) + { + tasks.Add(context.CallEntityAsync(entityIds[i], "delete")); + } + + await Task.WhenAll(tasks); + return "ok"; + } + } + + [KnownType(typeof(Entities.Counter))] + [KnownType(typeof(Activities.Hello))] + public sealed class LockThenFailReplay : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext context, (EntityId entityId, bool createNondeterminismFailure) input) + { + if (!(input.createNondeterminismFailure && context.IsReplaying)) + { + await context.LockEntitiesAsync(input.entityId); + + await context.ScheduleTask(typeof(Activities.Hello), "Tokyo"); + } + + return "ok"; + } + } + + [KnownType(typeof(Entities.Counter))] + public sealed class LockedTransfer : TaskOrchestration<(int, int), (EntityId, EntityId)> + { + public override async Task<(int, int)> RunTask(OrchestrationContext ctx, (EntityId, EntityId) input) + { + var (from, to) = input; + + if (from.Equals(to)) + { + throw new ArgumentException("from and to must be distinct"); + } + + if (ctx.IsInsideCriticalSection) + { + throw new Exception("test failed: lock context is incorrect"); + } + + int fromBalance; + int toBalance; + + using (await ctx.LockEntitiesAsync(from, to)) + { + if (!ctx.IsInsideCriticalSection) + { + throw new Exception("test failed: lock context is incorrect, must be in critical section"); + } + + var availableEntities = ctx.GetAvailableEntities().ToList(); + + if (availableEntities.Count != 2 || !availableEntities.Contains(from) || !availableEntities.Contains(to)) + { + throw new Exception("test failed: lock context is incorrect: available locks do not match"); + } + + // read balances in parallel + var t1 = ctx.CallEntityAsync(from, "get"); + + availableEntities = ctx.GetAvailableEntities().ToList(); + if (availableEntities.Count != 1 || !availableEntities.Contains(to)) + { + throw new Exception("test failed: lock context is incorrect: available locks wrong"); + } + + var t2 = ctx.CallEntityAsync(to, "get"); + + availableEntities = ctx.GetAvailableEntities().ToList(); + if (availableEntities.Count != 0) + { + throw new Exception("test failed: lock context is incorrect: available locks wrong"); + } + + fromBalance = await t1; + toBalance = await t2; + + availableEntities = ctx.GetAvailableEntities().ToList(); + if (availableEntities.Count != 2 || !availableEntities.Contains(from) || !availableEntities.Contains(to)) + { + throw new Exception("test failed: lock context is incorrect: available locks do not match"); + } + + // modify + fromBalance--; + toBalance++; + + // write balances in parallel + var t3 = ctx.CallEntityAsync(from, "set", fromBalance); + var t4 = ctx.CallEntityAsync(to, "set", toBalance); + await t3; + await t4; + } + + if (ctx.IsInsideCriticalSection) + { + throw new Exception("test failed: lock context is incorrect"); + } + + return (fromBalance, toBalance); + } + } + + + [KnownType(typeof(Entities.StringStore))] + [KnownType(typeof(Entities.StringStore2))] + public sealed class SignalAndCallStringStore : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext ctx, EntityId entity) + { + ctx.SignalEntity(entity, "set", "333"); + + var result = await ctx.CallEntityAsync(entity, "get"); + + if (result != "333") + { + return $"fail: wrong entity state: expected 333, got {result}"; + } + + return "ok"; + } + } + + [KnownType(typeof(Entities.StringStore))] + [KnownType(typeof(Entities.StringStore2))] + public sealed class CallAndDeleteStringStore : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext ctx, EntityId entity) + { + await ctx.CallEntityAsync(entity, "set", "333"); + + await ctx.CallEntityAsync(entity, "delete"); + + return "ok"; + } + } + + [KnownType(typeof(Orchestrations.DelayedSignal))] + [KnownType(typeof(Entities.Launcher))] + public sealed class LaunchOrchestrationFromEntity : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext ctx, EntityId entityId) + { + await ctx.CallEntityAsync(entityId, "launch", "hello"); + + while (true) + { + var orchestrationId = await ctx.CallEntityAsync(entityId, "get"); + + if (orchestrationId != null) + { + return orchestrationId; + } + + await ctx.CreateTimer(DateTime.UtcNow + TimeSpan.FromSeconds(1), CancellationToken.None); + } + } + } + + public sealed class DelayedSignal : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext ctx, EntityId entityId) + { + await ctx.CreateTimer(ctx.CurrentUtcDateTime + TimeSpan.FromSeconds(.2), CancellationToken.None); + + ctx.SignalEntity(entityId, "done"); + + return ""; + } + } + + [KnownType(typeof(Entities.FaultyEntityWithRollback))] + [KnownType(typeof(Entities.FaultyEntityWithoutRollback))] + public sealed class CallFaultyEntity : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext ctx, (EntityId, bool, ErrorPropagationMode) input) + { + (EntityId entityId, bool rollbackOnException, ErrorPropagationMode mode) = input; + + async Task ExpectException(Task t) + { + try + { + await t; + Assert.IsTrue(false, "expected exception"); + } + catch (OperationFailedException e) + { + if (mode == ErrorPropagationMode.SerializeExceptions) + { + Assert.IsTrue(e.InnerException != null && e.InnerException is TInner); + Assert.IsTrue(e.FailureDetails == null); + } + else // check that we received the failure details instead of the exception + { + Assert.IsTrue(e.InnerException == null); + Assert.IsTrue(e.FailureDetails != null); + Assert.IsTrue(e.FailureDetails.ErrorType == typeof(TInner).FullName); + } + } + catch (Exception e) + { + Assert.IsTrue(false, $"wrong exception: {e}"); + } + } + + Task Get() => ctx.CallEntityAsync(entityId, "Get"); + Task Set(int val) => ctx.CallEntityAsync(entityId, "Set", val); + Task SetThenThrow(int val) => ctx.CallEntityAsync(entityId, "SetThenThrow", val); + Task SetToUnserializable() => ctx.CallEntityAsync(entityId, "SetToUnserializable"); + Task SetToUnDeserializable() => ctx.CallEntityAsync(entityId, "SetToUnDeserializable"); + Task DeleteThenThrow () => ctx.CallEntityAsync(entityId, "DeleteThenThrow"); + Task Delete() => ctx.CallEntityAsync(entityId, "Delete"); + + + async Task TestRollbackToNonexistent() + { + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); + await ExpectException(SetToUnserializable()); + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); + } + + async Task TestNondeserializableState() + { + await SetToUnDeserializable(); + Assert.IsTrue(await ctx.CallEntityAsync(entityId, "exists")); + await ExpectException(Get()); + await ctx.CallEntityAsync(entityId, "deletewithoutreading"); + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); + } + + async Task TestSecondOperationRollback() + { + await Set(3); + Assert.AreEqual(3, await Get()); + await ExpectException(SetThenThrow(333)); + if (rollbackOnException) + { + Assert.AreEqual(3, await Get()); + } + else + { + Assert.AreEqual(333, await Get()); + } + await Delete(); + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); + } + + async Task TestDeleteOperationRollback() + { + await Set(3); + Assert.AreEqual(3, await Get()); + await ExpectException(SetThenThrow(333)); + if (rollbackOnException) + { + Assert.AreEqual(3, await Get()); + } + else + { + Assert.AreEqual(333, await Get()); + } + await ExpectException(DeleteThenThrow()); + if (rollbackOnException) + { + Assert.AreEqual(3, await Get()); + } + else + { + Assert.AreEqual(0, await Get()); + } + await Delete(); + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); + } + + async Task TestFirstOperationRollback() + { + await ExpectException(SetThenThrow(333)); + + if (!rollbackOnException) + { + Assert.IsTrue(await ctx.CallEntityAsync(entityId, "exists")); + await Delete(); + } + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); } - string[] data = await Task.WhenAll(results); - Result = string.Concat(data); - return Result; - } - } + // we use this utility function to try to enforce that a bunch of signals is delivered as a single batch. + // This is required for some of the tests here to work, since the batching affects the entity state management. + // The "enforcement" mechanism we use is not 100% failsafe (it still makes timing assumptions about the provider) + // but it should be more reliable than the original version of this test which failed quite frequently, as it was + // simply assuming that signals that are sent at the same time are always processed as a batch. + async Task ProcessAllSignalsInSingleBatch(Action sendSignals) + { + // first issue a signal that, when delivered, keeps the entity busy for a second + ctx.SignalEntity(entityId, "delay", 1); - [KnownType(typeof(Orchestrations.ParentWorkflowNestedActivityFail))] - [KnownType(typeof(Orchestrations.ChildWorkflowNestedActivityFail))] - [KnownType(typeof(Activities.HelloFailNestedSuborchestration))] - public class GrandparentWorkflowNestedActivityFail : TaskOrchestration - { - public static string Result; - public override async Task RunTask(OrchestrationContext context, bool waitForCompletion) - { - var results = new Task[2]; - for (int i = 0; i < 2; i++) + // we now need to yield briefly so that the delay signal is sent before the others + await ctx.CreateTimer(ctx.CurrentUtcDateTime + TimeSpan.FromMilliseconds(1), CancellationToken.None); + + // now send the signals in the batch. These should all arrive and get queued (inside the storage provider) + // while the entity is executing the delay operation. Therefore, after the delay operation finishes, + // all of the signals are processed in a single batch. + sendSignals(); + } + + async Task TestRollbackInBatch1() { - Task r = context.CreateSubOrchestrationInstance(typeof(Orchestrations.ParentWorkflowNestedActivityFail), i); - if (waitForCompletion) + await ProcessAllSignalsInSingleBatch(() => { - await r; + ctx.SignalEntity(entityId, "Set", 42); + ctx.SignalEntity(entityId, "SetThenThrow", 333); + ctx.SignalEntity(entityId, "DeleteThenThrow"); + }); + + if (rollbackOnException) + { + Assert.AreEqual(42, await Get()); + } + else + { + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); + ctx.SignalEntity(entityId, "Set", 42); } - results[i] = r; } - string[] data = await Task.WhenAll(results); - Result = string.Concat(data); - return Result; - } - } + async Task TestRollbackInBatch2() + { + await ProcessAllSignalsInSingleBatch(() => + { + ctx.SignalEntity(entityId, "Get"); + ctx.SignalEntity(entityId, "Set", 42); + ctx.SignalEntity(entityId, "Delete"); + ctx.SignalEntity(entityId, "Set", 43); + ctx.SignalEntity(entityId, "DeleteThenThrow"); + }); + + if (rollbackOnException) + { + Assert.AreEqual(43, await Get()); + } + else + { + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); + } + } - internal class Counter : TaskOrchestration - { - TaskCompletionSource waitForOperationHandle; + async Task TestRollbackInBatch3() + { + Task getTask = null; - public override async Task RunTask(OrchestrationContext context, int currentValue) - { - string operation = await this.WaitForOperation(); + await ProcessAllSignalsInSingleBatch(() => + { + ctx.SignalEntity(entityId, "Set", 55); + ctx.SignalEntity(entityId, "SetToUnserializable"); + getTask = ctx.CallEntityAsync(entityId, "Get"); + }); - bool done = false; - switch (operation?.ToLowerInvariant()) - { - case "incr": - currentValue++; - break; - case "decr": - currentValue--; - break; - case "end": - done = true; - break; + if (rollbackOnException) + { + Assert.AreEqual(55, await getTask); + await Delete(); + } + else + { + await ExpectException(getTask); + await ctx.CallEntityAsync(entityId, "deletewithoutreading"); + } + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); } - if (!done) + async Task TestRollbackInBatch4() { - context.ContinueAsNew(currentValue); - } + Task getTask = null; + + await ProcessAllSignalsInSingleBatch(() => + { + ctx.SignalEntity(entityId, "Set", 56); + ctx.SignalEntity(entityId, "SetToUnDeserializable"); + ctx.SignalEntity(entityId, "SetThenThrow", 999); + getTask = ctx.CallEntityAsync(entityId, "Get"); + }); - return currentValue; + if (rollbackOnException) + { + // we rolled back to an un-deserializable state + await ExpectException(getTask); + } + else + { + // we don't roll back the 999 when throwing + Assert.AreEqual(999, await getTask); + } - } + await ctx.CallEntityAsync(entityId, "deletewithoutreading"); + } - async Task WaitForOperation() - { - this.waitForOperationHandle = new TaskCompletionSource(); - string operation = await this.waitForOperationHandle.Task; - this.waitForOperationHandle = null; - return operation; - } - public override void OnEvent(OrchestrationContext context, string name, string input) - { - Assert.AreEqual("operation", name, true, "Unknown signal recieved..."); - if (this.waitForOperationHandle != null) + async Task TestRollbackInBatch5() + { + await ProcessAllSignalsInSingleBatch(() => + { + ctx.SignalEntity(entityId, "Set", 1); + ctx.SignalEntity(entityId, "Delete"); + ctx.SignalEntity(entityId, "Set", 2); + ctx.SignalEntity(entityId, "Delete"); + ctx.SignalEntity(entityId, "SetThenThrow", 3); + }); + + if (rollbackOnException) + { + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); + } + else + { + Assert.AreEqual(3, await Get()); // not rolled back + } + } + + try { - this.waitForOperationHandle.SetResult(input); + await TestRollbackToNonexistent(); + await TestNondeserializableState(); + await TestSecondOperationRollback(); + await TestDeleteOperationRollback(); + await TestFirstOperationRollback(); + + await TestRollbackInBatch1(); + await TestRollbackInBatch2(); + await TestRollbackInBatch3(); + await TestRollbackInBatch4(); + await TestRollbackInBatch5(); + + return "ok"; + } + catch (Exception e) + { + return e.ToString(); } } } @@ -3074,6 +4230,342 @@ public override Task RunTask(OrchestrationContext context, string inpu } } + static class Entities + { + internal class Counter : TaskEntity + { + public override int CreateInitialState(EntityContext ctx) + { + return 0; + } + + public override ValueTask ExecuteOperationAsync(EntityContext ctx) + { + switch (ctx.OperationName) + { + case "get": + ctx.Return(ctx.State); + break; + + case "increment": + ctx.State++; + ctx.Return(ctx.State); + break; + + case "add": + ctx.State += ctx.GetInput(); + ctx.Return(ctx.State); + break; + + case "set": + ctx.State = ctx.GetInput(); + break; + + case "delete": + ctx.DeleteState(); + break; + } + + return default; + } + } + + //-------------- An entity that forwards a signal ----------------- + + internal class Relay : TaskEntity + { + public override ValueTask ExecuteOperationAsync(EntityContext context) + { + var (destination, operation) = context.GetInput<(EntityId, string)>(); + + context.SignalEntity(destination, operation); + + return default; + } + } + + // -------------- An entity that records all batch positions and batch sizes ----------------- + internal class BatchEntity : TaskEntity> + { + public override List<(int, int)> CreateInitialState(EntityContext> context) + => new List<(int, int)>(); + + public override ValueTask ExecuteOperationAsync(EntityContext> context) + { + context.State.Add((context.BatchPosition, context.BatchSize)); + return default; + } + } + + //-------------- a very simple entity that stores a string ----------------- + // it offers two operations: + // "set" (takes a string, assigns it to the current state, does not return anything) + // "get" (returns a string containing the current state)// An entity that records all batch positions and batch sizes + internal class StringStore : TaskEntity + { + public override ValueTask ExecuteOperationAsync(EntityContext context) + { + switch (context.OperationName) + { + case "set": + context.State = context.GetInput(); + break; + + case "get": + context.Return(context.State); + break; + + default: + throw new NotImplementedException("no such operation"); + } + return default; + } + } + + //-------------- a slightly less trivial version of the same ----------------- + // as before with two differences: + // - "get" throws an exception if the entity does not already exist, i.e. state was not set to anything + // - a new operation "delete" deletes the entity, i.e. clears all state + internal class StringStore2 : TaskEntity + { + public override ValueTask ExecuteOperationAsync(EntityContext context) + { + switch (context.OperationName) + { + case "delete": + context.DeleteState(); + break; + + case "set": + context.State = context.GetInput(); + break; + + case "get": + if (!context.HasState) + { + throw new InvalidOperationException("this entity does not like 'get' when it does not have state yet"); + } + + context.Return(context.State); + break; + + default: + throw new NotImplementedException("no such operation"); + } + return default; + } + } + + //-------------- An entity that launches an orchestration ----------------- + + internal class Launcher : TaskEntity + { + public class State + { + public string Id; + public bool Done; + } + + public override State CreateInitialState(EntityContext context) => new State(); + + public override ValueTask ExecuteOperationAsync(EntityContext context) + { + switch (context.OperationName) + { + case "launch": + { + context.State.Id = context.StartNewOrchestration(typeof(Orchestrations.DelayedSignal), input: context.EntityId); + break; + } + + case "done": + { + context.State.Done = true; + break; + } + + case "get": + { + context.Return(context.State.Done ? context.State.Id : null); + break; + } + + default: + throw new NotImplementedException("no such entity operation"); + } + + return default; + } + } + + //-------------- an entity that is designed to throw certain exceptions during operations, serialization, or deserialization ----------------- + + internal class FaultyEntityWithRollback : FaultyEntity + { + public override bool RollbackOnExceptions => true; + public override FaultyEntity CreateInitialState(EntityContext context) => new FaultyEntityWithRollback(); + } + + internal class FaultyEntityWithoutRollback : FaultyEntity + { + public override bool RollbackOnExceptions => false; + public override FaultyEntity CreateInitialState(EntityContext context) => new FaultyEntityWithoutRollback(); + } + + [JsonObject(MemberSerialization.OptIn)] + internal abstract class FaultyEntity : TaskEntity + { + [JsonProperty] + public int Value { get; set; } + + [JsonProperty()] + [JsonConverter(typeof(CustomJsonConverter))] + public object ObjectWithFaultySerialization { get; set; } + + [JsonProperty] + public int NumberIncrementsSent { get; set; } + + public override DataConverter ErrorDataConverter => new JsonDataConverter(new JsonSerializerSettings + { + TypeNameHandling = TypeNameHandling.Objects, + ReferenceLoopHandling = ReferenceLoopHandling.Ignore, + }); + + public override ValueTask ExecuteOperationAsync(EntityContext context) + { + switch (context.OperationName) + { + case "exists": + context.Return(context.HasState); + break; + + case "deletewithoutreading": + context.DeleteState(); + break; + + case "Get": + if (!context.HasState) + { + context.Return(0); + } + else + { + context.Return(context.State.Value); + } + break; + + case "GetNumberIncrementsSent": + context.Return(context.State.NumberIncrementsSent); + break; + + case "Set": + context.State.Value = context.GetInput(); + break; + + case "SetToUnserializable": + context.State.ObjectWithFaultySerialization = new UnserializableKaboom(); + break; + + case "SetToUnDeserializable": + context.State.ObjectWithFaultySerialization = new UnDeserializableKaboom(); + break; + + case "SetThenThrow": + context.State.Value = context.GetInput(); + throw new FaultyEntity.SerializableKaboom(); + + case "Send": + Send(); + break; + + case "SendThenThrow": + Send(); + throw new FaultyEntity.SerializableKaboom(); + + case "SendThenMakeUnserializable": + Send(); + context.State.ObjectWithFaultySerialization = new UnserializableKaboom(); + break; + + case "Delete": + context.DeleteState(); + break; + + case "DeleteThenThrow": + context.DeleteState(); + throw new FaultyEntity.SerializableKaboom(); + + case "Throw": + throw new FaultyEntity.SerializableKaboom(); + + case "ThrowUnserializable": + throw new FaultyEntity.UnserializableKaboom(); + + case "ThrowUnDeserializable": + throw new FaultyEntity.UnDeserializableKaboom(); + } + return default; + + void Send() + { + var desc = $"{++context.State.NumberIncrementsSent}:{context.State.Value}"; + context.SignalEntity(context.GetInput(), desc); + } + } + + public class CustomJsonConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(SerializableKaboom) + || objectType == typeof(UnserializableKaboom) + || objectType == typeof(UnDeserializableKaboom); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.Null) + { + return null; + } + + var typename = serializer.Deserialize(reader); + + if (typename != nameof(SerializableKaboom)) + { + throw new JsonSerializationException("purposefully designed to not be deserializable"); + } + + return new SerializableKaboom(); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + if (value is UnserializableKaboom) + { + throw new JsonSerializationException("purposefully designed to not be serializable"); + } + + serializer.Serialize(writer, value.GetType().Name); + } + } + + public class UnserializableKaboom : Exception + { + } + + public class SerializableKaboom : Exception + { + } + + public class UnDeserializableKaboom : Exception + { + } + } + + } + + static class Activities { internal class HelloFailActivity : TaskActivity diff --git a/test/DurableTask.AzureStorage.Tests/TestHelpers.cs b/test/DurableTask.AzureStorage.Tests/TestHelpers.cs index 37073cbb3..d4e2e8b05 100644 --- a/test/DurableTask.AzureStorage.Tests/TestHelpers.cs +++ b/test/DurableTask.AzureStorage.Tests/TestHelpers.cs @@ -18,6 +18,7 @@ namespace DurableTask.AzureStorage.Tests using System.Diagnostics; using System.Diagnostics.Tracing; using System.Threading.Tasks; + using DurableTask.Core; using DurableTask.Core.Logging; using Microsoft.Extensions.Logging; @@ -28,6 +29,7 @@ public static TestOrchestrationHost GetTestOrchestrationHost( bool enableExtendedSessions, int extendedSessionTimeoutInSeconds = 30, bool fetchLargeMessages = true, + ErrorPropagationMode errorPropagationMode = ErrorPropagationMode.SerializeExceptions, Action? modifySettingsAction = null) { string storageConnectionString = GetTestStorageAccountConnectionString(); @@ -48,7 +50,7 @@ public static TestOrchestrationHost GetTestOrchestrationHost( // Give the caller a chance to make test-specific changes to the settings modifySettingsAction?.Invoke(settings); - return new TestOrchestrationHost(settings); + return new TestOrchestrationHost(settings, errorPropagationMode); } public static string GetTestStorageAccountConnectionString() diff --git a/test/DurableTask.AzureStorage.Tests/TestOrchestrationClient.cs b/test/DurableTask.AzureStorage.Tests/TestOrchestrationClient.cs index 92d018eed..236d0a231 100644 --- a/test/DurableTask.AzureStorage.Tests/TestOrchestrationClient.cs +++ b/test/DurableTask.AzureStorage.Tests/TestOrchestrationClient.cs @@ -45,6 +45,8 @@ public TestOrchestrationClient( public string InstanceId => this.instanceId; + public TaskHubClient InnerClient => client; + public async Task WaitForCompletionAsync(TimeSpan timeout) { timeout = AdjustTimeout(timeout); diff --git a/test/DurableTask.AzureStorage.Tests/TestOrchestrationHost.cs b/test/DurableTask.AzureStorage.Tests/TestOrchestrationHost.cs index b88e0dfcf..e9813c4ea 100644 --- a/test/DurableTask.AzureStorage.Tests/TestOrchestrationHost.cs +++ b/test/DurableTask.AzureStorage.Tests/TestOrchestrationHost.cs @@ -20,6 +20,7 @@ namespace DurableTask.AzureStorage.Tests using System.Runtime.Serialization; using System.Threading.Tasks; using DurableTask.Core; + using DurableTask.Core.Entities; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -32,8 +33,9 @@ internal sealed class TestOrchestrationHost : IDisposable readonly TaskHubClient client; readonly HashSet addedOrchestrationTypes; readonly HashSet addedActivityTypes; + readonly HashSet addedEntityTypes; - public TestOrchestrationHost(AzureStorageOrchestrationServiceSettings settings) + public TestOrchestrationHost(AzureStorageOrchestrationServiceSettings settings, ErrorPropagationMode errorPropagationMode = ErrorPropagationMode.SerializeExceptions) { this.service = new AzureStorageOrchestrationService(settings); this.service.CreateAsync().GetAwaiter().GetResult(); @@ -43,6 +45,9 @@ public TestOrchestrationHost(AzureStorageOrchestrationServiceSettings settings) this.client = new TaskHubClient(service, loggerFactory: settings.LoggerFactory); this.addedOrchestrationTypes = new HashSet(); this.addedActivityTypes = new HashSet(); + this.addedEntityTypes = new HashSet(); + + worker.ErrorPropagationMode = errorPropagationMode; } public string TaskHub => this.settings.TaskHubName; @@ -86,7 +91,7 @@ public async Task StartOrchestrationAsync( this.addedOrchestrationTypes.Add(orchestrationType); } - // Allow orchestration types to declare which activity types they depend on. + // Allow orchestration types to declare which activity and entity types they depend on. // CONSIDER: Make this a supported pattern in DTFx? KnownTypeAttribute[] knownTypes = (KnownTypeAttribute[])orchestrationType.GetCustomAttributes(typeof(KnownTypeAttribute), false); @@ -95,6 +100,7 @@ public async Task StartOrchestrationAsync( { bool orch = referencedKnownType.Type.IsSubclassOf(typeof(TaskOrchestration)); bool activ = referencedKnownType.Type.IsSubclassOf(typeof(TaskActivity)); + bool entit = referencedKnownType.Type.IsSubclassOf(typeof(Core.Entities.TaskEntity)); if (orch && !this.addedOrchestrationTypes.Contains(referencedKnownType.Type)) { this.worker.AddTaskOrchestrations(referencedKnownType.Type); @@ -106,6 +112,12 @@ public async Task StartOrchestrationAsync( this.worker.AddTaskActivities(referencedKnownType.Type); this.addedActivityTypes.Add(referencedKnownType.Type); } + + else if (entit && !this.addedEntityTypes.Contains(referencedKnownType.Type)) + { + this.worker.AddTaskEntities(referencedKnownType.Type); + this.addedEntityTypes.Add(referencedKnownType.Type); + } } DateTime creationTime = DateTime.UtcNow; @@ -139,6 +151,17 @@ public async Task StartOrchestrationAsync( return new TestOrchestrationClient(this.client, orchestrationType, instance.InstanceId, creationTime); } + public Task GetEntityClientAsync(Type entityType, EntityId entityId) + { + if (!this.addedEntityTypes.Contains(entityType)) + { + this.worker.AddTaskEntities(entityType); + this.addedEntityTypes.Add(entityType); + } + + return Task.FromResult(new TestEntityClient(new TaskHubEntityClient(this.client), entityId)); + } + public Task> StartInlineOrchestration( TInput input, string orchestrationName, From 5b625f862e0d1eaae9f74980317c068cb3f2104e Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Tue, 21 Mar 2023 08:36:43 -0700 Subject: [PATCH 2/8] address PR feedback. --- .../Entities/ClientEntityContext.cs | 14 ++-- src/DurableTask.Core/Entities/EntityId.cs | 64 ++++++++----------- .../Entities/EntityWorkItemProcessor.cs | 4 +- .../Entities/EventFormat/ReleaseMessage.cs | 6 +- .../Entities/EventFormat/RequestMessage.cs | 26 ++++---- .../Entities/EventFormat/ResponseMessage.cs | 12 ++-- src/DurableTask.Core/Entities/EventToSend.cs | 55 ++++++++++++++++ .../Entities/LocalSDK/TaskEntityContext.cs | 2 +- .../Entities/LocalSDK/TaskHubEntityClient.cs | 39 +++++------ .../Entities/OrchestrationEntityContext.cs | 24 +++---- .../Entities/StateFormat/EntityStatus.cs | 8 ++- .../Entities/StateFormat/MessageSorter.cs | 19 +++--- .../Entities/StateFormat/SchedulerState.cs | 17 ++--- src/DurableTask.Core/Entities/TaskEntity.cs | 2 +- .../OrchestratorExecutionResult.cs | 6 +- .../TaskOrchestrationContext.cs | 14 ++-- 16 files changed, 184 insertions(+), 128 deletions(-) create mode 100644 src/DurableTask.Core/Entities/EventToSend.cs diff --git a/src/DurableTask.Core/Entities/ClientEntityContext.cs b/src/DurableTask.Core/Entities/ClientEntityContext.cs index c92647c37..077aab944 100644 --- a/src/DurableTask.Core/Entities/ClientEntityContext.cs +++ b/src/DurableTask.Core/Entities/ClientEntityContext.cs @@ -27,12 +27,13 @@ public static class ClientEntityContext /// /// Create an event to represent an entity signal. /// + /// The target instance. /// A unique identifier for the request. /// The name of the operation. /// The serialized input for the operation. /// The time to schedule this signal, or null if not a scheduled signal - /// - public static (string eventName, object eventContent) EmitOperationSignal(Guid requestId, string operationName, string input, (DateTime original, DateTime capped)? scheduledTimeUtc) + /// The event to send. + public static EventToSend EmitOperationSignal(OrchestrationInstance targetInstance, Guid requestId, string operationName, string input, (DateTime original, DateTime capped)? scheduledTimeUtc) { var request = new RequestMessage() { @@ -51,15 +52,16 @@ public static (string eventName, object eventContent) EmitOperationSignal(Guid r ? EntityMessageEventNames.ScheduledRequestMessageEventName(scheduledTimeUtc.Value.capped) : EntityMessageEventNames.RequestMessageEventName; - return (eventName, jrequest); + return new EventToSend(eventName, jrequest, targetInstance); } /// /// Create an event to represent an entity unlock, which is called by clients to fix orphaned locks. /// + /// The target instance. /// The instance id of the entity to be unlocked. - /// - public static (string eventName, object eventContent) EmitUnlockForOrphanedLock(string lockOwnerInstanceId) + /// The event to send. + public static EventToSend EmitUnlockForOrphanedLock(OrchestrationInstance targetInstance, string lockOwnerInstanceId) { var message = new ReleaseMessage() { @@ -69,7 +71,7 @@ public static (string eventName, object eventContent) EmitUnlockForOrphanedLock( var jmessage = JToken.FromObject(message, Serializer.InternalSerializer); - return (EntityMessageEventNames.ReleaseMessageEventName, jmessage); + return new EventToSend(EntityMessageEventNames.ReleaseMessageEventName, jmessage, targetInstance); } /// diff --git a/src/DurableTask.Core/Entities/EntityId.cs b/src/DurableTask.Core/Entities/EntityId.cs index 4b7258278..1f4e18f12 100644 --- a/src/DurableTask.Core/Entities/EntityId.cs +++ b/src/DurableTask.Core/Entities/EntityId.cs @@ -13,16 +13,15 @@ namespace DurableTask.Core.Entities { using System; + using System.Runtime.Serialization; using Newtonsoft.Json; /// /// A unique identifier for an entity, consisting of entity name and entity key. /// - public struct EntityId : IEquatable, IComparable + [DataContract] + public readonly struct EntityId : IEquatable, IComparable { - [JsonIgnore] - private string schedulerId; - /// /// Create an entity id for an entity. /// @@ -35,41 +34,31 @@ public EntityId(string entityName, string entityKey) throw new ArgumentNullException(nameof(entityName), "Invalid entity id: entity name must not be a null or empty string."); } - this.EntityName = entityName.ToLowerInvariant(); + this.EntityName = entityName; this.EntityKey = entityKey ?? throw new ArgumentNullException(nameof(entityKey), "Invalid entity id: entity key must not be null."); - this.schedulerId = GetSchedulerId(this.EntityName, this.EntityKey); } /// /// The name for this class of entities. /// - [JsonProperty(PropertyName = "name", Required = Required.Always)] - public string EntityName { get; private set; } // do not remove set, is needed by Json Deserializer + [DataMember(Name = "name", IsRequired = true)] + public readonly string EntityName; /// /// The entity key. Uniquely identifies an entity among all entities of the same name. /// - [JsonProperty(PropertyName = "key", Required = Required.Always)] - public string EntityKey { get; private set; } // do not remove set, is needed by Json Deserializer - - /// - /// Returns the instance ID for a given entity ID. - /// - /// The entity ID. - /// The corresponding instance ID. - public static string GetInstanceIdFromEntityId(EntityId entityId) - { - return GetSchedulerId(entityId.EntityName, entityId.EntityKey); - } + [DataMember(Name = "key", IsRequired = true)] + public readonly string EntityKey; - private static string GetSchedulerId(string entityName, string entityKey) + /// + public override string ToString() { - return $"@{entityName}@{entityKey}"; + return $"@{this.EntityName}@{this.EntityKey}"; } internal static string GetSchedulerIdPrefixFromEntityName(string entityName) { - return $"@{entityName.ToLowerInvariant()}@"; + return $"@{entityName}@"; } /// @@ -77,26 +66,23 @@ internal static string GetSchedulerIdPrefixFromEntityName(string entityName) /// /// The instance ID. /// the corresponding entity ID. - public static EntityId GetEntityIdFromInstanceId(string instanceId) + public static EntityId FromString(string instanceId) { + if (instanceId == null) + { + throw new ArgumentNullException(nameof(instanceId)); + } var pos = instanceId.IndexOf('@', 1); + if (pos <= 0) + { + throw new ArgumentException("instanceId is not a valid entityId", nameof(instanceId)); + } var entityName = instanceId.Substring(1, pos - 1); var entityKey = instanceId.Substring(pos + 1); return new EntityId(entityName, entityKey); } - /// - public override string ToString() - { - // The scheduler id could be null if the object was deserialized. - if (this.schedulerId == null) - { - this.schedulerId = GetInstanceIdFromEntityId(this); - } - - return this.schedulerId; - } - + /// public override bool Equals(object obj) { @@ -106,20 +92,20 @@ public override bool Equals(object obj) /// public bool Equals(EntityId other) { - return this.ToString().Equals(other.ToString()); + return (this.EntityName,this.EntityKey).Equals((other.EntityName, other.EntityKey)); } /// public override int GetHashCode() { - return this.ToString().GetHashCode(); + return (this.EntityName, this.EntityKey).GetHashCode(); } /// public int CompareTo(object obj) { var other = (EntityId)obj; - return this.ToString().CompareTo(other.ToString()); + return (this.EntityName, this.EntityKey).CompareTo((other.EntityName, other.EntityKey)); } } } diff --git a/src/DurableTask.Core/Entities/EntityWorkItemProcessor.cs b/src/DurableTask.Core/Entities/EntityWorkItemProcessor.cs index f1d698f5a..a3cf3a47d 100644 --- a/src/DurableTask.Core/Entities/EntityWorkItemProcessor.cs +++ b/src/DurableTask.Core/Entities/EntityWorkItemProcessor.cs @@ -174,7 +174,7 @@ void ProcessLockRequest(RequestMessage request) if (request.Position < request.LockSet.Length) { // send lock request to next entity in the lock set - var target = new OrchestrationInstance() { InstanceId = EntityId.GetInstanceIdFromEntityId(request.LockSet[request.Position]) }; + var target = new OrchestrationInstance() { InstanceId = request.LockSet[request.Position].ToString() }; this.SendLockRequestMessage(target, request); } else @@ -586,7 +586,7 @@ async Task ExecuteViaMiddlewareAsync(Work workToDoNow) this.logHelper.EntityBatchExecuting(request); - string entityName = EntityId.GetEntityIdFromInstanceId(this.instanceId).EntityName; + string entityName = EntityId.FromString(this.instanceId).EntityName; string entityVersion = string.Empty; // TODO consider whether we should support explicit versions // Get the TaskOrchestration implementation. If it's not found, it either means that the developer never diff --git a/src/DurableTask.Core/Entities/EventFormat/ReleaseMessage.cs b/src/DurableTask.Core/Entities/EventFormat/ReleaseMessage.cs index db54b6a98..55c538f50 100644 --- a/src/DurableTask.Core/Entities/EventFormat/ReleaseMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/ReleaseMessage.cs @@ -12,14 +12,16 @@ // ---------------------------------------------------------------------------------- namespace DurableTask.Core.Entities.EventFormat { + using System.Runtime.Serialization; using Newtonsoft.Json; + [DataContract] internal class ReleaseMessage { - [JsonProperty(PropertyName = "parent")] + [DataMember(Name = "parent")] public string ParentInstanceId { get; set; } - [JsonProperty(PropertyName = "id")] + [DataMember(Name = "id")] public string Id { get; set; } public override string ToString() diff --git a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs index 23d297e9b..d87d0de8a 100644 --- a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs @@ -13,88 +13,90 @@ namespace DurableTask.Core.Entities.EventFormat { using System; + using System.Runtime.Serialization; using Newtonsoft.Json; using Newtonsoft.Json.Linq; /// /// A message sent to an entity, such as operation, signal, lock, or continue messages. /// + [DataContract] internal class RequestMessage { /// /// The name of the operation being called (if this is an operation message) or null /// (if this is a lock request). /// - [JsonProperty(PropertyName = "op")] + [DataMember(Name = "op")] public string Operation { get; set; } /// /// Whether or not this is a one-way message. /// - [JsonProperty(PropertyName = "signal", DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(Name = "signal", EmitDefaultValue = false)] public bool IsSignal { get; set; } /// /// The operation input. /// - [JsonProperty(PropertyName = "input", DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(Name = "input", EmitDefaultValue = false)] public string Input { get; set; } /// /// A unique identifier for this operation. /// - [JsonProperty(PropertyName = "id", Required = Required.Always)] + [DataMember(Name = "id", IsRequired = true)] public Guid Id { get; set; } /// /// The parent instance that called this operation. /// - [JsonProperty(PropertyName = "parent", DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(Name = "parent", EmitDefaultValue = false)] public string ParentInstanceId { get; set; } /// /// The parent instance that called this operation. /// - [JsonProperty(PropertyName = "parentExecution", DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(Name = "parentExecution", EmitDefaultValue = false)] public string ParentExecutionId { get; set; } /// /// Optionally, a scheduled time at which to start the operation. /// - [JsonProperty(PropertyName = "due", DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(Name = "due", EmitDefaultValue = false)] public DateTime? ScheduledTime { get; set; } /// /// A timestamp for this request. /// Used for duplicate filtering and in-order delivery. /// - [JsonProperty] + [DataMember] public DateTime Timestamp { get; set; } /// /// A timestamp for the predecessor request in the stream, or DateTime.MinValue if none. /// Used for duplicate filtering and in-order delivery. /// - [JsonProperty] + [DataMember] public DateTime Predecessor { get; set; } /// /// For lock requests, the set of locks being acquired. Is sorted, /// contains at least one element, and has no repetitions. /// - [JsonProperty(PropertyName = "lockset", DefaultValueHandling = DefaultValueHandling.Ignore, TypeNameHandling = TypeNameHandling.None)] + [DataMember(Name = "lockset", EmitDefaultValue = false)] public EntityId[] LockSet { get; set; } /// /// For lock requests involving multiple locks, the message number. /// - [JsonProperty(PropertyName = "pos", DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(Name = "pos", EmitDefaultValue = false)] public int Position { get; set; } /// /// whether this message is a lock request /// - [JsonIgnore] + [DataMember] public bool IsLockRequest => LockSet != null; /// diff --git a/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs index 1588bba60..85b1ccfc3 100644 --- a/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs @@ -13,21 +13,23 @@ namespace DurableTask.Core.Entities.EventFormat { using System; + using System.Runtime.Serialization; using Newtonsoft.Json; using Newtonsoft.Json.Linq; - + + [DataContract] internal class ResponseMessage { - [JsonProperty(PropertyName = "result")] + [DataMember(Name = "result")] public string Result { get; set; } - [JsonProperty(PropertyName = "exceptionType", DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(Name = "exceptionType", EmitDefaultValue = false)] public string ErrorMessage { get; set; } - [JsonProperty(PropertyName = "failureDetails", DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(Name = "failureDetails", EmitDefaultValue = false)] public FailureDetails FailureDetails { get; set; } - [JsonIgnore] + [IgnoreDataMember] public bool IsErrorResult => this.ErrorMessage != null; public override string ToString() diff --git a/src/DurableTask.Core/Entities/EventToSend.cs b/src/DurableTask.Core/Entities/EventToSend.cs new file mode 100644 index 000000000..2a567fb7a --- /dev/null +++ b/src/DurableTask.Core/Entities/EventToSend.cs @@ -0,0 +1,55 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +namespace DurableTask.Core.Entities +{ + using DurableTask.Core.Entities.EventFormat; + using DurableTask.Core.Entities.OperationFormat; + using DurableTask.Core.Entities.StateFormat; + using Newtonsoft.Json.Linq; + using Newtonsoft.Json; + using System; + + /// + /// The data associated with sending an event to an orchestration. + /// + public readonly struct EventToSend + { + /// + /// The name of the event. + /// + public readonly string EventName; + + /// + /// The content of the event. + /// + public readonly object EventContent; + + /// + /// The target instance for the event. + /// + public readonly OrchestrationInstance TargetInstance; + + /// + /// Construct an entity message event with the given members. + /// + /// The name of the event. + /// The content of the event. + /// The target of the event. + public EventToSend(string name, object content, OrchestrationInstance target) + { + EventName = name; + EventContent = content; + TargetInstance = target; + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/LocalSDK/TaskEntityContext.cs b/src/DurableTask.Core/Entities/LocalSDK/TaskEntityContext.cs index 68780d4f8..ef0f5a300 100644 --- a/src/DurableTask.Core/Entities/LocalSDK/TaskEntityContext.cs +++ b/src/DurableTask.Core/Entities/LocalSDK/TaskEntityContext.cs @@ -258,7 +258,7 @@ private void SignalEntityInternal(EntityId entity, DateTime? scheduledTimeUtc, s var action = new SendSignalOperationAction() { - InstanceId = EntityId.GetInstanceIdFromEntityId(entity), + InstanceId = entity.ToString(), Name = operationName, ScheduledTime = scheduledTimeUtc, Input = null, diff --git a/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs b/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs index 01da29f08..6f2abbd48 100644 --- a/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs +++ b/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs @@ -101,7 +101,7 @@ public async Task SignalEntityAsync(EntityId entityId, string operationName, obj } var guid = Guid.NewGuid(); // unique id for this request - var instanceId = EntityId.GetInstanceIdFromEntityId(entityId); + var instanceId = entityId.ToString(); var instance = new OrchestrationInstance() { InstanceId = instanceId }; string serializedInput = null; @@ -110,17 +110,18 @@ public async Task SignalEntityAsync(EntityId entityId, string operationName, obj serializedInput = this.messageDataConverter.Serialize(operationInput); } - (string name, object content) eventToSend = ClientEntityContext.EmitOperationSignal( - guid, - operationName, - serializedInput, - scheduledTime); + EventToSend eventToSend = ClientEntityContext.EmitOperationSignal( + instance, + guid, + operationName, + serializedInput, + scheduledTime); - string serializedEventContent = this.messageDataConverter.Serialize(eventToSend.content); + string serializedEventContent = this.messageDataConverter.Serialize(eventToSend.EventContent); var eventRaisedEvent = new EventRaisedEvent(-1, serializedEventContent) { - Name = eventToSend.name + Name = eventToSend.EventName }; var taskMessage = new TaskMessage @@ -142,7 +143,7 @@ public async Task SignalEntityAsync(EntityId entityId, string operationName, obj /// a response containing the current state of the entity. public async Task> ReadEntityStateAsync(EntityId entityId) { - var instanceId = EntityId.GetInstanceIdFromEntityId(entityId); + var instanceId = entityId.ToString(); this.logHelper.FetchingInstanceState(instanceId); IList stateList = await this.ServiceClient.GetOrchestrationStateAsync(instanceId, allExecutions:false); @@ -287,7 +288,7 @@ EntityStatus ConvertStatusResult(OrchestrationState orchestrationState) { return new EntityStatus() { - EntityId = EntityId.GetEntityIdFromInstanceId(orchestrationState.OrchestrationInstance.InstanceId), + EntityId = EntityId.FromString(orchestrationState.OrchestrationInstance.InstanceId), LastOperationTime = orchestrationState.CreatedTime, State = state, }; @@ -482,21 +483,21 @@ async Task CheckForOrphanedLockAndFixIt(string instanceId, string lockOwner) if (!lockOwnerIsStillRunning) { // the owner is not a running orchestration. Send a lock release. - (string name, object content) eventToSend = ClientEntityContext.EmitUnlockForOrphanedLock(lockOwner); - - string serializedEventContent = this.messageDataConverter.Serialize(eventToSend.content); - - var eventRaisedEvent = new EventRaisedEvent(-1, serializedEventContent) + OrchestrationInstance targetInstance = new OrchestrationInstance() { - Name = eventToSend.name + InstanceId = instanceId, }; - OrchestrationInstance targetInstance = new OrchestrationInstance() + var eventToSend = ClientEntityContext.EmitUnlockForOrphanedLock(targetInstance, lockOwner); + + string serializedEventContent = this.messageDataConverter.Serialize(eventToSend.EventContent); + + var eventRaisedEvent = new EventRaisedEvent(-1, serializedEventContent) { - InstanceId = instanceId, + Name = eventToSend.EventName }; - var taskMessage = new TaskMessage + var taskMessage = new TaskMessage { OrchestrationInstance = targetInstance, Event = eventRaisedEvent, diff --git a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs index 11a2447a5..40096d6fb 100644 --- a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs +++ b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs @@ -110,7 +110,7 @@ public bool ValidateOperationTransition(string targetInstanceId, bool oneWay, ou { if (this.IsInsideCriticalSection) { - var lockToUse = EntityId.GetEntityIdFromInstanceId(targetInstanceId); + var lockToUse = EntityId.FromString(targetInstanceId); if (oneWay) { if (this.criticalSectionLocks.Contains(lockToUse)) @@ -171,7 +171,7 @@ public void RecoverLockAfterCall(string targetInstanceId) { if (this.IsInsideCriticalSection) { - var lockToUse = EntityId.GetEntityIdFromInstanceId(targetInstanceId); + var lockToUse = EntityId.FromString(targetInstanceId); this.availableLocks.Add(lockToUse); } } @@ -179,7 +179,7 @@ public void RecoverLockAfterCall(string targetInstanceId) /// /// Get release messages for all locks in the critical section, and release them /// - public IEnumerable<(OrchestrationInstance target, string eventName, object eventContent)> EmitLockReleaseMessages() + public IEnumerable EmitLockReleaseMessages() { if (this.IsInsideCriticalSection) { @@ -191,9 +191,9 @@ public void RecoverLockAfterCall(string targetInstanceId) foreach (var entityId in this.criticalSectionLocks) { - var instance = new OrchestrationInstance() { InstanceId = EntityId.GetInstanceIdFromEntityId(entityId) }; + var instance = new OrchestrationInstance() { InstanceId = entityId.ToString() }; var jmessage = JObject.FromObject(message, Serializer.InternalSerializer); - yield return (instance, EntityMessageEventNames.ReleaseMessageEventName, jmessage); + yield return new EventToSend(EntityMessageEventNames.ReleaseMessageEventName, jmessage, instance); } this.criticalSectionLocks = null; @@ -211,8 +211,8 @@ public void RecoverLockAfterCall(string targetInstanceId) /// A unique identifier for this request. /// A time for which to schedule the delivery, or null if this is not a scheduled message /// The operation input - /// - public (string eventName, object eventContent) EmitRequestMessage( + /// The event to send. + public EventToSend EmitRequestMessage( OrchestrationInstance target, string operationName, bool oneWay, @@ -236,7 +236,7 @@ public void RecoverLockAfterCall(string targetInstanceId) // we pre-serialize to JObject so we can avoid exposure to application-specific serialization settings var jrequest = JObject.FromObject(request, Serializer.InternalSerializer); - return (eventName, jrequest); + return new EventToSend(eventName, jrequest, target); } /// @@ -244,8 +244,8 @@ public void RecoverLockAfterCall(string targetInstanceId) /// /// A unique request id. /// All the entities that are to be acquired. - /// - public (OrchestrationInstance target, string eventName, object eventContent) EmitAcquireMessage(Guid lockRequestId, EntityId[] entities) + /// The event to send. + public EventToSend EmitAcquireMessage(Guid lockRequestId, EntityId[] entities) { // All the entities in entity[] need to be locked, but to avoid deadlock, the locks have to be acquired // sequentially, in order. So, we send the lock request to the first entity; when the first lock @@ -266,7 +266,7 @@ public void RecoverLockAfterCall(string targetInstanceId) } // send lock request to first entity in the lock set - var target = new OrchestrationInstance() { InstanceId = EntityId.GetInstanceIdFromEntityId(entities[0]) }; + var target = new OrchestrationInstance() { InstanceId = entities[0].ToString() }; var request = new RequestMessage() { Id = lockRequestId, @@ -285,7 +285,7 @@ public void RecoverLockAfterCall(string targetInstanceId) // we pre-serialize to JObject so we can avoid exposure to application-specific serialization settings var jrequest = JObject.FromObject(request, Serializer.InternalSerializer); - return (target, eventName, jrequest); + return new EventToSend(eventName, jrequest, target); } /// diff --git a/src/DurableTask.Core/Entities/StateFormat/EntityStatus.cs b/src/DurableTask.Core/Entities/StateFormat/EntityStatus.cs index 9118c9d4f..19c74dd3c 100644 --- a/src/DurableTask.Core/Entities/StateFormat/EntityStatus.cs +++ b/src/DurableTask.Core/Entities/StateFormat/EntityStatus.cs @@ -13,30 +13,32 @@ namespace DurableTask.Core.Entities.StateFormat { + using System.Runtime.Serialization; using Newtonsoft.Json; /// /// Information about the current status of an entity. Excludes potentially large data /// (such as the entity state, or the contents of the queue) so it can always be read with low latency. /// + [DataContract] public class EntityStatus { /// /// Whether this entity exists or not. /// - [JsonProperty(PropertyName = "entityExists", DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(Name = "entityExists", EmitDefaultValue = false)] public bool EntityExists { get; set; } /// /// The size of the queue, i.e. the number of operations that are waiting for the current operation to complete. /// - [JsonProperty(PropertyName = "queueSize", DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(Name = "queueSize", EmitDefaultValue = false)] public int QueueSize { get; set; } /// /// The instance id of the orchestration that currently holds the lock of this entity. /// - [JsonProperty(PropertyName = "lockedBy", DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(Name = "lockedBy", EmitDefaultValue = false)] public string LockedBy { get; set; } } } diff --git a/src/DurableTask.Core/Entities/StateFormat/MessageSorter.cs b/src/DurableTask.Core/Entities/StateFormat/MessageSorter.cs index 517773ce3..1d824307f 100644 --- a/src/DurableTask.Core/Entities/StateFormat/MessageSorter.cs +++ b/src/DurableTask.Core/Entities/StateFormat/MessageSorter.cs @@ -16,6 +16,7 @@ namespace DurableTask.Core.Entities.StateFormat using System; using System.Collections.Generic; using System.Linq; + using System.Runtime.Serialization; using DurableTask.Core.Entities.EventFormat; using Newtonsoft.Json; @@ -23,27 +24,28 @@ namespace DurableTask.Core.Entities.StateFormat /// provides message ordering and deduplication of request messages (operations or lock requests) /// that are sent to entities, from other entities, or from orchestrations. /// + [DataContract] internal class MessageSorter { // don't update the reorder window too often since the garbage collection incurs some overhead. private static readonly TimeSpan MinIntervalBetweenCollections = TimeSpan.FromSeconds(10); - [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(EmitDefaultValue = false)] public Dictionary LastSentToInstance { get; set; } - [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(EmitDefaultValue = false)] public Dictionary ReceivedFromInstance { get; set; } - [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(EmitDefaultValue = false)] public DateTime ReceiveHorizon { get; set; } - [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(EmitDefaultValue = false)] public DateTime SendHorizon { get; set; } /// /// Used for testing purposes. /// - [JsonIgnore] + [IgnoreDataMember] internal int NumberBufferedRequests => ReceivedFromInstance?.Select(kvp => kvp.Value.Buffered?.Count ?? 0).Sum() ?? 0; @@ -256,15 +258,16 @@ private bool TryDeliverNextMessage(ReceiveBuffer buffer, out RequestMessage mess return false; } + [DataContract] public class ReceiveBuffer { - [JsonProperty] + [DataMember] public DateTime Last { get; set; }// last message delivered, or DateTime.Min if none - [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(EmitDefaultValue = false)] public string ExecutionId { get; set; } // execution id of last message, if any - [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(EmitDefaultValue = false)] public SortedDictionary Buffered { get; set; } } } diff --git a/src/DurableTask.Core/Entities/StateFormat/SchedulerState.cs b/src/DurableTask.Core/Entities/StateFormat/SchedulerState.cs index 86d81b84b..db7997aed 100644 --- a/src/DurableTask.Core/Entities/StateFormat/SchedulerState.cs +++ b/src/DurableTask.Core/Entities/StateFormat/SchedulerState.cs @@ -14,52 +14,53 @@ namespace DurableTask.Core.Entities.StateFormat { using System.Collections.Generic; + using System.Runtime.Serialization; using DurableTask.Core.Entities.EventFormat; using Newtonsoft.Json; /// /// The persisted state of an entity scheduler, as handed forward between ContinueAsNew instances. /// - [JsonObject(MemberSerialization.OptIn)] + [DataContract] internal class SchedulerState { /// /// Whether this entity exists or not. /// - [JsonProperty(PropertyName = "exists", DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(Name = "exists", EmitDefaultValue = false)] public bool EntityExists { get; set; } /// /// The last serialized entity state. /// - [JsonProperty(PropertyName = "state", DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(Name = "state", EmitDefaultValue = false)] public string EntityState { get; set; } /// /// The queue of waiting operations, or null if none. /// - [JsonProperty(PropertyName = "queue", DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(Name = "queue", EmitDefaultValue = false)] public Queue Queue { get; private set; } /// /// The instance id of the orchestration that currently holds the lock of this entity. /// - [JsonProperty(PropertyName = "lockedBy", DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(Name = "lockedBy", EmitDefaultValue = false)] public string LockedBy { get; set; } /// /// Whether processing on this entity is currently suspended. /// - [JsonProperty(PropertyName = "suspended", DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(Name = "suspended", EmitDefaultValue = false)] public bool Suspended { get; set; } /// /// The metadata used for reordering and deduplication of messages sent to entities. /// - [JsonProperty(PropertyName = "sorter", DefaultValueHandling = DefaultValueHandling.Ignore)] + [DataMember(Name = "sorter", EmitDefaultValue = false)] public MessageSorter MessageSorter { get; set; } = new MessageSorter(); - [JsonIgnore] + [IgnoreDataMember] public bool IsEmpty => !EntityExists && (Queue == null || Queue.Count == 0) && LockedBy == null; internal void Enqueue(RequestMessage operationMessage) diff --git a/src/DurableTask.Core/Entities/TaskEntity.cs b/src/DurableTask.Core/Entities/TaskEntity.cs index f81877a98..11e5ff19c 100644 --- a/src/DurableTask.Core/Entities/TaskEntity.cs +++ b/src/DurableTask.Core/Entities/TaskEntity.cs @@ -109,7 +109,7 @@ public override async Task ExecuteOperationBatchAsync(Oper var entityContext = new TaskEntityContext( this, - EntityId.GetEntityIdFromInstanceId(operations.InstanceId), + EntityId.FromString(operations.InstanceId), options, operations, result); diff --git a/src/DurableTask.Core/OrchestratorExecutionResult.cs b/src/DurableTask.Core/OrchestratorExecutionResult.cs index 849692e3d..97f1f72c0 100644 --- a/src/DurableTask.Core/OrchestratorExecutionResult.cs +++ b/src/DurableTask.Core/OrchestratorExecutionResult.cs @@ -15,6 +15,7 @@ namespace DurableTask.Core { using System; using System.Collections.Generic; + using System.Runtime.Serialization; using DurableTask.Core.Command; using DurableTask.Core.Common; using Newtonsoft.Json; @@ -22,18 +23,19 @@ namespace DurableTask.Core /// /// The result of an orchestration execution. /// + [DataContract] public class OrchestratorExecutionResult { /// /// The list of actions resulting from the orchestrator execution. /// - [JsonProperty("actions")] + [DataMember(Name = "actions")] public IEnumerable Actions { get; set; } = Array.Empty(); /// /// The custom status, if any, of the orchestrator. /// - [JsonProperty("customStatus")] + [DataMember(Name = "customStatus")] public string? CustomStatus { get; set; } /// diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index 6b3acfaac..f8a84519d 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -249,7 +249,7 @@ public override void SignalEntity(EntityId entityId, DateTime scheduledTimeUtc, { OrchestrationInstance target = new OrchestrationInstance() { - InstanceId = EntityId.GetInstanceIdFromEntityId(entityId), + InstanceId = entityId.ToString(), }; if (!this.EntityContext.ValidateOperationTransition(target.InstanceId, oneWay, out string errorMessage)) @@ -262,10 +262,9 @@ public override void SignalEntity(EntityId entityId, DateTime scheduledTimeUtc, string serializedInput = this.MessageDataConverter.Serialize(input); - (string name, object content) eventToSend - = this.entityContext.EmitRequestMessage(target, operationName, oneWay, operationId, scheduledTimeUtc, serializedInput); + var eventToSend = this.entityContext.EmitRequestMessage(target, operationName, oneWay, operationId, scheduledTimeUtc, serializedInput); - this.SendEvent(target, eventToSend.name, eventToSend.content); + this.SendEvent(eventToSend.TargetInstance, eventToSend.EventName, eventToSend.EventContent); return (operationId, target.InstanceId, taskId); } @@ -288,10 +287,9 @@ public override Task LockEntitiesAsync(params EntityId[] entities) Guid criticalSectionId = Utils.CreateGuidFromHash(string.Concat(OrchestrationInstance.ExecutionId, ":", taskId)); // send a message to the first entity to be acquired - (OrchestrationInstance target, string name, object content) eventToSend = - this.entityContext.EmitAcquireMessage(criticalSectionId, entities); + EventToSend eventToSend = this.entityContext.EmitAcquireMessage(criticalSectionId, entities); - this.SendEvent(eventToSend.target, eventToSend.name, eventToSend.content); + this.SendEvent(eventToSend.TargetInstance, eventToSend.EventName, eventToSend.EventContent); return this.entityContext.WaitForLockResponseAsync(criticalSectionId, taskId); } @@ -302,7 +300,7 @@ internal void ExitCriticalSection() { foreach (var releaseMessage in this.entityContext.EmitLockReleaseMessages()) { - this.SendEvent(releaseMessage.target, releaseMessage.eventName, releaseMessage.eventContent); + this.SendEvent(releaseMessage.TargetInstance, releaseMessage.EventName, releaseMessage.EventContent); } } } From d09f24b6b1b46ebabfe0b1dd43cd7ef3124781c4 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Thu, 30 Mar 2023 14:40:36 -0700 Subject: [PATCH 3/8] fix bug in last commit (forgot to remove ToLowerInvariant() in one place) --- src/DurableTask.Core/TaskHubWorker.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.Core/TaskHubWorker.cs b/src/DurableTask.Core/TaskHubWorker.cs index 1ea4f393f..baddcb990 100644 --- a/src/DurableTask.Core/TaskHubWorker.cs +++ b/src/DurableTask.Core/TaskHubWorker.cs @@ -314,7 +314,7 @@ public TaskHubWorker AddTaskEntities(params Type[] taskEntityTypes) foreach (Type type in taskEntityTypes) { ObjectCreator creator = new NameValueObjectCreator( - type.Name.ToLowerInvariant(), // entity names are always case-normalized + type.Name, string.Empty, type); From 5720c7de9dc3b3385d906351e1f460eadb358d87 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Fri, 31 Mar 2023 10:03:56 -0700 Subject: [PATCH 4/8] fix unit tests (cherry picked from commit ca50451425895f79d6def07eb61ccf0608294fb2) --- .../AzureStorageScenarioTests.cs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index d82db750e..e3076e050 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -629,7 +629,7 @@ public async Task EntityQueries() }, result => { - Assert.AreEqual(0, result.Count()); + Assert.AreEqual(4, result.Count()); }), (new TaskHubEntityClient.Query @@ -638,12 +638,12 @@ public async Task EntityQueries() }, result => { - Assert.AreEqual(4, result.Count()); + Assert.AreEqual(0, result.Count()); }), (new TaskHubEntityClient.Query { - InstanceIdPrefix = "@stringstore@", + InstanceIdPrefix = "@StringStore@", }, result => { @@ -652,8 +652,8 @@ public async Task EntityQueries() (new TaskHubEntityClient.Query { - InstanceIdPrefix = "@stringstore", - EntityName = "stringstore", + InstanceIdPrefix = "@StringStore", + EntityName = "StringStore", }, result => { @@ -662,7 +662,7 @@ public async Task EntityQueries() (new TaskHubEntityClient.Query { - InstanceIdPrefix = "@stringstore@", + InstanceIdPrefix = "@StringStore@", EntityName = "StringStore", }, result => @@ -672,7 +672,7 @@ public async Task EntityQueries() (new TaskHubEntityClient.Query { - InstanceIdPrefix = "@stringstore@b", + InstanceIdPrefix = "@StringStore@b", EntityName = "StringStore", }, result => @@ -4550,14 +4550,17 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s } } + [JsonObject] public class UnserializableKaboom : Exception { } + [JsonObject] public class SerializableKaboom : Exception { } + [JsonObject] public class UnDeserializableKaboom : Exception { } From a4afe9b0e25339e81db5800ee0133c5d363cc411 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Fri, 31 Mar 2023 10:09:35 -0700 Subject: [PATCH 5/8] address PR feedback. --- .../AzureStorageOrchestrationService.cs | 2 +- ...zureStorageOrchestrationServiceSettings.cs | 5 ++- .../Partitioning/AppLeaseManager.cs | 2 +- .../Common}/Fnv1aHashHelper.cs | 34 +++++++++++++++++-- src/DurableTask.Core/Common/Utils.cs | 28 +++++++++------ 5 files changed, 53 insertions(+), 18 deletions(-) rename src/{DurableTask.AzureStorage => DurableTask.Core/Common}/Fnv1aHashHelper.cs (61%) diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs index 5b8ed18c4..5f2062ea1 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs @@ -1923,7 +1923,7 @@ public Task DownloadBlobAsync(string blobUri) // be supported: https://github.com/Azure/azure-functions-durable-extension/issues/1 async Task GetControlQueueAsync(string instanceId) { - uint partitionIndex = Fnv1aHashHelper.ComputeHash(instanceId) % (uint)this.settings.PartitionCount; + uint partitionIndex = DurableTask.Core.Common.Fnv1aHashHelper.ComputeHash(instanceId) % (uint)this.settings.PartitionCount; string queueName = GetControlQueueName(this.settings.TaskHubName, (int)partitionIndex); ControlQueue cachedQueue; diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs index 18b43c329..c2b824b9d 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs @@ -14,15 +14,14 @@ namespace DurableTask.AzureStorage { using System; + using System.Runtime.Serialization; + using System.Threading.Tasks; using DurableTask.AzureStorage.Partitioning; using DurableTask.AzureStorage.Logging; using DurableTask.Core; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Queue; using Microsoft.WindowsAzure.Storage.Table; - using System.Runtime.Serialization; - using System.Threading.Tasks; - using DurableTask.Core.Entities; /// /// Settings that impact the runtime behavior of the . diff --git a/src/DurableTask.AzureStorage/Partitioning/AppLeaseManager.cs b/src/DurableTask.AzureStorage/Partitioning/AppLeaseManager.cs index 5ea594da4..62576898f 100644 --- a/src/DurableTask.AzureStorage/Partitioning/AppLeaseManager.cs +++ b/src/DurableTask.AzureStorage/Partitioning/AppLeaseManager.cs @@ -71,7 +71,7 @@ public AppLeaseManager( this.appLeaseContainer = this.azureStorageClient.GetBlobContainerReference(this.appLeaseContainerName); this.appLeaseInfoBlob = this.appLeaseContainer.GetBlobReference(this.appLeaseInfoBlobName); - var appNameHashInBytes = BitConverter.GetBytes(Fnv1aHashHelper.ComputeHash(this.appName)); + var appNameHashInBytes = BitConverter.GetBytes(DurableTask.Core.Common.Fnv1aHashHelper.ComputeHash(this.appName)); Array.Resize(ref appNameHashInBytes, 16); this.appLeaseId = new Guid(appNameHashInBytes).ToString(); diff --git a/src/DurableTask.AzureStorage/Fnv1aHashHelper.cs b/src/DurableTask.Core/Common/Fnv1aHashHelper.cs similarity index 61% rename from src/DurableTask.AzureStorage/Fnv1aHashHelper.cs rename to src/DurableTask.Core/Common/Fnv1aHashHelper.cs index fbff51089..4ac17f9c3 100644 --- a/src/DurableTask.AzureStorage/Fnv1aHashHelper.cs +++ b/src/DurableTask.Core/Common/Fnv1aHashHelper.cs @@ -10,8 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // ---------------------------------------------------------------------------------- - -namespace DurableTask.AzureStorage +namespace DurableTask.Core.Common { using System.Text; @@ -22,32 +21,61 @@ namespace DurableTask.AzureStorage /// See https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function. /// Tested with production data and random guids. The result was good distribution. /// - static class Fnv1aHashHelper + public static class Fnv1aHashHelper { const uint FnvPrime = unchecked(16777619); const uint FnvOffsetBasis = unchecked(2166136261); + /// + /// Compute a hash for a given string. + /// + /// The string to hash. + /// a four-byte hash public static uint ComputeHash(string value) { return ComputeHash(value, encoding: null); } + /// + /// Compute a hash for a given string and encoding. + /// + /// The string to hash. + /// The encoding. + /// a four-byte hash public static uint ComputeHash(string value, Encoding encoding) { return ComputeHash(value, encoding, hash: FnvOffsetBasis); } + /// + /// Compute a hash for a given string, encoding, and hash modifier. + /// + /// The string to hash. + /// The encoding. + /// The modifier hash. + /// a four-byte hash public static uint ComputeHash(string value, Encoding encoding, uint hash) { byte[] bytes = (encoding ?? Encoding.UTF8).GetBytes(value); return ComputeHash(bytes, hash); } + /// + /// Compute a hash for the given byte array. + /// + /// The byte array to hash. + /// a four-byte hash public static uint ComputeHash(byte[] array) { return ComputeHash(array, hash: FnvOffsetBasis); } + /// + /// Compute a hash for the given byte array. + /// + /// The byte array to hash. + /// The modifier hash. + /// a four-byte hash public static uint ComputeHash(byte[] array, uint hash) { for (var i = 0; i < array.Length; i++) diff --git a/src/DurableTask.Core/Common/Utils.cs b/src/DurableTask.Core/Common/Utils.cs index ea03a1cd3..942c22854 100644 --- a/src/DurableTask.Core/Common/Utils.cs +++ b/src/DurableTask.Core/Common/Utils.cs @@ -640,17 +640,25 @@ internal static Guid CreateGuidFromHash(string stringToHash) throw new ArgumentException("string to hash must not be null or empty", nameof(stringToHash)); } - byte[] hashByteArray; - using (HashAlgorithm hashAlgorithm = (HashAlgorithm)SHA1.Create()) - { - hashByteArray = hashAlgorithm.ComputeHash(Encoding.UTF8.GetBytes(stringToHash)); - } - - byte[] newGuidByteArray = new byte[16]; - Array.Copy(hashByteArray, 0, newGuidByteArray, 0, 16); - return new Guid(newGuidByteArray); + var bytes = Encoding.UTF8.GetBytes(stringToHash); + uint hash1 = Fnv1aHashHelper.ComputeHash(bytes, 0xdf0dd395); + uint hash2 = Fnv1aHashHelper.ComputeHash(bytes, 0xa19df4df); + uint hash3 = Fnv1aHashHelper.ComputeHash(bytes, 0xc88599c5); + uint hash4 = Fnv1aHashHelper.ComputeHash(bytes, 0xe24e3e64); + return new Guid( + hash1, + (ushort)(hash2 & 0xFFFF), + (ushort)((hash2 >> 16) & 0xFFFF), + (byte)(hash3 & 0xFF), + (byte)((hash3 >> 8) & 0xFF), + (byte)((hash3 >> 16) & 0xFF), + (byte)((hash3 >> 24) & 0xFF), + (byte)(hash4 & 0xFF), + (byte)((hash4 >> 8) & 0xFF), + (byte)((hash4 >> 16) & 0xFF), + (byte)((hash4 >> 24) & 0xFF)); } - + /// /// Gets the generic return type for a specific . /// From ad534915fd3e62e9becb69e490cec5c3fab87519 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Thu, 6 Apr 2023 15:08:18 -0700 Subject: [PATCH 6/8] address feedback. --- .../AzureStorageOrchestrationService.cs | 44 +- ...zureStorageOrchestrationServiceSettings.cs | 7 +- .../OrchestrationSessionManager.cs | 29 +- .../Entities/EntityBackendInformation.cs | 40 +- .../Entities/EntityExecutionOptions.cs | 23 + src/DurableTask.Core/Entities/EntityId.cs | 16 +- .../Entities/EntityWorkItemProcessor.cs | 638 ------------- .../Entities/IEntityExecutor.cs | 30 + .../Entities/IEntityOrchestrationService.cs | 47 + .../Entities/LocalSDK/EntityContext.cs | 24 +- .../Entities/LocalSDK/TaskEntity.cs | 63 ++ .../Entities/LocalSDK/TaskEntityContext.cs | 184 ++-- .../Entities/LocalSDK/TaskHubEntityClient.cs | 12 +- .../Entities/OrchestrationEntityContext.cs | 2 +- src/DurableTask.Core/Entities/TaskEntity.cs | 100 +- .../Logging/StructuredEventSource.cs | 4 +- src/DurableTask.Core/NameObjectManager.cs | 65 ++ src/DurableTask.Core/OrchestrationContext.cs | 20 +- .../OrchestrationWorkItemProcessor.cs | 604 ------------ .../OrchestratorExecutionResult.cs | 4 +- src/DurableTask.Core/TaskEntityDispatcher.cs | 896 ++++++++++++++++++ src/DurableTask.Core/TaskHubWorker.cs | 41 +- .../TaskOrchestrationContext.cs | 3 +- .../TaskOrchestrationDispatcher.cs | 727 +++++++++++--- .../AzureStorageScenarioTests.cs | 125 +-- .../TestOrchestrationHost.cs | 3 +- 26 files changed, 2057 insertions(+), 1694 deletions(-) delete mode 100644 src/DurableTask.Core/Entities/EntityWorkItemProcessor.cs create mode 100644 src/DurableTask.Core/Entities/IEntityExecutor.cs create mode 100644 src/DurableTask.Core/Entities/IEntityOrchestrationService.cs create mode 100644 src/DurableTask.Core/Entities/LocalSDK/TaskEntity.cs create mode 100644 src/DurableTask.Core/NameObjectManager.cs delete mode 100644 src/DurableTask.Core/OrchestrationWorkItemProcessor.cs create mode 100644 src/DurableTask.Core/TaskEntityDispatcher.cs diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs index 5f2062ea1..535ee570a 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs @@ -44,7 +44,7 @@ public sealed class AzureStorageOrchestrationService : IDisposable, IOrchestrationServiceQueryClient, IOrchestrationServicePurgeClient, - EntityBackendInformation.IInformationProvider + IEntityOrchestrationService { static readonly HistoryEvent[] EmptyHistoryEventList = new HistoryEvent[0]; @@ -270,7 +270,9 @@ public BehaviorOnContinueAsNew EventBehaviourForContinueAsNew /// public int TaskOrchestrationDispatcherCount { get; } = 1; - EntityBackendInformation EntityBackendInformation.IInformationProvider.GetEntityBackendInformation() + #region IEntityOrchestrationService + + EntityBackendInformation IEntityOrchestrationService.GetEntityBackendInformation() => new EntityBackendInformation() { EntityMessageReorderWindow = TimeSpan.FromMinutes(this.settings.EntityMessageReorderWindowInMinutes), @@ -279,6 +281,35 @@ EntityBackendInformation EntityBackendInformation.IInformationProvider.GetEntity MaximumSignalDelayTime = TimeSpan.FromDays(6), }; + void IEntityOrchestrationService.ProcessEntitiesSeparately() + { + this.orchestrationSessionManager.ProcessEntitiesSeparately = true; + } + + Task IEntityOrchestrationService.LockNextOrchestrationWorkItemAsync( + TimeSpan receiveTimeout, + CancellationToken cancellationToken) + { + if (!orchestrationSessionManager.ProcessEntitiesSeparately) + { + throw new InvalidOperationException("backend was not configured for separate entity processing"); + } + return this.LockNextTaskOrchestrationWorkItemAsync(false, cancellationToken); + } + + Task IEntityOrchestrationService.LockNextEntityWorkItemAsync( + TimeSpan receiveTimeout, + CancellationToken cancellationToken) + { + if (!orchestrationSessionManager.ProcessEntitiesSeparately) + { + throw new InvalidOperationException("backend was not configured for separate entity processing"); + } + return this.LockNextTaskOrchestrationWorkItemAsync(true, cancellationToken); + } + + #endregion + #region Management Operations (Create/Delete/Start/Stop) /// /// Deletes and creates the neccesary Azure Storage resources for the orchestration service. @@ -568,9 +599,14 @@ static TaskHubInfo GetTaskHubInfo(string taskHub, int partitionCount) #region Orchestration Work Item Methods /// - public async Task LockNextTaskOrchestrationWorkItemAsync( + public Task LockNextTaskOrchestrationWorkItemAsync( TimeSpan receiveTimeout, CancellationToken cancellationToken) + { + return LockNextTaskOrchestrationWorkItemAsync(false, cancellationToken); + } + + async Task LockNextTaskOrchestrationWorkItemAsync(bool entitiesOnly, CancellationToken cancellationToken) { Guid traceActivityId = StartNewLogicalTraceScope(useExisting: true); @@ -584,7 +620,7 @@ public async Task LockNextTaskOrchestrationWorkItemAs try { // This call will block until the next session is ready - session = await this.orchestrationSessionManager.GetNextSessionAsync(linkedCts.Token); + session = await this.orchestrationSessionManager.GetNextSessionAsync(entitiesOnly, linkedCts.Token); if (session == null) { return null; diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs index c2b824b9d..c36137f47 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs @@ -276,14 +276,15 @@ internal LogHelper Logger } /// - /// Gets or sets the maximum number of entity operations that are processed as a single batch. + /// Gets or sets the limit on the number of entity operations that should be processed as a single batch. + /// A null value indicates that no particular limit should be enforced. /// /// - /// Reducing this number can help to avoid timeouts in execution environments that impose time limitations on work items. + /// Limiting the batch size can help to avoid timeouts in execution environments that impose time limitations on work items. /// If set to 1, batching is disabled, and each operation executes as a separate work item. /// /// - /// A positive integer configured by the host. + /// A positive integer, or null. /// public int? MaxEntityOperationBatchSize { get; set; } = null; diff --git a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs index e18853031..c2ee22292 100644 --- a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs +++ b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs @@ -32,7 +32,8 @@ class OrchestrationSessionManager : IDisposable readonly Dictionary activeOrchestrationSessions = new Dictionary(StringComparer.OrdinalIgnoreCase); readonly ConcurrentDictionary ownedControlQueues = new ConcurrentDictionary(); readonly LinkedList pendingOrchestrationMessageBatches = new LinkedList(); - readonly AsyncQueue> readyForProcessingQueue = new AsyncQueue>(); + readonly AsyncQueue> orchestrationsReadyForProcessingQueue = new AsyncQueue>(); + readonly AsyncQueue> entitiesReadyForProcessingQueue = new AsyncQueue>(); readonly object messageAndSessionLock = new object(); readonly string storageAccountName; @@ -57,6 +58,8 @@ public OrchestrationSessionManager( internal IEnumerable Queues => this.ownedControlQueues.Values; + internal bool ProcessEntitiesSeparately { get; set; } + public void AddQueue(string partitionId, ControlQueue controlQueue, CancellationToken cancellationToken) { if (this.ownedControlQueues.TryAdd(partitionId, controlQueue)) @@ -480,7 +483,14 @@ async Task ScheduleOrchestrationStatePrefetch( batch.LastCheckpointTime = history.LastCheckpointTime; } - this.readyForProcessingQueue.Enqueue(node); + if (this.ProcessEntitiesSeparately && DurableTask.Core.Common.Entities.IsEntityInstance(batch.OrchestrationInstanceId)) + { + this.entitiesReadyForProcessingQueue.Enqueue(node); + } + else + { + this.orchestrationsReadyForProcessingQueue.Enqueue(node); + } } catch (OperationCanceledException) { @@ -504,14 +514,16 @@ async Task ScheduleOrchestrationStatePrefetch( } } - public async Task GetNextSessionAsync(CancellationToken cancellationToken) + public async Task GetNextSessionAsync(bool entitiesOnly, CancellationToken cancellationToken) { + var readyForProcessingQueue = entitiesOnly? this.entitiesReadyForProcessingQueue : this.orchestrationsReadyForProcessingQueue; + while (!cancellationToken.IsCancellationRequested) { // This call will block until: // 1) a batch of messages has been received for a particular instance and // 2) the history for that instance has been fetched - LinkedListNode node = await this.readyForProcessingQueue.DequeueAsync(cancellationToken); + LinkedListNode node = await readyForProcessingQueue.DequeueAsync(cancellationToken); lock (this.messageAndSessionLock) { @@ -556,7 +568,7 @@ async Task ScheduleOrchestrationStatePrefetch( // A message arrived for a different generation of an existing orchestration instance. // Put it back into the ready queue so that it can be processed once the current generation // is done executing. - if (this.readyForProcessingQueue.Count == 0) + if (readyForProcessingQueue.Count == 0) { // To avoid a tight dequeue loop, delay for a bit before putting this node back into the queue. // This is only necessary when the queue is empty. The main dequeue thread must not be blocked @@ -566,14 +578,14 @@ async Task ScheduleOrchestrationStatePrefetch( lock (this.messageAndSessionLock) { this.pendingOrchestrationMessageBatches.AddLast(node); - this.readyForProcessingQueue.Enqueue(node); + readyForProcessingQueue.Enqueue(node); } }); } else { this.pendingOrchestrationMessageBatches.AddLast(node); - this.readyForProcessingQueue.Enqueue(node); + readyForProcessingQueue.Enqueue(node); } } } @@ -635,7 +647,8 @@ public void GetStats( public virtual void Dispose() { this.fetchRuntimeStateQueue.Dispose(); - this.readyForProcessingQueue.Dispose(); + this.orchestrationsReadyForProcessingQueue.Dispose(); + this.entitiesReadyForProcessingQueue.Dispose(); } class PendingMessageBatch diff --git a/src/DurableTask.Core/Entities/EntityBackendInformation.cs b/src/DurableTask.Core/Entities/EntityBackendInformation.cs index 2d1ea25d9..bf95da387 100644 --- a/src/DurableTask.Core/Entities/EntityBackendInformation.cs +++ b/src/DurableTask.Core/Entities/EntityBackendInformation.cs @@ -19,38 +19,6 @@ namespace DurableTask.Core.Entities /// public class EntityBackendInformation { - /// - /// Get the entity options specified by the orchestration service, or the default options if the service does not specify options. - /// - /// The orchestration service. - /// The options that the provider specifies. - /// The entity options - public static bool BackendSupportsEntities(IOrchestrationService orchestrationService, out EntityBackendInformation entityBackendInformation) - { - if (orchestrationService is IInformationProvider optionsProvider) - { - entityBackendInformation = optionsProvider.GetEntityBackendInformation(); - return true; - } - else - { - entityBackendInformation = null; - return false; - } - } - - /// - /// Interface for objects that provide entity backend information. - /// - public interface IInformationProvider - { - /// - /// The entity backend info. - /// - /// The entity backend information object. - EntityBackendInformation GetEntityBackendInformation(); - } - /// /// The time window within which entity messages should be deduplicated and reordered. /// This is zero for providers that already guarantee exactly-once and ordered delivery. @@ -58,7 +26,7 @@ public interface IInformationProvider public TimeSpan EntityMessageReorderWindow { get; set; } /// - /// The maximum number of entity operations that should be processed as a single batch. + /// A limit on the number of entity operations that should be processed as a single batch, or null if there is no limit. /// public int? MaxEntityOperationBatchSize { get; set; } @@ -78,15 +46,15 @@ public interface IInformationProvider /// /// /// - public (DateTime original, DateTime capped) GetCappedScheduledTime(DateTime nowUtc, DateTime scheduledUtcTime) + public DateTime GetCappedScheduledTime(DateTime nowUtc, DateTime scheduledUtcTime) { if ((scheduledUtcTime - nowUtc) <= this.MaximumSignalDelayTime) { - return (scheduledUtcTime, scheduledUtcTime); + return scheduledUtcTime; } else { - return (scheduledUtcTime, nowUtc + this.MaximumSignalDelayTime); + return nowUtc + this.MaximumSignalDelayTime; } } } diff --git a/src/DurableTask.Core/Entities/EntityExecutionOptions.cs b/src/DurableTask.Core/Entities/EntityExecutionOptions.cs index 18ddea780..15bcdde4c 100644 --- a/src/DurableTask.Core/Entities/EntityExecutionOptions.cs +++ b/src/DurableTask.Core/Entities/EntityExecutionOptions.cs @@ -12,6 +12,7 @@ // ---------------------------------------------------------------------------------- using System; +using DurableTask.Core.Serializing; namespace DurableTask.Core.Entities { @@ -20,6 +21,28 @@ namespace DurableTask.Core.Entities /// public class EntityExecutionOptions { + /// + /// The data converter used for converting inputs and outputs for operations. + /// + public DataConverter MessageDataConverter { get; set; } = JsonDataConverter.Default; + + /// + /// The data converter used for the entity state. + /// + public DataConverter StateDataConverter { get; set; } = JsonDataConverter.Default; + + /// + /// The data converter used for exceptions. + /// + public DataConverter ErrorDataConverter { get; set; } = JsonDataConverter.Default; + + /// + /// If true, all effects of an entity operation (all state changes and all actions) are rolled back + /// if the entity operation completes with an exception. + /// Implementations may override this setting. + /// + public bool RollbackOnExceptions { get; set; } = true; + /// /// Information about backend entity support. /// diff --git a/src/DurableTask.Core/Entities/EntityId.cs b/src/DurableTask.Core/Entities/EntityId.cs index 1f4e18f12..12724732f 100644 --- a/src/DurableTask.Core/Entities/EntityId.cs +++ b/src/DurableTask.Core/Entities/EntityId.cs @@ -34,26 +34,26 @@ public EntityId(string entityName, string entityKey) throw new ArgumentNullException(nameof(entityName), "Invalid entity id: entity name must not be a null or empty string."); } - this.EntityName = entityName; - this.EntityKey = entityKey ?? throw new ArgumentNullException(nameof(entityKey), "Invalid entity id: entity key must not be null."); + this.Name = entityName; + this.Key = entityKey ?? throw new ArgumentNullException(nameof(entityKey), "Invalid entity id: entity key must not be null."); } /// /// The name for this class of entities. /// [DataMember(Name = "name", IsRequired = true)] - public readonly string EntityName; + public readonly string Name; /// /// The entity key. Uniquely identifies an entity among all entities of the same name. /// [DataMember(Name = "key", IsRequired = true)] - public readonly string EntityKey; + public readonly string Key; /// public override string ToString() { - return $"@{this.EntityName}@{this.EntityKey}"; + return $"@{this.Name}@{this.Key}"; } internal static string GetSchedulerIdPrefixFromEntityName(string entityName) @@ -92,20 +92,20 @@ public override bool Equals(object obj) /// public bool Equals(EntityId other) { - return (this.EntityName,this.EntityKey).Equals((other.EntityName, other.EntityKey)); + return (this.Name,this.Key).Equals((other.Name, other.Key)); } /// public override int GetHashCode() { - return (this.EntityName, this.EntityKey).GetHashCode(); + return (this.Name, this.Key).GetHashCode(); } /// public int CompareTo(object obj) { var other = (EntityId)obj; - return (this.EntityName, this.EntityKey).CompareTo((other.EntityName, other.EntityKey)); + return (this.Name, this.Key).CompareTo((other.Name, other.Key)); } } } diff --git a/src/DurableTask.Core/Entities/EntityWorkItemProcessor.cs b/src/DurableTask.Core/Entities/EntityWorkItemProcessor.cs deleted file mode 100644 index a3cf3a47d..000000000 --- a/src/DurableTask.Core/Entities/EntityWorkItemProcessor.cs +++ /dev/null @@ -1,638 +0,0 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- -namespace DurableTask.Core.Entities -{ - using DurableTask.Core.Common; - using DurableTask.Core.Entities.EventFormat; - using DurableTask.Core.Entities.OperationFormat; - using DurableTask.Core.Entities.StateFormat; - using DurableTask.Core.Exceptions; - using DurableTask.Core.History; - using DurableTask.Core.Logging; - using DurableTask.Core.Middleware; - using DurableTask.Core.Tracing; - using Newtonsoft.Json; - using System; - using System.Collections.Generic; - using System.Diagnostics; - using System.Linq; - using System.Threading.Tasks; - - internal class EntityWorkItemProcessor : TaskOrchestrationDispatcher.WorkItemProcessor - { - readonly LogHelper logHelper; - readonly INameVersionObjectManager objectManager; - readonly DispatchMiddlewarePipeline dispatchPipeline; - readonly EntityBackendInformation entityBackendInformation; - readonly string instanceId; - readonly ErrorPropagationMode errorPropagationMode; - - SchedulerState schedulerState; - int idCounter; - - public EntityWorkItemProcessor( - TaskOrchestrationDispatcher dispatcher, - TaskOrchestrationWorkItem workItem, - LogHelper logHelper, - INameVersionObjectManager objectManager, - DispatchMiddlewarePipeline dispatchPipeline, - EntityBackendInformation entityBackendInformation, - ErrorPropagationMode errorPropagationMode) : base(dispatcher, workItem) - { - this.logHelper = logHelper; - this.objectManager = objectManager; - this.dispatchPipeline = dispatchPipeline; - this.entityBackendInformation = entityBackendInformation; - this.instanceId = workItem.InstanceId; - this.errorPropagationMode = errorPropagationMode; - } - - public override async Task ProcessWorkItemAsync() - { - // we start with processing all the requests and figuring out which ones to execute - // results can depend on whether the entity is locked, what the maximum batch size is, - // and whether the messages arrived out of order - var workToDoNow = this.DetermineWork(); - - if (workToDoNow.OperationCount > 0) - { - // execute the user-defined operations on this entity, via the middleware - var result = await this.ExecuteViaMiddlewareAsync(workToDoNow); - - // go through all results - // for each operation that is not a signal, send a result message back to the calling orchestrator - for (int i = 0; i < result.Results.Count; i++) - { - var req = workToDoNow.Operations[i]; - if (!req.IsSignal) - { - this.SendResultMessage(req, result.Results[i]); - } - } - - if (result.Results.Count < workToDoNow.OperationCount) - { - // some operations were not processed - var deferred = workToDoNow.RemoveDeferredWork(result.Results.Count); - this.schedulerState.PutBack(deferred); - workToDoNow.ToBeContinued(this.schedulerState); - } - - // update the entity state based on the result - this.schedulerState.EntityState = result.EntityState; - this.schedulerState.EntityExists = result.EntityState != null; - - // perform the actions - foreach (var action in result.Actions) - { - switch (action) - { - case (SendSignalOperationAction sendSignalAction): - this.SendSignalMessage(sendSignalAction); - break; - case (StartNewOrchestrationOperationAction startAction): - this.ProcessSendStartMessage(startAction); - break; - } - } - } - - // process the lock request, if any - if (workToDoNow.LockRequest != null) - { - this.ProcessLockRequest(workToDoNow.LockRequest); - } - - if (workToDoNow.ToBeRescheduled != null) - { - foreach (var request in workToDoNow.ToBeRescheduled) - { - // Reschedule all signals that were received before their time - this.SendScheduledSelfMessage(request); - } - } - - if (workToDoNow.SuspendAndContinue) - { - this.SendContinueSelfMessage(); - } - - // this batch is complete. Since this is an entity, we now - // (always) start a new execution, as in continue-as-new - - var serializedSchedulerState = this.SerializeSchedulerStateForNextExecution(); - var nextExecutionStartedEvent = new ExecutionStartedEvent(-1, serializedSchedulerState) - { - OrchestrationInstance = new OrchestrationInstance - { - InstanceId = this.instanceId, - ExecutionId = Guid.NewGuid().ToString("N") - }, - Tags = runtimeState.Tags, - ParentInstance = runtimeState.ParentInstance, - Name = runtimeState.Name, - Version = runtimeState.Version - }; - var entityStatus = new EntityStatus() - { - EntityExists = this.schedulerState.EntityExists, - QueueSize = this.schedulerState.Queue?.Count ?? 0, - LockedBy = this.schedulerState.LockedBy, - }; - var serializedEntityStatus = JsonConvert.SerializeObject(entityStatus, Serializer.InternalSerializerSettings); - - // create the runtime state for the next execution - this.runtimeState = new OrchestrationRuntimeState(); - this.runtimeState.Status = serializedEntityStatus; - this.runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); - this.runtimeState.AddEvent(nextExecutionStartedEvent); - this.runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); - - this.workItem.OrchestrationRuntimeState = this.runtimeState; - this.instanceState = Utils.BuildOrchestrationState(this.runtimeState); - } - - void ProcessLockRequest(RequestMessage request) - { - this.logHelper.EntityLockAcquired(this.instanceId, request); - - // mark the entity state as locked - this.schedulerState.LockedBy = request.ParentInstanceId; - - request.Position++; - - if (request.Position < request.LockSet.Length) - { - // send lock request to next entity in the lock set - var target = new OrchestrationInstance() { InstanceId = request.LockSet[request.Position].ToString() }; - this.SendLockRequestMessage(target, request); - } - else - { - // send lock acquisition completed response back to originating orchestration instance - var target = new OrchestrationInstance() { InstanceId = request.ParentInstanceId, ExecutionId = request.ParentExecutionId }; - this.SendLockResponseMessage(target, request.Id); - } - } - - string SerializeSchedulerStateForNextExecution() - { - if (this.entityBackendInformation.SupportsImplicitEntityDeletion && this.schedulerState.IsEmpty && !this.schedulerState.Suspended) - { - // this entity scheduler is idle and the entity is deleted, so the instance and history can be removed from storage - // we convey this to the durability provider by issuing a continue-as-new with null input - return null; - } - else - { - // we persist the state of the entity scheduler and entity - return JsonConvert.SerializeObject(this.schedulerState, typeof(SchedulerState), Serializer.InternalSerializerSettings); - } - } - - #region Preprocess to determine work - - Work DetermineWork() - { - Queue lockHolderMessages = null; - Work batch = new Work(); - - foreach (HistoryEvent e in this.runtimeState.Events) - { - switch (e.EventType) - { - case EventType.ExecutionStarted: - - this.schedulerState = new SchedulerState(); - - if (runtimeState.Input != null) - { - try - { - // restore the scheduler state from the input - JsonConvert.PopulateObject(runtimeState.Input, this.schedulerState, Serializer.InternalSerializerSettings); - } - catch (Exception exception) - { - throw new EntitySchedulerException("Failed to deserialize entity scheduler state - may be corrupted or wrong version.", exception); - } - } - break; - - case EventType.EventRaised: - EventRaisedEvent eventRaisedEvent = (EventRaisedEvent)e; - - if (EntityMessageEventNames.IsRequestMessage(eventRaisedEvent.Name)) - { - // we are receiving an operation request or a lock request - var requestMessage = new RequestMessage(); - - try - { - JsonConvert.PopulateObject(eventRaisedEvent.Input, requestMessage, Serializer.InternalSerializerSettings); - } - catch (Exception exception) - { - throw new EntitySchedulerException("Failed to deserialize incoming request message - may be corrupted or wrong version.", exception); - } - - IEnumerable deliverNow; - - if (requestMessage.ScheduledTime.HasValue) - { - if ((requestMessage.ScheduledTime.Value - DateTime.UtcNow) > TimeSpan.FromMilliseconds(100)) - { - // message was delivered too early. This can happen e.g. if the orchestration service has limits on the delay times for messages. - // We handle this by rescheduling the message instead of processing it. - deliverNow = Array.Empty(); - batch.AddMessageToBeRescheduled(requestMessage); - } - else - { - // the message is scheduled to be delivered immediately. - // There are no FIFO guarantees for scheduled messages, so we skip the message sorter. - deliverNow = new RequestMessage[] { requestMessage }; - } - } - else - { - // run this through the message sorter to help with reordering and duplicate filtering - deliverNow = this.schedulerState.MessageSorter.ReceiveInOrder(requestMessage, this.entityBackendInformation.EntityMessageReorderWindow); - } - - foreach (var message in deliverNow) - { - if (this.schedulerState.LockedBy != null && this.schedulerState.LockedBy == message.ParentInstanceId) - { - if (lockHolderMessages == null) - { - lockHolderMessages = new Queue(); - } - - lockHolderMessages.Enqueue(message); - } - else - { - this.schedulerState.Enqueue(message); - } - } - } - else if (EntityMessageEventNames.IsReleaseMessage(eventRaisedEvent.Name)) - { - // we are receiving a lock release - var message = new ReleaseMessage(); - try - { - // restore the scheduler state from the input - JsonConvert.PopulateObject(eventRaisedEvent.Input, message, Serializer.InternalSerializerSettings); - } - catch (Exception exception) - { - throw new EntitySchedulerException("Failed to deserialize lock release message - may be corrupted or wrong version.", exception); - } - - if (this.schedulerState.LockedBy == message.ParentInstanceId) - { - this.logHelper.EntityLockReleased(this.instanceId, message); - this.schedulerState.LockedBy = null; - } - } - else - { - // this is a continue message. - // Resumes processing of previously queued operations, if any. - this.schedulerState.Suspended = false; - } - - break; - } - } - - // lock holder messages go to the front of the queue - if (lockHolderMessages != null) - { - this.schedulerState.PutBack(lockHolderMessages); - } - - if (!this.schedulerState.Suspended) - { - // 2. We add as many requests from the queue to the batch as possible, - // stopping at lock requests or when the maximum batch size is reached - while (this.schedulerState.MayDequeue()) - { - if (batch.OperationCount == this.entityBackendInformation.MaxEntityOperationBatchSize) - { - // we have reached the maximum batch size already - // insert a delay after this batch to ensure write back - batch.ToBeContinued(this.schedulerState); - break; - } - - var request = this.schedulerState.Dequeue(); - - if (request.IsLockRequest) - { - batch.AddLockRequest(request); - break; - } - else - { - batch.AddOperation(request); - } - } - } - - return batch; - } - - class Work - { - List operationBatch; // a (possibly empty) sequence of operations to be executed on the entity - RequestMessage lockRequest = null; // zero or one lock request to be executed after all the operations - List toBeRescheduled; // a (possibly empty) list of timed messages that were delivered too early and should be rescheduled - bool suspendAndContinue; // a flag telling as to send ourselves a continue signal - - public int OperationCount => this.operationBatch?.Count ?? 0; - public IReadOnlyList Operations => this.operationBatch; - public IReadOnlyList ToBeRescheduled => this.toBeRescheduled; - public RequestMessage LockRequest => this.lockRequest; - public bool SuspendAndContinue => this.suspendAndContinue; - - public void AddOperation(RequestMessage operationMessage) - { - if (this.operationBatch == null) - { - this.operationBatch = new List(); - } - this.operationBatch.Add(operationMessage); - } - - public void AddLockRequest(RequestMessage lockRequest) - { - Debug.Assert(this.lockRequest == null); - this.lockRequest = lockRequest; - } - - public void AddMessageToBeRescheduled(RequestMessage requestMessage) - { - if (this.toBeRescheduled == null) - { - this.toBeRescheduled = new List(); - } - this.toBeRescheduled.Add(requestMessage); - } - - public void ToBeContinued(SchedulerState schedulerState) - { - if (!schedulerState.Suspended) - { - this.suspendAndContinue = true; - } - } - - public List GetOperationRequests() - { - var operations = new List(this.operationBatch.Count); - for (int i = 0; i < this.operationBatch.Count; i++) - { - var request = this.operationBatch[i]; - operations.Add(new OperationRequest() - { - Operation = request.Operation, - Id = request.Id, - Input = request.Input, - }); - } - return operations; - } - - public Queue RemoveDeferredWork(int index) - { - var deferred = new Queue(); - for (int i = index; i < this.operationBatch.Count; i++) - { - deferred.Enqueue(this.operationBatch[i]); - } - this.operationBatch.RemoveRange(index, this.operationBatch.Count - index); - if (this.lockRequest != null) - { - deferred.Enqueue(this.lockRequest); - this.lockRequest = null; - } - return deferred; - } - } - - #endregion - - #region Send Messages - - void SendResultMessage(RequestMessage request, OperationResult result) - { - var destination = new OrchestrationInstance() - { - InstanceId = request.ParentInstanceId, - ExecutionId = request.ParentExecutionId, - }; - var responseMessage = new ResponseMessage() - { - Result = result.Result, - ErrorMessage = result.ErrorMessage, - FailureDetails = result.FailureDetails, - }; - this.ProcessSendEventMessage(destination, EntityMessageEventNames.ResponseMessageEventName(request.Id), responseMessage); - } - - void SendSignalMessage(SendSignalOperationAction action) - { - OrchestrationInstance destination = new OrchestrationInstance() - { - InstanceId = action.InstanceId - }; - RequestMessage message = new RequestMessage() - { - ParentInstanceId = this.instanceId, - ParentExecutionId = null, // for entities, message sorter persists across executions - Id = Guid.NewGuid(), - IsSignal = true, - Operation = action.Name, - ScheduledTime = action.ScheduledTime, - }; - string eventName; - if (action.ScheduledTime.HasValue) - { - (DateTime original, DateTime capped) = this.entityBackendInformation.GetCappedScheduledTime(DateTime.UtcNow, action.ScheduledTime.Value); - eventName = EntityMessageEventNames.ScheduledRequestMessageEventName(capped); - } - else - { - eventName = EntityMessageEventNames.RequestMessageEventName; - this.schedulerState.MessageSorter.LabelOutgoingMessage(message, action.InstanceId, DateTime.UtcNow, this.entityBackendInformation.EntityMessageReorderWindow); - } - this.ProcessSendEventMessage(destination, eventName, message); - } - - internal void SendLockRequestMessage(OrchestrationInstance target, RequestMessage message) - { - this.schedulerState.MessageSorter.LabelOutgoingMessage(message, target.InstanceId, DateTime.UtcNow, this.entityBackendInformation.EntityMessageReorderWindow); - this.ProcessSendEventMessage(target, EntityMessageEventNames.RequestMessageEventName, message); - } - - internal void SendLockResponseMessage(OrchestrationInstance target, Guid requestId) - { - var message = new ResponseMessage() - { - Result = "Lock Acquisition Completed", // ignored by receiver but shows up in traces - }; - this.ProcessSendEventMessage(target, EntityMessageEventNames.ResponseMessageEventName(requestId), message); - } - - void SendScheduledSelfMessage(RequestMessage request) - { - var self = new OrchestrationInstance() - { - InstanceId = this.instanceId, - }; - this.ProcessSendEventMessage(self, EntityMessageEventNames.ScheduledRequestMessageEventName(request.ScheduledTime.Value), request); - } - - void SendContinueSelfMessage() - { - var self = new OrchestrationInstance() - { - InstanceId = this.instanceId, - }; - this.ProcessSendEventMessage(self, EntityMessageEventNames.ContinueMessageEventName, null); - } - - void ProcessSendEventMessage(OrchestrationInstance destination, string eventName, object eventContent) - { - string serializedContent = null; - if (eventContent != null) - { - serializedContent = JsonConvert.SerializeObject(eventContent, Serializer.InternalSerializerSettings); - } - - var eventSentEvent = new EventSentEvent(this.idCounter++) - { - InstanceId = destination.InstanceId, - Name = eventName, - Input = serializedContent, - }; - this.logHelper.RaisingEvent(runtimeState.OrchestrationInstance!, eventSentEvent); - - this.orchestratorMessages.Add(new TaskMessage - { - OrchestrationInstance = destination, - Event = new EventRaisedEvent(-1, serializedContent) - { - Name = eventName, - Input = serializedContent, - }, - }); - } - - void ProcessSendStartMessage(StartNewOrchestrationOperationAction action) - { - OrchestrationInstance destination = new OrchestrationInstance() - { - InstanceId = action.InstanceId, - ExecutionId = Guid.NewGuid().ToString("N"), - }; - var executionStartedEvent = new ExecutionStartedEvent(-1, action.Input) - { - Tags = OrchestrationTags.MergeTags(action.Tags, runtimeState.Tags), - OrchestrationInstance = destination, - ParentInstance = new ParentInstance - { - OrchestrationInstance = runtimeState.OrchestrationInstance, - Name = runtimeState.Name, - Version = runtimeState.Version, - TaskScheduleId = idCounter++, - }, - Name = action.Name, - Version = action.Version, - }; - this.logHelper.SchedulingOrchestration(executionStartedEvent); - - this.orchestratorMessages.Add(new TaskMessage - { - OrchestrationInstance = destination, - Event = executionStartedEvent, - }); - } - - #endregion - - async Task ExecuteViaMiddlewareAsync(Work workToDoNow) - { - // the request object that will be passed to the worker - var request = new OperationBatchRequest() - { - InstanceId = this.instanceId, - EntityState = this.schedulerState.EntityState, - Operations = workToDoNow.GetOperationRequests(), - }; - - this.logHelper.EntityBatchExecuting(request); - - string entityName = EntityId.FromString(this.instanceId).EntityName; - string entityVersion = string.Empty; // TODO consider whether we should support explicit versions - - // Get the TaskOrchestration implementation. If it's not found, it either means that the developer never - // registered it (which is an error, and we'll throw for this further down) or it could be that some custom - // middleware (e.g. out-of-process execution middleware) is intended to implement the orchestration logic. - TaskEntity taskEntity = this.objectManager.GetObject(entityName, entityVersion); - - var dispatchContext = new DispatchMiddlewareContext(); - dispatchContext.SetProperty(request); - - await this.dispatchPipeline.RunAsync(dispatchContext, async _ => - { - // Check to see if the custom middleware intercepted and substituted the orchestration execution - // with its own execution behavior, providing us with the end results. If so, we can terminate - // the dispatch pipeline here. - var resultFromMiddleware = dispatchContext.GetProperty(); - if (resultFromMiddleware != null) - { - return; - } - - if (taskEntity == null) - { - throw TraceHelper.TraceExceptionInstance( - TraceEventType.Error, - "TaskOrchestrationDispatcher-EntityTypeMissing", - runtimeState.OrchestrationInstance, - new TypeMissingException($"Entity not found: {entityName}")); - } - - var options = new EntityExecutionOptions() - { - EntityBackendInformation = this.entityBackendInformation, - ErrorPropagationMode = this.errorPropagationMode, - }; - - var resultFromTaskEntityObject = await taskEntity.ExecuteOperationBatchAsync(request, options); - - dispatchContext.SetProperty(resultFromTaskEntityObject); - }); - - var result = dispatchContext.GetProperty(); - - this.logHelper.EntityBatchExecuted(request, result); - - return result; - } - } -} diff --git a/src/DurableTask.Core/Entities/IEntityExecutor.cs b/src/DurableTask.Core/Entities/IEntityExecutor.cs new file mode 100644 index 000000000..7729662e3 --- /dev/null +++ b/src/DurableTask.Core/Entities/IEntityExecutor.cs @@ -0,0 +1,30 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +using System; +using System.Threading; +using System.Threading.Tasks; +using DurableTask.Core.Entities.OperationFormat; + +namespace DurableTask.Core.Entities +{ + /// + /// Untyped interface for processing batches of entity operations. + /// + public interface IEntityExecutor + { + /// + /// processes a batch of entity opreations + /// + internal abstract Task ExecuteOperationBatchAsync(OperationBatchRequest operations, EntityExecutionOptions options); + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/IEntityOrchestrationService.cs b/src/DurableTask.Core/Entities/IEntityOrchestrationService.cs new file mode 100644 index 000000000..3944aea1e --- /dev/null +++ b/src/DurableTask.Core/Entities/IEntityOrchestrationService.cs @@ -0,0 +1,47 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace DurableTask.Core.Entities +{ + /// + /// Interface for objects that provide entity backend information. + /// + public interface IEntityOrchestrationService + { + /// + /// The entity orchestration service. + /// + /// The entity backend information object. + EntityBackendInformation GetEntityBackendInformation(); + + /// + /// Configures the orchestration service backend so entities and orchestrations are kept in two separate queues, and can be fetched separately. + /// + void ProcessEntitiesSeparately(); + + /// + /// Specialized variant of that + /// fetches only work items for true orchestrations, not entities. + /// + Task LockNextOrchestrationWorkItemAsync(TimeSpan receiveTimeout, CancellationToken cancellationToken); + + /// + /// Specialized variant of that + /// fetches only work items for entities, not plain orchestrations. + /// + Task LockNextEntityWorkItemAsync(TimeSpan receiveTimeout, CancellationToken cancellationToken); + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/LocalSDK/EntityContext.cs b/src/DurableTask.Core/Entities/LocalSDK/EntityContext.cs index cf872309c..b754d4fad 100644 --- a/src/DurableTask.Core/Entities/LocalSDK/EntityContext.cs +++ b/src/DurableTask.Core/Entities/LocalSDK/EntityContext.cs @@ -14,6 +14,7 @@ namespace DurableTask.Core { using DurableTask.Core.Entities; + using DurableTask.Core.Serializing; using System; using System.Reflection; using System.Threading.Tasks; @@ -22,17 +23,12 @@ namespace DurableTask.Core /// Context for an entity, which is available to the application code while it is executing entity operations. /// /// The JSON-serializable type of the entity state. - public abstract class EntityContext + public abstract class EntityContext { /// - /// Gets the name of the currently executing entity. + /// Gives access to various options to control entity execution. /// - public abstract string EntityName { get; } - - /// - /// Gets the key of the currently executing entity. - /// - public abstract string EntityKey { get; } + public abstract EntityExecutionOptions EntityExecutionOptions { get; } /// /// Gets the id of the currently executing entity. @@ -88,7 +84,7 @@ public abstract class EntityContext /// An operation invocation on an entity includes an operation name, which states what /// operation to perform, and optionally an operation input. /// - public abstract TInput GetInput(); + public virtual TInput GetInput() => (TInput)this.GetInput(typeof(TInput)); /// /// Gets the input for this operation, as a deserialized value. @@ -101,12 +97,6 @@ public abstract class EntityContext /// public abstract object GetInput(Type inputType); - /// - /// Returns the given result to the caller of this operation. - /// - /// the result to return. - public abstract void Return(object result); - /// /// Signals an entity to perform an operation, without waiting for a response. Any result or exception is ignored (fire and forget). /// @@ -125,7 +115,7 @@ public abstract class EntityContext public abstract void SignalEntity(EntityId entity, DateTime scheduledTimeUtc, string operationName, object operationInput = null); /// - /// Schedules an orchestration function with the given name and version for execution./>. + /// Schedules an orchestration function with the given name and version for execution. /// Any result or exception is ignored (fire and forget). /// /// The name of the orchestrator, as specified by the ObjectCreator. @@ -136,7 +126,7 @@ public abstract class EntityContext public abstract string StartNewOrchestration(string name, string version, object input, string instanceId = null); /// - /// Schedules an orchestration function with the given type for execution./>. + /// Schedules an orchestration function with the given type for execution. /// Any result or exception is ignored (fire and forget). /// /// Type of the TaskOrchestration derived class to instantiate diff --git a/src/DurableTask.Core/Entities/LocalSDK/TaskEntity.cs b/src/DurableTask.Core/Entities/LocalSDK/TaskEntity.cs new file mode 100644 index 000000000..9b29d23f7 --- /dev/null +++ b/src/DurableTask.Core/Entities/LocalSDK/TaskEntity.cs @@ -0,0 +1,63 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Entities +{ + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + using DurableTask.Core.Common; + using DurableTask.Core.Entities.EventFormat; + using DurableTask.Core.Entities.OperationFormat; + using DurableTask.Core.Exceptions; + using DurableTask.Core.Serializing; + using Newtonsoft.Json; + + /// + /// TaskEntity representing an entity with typed state. + /// + /// The type of the entity state. + public abstract class TaskEntity : TaskEntity + { + /// + /// Defines how this entity processes operations. Implementations must override this. + /// + /// The context for the operation. + /// A task that completes with the return value of the operation. + public abstract ValueTask ExecuteOperationAsync(EntityContext context); + + /// + /// A function for creating the initial state of the entity. + /// Implementations may override this if they want to perform a different initialization. + /// + /// This is only called when the entity state is created, not when it is reloaded from storage. + /// When loading the entity from storage, the entity state is created using a DataConverter. + public virtual TState CreateInitialState(EntityContext context) + { + return default; + } + + /// + /// We cache the entity context inside the TaskEntity object. That way, it is possible for applications + /// to use a caching implementation of if they want + /// to avoid constructing a fresh execution context, and allow deserialized state objects to be reused. + /// + internal TaskEntityContext CachedContext; + + internal override Task ExecuteOperationBatchAsync(OperationBatchRequest operations, EntityExecutionOptions options) + { + this.CachedContext ??= new TaskEntityContext(this, options); + return this.CachedContext.ExecuteBatchAsync(operations); + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/LocalSDK/TaskEntityContext.cs b/src/DurableTask.Core/Entities/LocalSDK/TaskEntityContext.cs index ef0f5a300..732a3b46b 100644 --- a/src/DurableTask.Core/Entities/LocalSDK/TaskEntityContext.cs +++ b/src/DurableTask.Core/Entities/LocalSDK/TaskEntityContext.cs @@ -28,34 +28,32 @@ internal class TaskEntityContext : EntityContext { readonly TaskEntity taskEntity; readonly EntityExecutionOptions executionOptions; - readonly OperationBatchRequest batchRequest; - readonly OperationBatchResult batchResult; - int batchPosition; + EntityId entityId; + int batchPosition = -1; + int batchSize; + string lastSerializedState; + OperationRequest currentOperationRequest; OperationResult currentOperationResult; StateAccess currentStateAccess; TState currentState; + List actions; - public TaskEntityContext(TaskEntity taskEntity, EntityId entityId, EntityExecutionOptions options, OperationBatchRequest batchRequest, OperationBatchResult batchResult) + public TaskEntityContext(TaskEntity taskEntity, EntityExecutionOptions options) { this.taskEntity = taskEntity; - this.EntityId = entityId; this.executionOptions = options; - this.batchRequest = batchRequest; - this.batchResult = batchResult; } - public override string EntityName => this.EntityId.EntityName; - public override string EntityKey => this.EntityId.EntityKey; - public override EntityId EntityId { get; } + public override EntityId EntityId => this.entityId; - public EntityExecutionOptions ExecutionOptions => this.executionOptions; + public override EntityExecutionOptions EntityExecutionOptions => this.executionOptions; - public override string OperationName => this.batchRequest.Operations[this.batchPosition].Operation; - public override int BatchSize => this.batchRequest.Operations.Count; + public override string OperationName => this.currentOperationRequest.Operation; + public override int BatchSize => this.batchSize; public override int BatchPosition => this.batchPosition; - OperationRequest CurrentOperation => this.batchRequest.Operations[batchPosition]; + OperationRequest CurrentOperation => this.currentOperationRequest; // The last serialized checkpoint of the entity state is always stored in // this.LastSerializedState. The current state is determined by this.CurrentStateAccess and this.CurrentState. @@ -67,12 +65,6 @@ internal enum StateAccess Deleted, // current state is deleted } - internal string LastSerializedState - { - get { return this.batchResult.EntityState; } - set { this.batchResult.EntityState = value; } - } - public override bool HasState { get @@ -86,7 +78,7 @@ public override bool HasState case StateAccess.Deleted: return false; - default: return this.LastSerializedState != null; + default: return this.lastSerializedState != null; } } } @@ -107,11 +99,11 @@ public override TState State TState result; - if (this.LastSerializedState != null && this.currentStateAccess != StateAccess.Deleted) + if (this.lastSerializedState != null && this.currentStateAccess != StateAccess.Deleted) { try { - result = taskEntity.StateDataConverter.Deserialize(this.LastSerializedState); + result = this.executionOptions.StateDataConverter.Deserialize(this.lastSerializedState); } catch (Exception e) { @@ -155,25 +147,25 @@ public void Rollback(int positionBeforeCurrentOperation) // we also roll back the list of outgoing messages, // so any signals sent by this operation are discarded. - this.batchResult.Actions.RemoveRange(positionBeforeCurrentOperation, this.batchResult.Actions.Count - positionBeforeCurrentOperation); + this.actions.RemoveRange(positionBeforeCurrentOperation, this.actions.Count - positionBeforeCurrentOperation); } private bool TryWriteback(out OperationResult serializationErrorResult, OperationRequest operationRequest = null) { if (this.currentStateAccess == StateAccess.Deleted) { - this.LastSerializedState = null; + this.lastSerializedState = null; this.currentStateAccess = StateAccess.NotAccessed; } else if (this.currentStateAccess == StateAccess.Accessed) { try { - string serializedState = this.taskEntity.StateDataConverter.Serialize(this.currentState); - this.LastSerializedState = serializedState; - this.currentStateAccess = StateAccess.Clean; + string serializedState = this.executionOptions.StateDataConverter.Serialize(this.currentState); + this.lastSerializedState = serializedState; + this.currentStateAccess = StateAccess.Clean; } - catch (Exception serializationException) + catch (Exception serializationException) when (!Utils.IsFatal(serializationException)) { // we cannot serialize the entity state - this is an application error. To help users diagnose this, // we wrap it into a descriptive exception, and propagate this error result to any calling orchestrations. @@ -199,23 +191,11 @@ private bool TryWriteback(out OperationResult serializationErrorResult, Operatio return true; } - public override TInput GetInput() - { - try - { - return this.taskEntity.MessageDataConverter.Deserialize(this.CurrentOperation.Input); - } - catch(Exception e) - { - throw new EntitySchedulerException($"Failed to deserialize input for operation '{this.CurrentOperation.Operation}': {e.Message}", e); - } - } - public override object GetInput(Type inputType) { try { - return this.taskEntity.MessageDataConverter.Deserialize(this.CurrentOperation.Input, inputType); + return this.executionOptions.MessageDataConverter.Deserialize(this.CurrentOperation.Input, inputType); } catch (Exception e) { @@ -223,25 +203,13 @@ public override object GetInput(Type inputType) } } - public override void Return(object result) - { - try - { - this.currentOperationResult.Result = this.taskEntity.MessageDataConverter.Serialize(result); - } - catch (Exception e) - { - throw new EntitySchedulerException($"Failed to serialize output for operation '{this.CurrentOperation.Operation}': {e.Message}", e); - } - } - public override void SignalEntity(EntityId entity, string operationName, object operationInput = null) { this.SignalEntityInternal(entity, null, operationName, operationInput); } - + public override void SignalEntity(EntityId entity, DateTime scheduledTimeUtc, string operationName, object operationInput = null) { this.SignalEntityInternal(entity, scheduledTimeUtc, operationName, operationInput); @@ -254,7 +222,7 @@ private void SignalEntityInternal(EntityId entity, DateTime? scheduledTimeUtc, s throw new ArgumentNullException(nameof(operationName)); } - string functionName = entity.EntityName; + string functionName = entity.Name; var action = new SendSignalOperationAction() { @@ -268,7 +236,7 @@ private void SignalEntityInternal(EntityId entity, DateTime? scheduledTimeUtc, s { try { - action.Input = taskEntity.MessageDataConverter.Serialize(operationInput); + action.Input = this.executionOptions.MessageDataConverter.Serialize(operationInput); } catch (Exception e) { @@ -277,9 +245,9 @@ private void SignalEntityInternal(EntityId entity, DateTime? scheduledTimeUtc, s } // add the action to the results, under a lock since user code may be concurrent - lock (this.batchResult.Actions) + lock (this.actions) { - this.batchResult.Actions.Add(action); + this.actions.Add(action); } } @@ -306,7 +274,7 @@ public override string StartNewOrchestration(string name, string version, object { try { - action.Input = taskEntity.MessageDataConverter.Serialize(input); + action.Input = this.executionOptions.MessageDataConverter.Serialize(input); } catch (Exception e) { @@ -315,23 +283,49 @@ public override string StartNewOrchestration(string name, string version, object } // add the action to the results, under a lock since user code may be concurrent - lock (this.batchResult.Actions) + lock (this.actions) { - this.batchResult.Actions.Add(action); + this.actions.Add(action); } return instanceId; } - public async Task ExecuteBatchAsync() + public async Task ExecuteBatchAsync(OperationBatchRequest batchRequest) { + var entityId = EntityId.FromString(batchRequest.InstanceId); + + if (entityId.Equals(this.entityId) && batchRequest.EntityState == this.lastSerializedState) + { + // we were called before, with the same entityId, and the same state. + // We can therefore keep the current state as is. + } + else + { + this.entityId = entityId; + this.lastSerializedState = batchRequest.EntityState; + this.currentState = default; + this.currentStateAccess = StateAccess.NotAccessed; + } + + this.batchSize = batchRequest.Operations.Count; + var actions = this.actions = new List(); + var results = new List(); + // execute all the operations in a loop and record the results. - for (int i = 0; i < this.batchRequest.Operations.Count; i++) + for (int i = 0; i < batchRequest.Operations.Count; i++) { - await this.ProcessOperationRequestAsync(i); + this.batchPosition = i; + this.currentOperationRequest = batchRequest.Operations[i]; + this.currentOperationResult = new OperationResult(); + + // process the operation (handling any errors if necessary) + await this.ProcessOperationRequestAsync(); + + results.Add(this.currentOperationResult); } - if (this.taskEntity.RollbackOnExceptions) + if (this.executionOptions.RollbackOnExceptions) { // the state has already been written back, since it is // done right after each operation. @@ -342,49 +336,70 @@ public async Task ExecuteBatchAsync() var writeBackSuccessful = this.TryWriteback(out OperationResult serializationErrorMessage); - if (!writeBackSuccessful) + if (!writeBackSuccessful) { // failed to write state, we now consider all operations in the batch as failed. // We thus record the results (which are now fail results) and commit the batch // with the original state. // we clear the actions (if we cannot update the state, we should not take other actions either) - this.batchResult.Actions.Clear(); + actions.Clear(); // we replace all response messages with the serialization error message, // so that callers get to know that this operation failed, and why - for (int i = 0; i < this.batchResult.Results.Count; i++) + for (int i = 0; i < results.Count; i++) { - this.batchResult.Results[i] = serializationErrorMessage; + results[i] = serializationErrorMessage; } } } + + // clear these fields before returning, so if this context is being cached somewhere, it keeps only the relevant information. + this.batchPosition = -1; + this.batchSize = 0; + this.currentOperationRequest = null; + this.currentOperationResult = null; + this.actions = null; + + return new OperationBatchResult() + { + Results = results, + Actions = actions, + EntityState = this.lastSerializedState, + }; } - async ValueTask ProcessOperationRequestAsync(int index) + async ValueTask ProcessOperationRequestAsync() { - // set context for operation - var operation = this.batchRequest.Operations[index]; - this.batchPosition = index; - this.currentOperationResult = new OperationResult(); - - var actionPositionCheckpoint = this.batchResult.Actions.Count; + var actionPositionCheckpoint = this.actions.Count; try { - await taskEntity.ExecuteOperationAsync(this); + object returnedResult = await taskEntity.ExecuteOperationAsync(this); + + if (returnedResult != null) + { + try + { + this.currentOperationResult.Result = this.executionOptions.MessageDataConverter.Serialize(returnedResult); + } + catch (Exception e) + { + throw new EntitySchedulerException($"Failed to serialize output for operation '{this.CurrentOperation.Operation}': {e.Message}", e); + } + } } catch (Exception e) when (!Utils.IsFatal(e) && !Utils.IsExecutionAborting(e)) { this.CaptureExceptionInOperationResult(this.currentOperationResult, e); } - if (this.taskEntity.RollbackOnExceptions) + if (this.executionOptions.RollbackOnExceptions) { // we write back the entity state after each successful operation if (this.currentOperationResult.ErrorMessage == null) { - if (!this.TryWriteback(out OperationResult errorResult, operation)) + if (!this.TryWriteback(out OperationResult errorResult, this.currentOperationRequest)) { // state serialization failed; create error response and roll back. this.currentOperationResult = errorResult; @@ -397,9 +412,6 @@ async ValueTask ProcessOperationRequestAsync(int index) this.Rollback(actionPositionCheckpoint); } } - - // write the result to the list of results for the batch - this.batchResult.Results.Add(this.currentOperationResult); } public void CaptureExceptionInOperationResult(OperationResult result, Exception originalException) @@ -413,7 +425,7 @@ public void CaptureExceptionInOperationResult(OperationResult result, Exception case ErrorPropagationMode.SerializeExceptions: try { - result.Result = this.taskEntity.ErrorDataConverter.Serialize(originalException); + result.Result = this.executionOptions.ErrorDataConverter.Serialize(originalException); } catch (Exception serializationException) when (!Utils.IsFatal(serializationException)) { @@ -422,10 +434,10 @@ public void CaptureExceptionInOperationResult(OperationResult result, Exception // because this information may help users that are trying to troubleshoot their application. try { - result.Result = this.taskEntity.ErrorDataConverter.Serialize(serializationException); + result.Result = this.executionOptions.ErrorDataConverter.Serialize(serializationException); } catch (Exception serializationExceptionSerializationException) when (!Utils.IsFatal(serializationExceptionSerializationException)) - { + { // there seems to be nothing we can do. } } diff --git a/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs b/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs index 6f2abbd48..dc6c27cf2 100644 --- a/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs +++ b/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs @@ -51,6 +51,7 @@ private void CheckEntitySupport(string name) throw new InvalidOperationException($"{nameof(TaskHubEntityClient)}.{name} is not supported because the chosen backend does not support entities."); } } + private void CheckQuerySupport(string name) { if (this.queryClient == null) @@ -58,6 +59,7 @@ private void CheckQuerySupport(string name) throw new InvalidOperationException($"{nameof(TaskHubEntityClient)}.{name} is not supported because the chosen backend does not implement {nameof(IOrchestrationServiceQueryClient)}."); } } + private void CheckPurgeSupport(string name) { if (this.purgeClient == null) @@ -77,7 +79,7 @@ public TaskHubEntityClient(TaskHubClient client, DataConverter stateDataConverte this.messageDataConverter = client.DefaultConverter; this.stateDataConverter = stateDataConverter ?? client.DefaultConverter; this.logHelper = client.LogHelper; - this.backendInformation = (client.ServiceClient as EntityBackendInformation.IInformationProvider)?.GetEntityBackendInformation(); + this.backendInformation = (client.ServiceClient as IEntityOrchestrationService)?.GetEntityBackendInformation(); this.queryClient = client.ServiceClient as IOrchestrationServiceQueryClient; this.purgeClient = client.ServiceClient as IOrchestrationServicePurgeClient; } @@ -97,7 +99,9 @@ public async Task SignalEntityAsync(EntityId entityId, string operationName, obj (DateTime original, DateTime capped)? scheduledTime = null; if (scheduledTimeUtc.HasValue) { - scheduledTime = this.backendInformation.GetCappedScheduledTime(DateTime.UtcNow, scheduledTimeUtc.Value.ToUniversalTime()); + DateTime original = scheduledTimeUtc.Value.ToUniversalTime(); + DateTime capped = this.backendInformation.GetCappedScheduledTime(DateTime.UtcNow, original); + scheduledTime = (original, capped); } var guid = Guid.NewGuid(); // unique id for this request @@ -463,8 +467,8 @@ public async Task CleanEntityStorageAsync(bool removeE async Task DeleteIdleOrchestrationEntity(OrchestrationState state) { - await this.purgeClient.PurgeInstanceStateAsync(state.OrchestrationInstance.InstanceId); - Interlocked.Increment(ref finalResult.NumberOfEmptyEntitiesRemoved); + var purgeResult = await this.purgeClient.PurgeInstanceStateAsync(state.OrchestrationInstance.InstanceId); + Interlocked.Add(ref finalResult.NumberOfEmptyEntitiesRemoved, purgeResult.DeletedInstanceCount); } async Task CheckForOrphanedLockAndFixIt(string instanceId, string lockOwner) diff --git a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs index 40096d6fb..fae85699e 100644 --- a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs +++ b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs @@ -77,7 +77,7 @@ public OrchestrationEntityContext( { foreach(var e in this.availableLocks) { - yield return (e.EntityName, e.EntityKey); + yield return (e.Name, e.Key); } } } diff --git a/src/DurableTask.Core/Entities/TaskEntity.cs b/src/DurableTask.Core/Entities/TaskEntity.cs index 11e5ff19c..d5ed96af1 100644 --- a/src/DurableTask.Core/Entities/TaskEntity.cs +++ b/src/DurableTask.Core/Entities/TaskEntity.cs @@ -13,110 +13,18 @@ namespace DurableTask.Core.Entities { - using System; - using System.Collections.Generic; using System.Threading.Tasks; - using DurableTask.Core.Common; - using DurableTask.Core.Entities.EventFormat; using DurableTask.Core.Entities.OperationFormat; - using DurableTask.Core.Exceptions; - using DurableTask.Core.Serializing; - using Newtonsoft.Json; /// - /// Base class for TaskEntity. + /// Abstract base class for entities. /// - /// For in-process user code, we recommend using the specialized - /// class which has a per-operation interface, - /// provides type-safe access to the state, and handles state management, including initialization, - /// serialization, and deserialization. + /// To implement task entities, use the subclass . public abstract class TaskEntity { /// - /// Executes the given operation batch and return the results. + /// Internal untyped interface for entity batch operations. /// - /// The batch of operations to execute. - /// Options to control entity execution. - /// - public abstract Task ExecuteOperationBatchAsync(OperationBatchRequest operations, EntityExecutionOptions options); - } - - /// - /// TaskEntity representing an entity with typed state and options for controlling serialization and batch execution. - /// - /// The type of the entity state. - public abstract class TaskEntity : TaskEntity - { - /// - /// Defines how this entity processes operations. Implementations must override this. - /// - /// The context for the operation. - /// a task that indicates completion of the operation - public abstract ValueTask ExecuteOperationAsync(EntityContext context); - - /// - /// A function for creating the initial state of the entity. This is - /// automatically called when the entity is first accessed, or - /// when it is accessed after having been deleted. - /// Implementations may override this if they want to perform a different initialization. - /// - public virtual TState CreateInitialState(EntityContext context) - { - return default(TState); - } - - /// - /// The data converter used for converting inputs and outputs for operations. - /// Implementations may override this setting. - /// - public virtual DataConverter MessageDataConverter => JsonDataConverter.Default; - - /// - /// The data converter used for the entity state. - /// Implementations may override this setting. - /// - public virtual DataConverter StateDataConverter => this.MessageDataConverter; - - /// - /// The data converter used for exceptions. - /// Implementations may override this setting. - /// - public virtual DataConverter ErrorDataConverter => this.MessageDataConverter; - - /// - /// If true, all effects of an entity operation (all state changes and all actions) are rolled back - /// if the entity operation completes with an exception. - /// Implementations may override this setting. - /// - public virtual bool RollbackOnExceptions => true; - - /// - /// Options for executing entities. - /// - public EntityExecutionOptions EntityExecutionOptions { get; set; } - - /// - public override async Task ExecuteOperationBatchAsync(OperationBatchRequest operations, EntityExecutionOptions options) - { - this.EntityExecutionOptions = options; - - var result = new OperationBatchResult() - { - Results = new List(), - Actions = new List(), - EntityState = operations.EntityState, - }; - - var entityContext = new TaskEntityContext( - this, - EntityId.FromString(operations.InstanceId), - options, - operations, - result); - - await entityContext.ExecuteBatchAsync(); - - return result; - } + internal abstract Task ExecuteOperationBatchAsync(OperationBatchRequest operations, EntityExecutionOptions options); } } \ No newline at end of file diff --git a/src/DurableTask.Core/Logging/StructuredEventSource.cs b/src/DurableTask.Core/Logging/StructuredEventSource.cs index a738425df..26dc3178c 100644 --- a/src/DurableTask.Core/Logging/StructuredEventSource.cs +++ b/src/DurableTask.Core/Logging/StructuredEventSource.cs @@ -687,7 +687,7 @@ internal void EntityLockAcquired( { // TODO: Use WriteEventCore for better performance this.WriteEvent( - EventIds.EntityBatchExecuted, + EventIds.EntityLockAcquired, EntityId, InstanceId, ExecutionId, @@ -711,7 +711,7 @@ internal void EntityLockReleased( { // TODO: Use WriteEventCore for better performance this.WriteEvent( - EventIds.EntityBatchExecuted, + EventIds.EntityLockReleased, EntityId, InstanceId, Id, diff --git a/src/DurableTask.Core/NameObjectManager.cs b/src/DurableTask.Core/NameObjectManager.cs new file mode 100644 index 000000000..c8a5d5742 --- /dev/null +++ b/src/DurableTask.Core/NameObjectManager.cs @@ -0,0 +1,65 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core +{ + using System; + using System.Collections.Generic; + + internal class NameObjectManager : INameVersionObjectManager + { + readonly IDictionary> creators; + readonly object thisLock = new object(); + + public NameObjectManager() + { + this.creators = new Dictionary>(); + } + + public void Add(ObjectCreator creator) + { + lock (this.thisLock) + { + string key = GetKey(creator.Name, creator.Version); + + if (this.creators.ContainsKey(key)) + { + throw new InvalidOperationException("Duplicate entry detected: " + creator.Name + " " + + creator.Version); + } + + this.creators.Add(key, creator); + } + } + + public T GetObject(string name, string version) + { + string key = GetKey(name, version); + + lock (this.thisLock) + { + if (this.creators.TryGetValue(key, out ObjectCreator creator)) + { + return creator.Create(); + } + + return default(T); + } + } + + string GetKey(string name, string version) + { + return name; + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/OrchestrationContext.cs b/src/DurableTask.Core/OrchestrationContext.cs index bb057b4a5..4aab9b155 100644 --- a/src/DurableTask.Core/OrchestrationContext.cs +++ b/src/DurableTask.Core/OrchestrationContext.cs @@ -385,7 +385,8 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve /// The input for the operation. /// if the orchestration is inside a critical section and the lock for this entity is not available. /// A task representing the result of the operation. - public abstract Task CallEntityAsync(Entities.EntityId entityId, string operationName, object operationInput = null); + public virtual Task CallEntityAsync(Entities.EntityId entityId, string operationName, object operationInput = null) + => throw new NotImplementedException(); /// /// Calls an operation on an entity and waits for it to complete. @@ -395,7 +396,8 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve /// The input for the operation. /// A task representing the completion of the operation on the entity. /// if the orchestration is inside a critical section and the lock for this entity is not available. - public abstract Task CallEntityAsync(Entities.EntityId entityId, string operationName, object operationInput = null); + public virtual Task CallEntityAsync(Entities.EntityId entityId, string operationName, object operationInput = null) + => throw new NotImplementedException(); /// /// Signals an entity operation on the specified entity. @@ -404,7 +406,8 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve /// The name of the operation. /// The input for the operation. /// if the orchestration is inside a critical section that locked this entity. - public abstract void SignalEntity(Entities.EntityId entityId, string operationName, object operationInput = null); + public virtual void SignalEntity(Entities.EntityId entityId, string operationName, object operationInput = null) + => throw new NotImplementedException(); /// /// Signals an entity to perform an operation, at a specified time. Any result or exception is ignored (fire and forget). @@ -414,7 +417,8 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve /// The name of the operation. /// if the orchestration is inside a critical section that locked this entity. /// The input for the operation. - public abstract void SignalEntity(Entities.EntityId entityId, DateTime scheduledTimeUtc, string operationName, object operationInput = null); + public virtual void SignalEntity(Entities.EntityId entityId, DateTime scheduledTimeUtc, string operationName, object operationInput = null) + => throw new NotImplementedException(); /// /// Locks one or more entities for the duration of the critical section. @@ -425,13 +429,14 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve /// The entities whose locks should be acquired. /// A task that must be awaited prior to entering the critical section, and which returns a IDisposable that must be disposed after exiting the critical section. /// if the context is already in a critical section. - public abstract Task LockEntitiesAsync(params EntityId[] entities); + public virtual Task LockEntitiesAsync(params EntityId[] entities) + => throw new NotImplementedException(); /// /// Whether this orchestration is currently inside a critical section. Critical sections are entered when calling /// , and are exited when disposing the returned IDisposable. /// - public abstract bool IsInsideCriticalSection { get; } + public virtual bool IsInsideCriticalSection { get { throw new NotImplementedException(); } } /// /// Enumerates all the entities that can be called from within the current critical section. @@ -439,7 +444,8 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve /// and for which there is not currently an operation call pending. /// /// An enumeration of all the currently available entities. - public abstract IEnumerable GetAvailableEntities(); + public virtual IEnumerable GetAvailableEntities() + => throw new NotImplementedException(); /// /// Raises an event for the specified orchestration instance, which eventually causes the OnEvent() method in the diff --git a/src/DurableTask.Core/OrchestrationWorkItemProcessor.cs b/src/DurableTask.Core/OrchestrationWorkItemProcessor.cs deleted file mode 100644 index 9e133b229..000000000 --- a/src/DurableTask.Core/OrchestrationWorkItemProcessor.cs +++ /dev/null @@ -1,604 +0,0 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- -#nullable enable -using DurableTask.Core.Command; -using DurableTask.Core.Common; -using DurableTask.Core.Exceptions; -using DurableTask.Core.History; -using DurableTask.Core.Logging; -using DurableTask.Core.Middleware; -using DurableTask.Core.Serializing; -using DurableTask.Core.Tracing; -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; - -namespace DurableTask.Core -{ - /// - /// Handles orchestration work items. - /// - class OrchestrationWorkItemProcessor : TaskOrchestrationDispatcher.WorkItemProcessor - { - static readonly Task CompletedTask = Task.FromResult(0); - - readonly IOrchestrationService orchestrationService; - readonly LogHelper logHelper; - ErrorPropagationMode errorPropagationMode; - readonly INameVersionObjectManager objectManager; - readonly DispatchMiddlewarePipeline dispatchPipeline; - - public OrchestrationWorkItemProcessor( - TaskOrchestrationDispatcher dispatcher, - TaskOrchestrationWorkItem workItem, - IOrchestrationService orchestrationService, - ErrorPropagationMode errorPropagationMode, - LogHelper logHelper, - INameVersionObjectManager orchestrationObjectManager, - DispatchMiddlewarePipeline dispatchPipeline) : base(dispatcher, workItem) - { - this.orchestrationService = orchestrationService; - this.errorPropagationMode = errorPropagationMode; - this.logHelper = logHelper; - this.objectManager = orchestrationObjectManager; - this.dispatchPipeline = dispatchPipeline; - } - - public override async Task ProcessWorkItemAsync() - { - ExecutionStartedEvent? continueAsNewExecutionStarted = null; - IList? carryOverEvents = null; - string? carryOverStatus = null; - - do - { - this.continuedAsNew = false; - this.continuedAsNewMessage = null; - - this.logHelper.OrchestrationExecuting(this.runtimeState.OrchestrationInstance!, this.runtimeState.Name); - TraceHelper.TraceInstance( - TraceEventType.Verbose, - "TaskOrchestrationDispatcher-ExecuteUserOrchestration-Begin", - this.runtimeState.OrchestrationInstance, - "Executing user orchestration: {0}", - JsonDataConverter.Default.Serialize(this.runtimeState.GetOrchestrationRuntimeStateDump(), true)); - - if (this.workItem.Cursor == null) - { - this.workItem.Cursor = await this.ExecuteOrchestrationAsync(this.runtimeState, this.workItem); - } - else - { - await this.ResumeOrchestrationAsync(this.workItem); - } - - IReadOnlyList decisions = this.workItem.Cursor.LatestDecisions.ToList(); - - this.logHelper.OrchestrationExecuted( - this.runtimeState.OrchestrationInstance!, - this.runtimeState.Name, - decisions); - TraceHelper.TraceInstance( - TraceEventType.Information, - "TaskOrchestrationDispatcher-ExecuteUserOrchestration-End", - this.runtimeState.OrchestrationInstance, - "Executed user orchestration. Received {0} orchestrator actions: {1}", - decisions.Count, - string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType))); - - // TODO: Exception handling for invalid decisions, which is increasingly likely - // when custom middleware is involved (e.g. out-of-process scenarios). - foreach (OrchestratorAction decision in decisions) - { - TraceHelper.TraceInstance( - TraceEventType.Information, - "TaskOrchestrationDispatcher-ProcessOrchestratorAction", - this.runtimeState.OrchestrationInstance, - "Processing orchestrator action of type {0}", - decision.OrchestratorActionType); - switch (decision.OrchestratorActionType) - { - case OrchestratorActionType.ScheduleOrchestrator: - var scheduleTaskAction = (ScheduleTaskOrchestratorAction)decision; - var message = this.ProcessScheduleTaskDecision( - scheduleTaskAction, - this.runtimeState, - this.dispatcher.IncludeParameters); - this.messagesToSend.Add(message); - break; - case OrchestratorActionType.CreateTimer: - var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision; - this.timerMessages.Add(this.ProcessCreateTimerDecision( - timerOrchestratorAction, - this.runtimeState, - isInternal: false)); - break; - case OrchestratorActionType.CreateSubOrchestration: - var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; - this.orchestratorMessages.Add( - this.ProcessCreateSubOrchestrationInstanceDecision( - createSubOrchestrationAction, - this.runtimeState, - this.dispatcher.IncludeParameters)); - break; - case OrchestratorActionType.SendEvent: - var sendEventAction = (SendEventOrchestratorAction)decision; - this.orchestratorMessages.Add( - this.ProcessSendEventDecision(sendEventAction, this.runtimeState)); - break; - case OrchestratorActionType.OrchestrationComplete: - OrchestrationCompleteOrchestratorAction completeDecision = (OrchestrationCompleteOrchestratorAction)decision; - TaskMessage? workflowInstanceCompletedMessage = - this.ProcessWorkflowCompletedTaskDecision(completeDecision, this.runtimeState, this.dispatcher.IncludeDetails, out this.continuedAsNew); - if (workflowInstanceCompletedMessage != null) - { - // Send complete message to parent workflow or to itself to start a new execution - // Store the event so we can rebuild the state - carryOverEvents = null; - if (this.continuedAsNew) - { - this.continuedAsNewMessage = workflowInstanceCompletedMessage; - continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent; - if (completeDecision.CarryoverEvents.Any()) - { - carryOverEvents = completeDecision.CarryoverEvents.ToList(); - completeDecision.CarryoverEvents.Clear(); - } - } - else - { - this.orchestratorMessages.Add(workflowInstanceCompletedMessage); - } - } - - this.isCompleted = !this.continuedAsNew; - break; - default: - throw TraceHelper.TraceExceptionInstance( - TraceEventType.Error, - "TaskOrchestrationDispatcher-UnsupportedDecisionType", - this.runtimeState.OrchestrationInstance, - new NotSupportedException($"Decision type '{decision.OrchestratorActionType}' not supported")); - } - - // Underlying orchestration service provider may have a limit of messages per call, to avoid the situation - // we keep on asking the provider if message count is ok and stop processing new decisions if not. - // - // We also put in a fake timer to force next orchestration task for remaining messages - int totalMessages = this.messagesToSend.Count + this.orchestratorMessages.Count + this.timerMessages.Count; - if (this.orchestrationService.IsMaxMessageCountExceeded(totalMessages, this.runtimeState)) - { - TraceHelper.TraceInstance( - TraceEventType.Information, - "TaskOrchestrationDispatcher-MaxMessageCountReached", - this.runtimeState.OrchestrationInstance, - "MaxMessageCount reached. Adding timer to process remaining events in next attempt."); - - if (this.isCompleted || this.continuedAsNew) - { - TraceHelper.TraceInstance( - TraceEventType.Information, - "TaskOrchestrationDispatcher-OrchestrationAlreadyCompleted", - this.runtimeState.OrchestrationInstance, - "Orchestration already completed. Skip adding timer for splitting messages."); - break; - } - - var dummyTimer = new CreateTimerOrchestratorAction - { - Id = FrameworkConstants.FakeTimerIdToSplitDecision, - FireAt = DateTime.UtcNow - }; - - this.timerMessages.Add(this.ProcessCreateTimerDecision( - dummyTimer, - this.runtimeState, - isInternal: true)); - this.isInterrupted = true; - break; - } - } - - // correlation - CorrelationTraceClient.Propagate(() => { - if (this.runtimeState.ExecutionStartedEvent != null) - this.runtimeState.ExecutionStartedEvent.Correlation = CorrelationTraceContext.Current.SerializableTraceContext; - }); - - // finish up processing of the work item - if (!this.continuedAsNew && this.runtimeState.Events.Last().EventType != EventType.OrchestratorCompleted) - { - this.runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); - } - - if (this.isCompleted) - { - TraceHelper.TraceSession(TraceEventType.Information, "TaskOrchestrationDispatcher-DeletingSessionState", this.workItem.InstanceId, "Deleting session state"); - if (this.runtimeState.ExecutionStartedEvent != null) - { - this.instanceState = Utils.BuildOrchestrationState(this.runtimeState); - } - } - else - { - if (this.continuedAsNew) - { - TraceHelper.TraceSession( - TraceEventType.Information, - "TaskOrchestrationDispatcher-UpdatingStateForContinuation", - this.workItem.InstanceId, - "Updating state for continuation"); - - // correlation - CorrelationTraceClient.Propagate(() => - { - continueAsNewExecutionStarted!.Correlation = CorrelationTraceContext.Current.SerializableTraceContext; - }); - - this.runtimeState = new OrchestrationRuntimeState(); - this.runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); - this.runtimeState.AddEvent(continueAsNewExecutionStarted!); - this.runtimeState.Status = this.workItem.OrchestrationRuntimeState.Status ?? carryOverStatus; - carryOverStatus = this.workItem.OrchestrationRuntimeState.Status; - - if (carryOverEvents != null) - { - foreach (var historyEvent in carryOverEvents) - { - this.runtimeState.AddEvent(historyEvent); - } - } - - this.runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); - this.workItem.OrchestrationRuntimeState = this.runtimeState; - - this.workItem.Cursor = null; - } - - this.instanceState = Utils.BuildOrchestrationState(this.runtimeState); - } - } while (this.continuedAsNew); - - this.runtimeState.Status = this.runtimeState.Status ?? carryOverStatus; - - if (this.instanceState != null) - { - this.instanceState.Status = this.runtimeState.Status; - } - } - - static OrchestrationExecutionContext GetOrchestrationExecutionContext(OrchestrationRuntimeState runtimeState) - { - return new OrchestrationExecutionContext { OrchestrationTags = runtimeState.Tags ?? new Dictionary(capacity: 0) }; - } - - async Task ExecuteOrchestrationAsync(OrchestrationRuntimeState runtimeState, TaskOrchestrationWorkItem workItem) - { - // Get the TaskOrchestration implementation. If it's not found, it either means that the developer never - // registered it (which is an error, and we'll throw for this further down) or it could be that some custom - // middleware (e.g. out-of-process execution middleware) is intended to implement the orchestration logic. - TaskOrchestration? taskOrchestration = this.objectManager.GetObject(runtimeState.Name, runtimeState.Version!); - - var dispatchContext = new DispatchMiddlewareContext(); - dispatchContext.SetProperty(runtimeState.OrchestrationInstance); - dispatchContext.SetProperty(taskOrchestration); - dispatchContext.SetProperty(runtimeState); - dispatchContext.SetProperty(workItem); - dispatchContext.SetProperty(GetOrchestrationExecutionContext(runtimeState)); - - TaskOrchestrationExecutor? executor = null; - - await this.dispatchPipeline.RunAsync(dispatchContext, _ => - { - // Check to see if the custom middleware intercepted and substituted the orchestration execution - // with its own execution behavior, providing us with the end results. If so, we can terminate - // the dispatch pipeline here. - var resultFromMiddleware = dispatchContext.GetProperty(); - if (resultFromMiddleware != null) - { - return CompletedTask; - } - - if (taskOrchestration == null) - { - throw TraceHelper.TraceExceptionInstance( - TraceEventType.Error, - "TaskOrchestrationDispatcher-TypeMissing", - runtimeState.OrchestrationInstance, - new TypeMissingException($"Orchestration not found: ({runtimeState.Name}, {runtimeState.Version})")); - } - - executor = new TaskOrchestrationExecutor( - runtimeState, - taskOrchestration, - this.orchestrationService.EventBehaviourForContinueAsNew, - this.dispatcher.EntityBackendInformation, - this.errorPropagationMode); - OrchestratorExecutionResult resultFromOrchestrator = executor.Execute(); - dispatchContext.SetProperty(resultFromOrchestrator); - return CompletedTask; - }); - - var result = dispatchContext.GetProperty(); - IEnumerable decisions = result?.Actions ?? Enumerable.Empty(); - runtimeState.Status = result?.CustomStatus; - - return new OrchestrationExecutionCursor(runtimeState, taskOrchestration, executor, decisions); - } - - async Task ResumeOrchestrationAsync(TaskOrchestrationWorkItem workItem) - { - OrchestrationExecutionCursor cursor = workItem.Cursor; - - var dispatchContext = new DispatchMiddlewareContext(); - dispatchContext.SetProperty(cursor.RuntimeState.OrchestrationInstance); - dispatchContext.SetProperty(cursor.TaskOrchestration); - dispatchContext.SetProperty(cursor.RuntimeState); - dispatchContext.SetProperty(workItem); - - cursor.LatestDecisions = Enumerable.Empty(); - await this.dispatchPipeline.RunAsync(dispatchContext, _ => - { - OrchestratorExecutionResult result = cursor.OrchestrationExecutor.ExecuteNewEvents(); - dispatchContext.SetProperty(result); - return CompletedTask; - }); - - var result = dispatchContext.GetProperty(); - cursor.LatestDecisions = result?.Actions ?? Enumerable.Empty(); - cursor.RuntimeState.Status = result?.CustomStatus; - } - - - TaskMessage? ProcessWorkflowCompletedTaskDecision( - OrchestrationCompleteOrchestratorAction completeOrchestratorAction, - OrchestrationRuntimeState runtimeState, - bool includeDetails, - out bool continuedAsNew) - { - ExecutionCompletedEvent executionCompletedEvent; - continuedAsNew = (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew); - if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew) - { - executionCompletedEvent = new ContinueAsNewEvent(completeOrchestratorAction.Id, - completeOrchestratorAction.Result); - } - else - { - executionCompletedEvent = new ExecutionCompletedEvent(completeOrchestratorAction.Id, - completeOrchestratorAction.Result, - completeOrchestratorAction.OrchestrationStatus, - completeOrchestratorAction.FailureDetails); - } - - runtimeState.AddEvent(executionCompletedEvent); - - this.logHelper.OrchestrationCompleted(runtimeState, completeOrchestratorAction); - TraceHelper.TraceInstance( - runtimeState.OrchestrationStatus == OrchestrationStatus.Failed ? TraceEventType.Warning : TraceEventType.Information, - "TaskOrchestrationDispatcher-InstanceCompleted", - runtimeState.OrchestrationInstance, - "Instance Id '{0}' completed in state {1} with result: {2}", - runtimeState.OrchestrationInstance, - runtimeState.OrchestrationStatus, - completeOrchestratorAction.Result); - TraceHelper.TraceInstance( - TraceEventType.Information, - "TaskOrchestrationDispatcher-InstanceCompletionEvents", - runtimeState.OrchestrationInstance, - () => Utils.EscapeJson(JsonDataConverter.Default.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true))); - - // Check to see if we need to start a new execution - if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew) - { - var taskMessage = new TaskMessage(); - var startedEvent = new ExecutionStartedEvent(-1, completeOrchestratorAction.Result) - { - OrchestrationInstance = new OrchestrationInstance - { - InstanceId = runtimeState.OrchestrationInstance!.InstanceId, - ExecutionId = Guid.NewGuid().ToString("N") - }, - Tags = runtimeState.Tags, - ParentInstance = runtimeState.ParentInstance, - Name = runtimeState.Name, - Version = completeOrchestratorAction.NewVersion ?? runtimeState.Version - }; - - taskMessage.OrchestrationInstance = startedEvent.OrchestrationInstance; - taskMessage.Event = startedEvent; - - return taskMessage; - } - - // If this is a Sub Orchestration, and not tagged as fire-and-forget, - // then notify the parent by sending a complete message - if (runtimeState.ParentInstance != null - && !OrchestrationTags.IsTaggedAsFireAndForget(runtimeState.Tags)) - { - var taskMessage = new TaskMessage(); - if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Completed) - { - var subOrchestrationCompletedEvent = - new SubOrchestrationInstanceCompletedEvent(-1, runtimeState.ParentInstance.TaskScheduleId, - completeOrchestratorAction.Result); - - taskMessage.Event = subOrchestrationCompletedEvent; - } - else if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Failed || - completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Terminated) - { - var subOrchestrationFailedEvent = - new SubOrchestrationInstanceFailedEvent(-1, runtimeState.ParentInstance.TaskScheduleId, - completeOrchestratorAction.Result, - includeDetails ? completeOrchestratorAction.Details : null); - subOrchestrationFailedEvent.FailureDetails = completeOrchestratorAction.FailureDetails; - - taskMessage.Event = subOrchestrationFailedEvent; - } - - if (taskMessage.Event != null) - { - taskMessage.OrchestrationInstance = runtimeState.ParentInstance.OrchestrationInstance; - return taskMessage; - } - } - - return null; - } - - TaskMessage ProcessScheduleTaskDecision( - ScheduleTaskOrchestratorAction scheduleTaskOrchestratorAction, - OrchestrationRuntimeState runtimeState, - bool includeParameters) - { - if (scheduleTaskOrchestratorAction.Name == null) - { - throw new ArgumentException("No name was given for the task activity to schedule!", nameof(scheduleTaskOrchestratorAction)); - } - - var taskMessage = new TaskMessage(); - - var scheduledEvent = new TaskScheduledEvent( - eventId: scheduleTaskOrchestratorAction.Id, - name: scheduleTaskOrchestratorAction.Name, - version: scheduleTaskOrchestratorAction.Version, - input: scheduleTaskOrchestratorAction.Input); - - taskMessage.Event = scheduledEvent; - taskMessage.OrchestrationInstance = runtimeState.OrchestrationInstance; - taskMessage.OrchestrationExecutionContext = GetOrchestrationExecutionContext(runtimeState); - - if (!includeParameters) - { - scheduledEvent = new TaskScheduledEvent( - eventId: scheduleTaskOrchestratorAction.Id, - name: scheduleTaskOrchestratorAction.Name, - version: scheduleTaskOrchestratorAction.Version); - } - - this.logHelper.SchedulingActivity( - runtimeState.OrchestrationInstance!, - scheduledEvent); - - runtimeState.AddEvent(scheduledEvent); - return taskMessage; - } - - TaskMessage ProcessCreateTimerDecision( - CreateTimerOrchestratorAction createTimerOrchestratorAction, - OrchestrationRuntimeState runtimeState, - bool isInternal) - { - var taskMessage = new TaskMessage(); - - var timerCreatedEvent = new TimerCreatedEvent(createTimerOrchestratorAction.Id) - { - FireAt = createTimerOrchestratorAction.FireAt - }; - - runtimeState.AddEvent(timerCreatedEvent); - - taskMessage.Event = new TimerFiredEvent(-1) - { - TimerId = createTimerOrchestratorAction.Id, - FireAt = createTimerOrchestratorAction.FireAt - }; - - this.logHelper.CreatingTimer( - runtimeState.OrchestrationInstance!, - timerCreatedEvent, - isInternal); - - taskMessage.OrchestrationInstance = runtimeState.OrchestrationInstance; - - return taskMessage; - } - - TaskMessage ProcessCreateSubOrchestrationInstanceDecision( - CreateSubOrchestrationAction createSubOrchestrationAction, - OrchestrationRuntimeState runtimeState, - bool includeParameters) - { - var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id) - { - Name = createSubOrchestrationAction.Name, - Version = createSubOrchestrationAction.Version, - InstanceId = createSubOrchestrationAction.InstanceId - }; - if (includeParameters) - { - historyEvent.Input = createSubOrchestrationAction.Input; - } - - runtimeState.AddEvent(historyEvent); - - var taskMessage = new TaskMessage(); - - var startedEvent = new ExecutionStartedEvent(-1, createSubOrchestrationAction.Input) - { - Tags = OrchestrationTags.MergeTags(createSubOrchestrationAction.Tags, runtimeState.Tags), - OrchestrationInstance = new OrchestrationInstance - { - InstanceId = createSubOrchestrationAction.InstanceId, - ExecutionId = Guid.NewGuid().ToString("N") - }, - ParentInstance = new ParentInstance - { - OrchestrationInstance = runtimeState.OrchestrationInstance, - Name = runtimeState.Name, - Version = runtimeState.Version, - TaskScheduleId = createSubOrchestrationAction.Id - }, - Name = createSubOrchestrationAction.Name, - Version = createSubOrchestrationAction.Version - }; - - this.logHelper.SchedulingOrchestration(startedEvent); - - taskMessage.OrchestrationInstance = startedEvent.OrchestrationInstance; - taskMessage.Event = startedEvent; - taskMessage.OrchestrationExecutionContext = GetOrchestrationExecutionContext(runtimeState); - - return taskMessage; - } - - TaskMessage ProcessSendEventDecision( - SendEventOrchestratorAction sendEventAction, - OrchestrationRuntimeState runtimeState) - { - var historyEvent = new EventSentEvent(sendEventAction.Id) - { - InstanceId = sendEventAction.Instance?.InstanceId, - Name = sendEventAction.EventName, - Input = sendEventAction.EventData - }; - - runtimeState.AddEvent(historyEvent); - - this.logHelper.RaisingEvent(runtimeState.OrchestrationInstance!, historyEvent); - - return new TaskMessage - { - OrchestrationInstance = sendEventAction.Instance, - Event = new EventRaisedEvent(-1, sendEventAction.EventData) - { - Name = sendEventAction.EventName - } - }; - } - } -} \ No newline at end of file diff --git a/src/DurableTask.Core/OrchestratorExecutionResult.cs b/src/DurableTask.Core/OrchestratorExecutionResult.cs index 97f1f72c0..86c91c69e 100644 --- a/src/DurableTask.Core/OrchestratorExecutionResult.cs +++ b/src/DurableTask.Core/OrchestratorExecutionResult.cs @@ -29,13 +29,13 @@ public class OrchestratorExecutionResult /// /// The list of actions resulting from the orchestrator execution. /// - [DataMember(Name = "actions")] + [JsonProperty("actions")] public IEnumerable Actions { get; set; } = Array.Empty(); /// /// The custom status, if any, of the orchestrator. /// - [DataMember(Name = "customStatus")] + [JsonProperty("customStatus")] public string? CustomStatus { get; set; } /// diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs new file mode 100644 index 000000000..1baf86851 --- /dev/null +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -0,0 +1,896 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +namespace DurableTask.Core +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Threading; + using System.Threading.Tasks; + using DurableTask.Core.Common; + using DurableTask.Core.Entities; + using DurableTask.Core.Entities.EventFormat; + using DurableTask.Core.Entities.OperationFormat; + using DurableTask.Core.Entities.StateFormat; + using DurableTask.Core.Exceptions; + using DurableTask.Core.History; + using DurableTask.Core.Logging; + using DurableTask.Core.Middleware; + using DurableTask.Core.Tracing; + using Newtonsoft.Json; + + /// + /// Dispatcher for orchestrations and entities to handle processing and renewing, completion of orchestration events. + /// + public class TaskEntityDispatcher + { + readonly INameVersionObjectManager objectManager; + readonly IOrchestrationService orchestrationService; + readonly IEntityOrchestrationService entityOrchestrationService; + readonly WorkItemDispatcher dispatcher; + readonly DispatchMiddlewarePipeline dispatchPipeline; + readonly EntityBackendInformation entityBackendInformation; + readonly LogHelper logHelper; + readonly ErrorPropagationMode errorPropagationMode; + readonly TaskOrchestrationDispatcher.NonBlockingCountdownLock concurrentSessionLock; + + + internal TaskEntityDispatcher( + IOrchestrationService orchestrationService, + INameVersionObjectManager entityObjectManager, + DispatchMiddlewarePipeline entityDispatchPipeline, + LogHelper logHelper, + ErrorPropagationMode errorPropagationMode) + { + this.objectManager = entityObjectManager ?? throw new ArgumentNullException(nameof(entityObjectManager)); + this.orchestrationService = orchestrationService ?? throw new ArgumentNullException(nameof(orchestrationService)); + this.dispatchPipeline = entityDispatchPipeline ?? throw new ArgumentNullException(nameof(entityDispatchPipeline)); + this.logHelper = logHelper ?? throw new ArgumentNullException(nameof(logHelper)); + this.errorPropagationMode = errorPropagationMode; + this.entityOrchestrationService = (orchestrationService as IEntityOrchestrationService)!; + this.entityBackendInformation = entityOrchestrationService.GetEntityBackendInformation(); + + this.dispatcher = new WorkItemDispatcher( + "TaskEntityDispatcher", + item => item == null ? string.Empty : item.InstanceId, + this.OnFetchWorkItemAsync, + this.OnProcessWorkItemSessionAsync) + { + GetDelayInSecondsAfterOnFetchException = orchestrationService.GetDelayInSecondsAfterOnFetchException, + GetDelayInSecondsAfterOnProcessException = orchestrationService.GetDelayInSecondsAfterOnProcessException, + SafeReleaseWorkItem = orchestrationService.ReleaseTaskOrchestrationWorkItemAsync, + AbortWorkItem = orchestrationService.AbandonTaskOrchestrationWorkItemAsync, + DispatcherCount = orchestrationService.TaskOrchestrationDispatcherCount, + MaxConcurrentWorkItems = orchestrationService.MaxConcurrentTaskOrchestrationWorkItems, + LogHelper = logHelper, + }; + + // To avoid starvation, we only allow half of all concurrently execution orchestrations to + // leverage extended sessions. + var maxConcurrentSessions = (int)Math.Ceiling(this.dispatcher.MaxConcurrentWorkItems / 2.0); + this.concurrentSessionLock = new TaskOrchestrationDispatcher.NonBlockingCountdownLock(maxConcurrentSessions); + } + + /// + /// The entity options configured, or null if the backend does not support entities. + /// + public EntityBackendInformation EntityBackendInformation => this.entityBackendInformation; + + /// + /// Starts the dispatcher to start getting and processing orchestration events + /// + public async Task StartAsync() + { + await this.dispatcher.StartAsync(); + } + + /// + /// Stops the dispatcher to stop getting and processing orchestration events + /// + /// Flag indicating whether to stop gracefully or immediately + public async Task StopAsync(bool forced) + { + await this.dispatcher.StopAsync(forced); + } + + + /// + /// Method to get the next work item to process within supplied timeout + /// + /// The max timeout to wait + /// A cancellation token used to cancel a fetch operation. + /// A new TaskOrchestrationWorkItem + protected Task OnFetchWorkItemAsync(TimeSpan receiveTimeout, CancellationToken cancellationToken) + { + return this.entityOrchestrationService.LockNextEntityWorkItemAsync(receiveTimeout, cancellationToken); + } + + async Task OnProcessWorkItemSessionAsync(TaskOrchestrationWorkItem workItem) + { + try + { + if (workItem.Session == null) + { + // Legacy behavior + await this.OnProcessWorkItemAsync(workItem); + return; + } + + var isExtendedSession = false; + + CorrelationTraceClient.Propagate( + () => + { + // Check if it is extended session. + isExtendedSession = this.concurrentSessionLock.Acquire(); + this.concurrentSessionLock.Release(); + workItem.IsExtendedSession = isExtendedSession; + }); + + var processCount = 0; + try + { + while (true) + { + // If the provider provided work items, execute them. + if (workItem.NewMessages?.Count > 0) + { + bool isCompletedOrInterrupted = await this.OnProcessWorkItemAsync(workItem); + if (isCompletedOrInterrupted) + { + break; + } + + processCount++; + } + + // Fetches beyond the first require getting an extended session lock, used to prevent starvation. + if (processCount > 0 && !isExtendedSession) + { + isExtendedSession = this.concurrentSessionLock.Acquire(); + if (!isExtendedSession) + { + TraceHelper.Trace(TraceEventType.Verbose, "OnProcessWorkItemSession-MaxOperations", "Failed to acquire concurrent session lock."); + break; + } + } + + TraceHelper.Trace(TraceEventType.Verbose, "OnProcessWorkItemSession-StartFetch", "Starting fetch of existing session."); + Stopwatch timer = Stopwatch.StartNew(); + + // Wait for new messages to arrive for the session. This call is expected to block (asynchronously) + // until either new messages are available or until a provider-specific timeout has expired. + workItem.NewMessages = await workItem.Session.FetchNewOrchestrationMessagesAsync(workItem); + if (workItem.NewMessages == null) + { + break; + } + + TraceHelper.Trace( + TraceEventType.Verbose, + "OnProcessWorkItemSession-EndFetch", + $"Fetched {workItem.NewMessages.Count} new message(s) after {timer.ElapsedMilliseconds} ms from existing session."); + workItem.OrchestrationRuntimeState.NewEvents.Clear(); + } + } + finally + { + if (isExtendedSession) + { + TraceHelper.Trace( + TraceEventType.Verbose, + "OnProcessWorkItemSession-Release", + $"Releasing extended session after {processCount} batch(es)."); + this.concurrentSessionLock.Release(); + } + } + } + catch (SessionAbortedException e) + { + // Either the orchestration or the orchestration service explicitly abandoned the session. + OrchestrationInstance instance = workItem.OrchestrationRuntimeState?.OrchestrationInstance ?? new OrchestrationInstance { InstanceId = workItem.InstanceId }; + this.logHelper.OrchestrationAborted(instance, e.Message); + TraceHelper.TraceInstance(TraceEventType.Warning, "TaskOrchestrationDispatcher-ExecutionAborted", instance, "{0}", e.Message); + await this.orchestrationService.AbandonTaskOrchestrationWorkItemAsync(workItem); + } + } + + class WorkItemEffects + { + public List ActivityMessages; + public List TimerMessages; + public List InstanceMessages; + public int taskIdCounter; + public string InstanceId; + public OrchestrationRuntimeState RuntimeState; + } + + /// + /// Method to process a new work item + /// + /// The work item to process + protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem workItem) + { + // correlation + CorrelationTraceClient.Propagate(() => CorrelationTraceContext.Current = workItem.TraceContext); + + OrchestrationRuntimeState originalOrchestrationRuntimeState = workItem.OrchestrationRuntimeState; + + OrchestrationRuntimeState runtimeState = workItem.OrchestrationRuntimeState; + runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); + + Task renewTask = null; + using var renewCancellationTokenSource = new CancellationTokenSource(); + if (workItem.LockedUntilUtc < DateTime.MaxValue) + { + // start a task to run RenewUntil + renewTask = Task.Factory.StartNew( + () => TaskOrchestrationDispatcher.RenewUntil(workItem, this.orchestrationService, this.logHelper, nameof(TaskEntityDispatcher), renewCancellationTokenSource.Token), + renewCancellationTokenSource.Token); + } + + WorkItemEffects effects = new WorkItemEffects() + { + ActivityMessages = new List(), + TimerMessages = new List(), + InstanceMessages = new List(), + taskIdCounter = 0, + InstanceId = workItem.InstanceId, + RuntimeState = runtimeState, + }; + + try + { + // Assumes that: if the batch contains a new "ExecutionStarted" event, it is the first message in the batch. + if (!TaskOrchestrationDispatcher.ReconcileMessagesWithState(workItem, "TaskEntityDispatcher", this.logHelper)) + { + // TODO : mark an orchestration as faulted if there is data corruption + this.logHelper.DroppingOrchestrationWorkItem(workItem, "Received work-item for an invalid orchestration"); + TraceHelper.TraceSession( + TraceEventType.Error, + "TaskEntityDispatcher-DeletedOrchestration", + runtimeState.OrchestrationInstance?.InstanceId, + "Received work-item for an invalid orchestration"); + } + else + { + + // we start with processing all the requests and figuring out which ones to execute now + // results can depend on whether the entity is locked, what the maximum batch size is, + // and whether the messages arrived out of order + + this.DetermineWork(workItem.OrchestrationRuntimeState, + out SchedulerState schedulerState, + out Work workToDoNow); + + if (workToDoNow.OperationCount > 0) + { + // execute the user-defined operations on this entity, via the middleware + var result = await this.ExecuteViaMiddlewareAsync(workToDoNow, runtimeState.OrchestrationInstance, schedulerState.EntityState); + + // go through all results + // for each operation that is not a signal, send a result message back to the calling orchestrator + for (int i = 0; i < result.Results!.Count; i++) + { + var req = workToDoNow.Operations[i]; + if (!req.IsSignal) + { + this.SendResultMessage(effects, req, result.Results[i]); + } + } + + if (result.Results.Count < workToDoNow.OperationCount) + { + // some operations were not processed + var deferred = workToDoNow.RemoveDeferredWork(result.Results.Count); + schedulerState.PutBack(deferred); + workToDoNow.ToBeContinued(schedulerState); + } + + // update the entity state based on the result + schedulerState.EntityState = result.EntityState; + schedulerState.EntityExists = result.EntityState != null; + + // perform the actions + foreach (var action in result.Actions!) + { + switch (action) + { + case (SendSignalOperationAction sendSignalAction): + this.SendSignalMessage(effects, schedulerState, sendSignalAction); + break; + case (StartNewOrchestrationOperationAction startAction): + this.ProcessSendStartMessage(effects, runtimeState, startAction); + break; + } + } + } + + // process the lock request, if any + if (workToDoNow.LockRequest != null) + { + this.ProcessLockRequest(effects, schedulerState, workToDoNow.LockRequest); + } + + if (workToDoNow.ToBeRescheduled != null) + { + foreach (var request in workToDoNow.ToBeRescheduled) + { + // Reschedule all signals that were received before their time + this.SendScheduledSelfMessage(effects, request); + } + } + + if (workToDoNow.SuspendAndContinue) + { + this.SendContinueSelfMessage(effects); + } + + // this batch is complete. Since this is an entity, we now + // (always) start a new execution, as in continue-as-new + + var serializedSchedulerState = this.SerializeSchedulerStateForNextExecution(schedulerState); + var nextExecutionStartedEvent = new ExecutionStartedEvent(-1, serializedSchedulerState) + { + OrchestrationInstance = new OrchestrationInstance + { + InstanceId = workItem.InstanceId, + ExecutionId = Guid.NewGuid().ToString("N") + }, + Tags = runtimeState.Tags, + ParentInstance = runtimeState.ParentInstance, + Name = runtimeState.Name, + Version = runtimeState.Version + }; + var entityStatus = new EntityStatus() + { + EntityExists = schedulerState.EntityExists, + QueueSize = schedulerState.Queue?.Count ?? 0, + LockedBy = schedulerState.LockedBy, + }; + var serializedEntityStatus = JsonConvert.SerializeObject(entityStatus, Serializer.InternalSerializerSettings); + + // create the new runtime state for the next execution + runtimeState = new OrchestrationRuntimeState(); + runtimeState.Status = serializedEntityStatus; + runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); + runtimeState.AddEvent(nextExecutionStartedEvent); + runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); + } + } + finally + { + if (renewTask != null) + { + try + { + renewCancellationTokenSource.Cancel(); + await renewTask; + } + catch (ObjectDisposedException) + { + // ignore + } + catch (OperationCanceledException) + { + // ignore + } + } + } + + OrchestrationState instanceState = (runtimeState.ExecutionStartedEvent != null) ? + instanceState = Utils.BuildOrchestrationState(runtimeState) : null; + + if (workItem.RestoreOriginalRuntimeStateDuringCompletion) + { + // some backends expect the original runtime state object + workItem.OrchestrationRuntimeState = originalOrchestrationRuntimeState; + } + else + { + workItem.OrchestrationRuntimeState = runtimeState; + } + + await this.orchestrationService.CompleteTaskOrchestrationWorkItemAsync( + workItem, + runtimeState, + effects.ActivityMessages, + effects.InstanceMessages, + effects.TimerMessages, + null, + instanceState); + + if (workItem.RestoreOriginalRuntimeStateDuringCompletion) + { + workItem.OrchestrationRuntimeState = runtimeState; + } + + return true; + } + + void ProcessLockRequest(WorkItemEffects effects, SchedulerState schedulerState, RequestMessage request) + { + this.logHelper.EntityLockAcquired(effects.InstanceId, request); + + // mark the entity state as locked + schedulerState.LockedBy = request.ParentInstanceId; + + request.Position++; + + if (request.Position < request.LockSet.Length) + { + // send lock request to next entity in the lock set + var target = new OrchestrationInstance() { InstanceId = request.LockSet[request.Position].ToString() }; + this.SendLockRequestMessage(effects, schedulerState, target, request); + } + else + { + // send lock acquisition completed response back to originating orchestration instance + var target = new OrchestrationInstance() { InstanceId = request.ParentInstanceId, ExecutionId = request.ParentExecutionId }; + this.SendLockResponseMessage(effects, target, request.Id); + } + } + + string SerializeSchedulerStateForNextExecution(SchedulerState schedulerState) + { + if (this.entityBackendInformation.SupportsImplicitEntityDeletion && schedulerState.IsEmpty && !schedulerState.Suspended) + { + // this entity scheduler is idle and the entity is deleted, so the instance and history can be removed from storage + // we convey this to the durability provider by issuing a continue-as-new with null input + return null; + } + else + { + // we persist the state of the entity scheduler and entity + return JsonConvert.SerializeObject(schedulerState, typeof(SchedulerState), Serializer.InternalSerializerSettings); + } + } + + #region Preprocess to determine work + + void DetermineWork(OrchestrationRuntimeState runtimeState, out SchedulerState schedulerState, out Work batch) + { + string instanceId = runtimeState.OrchestrationInstance.InstanceId; + schedulerState = new SchedulerState(); + batch = new Work(); + + Queue lockHolderMessages = null; + + foreach (HistoryEvent e in runtimeState.Events) + { + switch (e.EventType) + { + case EventType.ExecutionStarted: + + + if (runtimeState.Input != null) + { + try + { + // restore the scheduler state from the input + JsonConvert.PopulateObject(runtimeState.Input, schedulerState, Serializer.InternalSerializerSettings); + } + catch (Exception exception) + { + throw new EntitySchedulerException("Failed to deserialize entity scheduler state - may be corrupted or wrong version.", exception); + } + } + break; + + case EventType.EventRaised: + EventRaisedEvent eventRaisedEvent = (EventRaisedEvent)e; + + if (EntityMessageEventNames.IsRequestMessage(eventRaisedEvent.Name)) + { + // we are receiving an operation request or a lock request + var requestMessage = new RequestMessage(); + + try + { + JsonConvert.PopulateObject(eventRaisedEvent.Input, requestMessage, Serializer.InternalSerializerSettings); + } + catch (Exception exception) + { + throw new EntitySchedulerException("Failed to deserialize incoming request message - may be corrupted or wrong version.", exception); + } + + IEnumerable deliverNow; + + if (requestMessage.ScheduledTime.HasValue) + { + if ((requestMessage.ScheduledTime.Value - DateTime.UtcNow) > TimeSpan.FromMilliseconds(100)) + { + // message was delivered too early. This can happen e.g. if the orchestration service has limits on the delay times for messages. + // We handle this by rescheduling the message instead of processing it. + deliverNow = Array.Empty(); + batch.AddMessageToBeRescheduled(requestMessage); + } + else + { + // the message is scheduled to be delivered immediately. + // There are no FIFO guarantees for scheduled messages, so we skip the message sorter. + deliverNow = new RequestMessage[] { requestMessage }; + } + } + else + { + // run this through the message sorter to help with reordering and duplicate filtering + deliverNow = schedulerState.MessageSorter.ReceiveInOrder(requestMessage, this.entityBackendInformation.EntityMessageReorderWindow); + } + + foreach (var message in deliverNow) + { + if (schedulerState.LockedBy != null && schedulerState.LockedBy == message.ParentInstanceId) + { + if (lockHolderMessages == null) + { + lockHolderMessages = new Queue(); + } + + lockHolderMessages.Enqueue(message); + } + else + { + schedulerState.Enqueue(message); + } + } + } + else if (EntityMessageEventNames.IsReleaseMessage(eventRaisedEvent.Name)) + { + // we are receiving a lock release + var message = new ReleaseMessage(); + try + { + // restore the scheduler state from the input + JsonConvert.PopulateObject(eventRaisedEvent.Input, message, Serializer.InternalSerializerSettings); + } + catch (Exception exception) + { + throw new EntitySchedulerException("Failed to deserialize lock release message - may be corrupted or wrong version.", exception); + } + + if (schedulerState.LockedBy == message.ParentInstanceId) + { + this.logHelper.EntityLockReleased(instanceId, message); + schedulerState.LockedBy = null; + } + } + else + { + // this is a continue message. + // Resumes processing of previously queued operations, if any. + schedulerState.Suspended = false; + } + + break; + } + } + + // lock holder messages go to the front of the queue + if (lockHolderMessages != null) + { + schedulerState.PutBack(lockHolderMessages); + } + + if (!schedulerState.Suspended) + { + // 2. We add as many requests from the queue to the batch as possible, + // stopping at lock requests or when the maximum batch size is reached + while (schedulerState.MayDequeue()) + { + if (batch.OperationCount == this.entityBackendInformation.MaxEntityOperationBatchSize) + { + // we have reached the maximum batch size already + // insert a delay after this batch to ensure write back + batch.ToBeContinued(schedulerState); + break; + } + + var request = schedulerState.Dequeue(); + + if (request.IsLockRequest) + { + batch.AddLockRequest(request); + break; + } + else + { + batch.AddOperation(request); + } + } + } + } + + class Work + { + List operationBatch; // a (possibly empty) sequence of operations to be executed on the entity + RequestMessage lockRequest = null; // zero or one lock request to be executed after all the operations + List toBeRescheduled; // a (possibly empty) list of timed messages that were delivered too early and should be rescheduled + bool suspendAndContinue; // a flag telling as to send ourselves a continue signal + + public int OperationCount => this.operationBatch?.Count ?? 0; + public IReadOnlyList Operations => this.operationBatch; + public IReadOnlyList ToBeRescheduled => this.toBeRescheduled; + public RequestMessage LockRequest => this.lockRequest; + public bool SuspendAndContinue => this.suspendAndContinue; + + public void AddOperation(RequestMessage operationMessage) + { + if (this.operationBatch == null) + { + this.operationBatch = new List(); + } + this.operationBatch.Add(operationMessage); + } + + public void AddLockRequest(RequestMessage lockRequest) + { + Debug.Assert(this.lockRequest == null); + this.lockRequest = lockRequest; + } + + public void AddMessageToBeRescheduled(RequestMessage requestMessage) + { + if (this.toBeRescheduled == null) + { + this.toBeRescheduled = new List(); + } + this.toBeRescheduled.Add(requestMessage); + } + + public void ToBeContinued(SchedulerState schedulerState) + { + if (!schedulerState.Suspended) + { + this.suspendAndContinue = true; + } + } + + public List GetOperationRequests() + { + var operations = new List(this.operationBatch.Count); + for (int i = 0; i < this.operationBatch.Count; i++) + { + var request = this.operationBatch[i]; + operations.Add(new OperationRequest() + { + Operation = request.Operation, + Id = request.Id, + Input = request.Input, + }); + } + return operations; + } + + public Queue RemoveDeferredWork(int index) + { + var deferred = new Queue(); + for (int i = index; i < this.operationBatch.Count; i++) + { + deferred.Enqueue(this.operationBatch[i]); + } + this.operationBatch.RemoveRange(index, this.operationBatch.Count - index); + if (this.lockRequest != null) + { + deferred.Enqueue(this.lockRequest); + this.lockRequest = null; + } + return deferred; + } + } + + #endregion + + #region Send Messages + + void SendResultMessage(WorkItemEffects effects, RequestMessage request, OperationResult result) + { + var destination = new OrchestrationInstance() + { + InstanceId = request.ParentInstanceId, + ExecutionId = request.ParentExecutionId, + }; + var responseMessage = new ResponseMessage() + { + Result = result.Result, + ErrorMessage = result.ErrorMessage, + FailureDetails = result.FailureDetails, + }; + this.ProcessSendEventMessage(effects, destination, EntityMessageEventNames.ResponseMessageEventName(request.Id), responseMessage); + } + + void SendSignalMessage(WorkItemEffects effects, SchedulerState schedulerState, SendSignalOperationAction action) + { + OrchestrationInstance destination = new OrchestrationInstance() + { + InstanceId = action.InstanceId + }; + RequestMessage message = new RequestMessage() + { + ParentInstanceId = effects.InstanceId, + ParentExecutionId = null, // for entities, message sorter persists across executions + Id = Guid.NewGuid(), + IsSignal = true, + Operation = action.Name, + ScheduledTime = action.ScheduledTime, + }; + string eventName; + if (action.ScheduledTime.HasValue) + { + DateTime original = action.ScheduledTime.Value; + DateTime capped = this.entityBackendInformation.GetCappedScheduledTime(DateTime.UtcNow, original); + eventName = EntityMessageEventNames.ScheduledRequestMessageEventName(capped); + } + else + { + eventName = EntityMessageEventNames.RequestMessageEventName; + schedulerState.MessageSorter.LabelOutgoingMessage(message, action.InstanceId, DateTime.UtcNow, this.entityBackendInformation.EntityMessageReorderWindow); + } + this.ProcessSendEventMessage(effects, destination, eventName, message); + } + + void SendLockRequestMessage(WorkItemEffects effects, SchedulerState schedulerState, OrchestrationInstance target, RequestMessage message) + { + schedulerState.MessageSorter.LabelOutgoingMessage(message, target.InstanceId, DateTime.UtcNow, this.entityBackendInformation.EntityMessageReorderWindow); + this.ProcessSendEventMessage(effects, target, EntityMessageEventNames.RequestMessageEventName, message); + } + + void SendLockResponseMessage(WorkItemEffects effects, OrchestrationInstance target, Guid requestId) + { + var message = new ResponseMessage() + { + Result = "Lock Acquisition Completed", // ignored by receiver but shows up in traces + }; + this.ProcessSendEventMessage(effects, target, EntityMessageEventNames.ResponseMessageEventName(requestId), message); + } + + void SendScheduledSelfMessage(WorkItemEffects effects, RequestMessage request) + { + var self = new OrchestrationInstance() + { + InstanceId = effects.InstanceId, + }; + this.ProcessSendEventMessage(effects, self, EntityMessageEventNames.ScheduledRequestMessageEventName(request.ScheduledTime.Value), request); + } + + void SendContinueSelfMessage(WorkItemEffects effects) + { + var self = new OrchestrationInstance() + { + InstanceId = effects.InstanceId, + }; + this.ProcessSendEventMessage(effects, self, EntityMessageEventNames.ContinueMessageEventName, null); + } + + void ProcessSendEventMessage(WorkItemEffects effects, OrchestrationInstance destination, string eventName, object eventContent) + { + string serializedContent = null; + if (eventContent != null) + { + serializedContent = JsonConvert.SerializeObject(eventContent, Serializer.InternalSerializerSettings); + } + + var eventSentEvent = new EventSentEvent(effects.taskIdCounter++) + { + InstanceId = destination.InstanceId, + Name = eventName, + Input = serializedContent, + }; + this.logHelper.RaisingEvent(effects.RuntimeState.OrchestrationInstance, eventSentEvent); + + effects.InstanceMessages.Add(new TaskMessage + { + OrchestrationInstance = destination, + Event = new EventRaisedEvent(-1, serializedContent) + { + Name = eventName, + Input = serializedContent, + }, + }); + } + + void ProcessSendStartMessage(WorkItemEffects effects, OrchestrationRuntimeState runtimeState, StartNewOrchestrationOperationAction action) + { + OrchestrationInstance destination = new OrchestrationInstance() + { + InstanceId = action.InstanceId, + ExecutionId = Guid.NewGuid().ToString("N"), + }; + var executionStartedEvent = new ExecutionStartedEvent(-1, action.Input) + { + Tags = OrchestrationTags.MergeTags(action.Tags, runtimeState.Tags), + OrchestrationInstance = destination, + ParentInstance = new ParentInstance + { + OrchestrationInstance = runtimeState.OrchestrationInstance, + Name = runtimeState.Name, + Version = runtimeState.Version, + TaskScheduleId = effects.taskIdCounter++, + }, + Name = action.Name, + Version = action.Version, + }; + this.logHelper.SchedulingOrchestration(executionStartedEvent); + + effects.InstanceMessages.Add(new TaskMessage + { + OrchestrationInstance = destination, + Event = executionStartedEvent, + }); + } + + #endregion + + async Task ExecuteViaMiddlewareAsync(Work workToDoNow, OrchestrationInstance instance, string serializedEntityState) + { + // the request object that will be passed to the worker + var request = new OperationBatchRequest() + { + InstanceId = instance.InstanceId, + EntityState = serializedEntityState, + Operations = workToDoNow.GetOperationRequests(), + }; + + this.logHelper.EntityBatchExecuting(request); + + var entityId = EntityId.FromString(instance.InstanceId); + string entityName = entityId.Name; + string entityKey = entityId.Key; + + // Get the TaskOrchestration implementation. If it's not found, it either means that the developer never + // registered it (which is an error, and we'll throw for this further down) or it could be that some custom + // middleware (e.g. out-of-process execution middleware) is intended to implement the orchestration logic. + TaskEntity taskEntity = this.objectManager.GetObject(entityName, entityKey); + + var dispatchContext = new DispatchMiddlewareContext(); + dispatchContext.SetProperty(request); + + await this.dispatchPipeline.RunAsync(dispatchContext, async _ => + { + // Check to see if the custom middleware intercepted and substituted the orchestration execution + // with its own execution behavior, providing us with the end results. If so, we can terminate + // the dispatch pipeline here. + var resultFromMiddleware = dispatchContext.GetProperty(); + if (resultFromMiddleware != null) + { + return; + } + + if (taskEntity == null) + { + throw TraceHelper.TraceExceptionInstance( + TraceEventType.Error, + "TaskOrchestrationDispatcher-EntityTypeMissing", + instance, + new TypeMissingException($"Entity not found: {entityName}")); + } + + var options = new EntityExecutionOptions() + { + EntityBackendInformation = this.entityBackendInformation, + ErrorPropagationMode = this.errorPropagationMode, + }; + + var result = await taskEntity.ExecuteOperationBatchAsync(request, options); + + dispatchContext.SetProperty(result); + }); + + var result = dispatchContext.GetProperty(); + + this.logHelper.EntityBatchExecuted(request, result); + + return result; + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/TaskHubWorker.cs b/src/DurableTask.Core/TaskHubWorker.cs index baddcb990..2d029006b 100644 --- a/src/DurableTask.Core/TaskHubWorker.cs +++ b/src/DurableTask.Core/TaskHubWorker.cs @@ -37,6 +37,8 @@ public sealed class TaskHubWorker : IDisposable readonly INameVersionObjectManager orchestrationManager; readonly INameVersionObjectManager entityManager; + readonly IEntityOrchestrationService entityOrchestrationService; + readonly DispatchMiddlewarePipeline orchestrationDispatchPipeline = new DispatchMiddlewarePipeline(); readonly DispatchMiddlewarePipeline entityDispatchPipeline = new DispatchMiddlewarePipeline(); readonly DispatchMiddlewarePipeline activityDispatchPipeline = new DispatchMiddlewarePipeline(); @@ -50,10 +52,16 @@ public sealed class TaskHubWorker : IDisposable // ReSharper disable once InconsistentNaming (avoid breaking change) public IOrchestrationService orchestrationService { get; } + /// + /// Indicates whether the configured backend supports entities. + /// + public bool SupportsEntities => this.entityOrchestrationService != null; + volatile bool isStarted; TaskActivityDispatcher activityDispatcher; - TaskOrchestrationDispatcher orchestrationDispatcher; // used for both orchestrations and entities + TaskOrchestrationDispatcher orchestrationDispatcher; + TaskEntityDispatcher entityDispatcher; /// /// Create a new TaskHubWorker with given OrchestrationService @@ -64,7 +72,7 @@ public TaskHubWorker(IOrchestrationService orchestrationService) orchestrationService, new NameVersionObjectManager(), new NameVersionObjectManager(), - new NameVersionObjectManager()) + new NameObjectManager()) { } @@ -79,7 +87,7 @@ public TaskHubWorker(IOrchestrationService orchestrationService, ILoggerFactory orchestrationService, new NameVersionObjectManager(), new NameVersionObjectManager(), - new NameVersionObjectManager(), + new NameObjectManager(), loggerFactory) { } @@ -126,10 +134,12 @@ public TaskHubWorker( this.entityManager = entityObjectManager ?? throw new ArgumentException("entityObjectManager"); this.orchestrationService = orchestrationService ?? throw new ArgumentException("orchestrationService"); this.logHelper = new LogHelper(loggerFactory?.CreateLogger("DurableTask.Core")); + this.entityOrchestrationService = orchestrationService as IEntityOrchestrationService; + this.entityOrchestrationService?.ProcessEntitiesSeparately(); // lets the backend know that this worker wants them separately } /// - /// Gets the orchestration and entity dispatcher + /// Gets the orchestration dispatcher /// public TaskOrchestrationDispatcher TaskOrchestrationDispatcher => this.orchestrationDispatcher; @@ -138,6 +148,11 @@ public TaskHubWorker( /// public TaskActivityDispatcher TaskActivityDispatcher => this.activityDispatcher; + /// + /// Gets the entity dispatcher + /// + public TaskEntityDispatcher TaskEntityDispatcher => this.entityDispatcher; + /// /// Gets or sets the error propagation behavior when an activity or orchestration fails with an unhandled exception. /// @@ -202,9 +217,7 @@ public async Task StartAsync() this.orchestrationDispatcher = new TaskOrchestrationDispatcher( this.orchestrationService, this.orchestrationManager, - this.entityManager, this.orchestrationDispatchPipeline, - this.entityDispatchPipeline, this.logHelper, this.ErrorPropagationMode); this.activityDispatcher = new TaskActivityDispatcher( @@ -214,10 +227,25 @@ public async Task StartAsync() this.logHelper, this.ErrorPropagationMode); + if (this.SupportsEntities) + { + this.entityDispatcher = new TaskEntityDispatcher( + this.orchestrationService, + this.entityManager, + this.entityDispatchPipeline, + this.logHelper, + this.ErrorPropagationMode); + } + await this.orchestrationService.StartAsync(); await this.orchestrationDispatcher.StartAsync(); await this.activityDispatcher.StartAsync(); + if (this.SupportsEntities) + { + await this.entityDispatcher.StartAsync(); + } + this.logHelper.TaskHubWorkerStarted(sw.Elapsed); this.isStarted = true; } @@ -255,6 +283,7 @@ public async Task StopAsync(bool isForced) { this.orchestrationDispatcher.StopAsync(isForced), this.activityDispatcher.StopAsync(isForced), + this.SupportsEntities ? this.entityDispatcher.StopAsync(isForced) : Task.CompletedTask, }; await Task.WhenAll(dispatcherShutdowns); diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index f8a84519d..a696e4fb7 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -242,7 +242,8 @@ public override void SignalEntity(EntityId entityId, string operationName, objec public override void SignalEntity(EntityId entityId, DateTime scheduledTimeUtc, string operationName, object input = null) { this.CheckEntitySupport(); - this.EntityOperationCore(entityId, operationName, input, true, EntityBackendInformation.GetCappedScheduledTime(this.CurrentUtcDateTime, scheduledTimeUtc)); + DateTime cappedTime = EntityBackendInformation.GetCappedScheduledTime(this.CurrentUtcDateTime, scheduledTimeUtc); + this.EntityOperationCore(entityId, operationName, input, true, (scheduledTimeUtc, cappedTime)); } (Guid operationId, string targetInstanceId, int taskId) EntityOperationCore(EntityId entityId, string operationName, object input, bool oneWay, (DateTime original, DateTime capped)? scheduledTimeUtc = null) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 6ef1a2bf8..0133287dd 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -15,54 +15,52 @@ namespace DurableTask.Core { using System; using System.Collections.Generic; + using System.Collections.ObjectModel; using System.Diagnostics; + using System.Linq; using System.Threading; using System.Threading.Tasks; + using DurableTask.Core.Command; using DurableTask.Core.Common; using DurableTask.Core.Entities; using DurableTask.Core.Exceptions; using DurableTask.Core.History; using DurableTask.Core.Logging; using DurableTask.Core.Middleware; + using DurableTask.Core.Serializing; using DurableTask.Core.Tracing; /// - /// Dispatcher for orchestrations and entities to handle processing and renewing, completion of orchestration events. + /// Dispatcher for orchestrations to handle processing and renewing, completion of orchestration events /// public class TaskOrchestrationDispatcher { - readonly INameVersionObjectManager orchestrationObjectManager; - readonly INameVersionObjectManager entityObjectManager; + static readonly Task CompletedTask = Task.FromResult(0); + + readonly INameVersionObjectManager objectManager; readonly IOrchestrationService orchestrationService; readonly WorkItemDispatcher dispatcher; - readonly DispatchMiddlewarePipeline orchestrationDispatchPipeline; - readonly DispatchMiddlewarePipeline entityDispatchPipeline; - readonly EntityBackendInformation? entityBackendInformation; + readonly DispatchMiddlewarePipeline dispatchPipeline; readonly LogHelper logHelper; ErrorPropagationMode errorPropagationMode; readonly NonBlockingCountdownLock concurrentSessionLock; + readonly IEntityOrchestrationService? entityOrchestrationService; + readonly EntityBackendInformation? entityBackendInformation; internal TaskOrchestrationDispatcher( IOrchestrationService orchestrationService, - INameVersionObjectManager orchestrationObjectManager, - INameVersionObjectManager entityObjectManager, - DispatchMiddlewarePipeline orchestrationDispatchPipeline, - DispatchMiddlewarePipeline entityDispatchPipeline, + INameVersionObjectManager objectManager, + DispatchMiddlewarePipeline dispatchPipeline, LogHelper logHelper, ErrorPropagationMode errorPropagationMode) { - this.orchestrationObjectManager = orchestrationObjectManager ?? throw new ArgumentNullException(nameof(orchestrationObjectManager)); - this.entityObjectManager = entityObjectManager ?? throw new ArgumentNullException(nameof(entityObjectManager)); + this.objectManager = objectManager ?? throw new ArgumentNullException(nameof(objectManager)); this.orchestrationService = orchestrationService ?? throw new ArgumentNullException(nameof(orchestrationService)); - this.orchestrationDispatchPipeline = orchestrationDispatchPipeline ?? throw new ArgumentNullException(nameof(orchestrationDispatchPipeline)); - this.entityDispatchPipeline = entityDispatchPipeline ?? throw new ArgumentNullException(nameof(entityDispatchPipeline)); + this.dispatchPipeline = dispatchPipeline ?? throw new ArgumentNullException(nameof(dispatchPipeline)); this.logHelper = logHelper ?? throw new ArgumentNullException(nameof(logHelper)); this.errorPropagationMode = errorPropagationMode; - - if (EntityBackendInformation.BackendSupportsEntities(orchestrationService, out var options)) - { - this.entityBackendInformation = options; - } + this.entityOrchestrationService = orchestrationService as IEntityOrchestrationService; + this.entityBackendInformation = this.entityOrchestrationService?.GetEntityBackendInformation(); this.dispatcher = new WorkItemDispatcher( "TaskOrchestrationDispatcher", @@ -85,11 +83,6 @@ internal TaskOrchestrationDispatcher( this.concurrentSessionLock = new NonBlockingCountdownLock(maxConcurrentSessions); } - /// - /// The entity options configured, or null if the backend does not support entities. - /// - public EntityBackendInformation? EntityBackendInformation => this.entityBackendInformation; - /// /// Starts the dispatcher to start getting and processing orchestration events /// @@ -125,9 +118,18 @@ public async Task StopAsync(bool forced) /// A new TaskOrchestrationWorkItem protected Task OnFetchWorkItemAsync(TimeSpan receiveTimeout, CancellationToken cancellationToken) { - return this.orchestrationService.LockNextTaskOrchestrationWorkItemAsync(receiveTimeout, cancellationToken); + if (this.entityOrchestrationService != null) + { + // we call the entity interface to make sure we are receiving only true orchestrations here + return this.entityOrchestrationService.LockNextOrchestrationWorkItemAsync(receiveTimeout, cancellationToken); + } + else + { + return this.orchestrationService.LockNextTaskOrchestrationWorkItemAsync(receiveTimeout, cancellationToken); + } } + /// /// Ensures the first ExecutionStarted event in the batch (if any) appears at the beginning /// of its executionID history. @@ -290,35 +292,29 @@ async Task OnProcessWorkItemSessionAsync(TaskOrchestrationWorkItem workItem) /// /// The work item to process protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem workItem) - { + { + var messagesToSend = new List(); + var timerMessages = new List(); + var orchestratorMessages = new List(); + var isCompleted = false; + var continuedAsNew = false; + var isInterrupted = false; + // correlation CorrelationTraceClient.Propagate(() => CorrelationTraceContext.Current = workItem.TraceContext); - OrchestrationRuntimeState originalOrchestrationRuntimeState = workItem.OrchestrationRuntimeState; + ExecutionStartedEvent? continueAsNewExecutionStarted = null; + TaskMessage? continuedAsNewMessage = null; + IList? carryOverEvents = null; + string? carryOverStatus = null; - bool isEntity = Common.Entities.IsEntityInstance(workItem.InstanceId); + OrchestrationRuntimeState runtimeState = workItem.OrchestrationRuntimeState; - WorkItemProcessor specializedDispatcher = isEntity - ? new EntityWorkItemProcessor( - this, - workItem, - this.logHelper, - this.entityObjectManager, - this.entityDispatchPipeline, - this.entityBackendInformation, - this.errorPropagationMode) - : new OrchestrationWorkItemProcessor( - this, - workItem, - this.orchestrationService, - this.errorPropagationMode, - this.logHelper, - this.orchestrationObjectManager, - this.orchestrationDispatchPipeline); - - specializedDispatcher.workItem = workItem; - specializedDispatcher.runtimeState = workItem.OrchestrationRuntimeState; - specializedDispatcher.runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); + runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); + + OrchestrationRuntimeState originalOrchestrationRuntimeState = runtimeState; + + OrchestrationState? instanceState = null; Task? renewTask = null; using var renewCancellationTokenSource = new CancellationTokenSource(); @@ -326,31 +322,237 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work { // start a task to run RenewUntil renewTask = Task.Factory.StartNew( - () => this.RenewUntil(workItem, renewCancellationTokenSource.Token), + () => RenewUntil(workItem, this.orchestrationService, this.logHelper, nameof(TaskOrchestrationDispatcher), renewCancellationTokenSource.Token), renewCancellationTokenSource.Token); } try { // Assumes that: if the batch contains a new "ExecutionStarted" event, it is the first message in the batch. - if (!this.ReconcileMessagesWithState(workItem)) + if (!ReconcileMessagesWithState(workItem, "TaskOrchestrationDispatcher", logHelper)) { // TODO : mark an orchestration as faulted if there is data corruption this.logHelper.DroppingOrchestrationWorkItem(workItem, "Received work-item for an invalid orchestration"); TraceHelper.TraceSession( TraceEventType.Error, "TaskOrchestrationDispatcher-DeletedOrchestration", - specializedDispatcher.runtimeState.OrchestrationInstance?.InstanceId, + runtimeState.OrchestrationInstance?.InstanceId, "Received work-item for an invalid orchestration"); - - specializedDispatcher.isCompleted = true; + isCompleted = true; } else { - // now, do the actual processing of the work item - await specializedDispatcher.ProcessWorkItemAsync(); - } + do + { + continuedAsNew = false; + continuedAsNewMessage = null; + + this.logHelper.OrchestrationExecuting(runtimeState.OrchestrationInstance!, runtimeState.Name); + TraceHelper.TraceInstance( + TraceEventType.Verbose, + "TaskOrchestrationDispatcher-ExecuteUserOrchestration-Begin", + runtimeState.OrchestrationInstance, + "Executing user orchestration: {0}", + JsonDataConverter.Default.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true)); + + if (workItem.Cursor == null) + { + workItem.Cursor = await this.ExecuteOrchestrationAsync(runtimeState, workItem); + } + else + { + await this.ResumeOrchestrationAsync(workItem); + } + + IReadOnlyList decisions = workItem.Cursor.LatestDecisions.ToList(); + + this.logHelper.OrchestrationExecuted( + runtimeState.OrchestrationInstance!, + runtimeState.Name, + decisions); + TraceHelper.TraceInstance( + TraceEventType.Information, + "TaskOrchestrationDispatcher-ExecuteUserOrchestration-End", + runtimeState.OrchestrationInstance, + "Executed user orchestration. Received {0} orchestrator actions: {1}", + decisions.Count, + string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType))); + + // TODO: Exception handling for invalid decisions, which is increasingly likely + // when custom middleware is involved (e.g. out-of-process scenarios). + foreach (OrchestratorAction decision in decisions) + { + TraceHelper.TraceInstance( + TraceEventType.Information, + "TaskOrchestrationDispatcher-ProcessOrchestratorAction", + runtimeState.OrchestrationInstance, + "Processing orchestrator action of type {0}", + decision.OrchestratorActionType); + switch (decision.OrchestratorActionType) + { + case OrchestratorActionType.ScheduleOrchestrator: + var scheduleTaskAction = (ScheduleTaskOrchestratorAction)decision; + var message = this.ProcessScheduleTaskDecision( + scheduleTaskAction, + runtimeState, + this.IncludeParameters); + messagesToSend.Add(message); + break; + case OrchestratorActionType.CreateTimer: + var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision; + timerMessages.Add(this.ProcessCreateTimerDecision( + timerOrchestratorAction, + runtimeState, + isInternal: false)); + break; + case OrchestratorActionType.CreateSubOrchestration: + var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; + orchestratorMessages.Add( + this.ProcessCreateSubOrchestrationInstanceDecision( + createSubOrchestrationAction, + runtimeState, + this.IncludeParameters)); + break; + case OrchestratorActionType.SendEvent: + var sendEventAction = (SendEventOrchestratorAction)decision; + orchestratorMessages.Add( + this.ProcessSendEventDecision(sendEventAction, runtimeState)); + break; + case OrchestratorActionType.OrchestrationComplete: + OrchestrationCompleteOrchestratorAction completeDecision = (OrchestrationCompleteOrchestratorAction)decision; + TaskMessage? workflowInstanceCompletedMessage = + this.ProcessWorkflowCompletedTaskDecision(completeDecision, runtimeState, this.IncludeDetails, out continuedAsNew); + if (workflowInstanceCompletedMessage != null) + { + // Send complete message to parent workflow or to itself to start a new execution + // Store the event so we can rebuild the state + carryOverEvents = null; + if (continuedAsNew) + { + continuedAsNewMessage = workflowInstanceCompletedMessage; + continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent; + if (completeDecision.CarryoverEvents.Any()) + { + carryOverEvents = completeDecision.CarryoverEvents.ToList(); + completeDecision.CarryoverEvents.Clear(); + } + } + else + { + orchestratorMessages.Add(workflowInstanceCompletedMessage); + } + } + + isCompleted = !continuedAsNew; + break; + default: + throw TraceHelper.TraceExceptionInstance( + TraceEventType.Error, + "TaskOrchestrationDispatcher-UnsupportedDecisionType", + runtimeState.OrchestrationInstance, + new NotSupportedException($"Decision type '{decision.OrchestratorActionType}' not supported")); + } + + // Underlying orchestration service provider may have a limit of messages per call, to avoid the situation + // we keep on asking the provider if message count is ok and stop processing new decisions if not. + // + // We also put in a fake timer to force next orchestration task for remaining messages + int totalMessages = messagesToSend.Count + orchestratorMessages.Count + timerMessages.Count; + if (this.orchestrationService.IsMaxMessageCountExceeded(totalMessages, runtimeState)) + { + TraceHelper.TraceInstance( + TraceEventType.Information, + "TaskOrchestrationDispatcher-MaxMessageCountReached", + runtimeState.OrchestrationInstance, + "MaxMessageCount reached. Adding timer to process remaining events in next attempt."); + + if (isCompleted || continuedAsNew) + { + TraceHelper.TraceInstance( + TraceEventType.Information, + "TaskOrchestrationDispatcher-OrchestrationAlreadyCompleted", + runtimeState.OrchestrationInstance, + "Orchestration already completed. Skip adding timer for splitting messages."); + break; + } + + var dummyTimer = new CreateTimerOrchestratorAction + { + Id = FrameworkConstants.FakeTimerIdToSplitDecision, + FireAt = DateTime.UtcNow + }; + + timerMessages.Add(this.ProcessCreateTimerDecision( + dummyTimer, + runtimeState, + isInternal: true)); + isInterrupted = true; + break; + } + } + + // correlation + CorrelationTraceClient.Propagate(() => + { + if (runtimeState.ExecutionStartedEvent != null) + runtimeState.ExecutionStartedEvent.Correlation = CorrelationTraceContext.Current.SerializableTraceContext; + }); + + + // finish up processing of the work item + if (!continuedAsNew && runtimeState.Events.Last().EventType != EventType.OrchestratorCompleted) + { + runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); + } + + if (isCompleted) + { + TraceHelper.TraceSession(TraceEventType.Information, "TaskOrchestrationDispatcher-DeletingSessionState", workItem.InstanceId, "Deleting session state"); + if (runtimeState.ExecutionStartedEvent != null) + { + instanceState = Utils.BuildOrchestrationState(runtimeState); + } + } + else + { + if (continuedAsNew) + { + TraceHelper.TraceSession( + TraceEventType.Information, + "TaskOrchestrationDispatcher-UpdatingStateForContinuation", + workItem.InstanceId, + "Updating state for continuation"); + // correlation + CorrelationTraceClient.Propagate(() => + { + continueAsNewExecutionStarted!.Correlation = CorrelationTraceContext.Current.SerializableTraceContext; + }); + + runtimeState = new OrchestrationRuntimeState(); + runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); + runtimeState.AddEvent(continueAsNewExecutionStarted!); + runtimeState.Status = workItem.OrchestrationRuntimeState.Status ?? carryOverStatus; + carryOverStatus = workItem.OrchestrationRuntimeState.Status; + + if (carryOverEvents != null) + { + foreach (var historyEvent in carryOverEvents) + { + runtimeState.AddEvent(historyEvent); + } + } + + runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); + workItem.OrchestrationRuntimeState = runtimeState; + + workItem.Cursor = null; + } + + instanceState = Utils.BuildOrchestrationState(runtimeState); + } + } while (continuedAsNew); + } } finally { @@ -378,27 +580,39 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work workItem.OrchestrationRuntimeState = originalOrchestrationRuntimeState; } + runtimeState.Status = runtimeState.Status ?? carryOverStatus; + + if (instanceState != null) + { + instanceState.Status = runtimeState.Status; + } + await this.orchestrationService.CompleteTaskOrchestrationWorkItemAsync( workItem, - specializedDispatcher.runtimeState, - specializedDispatcher.continuedAsNew ? null : specializedDispatcher.messagesToSend, - specializedDispatcher.orchestratorMessages, - specializedDispatcher.continuedAsNew ? null : specializedDispatcher.timerMessages, - specializedDispatcher.continuedAsNewMessage, - specializedDispatcher.instanceState); + runtimeState, + continuedAsNew ? null : messagesToSend, + orchestratorMessages, + continuedAsNew ? null : timerMessages, + continuedAsNewMessage, + instanceState); if (workItem.RestoreOriginalRuntimeStateDuringCompletion) { - workItem.OrchestrationRuntimeState = specializedDispatcher.runtimeState; + workItem.OrchestrationRuntimeState = runtimeState; } - return specializedDispatcher.isCompleted || specializedDispatcher.continuedAsNew || specializedDispatcher.isInterrupted; + return isCompleted || continuedAsNew || isInterrupted; } - TimeSpan MinRenewalInterval = TimeSpan.FromSeconds(5); // prevents excessive retries if clocks are off - TimeSpan MaxRenewalInterval = TimeSpan.FromSeconds(30); + static OrchestrationExecutionContext GetOrchestrationExecutionContext(OrchestrationRuntimeState runtimeState) + { + return new OrchestrationExecutionContext { OrchestrationTags = runtimeState.Tags ?? new Dictionary(capacity: 0) }; + } + + static TimeSpan MinRenewalInterval = TimeSpan.FromSeconds(5); // prevents excessive retries if clocks are off + static TimeSpan MaxRenewalInterval = TimeSpan.FromSeconds(30); - async Task RenewUntil(TaskOrchestrationWorkItem workItem, CancellationToken cancellationToken) + internal static async Task RenewUntil(TaskOrchestrationWorkItem workItem, IOrchestrationService orchestrationService, LogHelper logHelper, string dispatcher, CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { @@ -421,28 +635,107 @@ async Task RenewUntil(TaskOrchestrationWorkItem workItem, CancellationToken canc try { - this.logHelper.RenewOrchestrationWorkItemStarting(workItem); - TraceHelper.Trace(TraceEventType.Information, "TaskOrchestrationDispatcher-RenewWorkItemStarting", "Renewing work item for instance {0}", workItem.InstanceId); - await this.orchestrationService.RenewTaskOrchestrationWorkItemLockAsync(workItem); - this.logHelper.RenewOrchestrationWorkItemCompleted(workItem); - TraceHelper.Trace(TraceEventType.Information, "TaskOrchestrationDispatcher-RenewWorkItemCompleted", "Successfully renewed work item for instance {0}", workItem.InstanceId); + logHelper.RenewOrchestrationWorkItemStarting(workItem); + TraceHelper.Trace(TraceEventType.Information, $"{dispatcher}-RenewWorkItemStarting", "Renewing work item for instance {0}", workItem.InstanceId); + await orchestrationService.RenewTaskOrchestrationWorkItemLockAsync(workItem); + logHelper.RenewOrchestrationWorkItemCompleted(workItem); + TraceHelper.Trace(TraceEventType.Information, $"{dispatcher}-RenewWorkItemCompleted", "Successfully renewed work item for instance {0}", workItem.InstanceId); } catch (Exception exception) when (!Utils.IsFatal(exception)) { - this.logHelper.RenewOrchestrationWorkItemFailed(workItem, exception); - TraceHelper.TraceException(TraceEventType.Warning, "TaskOrchestrationDispatcher-RenewWorkItemFailed", exception, "Failed to renew work item for instance {0}", workItem.InstanceId); + logHelper.RenewOrchestrationWorkItemFailed(workItem, exception); + TraceHelper.TraceException(TraceEventType.Warning, $"{dispatcher}-RenewWorkItemFailed", exception, "Failed to renew work item for instance {0}", workItem.InstanceId); } } } + async Task ExecuteOrchestrationAsync(OrchestrationRuntimeState runtimeState, TaskOrchestrationWorkItem workItem) + { + // Get the TaskOrchestration implementation. If it's not found, it either means that the developer never + // registered it (which is an error, and we'll throw for this further down) or it could be that some custom + // middleware (e.g. out-of-process execution middleware) is intended to implement the orchestration logic. + TaskOrchestration? taskOrchestration = this.objectManager.GetObject(runtimeState.Name, runtimeState.Version!); + + var dispatchContext = new DispatchMiddlewareContext(); + dispatchContext.SetProperty(runtimeState.OrchestrationInstance); + dispatchContext.SetProperty(taskOrchestration); + dispatchContext.SetProperty(runtimeState); + dispatchContext.SetProperty(workItem); + dispatchContext.SetProperty(GetOrchestrationExecutionContext(runtimeState)); + + TaskOrchestrationExecutor? executor = null; + + await this.dispatchPipeline.RunAsync(dispatchContext, _ => + { + // Check to see if the custom middleware intercepted and substituted the orchestration execution + // with its own execution behavior, providing us with the end results. If so, we can terminate + // the dispatch pipeline here. + var resultFromMiddleware = dispatchContext.GetProperty(); + if (resultFromMiddleware != null) + { + return CompletedTask; + } + + if (taskOrchestration == null) + { + throw TraceHelper.TraceExceptionInstance( + TraceEventType.Error, + "TaskOrchestrationDispatcher-TypeMissing", + runtimeState.OrchestrationInstance, + new TypeMissingException($"Orchestration not found: ({runtimeState.Name}, {runtimeState.Version})")); + } + + executor = new TaskOrchestrationExecutor( + runtimeState, + taskOrchestration, + this.orchestrationService.EventBehaviourForContinueAsNew, + this.entityBackendInformation, + this.errorPropagationMode); ; + OrchestratorExecutionResult resultFromOrchestrator = executor.Execute(); + dispatchContext.SetProperty(resultFromOrchestrator); + return CompletedTask; + }); + + var result = dispatchContext.GetProperty(); + IEnumerable decisions = result?.Actions ?? Enumerable.Empty(); + runtimeState.Status = result?.CustomStatus; + + return new OrchestrationExecutionCursor(runtimeState, taskOrchestration, executor, decisions); + } + + async Task ResumeOrchestrationAsync(TaskOrchestrationWorkItem workItem) + { + OrchestrationExecutionCursor cursor = workItem.Cursor; + + var dispatchContext = new DispatchMiddlewareContext(); + dispatchContext.SetProperty(cursor.RuntimeState.OrchestrationInstance); + dispatchContext.SetProperty(cursor.TaskOrchestration); + dispatchContext.SetProperty(cursor.RuntimeState); + dispatchContext.SetProperty(workItem); + + cursor.LatestDecisions = Enumerable.Empty(); + await this.dispatchPipeline.RunAsync(dispatchContext, _ => + { + OrchestratorExecutionResult result = cursor.OrchestrationExecutor.ExecuteNewEvents(); + dispatchContext.SetProperty(result); + return CompletedTask; + }); + + var result = dispatchContext.GetProperty(); + cursor.LatestDecisions = result?.Actions ?? Enumerable.Empty(); + cursor.RuntimeState.Status = result?.CustomStatus; + } + /// /// Converts new messages into history events that get appended to the existing orchestration state. /// Returns False if the workItem should be discarded. True if it should be processed further. /// Assumes that: if the batch contains a new "ExecutionStarted" event, it is the first message in the batch. /// /// A batch of work item messages. + /// The name of the dispatcher, used for tracing. + /// The log helper. /// True if workItem should be processed further. False otherwise. - bool ReconcileMessagesWithState(TaskOrchestrationWorkItem workItem) + internal static bool ReconcileMessagesWithState(TaskOrchestrationWorkItem workItem, string dispatcher, LogHelper logHelper) { foreach (TaskMessage message in workItem.NewMessages) { @@ -451,7 +744,7 @@ bool ReconcileMessagesWithState(TaskOrchestrationWorkItem workItem) { throw TraceHelper.TraceException( TraceEventType.Error, - "TaskOrchestrationDispatcher-OrchestrationInstanceMissing", + $"{dispatcher}-OrchestrationInstanceMissing", new InvalidOperationException("Message does not contain any OrchestrationInstance information")); } @@ -469,10 +762,10 @@ bool ReconcileMessagesWithState(TaskOrchestrationWorkItem workItem) return false; } - this.logHelper.ProcessingOrchestrationMessage(workItem, message); + logHelper.ProcessingOrchestrationMessage(workItem, message); TraceHelper.TraceInstance( TraceEventType.Information, - "TaskOrchestrationDispatcher-ProcessEvent", + $"{dispatcher}-ProcessEvent", orchestrationInstance, "Processing new event with Id {0} and type {1}", message.Event.EventId, @@ -483,10 +776,10 @@ bool ReconcileMessagesWithState(TaskOrchestrationWorkItem workItem) if (workItem.OrchestrationRuntimeState.ExecutionStartedEvent != null) { // this was caused due to a dupe execution started event, swallow this one - this.logHelper.DroppingOrchestrationMessage(workItem, message, "Duplicate start event"); + logHelper.DroppingOrchestrationMessage(workItem, message, "Duplicate start event"); TraceHelper.TraceInstance( TraceEventType.Warning, - "TaskOrchestrationDispatcher-DuplicateStartEvent", + $"{dispatcher}-DuplicateStartEvent", orchestrationInstance, "Duplicate start event. Ignoring event with Id {0} and type {1} ", message.Event.EventId, @@ -500,13 +793,13 @@ bool ReconcileMessagesWithState(TaskOrchestrationWorkItem workItem) workItem.OrchestrationRuntimeState.OrchestrationInstance?.ExecutionId)) { // eat up any events for previous executions - this.logHelper.DroppingOrchestrationMessage( + logHelper.DroppingOrchestrationMessage( workItem, message, $"ExecutionId of event ({orchestrationInstance.ExecutionId}) does not match current executionId"); TraceHelper.TraceInstance( TraceEventType.Warning, - "TaskOrchestrationDispatcher-ExecutionIdMismatch", + $"{dispatcher}-ExecutionIdMismatch", orchestrationInstance, "ExecutionId of event does not match current executionId. Ignoring event with Id {0} and type {1} ", message.Event.EventId, @@ -520,7 +813,245 @@ bool ReconcileMessagesWithState(TaskOrchestrationWorkItem workItem) return true; } - class NonBlockingCountdownLock + TaskMessage? ProcessWorkflowCompletedTaskDecision( + OrchestrationCompleteOrchestratorAction completeOrchestratorAction, + OrchestrationRuntimeState runtimeState, + bool includeDetails, + out bool continuedAsNew) + { + ExecutionCompletedEvent executionCompletedEvent; + continuedAsNew = (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew); + if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew) + { + executionCompletedEvent = new ContinueAsNewEvent(completeOrchestratorAction.Id, + completeOrchestratorAction.Result); + } + else + { + executionCompletedEvent = new ExecutionCompletedEvent(completeOrchestratorAction.Id, + completeOrchestratorAction.Result, + completeOrchestratorAction.OrchestrationStatus, + completeOrchestratorAction.FailureDetails); + } + + runtimeState.AddEvent(executionCompletedEvent); + + this.logHelper.OrchestrationCompleted(runtimeState, completeOrchestratorAction); + TraceHelper.TraceInstance( + runtimeState.OrchestrationStatus == OrchestrationStatus.Failed ? TraceEventType.Warning : TraceEventType.Information, + "TaskOrchestrationDispatcher-InstanceCompleted", + runtimeState.OrchestrationInstance, + "Instance Id '{0}' completed in state {1} with result: {2}", + runtimeState.OrchestrationInstance, + runtimeState.OrchestrationStatus, + completeOrchestratorAction.Result); + TraceHelper.TraceInstance( + TraceEventType.Information, + "TaskOrchestrationDispatcher-InstanceCompletionEvents", + runtimeState.OrchestrationInstance, + () => Utils.EscapeJson(JsonDataConverter.Default.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true))); + + // Check to see if we need to start a new execution + if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew) + { + var taskMessage = new TaskMessage(); + var startedEvent = new ExecutionStartedEvent(-1, completeOrchestratorAction.Result) + { + OrchestrationInstance = new OrchestrationInstance + { + InstanceId = runtimeState.OrchestrationInstance!.InstanceId, + ExecutionId = Guid.NewGuid().ToString("N") + }, + Tags = runtimeState.Tags, + ParentInstance = runtimeState.ParentInstance, + Name = runtimeState.Name, + Version = completeOrchestratorAction.NewVersion ?? runtimeState.Version + }; + + taskMessage.OrchestrationInstance = startedEvent.OrchestrationInstance; + taskMessage.Event = startedEvent; + + return taskMessage; + } + + // If this is a Sub Orchestration, and not tagged as fire-and-forget, + // then notify the parent by sending a complete message + if (runtimeState.ParentInstance != null + && !OrchestrationTags.IsTaggedAsFireAndForget(runtimeState.Tags)) + { + var taskMessage = new TaskMessage(); + if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Completed) + { + var subOrchestrationCompletedEvent = + new SubOrchestrationInstanceCompletedEvent(-1, runtimeState.ParentInstance.TaskScheduleId, + completeOrchestratorAction.Result); + + taskMessage.Event = subOrchestrationCompletedEvent; + } + else if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Failed || + completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Terminated) + { + var subOrchestrationFailedEvent = + new SubOrchestrationInstanceFailedEvent(-1, runtimeState.ParentInstance.TaskScheduleId, + completeOrchestratorAction.Result, + includeDetails ? completeOrchestratorAction.Details : null); + subOrchestrationFailedEvent.FailureDetails = completeOrchestratorAction.FailureDetails; + + taskMessage.Event = subOrchestrationFailedEvent; + } + + if (taskMessage.Event != null) + { + taskMessage.OrchestrationInstance = runtimeState.ParentInstance.OrchestrationInstance; + return taskMessage; + } + } + + return null; + } + + TaskMessage ProcessScheduleTaskDecision( + ScheduleTaskOrchestratorAction scheduleTaskOrchestratorAction, + OrchestrationRuntimeState runtimeState, + bool includeParameters) + { + if (scheduleTaskOrchestratorAction.Name == null) + { + throw new ArgumentException("No name was given for the task activity to schedule!", nameof(scheduleTaskOrchestratorAction)); + } + + var taskMessage = new TaskMessage(); + + var scheduledEvent = new TaskScheduledEvent( + eventId: scheduleTaskOrchestratorAction.Id, + name: scheduleTaskOrchestratorAction.Name, + version: scheduleTaskOrchestratorAction.Version, + input: scheduleTaskOrchestratorAction.Input); + + taskMessage.Event = scheduledEvent; + taskMessage.OrchestrationInstance = runtimeState.OrchestrationInstance; + taskMessage.OrchestrationExecutionContext = GetOrchestrationExecutionContext(runtimeState); + + if (!includeParameters) + { + scheduledEvent = new TaskScheduledEvent( + eventId: scheduleTaskOrchestratorAction.Id, + name: scheduleTaskOrchestratorAction.Name, + version: scheduleTaskOrchestratorAction.Version); + } + + this.logHelper.SchedulingActivity( + runtimeState.OrchestrationInstance!, + scheduledEvent); + + runtimeState.AddEvent(scheduledEvent); + return taskMessage; + } + + TaskMessage ProcessCreateTimerDecision( + CreateTimerOrchestratorAction createTimerOrchestratorAction, + OrchestrationRuntimeState runtimeState, + bool isInternal) + { + var taskMessage = new TaskMessage(); + + var timerCreatedEvent = new TimerCreatedEvent(createTimerOrchestratorAction.Id) + { + FireAt = createTimerOrchestratorAction.FireAt + }; + + runtimeState.AddEvent(timerCreatedEvent); + + taskMessage.Event = new TimerFiredEvent(-1) + { + TimerId = createTimerOrchestratorAction.Id, + FireAt = createTimerOrchestratorAction.FireAt + }; + + this.logHelper.CreatingTimer( + runtimeState.OrchestrationInstance!, + timerCreatedEvent, + isInternal); + + taskMessage.OrchestrationInstance = runtimeState.OrchestrationInstance; + + return taskMessage; + } + + TaskMessage ProcessCreateSubOrchestrationInstanceDecision( + CreateSubOrchestrationAction createSubOrchestrationAction, + OrchestrationRuntimeState runtimeState, + bool includeParameters) + { + var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id) + { + Name = createSubOrchestrationAction.Name, + Version = createSubOrchestrationAction.Version, + InstanceId = createSubOrchestrationAction.InstanceId + }; + if (includeParameters) + { + historyEvent.Input = createSubOrchestrationAction.Input; + } + + runtimeState.AddEvent(historyEvent); + + var taskMessage = new TaskMessage(); + + var startedEvent = new ExecutionStartedEvent(-1, createSubOrchestrationAction.Input) + { + Tags = OrchestrationTags.MergeTags(createSubOrchestrationAction.Tags, runtimeState.Tags), + OrchestrationInstance = new OrchestrationInstance + { + InstanceId = createSubOrchestrationAction.InstanceId, + ExecutionId = Guid.NewGuid().ToString("N") + }, + ParentInstance = new ParentInstance + { + OrchestrationInstance = runtimeState.OrchestrationInstance, + Name = runtimeState.Name, + Version = runtimeState.Version, + TaskScheduleId = createSubOrchestrationAction.Id + }, + Name = createSubOrchestrationAction.Name, + Version = createSubOrchestrationAction.Version + }; + + this.logHelper.SchedulingOrchestration(startedEvent); + + taskMessage.OrchestrationInstance = startedEvent.OrchestrationInstance; + taskMessage.Event = startedEvent; + taskMessage.OrchestrationExecutionContext = GetOrchestrationExecutionContext(runtimeState); + + return taskMessage; + } + + TaskMessage ProcessSendEventDecision( + SendEventOrchestratorAction sendEventAction, + OrchestrationRuntimeState runtimeState) + { + var historyEvent = new EventSentEvent(sendEventAction.Id) + { + InstanceId = sendEventAction.Instance?.InstanceId, + Name = sendEventAction.EventName, + Input = sendEventAction.EventData + }; + + runtimeState.AddEvent(historyEvent); + + this.logHelper.RaisingEvent(runtimeState.OrchestrationInstance!, historyEvent); + + return new TaskMessage + { + OrchestrationInstance = sendEventAction.Instance, + Event = new EventRaisedEvent(-1, sendEventAction.EventData) + { + Name = sendEventAction.EventName + } + }; + } + + internal class NonBlockingCountdownLock { int available; @@ -559,33 +1090,5 @@ public void Release() Interlocked.Increment(ref this.available); } } - - /// - /// Base class for the specialized work item processing for entity and orchestration work items, respectively - /// - internal abstract class WorkItemProcessor - { - public TaskOrchestrationDispatcher dispatcher; - public TaskOrchestrationWorkItem workItem; - public OrchestrationRuntimeState runtimeState; - - public List messagesToSend = new List(); - public List timerMessages = new List(); - public List orchestratorMessages = new List(); - public bool isCompleted; - public bool continuedAsNew; - public bool isInterrupted; - public OrchestrationState? instanceState; - public TaskMessage? continuedAsNewMessage; - - public WorkItemProcessor(TaskOrchestrationDispatcher dispatcher, TaskOrchestrationWorkItem workItem) - { - this.dispatcher = dispatcher; - this.workItem = workItem; - this.runtimeState = workItem.OrchestrationRuntimeState; - } - - public abstract Task ProcessWorkItemAsync(); - } } } \ No newline at end of file diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index e3076e050..d1b32eaaa 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -838,13 +838,16 @@ public async Task EntityFireAndForget(bool extendedSessions) } /// - /// End-to-end test which validates a simple entity scenario which sends a signal - /// to a relay which forwards it to counter, and polls until the signal is delivered. + /// End-to-end test which validates two simple entity scenarios: + /// a) send a signal to a counter, then poll until the signal is delivered. + /// b) same, but send the signal indirectly via a relay entity. /// [DataTestMethod] - [DataRow(false)] - [DataRow(true)] - public async Task DurableEntity_SignalThenPoll(bool extendedSessions) + [DataRow(false, false)] + [DataRow(true, false)] + [DataRow(false, true)] + [DataRow(true, true)] + public async Task DurableEntity_SignalThenPoll(bool extendedSessions, bool viaRelay) { using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: extendedSessions)) { @@ -854,9 +857,16 @@ public async Task DurableEntity_SignalThenPoll(bool extendedSessions) var counterEntityId = new EntityId("Counter", Guid.NewGuid().ToString()); var client = await host.StartOrchestrationAsync(typeof(Orchestrations.PollCounterEntity), counterEntityId); - var entityClient = new TaskHubEntityClient(client.InnerClient); - await entityClient.SignalEntityAsync(relayEntityId, "", (counterEntityId, "increment")); + + if (viaRelay) + { + await entityClient.SignalEntityAsync(relayEntityId, "", (counterEntityId, "increment")); + } + else + { + await entityClient.SignalEntityAsync(counterEntityId, "increment"); + } var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); @@ -4239,34 +4249,32 @@ public override int CreateInitialState(EntityContext ctx) return 0; } - public override ValueTask ExecuteOperationAsync(EntityContext ctx) + public override ValueTask ExecuteOperationAsync(EntityContext ctx) { switch (ctx.OperationName) { case "get": - ctx.Return(ctx.State); - break; + return new ValueTask(ctx.State); case "increment": ctx.State++; - ctx.Return(ctx.State); - break; + return new ValueTask(ctx.State); case "add": ctx.State += ctx.GetInput(); - ctx.Return(ctx.State); - break; + return new ValueTask(ctx.State); case "set": ctx.State = ctx.GetInput(); - break; + return default; case "delete": ctx.DeleteState(); - break; - } + return default; - return default; + default: + throw new InvalidOperationException($"unknown operation: {ctx.OperationName}"); + } } } @@ -4274,7 +4282,7 @@ public override ValueTask ExecuteOperationAsync(EntityContext ctx) internal class Relay : TaskEntity { - public override ValueTask ExecuteOperationAsync(EntityContext context) + public override ValueTask ExecuteOperationAsync(EntityContext context) { var (destination, operation) = context.GetInput<(EntityId, string)>(); @@ -4290,7 +4298,7 @@ internal class BatchEntity : TaskEntity> public override List<(int, int)> CreateInitialState(EntityContext> context) => new List<(int, int)>(); - public override ValueTask ExecuteOperationAsync(EntityContext> context) + public override ValueTask ExecuteOperationAsync(EntityContext> context) { context.State.Add((context.BatchPosition, context.BatchSize)); return default; @@ -4303,22 +4311,20 @@ public override ValueTask ExecuteOperationAsync(EntityContext> // "get" (returns a string containing the current state)// An entity that records all batch positions and batch sizes internal class StringStore : TaskEntity { - public override ValueTask ExecuteOperationAsync(EntityContext context) + public override ValueTask ExecuteOperationAsync(EntityContext context) { switch (context.OperationName) { case "set": context.State = context.GetInput(); - break; + return default; case "get": - context.Return(context.State); - break; + return new ValueTask(context.State); default: throw new NotImplementedException("no such operation"); } - return default; } } @@ -4328,17 +4334,17 @@ public override ValueTask ExecuteOperationAsync(EntityContext context) // - a new operation "delete" deletes the entity, i.e. clears all state internal class StringStore2 : TaskEntity { - public override ValueTask ExecuteOperationAsync(EntityContext context) + public override ValueTask ExecuteOperationAsync(EntityContext context) { switch (context.OperationName) { case "delete": context.DeleteState(); - break; + return default; case "set": context.State = context.GetInput(); - break; + return default; case "get": if (!context.HasState) @@ -4346,13 +4352,11 @@ public override ValueTask ExecuteOperationAsync(EntityContext context) throw new InvalidOperationException("this entity does not like 'get' when it does not have state yet"); } - context.Return(context.State); - break; + return new ValueTask(context.State); default: throw new NotImplementedException("no such operation"); } - return default; } } @@ -4368,33 +4372,30 @@ public class State public override State CreateInitialState(EntityContext context) => new State(); - public override ValueTask ExecuteOperationAsync(EntityContext context) + public override ValueTask ExecuteOperationAsync(EntityContext context) { switch (context.OperationName) { case "launch": { context.State.Id = context.StartNewOrchestration(typeof(Orchestrations.DelayedSignal), input: context.EntityId); - break; + return default; } case "done": { context.State.Done = true; - break; + return default; } case "get": { - context.Return(context.State.Done ? context.State.Id : null); - break; + return new ValueTask(context.State.Done ? context.State.Id : null); } default: throw new NotImplementedException("no such entity operation"); } - - return default; } } @@ -4402,13 +4403,21 @@ public override ValueTask ExecuteOperationAsync(EntityContext context) internal class FaultyEntityWithRollback : FaultyEntity { - public override bool RollbackOnExceptions => true; + public override ValueTask ExecuteOperationAsync(EntityContext context) + { + context.EntityExecutionOptions.RollbackOnExceptions = true; + return base.ExecuteOperationAsync(context); + } public override FaultyEntity CreateInitialState(EntityContext context) => new FaultyEntityWithRollback(); } internal class FaultyEntityWithoutRollback : FaultyEntity { - public override bool RollbackOnExceptions => false; + public override ValueTask ExecuteOperationAsync(EntityContext context) + { + context.EntityExecutionOptions.RollbackOnExceptions = false; + return base.ExecuteOperationAsync(context); + } public override FaultyEntity CreateInitialState(EntityContext context) => new FaultyEntityWithoutRollback(); } @@ -4425,50 +4434,49 @@ internal abstract class FaultyEntity : TaskEntity [JsonProperty] public int NumberIncrementsSent { get; set; } - public override DataConverter ErrorDataConverter => new JsonDataConverter(new JsonSerializerSettings + DataConverter ErrorDataConverter = new JsonDataConverter(new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Objects, ReferenceLoopHandling = ReferenceLoopHandling.Ignore, }); - public override ValueTask ExecuteOperationAsync(EntityContext context) + public override ValueTask ExecuteOperationAsync(EntityContext context) { + context.EntityExecutionOptions.ErrorDataConverter = this.ErrorDataConverter; + switch (context.OperationName) { case "exists": - context.Return(context.HasState); - break; + return new ValueTask(context.HasState); case "deletewithoutreading": context.DeleteState(); - break; + return default; case "Get": if (!context.HasState) { - context.Return(0); + return new ValueTask(0); } else { - context.Return(context.State.Value); + return new ValueTask(context.State.Value); } - break; case "GetNumberIncrementsSent": - context.Return(context.State.NumberIncrementsSent); - break; + return new ValueTask(context.State.NumberIncrementsSent); case "Set": context.State.Value = context.GetInput(); - break; + return default; case "SetToUnserializable": context.State.ObjectWithFaultySerialization = new UnserializableKaboom(); - break; + return default; case "SetToUnDeserializable": context.State.ObjectWithFaultySerialization = new UnDeserializableKaboom(); - break; + return default; case "SetThenThrow": context.State.Value = context.GetInput(); @@ -4476,7 +4484,7 @@ public override ValueTask ExecuteOperationAsync(EntityContext cont case "Send": Send(); - break; + return default; case "SendThenThrow": Send(); @@ -4485,11 +4493,11 @@ public override ValueTask ExecuteOperationAsync(EntityContext cont case "SendThenMakeUnserializable": Send(); context.State.ObjectWithFaultySerialization = new UnserializableKaboom(); - break; + return default; case "Delete": context.DeleteState(); - break; + return default; case "DeleteThenThrow": context.DeleteState(); @@ -4503,8 +4511,10 @@ public override ValueTask ExecuteOperationAsync(EntityContext cont case "ThrowUnDeserializable": throw new FaultyEntity.UnDeserializableKaboom(); + + default: + throw new NotImplementedException("no such entity operation"); } - return default; void Send() { @@ -4565,7 +4575,6 @@ public class UnDeserializableKaboom : Exception { } } - } diff --git a/test/DurableTask.AzureStorage.Tests/TestOrchestrationHost.cs b/test/DurableTask.AzureStorage.Tests/TestOrchestrationHost.cs index e9813c4ea..ae4ca843d 100644 --- a/test/DurableTask.AzureStorage.Tests/TestOrchestrationHost.cs +++ b/test/DurableTask.AzureStorage.Tests/TestOrchestrationHost.cs @@ -100,7 +100,8 @@ public async Task StartOrchestrationAsync( { bool orch = referencedKnownType.Type.IsSubclassOf(typeof(TaskOrchestration)); bool activ = referencedKnownType.Type.IsSubclassOf(typeof(TaskActivity)); - bool entit = referencedKnownType.Type.IsSubclassOf(typeof(Core.Entities.TaskEntity)); + bool entit = referencedKnownType.Type.IsSubclassOf(typeof(TaskEntity)); + if (orch && !this.addedOrchestrationTypes.Contains(referencedKnownType.Type)) { this.worker.AddTaskOrchestrations(referencedKnownType.Type); From 1d69426876592b786a829f6829cc71db49efdd62 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Thu, 13 Apr 2023 10:49:50 -0700 Subject: [PATCH 7/8] address PR feedback --- .../AzureStorageOrchestrationService.cs | 4 ++-- ...zureStorageOrchestrationServiceSettings.cs | 6 +++++ .../OrchestrationSessionManager.cs | 6 +++++ .../Entities/EntityBackendInformation.cs | 9 ++++++-- .../Entities/EntityExecutionOptions.cs | 2 +- src/DurableTask.Core/Entities/EntityId.cs | 4 ++-- src/DurableTask.Core/Entities/EventToSend.cs | 4 ++-- .../Entities/IEntityOrchestrationService.cs | 4 ++-- .../Entities/LocalSDK/TaskHubEntityClient.cs | 14 ++++++------ .../Entities/OrchestrationEntityContext.cs | 2 +- src/DurableTask.Core/FailureDetails.cs | 8 +++---- src/DurableTask.Core/Logging/LogEvents.cs | 17 +++++++++++--- src/DurableTask.Core/Logging/LogHelper.cs | 1 - .../Logging/StructuredEventSource.cs | 4 ++-- src/DurableTask.Core/OrchestrationContext.cs | 2 +- src/DurableTask.Core/TaskEntityDispatcher.cs | 22 +++++++++---------- src/DurableTask.Core/TaskHubWorker.cs | 14 ++++++++---- .../TaskOrchestrationContext.cs | 8 +++---- .../TaskOrchestrationDispatcher.cs | 6 ++--- .../TaskOrchestrationExecutor.cs | 8 +++---- 20 files changed, 89 insertions(+), 56 deletions(-) diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs index 535ee570a..5e9a04cb5 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs @@ -272,8 +272,8 @@ public BehaviorOnContinueAsNew EventBehaviourForContinueAsNew #region IEntityOrchestrationService - EntityBackendInformation IEntityOrchestrationService.GetEntityBackendInformation() - => new EntityBackendInformation() + EntityBackendProperties IEntityOrchestrationService.GetEntityBackendProperties() + => new EntityBackendProperties() { EntityMessageReorderWindow = TimeSpan.FromMinutes(this.settings.EntityMessageReorderWindowInMinutes), MaxEntityOperationBatchSize = this.settings.MaxEntityOperationBatchSize, diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs index c36137f47..eee9c1947 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs @@ -111,6 +111,12 @@ public class AzureStorageOrchestrationServiceSettings /// public int MaxConcurrentTaskOrchestrationWorkItems { get; set; } = 100; + /// + /// Gets or sets the maximum number of entity operation batches that can be processed concurrently on a single node. + /// The default value is 100. + /// + public int MaxConcurrentTaskEntityWorkItems { get; set; } = 100; + /// /// Gets or sets the maximum number of concurrent storage operations that can be executed in the context /// of a single orchestration instance. diff --git a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs index c2ee22292..333052b9a 100644 --- a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs +++ b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs @@ -58,6 +58,12 @@ public OrchestrationSessionManager( internal IEnumerable Queues => this.ownedControlQueues.Values; + /// + /// Recent versions of DurableTask.Core can be configured to use a separate pipeline for processing entity work items, + /// while older versions use a single pipeline for both orchestration and entity work items. To support both scenarios, + /// this property can be modified prior to starting the orchestration service. If set to true, the work items that are ready for + /// processing are stored in and , respectively. + /// internal bool ProcessEntitiesSeparately { get; set; } public void AddQueue(string partitionId, ControlQueue controlQueue, CancellationToken cancellationToken) diff --git a/src/DurableTask.Core/Entities/EntityBackendInformation.cs b/src/DurableTask.Core/Entities/EntityBackendInformation.cs index bf95da387..1d765c65a 100644 --- a/src/DurableTask.Core/Entities/EntityBackendInformation.cs +++ b/src/DurableTask.Core/Entities/EntityBackendInformation.cs @@ -17,7 +17,7 @@ namespace DurableTask.Core.Entities /// /// Entity processing characteristics that are controlled by the backend provider, i.e. the orchestration service. /// - public class EntityBackendInformation + public class EntityBackendProperties { /// /// The time window within which entity messages should be deduplicated and reordered. @@ -28,7 +28,12 @@ public class EntityBackendInformation /// /// A limit on the number of entity operations that should be processed as a single batch, or null if there is no limit. /// - public int? MaxEntityOperationBatchSize { get; set; } + public int? MaxEntityOperationBatchSize { get; set; } + + /// + /// The maximum number of entity operation batches that can be processed concurrently on a single node. + /// + public int MaxConcurrentTaskEntityWorkItems { get; set; } /// /// Whether the backend supports implicit deletion, i.e. setting the entity scheduler state to null implicitly deletes the storage record. diff --git a/src/DurableTask.Core/Entities/EntityExecutionOptions.cs b/src/DurableTask.Core/Entities/EntityExecutionOptions.cs index 15bcdde4c..6e6603b92 100644 --- a/src/DurableTask.Core/Entities/EntityExecutionOptions.cs +++ b/src/DurableTask.Core/Entities/EntityExecutionOptions.cs @@ -46,7 +46,7 @@ public class EntityExecutionOptions /// /// Information about backend entity support. /// - internal EntityBackendInformation EntityBackendInformation { get; set; } + internal EntityBackendProperties EntityBackendProperties { get; set; } /// /// The mode that is used for propagating errors, as specified in the . diff --git a/src/DurableTask.Core/Entities/EntityId.cs b/src/DurableTask.Core/Entities/EntityId.cs index 12724732f..a7de3480a 100644 --- a/src/DurableTask.Core/Entities/EntityId.cs +++ b/src/DurableTask.Core/Entities/EntityId.cs @@ -73,9 +73,9 @@ public static EntityId FromString(string instanceId) throw new ArgumentNullException(nameof(instanceId)); } var pos = instanceId.IndexOf('@', 1); - if (pos <= 0) + if ( pos <= 0 || instanceId[0] != '@') { - throw new ArgumentException("instanceId is not a valid entityId", nameof(instanceId)); + throw new ArgumentException($"Instance ID '{instanceId}' is not a valid entity ID.", nameof(instanceId)); } var entityName = instanceId.Substring(1, pos - 1); var entityKey = instanceId.Substring(pos + 1); diff --git a/src/DurableTask.Core/Entities/EventToSend.cs b/src/DurableTask.Core/Entities/EventToSend.cs index 2a567fb7a..693db16bc 100644 --- a/src/DurableTask.Core/Entities/EventToSend.cs +++ b/src/DurableTask.Core/Entities/EventToSend.cs @@ -27,12 +27,12 @@ public readonly struct EventToSend /// /// The name of the event. /// - public readonly string EventName; + public readonly string EventName { get; } /// /// The content of the event. /// - public readonly object EventContent; + public readonly object EventContent { get; } /// /// The target instance for the event. diff --git a/src/DurableTask.Core/Entities/IEntityOrchestrationService.cs b/src/DurableTask.Core/Entities/IEntityOrchestrationService.cs index 3944aea1e..8c63dca6b 100644 --- a/src/DurableTask.Core/Entities/IEntityOrchestrationService.cs +++ b/src/DurableTask.Core/Entities/IEntityOrchestrationService.cs @@ -24,8 +24,8 @@ public interface IEntityOrchestrationService /// /// The entity orchestration service. /// - /// The entity backend information object. - EntityBackendInformation GetEntityBackendInformation(); + /// An object containing properties of the entity backend. + EntityBackendProperties GetEntityBackendProperties(); /// /// Configures the orchestration service backend so entities and orchestrations are kept in two separate queues, and can be fetched separately. diff --git a/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs b/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs index dc6c27cf2..939a8b6ee 100644 --- a/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs +++ b/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs @@ -35,7 +35,7 @@ public sealed class TaskHubEntityClient readonly DataConverter messageDataConverter; readonly DataConverter stateDataConverter; readonly LogHelper logHelper; - readonly EntityBackendInformation backendInformation; + readonly EntityBackendProperties backendProperties; readonly IOrchestrationServiceQueryClient queryClient; readonly IOrchestrationServicePurgeClient purgeClient; @@ -46,7 +46,7 @@ public sealed class TaskHubEntityClient private void CheckEntitySupport(string name) { - if (this.backendInformation == null) + if (this.backendProperties == null) { throw new InvalidOperationException($"{nameof(TaskHubEntityClient)}.{name} is not supported because the chosen backend does not support entities."); } @@ -79,7 +79,7 @@ public TaskHubEntityClient(TaskHubClient client, DataConverter stateDataConverte this.messageDataConverter = client.DefaultConverter; this.stateDataConverter = stateDataConverter ?? client.DefaultConverter; this.logHelper = client.LogHelper; - this.backendInformation = (client.ServiceClient as IEntityOrchestrationService)?.GetEntityBackendInformation(); + this.backendProperties = (client.ServiceClient as IEntityOrchestrationService)?.GetEntityBackendProperties(); this.queryClient = client.ServiceClient as IOrchestrationServiceQueryClient; this.purgeClient = client.ServiceClient as IOrchestrationServicePurgeClient; } @@ -100,7 +100,7 @@ public async Task SignalEntityAsync(EntityId entityId, string operationName, obj if (scheduledTimeUtc.HasValue) { DateTime original = scheduledTimeUtc.Value.ToUniversalTime(); - DateTime capped = this.backendInformation.GetCappedScheduledTime(DateTime.UtcNow, original); + DateTime capped = this.backendProperties.GetCappedScheduledTime(DateTime.UtcNow, original); scheduledTime = (original, capped); } @@ -409,7 +409,7 @@ public class EntityStatus /// Removes empty entities from storage and releases orphaned locks. /// /// An entity is considered empty, and is removed, if it has no state, is not locked, and has - /// been idle for more than minutes. + /// been idle for more than minutes. /// Locks are considered orphaned, and are released, if the orchestration that holds them is not in state . This /// should not happen under normal circumstances, but can occur if the orchestration instance holding the lock /// exhibits replay nondeterminism failures, or if it is explicitly purged. @@ -456,8 +456,8 @@ public async Task CleanEntityStorageAsync(bool removeE if (removeEmptyEntities) { bool isEmptyEntity = !status.EntityExists && status.LockedBy == null && status.QueueSize == 0; - bool safeToRemoveWithoutBreakingMessageSorterLogic = (this.backendInformation.EntityMessageReorderWindow == TimeSpan.Zero) ? - true : (now - state.LastUpdatedTime > this.backendInformation.EntityMessageReorderWindow); + bool safeToRemoveWithoutBreakingMessageSorterLogic = (this.backendProperties.EntityMessageReorderWindow == TimeSpan.Zero) ? + true : (now - state.LastUpdatedTime > this.backendProperties.EntityMessageReorderWindow); if (isEmptyEntity && safeToRemoveWithoutBreakingMessageSorterLogic) { tasks.Add(DeleteIdleOrchestrationEntity(state)); diff --git a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs index fae85699e..fd516dc4e 100644 --- a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs +++ b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs @@ -311,7 +311,7 @@ internal void AdjustOutgoingMessage(string instanceId, RequestMessage requestMes requestMessage, instanceId, this.innerContext.CurrentUtcDateTime, - this.innerContext.EntityBackendInformation.EntityMessageReorderWindow); + this.innerContext.EntityBackendProperties.EntityMessageReorderWindow); eventName = EntityMessageEventNames.RequestMessageEventName; } diff --git a/src/DurableTask.Core/FailureDetails.cs b/src/DurableTask.Core/FailureDetails.cs index 6e929afec..55bce70d7 100644 --- a/src/DurableTask.Core/FailureDetails.cs +++ b/src/DurableTask.Core/FailureDetails.cs @@ -53,12 +53,12 @@ public FailureDetails(Exception e) /// /// Initializes a new instance of the class from an exception object and - /// an explicitly specified inner exception. + /// an explicitly specified inner exception. The specified inner exception replaces any inner exception already present. /// /// The exception used to generate the failure details. - /// The inner exception to use. - public FailureDetails(Exception e, Exception InnerException) - : this(e.GetType().FullName, GetErrorMessage(e), e.StackTrace, FromException(InnerException), false) + /// The inner exception to use. + public FailureDetails(Exception e, Exception innerException) + : this(e.GetType().FullName, GetErrorMessage(e), e.StackTrace, FromException(innerException), false) { } diff --git a/src/DurableTask.Core/Logging/LogEvents.cs b/src/DurableTask.Core/Logging/LogEvents.cs index 258172173..3dc1317c7 100644 --- a/src/DurableTask.Core/Logging/LogEvents.cs +++ b/src/DurableTask.Core/Logging/LogEvents.cs @@ -22,6 +22,8 @@ namespace DurableTask.Core.Logging using DurableTask.Core.Entities.OperationFormat; using DurableTask.Core.History; using Microsoft.Extensions.Logging; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; /// /// This class defines all log events supported by DurableTask.Core. @@ -1284,8 +1286,17 @@ public EntityLockAcquired(string entityId, Core.Entities.EventFormat.RequestMess this.InstanceId = message.ParentInstanceId; this.ExecutionId = message.ParentExecutionId; this.CriticalSectionId = message.Id; - this.LockSet = message.LockSet; this.Position = message.Position; + + if (message.LockSet != null) + { + var jArray = new JArray(); + foreach (var id in message.LockSet) + { + jArray.Add((JToken) id.ToString()); + } + this.LockSet = jArray.ToString(Formatting.None); + } } [StructuredLogField] @@ -1301,7 +1312,7 @@ public EntityLockAcquired(string entityId, Core.Entities.EventFormat.RequestMess public Guid CriticalSectionId { get; set; } [StructuredLogField] - public EntityId[] LockSet { get; set; } + public string LockSet { get; set; } [StructuredLogField] public int Position { get; set; } @@ -1321,7 +1332,7 @@ void IEventSourceEvent.WriteEventSource() => this.InstanceId ?? string.Empty, this.ExecutionId ?? string.Empty, this.CriticalSectionId, - this.LockSet.Length, + this.LockSet ?? string.Empty, this.Position, Utils.AppName, Utils.PackageVersion); diff --git a/src/DurableTask.Core/Logging/LogHelper.cs b/src/DurableTask.Core/Logging/LogHelper.cs index b6ff3534f..96e74ee60 100644 --- a/src/DurableTask.Core/Logging/LogHelper.cs +++ b/src/DurableTask.Core/Logging/LogHelper.cs @@ -16,7 +16,6 @@ namespace DurableTask.Core.Logging using System; using System.Collections.Generic; using System.Text; - using System.Web; using DurableTask.Core.Command; using DurableTask.Core.Common; using DurableTask.Core.Entities; diff --git a/src/DurableTask.Core/Logging/StructuredEventSource.cs b/src/DurableTask.Core/Logging/StructuredEventSource.cs index 26dc3178c..b6ec19a32 100644 --- a/src/DurableTask.Core/Logging/StructuredEventSource.cs +++ b/src/DurableTask.Core/Logging/StructuredEventSource.cs @@ -678,7 +678,7 @@ internal void EntityLockAcquired( string InstanceId, string ExecutionId, Guid CriticalSectionId, - int LockSetSize, + string LockSet, int Position, string AppName, string ExtensionVersion) @@ -692,7 +692,7 @@ internal void EntityLockAcquired( InstanceId, ExecutionId, CriticalSectionId, - LockSetSize, + LockSet, Position, AppName, ExtensionVersion); diff --git a/src/DurableTask.Core/OrchestrationContext.cs b/src/DurableTask.Core/OrchestrationContext.cs index 4aab9b155..dffc84df8 100644 --- a/src/DurableTask.Core/OrchestrationContext.cs +++ b/src/DurableTask.Core/OrchestrationContext.cs @@ -73,7 +73,7 @@ public abstract class OrchestrationContext /// /// Information about backend entity support, or null if the configured backend does not support entities. /// - internal EntityBackendInformation EntityBackendInformation { get; set; } + internal EntityBackendProperties EntityBackendProperties { get; set; } /// /// Create a proxy client class to schedule remote TaskActivities via a strongly typed interface. diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index 1baf86851..fadb39384 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -39,7 +39,7 @@ public class TaskEntityDispatcher readonly IEntityOrchestrationService entityOrchestrationService; readonly WorkItemDispatcher dispatcher; readonly DispatchMiddlewarePipeline dispatchPipeline; - readonly EntityBackendInformation entityBackendInformation; + readonly EntityBackendProperties entityBackendProperties; readonly LogHelper logHelper; readonly ErrorPropagationMode errorPropagationMode; readonly TaskOrchestrationDispatcher.NonBlockingCountdownLock concurrentSessionLock; @@ -58,7 +58,7 @@ internal TaskEntityDispatcher( this.logHelper = logHelper ?? throw new ArgumentNullException(nameof(logHelper)); this.errorPropagationMode = errorPropagationMode; this.entityOrchestrationService = (orchestrationService as IEntityOrchestrationService)!; - this.entityBackendInformation = entityOrchestrationService.GetEntityBackendInformation(); + this.entityBackendProperties = entityOrchestrationService.GetEntityBackendProperties(); this.dispatcher = new WorkItemDispatcher( "TaskEntityDispatcher", @@ -71,7 +71,7 @@ internal TaskEntityDispatcher( SafeReleaseWorkItem = orchestrationService.ReleaseTaskOrchestrationWorkItemAsync, AbortWorkItem = orchestrationService.AbandonTaskOrchestrationWorkItemAsync, DispatcherCount = orchestrationService.TaskOrchestrationDispatcherCount, - MaxConcurrentWorkItems = orchestrationService.MaxConcurrentTaskOrchestrationWorkItems, + MaxConcurrentWorkItems = this.entityBackendProperties.MaxConcurrentTaskEntityWorkItems, LogHelper = logHelper, }; @@ -84,7 +84,7 @@ internal TaskEntityDispatcher( /// /// The entity options configured, or null if the backend does not support entities. /// - public EntityBackendInformation EntityBackendInformation => this.entityBackendInformation; + public EntityBackendProperties EntityBackendProperties => this.entityBackendProperties; /// /// Starts the dispatcher to start getting and processing orchestration events @@ -443,7 +443,7 @@ void ProcessLockRequest(WorkItemEffects effects, SchedulerState schedulerState, string SerializeSchedulerStateForNextExecution(SchedulerState schedulerState) { - if (this.entityBackendInformation.SupportsImplicitEntityDeletion && schedulerState.IsEmpty && !schedulerState.Suspended) + if (this.entityBackendProperties.SupportsImplicitEntityDeletion && schedulerState.IsEmpty && !schedulerState.Suspended) { // this entity scheduler is idle and the entity is deleted, so the instance and history can be removed from storage // we convey this to the durability provider by issuing a continue-as-new with null input @@ -525,7 +525,7 @@ void DetermineWork(OrchestrationRuntimeState runtimeState, out SchedulerState sc else { // run this through the message sorter to help with reordering and duplicate filtering - deliverNow = schedulerState.MessageSorter.ReceiveInOrder(requestMessage, this.entityBackendInformation.EntityMessageReorderWindow); + deliverNow = schedulerState.MessageSorter.ReceiveInOrder(requestMessage, this.entityBackendProperties.EntityMessageReorderWindow); } foreach (var message in deliverNow) @@ -588,7 +588,7 @@ void DetermineWork(OrchestrationRuntimeState runtimeState, out SchedulerState sc // stopping at lock requests or when the maximum batch size is reached while (schedulerState.MayDequeue()) { - if (batch.OperationCount == this.entityBackendInformation.MaxEntityOperationBatchSize) + if (batch.OperationCount == this.entityBackendProperties.MaxEntityOperationBatchSize) { // we have reached the maximum batch size already // insert a delay after this batch to ensure write back @@ -728,20 +728,20 @@ void SendSignalMessage(WorkItemEffects effects, SchedulerState schedulerState, S if (action.ScheduledTime.HasValue) { DateTime original = action.ScheduledTime.Value; - DateTime capped = this.entityBackendInformation.GetCappedScheduledTime(DateTime.UtcNow, original); + DateTime capped = this.entityBackendProperties.GetCappedScheduledTime(DateTime.UtcNow, original); eventName = EntityMessageEventNames.ScheduledRequestMessageEventName(capped); } else { eventName = EntityMessageEventNames.RequestMessageEventName; - schedulerState.MessageSorter.LabelOutgoingMessage(message, action.InstanceId, DateTime.UtcNow, this.entityBackendInformation.EntityMessageReorderWindow); + schedulerState.MessageSorter.LabelOutgoingMessage(message, action.InstanceId, DateTime.UtcNow, this.entityBackendProperties.EntityMessageReorderWindow); } this.ProcessSendEventMessage(effects, destination, eventName, message); } void SendLockRequestMessage(WorkItemEffects effects, SchedulerState schedulerState, OrchestrationInstance target, RequestMessage message) { - schedulerState.MessageSorter.LabelOutgoingMessage(message, target.InstanceId, DateTime.UtcNow, this.entityBackendInformation.EntityMessageReorderWindow); + schedulerState.MessageSorter.LabelOutgoingMessage(message, target.InstanceId, DateTime.UtcNow, this.entityBackendProperties.EntityMessageReorderWindow); this.ProcessSendEventMessage(effects, target, EntityMessageEventNames.RequestMessageEventName, message); } @@ -877,7 +877,7 @@ await this.dispatchPipeline.RunAsync(dispatchContext, async _ => var options = new EntityExecutionOptions() { - EntityBackendInformation = this.entityBackendInformation, + EntityBackendProperties = this.entityBackendProperties, ErrorPropagationMode = this.errorPropagationMode, }; diff --git a/src/DurableTask.Core/TaskHubWorker.cs b/src/DurableTask.Core/TaskHubWorker.cs index 2d029006b..c7892a051 100644 --- a/src/DurableTask.Core/TaskHubWorker.cs +++ b/src/DurableTask.Core/TaskHubWorker.cs @@ -98,7 +98,8 @@ public TaskHubWorker(IOrchestrationService orchestrationService, ILoggerFactory /// Reference the orchestration service implementation /// NameVersionObjectManager for Orchestrations /// NameVersionObjectManager for Activities - /// The for entities + /// The NameVersionObjectManager for entities. The version is the entity key. + /// public TaskHubWorker( IOrchestrationService orchestrationService, INameVersionObjectManager orchestrationObjectManager, @@ -120,7 +121,7 @@ public TaskHubWorker( /// The orchestration service implementation /// The for orchestrations /// The for activities - /// The for entities + /// The for entities. The version is the entity key. /// The to use for logging public TaskHubWorker( IOrchestrationService orchestrationService, @@ -134,8 +135,13 @@ public TaskHubWorker( this.entityManager = entityObjectManager ?? throw new ArgumentException("entityObjectManager"); this.orchestrationService = orchestrationService ?? throw new ArgumentException("orchestrationService"); this.logHelper = new LogHelper(loggerFactory?.CreateLogger("DurableTask.Core")); - this.entityOrchestrationService = orchestrationService as IEntityOrchestrationService; - this.entityOrchestrationService?.ProcessEntitiesSeparately(); // lets the backend know that this worker wants them separately + + // if the backend supports entities, configure it to collect entity work items in a separate queue. + if (orchestrationService is IEntityOrchestrationService entityOrchestrationService) + { + this.entityOrchestrationService = entityOrchestrationService; + entityOrchestrationService.ProcessEntitiesSeparately(); + } } /// diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index a696e4fb7..62c027816 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -56,7 +56,7 @@ internal OrchestrationEntityContext EntityContext public TaskOrchestrationContext( OrchestrationInstance orchestrationInstance, TaskScheduler taskScheduler, - EntityBackendInformation entityBackendInformation = null, + EntityBackendProperties entityBackendProperties = null, ErrorPropagationMode errorPropagationMode = ErrorPropagationMode.SerializeExceptions) { Utils.UnusedParameter(taskScheduler); @@ -68,7 +68,7 @@ public TaskOrchestrationContext( this.ErrorDataConverter = JsonDataConverter.Default; OrchestrationInstance = orchestrationInstance; IsReplaying = false; - this.EntityBackendInformation = entityBackendInformation; + this.EntityBackendProperties = entityBackendProperties; ErrorPropagationMode = errorPropagationMode; this.eventsWhileSuspended = new Queue(); } @@ -81,7 +81,7 @@ public TaskOrchestrationContext( private void CheckEntitySupport() { - if (this.EntityBackendInformation == null) + if (this.EntityBackendProperties == null) { throw new InvalidOperationException("Entities are not supported by the configured persistence backend."); } @@ -242,7 +242,7 @@ public override void SignalEntity(EntityId entityId, string operationName, objec public override void SignalEntity(EntityId entityId, DateTime scheduledTimeUtc, string operationName, object input = null) { this.CheckEntitySupport(); - DateTime cappedTime = EntityBackendInformation.GetCappedScheduledTime(this.CurrentUtcDateTime, scheduledTimeUtc); + DateTime cappedTime = EntityBackendProperties.GetCappedScheduledTime(this.CurrentUtcDateTime, scheduledTimeUtc); this.EntityOperationCore(entityId, operationName, input, true, (scheduledTimeUtc, cappedTime)); } diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 0133287dd..f4cf20dbc 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -45,7 +45,7 @@ public class TaskOrchestrationDispatcher ErrorPropagationMode errorPropagationMode; readonly NonBlockingCountdownLock concurrentSessionLock; readonly IEntityOrchestrationService? entityOrchestrationService; - readonly EntityBackendInformation? entityBackendInformation; + readonly EntityBackendProperties? entityBackendProperties; internal TaskOrchestrationDispatcher( IOrchestrationService orchestrationService, @@ -60,7 +60,7 @@ internal TaskOrchestrationDispatcher( this.logHelper = logHelper ?? throw new ArgumentNullException(nameof(logHelper)); this.errorPropagationMode = errorPropagationMode; this.entityOrchestrationService = orchestrationService as IEntityOrchestrationService; - this.entityBackendInformation = this.entityOrchestrationService?.GetEntityBackendInformation(); + this.entityBackendProperties = this.entityOrchestrationService?.GetEntityBackendProperties(); this.dispatcher = new WorkItemDispatcher( "TaskOrchestrationDispatcher", @@ -689,7 +689,7 @@ await this.dispatchPipeline.RunAsync(dispatchContext, _ => runtimeState, taskOrchestration, this.orchestrationService.EventBehaviourForContinueAsNew, - this.entityBackendInformation, + this.entityBackendProperties, this.errorPropagationMode); ; OrchestratorExecutionResult resultFromOrchestrator = executor.Execute(); dispatchContext.SetProperty(resultFromOrchestrator); diff --git a/src/DurableTask.Core/TaskOrchestrationExecutor.cs b/src/DurableTask.Core/TaskOrchestrationExecutor.cs index 2036cb105..c9fc3f026 100644 --- a/src/DurableTask.Core/TaskOrchestrationExecutor.cs +++ b/src/DurableTask.Core/TaskOrchestrationExecutor.cs @@ -44,20 +44,20 @@ public class TaskOrchestrationExecutor /// /// /// - /// + /// /// public TaskOrchestrationExecutor( OrchestrationRuntimeState orchestrationRuntimeState, TaskOrchestration taskOrchestration, BehaviorOnContinueAsNew eventBehaviourForContinueAsNew, - EntityBackendInformation? entityBackendInformation, + EntityBackendProperties? entityBackendProperties, ErrorPropagationMode errorPropagationMode = ErrorPropagationMode.SerializeExceptions) { this.decisionScheduler = new SynchronousTaskScheduler(); this.context = new TaskOrchestrationContext( orchestrationRuntimeState.OrchestrationInstance, this.decisionScheduler, - entityBackendInformation, + entityBackendProperties, errorPropagationMode ); this.orchestrationRuntimeState = orchestrationRuntimeState; @@ -78,7 +78,7 @@ public TaskOrchestrationExecutor( TaskOrchestration taskOrchestration, BehaviorOnContinueAsNew eventBehaviourForContinueAsNew, ErrorPropagationMode errorPropagationMode = ErrorPropagationMode.SerializeExceptions) - : this(orchestrationRuntimeState, taskOrchestration, eventBehaviourForContinueAsNew, new EntityBackendInformation(), errorPropagationMode) + : this(orchestrationRuntimeState, taskOrchestration, eventBehaviourForContinueAsNew, new EntityBackendProperties(), errorPropagationMode) { } From 69b2819f5fb0093bfe08a4de13c53895dbde5e80 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Fri, 14 Apr 2023 10:27:36 -0700 Subject: [PATCH 8/8] address PR feedback --- src/DurableTask.Core/Entities/EntityId.cs | 4 ++-- src/DurableTask.Core/Logging/LogEvents.cs | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/DurableTask.Core/Entities/EntityId.cs b/src/DurableTask.Core/Entities/EntityId.cs index a7de3480a..9c63896ee 100644 --- a/src/DurableTask.Core/Entities/EntityId.cs +++ b/src/DurableTask.Core/Entities/EntityId.cs @@ -68,9 +68,9 @@ internal static string GetSchedulerIdPrefixFromEntityName(string entityName) /// the corresponding entity ID. public static EntityId FromString(string instanceId) { - if (instanceId == null) + if (string.IsNullOrEmpty(instanceId)) { - throw new ArgumentNullException(nameof(instanceId)); + throw new ArgumentException(nameof(instanceId)); } var pos = instanceId.IndexOf('@', 1); if ( pos <= 0 || instanceId[0] != '@') diff --git a/src/DurableTask.Core/Logging/LogEvents.cs b/src/DurableTask.Core/Logging/LogEvents.cs index 3dc1317c7..184a97b87 100644 --- a/src/DurableTask.Core/Logging/LogEvents.cs +++ b/src/DurableTask.Core/Logging/LogEvents.cs @@ -1290,12 +1290,7 @@ public EntityLockAcquired(string entityId, Core.Entities.EventFormat.RequestMess if (message.LockSet != null) { - var jArray = new JArray(); - foreach (var id in message.LockSet) - { - jArray.Add((JToken) id.ToString()); - } - this.LockSet = jArray.ToString(Formatting.None); + this.LockSet = string.Join(",", message.LockSet.Select(id => id.ToString())); } }