diff --git a/Test/DurableTask.AzureStorage.Tests/TestEntityClient.cs b/Test/DurableTask.AzureStorage.Tests/TestEntityClient.cs new file mode 100644 index 000000000..46627a792 --- /dev/null +++ b/Test/DurableTask.AzureStorage.Tests/TestEntityClient.cs @@ -0,0 +1,116 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.AzureStorage.Tests +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Threading; + using System.Threading.Tasks; + using DurableTask.AzureStorage.Tracking; + using DurableTask.Core; + using DurableTask.Core.Entities; + using DurableTask.Core.History; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json; + + class TestEntityClient + { + readonly EntityId entityId; + + public TestEntityClient( + TaskHubEntityClient client, + EntityId entityId) + { + this.InnerClient = client; + this.entityId = entityId; + } + + public TaskHubEntityClient InnerClient { get; } + + public async Task SignalEntity( + string operationName, + object operationContent = null) + { + Trace.TraceInformation($"Signaling entity {this.entityId} with operation named {operationName}."); + await this.InnerClient.SignalEntityAsync(this.entityId, operationName, operationContent); + } + + public async Task SignalEntity( + DateTime startTimeUtc, + string operationName, + object operationContent = null) + { + Trace.TraceInformation($"Signaling entity {this.entityId} with operation named {operationName}."); + await this.InnerClient.SignalEntityAsync(this.entityId, operationName, operationContent, startTimeUtc); + } + + public async Task WaitForEntityState( + TimeSpan? timeout = null, + Func describeWhatWeAreWaitingFor = null) + { + if (timeout == null) + { + timeout = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(30); + } + + Stopwatch sw = Stopwatch.StartNew(); + + TaskHubEntityClient.StateResponse response; + + do + { + response = await this.InnerClient.ReadEntityStateAsync(this.entityId); + if (response.EntityExists) + { + if (describeWhatWeAreWaitingFor == null) + { + break; + } + else + { + var waitForResult = describeWhatWeAreWaitingFor(response.EntityState); + + if (string.IsNullOrEmpty(waitForResult)) + { + break; + } + else + { + Trace.TraceInformation($"Waiting for {this.entityId} : {waitForResult}"); + } + } + } + else + { + Trace.TraceInformation($"Waiting for {this.entityId} to have state."); + } + + await Task.Delay(TimeSpan.FromMilliseconds(100)); + } + while (sw.Elapsed < timeout); + + if (response.EntityExists) + { + string serializedState = JsonConvert.SerializeObject(response.EntityState); + Trace.TraceInformation($"Found state: {serializedState}"); + return response.EntityState; + } + else + { + throw new TimeoutException($"Durable entity '{this.entityId}' still doesn't have any state!"); + } + } + } +} diff --git a/Test/DurableTask.Core.Tests/MessageSorterTests.cs b/Test/DurableTask.Core.Tests/MessageSorterTests.cs new file mode 100644 index 000000000..f68268bb5 --- /dev/null +++ b/Test/DurableTask.Core.Tests/MessageSorterTests.cs @@ -0,0 +1,340 @@ +// --------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// --------------------------------------------------------------- + +namespace DurableTask.Core.Tests +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Threading.Tasks; + using DurableTask.Core.Entities.EventFormat; + using DurableTask.Core.Entities.StateFormat; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class MessageSorterTests + { + private static readonly TimeSpan ReorderWindow = TimeSpan.FromMinutes(30); + + [TestMethod] + public void SimpleInOrder() + { + var senderId = "A"; + var receiverId = "B"; + + var senderSorter = new MessageSorter(); + + var message1 = Send(senderId, receiverId, "1", senderSorter, DateTime.UtcNow); + var message2 = Send(senderId, receiverId, "2", senderSorter, DateTime.UtcNow); + var message3 = Send(senderId, receiverId, "3", senderSorter, DateTime.UtcNow); + + List batch; + MessageSorter receiverSorter = new MessageSorter(); + + // delivering the sequence in order produces 1 message each time + batch = receiverSorter.ReceiveInOrder(message1, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("1"); + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("2"); + batch = receiverSorter.ReceiveInOrder(message3, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("3"); + + Assert.AreEqual(0, receiverSorter.NumberBufferedRequests); + } + + [TestMethod] + public void WackySystemClock() + { + var senderId = "A"; + var receiverId = "B"; + + var senderSorter = new MessageSorter(); + + // simulate system clock that goes backwards - mechanism should still guarantee monotonicitty + var message1 = Send(senderId, receiverId, "1", senderSorter, DateTime.UtcNow); + var message2 = Send(senderId, receiverId, "2", senderSorter, DateTime.UtcNow - TimeSpan.FromSeconds(1)); + var message3 = Send(senderId, receiverId, "3", senderSorter, DateTime.UtcNow - TimeSpan.FromSeconds(2)); + + List batch; + MessageSorter receiverSorter = new MessageSorter(); + + // delivering the sequence in order produces 1 message each time + batch = receiverSorter.ReceiveInOrder(message1, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("1"); + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("2"); + batch = receiverSorter.ReceiveInOrder(message3, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("3"); + + Assert.AreEqual(0, receiverSorter.NumberBufferedRequests); + } + + [TestMethod] + public void DelayedElement() + { + var senderId = "A"; + var receiverId = "B"; + + var senderSorter = new MessageSorter(); + + var message1 = Send(senderId, receiverId, "1", senderSorter, DateTime.UtcNow); + var message2 = Send(senderId, receiverId, "2", senderSorter, DateTime.UtcNow); + var message3 = Send(senderId, receiverId, "3", senderSorter, DateTime.UtcNow); + + List batch; + MessageSorter receiverSorter; + + // delivering first message last delays all messages until getting the first one + receiverSorter = new MessageSorter(); + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Empty(batch); + batch = receiverSorter.ReceiveInOrder(message3, ReorderWindow).ToList(); + Assert.That.Empty(batch); + batch = receiverSorter.ReceiveInOrder(message1, ReorderWindow).ToList(); + Assert.That.Collection( + batch, + first => first.Input.Equals("1"), + second => second.Input.Equals("2"), + third => third.Input.Equals("3")); + + Assert.AreEqual(0, receiverSorter.NumberBufferedRequests); + } + + [TestMethod] + public void NoFilteringOrSortingPastReorderWindow() + { + var senderId = "A"; + var receiverId = "B"; + + var senderSorter = new MessageSorter(); + var now = DateTime.UtcNow; + + // last message is sent after an interval exceeding the reorder window + var message1 = Send(senderId, receiverId, "1", senderSorter, now); + var message2 = Send(senderId, receiverId, "2", senderSorter, now + TimeSpan.FromTicks(1)); + var message3 = Send(senderId, receiverId, "3", senderSorter, now + TimeSpan.FromTicks(2) + ReorderWindow); + + List batch; + MessageSorter receiverSorter = new MessageSorter(); + + // delivering the sequence in order produces 1 message each time + receiverSorter = new MessageSorter(); + batch = receiverSorter.ReceiveInOrder(message1, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("1"); + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("2"); + batch = receiverSorter.ReceiveInOrder(message3, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("3"); + + // duplicates are not filtered or sorted, but simply passed through, because we are past the reorder window + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("2"); + batch = receiverSorter.ReceiveInOrder(message1, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("1"); + + Assert.AreEqual(0, receiverSorter.NumberBufferedRequests); + } + + [TestMethod] + public void DuplicatedElements() + { + var senderId = "A"; + var receiverId = "B"; + + var senderSorter = new MessageSorter(); + + var message1 = Send(senderId, receiverId, "1", senderSorter, DateTime.UtcNow); + var message2 = Send(senderId, receiverId, "2", senderSorter, DateTime.UtcNow); + var message3 = Send(senderId, receiverId, "3", senderSorter, DateTime.UtcNow); + + List batch; + MessageSorter receiverSorter; + + // delivering first message last delays all messages until getting the first one + receiverSorter = new MessageSorter(); + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Empty(batch); + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Empty(batch); + batch = receiverSorter.ReceiveInOrder(message1, ReorderWindow).ToList(); + Assert.That.Collection( + batch, + first => first.Input.Equals("1"), + second => second.Input.Equals("2")); + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Empty(batch); + batch = receiverSorter.ReceiveInOrder(message1, ReorderWindow).ToList(); + Assert.That.Empty(batch); + batch = receiverSorter.ReceiveInOrder(message3, ReorderWindow).ToList(); + Assert.That.Single(batch).Input.Equals("3"); + batch = receiverSorter.ReceiveInOrder(message3, ReorderWindow).ToList(); + Assert.That.Empty(batch); + batch = receiverSorter.ReceiveInOrder(message2, ReorderWindow).ToList(); + Assert.That.Empty(batch); + batch = receiverSorter.ReceiveInOrder(message1, ReorderWindow).ToList(); + Assert.That.Empty(batch); + + Assert.AreEqual(0, receiverSorter.NumberBufferedRequests); + } + + [TestMethod] + public void RandomShuffleAndDuplication() + { + var senderId = "A"; + var receiverId = "B"; + + var senderSorter = new MessageSorter(); + var receiverSorter = new MessageSorter(); + + var messageCount = 100; + var duplicateCount = 100; + + // create a ordered sequence of messages + var messages = new List(); + for (int i = 0; i < messageCount; i++) + { + messages.Add(Send(senderId, receiverId, i.ToString(), senderSorter, DateTime.UtcNow)); + } + + // add some random duplicates + var random = new Random(0); + for (int i = 0; i < duplicateCount; i++) + { + messages.Add(messages[random.Next(messageCount)]); + } + + // shuffle the messages + Shuffle(messages, random); + + // deliver all the messages + var deliveredMessages = new List(); + + foreach (var msg in messages) + { + foreach (var deliveredMessage in receiverSorter.ReceiveInOrder(msg, ReorderWindow)) + { + deliveredMessages.Add(deliveredMessage); + } + } + + // check that the delivered messages are the original sequence + Assert.AreEqual(messageCount, deliveredMessages.Count()); + for (int i = 0; i < messageCount; i++) + { + Assert.AreEqual(i.ToString(), deliveredMessages[i].Input); + } + + Assert.AreEqual(0, receiverSorter.NumberBufferedRequests); + } + + /// + /// Tests that if messages get reordered beyond the supported reorder window, + /// we still deliver them all but they may now be out of order. + /// + [TestMethod] + public void RandomCollection() + { + var senderId = "A"; + var receiverId = "B"; + + var senderSorter = new MessageSorter(); + var receiverSorter = new MessageSorter(); + + var messageCount = 100; + + var random = new Random(0); + var now = DateTime.UtcNow; + + // create a ordered sequence of messages + var messages = new List(); + for (int i = 0; i < messageCount; i++) + { + messages.Add(Send(senderId, receiverId, i.ToString(), senderSorter, now + TimeSpan.FromSeconds(random.Next(5)), TimeSpan.FromSeconds(10))); + } + + // shuffle the messages + Shuffle(messages, random); + + // add a final message + messages.Add(Send(senderId, receiverId, (messageCount + 1).ToString(), senderSorter, now + TimeSpan.FromSeconds(1000), TimeSpan.FromSeconds(10))); + + // deliver all the messages + var deliveredMessages = new List(); + + for (int i = 0; i < messageCount; i++) + { + foreach (var deliveredMessage in receiverSorter.ReceiveInOrder(messages[i], TimeSpan.FromSeconds(10))) + { + deliveredMessages.Add(deliveredMessage); + } + + Assert.AreEqual(i + 1, deliveredMessages.Count + receiverSorter.NumberBufferedRequests); + } + + // receive the final messages + foreach (var deliveredMessage in receiverSorter.ReceiveInOrder(messages[messageCount], TimeSpan.FromSeconds(10))) + { + deliveredMessages.Add(deliveredMessage); + } + + // check that all messages were delivered + Assert.AreEqual(messageCount + 1, deliveredMessages.Count()); + + Assert.AreEqual(0, receiverSorter.NumberBufferedRequests); + } + + private static RequestMessage Send(string senderId, string receiverId, string input, MessageSorter sorter, DateTime now, TimeSpan? reorderWindow = null) + { + var msg = new RequestMessage() + { + Id = Guid.NewGuid(), + ParentInstanceId = senderId, + Input = input, + }; + sorter.LabelOutgoingMessage(msg, receiverId, now, reorderWindow.HasValue ? reorderWindow.Value : ReorderWindow); + return msg; + } + + private static void Shuffle(IList list, Random random) + { + int n = list.Count; + while (n > 1) + { + n--; + int k = random.Next(n + 1); + T value = list[k]; + list[k] = list[n]; + list[n] = value; + } + } + } + + internal static class AssertExtensions + { + public static void Empty(this Assert assert, IEnumerable collection) + { + Assert.AreEqual(0, collection.Count()); + } + + public static T Single(this Assert assert, IEnumerable collection) + { + var e = collection.GetEnumerator(); + Assert.IsTrue(e.MoveNext()); + T element = e.Current; + Assert.IsFalse(e.MoveNext()); + return element; + } + + public static void Collection(this Assert assert, IEnumerable collection, params Action[] elementInspectors) + { + var list = collection.ToList(); + Assert.AreEqual(elementInspectors.Length, list.Count); + for(int i = 0; i < elementInspectors.Length; i++) + { + elementInspectors[i](list[i]); + } + } + } +} \ No newline at end of file diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs index 3b1993d4c..5e9a04cb5 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs @@ -28,6 +28,7 @@ namespace DurableTask.AzureStorage using DurableTask.AzureStorage.Storage; using DurableTask.AzureStorage.Tracking; using DurableTask.Core; + using DurableTask.Core.Entities; using DurableTask.Core.Exceptions; using DurableTask.Core.History; using DurableTask.Core.Query; @@ -42,7 +43,8 @@ public sealed class AzureStorageOrchestrationService : IOrchestrationServiceClient, IDisposable, IOrchestrationServiceQueryClient, - IOrchestrationServicePurgeClient + IOrchestrationServicePurgeClient, + IEntityOrchestrationService { static readonly HistoryEvent[] EmptyHistoryEventList = new HistoryEvent[0]; @@ -268,6 +270,46 @@ public BehaviorOnContinueAsNew EventBehaviourForContinueAsNew /// public int TaskOrchestrationDispatcherCount { get; } = 1; + #region IEntityOrchestrationService + + EntityBackendProperties IEntityOrchestrationService.GetEntityBackendProperties() + => new EntityBackendProperties() + { + EntityMessageReorderWindow = TimeSpan.FromMinutes(this.settings.EntityMessageReorderWindowInMinutes), + MaxEntityOperationBatchSize = this.settings.MaxEntityOperationBatchSize, + SupportsImplicitEntityDeletion = false, // not supported by this backend + MaximumSignalDelayTime = TimeSpan.FromDays(6), + }; + + void IEntityOrchestrationService.ProcessEntitiesSeparately() + { + this.orchestrationSessionManager.ProcessEntitiesSeparately = true; + } + + Task IEntityOrchestrationService.LockNextOrchestrationWorkItemAsync( + TimeSpan receiveTimeout, + CancellationToken cancellationToken) + { + if (!orchestrationSessionManager.ProcessEntitiesSeparately) + { + throw new InvalidOperationException("backend was not configured for separate entity processing"); + } + return this.LockNextTaskOrchestrationWorkItemAsync(false, cancellationToken); + } + + Task IEntityOrchestrationService.LockNextEntityWorkItemAsync( + TimeSpan receiveTimeout, + CancellationToken cancellationToken) + { + if (!orchestrationSessionManager.ProcessEntitiesSeparately) + { + throw new InvalidOperationException("backend was not configured for separate entity processing"); + } + return this.LockNextTaskOrchestrationWorkItemAsync(true, cancellationToken); + } + + #endregion + #region Management Operations (Create/Delete/Start/Stop) /// /// Deletes and creates the neccesary Azure Storage resources for the orchestration service. @@ -557,9 +599,14 @@ static TaskHubInfo GetTaskHubInfo(string taskHub, int partitionCount) #region Orchestration Work Item Methods /// - public async Task LockNextTaskOrchestrationWorkItemAsync( + public Task LockNextTaskOrchestrationWorkItemAsync( TimeSpan receiveTimeout, CancellationToken cancellationToken) + { + return LockNextTaskOrchestrationWorkItemAsync(false, cancellationToken); + } + + async Task LockNextTaskOrchestrationWorkItemAsync(bool entitiesOnly, CancellationToken cancellationToken) { Guid traceActivityId = StartNewLogicalTraceScope(useExisting: true); @@ -573,7 +620,7 @@ public async Task LockNextTaskOrchestrationWorkItemAs try { // This call will block until the next session is ready - session = await this.orchestrationSessionManager.GetNextSessionAsync(linkedCts.Token); + session = await this.orchestrationSessionManager.GetNextSessionAsync(entitiesOnly, linkedCts.Token); if (session == null) { return null; @@ -1912,7 +1959,7 @@ public Task DownloadBlobAsync(string blobUri) // be supported: https://github.com/Azure/azure-functions-durable-extension/issues/1 async Task GetControlQueueAsync(string instanceId) { - uint partitionIndex = Fnv1aHashHelper.ComputeHash(instanceId) % (uint)this.settings.PartitionCount; + uint partitionIndex = DurableTask.Core.Common.Fnv1aHashHelper.ComputeHash(instanceId) % (uint)this.settings.PartitionCount; string queueName = GetControlQueueName(this.settings.TaskHubName, (int)partitionIndex); ControlQueue cachedQueue; diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs index c84621951..eee9c1947 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs @@ -14,14 +14,14 @@ namespace DurableTask.AzureStorage { using System; + using System.Runtime.Serialization; + using System.Threading.Tasks; using DurableTask.AzureStorage.Partitioning; using DurableTask.AzureStorage.Logging; using DurableTask.Core; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Queue; using Microsoft.WindowsAzure.Storage.Table; - using System.Runtime.Serialization; - using System.Threading.Tasks; /// /// Settings that impact the runtime behavior of the . @@ -111,6 +111,12 @@ public class AzureStorageOrchestrationServiceSettings /// public int MaxConcurrentTaskOrchestrationWorkItems { get; set; } = 100; + /// + /// Gets or sets the maximum number of entity operation batches that can be processed concurrently on a single node. + /// The default value is 100. + /// + public int MaxConcurrentTaskEntityWorkItems { get; set; } = 100; + /// /// Gets or sets the maximum number of concurrent storage operations that can be executed in the context /// of a single orchestration instance. @@ -274,5 +280,24 @@ internal LogHelper Logger return this.logHelper; } } + + /// + /// Gets or sets the limit on the number of entity operations that should be processed as a single batch. + /// A null value indicates that no particular limit should be enforced. + /// + /// + /// Limiting the batch size can help to avoid timeouts in execution environments that impose time limitations on work items. + /// If set to 1, batching is disabled, and each operation executes as a separate work item. + /// + /// + /// A positive integer, or null. + /// + public int? MaxEntityOperationBatchSize { get; set; } = null; + + /// + /// Gets or sets the time window within which entity messages get deduplicated and reordered. + /// If set to zero, there is no sorting or deduplication, and all messages are just passed through. + /// + public int EntityMessageReorderWindowInMinutes { get; set; } = 30; } } diff --git a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs index e18853031..333052b9a 100644 --- a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs +++ b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs @@ -32,7 +32,8 @@ class OrchestrationSessionManager : IDisposable readonly Dictionary activeOrchestrationSessions = new Dictionary(StringComparer.OrdinalIgnoreCase); readonly ConcurrentDictionary ownedControlQueues = new ConcurrentDictionary(); readonly LinkedList pendingOrchestrationMessageBatches = new LinkedList(); - readonly AsyncQueue> readyForProcessingQueue = new AsyncQueue>(); + readonly AsyncQueue> orchestrationsReadyForProcessingQueue = new AsyncQueue>(); + readonly AsyncQueue> entitiesReadyForProcessingQueue = new AsyncQueue>(); readonly object messageAndSessionLock = new object(); readonly string storageAccountName; @@ -57,6 +58,14 @@ public OrchestrationSessionManager( internal IEnumerable Queues => this.ownedControlQueues.Values; + /// + /// Recent versions of DurableTask.Core can be configured to use a separate pipeline for processing entity work items, + /// while older versions use a single pipeline for both orchestration and entity work items. To support both scenarios, + /// this property can be modified prior to starting the orchestration service. If set to true, the work items that are ready for + /// processing are stored in and , respectively. + /// + internal bool ProcessEntitiesSeparately { get; set; } + public void AddQueue(string partitionId, ControlQueue controlQueue, CancellationToken cancellationToken) { if (this.ownedControlQueues.TryAdd(partitionId, controlQueue)) @@ -480,7 +489,14 @@ async Task ScheduleOrchestrationStatePrefetch( batch.LastCheckpointTime = history.LastCheckpointTime; } - this.readyForProcessingQueue.Enqueue(node); + if (this.ProcessEntitiesSeparately && DurableTask.Core.Common.Entities.IsEntityInstance(batch.OrchestrationInstanceId)) + { + this.entitiesReadyForProcessingQueue.Enqueue(node); + } + else + { + this.orchestrationsReadyForProcessingQueue.Enqueue(node); + } } catch (OperationCanceledException) { @@ -504,14 +520,16 @@ async Task ScheduleOrchestrationStatePrefetch( } } - public async Task GetNextSessionAsync(CancellationToken cancellationToken) + public async Task GetNextSessionAsync(bool entitiesOnly, CancellationToken cancellationToken) { + var readyForProcessingQueue = entitiesOnly? this.entitiesReadyForProcessingQueue : this.orchestrationsReadyForProcessingQueue; + while (!cancellationToken.IsCancellationRequested) { // This call will block until: // 1) a batch of messages has been received for a particular instance and // 2) the history for that instance has been fetched - LinkedListNode node = await this.readyForProcessingQueue.DequeueAsync(cancellationToken); + LinkedListNode node = await readyForProcessingQueue.DequeueAsync(cancellationToken); lock (this.messageAndSessionLock) { @@ -556,7 +574,7 @@ async Task ScheduleOrchestrationStatePrefetch( // A message arrived for a different generation of an existing orchestration instance. // Put it back into the ready queue so that it can be processed once the current generation // is done executing. - if (this.readyForProcessingQueue.Count == 0) + if (readyForProcessingQueue.Count == 0) { // To avoid a tight dequeue loop, delay for a bit before putting this node back into the queue. // This is only necessary when the queue is empty. The main dequeue thread must not be blocked @@ -566,14 +584,14 @@ async Task ScheduleOrchestrationStatePrefetch( lock (this.messageAndSessionLock) { this.pendingOrchestrationMessageBatches.AddLast(node); - this.readyForProcessingQueue.Enqueue(node); + readyForProcessingQueue.Enqueue(node); } }); } else { this.pendingOrchestrationMessageBatches.AddLast(node); - this.readyForProcessingQueue.Enqueue(node); + readyForProcessingQueue.Enqueue(node); } } } @@ -635,7 +653,8 @@ public void GetStats( public virtual void Dispose() { this.fetchRuntimeStateQueue.Dispose(); - this.readyForProcessingQueue.Dispose(); + this.orchestrationsReadyForProcessingQueue.Dispose(); + this.entitiesReadyForProcessingQueue.Dispose(); } class PendingMessageBatch diff --git a/src/DurableTask.AzureStorage/Partitioning/AppLeaseManager.cs b/src/DurableTask.AzureStorage/Partitioning/AppLeaseManager.cs index 5ea594da4..62576898f 100644 --- a/src/DurableTask.AzureStorage/Partitioning/AppLeaseManager.cs +++ b/src/DurableTask.AzureStorage/Partitioning/AppLeaseManager.cs @@ -71,7 +71,7 @@ public AppLeaseManager( this.appLeaseContainer = this.azureStorageClient.GetBlobContainerReference(this.appLeaseContainerName); this.appLeaseInfoBlob = this.appLeaseContainer.GetBlobReference(this.appLeaseInfoBlobName); - var appNameHashInBytes = BitConverter.GetBytes(Fnv1aHashHelper.ComputeHash(this.appName)); + var appNameHashInBytes = BitConverter.GetBytes(DurableTask.Core.Common.Fnv1aHashHelper.ComputeHash(this.appName)); Array.Resize(ref appNameHashInBytes, 16); this.appLeaseId = new Guid(appNameHashInBytes).ToString(); diff --git a/src/DurableTask.Core/Common/Entities.cs b/src/DurableTask.Core/Common/Entities.cs index 2484153de..64fce9486 100644 --- a/src/DurableTask.Core/Common/Entities.cs +++ b/src/DurableTask.Core/Common/Entities.cs @@ -10,14 +10,14 @@ // See the License for the specific language governing permissions and // limitations under the License. // ---------------------------------------------------------------------------------- - -using DurableTask.Core.History; -using System; -using System.Collections.Generic; -using System.Text; - +#nullable enable namespace DurableTask.Core.Common { + using DurableTask.Core.History; + using System; + using System.Collections.Generic; + using System.Text; + /// /// Helpers for dealing with special naming conventions around auto-started orchestrations (entities) /// diff --git a/src/DurableTask.AzureStorage/Fnv1aHashHelper.cs b/src/DurableTask.Core/Common/Fnv1aHashHelper.cs similarity index 61% rename from src/DurableTask.AzureStorage/Fnv1aHashHelper.cs rename to src/DurableTask.Core/Common/Fnv1aHashHelper.cs index fbff51089..4ac17f9c3 100644 --- a/src/DurableTask.AzureStorage/Fnv1aHashHelper.cs +++ b/src/DurableTask.Core/Common/Fnv1aHashHelper.cs @@ -10,8 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // ---------------------------------------------------------------------------------- - -namespace DurableTask.AzureStorage +namespace DurableTask.Core.Common { using System.Text; @@ -22,32 +21,61 @@ namespace DurableTask.AzureStorage /// See https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function. /// Tested with production data and random guids. The result was good distribution. /// - static class Fnv1aHashHelper + public static class Fnv1aHashHelper { const uint FnvPrime = unchecked(16777619); const uint FnvOffsetBasis = unchecked(2166136261); + /// + /// Compute a hash for a given string. + /// + /// The string to hash. + /// a four-byte hash public static uint ComputeHash(string value) { return ComputeHash(value, encoding: null); } + /// + /// Compute a hash for a given string and encoding. + /// + /// The string to hash. + /// The encoding. + /// a four-byte hash public static uint ComputeHash(string value, Encoding encoding) { return ComputeHash(value, encoding, hash: FnvOffsetBasis); } + /// + /// Compute a hash for a given string, encoding, and hash modifier. + /// + /// The string to hash. + /// The encoding. + /// The modifier hash. + /// a four-byte hash public static uint ComputeHash(string value, Encoding encoding, uint hash) { byte[] bytes = (encoding ?? Encoding.UTF8).GetBytes(value); return ComputeHash(bytes, hash); } + /// + /// Compute a hash for the given byte array. + /// + /// The byte array to hash. + /// a four-byte hash public static uint ComputeHash(byte[] array) { return ComputeHash(array, hash: FnvOffsetBasis); } + /// + /// Compute a hash for the given byte array. + /// + /// The byte array to hash. + /// The modifier hash. + /// a four-byte hash public static uint ComputeHash(byte[] array, uint hash) { for (var i = 0; i < array.Length; i++) diff --git a/src/DurableTask.Core/Common/Utils.cs b/src/DurableTask.Core/Common/Utils.cs index 2d63eb895..942c22854 100644 --- a/src/DurableTask.Core/Common/Utils.cs +++ b/src/DurableTask.Core/Common/Utils.cs @@ -21,6 +21,7 @@ namespace DurableTask.Core.Common using System.Linq; using System.Reflection; using System.Runtime.ExceptionServices; + using System.Security.Cryptography; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -152,7 +153,7 @@ public static object DeserializeFromJson(JsonSerializer serializer, string jsonS /// The default value comes from the WEBSITE_SITE_NAME environment variable, which is defined /// in Azure App Service. Other environments can use DTFX_APP_NAME to set this value. /// - public static string AppName { get; set; } = + public static string AppName { get; set; } = Environment.GetEnvironmentVariable("WEBSITE_SITE_NAME") ?? Environment.GetEnvironmentVariable("DTFX_APP_NAME") ?? string.Empty; @@ -624,6 +625,40 @@ public static bool TryGetTaskScheduledId(HistoryEvent historyEvent, out int task } } + /// + /// Creates a determinstic Guid from a string using a hash function. This is a simple hash + /// meant to produce pseudo-random Guids, it is not meant to be cryptographically secure, + /// and does not follow any formatting conventions for UUIDs (such as RFC 4122). + /// + /// The string to hash. + /// A Guid constructed from the hash. + /// + internal static Guid CreateGuidFromHash(string stringToHash) + { + if (string.IsNullOrEmpty(stringToHash)) + { + throw new ArgumentException("string to hash must not be null or empty", nameof(stringToHash)); + } + + var bytes = Encoding.UTF8.GetBytes(stringToHash); + uint hash1 = Fnv1aHashHelper.ComputeHash(bytes, 0xdf0dd395); + uint hash2 = Fnv1aHashHelper.ComputeHash(bytes, 0xa19df4df); + uint hash3 = Fnv1aHashHelper.ComputeHash(bytes, 0xc88599c5); + uint hash4 = Fnv1aHashHelper.ComputeHash(bytes, 0xe24e3e64); + return new Guid( + hash1, + (ushort)(hash2 & 0xFFFF), + (ushort)((hash2 >> 16) & 0xFFFF), + (byte)(hash3 & 0xFF), + (byte)((hash3 >> 8) & 0xFF), + (byte)((hash3 >> 16) & 0xFF), + (byte)((hash3 >> 24) & 0xFF), + (byte)(hash4 & 0xFF), + (byte)((hash4 >> 8) & 0xFF), + (byte)((hash4 >> 16) & 0xFF), + (byte)((hash4 >> 24) & 0xFF)); + } + /// /// Gets the generic return type for a specific . /// diff --git a/src/DurableTask.Core/Entities/ClientEntityContext.cs b/src/DurableTask.Core/Entities/ClientEntityContext.cs new file mode 100644 index 000000000..077aab944 --- /dev/null +++ b/src/DurableTask.Core/Entities/ClientEntityContext.cs @@ -0,0 +1,90 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +namespace DurableTask.Core.Entities +{ + using DurableTask.Core.Entities.EventFormat; + using DurableTask.Core.Entities.OperationFormat; + using DurableTask.Core.Entities.StateFormat; + using Newtonsoft.Json.Linq; + using Newtonsoft.Json; + using System; + + /// + /// Utility functions for clients that interact with entities, either by sending events or by accessing the entity state directly in storage + /// + public static class ClientEntityContext + { + /// + /// Create an event to represent an entity signal. + /// + /// The target instance. + /// A unique identifier for the request. + /// The name of the operation. + /// The serialized input for the operation. + /// The time to schedule this signal, or null if not a scheduled signal + /// The event to send. + public static EventToSend EmitOperationSignal(OrchestrationInstance targetInstance, Guid requestId, string operationName, string input, (DateTime original, DateTime capped)? scheduledTimeUtc) + { + var request = new RequestMessage() + { + ParentInstanceId = null, // means this was sent by a client + ParentExecutionId = null, + Id = requestId, + IsSignal = true, + Operation = operationName, + ScheduledTime = scheduledTimeUtc?.original, + Input = input, + }; + + var jrequest = JToken.FromObject(request, Serializer.InternalSerializer); + + var eventName = scheduledTimeUtc.HasValue + ? EntityMessageEventNames.ScheduledRequestMessageEventName(scheduledTimeUtc.Value.capped) + : EntityMessageEventNames.RequestMessageEventName; + + return new EventToSend(eventName, jrequest, targetInstance); + } + + /// + /// Create an event to represent an entity unlock, which is called by clients to fix orphaned locks. + /// + /// The target instance. + /// The instance id of the entity to be unlocked. + /// The event to send. + public static EventToSend EmitUnlockForOrphanedLock(OrchestrationInstance targetInstance, string lockOwnerInstanceId) + { + var message = new ReleaseMessage() + { + ParentInstanceId = lockOwnerInstanceId, + Id = "fix-orphaned-lock", // we don't know the original id but it does not matter + }; + + var jmessage = JToken.FromObject(message, Serializer.InternalSerializer); + + return new EventToSend(EntityMessageEventNames.ReleaseMessageEventName, jmessage, targetInstance); + } + + /// + /// Extracts the user-defined entity state (as a serialized string) from the scheduler state (also a serialized string). + /// + /// The state of the scheduler, as a serialized string. + /// The entity state + /// True if the entity exists, or false otherwise + public static bool TryGetEntityStateFromSerializedSchedulerState(string serializedSchedulerState, out string entityState) + { + var schedulerState = JsonConvert.DeserializeObject(serializedSchedulerState, Serializer.InternalSerializerSettings); + entityState = schedulerState.EntityState; + return schedulerState.EntityExists; + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/EntityBackendInformation.cs b/src/DurableTask.Core/Entities/EntityBackendInformation.cs new file mode 100644 index 000000000..1d765c65a --- /dev/null +++ b/src/DurableTask.Core/Entities/EntityBackendInformation.cs @@ -0,0 +1,66 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +using System; + +namespace DurableTask.Core.Entities +{ + /// + /// Entity processing characteristics that are controlled by the backend provider, i.e. the orchestration service. + /// + public class EntityBackendProperties + { + /// + /// The time window within which entity messages should be deduplicated and reordered. + /// This is zero for providers that already guarantee exactly-once and ordered delivery. + /// + public TimeSpan EntityMessageReorderWindow { get; set; } + + /// + /// A limit on the number of entity operations that should be processed as a single batch, or null if there is no limit. + /// + public int? MaxEntityOperationBatchSize { get; set; } + + /// + /// The maximum number of entity operation batches that can be processed concurrently on a single node. + /// + public int MaxConcurrentTaskEntityWorkItems { get; set; } + + /// + /// Whether the backend supports implicit deletion, i.e. setting the entity scheduler state to null implicitly deletes the storage record. + /// + public bool SupportsImplicitEntityDeletion { get; set; } + + /// + /// Value of maximum durable timer delay. Used for delayed signals. + /// + public TimeSpan MaximumSignalDelayTime { get; set; } + + /// + /// Computes a cap on the scheduled time of an entity signal, based on the maximum signal delay time + /// + /// + /// + /// + public DateTime GetCappedScheduledTime(DateTime nowUtc, DateTime scheduledUtcTime) + { + if ((scheduledUtcTime - nowUtc) <= this.MaximumSignalDelayTime) + { + return scheduledUtcTime; + } + else + { + return nowUtc + this.MaximumSignalDelayTime; + } + } + } +} diff --git a/src/DurableTask.Core/Entities/EntityExecutionOptions.cs b/src/DurableTask.Core/Entities/EntityExecutionOptions.cs new file mode 100644 index 000000000..6e6603b92 --- /dev/null +++ b/src/DurableTask.Core/Entities/EntityExecutionOptions.cs @@ -0,0 +1,56 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using DurableTask.Core.Serializing; + +namespace DurableTask.Core.Entities +{ + /// + /// Options that are used for configuring how a TaskEntity executes entity operations. + /// + public class EntityExecutionOptions + { + /// + /// The data converter used for converting inputs and outputs for operations. + /// + public DataConverter MessageDataConverter { get; set; } = JsonDataConverter.Default; + + /// + /// The data converter used for the entity state. + /// + public DataConverter StateDataConverter { get; set; } = JsonDataConverter.Default; + + /// + /// The data converter used for exceptions. + /// + public DataConverter ErrorDataConverter { get; set; } = JsonDataConverter.Default; + + /// + /// If true, all effects of an entity operation (all state changes and all actions) are rolled back + /// if the entity operation completes with an exception. + /// Implementations may override this setting. + /// + public bool RollbackOnExceptions { get; set; } = true; + + /// + /// Information about backend entity support. + /// + internal EntityBackendProperties EntityBackendProperties { get; set; } + + /// + /// The mode that is used for propagating errors, as specified in the . + /// + internal ErrorPropagationMode ErrorPropagationMode { get; set; } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/EntityId.cs b/src/DurableTask.Core/Entities/EntityId.cs new file mode 100644 index 000000000..9c63896ee --- /dev/null +++ b/src/DurableTask.Core/Entities/EntityId.cs @@ -0,0 +1,111 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +namespace DurableTask.Core.Entities +{ + using System; + using System.Runtime.Serialization; + using Newtonsoft.Json; + + /// + /// A unique identifier for an entity, consisting of entity name and entity key. + /// + [DataContract] + public readonly struct EntityId : IEquatable, IComparable + { + /// + /// Create an entity id for an entity. + /// + /// The name of this class of entities. + /// The entity key. + public EntityId(string entityName, string entityKey) + { + if (string.IsNullOrEmpty(entityName)) + { + throw new ArgumentNullException(nameof(entityName), "Invalid entity id: entity name must not be a null or empty string."); + } + + this.Name = entityName; + this.Key = entityKey ?? throw new ArgumentNullException(nameof(entityKey), "Invalid entity id: entity key must not be null."); + } + + /// + /// The name for this class of entities. + /// + [DataMember(Name = "name", IsRequired = true)] + public readonly string Name; + + /// + /// The entity key. Uniquely identifies an entity among all entities of the same name. + /// + [DataMember(Name = "key", IsRequired = true)] + public readonly string Key; + + /// + public override string ToString() + { + return $"@{this.Name}@{this.Key}"; + } + + internal static string GetSchedulerIdPrefixFromEntityName(string entityName) + { + return $"@{entityName}@"; + } + + /// + /// Returns the entity ID for a given instance ID. + /// + /// The instance ID. + /// the corresponding entity ID. + public static EntityId FromString(string instanceId) + { + if (string.IsNullOrEmpty(instanceId)) + { + throw new ArgumentException(nameof(instanceId)); + } + var pos = instanceId.IndexOf('@', 1); + if ( pos <= 0 || instanceId[0] != '@') + { + throw new ArgumentException($"Instance ID '{instanceId}' is not a valid entity ID.", nameof(instanceId)); + } + var entityName = instanceId.Substring(1, pos - 1); + var entityKey = instanceId.Substring(pos + 1); + return new EntityId(entityName, entityKey); + } + + + /// + public override bool Equals(object obj) + { + return (obj is EntityId other) && this.Equals(other); + } + + /// + public bool Equals(EntityId other) + { + return (this.Name,this.Key).Equals((other.Name, other.Key)); + } + + /// + public override int GetHashCode() + { + return (this.Name, this.Key).GetHashCode(); + } + + /// + public int CompareTo(object obj) + { + var other = (EntityId)obj; + return (this.Name, this.Key).CompareTo((other.Name, other.Key)); + } + } +} diff --git a/src/DurableTask.Core/Entities/EntitySchedulerException.cs b/src/DurableTask.Core/Entities/EntitySchedulerException.cs new file mode 100644 index 000000000..520e6d1b2 --- /dev/null +++ b/src/DurableTask.Core/Entities/EntitySchedulerException.cs @@ -0,0 +1,52 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities +{ + using System; + using System.Runtime.Serialization; + + /// + /// Exception used to describe various issues encountered by the entity scheduler. + /// + [Serializable] + public class EntitySchedulerException : Exception + { + /// + /// Initializes a new instance of the class. + /// + public EntitySchedulerException() + { + } + + /// + /// Initializes an new instance of the class. + /// + /// The message that describes the error. + /// The exception that was caught. + public EntitySchedulerException(string errorMessage, Exception innerException) + : base(errorMessage, innerException) + { + } + + /// + /// Initializes a new instance of the class with serialized data. + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized object data about the exception being thrown. + /// The System.Runtime.Serialization.StreamingContext that contains contextual information about the source or destination. + protected EntitySchedulerException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/EventFormat/EntityMessageEventNames.cs b/src/DurableTask.Core/Entities/EventFormat/EntityMessageEventNames.cs new file mode 100644 index 000000000..adecea1dc --- /dev/null +++ b/src/DurableTask.Core/Entities/EventFormat/EntityMessageEventNames.cs @@ -0,0 +1,37 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities.EventFormat +{ + using System; + + /// + /// Determines event names to use for messages sent to and from entities. + /// + internal static class EntityMessageEventNames + { + public static string RequestMessageEventName => "op"; + + public static string ReleaseMessageEventName => "release"; + + public static string ContinueMessageEventName => "continue"; + + public static string ScheduledRequestMessageEventName(DateTime scheduledUtc) => $"op@{scheduledUtc:o}"; + + public static string ResponseMessageEventName(Guid requestId) => requestId.ToString(); + + public static bool IsRequestMessage(string eventName) => eventName.StartsWith("op"); + + public static bool IsReleaseMessage(string eventName) => eventName == "release"; + } +} diff --git a/src/DurableTask.Core/Entities/EventFormat/ReleaseMessage.cs b/src/DurableTask.Core/Entities/EventFormat/ReleaseMessage.cs new file mode 100644 index 000000000..55c538f50 --- /dev/null +++ b/src/DurableTask.Core/Entities/EventFormat/ReleaseMessage.cs @@ -0,0 +1,32 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +namespace DurableTask.Core.Entities.EventFormat +{ + using System.Runtime.Serialization; + using Newtonsoft.Json; + + [DataContract] + internal class ReleaseMessage + { + [DataMember(Name = "parent")] + public string ParentInstanceId { get; set; } + + [DataMember(Name = "id")] + public string Id { get; set; } + + public override string ToString() + { + return $"[Release lock {Id} by {ParentInstanceId}]"; + } + } +} diff --git a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs new file mode 100644 index 000000000..d87d0de8a --- /dev/null +++ b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs @@ -0,0 +1,115 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +namespace DurableTask.Core.Entities.EventFormat +{ + using System; + using System.Runtime.Serialization; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + /// + /// A message sent to an entity, such as operation, signal, lock, or continue messages. + /// + [DataContract] + internal class RequestMessage + { + /// + /// The name of the operation being called (if this is an operation message) or null + /// (if this is a lock request). + /// + [DataMember(Name = "op")] + public string Operation { get; set; } + + /// + /// Whether or not this is a one-way message. + /// + [DataMember(Name = "signal", EmitDefaultValue = false)] + public bool IsSignal { get; set; } + + /// + /// The operation input. + /// + [DataMember(Name = "input", EmitDefaultValue = false)] + public string Input { get; set; } + + /// + /// A unique identifier for this operation. + /// + [DataMember(Name = "id", IsRequired = true)] + public Guid Id { get; set; } + + /// + /// The parent instance that called this operation. + /// + [DataMember(Name = "parent", EmitDefaultValue = false)] + public string ParentInstanceId { get; set; } + + /// + /// The parent instance that called this operation. + /// + [DataMember(Name = "parentExecution", EmitDefaultValue = false)] + public string ParentExecutionId { get; set; } + + /// + /// Optionally, a scheduled time at which to start the operation. + /// + [DataMember(Name = "due", EmitDefaultValue = false)] + public DateTime? ScheduledTime { get; set; } + + /// + /// A timestamp for this request. + /// Used for duplicate filtering and in-order delivery. + /// + [DataMember] + public DateTime Timestamp { get; set; } + + /// + /// A timestamp for the predecessor request in the stream, or DateTime.MinValue if none. + /// Used for duplicate filtering and in-order delivery. + /// + [DataMember] + public DateTime Predecessor { get; set; } + + /// + /// For lock requests, the set of locks being acquired. Is sorted, + /// contains at least one element, and has no repetitions. + /// + [DataMember(Name = "lockset", EmitDefaultValue = false)] + public EntityId[] LockSet { get; set; } + + /// + /// For lock requests involving multiple locks, the message number. + /// + [DataMember(Name = "pos", EmitDefaultValue = false)] + public int Position { get; set; } + + /// + /// whether this message is a lock request + /// + [DataMember] + public bool IsLockRequest => LockSet != null; + + /// + public override string ToString() + { + if (IsLockRequest) + { + return $"[Request lock {Id} by {ParentInstanceId} {ParentExecutionId}, position {Position}]"; + } + else + { + return $"[{(IsSignal ? "Signal" : "Call")} '{Operation}' operation {Id} by {ParentInstanceId} {ParentExecutionId}]"; + } + } + } +} diff --git a/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs new file mode 100644 index 000000000..85b1ccfc3 --- /dev/null +++ b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs @@ -0,0 +1,47 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +namespace DurableTask.Core.Entities.EventFormat +{ + using System; + using System.Runtime.Serialization; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + [DataContract] + internal class ResponseMessage + { + [DataMember(Name = "result")] + public string Result { get; set; } + + [DataMember(Name = "exceptionType", EmitDefaultValue = false)] + public string ErrorMessage { get; set; } + + [DataMember(Name = "failureDetails", EmitDefaultValue = false)] + public FailureDetails FailureDetails { get; set; } + + [IgnoreDataMember] + public bool IsErrorResult => this.ErrorMessage != null; + + public override string ToString() + { + if (this.IsErrorResult) + { + return $"[ErrorResponse {this.Result}]"; + } + else + { + return $"[Response {this.Result}]"; + } + } + } +} diff --git a/src/DurableTask.Core/Entities/EventToSend.cs b/src/DurableTask.Core/Entities/EventToSend.cs new file mode 100644 index 000000000..693db16bc --- /dev/null +++ b/src/DurableTask.Core/Entities/EventToSend.cs @@ -0,0 +1,55 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +namespace DurableTask.Core.Entities +{ + using DurableTask.Core.Entities.EventFormat; + using DurableTask.Core.Entities.OperationFormat; + using DurableTask.Core.Entities.StateFormat; + using Newtonsoft.Json.Linq; + using Newtonsoft.Json; + using System; + + /// + /// The data associated with sending an event to an orchestration. + /// + public readonly struct EventToSend + { + /// + /// The name of the event. + /// + public readonly string EventName { get; } + + /// + /// The content of the event. + /// + public readonly object EventContent { get; } + + /// + /// The target instance for the event. + /// + public readonly OrchestrationInstance TargetInstance; + + /// + /// Construct an entity message event with the given members. + /// + /// The name of the event. + /// The content of the event. + /// The target of the event. + public EventToSend(string name, object content, OrchestrationInstance target) + { + EventName = name; + EventContent = content; + TargetInstance = target; + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/IEntityExecutor.cs b/src/DurableTask.Core/Entities/IEntityExecutor.cs new file mode 100644 index 000000000..7729662e3 --- /dev/null +++ b/src/DurableTask.Core/Entities/IEntityExecutor.cs @@ -0,0 +1,30 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +using System; +using System.Threading; +using System.Threading.Tasks; +using DurableTask.Core.Entities.OperationFormat; + +namespace DurableTask.Core.Entities +{ + /// + /// Untyped interface for processing batches of entity operations. + /// + public interface IEntityExecutor + { + /// + /// processes a batch of entity opreations + /// + internal abstract Task ExecuteOperationBatchAsync(OperationBatchRequest operations, EntityExecutionOptions options); + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/IEntityOrchestrationService.cs b/src/DurableTask.Core/Entities/IEntityOrchestrationService.cs new file mode 100644 index 000000000..8c63dca6b --- /dev/null +++ b/src/DurableTask.Core/Entities/IEntityOrchestrationService.cs @@ -0,0 +1,47 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace DurableTask.Core.Entities +{ + /// + /// Interface for objects that provide entity backend information. + /// + public interface IEntityOrchestrationService + { + /// + /// The entity orchestration service. + /// + /// An object containing properties of the entity backend. + EntityBackendProperties GetEntityBackendProperties(); + + /// + /// Configures the orchestration service backend so entities and orchestrations are kept in two separate queues, and can be fetched separately. + /// + void ProcessEntitiesSeparately(); + + /// + /// Specialized variant of that + /// fetches only work items for true orchestrations, not entities. + /// + Task LockNextOrchestrationWorkItemAsync(TimeSpan receiveTimeout, CancellationToken cancellationToken); + + /// + /// Specialized variant of that + /// fetches only work items for entities, not plain orchestrations. + /// + Task LockNextEntityWorkItemAsync(TimeSpan receiveTimeout, CancellationToken cancellationToken); + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/LocalSDK/EntityContext.cs b/src/DurableTask.Core/Entities/LocalSDK/EntityContext.cs new file mode 100644 index 000000000..b754d4fad --- /dev/null +++ b/src/DurableTask.Core/Entities/LocalSDK/EntityContext.cs @@ -0,0 +1,139 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core +{ + using DurableTask.Core.Entities; + using DurableTask.Core.Serializing; + using System; + using System.Reflection; + using System.Threading.Tasks; + + /// + /// Context for an entity, which is available to the application code while it is executing entity operations. + /// + /// The JSON-serializable type of the entity state. + public abstract class EntityContext + { + /// + /// Gives access to various options to control entity execution. + /// + public abstract EntityExecutionOptions EntityExecutionOptions { get; } + + /// + /// Gets the id of the currently executing entity. + /// + public abstract EntityId EntityId { get; } + + /// + /// Gets the name of the operation that was called. + /// + /// + /// An operation invocation on an entity includes an operation name, which states what + /// operation to perform, and optionally an operation input. + /// + public abstract string OperationName { get; } + + /// + /// Whether this entity has state. + /// + /// The value of changes as entity operations access or delete the state during their execution. It is set to true after is accessed from within an operation, + /// and it is set to false after is called. This can happen repeatedly; neither creation nor deletion are permanent. + public abstract bool HasState { get; } + + /// + /// The size of the current batch of operations. + /// + public abstract int BatchSize { get; } + + /// + /// The position of the currently executing operation within the current batch of operations. + /// + public abstract int BatchPosition { get; } + + /// + /// Gets or sets the current state of the entity. Implicitly creates the state if the entity does not have state yet. + /// + /// If the entity does not have a state yet (first time access), + /// or does not have a state anymore (was deleted), accessing this property implicitly creates state for the entity. For the 'get' accessor, this means + /// that is called to create the state. + /// For the 'set' accessor, the given value is used. + public abstract TState State { get; set; } + + /// + /// Deletes the state of this entity. + /// + public abstract void DeleteState(); + + /// + /// Gets the input for this operation, as a deserialized value. + /// + /// The JSON-serializable type used for the operation input. + /// The operation input, or default() if none. + /// + /// An operation invocation on an entity includes an operation name, which states what + /// operation to perform, and optionally an operation input. + /// + public virtual TInput GetInput() => (TInput)this.GetInput(typeof(TInput)); + + /// + /// Gets the input for this operation, as a deserialized value. + /// + /// The JSON-serializable type used for the operation input. + /// The operation input, or default() if none. + /// + /// An operation invocation on an entity includes an operation name, which states what + /// operation to perform, and optionally an operation input. + /// + public abstract object GetInput(Type inputType); + + /// + /// Signals an entity to perform an operation, without waiting for a response. Any result or exception is ignored (fire and forget). + /// + /// The target entity. + /// The name of the operation. + /// The operation input. + public abstract void SignalEntity(EntityId entity, string operationName, object operationInput = null); + + /// + /// Signals an entity to perform an operation, at a specified time. Any result or exception is ignored (fire and forget). + /// + /// The target entity. + /// The time at which to start the operation. + /// The name of the operation. + /// The input for the operation. + public abstract void SignalEntity(EntityId entity, DateTime scheduledTimeUtc, string operationName, object operationInput = null); + + /// + /// Schedules an orchestration function with the given name and version for execution. + /// Any result or exception is ignored (fire and forget). + /// + /// The name of the orchestrator, as specified by the ObjectCreator. + /// The version of the orchestrator. + /// the input to pass to the orchestrator function. + /// optionally, an instance id for the orchestration. By default, a random GUID is used. + /// The instance id of the new orchestration. + public abstract string StartNewOrchestration(string name, string version, object input, string instanceId = null); + + /// + /// Schedules an orchestration function with the given type for execution. + /// Any result or exception is ignored (fire and forget). + /// + /// Type of the TaskOrchestration derived class to instantiate + /// the input to pass to the orchestrator function. + /// optionally, an instance id for the orchestration. By default, a random GUID is used. + /// The instance id of the new orchestration. + public virtual string StartNewOrchestration(Type orchestrationType, object input, string instanceId = null) + => this.StartNewOrchestration(NameVersionHelper.GetDefaultName(orchestrationType), NameVersionHelper.GetDefaultVersion(orchestrationType), input, instanceId); + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/LocalSDK/TaskEntity.cs b/src/DurableTask.Core/Entities/LocalSDK/TaskEntity.cs new file mode 100644 index 000000000..9b29d23f7 --- /dev/null +++ b/src/DurableTask.Core/Entities/LocalSDK/TaskEntity.cs @@ -0,0 +1,63 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Entities +{ + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + using DurableTask.Core.Common; + using DurableTask.Core.Entities.EventFormat; + using DurableTask.Core.Entities.OperationFormat; + using DurableTask.Core.Exceptions; + using DurableTask.Core.Serializing; + using Newtonsoft.Json; + + /// + /// TaskEntity representing an entity with typed state. + /// + /// The type of the entity state. + public abstract class TaskEntity : TaskEntity + { + /// + /// Defines how this entity processes operations. Implementations must override this. + /// + /// The context for the operation. + /// A task that completes with the return value of the operation. + public abstract ValueTask ExecuteOperationAsync(EntityContext context); + + /// + /// A function for creating the initial state of the entity. + /// Implementations may override this if they want to perform a different initialization. + /// + /// This is only called when the entity state is created, not when it is reloaded from storage. + /// When loading the entity from storage, the entity state is created using a DataConverter. + public virtual TState CreateInitialState(EntityContext context) + { + return default; + } + + /// + /// We cache the entity context inside the TaskEntity object. That way, it is possible for applications + /// to use a caching implementation of if they want + /// to avoid constructing a fresh execution context, and allow deserialized state objects to be reused. + /// + internal TaskEntityContext CachedContext; + + internal override Task ExecuteOperationBatchAsync(OperationBatchRequest operations, EntityExecutionOptions options) + { + this.CachedContext ??= new TaskEntityContext(this, options); + return this.CachedContext.ExecuteBatchAsync(operations); + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/LocalSDK/TaskEntityContext.cs b/src/DurableTask.Core/Entities/LocalSDK/TaskEntityContext.cs new file mode 100644 index 000000000..732a3b46b --- /dev/null +++ b/src/DurableTask.Core/Entities/LocalSDK/TaskEntityContext.cs @@ -0,0 +1,452 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core +{ + using DurableTask.Core.Common; + using DurableTask.Core.Entities; + using DurableTask.Core.Entities.OperationFormat; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + using System; + using System.Collections.Generic; + using System.Reflection; + using System.Runtime.Serialization; + using System.Threading.Tasks; + + internal class TaskEntityContext : EntityContext + { + readonly TaskEntity taskEntity; + readonly EntityExecutionOptions executionOptions; + + EntityId entityId; + int batchPosition = -1; + int batchSize; + string lastSerializedState; + OperationRequest currentOperationRequest; + OperationResult currentOperationResult; + StateAccess currentStateAccess; + TState currentState; + List actions; + + public TaskEntityContext(TaskEntity taskEntity, EntityExecutionOptions options) + { + this.taskEntity = taskEntity; + this.executionOptions = options; + } + + public override EntityId EntityId => this.entityId; + + public override EntityExecutionOptions EntityExecutionOptions => this.executionOptions; + + public override string OperationName => this.currentOperationRequest.Operation; + public override int BatchSize => this.batchSize; + public override int BatchPosition => this.batchPosition; + + OperationRequest CurrentOperation => this.currentOperationRequest; + + // The last serialized checkpoint of the entity state is always stored in + // this.LastSerializedState. The current state is determined by this.CurrentStateAccess and this.CurrentState. + internal enum StateAccess + { + NotAccessed, // current state is stored in this.LastSerializedState + Accessed, // current state is stored in this.currentState + Clean, // current state is stored in both this.currentState (deserialized) and in this.LastSerializedState + Deleted, // current state is deleted + } + + public override bool HasState + { + get + { + switch (this.currentStateAccess) + { + case StateAccess.Accessed: + case StateAccess.Clean: + return true; + + case StateAccess.Deleted: + return false; + + default: return this.lastSerializedState != null; + } + } + } + + public override TState State + { + get + { + if (this.currentStateAccess == StateAccess.Accessed) + { + return this.currentState; + } + else if (this.currentStateAccess == StateAccess.Clean) + { + this.currentStateAccess = StateAccess.Accessed; + return this.currentState; + } + + TState result; + + if (this.lastSerializedState != null && this.currentStateAccess != StateAccess.Deleted) + { + try + { + result = this.executionOptions.StateDataConverter.Deserialize(this.lastSerializedState); + } + catch (Exception e) + { + throw new EntitySchedulerException($"Failed to deserialize entity state: {e.Message}", e); + } + } + else + { + try + { + result = this.taskEntity.CreateInitialState(this); + } + catch (Exception e) + { + throw new EntitySchedulerException($"Failed to initialize entity state: {e.Message}", e); + } + } + + this.currentStateAccess = StateAccess.Accessed; + this.currentState = result; + return result; + } + set + { + this.currentState = value; + this.currentStateAccess = StateAccess.Accessed; + } + } + + public override void DeleteState() + { + this.currentStateAccess = StateAccess.Deleted; + this.currentState = default; + } + + public void Rollback(int positionBeforeCurrentOperation) + { + // We discard the current state, which means we go back to the last serialized one + this.currentStateAccess = StateAccess.NotAccessed; + this.currentState = default; + + // we also roll back the list of outgoing messages, + // so any signals sent by this operation are discarded. + this.actions.RemoveRange(positionBeforeCurrentOperation, this.actions.Count - positionBeforeCurrentOperation); + } + + private bool TryWriteback(out OperationResult serializationErrorResult, OperationRequest operationRequest = null) + { + if (this.currentStateAccess == StateAccess.Deleted) + { + this.lastSerializedState = null; + this.currentStateAccess = StateAccess.NotAccessed; + } + else if (this.currentStateAccess == StateAccess.Accessed) + { + try + { + string serializedState = this.executionOptions.StateDataConverter.Serialize(this.currentState); + this.lastSerializedState = serializedState; + this.currentStateAccess = StateAccess.Clean; + } + catch (Exception serializationException) when (!Utils.IsFatal(serializationException)) + { + // we cannot serialize the entity state - this is an application error. To help users diagnose this, + // we wrap it into a descriptive exception, and propagate this error result to any calling orchestrations. + + serializationException = new EntitySchedulerException( + $"Operation was rolled back because state for entity '{this.EntityId}' could not be serialized: {serializationException.Message}", serializationException); + serializationErrorResult = new OperationResult(); + this.CaptureExceptionInOperationResult(serializationErrorResult, serializationException); + + // we have no choice but to roll back to the state prior to this operation + this.currentStateAccess = StateAccess.NotAccessed; + this.currentState = default; + + return false; + } + } + else + { + // the state was not accessed, or is clean, so we don't need to write anything back + } + + serializationErrorResult = null; + return true; + } + + public override object GetInput(Type inputType) + { + try + { + return this.executionOptions.MessageDataConverter.Deserialize(this.CurrentOperation.Input, inputType); + } + catch (Exception e) + { + throw new EntitySchedulerException($"Failed to deserialize input for operation '{this.CurrentOperation.Operation}': {e.Message}", e); + } + } + + + public override void SignalEntity(EntityId entity, string operationName, object operationInput = null) + { + this.SignalEntityInternal(entity, null, operationName, operationInput); + } + + + public override void SignalEntity(EntityId entity, DateTime scheduledTimeUtc, string operationName, object operationInput = null) + { + this.SignalEntityInternal(entity, scheduledTimeUtc, operationName, operationInput); + } + + private void SignalEntityInternal(EntityId entity, DateTime? scheduledTimeUtc, string operationName, object operationInput) + { + if (operationName == null) + { + throw new ArgumentNullException(nameof(operationName)); + } + + string functionName = entity.Name; + + var action = new SendSignalOperationAction() + { + InstanceId = entity.ToString(), + Name = operationName, + ScheduledTime = scheduledTimeUtc, + Input = null, + }; + + if (operationInput != null) + { + try + { + action.Input = this.executionOptions.MessageDataConverter.Serialize(operationInput); + } + catch (Exception e) + { + throw new EntitySchedulerException($"Failed to serialize input for operation '{operationName}': {e.Message}", e); + } + } + + // add the action to the results, under a lock since user code may be concurrent + lock (this.actions) + { + this.actions.Add(action); + } + } + + public override string StartNewOrchestration(string name, string version, object input, string instanceId = null) + { + if (string.IsNullOrEmpty(instanceId)) + { + instanceId = Guid.NewGuid().ToString(); // this is an entity, so we don't need to be deterministic + } + else if (DurableTask.Core.Common.Entities.IsEntityInstance(instanceId)) + { + throw new ArgumentException(nameof(instanceId), "Invalid orchestration instance ID, must not be an entity ID"); + } + + var action = new StartNewOrchestrationOperationAction() + { + InstanceId = instanceId, + Name = name, + Version = version, + Tags = new Dictionary() { { OrchestrationTags.FireAndForget, "" } }, + }; + + if (input != null) + { + try + { + action.Input = this.executionOptions.MessageDataConverter.Serialize(input); + } + catch (Exception e) + { + throw new EntitySchedulerException($"Failed to serialize input for orchestration '{name}': {e.Message}", e); + } + } + + // add the action to the results, under a lock since user code may be concurrent + lock (this.actions) + { + this.actions.Add(action); + } + + return instanceId; + } + + public async Task ExecuteBatchAsync(OperationBatchRequest batchRequest) + { + var entityId = EntityId.FromString(batchRequest.InstanceId); + + if (entityId.Equals(this.entityId) && batchRequest.EntityState == this.lastSerializedState) + { + // we were called before, with the same entityId, and the same state. + // We can therefore keep the current state as is. + } + else + { + this.entityId = entityId; + this.lastSerializedState = batchRequest.EntityState; + this.currentState = default; + this.currentStateAccess = StateAccess.NotAccessed; + } + + this.batchSize = batchRequest.Operations.Count; + var actions = this.actions = new List(); + var results = new List(); + + // execute all the operations in a loop and record the results. + for (int i = 0; i < batchRequest.Operations.Count; i++) + { + this.batchPosition = i; + this.currentOperationRequest = batchRequest.Operations[i]; + this.currentOperationResult = new OperationResult(); + + // process the operation (handling any errors if necessary) + await this.ProcessOperationRequestAsync(); + + results.Add(this.currentOperationResult); + } + + if (this.executionOptions.RollbackOnExceptions) + { + // the state has already been written back, since it is + // done right after each operation. + } + else + { + // we are writing back the state only now, after the whole batch is complete. + + var writeBackSuccessful = this.TryWriteback(out OperationResult serializationErrorMessage); + + if (!writeBackSuccessful) + { + // failed to write state, we now consider all operations in the batch as failed. + // We thus record the results (which are now fail results) and commit the batch + // with the original state. + + // we clear the actions (if we cannot update the state, we should not take other actions either) + actions.Clear(); + + // we replace all response messages with the serialization error message, + // so that callers get to know that this operation failed, and why + for (int i = 0; i < results.Count; i++) + { + results[i] = serializationErrorMessage; + } + } + } + + // clear these fields before returning, so if this context is being cached somewhere, it keeps only the relevant information. + this.batchPosition = -1; + this.batchSize = 0; + this.currentOperationRequest = null; + this.currentOperationResult = null; + this.actions = null; + + return new OperationBatchResult() + { + Results = results, + Actions = actions, + EntityState = this.lastSerializedState, + }; + } + + async ValueTask ProcessOperationRequestAsync() + { + var actionPositionCheckpoint = this.actions.Count; + + try + { + object returnedResult = await taskEntity.ExecuteOperationAsync(this); + + if (returnedResult != null) + { + try + { + this.currentOperationResult.Result = this.executionOptions.MessageDataConverter.Serialize(returnedResult); + } + catch (Exception e) + { + throw new EntitySchedulerException($"Failed to serialize output for operation '{this.CurrentOperation.Operation}': {e.Message}", e); + } + } + } + catch (Exception e) when (!Utils.IsFatal(e) && !Utils.IsExecutionAborting(e)) + { + this.CaptureExceptionInOperationResult(this.currentOperationResult, e); + } + + if (this.executionOptions.RollbackOnExceptions) + { + // we write back the entity state after each successful operation + if (this.currentOperationResult.ErrorMessage == null) + { + if (!this.TryWriteback(out OperationResult errorResult, this.currentOperationRequest)) + { + // state serialization failed; create error response and roll back. + this.currentOperationResult = errorResult; + } + } + + if (this.currentOperationResult.ErrorMessage != null) + { + // we must also roll back any actions that this operation has issued + this.Rollback(actionPositionCheckpoint); + } + } + } + + public void CaptureExceptionInOperationResult(OperationResult result, Exception originalException) + { + // a non-null ErrorMessage field is how we track internally that this result represents a failed operation + result.ErrorMessage = originalException.Message ?? originalException.GetType().FullName; + + // record additional information, based on the error propagation mode + switch (this.executionOptions.ErrorPropagationMode) + { + case ErrorPropagationMode.SerializeExceptions: + try + { + result.Result = this.executionOptions.ErrorDataConverter.Serialize(originalException); + } + catch (Exception serializationException) when (!Utils.IsFatal(serializationException)) + { + // we can't serialize the original exception. We can't throw it here. + // So let us try to at least serialize the serialization exception + // because this information may help users that are trying to troubleshoot their application. + try + { + result.Result = this.executionOptions.ErrorDataConverter.Serialize(serializationException); + } + catch (Exception serializationExceptionSerializationException) when (!Utils.IsFatal(serializationExceptionSerializationException)) + { + // there seems to be nothing we can do. + } + } + break; + + case ErrorPropagationMode.UseFailureDetails: + result.FailureDetails = new FailureDetails(originalException); + break; + } + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs b/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs new file mode 100644 index 000000000..939a8b6ee --- /dev/null +++ b/src/DurableTask.Core/Entities/LocalSDK/TaskHubEntityClient.cs @@ -0,0 +1,542 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core +{ + using DurableTask.Core.Logging; + using DurableTask.Core.Serializing; + using Microsoft.Extensions.Logging; + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + using System.Threading; + using DurableTask.Core.Entities; + using DurableTask.Core.History; + using System.Linq; + using System.Diagnostics; + using DurableTask.Core.Query; + using Newtonsoft.Json; + + /// + /// Client used to manage and query entity instances + /// + public sealed class TaskHubEntityClient + { + readonly DataConverter messageDataConverter; + readonly DataConverter stateDataConverter; + readonly LogHelper logHelper; + readonly EntityBackendProperties backendProperties; + readonly IOrchestrationServiceQueryClient queryClient; + readonly IOrchestrationServicePurgeClient purgeClient; + + /// + /// The orchestration service client for this task hub client + /// + public IOrchestrationServiceClient ServiceClient { get; } + + private void CheckEntitySupport(string name) + { + if (this.backendProperties == null) + { + throw new InvalidOperationException($"{nameof(TaskHubEntityClient)}.{name} is not supported because the chosen backend does not support entities."); + } + } + + private void CheckQuerySupport(string name) + { + if (this.queryClient == null) + { + throw new InvalidOperationException($"{nameof(TaskHubEntityClient)}.{name} is not supported because the chosen backend does not implement {nameof(IOrchestrationServiceQueryClient)}."); + } + } + + private void CheckPurgeSupport(string name) + { + if (this.purgeClient == null) + { + throw new InvalidOperationException($"{nameof(TaskHubEntityClient)}.{name} is not supported because the chosen backend does not implement {nameof(IOrchestrationServicePurgeClient)}."); + } + } + + /// + /// Create a new TaskHubEntityClient from a given TaskHubClient. + /// + /// The taskhub client. + /// The to use for entity state deserialization, or null if the default converter should be used for that purpose. + public TaskHubEntityClient(TaskHubClient client, DataConverter stateDataConverter = null) + { + this.ServiceClient = client.ServiceClient; + this.messageDataConverter = client.DefaultConverter; + this.stateDataConverter = stateDataConverter ?? client.DefaultConverter; + this.logHelper = client.LogHelper; + this.backendProperties = (client.ServiceClient as IEntityOrchestrationService)?.GetEntityBackendProperties(); + this.queryClient = client.ServiceClient as IOrchestrationServiceQueryClient; + this.purgeClient = client.ServiceClient as IOrchestrationServicePurgeClient; + } + + /// + /// Signals an entity to perform an operation. + /// + /// The target entity. + /// The name of the operation. + /// The input for the operation. + /// A future time for which to schedule the start of this operation, or null if is should start as soon as possible. + /// A task that completes when the message has been reliably enqueued. + public async Task SignalEntityAsync(EntityId entityId, string operationName, object operationInput = null, DateTime? scheduledTimeUtc = null) + { + this.CheckEntitySupport(nameof(SignalEntityAsync)); + + (DateTime original, DateTime capped)? scheduledTime = null; + if (scheduledTimeUtc.HasValue) + { + DateTime original = scheduledTimeUtc.Value.ToUniversalTime(); + DateTime capped = this.backendProperties.GetCappedScheduledTime(DateTime.UtcNow, original); + scheduledTime = (original, capped); + } + + var guid = Guid.NewGuid(); // unique id for this request + var instanceId = entityId.ToString(); + var instance = new OrchestrationInstance() { InstanceId = instanceId }; + + string serializedInput = null; + if (operationInput != null) + { + serializedInput = this.messageDataConverter.Serialize(operationInput); + } + + EventToSend eventToSend = ClientEntityContext.EmitOperationSignal( + instance, + guid, + operationName, + serializedInput, + scheduledTime); + + string serializedEventContent = this.messageDataConverter.Serialize(eventToSend.EventContent); + + var eventRaisedEvent = new EventRaisedEvent(-1, serializedEventContent) + { + Name = eventToSend.EventName + }; + + var taskMessage = new TaskMessage + { + OrchestrationInstance = instance, + Event = eventRaisedEvent, + }; + + this.logHelper.RaisingEvent(instance, eventRaisedEvent); + + await this.ServiceClient.SendTaskOrchestrationMessageAsync(taskMessage); + } + + /// + /// Tries to read the current state of an entity. + /// + /// The JSON-serializable type of the entity. + /// The target entity. + /// a response containing the current state of the entity. + public async Task> ReadEntityStateAsync(EntityId entityId) + { + var instanceId = entityId.ToString(); + + this.logHelper.FetchingInstanceState(instanceId); + IList stateList = await this.ServiceClient.GetOrchestrationStateAsync(instanceId, allExecutions:false); + + OrchestrationState state = stateList?.FirstOrDefault(); + if (state != null + && state.OrchestrationInstance != null + && state.Input != null) + { + if (ClientEntityContext.TryGetEntityStateFromSerializedSchedulerState(state.Input, out string serializedEntityState)) + { + return new StateResponse() + { + EntityExists = true, + EntityState = this.messageDataConverter.Deserialize(serializedEntityState), + }; + } + } + + return new StateResponse() + { + EntityExists = false, + EntityState = default, + }; + } + + /// + /// The response returned by . + /// + /// The JSON-serializable type of the entity. + public struct StateResponse + { + /// + /// Whether this entity has a state or not. + /// + /// An entity initially has no state, but a state is created and persisted in storage once operations access it. + public bool EntityExists { get; set; } + + /// + /// The current state of the entity, if it exists, or default() otherwise. + /// + public T EntityState { get; set; } + } + + /// + /// Gets the status of all entity instances that match the specified query conditions. + /// + /// Return entity instances that match the specified query conditions. + /// Cancellation token that can be used to cancel the query operation. + /// Returns a page of entity instances and a continuation token for fetching the next page. + public async Task ListEntitiesAsync(Query query, CancellationToken cancellationToken) + { + this.CheckEntitySupport(nameof(ListEntitiesAsync)); + this.CheckQuerySupport(nameof(ListEntitiesAsync)); + + OrchestrationQuery innerQuery = new OrchestrationQuery() + { + FetchInputsAndOutputs = query.FetchState, + ContinuationToken = query.ContinuationToken, + CreatedTimeFrom = query.LastOperationFrom, + CreatedTimeTo = query.LastOperationTo, + InstanceIdPrefix = "@", + PageSize = query.PageSize, + RuntimeStatus = null, + TaskHubNames = query.TaskHubNames, + }; + + bool unsatisfiable = false; + + void ApplyPrefixConjunction(string prefix) + { + if (innerQuery.InstanceIdPrefix.Length >= prefix.Length) + { + unsatisfiable = unsatisfiable || !innerQuery.InstanceIdPrefix.StartsWith(prefix); + } + else + { + unsatisfiable = unsatisfiable || !prefix.StartsWith(innerQuery.InstanceIdPrefix); + innerQuery.InstanceIdPrefix = prefix; + } + } + + if (query.InstanceIdPrefix != null) + { + ApplyPrefixConjunction(query.InstanceIdPrefix); + } + if (query.EntityName != null) + { + ApplyPrefixConjunction(EntityId.GetSchedulerIdPrefixFromEntityName(query.EntityName)); + } + + if (unsatisfiable) + { + return new QueryResult() + { + Entities = new List(), + ContinuationToken = null, + }; + } + + Stopwatch stopwatch = new Stopwatch(); + stopwatch.Start(); + QueryResult entityResult = new QueryResult() + { + Entities = new List(), + ContinuationToken = innerQuery.ContinuationToken, + }; + + do + { + var result = await queryClient.GetOrchestrationWithQueryAsync(innerQuery, cancellationToken).ConfigureAwait(false); + entityResult.Entities.AddRange(result.OrchestrationState + .Select(ConvertStatusResult) + .Where(status => status != null)); + entityResult.ContinuationToken = innerQuery.ContinuationToken = result.ContinuationToken; + } + while ( // run multiple queries if no records are found, but never in excess of 100ms + entityResult.ContinuationToken != null + && !entityResult.Entities.Any() + && stopwatch.ElapsedMilliseconds <= 100 + && !cancellationToken.IsCancellationRequested); + + return entityResult; + + EntityStatus ConvertStatusResult(OrchestrationState orchestrationState) + { + string state = null; + bool hasState = false; + + if (query.FetchState && orchestrationState.Input != null) + { + hasState = ClientEntityContext.TryGetEntityStateFromSerializedSchedulerState(orchestrationState.Input, out state); + } + else if (orchestrationState.Status != null && orchestrationState.Status != "null") + { + var entityStatus = new DurableTask.Core.Entities.StateFormat.EntityStatus(); + JsonConvert.PopulateObject(orchestrationState.Status, entityStatus, Serializer.InternalSerializerSettings); + hasState = entityStatus.EntityExists; + } + + if (hasState || query.IncludeDeleted) + { + return new EntityStatus() + { + EntityId = EntityId.FromString(orchestrationState.OrchestrationInstance.InstanceId), + LastOperationTime = orchestrationState.CreatedTime, + State = state, + }; + } + else + { + return null; + } + } + } + + /// + /// Query condition for searching the status of entity instances. + /// + public class Query + { + /// + /// If not null, return only entities whose name matches this name. + /// + public string EntityName { get; set; } + + /// + /// If not null, return only entities whose instance id starts with this prefix. + /// + public string InstanceIdPrefix { get; set; } + + /// + /// If not null, return only entity instances which had operations after this DateTime. + /// + public DateTime? LastOperationFrom { get; set; } + + /// + /// If not null, return only entity instances which had operations before this DateTime. + /// + public DateTime? LastOperationTo { get; set; } + + /// + /// If not null, return only entity instances from task hubs whose name is in this list. + /// + public ICollection TaskHubNames { get; set; } + + /// + /// Number of records per one request. The default value is 100. + /// + /// + /// Requests may return fewer records than the specified page size, even if there are more records. + /// Always check the continuation token to determine whether there are more records. + /// + public int PageSize { get; set; } = 100; + + /// + /// ContinuationToken of the pager. + /// + public string ContinuationToken { get; set; } + + /// + /// Determines whether the query results include the state of the entity. + /// + public bool FetchState { get; set; } = false; + + /// + /// Determines whether the results may include entities that currently have no state (such as deleted entities). + /// + /// The effects of this vary by backend. Some backends do not retain deleted entities, so this parameter is irrelevant in that situation. + public bool IncludeDeleted { get; set; } = false; + } + + /// + /// A partial result of an entity status query. + /// + public class QueryResult + { + /// + /// Gets or sets a collection of statuses of entity instances matching the query description. + /// + /// A collection of entity instance status values. + public List Entities { get; set; } + + /// + /// Gets or sets a token that can be used to resume the query with data not already returned by this query. + /// + /// A server-generated continuation token or null if there are no further continuations. + public string ContinuationToken { get; set; } + } + + /// + /// The status of an entity, as returned by entity queries. + /// + public class EntityStatus + { + /// + /// The EntityId of the queried entity instance. + /// + /// + /// The unique EntityId of the instance. + /// + public EntityId EntityId { get; set; } + + /// + /// The time of the last operation processed by the entity instance. + /// + /// + /// The last operation time in UTC. + /// + public DateTime LastOperationTime { get; set; } + + /// + /// The current state of the entity instance, or null if states were not fetched or the entity has no state. + /// + public string State { get; set; } + } + + /// + /// Removes empty entities from storage and releases orphaned locks. + /// + /// An entity is considered empty, and is removed, if it has no state, is not locked, and has + /// been idle for more than minutes. + /// Locks are considered orphaned, and are released, if the orchestration that holds them is not in state . This + /// should not happen under normal circumstances, but can occur if the orchestration instance holding the lock + /// exhibits replay nondeterminism failures, or if it is explicitly purged. + /// Whether to remove empty entities. + /// Whether to release orphaned locks. + /// Cancellation token that can be used to cancel the operation. + /// A task that completes when the operation is finished. + public async Task CleanEntityStorageAsync(bool removeEmptyEntities, bool releaseOrphanedLocks, CancellationToken cancellationToken) + { + this.CheckEntitySupport(nameof(CleanEntityStorageAsync)); + this.CheckQuerySupport(nameof(CleanEntityStorageAsync)); + + if (removeEmptyEntities) + { + this.CheckPurgeSupport(nameof(CleanEntityStorageAsync)); + } + + DateTime now = DateTime.UtcNow; + CleanEntityStorageResult finalResult = default; + + var query = new OrchestrationQuery() + { + InstanceIdPrefix = "@", + FetchInputsAndOutputs = false, + }; + + // list all entities (without fetching the input) and for each one that requires action, + // perform that action. Waits for all actions to finish after each page. + do + { + var page = await this.queryClient.GetOrchestrationWithQueryAsync(query, cancellationToken); + + List tasks = new List(); + foreach (var state in page.OrchestrationState) + { + var status = new DurableTask.Core.Entities.StateFormat.EntityStatus(); + JsonConvert.PopulateObject(state.Status, status, Serializer.InternalSerializerSettings); + + if (releaseOrphanedLocks && status.LockedBy != null) + { + tasks.Add(CheckForOrphanedLockAndFixIt(state.OrchestrationInstance.InstanceId, status.LockedBy)); + } + + if (removeEmptyEntities) + { + bool isEmptyEntity = !status.EntityExists && status.LockedBy == null && status.QueueSize == 0; + bool safeToRemoveWithoutBreakingMessageSorterLogic = (this.backendProperties.EntityMessageReorderWindow == TimeSpan.Zero) ? + true : (now - state.LastUpdatedTime > this.backendProperties.EntityMessageReorderWindow); + if (isEmptyEntity && safeToRemoveWithoutBreakingMessageSorterLogic) + { + tasks.Add(DeleteIdleOrchestrationEntity(state)); + } + } + } + + async Task DeleteIdleOrchestrationEntity(OrchestrationState state) + { + var purgeResult = await this.purgeClient.PurgeInstanceStateAsync(state.OrchestrationInstance.InstanceId); + Interlocked.Add(ref finalResult.NumberOfEmptyEntitiesRemoved, purgeResult.DeletedInstanceCount); + } + + async Task CheckForOrphanedLockAndFixIt(string instanceId, string lockOwner) + { + bool lockOwnerIsStillRunning = false; + + IList stateList = await this.ServiceClient.GetOrchestrationStateAsync(lockOwner, allExecutions: false); + OrchestrationState state = stateList?.FirstOrDefault(); + if (state != null) + { + lockOwnerIsStillRunning = + (state.OrchestrationStatus == OrchestrationStatus.Running + || state.OrchestrationStatus == OrchestrationStatus.Suspended); + } + + if (!lockOwnerIsStillRunning) + { + // the owner is not a running orchestration. Send a lock release. + OrchestrationInstance targetInstance = new OrchestrationInstance() + { + InstanceId = instanceId, + }; + + var eventToSend = ClientEntityContext.EmitUnlockForOrphanedLock(targetInstance, lockOwner); + + string serializedEventContent = this.messageDataConverter.Serialize(eventToSend.EventContent); + + var eventRaisedEvent = new EventRaisedEvent(-1, serializedEventContent) + { + Name = eventToSend.EventName + }; + + var taskMessage = new TaskMessage + { + OrchestrationInstance = targetInstance, + Event = eventRaisedEvent, + }; + + this.logHelper.RaisingEvent(targetInstance, eventRaisedEvent); + + await this.ServiceClient.SendTaskOrchestrationMessageAsync(taskMessage); + + Interlocked.Increment(ref finalResult.NumberOfOrphanedLocksRemoved); + } + } + + await Task.WhenAll(tasks); + query.ContinuationToken = page.ContinuationToken; + } + while (query.ContinuationToken != null); + + return finalResult; + } + + /// + /// The result of a clean entity storage operation. + /// + public struct CleanEntityStorageResult + { + /// + /// The number of orphaned locks that were removed. + /// + public int NumberOfOrphanedLocksRemoved; + + /// + /// The number of entities whose metadata was removed from storage. + /// + public int NumberOfEmptyEntitiesRemoved; + } + } +} diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationAction.cs new file mode 100644 index 000000000..467cecda4 --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationAction.cs @@ -0,0 +1,29 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities.OperationFormat +{ + using Newtonsoft.Json; + + /// + /// Defines a set of base properties for an operator action. + /// + [JsonConverter(typeof(OperationActionConverter))] + public abstract class OperationAction + { + /// + /// The type of the orchestrator action. + /// + public abstract OperationActionType OperationActionType { get; } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationActionConverter.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationActionConverter.cs new file mode 100644 index 000000000..e8081ad06 --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationActionConverter.cs @@ -0,0 +1,41 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Entities.OperationFormat +{ + using System; + using Newtonsoft.Json.Linq; + using DurableTask.Core.Serializing; + + internal class OperationActionConverter : JsonCreationConverter + { + protected override OperationAction CreateObject(Type objectType, JObject jObject) + { + if (jObject.TryGetValue("OperationActionType", StringComparison.OrdinalIgnoreCase, out JToken actionType)) + { + var type = (OperationActionType)int.Parse((string)actionType); + switch (type) + { + case OperationActionType.SendSignal: + return new SendSignalOperationAction(); + case OperationActionType.StartNewOrchestration: + return new StartNewOrchestrationOperationAction(); + default: + throw new NotSupportedException("Unrecognized action type."); + } + } + + throw new NotSupportedException("Action Type not provided."); + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationActionType.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationActionType.cs new file mode 100644 index 000000000..d0c2dc177 --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationActionType.cs @@ -0,0 +1,31 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Entities.OperationFormat +{ + /// + /// Enumeration of entity operation actions. + /// + public enum OperationActionType + { + /// + /// A signal was sent to an entity + /// + SendSignal, + + /// + /// A new fire-and-forget orchestration was started + /// + StartNewOrchestration, + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationBatchRequest.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationBatchRequest.cs new file mode 100644 index 000000000..69515b184 --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationBatchRequest.cs @@ -0,0 +1,44 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities.OperationFormat +{ + using Newtonsoft.Json; + using System; + using System.Collections.Generic; + using System.Text; + + /// + /// A request for execution of a batch of operations on an entity. + /// + public class OperationBatchRequest + { + // NOTE: Actions must be serializable by a variety of different serializer types to support out-of-process execution. + // To ensure maximum compatibility, all properties should be public and settable by default. + + /// + /// The instance id for this entity. + /// + public string? InstanceId { get; set; } + + /// + /// The current state of the entity, or null if the entity does not exist. + /// + public string? EntityState { get; set; } + + /// + /// The list of operations to be performed on the entity. + /// + public List? Operations { get; set; } + } +} diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationBatchResult.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationBatchResult.cs new file mode 100644 index 000000000..9a8e9deee --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationBatchResult.cs @@ -0,0 +1,48 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities.OperationFormat +{ + using Newtonsoft.Json; + using System; + using System.Collections.Generic; + using System.Text; + + /// + /// The results of executing a batch of operations on the entity out of process. + /// + public class OperationBatchResult + { + // NOTE: Actions must be serializable by a variety of different serializer types to support out-of-process execution. + // To ensure maximum compatibility, all properties should be public and settable by default. + + /// + /// The results of executing the operations in the batch. The length of this list must match + /// the size of the batch if all messages were processed; In particular, all execution errors must be reported as a result. + /// However, this list of results can be shorter than the list of operations if + /// some suffix of the operation list was skipped, e.g. due to shutdown, send throttling, or timeouts. + /// + public List? Results { get; set; } + + /// + /// The list of actions (outgoing messages) performed while executing the operations in the batch. Can be empty. + /// + public List? Actions { get; set; } + + /// + /// The state of the entity after executing the batch, + /// or null if the entity has no state (e.g. if it has been deleted). + /// + public string? EntityState { get; set; } + } +} diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs new file mode 100644 index 000000000..09405cb41 --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs @@ -0,0 +1,43 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities.OperationFormat +{ + using System; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + /// + /// A request message sent to an entity when calling or signaling the entity. + /// + public class OperationRequest + { + // NOTE: Actions must be serializable by a variety of different serializer types to support out-of-process execution. + // To ensure maximum compatibility, all properties should be public and settable by default. + + /// + /// The name of the operation. + /// + public string? Operation { get; set; } + + /// + /// The unique GUID of the operation. + /// + public Guid Id { get; set; } + + /// + /// The input for the operation. Can be null if no input was given. + /// + public string? Input { get; set; } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs new file mode 100644 index 000000000..2fdee8733 --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs @@ -0,0 +1,46 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities.OperationFormat +{ + using System; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + /// + /// A response message sent by an entity to a caller after it executes an operation. + /// + public class OperationResult + { + // NOTE: Actions must be serializable by a variety of different serializer types to support out-of-process execution. + // To ensure maximum compatibility, all properties should be public and settable by default. + + /// + /// The serialized result returned by the operation. Can be null, if the operation returned no result. + /// May contain error details, such as a serialized exception, if is not null. + /// + public string? Result { get; set; } + + /// + /// If non-null, this string indicates that this operation did not successfully complete. + /// The actual content and its interpretation varies depending on the SDK used. + /// + public string? ErrorMessage { get; set; } + + /// + /// A structured language-independent representation of the error. Whether this field is present + /// depends on which SDK is used, and on configuration settings. + /// + public FailureDetails? FailureDetails { get; set; } + } +} diff --git a/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs new file mode 100644 index 000000000..800613cad --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs @@ -0,0 +1,51 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities.OperationFormat +{ + using Newtonsoft.Json; + using System; + using System.Collections.Generic; + + /// + /// Operation action for sending a signal. + /// + public class SendSignalOperationAction : OperationAction + { + /// + public override OperationActionType OperationActionType => OperationActionType.SendSignal; + + // NOTE: Actions must be serializable by a variety of different serializer types to support out-of-process execution. + // To ensure maximum compatibility, all properties should be public and settable by default. + + /// + /// The destination entity for the signal. + /// + public string? InstanceId { get; set; } + + /// + /// The name of the operation being signaled. + /// + public string? Name { get; set; } + + /// + /// The input of the operation being signaled. + /// + public string? Input { get; set; } + + /// + /// Optionally, a scheduled delivery time for the signal. + /// + public DateTime? ScheduledTime { get; set; } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs new file mode 100644 index 000000000..27b486d08 --- /dev/null +++ b/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs @@ -0,0 +1,55 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Entities.OperationFormat +{ + using DurableTask.Core.Entities; + using System.Collections.Generic; + + /// + /// Orchestrator action for creating sub-orchestrations. + /// + public class StartNewOrchestrationOperationAction : OperationAction + { + /// + public override OperationActionType OperationActionType => OperationActionType.StartNewOrchestration; + + // NOTE: Actions must be serializable by a variety of different serializer types to support out-of-process execution. + // To ensure maximum compatibility, all properties should be public and settable by default. + + /// + /// The name of the sub-orchestrator to start. + /// + public string? Name { get; set; } + + /// + /// The version of the sub-orchestrator to start. + /// + public string? Version { get; set; } + + /// + /// The instance ID of the created sub-orchestration. + /// + public string? InstanceId { get; set; } + + /// + /// The input of the sub-orchestration. + /// + public string? Input { get; set; } + + /// + /// Tags to be applied to the sub-orchestration. + /// + public IDictionary? Tags { get; set; } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs new file mode 100644 index 000000000..fd516dc4e --- /dev/null +++ b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs @@ -0,0 +1,463 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +namespace DurableTask.Core.Entities +{ + using DurableTask.Core.Common; + using DurableTask.Core.Entities.EventFormat; + using DurableTask.Core.Entities.OperationFormat; + using DurableTask.Core.Entities.StateFormat; + using DurableTask.Core.History; + using DurableTask.Core.Serializing; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + + /// + /// Tracks the entity-related state of an orchestration. + /// Tracks and validates the synchronization state. + /// + public class OrchestrationEntityContext + { + private readonly string instanceId; + private readonly string executionId; + private readonly OrchestrationContext innerContext; + private readonly Dictionary pending; + private readonly MessageSorter messageSorter; + + private Guid? criticalSectionId; + private EntityId[] criticalSectionLocks; + private bool lockAcquisitionPending; + private HashSet availableLocks; + + /// + /// Constructs an OrchestrationEntityContext. + /// + /// The instance id. + /// The execution id. + /// The inner context. + public OrchestrationEntityContext( + string instanceId, + string executionId, + OrchestrationContext innerContext) + { + this.instanceId = instanceId; + this.executionId = executionId; + this.innerContext = innerContext; + this.pending = new Dictionary(); + this.messageSorter = new MessageSorter(); + } + + /// + /// Whether this orchestration is currently inside a critical section. + /// + public bool IsInsideCriticalSection => this.criticalSectionId != null; + + /// + /// Enumerate all the entities that are available for calling from within a critical section. + /// This set contains all the entities that were locked prior to entering the critical section, + /// and for which there is not currently an operation call pending. + /// + /// An enumeration of all the currently available entities. + public IEnumerable<(string entityName, string entityKey)> GetAvailableEntities() + { + if (this.IsInsideCriticalSection) + { + foreach(var e in this.availableLocks) + { + yield return (e.Name, e.Key); + } + } + } + + /// + /// Check that a suborchestration is a valid transition in the current state. + /// + /// The error message, if it is not valid, or null otherwise + /// whether the transition is valid + public bool ValidateSuborchestrationTransition(out string errorMessage) + { + if (this.IsInsideCriticalSection) + { + errorMessage = "While holding locks, cannot call suborchestrators."; + return false; + } + + errorMessage = null; + return true; + } + + /// + /// Check that acquire is a valid transition in the current state. + /// + /// Whether this is a signal or a call. + /// The target instance id. + /// The error message, if it is not valid, or null otherwise + /// whether the transition is valid + public bool ValidateOperationTransition(string targetInstanceId, bool oneWay, out string errorMessage) + { + if (this.IsInsideCriticalSection) + { + var lockToUse = EntityId.FromString(targetInstanceId); + if (oneWay) + { + if (this.criticalSectionLocks.Contains(lockToUse)) + { + errorMessage = "Must not signal a locked entity from a critical section."; + return false; + } + } + else + { + if (!this.availableLocks.Remove(lockToUse)) + { + if (this.lockAcquisitionPending) + { + errorMessage = "Must await the completion of the lock request prior to calling any entity."; + return false; + } + if (this.criticalSectionLocks.Contains(lockToUse)) + { + errorMessage = "Must not call an entity from a critical section while a prior call to the same entity is still pending."; + return false; + } + else + { + errorMessage = "Must not call an entity from a critical section if it is not one of the locked entities."; + return false; + } + } + } + } + + errorMessage = null; + return true; + } + + /// + /// Check that acquire is a valid transition in the current state. + /// + /// The error message, if it is not valid, or null otherwise + /// whether the transition is valid + public bool ValidateAcquireTransition(out string errorMessage) + { + if (this.IsInsideCriticalSection) + { + errorMessage = "Must not enter another critical section from within a critical section."; + return false; + } + + errorMessage = null; + return true; + } + + /// + /// Called after an operation call within a critical section completes. + /// + /// + public void RecoverLockAfterCall(string targetInstanceId) + { + if (this.IsInsideCriticalSection) + { + var lockToUse = EntityId.FromString(targetInstanceId); + this.availableLocks.Add(lockToUse); + } + } + + /// + /// Get release messages for all locks in the critical section, and release them + /// + public IEnumerable EmitLockReleaseMessages() + { + if (this.IsInsideCriticalSection) + { + var message = new ReleaseMessage() + { + ParentInstanceId = instanceId, + Id = this.criticalSectionId.Value.ToString(), + }; + + foreach (var entityId in this.criticalSectionLocks) + { + var instance = new OrchestrationInstance() { InstanceId = entityId.ToString() }; + var jmessage = JObject.FromObject(message, Serializer.InternalSerializer); + yield return new EventToSend(EntityMessageEventNames.ReleaseMessageEventName, jmessage, instance); + } + + this.criticalSectionLocks = null; + this.availableLocks = null; + this.criticalSectionId = null; + } + } + + /// + /// Creates a request message to be sent to an entity. + /// + /// The target entity. + /// The name of the operation. + /// If true, this is a signal, otherwise it is a call. + /// A unique identifier for this request. + /// A time for which to schedule the delivery, or null if this is not a scheduled message + /// The operation input + /// The event to send. + public EventToSend EmitRequestMessage( + OrchestrationInstance target, + string operationName, + bool oneWay, + Guid operationId, + (DateTime original, DateTime capped)? scheduledTimeUtc, + string input) + { + var request = new RequestMessage() + { + ParentInstanceId = this.instanceId, + ParentExecutionId = this.executionId, + Id = operationId, + IsSignal = oneWay, + Operation = operationName, + ScheduledTime = scheduledTimeUtc?.original, + Input = input, + }; + + this.AdjustOutgoingMessage(target.InstanceId, request, scheduledTimeUtc?.capped, out string eventName); + + // we pre-serialize to JObject so we can avoid exposure to application-specific serialization settings + var jrequest = JObject.FromObject(request, Serializer.InternalSerializer); + + return new EventToSend(eventName, jrequest, target); + } + + /// + /// Creates an acquire message to be sent to an entity. + /// + /// A unique request id. + /// All the entities that are to be acquired. + /// The event to send. + public EventToSend EmitAcquireMessage(Guid lockRequestId, EntityId[] entities) + { + // All the entities in entity[] need to be locked, but to avoid deadlock, the locks have to be acquired + // sequentially, in order. So, we send the lock request to the first entity; when the first lock + // is granted by the first entity, the first entity will forward the lock request to the second entity, + // and so on; after the last entity grants the last lock, a response is sent back here. + + // acquire the locks in a globally fixed order to avoid deadlocks + Array.Sort(entities); + + // remove duplicates if necessary. Probably quite rare, so no need to optimize more. + for (int i = 0; i < entities.Length - 1; i++) + { + if (entities[i].Equals(entities[i + 1])) + { + entities = entities.Distinct().ToArray(); + break; + } + } + + // send lock request to first entity in the lock set + var target = new OrchestrationInstance() { InstanceId = entities[0].ToString() }; + var request = new RequestMessage() + { + Id = lockRequestId, + ParentInstanceId = this.instanceId, + ParentExecutionId = this.executionId, + LockSet = entities, + Position = 0, + }; + + this.criticalSectionId = lockRequestId; + this.criticalSectionLocks = entities; + this.lockAcquisitionPending = true; + + this.AdjustOutgoingMessage(target.InstanceId, request, null, out string eventName); + + // we pre-serialize to JObject so we can avoid exposure to application-specific serialization settings + var jrequest = JObject.FromObject(request, Serializer.InternalSerializer); + + return new EventToSend(eventName, jrequest, target); + } + + /// + /// Called when a response to the acquire message is received from the last entity. + /// + /// The result returned. + /// The guid for the lock operation + public void CompleteAcquire(OperationResult result, Guid criticalSectionId) + { + this.availableLocks = new HashSet(this.criticalSectionLocks); + this.lockAcquisitionPending = false; + } + + internal void AdjustOutgoingMessage(string instanceId, RequestMessage requestMessage, DateTime? cappedTime, out string eventName) + { + if (requestMessage.ScheduledTime.HasValue) + { + eventName = EntityMessageEventNames.ScheduledRequestMessageEventName(cappedTime.Value); + } + else + { + this.messageSorter.LabelOutgoingMessage( + requestMessage, + instanceId, + this.innerContext.CurrentUtcDateTime, + this.innerContext.EntityBackendProperties.EntityMessageReorderWindow); + + eventName = EntityMessageEventNames.RequestMessageEventName; + } + } + + /// + /// Extracts the operation result from an event that represents an entity response. + /// + /// + /// + public OperationResult DeserializeEntityResponseEvent(string eventContent) + { + var responseMessage = new ResponseMessage(); + + // for compatibility, we deserialize in a way that is resilient to any typename presence/absence/mismatch + try + { + // restore the scheduler state from the input + JsonConvert.PopulateObject(eventContent, responseMessage, Serializer.InternalSerializerSettings); + } + catch (Exception exception) + { + throw new EntitySchedulerException("Failed to deserialize entity response.", exception); + } + + return new OperationResult() + { + Result = responseMessage.Result, + ErrorMessage = responseMessage.ErrorMessage, + FailureDetails = responseMessage.FailureDetails, + }; + } + + + private interface IEntityResponseContinuation + { + void DeliverResult(OperationResult operationResult, EventRaisedEvent eventRaisedEvent, TaskOrchestrationContext taskOrchestrationContext); + } + + private class OperationContinuation : TaskCompletionSource, IEntityResponseContinuation + { + readonly private int taskId; + readonly private string instanceId; + + public OperationContinuation(int taskId, string instanceId) + { + this.taskId = taskId; + this.instanceId = instanceId; + } + + public void DeliverResult(OperationResult operationResult, EventRaisedEvent eventRaisedEvent, TaskOrchestrationContext taskOrchestrationContext) + { + if (taskOrchestrationContext.EntityContext.IsInsideCriticalSection) + { + // the lock is available again now that the entity call returned + taskOrchestrationContext.EntityContext.RecoverLockAfterCall(this.instanceId); + } + + if (operationResult.ErrorMessage == null) + { + taskOrchestrationContext.HandleEntityOperationCompletedEvent(eventRaisedEvent, this.taskId, this.instanceId, operationResult, this); + } + else + { + taskOrchestrationContext.HandleEntityOperationFailedEvent(eventRaisedEvent, this.taskId, this.instanceId, operationResult, this); + } + } + } + + private class LockAcquisitionContinuation : TaskCompletionSource, IEntityResponseContinuation + { + readonly private Guid criticalSectionId; + readonly private int taskId; + + public LockAcquisitionContinuation(Guid lockRequestId, int taskId) + { + this.criticalSectionId = lockRequestId; + this.taskId = taskId; + } + + public void DeliverResult(OperationResult operationResult, EventRaisedEvent eventRaisedEvent, TaskOrchestrationContext taskOrchestrationContext) + { + taskOrchestrationContext.EntityContext.CompleteAcquire(operationResult, this.criticalSectionId); + this.SetResult(new LockReleaser(taskOrchestrationContext, criticalSectionId)); + } + } + + private class LockReleaser : IDisposable + { + private readonly TaskOrchestrationContext context; + private readonly Guid criticalSectionId; + + public LockReleaser(TaskOrchestrationContext context, Guid criticalSectionId) + { + this.context = context; + this.criticalSectionId = criticalSectionId; + } + + public void Dispose() + { + this.context.ExitCriticalSection(); + } + } + + internal Task WaitForOperationResponseAsync(Guid operationId, int taskId, string instanceId) + { + var promise = new OperationContinuation(taskId, instanceId); + lock (this.pending) + { + this.pending.Add(operationId, promise); + } + return promise.Task; + } + + internal Task WaitForLockResponseAsync(Guid criticalSectionId, int taskId) + { + var promise = new LockAcquisitionContinuation(criticalSectionId, taskId); + lock (this.pending) + { + this.pending.Add(criticalSectionId, promise); + } + return promise.Task; + } + + internal bool HandledAsEntityResponse(EventRaisedEvent evt, TaskOrchestrationContext taskOrchestrationContext) + { + Guid operationId; + if (!Guid.TryParse(evt.Name, out operationId)) + { + return false; + } + + IEntityResponseContinuation completionSource; + lock (this.pending) + { + if (!this.pending.TryGetValue(operationId, out completionSource)) + { + return false; + } + this.pending.Remove(operationId); + } + + var operationResult = this.DeserializeEntityResponseEvent(evt.Input); + completionSource.DeliverResult(operationResult, evt, taskOrchestrationContext); + + return true; + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/Serializer.cs b/src/DurableTask.Core/Entities/Serializer.cs new file mode 100644 index 000000000..19a662fa4 --- /dev/null +++ b/src/DurableTask.Core/Entities/Serializer.cs @@ -0,0 +1,34 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using DurableTask.Core.Serializing; +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Text; + +namespace DurableTask.Core.Entities +{ + internal static class Serializer + { + /// + /// This serializer is used exclusively for internally defined data structures and cannot be customized by user. + /// This is intentional, to avoid problems caused by our unability to control the exact format. + /// For example, including typenames can cause compatibility problems if the type name is later changed. + /// + public static JsonSerializer InternalSerializer = JsonSerializer.Create(InternalSerializerSettings); + + public static JsonSerializerSettings InternalSerializerSettings + = new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.None }; + } +} diff --git a/src/DurableTask.Core/Entities/StateFormat/EntityStatus.cs b/src/DurableTask.Core/Entities/StateFormat/EntityStatus.cs new file mode 100644 index 000000000..19c74dd3c --- /dev/null +++ b/src/DurableTask.Core/Entities/StateFormat/EntityStatus.cs @@ -0,0 +1,44 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Entities.StateFormat +{ + using System.Runtime.Serialization; + using Newtonsoft.Json; + + /// + /// Information about the current status of an entity. Excludes potentially large data + /// (such as the entity state, or the contents of the queue) so it can always be read with low latency. + /// + [DataContract] + public class EntityStatus + { + /// + /// Whether this entity exists or not. + /// + [DataMember(Name = "entityExists", EmitDefaultValue = false)] + public bool EntityExists { get; set; } + + /// + /// The size of the queue, i.e. the number of operations that are waiting for the current operation to complete. + /// + [DataMember(Name = "queueSize", EmitDefaultValue = false)] + public int QueueSize { get; set; } + + /// + /// The instance id of the orchestration that currently holds the lock of this entity. + /// + [DataMember(Name = "lockedBy", EmitDefaultValue = false)] + public string LockedBy { get; set; } + } +} diff --git a/src/DurableTask.Core/Entities/StateFormat/MessageSorter.cs b/src/DurableTask.Core/Entities/StateFormat/MessageSorter.cs new file mode 100644 index 000000000..1d824307f --- /dev/null +++ b/src/DurableTask.Core/Entities/StateFormat/MessageSorter.cs @@ -0,0 +1,274 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Entities.StateFormat +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Runtime.Serialization; + using DurableTask.Core.Entities.EventFormat; + using Newtonsoft.Json; + + /// + /// provides message ordering and deduplication of request messages (operations or lock requests) + /// that are sent to entities, from other entities, or from orchestrations. + /// + [DataContract] + internal class MessageSorter + { + // don't update the reorder window too often since the garbage collection incurs some overhead. + private static readonly TimeSpan MinIntervalBetweenCollections = TimeSpan.FromSeconds(10); + + [DataMember(EmitDefaultValue = false)] + public Dictionary LastSentToInstance { get; set; } + + [DataMember(EmitDefaultValue = false)] + public Dictionary ReceivedFromInstance { get; set; } + + [DataMember(EmitDefaultValue = false)] + public DateTime ReceiveHorizon { get; set; } + + [DataMember(EmitDefaultValue = false)] + public DateTime SendHorizon { get; set; } + + /// + /// Used for testing purposes. + /// + [IgnoreDataMember] + internal int NumberBufferedRequests => + ReceivedFromInstance?.Select(kvp => kvp.Value.Buffered?.Count ?? 0).Sum() ?? 0; + + /// + /// Called on the sending side, to fill in timestamp and predecessor fields. + /// + public void LabelOutgoingMessage(RequestMessage message, string destination, DateTime now, TimeSpan reorderWindow) + { + if (reorderWindow.Ticks == 0) + { + return; // we are not doing any message sorting. + } + + DateTime timestamp = now; + + if (SendHorizon + reorderWindow + MinIntervalBetweenCollections < now) + { + SendHorizon = now - reorderWindow; + + // clean out send clocks that are past the reorder window + + if (LastSentToInstance != null) + { + List expired = new List(); + + foreach (var kvp in LastSentToInstance) + { + if (kvp.Value < SendHorizon) + { + expired.Add(kvp.Key); + } + } + + foreach (var t in expired) + { + LastSentToInstance.Remove(t); + } + } + } + + if (LastSentToInstance == null) + { + LastSentToInstance = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + else if (LastSentToInstance.TryGetValue(destination, out var last)) + { + message.Predecessor = last; + + // ensure timestamps are monotonic even if system clock is not + if (timestamp <= last) + { + timestamp = new DateTime(last.Ticks + 1); + } + } + + message.Timestamp = timestamp; + LastSentToInstance[destination] = timestamp; + } + + /// + /// Called on the receiving side, to reorder and deduplicate within the window. + /// + public IEnumerable ReceiveInOrder(RequestMessage message, TimeSpan reorderWindow) + { + // messages sent from clients and forwarded lock messages are not participating in the sorting. + if (reorderWindow.Ticks == 0 || message.ParentInstanceId == null || message.Position > 0) + { + // Just pass the message through. + yield return message; + yield break; + } + + // advance the horizon based on the latest timestamp + if (ReceiveHorizon + reorderWindow + MinIntervalBetweenCollections < message.Timestamp) + { + ReceiveHorizon = message.Timestamp - reorderWindow; + + // deliver any messages that were held in the receive buffers + // but are now past the reorder window + + List emptyReceiveBuffers = new List(); + + if (ReceivedFromInstance != null) + { + foreach (var kvp in ReceivedFromInstance) + { + if (kvp.Value.Last < ReceiveHorizon) + { + kvp.Value.Last = DateTime.MinValue; + } + + while (TryDeliverNextMessage(kvp.Value, out var next)) + { + yield return next; + } + + if (kvp.Value.Last == DateTime.MinValue + && (kvp.Value.Buffered == null || kvp.Value.Buffered.Count == 0)) + { + emptyReceiveBuffers.Add(kvp.Key); + } + } + + foreach (var t in emptyReceiveBuffers) + { + ReceivedFromInstance.Remove(t); + } + + if (ReceivedFromInstance.Count == 0) + { + ReceivedFromInstance = null; + } + } + } + + // Messages older than the reorder window are not participating. + if (message.Timestamp < ReceiveHorizon) + { + // Just pass the message through. + yield return message; + yield break; + } + + ReceiveBuffer receiveBuffer; + + if (ReceivedFromInstance == null) + { + ReceivedFromInstance = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + if (!ReceivedFromInstance.TryGetValue(message.ParentInstanceId, out receiveBuffer)) + { + ReceivedFromInstance[message.ParentInstanceId] = receiveBuffer = new ReceiveBuffer() + { + ExecutionId = message.ParentExecutionId, + }; + } + else if (receiveBuffer.ExecutionId != message.ParentExecutionId) + { + // this message is from a new execution; release all buffered messages and start over + if (receiveBuffer.Buffered != null) + { + foreach (var kvp in receiveBuffer.Buffered) + { + yield return kvp.Value; + } + + receiveBuffer.Buffered.Clear(); + } + + receiveBuffer.Last = DateTime.MinValue; + receiveBuffer.ExecutionId = message.ParentExecutionId; + } + + if (message.Timestamp <= receiveBuffer.Last) + { + // This message was already delivered, it's a duplicate + yield break; + } + + if (message.Predecessor > receiveBuffer.Last + && message.Predecessor >= ReceiveHorizon) + { + // this message is waiting for a non-delivered predecessor in the window, buffer it + if (receiveBuffer.Buffered == null) + { + receiveBuffer.Buffered = new SortedDictionary(); + } + + receiveBuffer.Buffered[message.Timestamp] = message; + } + else + { + yield return message; + + receiveBuffer.Last = message.Timestamp >= ReceiveHorizon ? message.Timestamp : DateTime.MinValue; + + while (TryDeliverNextMessage(receiveBuffer, out var next)) + { + yield return next; + } + } + } + + private bool TryDeliverNextMessage(ReceiveBuffer buffer, out RequestMessage message) + { + if (buffer.Buffered != null) + { + using (var e = buffer.Buffered.GetEnumerator()) + { + if (e.MoveNext()) + { + var pred = e.Current.Value.Predecessor; + + if (pred <= buffer.Last || pred < ReceiveHorizon) + { + message = e.Current.Value; + + buffer.Last = message.Timestamp >= ReceiveHorizon ? message.Timestamp : DateTime.MinValue; + + buffer.Buffered.Remove(message.Timestamp); + + return true; + } + } + } + } + + message = null; + return false; + } + + [DataContract] + public class ReceiveBuffer + { + [DataMember] + public DateTime Last { get; set; }// last message delivered, or DateTime.Min if none + + [DataMember(EmitDefaultValue = false)] + public string ExecutionId { get; set; } // execution id of last message, if any + + [DataMember(EmitDefaultValue = false)] + public SortedDictionary Buffered { get; set; } + } + } +} diff --git a/src/DurableTask.Core/Entities/StateFormat/SchedulerState.cs b/src/DurableTask.Core/Entities/StateFormat/SchedulerState.cs new file mode 100644 index 000000000..db7997aed --- /dev/null +++ b/src/DurableTask.Core/Entities/StateFormat/SchedulerState.cs @@ -0,0 +1,113 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Entities.StateFormat +{ + using System.Collections.Generic; + using System.Runtime.Serialization; + using DurableTask.Core.Entities.EventFormat; + using Newtonsoft.Json; + + /// + /// The persisted state of an entity scheduler, as handed forward between ContinueAsNew instances. + /// + [DataContract] + internal class SchedulerState + { + /// + /// Whether this entity exists or not. + /// + [DataMember(Name = "exists", EmitDefaultValue = false)] + public bool EntityExists { get; set; } + + /// + /// The last serialized entity state. + /// + [DataMember(Name = "state", EmitDefaultValue = false)] + public string EntityState { get; set; } + + /// + /// The queue of waiting operations, or null if none. + /// + [DataMember(Name = "queue", EmitDefaultValue = false)] + public Queue Queue { get; private set; } + + /// + /// The instance id of the orchestration that currently holds the lock of this entity. + /// + [DataMember(Name = "lockedBy", EmitDefaultValue = false)] + public string LockedBy { get; set; } + + /// + /// Whether processing on this entity is currently suspended. + /// + [DataMember(Name = "suspended", EmitDefaultValue = false)] + public bool Suspended { get; set; } + + /// + /// The metadata used for reordering and deduplication of messages sent to entities. + /// + [DataMember(Name = "sorter", EmitDefaultValue = false)] + public MessageSorter MessageSorter { get; set; } = new MessageSorter(); + + [IgnoreDataMember] + public bool IsEmpty => !EntityExists && (Queue == null || Queue.Count == 0) && LockedBy == null; + + internal void Enqueue(RequestMessage operationMessage) + { + if (Queue == null) + { + Queue = new Queue(); + } + + Queue.Enqueue(operationMessage); + } + + internal void PutBack(Queue messages) + { + if (Queue != null) + { + foreach (var message in Queue) + { + messages.Enqueue(message); + } + } + + Queue = messages; + } + + internal bool MayDequeue() + { + return Queue != null + && Queue.Count > 0 + && (LockedBy == null || LockedBy == Queue.Peek().ParentInstanceId); + } + + internal RequestMessage Dequeue() + { + var result = Queue.Dequeue(); + + if (Queue.Count == 0) + { + Queue = null; + } + + return result; + } + + public override string ToString() + { + return $"exists={EntityExists} queue.count={(Queue != null ? Queue.Count : 0)}"; + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/TaskEntity.cs b/src/DurableTask.Core/Entities/TaskEntity.cs new file mode 100644 index 000000000..d5ed96af1 --- /dev/null +++ b/src/DurableTask.Core/Entities/TaskEntity.cs @@ -0,0 +1,30 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Entities +{ + using System.Threading.Tasks; + using DurableTask.Core.Entities.OperationFormat; + + /// + /// Abstract base class for entities. + /// + /// To implement task entities, use the subclass . + public abstract class TaskEntity + { + /// + /// Internal untyped interface for entity batch operations. + /// + internal abstract Task ExecuteOperationBatchAsync(OperationBatchRequest operations, EntityExecutionOptions options); + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Exceptions/EntityLockingRulesViolationException.cs b/src/DurableTask.Core/Exceptions/EntityLockingRulesViolationException.cs new file mode 100644 index 000000000..35da8a437 --- /dev/null +++ b/src/DurableTask.Core/Exceptions/EntityLockingRulesViolationException.cs @@ -0,0 +1,62 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Exceptions +{ + using System; + using System.Runtime.Serialization; + + /// + /// Indicates an invalid use of entities in critical sections. + /// + [Serializable] + public class EntityLockingRulesViolationException : InvalidOperationException + { + /// + /// Initializes a new instance. + /// + public EntityLockingRulesViolationException() + { + } + + /// + /// Initializes an new instance with a specified error message. + /// + /// The message that describes the error. + public EntityLockingRulesViolationException(string message) + : base(message) + { + } + + /// + /// Initializes an new instance with a specified error message. + /// and a reference to the inner exception that is the cause of this exception. + /// + /// The message that describes the error. + /// The exception that is the cause of the current exception, or a null reference if no inner exception is specified. + public EntityLockingRulesViolationException(string message, Exception innerException) + : base(message, innerException) + { + } + + /// + /// Initializes a new instance with serialized data. + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized object data about the exception being thrown. + /// The System.Runtime.Serialization.StreamingContext that contains contextual information about the source or destination. + protected EntityLockingRulesViolationException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/Exceptions/OperationFailedException.cs b/src/DurableTask.Core/Exceptions/OperationFailedException.cs new file mode 100644 index 000000000..b529c020d --- /dev/null +++ b/src/DurableTask.Core/Exceptions/OperationFailedException.cs @@ -0,0 +1,108 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core.Exceptions +{ + using System; + using System.Runtime.Serialization; + + /// + /// Represents errors experienced while executing an operation on an entity. + /// + [Serializable] + public class OperationFailedException : OrchestrationException + { + /// + /// Initializes an new instance of the OperationFailedException class + /// + public OperationFailedException() + { + } + + /// + /// Initializes an new instance of the OperationFailedException class with a specified error message + /// + /// The message that describes the error. + public OperationFailedException(string reason) + : base(reason) + { + } + + /// + /// Initializes an new instance of the OperationFailedException class with a specified error message + /// and a reference to the inner exception that is the cause of this exception. + /// + /// The message that describes the error. + /// The exception that is the cause of the current exception, or a null reference if no inner exception is specified. + public OperationFailedException(string reason, Exception innerException) + : base(reason, innerException) + { + } + + /// + /// Initializes an new instance of the OperationFailedException class with a specified event id, schedule id, name, version and error message + /// and a reference to the inner exception that is the cause of this exception. + /// + /// EventId of the error. + /// ScheduleId of the error. + /// The instance id of the entity. + /// The operation id of the operation. + /// The message that describes the error. + /// The exception that is the cause of the current exception, or a null reference if no cause is specified. + public OperationFailedException(int eventId, int scheduleId, string instanceId, string operationId, string reason, + Exception cause) + : base(eventId, reason, cause) + { + ScheduleId = scheduleId; + InstanceId = instanceId; + OperationId = operationId; + } + + /// + /// Initializes a new instance of the OperationFailedException class with serialized data. + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized object data about the exception being thrown. + /// The System.Runtime.Serialization.StreamingContext that contains contextual information about the source or destination. + protected OperationFailedException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + ScheduleId = info.GetInt32(nameof(ScheduleId)); + InstanceId = info.GetString(nameof(InstanceId)); + OperationId = info.GetString(nameof(OperationId)); + } + + /// + public override void GetObjectData(SerializationInfo info, StreamingContext context) + { + base.GetObjectData(info, context); + info.AddValue(nameof(ScheduleId), ScheduleId); + info.AddValue(nameof(InstanceId), InstanceId); + info.AddValue(nameof(OperationId), OperationId); + } + + /// + /// Schedule Id of the exception + /// + public int ScheduleId { get; set; } + + /// + /// The instance id of the entity that experienced the failure when executing the operation. + /// + public string InstanceId { get; set; } + + /// + /// The operation id of the operation that experienced the failure. + /// + public string OperationId { get; set; } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/FailureDetails.cs b/src/DurableTask.Core/FailureDetails.cs index b99584833..55bce70d7 100644 --- a/src/DurableTask.Core/FailureDetails.cs +++ b/src/DurableTask.Core/FailureDetails.cs @@ -19,7 +19,7 @@ namespace DurableTask.Core using Newtonsoft.Json; /// - /// Details of an activity or orchestration failure. + /// Details of an activity, orchestration, or entity operation failure. /// [Serializable] public class FailureDetails : IEquatable @@ -51,6 +51,17 @@ public FailureDetails(Exception e) { } + /// + /// Initializes a new instance of the class from an exception object and + /// an explicitly specified inner exception. The specified inner exception replaces any inner exception already present. + /// + /// The exception used to generate the failure details. + /// The inner exception to use. + public FailureDetails(Exception e, Exception innerException) + : this(e.GetType().FullName, GetErrorMessage(e), e.StackTrace, FromException(innerException), false) + { + } + /// /// For testing purposes only: Initializes a new, empty instance of the class. /// diff --git a/src/DurableTask.Core/Logging/EventIds.cs b/src/DurableTask.Core/Logging/EventIds.cs index 963de5f71..2998e5f8d 100644 --- a/src/DurableTask.Core/Logging/EventIds.cs +++ b/src/DurableTask.Core/Logging/EventIds.cs @@ -47,6 +47,10 @@ static class EventIds public const int OrchestrationExecuted = 52; public const int OrchestrationAborted = 53; public const int DiscardingMessage = 54; + public const int EntityBatchExecuting = 55; + public const int EntityBatchExecuted = 56; + public const int EntityLockAcquired = 57; + public const int EntityLockReleased = 58; public const int TaskActivityStarting = 60; public const int TaskActivityCompleted = 61; diff --git a/src/DurableTask.Core/Logging/LogEvents.cs b/src/DurableTask.Core/Logging/LogEvents.cs index a914154a2..184a97b87 100644 --- a/src/DurableTask.Core/Logging/LogEvents.cs +++ b/src/DurableTask.Core/Logging/LogEvents.cs @@ -14,11 +14,16 @@ namespace DurableTask.Core.Logging { using System; + using System.Linq; using System.Text; using DurableTask.Core.Command; using DurableTask.Core.Common; + using DurableTask.Core.Entities; + using DurableTask.Core.Entities.OperationFormat; using DurableTask.Core.History; using Microsoft.Extensions.Logging; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; /// /// This class defines all log events supported by DurableTask.Core. @@ -1177,6 +1182,196 @@ void IEventSourceEvent.WriteEventSource() => Utils.PackageVersion); } + /// + /// Log event representing a task hub worker executing a batch of entity operations. + /// + internal class EntityBatchExecuting : StructuredLogEvent, IEventSourceEvent + { + public EntityBatchExecuting(OperationBatchRequest request) + { + this.InstanceId = request.InstanceId; + this.OperationCount = request.Operations.Count; + this.EntityStateLength = request.EntityState?.Length ?? 0; + } + + [StructuredLogField] + public string InstanceId { get; } + + [StructuredLogField] + public int OperationCount { get; } + + [StructuredLogField] + public int EntityStateLength { get; } + + public override EventId EventId => new EventId( + EventIds.EntityBatchExecuting, + nameof(EventIds.EntityBatchExecuting)); + + public override LogLevel Level => LogLevel.Debug; + + protected override string CreateLogMessage() => + $"{this.InstanceId}: executing batch of {this.OperationCount} operations on entity state of length {this.EntityStateLength}."; + + void IEventSourceEvent.WriteEventSource() => + StructuredEventSource.Log.EntityBatchExecuting( + this.InstanceId, + this.OperationCount, + this.EntityStateLength, + Utils.AppName, + Utils.PackageVersion); + } + + /// + /// Log event representing a task hub worker executed a batch of entity operations. + /// + internal class EntityBatchExecuted : StructuredLogEvent, IEventSourceEvent + { + public EntityBatchExecuted(OperationBatchRequest request, OperationBatchResult result) + { + this.InstanceId = request.InstanceId; + this.OperationCount = request.Operations.Count; + this.ResultCount = result.Results.Count; + this.ErrorCount = result.Results.Count(x => x.ErrorMessage != null); + this.ActionCount = result.Actions.Count; + this.EntityStateLength = request.EntityState?.Length ?? 0; + } + + [StructuredLogField] + public string InstanceId { get; } + + [StructuredLogField] + public int OperationCount { get; } + + [StructuredLogField] + public int ResultCount { get; } + + [StructuredLogField] + public int ErrorCount { get; } + + [StructuredLogField] + public int ActionCount { get; } + + [StructuredLogField] + public int EntityStateLength { get; } + + public override EventId EventId => new EventId( + EventIds.EntityBatchExecuting, + nameof(EventIds.EntityBatchExecuting)); + + public override LogLevel Level => LogLevel.Information; + + protected override string CreateLogMessage() => + $"{this.InstanceId}: completed {this.ResultCount} of {this.OperationCount} entity operations, resulting in {this.ErrorCount} errors, {this.ActionCount} actions, and entity state of length {this.EntityStateLength}."; + + void IEventSourceEvent.WriteEventSource() => + StructuredEventSource.Log.EntityBatchExecuted( + this.InstanceId, + this.OperationCount, + this.ResultCount, + this.ErrorCount, + this.ActionCount, + this.EntityStateLength, + Utils.AppName, + Utils.PackageVersion); + } + + /// + /// Logs that an entity processed a lock acquire message. + /// + internal class EntityLockAcquired : StructuredLogEvent, IEventSourceEvent + { + public EntityLockAcquired(string entityId, Core.Entities.EventFormat.RequestMessage message) + { + this.EntityId = entityId; + this.InstanceId = message.ParentInstanceId; + this.ExecutionId = message.ParentExecutionId; + this.CriticalSectionId = message.Id; + this.Position = message.Position; + + if (message.LockSet != null) + { + this.LockSet = string.Join(",", message.LockSet.Select(id => id.ToString())); + } + } + + [StructuredLogField] + public string EntityId { get; } + + [StructuredLogField] + public string InstanceId { get; set; } + + [StructuredLogField] + public string ExecutionId { get; set; } + + [StructuredLogField] + public Guid CriticalSectionId { get; set; } + + [StructuredLogField] + public string LockSet { get; set; } + + [StructuredLogField] + public int Position { get; set; } + + public override EventId EventId => new EventId( + EventIds.EntityLockAcquired, + nameof(EventIds.EntityLockAcquired)); + + public override LogLevel Level => LogLevel.Information; + + protected override string CreateLogMessage() => + $"{this.EntityId}: acquired lock {this.Position+1}/{this.LockSet.Length} for orchestration instanceId={this.InstanceId} executionId={this.ExecutionId} criticalSectionId={this.CriticalSectionId}"; + + void IEventSourceEvent.WriteEventSource() => + StructuredEventSource.Log.EntityLockAcquired( + this.EntityId, + this.InstanceId ?? string.Empty, + this.ExecutionId ?? string.Empty, + this.CriticalSectionId, + this.LockSet ?? string.Empty, + this.Position, + Utils.AppName, + Utils.PackageVersion); + } + + /// + /// Logs that an entity processed a lock release message. + /// + internal class EntityLockReleased : StructuredLogEvent, IEventSourceEvent + { + public EntityLockReleased(string entityId, Core.Entities.EventFormat.ReleaseMessage message) + { + this.EntityId = entityId; + this.InstanceId = message.ParentInstanceId; + this.Id = message.Id; + } + + [StructuredLogField] + public string EntityId { get; } + + [StructuredLogField] + public string InstanceId { get; set; } + + [StructuredLogField] + public string Id { get; set; } + + public override EventId EventId => new EventId( + EventIds.EntityLockReleased, + nameof(EventIds.EntityLockReleased)); + + public override LogLevel Level => LogLevel.Information; + + protected override string CreateLogMessage() => + $"{this.EntityId}: released lock for orchestration instanceId={this.InstanceId} id={this.Id}"; + + void IEventSourceEvent.WriteEventSource() => + StructuredEventSource.Log.EntityLockReleased( + this.EntityId, + this.InstanceId ?? string.Empty, + this.Id ?? string.Empty, + Utils.AppName, + Utils.PackageVersion); + } + /// /// Log event indicating that an activity execution is starting. /// diff --git a/src/DurableTask.Core/Logging/LogHelper.cs b/src/DurableTask.Core/Logging/LogHelper.cs index f4d1ffe34..96e74ee60 100644 --- a/src/DurableTask.Core/Logging/LogHelper.cs +++ b/src/DurableTask.Core/Logging/LogHelper.cs @@ -18,6 +18,8 @@ namespace DurableTask.Core.Logging using System.Text; using DurableTask.Core.Command; using DurableTask.Core.Common; + using DurableTask.Core.Entities; + using DurableTask.Core.Entities.OperationFormat; using DurableTask.Core.History; using Microsoft.Extensions.Logging; @@ -561,6 +563,58 @@ internal void RenewOrchestrationWorkItemFailed(TaskOrchestrationWorkItem workIte } } + + /// + /// Logs that an entity operation batch is about to start executing. + /// + /// The batch request. + internal void EntityBatchExecuting(OperationBatchRequest request) + { + if (this.IsStructuredLoggingEnabled) + { + this.WriteStructuredLog(new LogEvents.EntityBatchExecuting(request)); + } + } + + /// + /// Logs that an entity operation batch completed its execution. + /// + /// The batch request. + /// The batch result. + internal void EntityBatchExecuted(OperationBatchRequest request, OperationBatchResult result) + { + if (this.IsStructuredLoggingEnabled) + { + this.WriteStructuredLog(new LogEvents.EntityBatchExecuted(request, result)); + } + } + + /// + /// Logs that an entity processed a lock acquire message. + /// + /// The entity id. + /// The message. + internal void EntityLockAcquired(string entityId, Core.Entities.EventFormat.RequestMessage message) + { + if (this.IsStructuredLoggingEnabled) + { + this.WriteStructuredLog(new LogEvents.EntityLockAcquired(entityId, message)); + } + } + + /// + /// Logs that an entity processed a lock release message. + /// + /// The entity id. + /// The message. + internal void EntityLockReleased(string entityId, Core.Entities.EventFormat.ReleaseMessage message) + { + if (this.IsStructuredLoggingEnabled) + { + this.WriteStructuredLog(new LogEvents.EntityLockReleased(entityId, message)); + } + } + #endregion #region Activity dispatcher diff --git a/src/DurableTask.Core/Logging/StructuredEventSource.cs b/src/DurableTask.Core/Logging/StructuredEventSource.cs index b9edc0a46..b6ec19a32 100644 --- a/src/DurableTask.Core/Logging/StructuredEventSource.cs +++ b/src/DurableTask.Core/Logging/StructuredEventSource.cs @@ -624,6 +624,102 @@ internal void DiscardingMessage( } } + [Event(EventIds.EntityBatchExecuting, Level = EventLevel.Informational, Version = 1)] + internal void EntityBatchExecuting( + string InstanceId, + int OperationCount, + int EntityStateLength, + string AppName, + string ExtensionVersion) + { + if (this.IsEnabled(EventLevel.Informational)) + { + // TODO: Use WriteEventCore for better performance + this.WriteEvent( + EventIds.EntityBatchExecuting, + InstanceId, + OperationCount, + EntityStateLength, + AppName, + ExtensionVersion); + } + } + + [Event(EventIds.EntityBatchExecuted, Level = EventLevel.Informational, Version = 1)] + internal void EntityBatchExecuted( + string InstanceId, + int OperationCount, + int ResultCount, + int ErrorCount, + int ActionCount, + int EntityStateLength, + string AppName, + string ExtensionVersion) + { + if (this.IsEnabled(EventLevel.Informational)) + { + // TODO: Use WriteEventCore for better performance + this.WriteEvent( + EventIds.EntityBatchExecuted, + InstanceId, + OperationCount, + ResultCount, + ErrorCount, + ActionCount, + EntityStateLength, + AppName, + ExtensionVersion); + } + } + + [Event(EventIds.EntityLockAcquired, Level = EventLevel.Informational, Version = 1)] + internal void EntityLockAcquired( + string EntityId, + string InstanceId, + string ExecutionId, + Guid CriticalSectionId, + string LockSet, + int Position, + string AppName, + string ExtensionVersion) + { + if (this.IsEnabled(EventLevel.Informational)) + { + // TODO: Use WriteEventCore for better performance + this.WriteEvent( + EventIds.EntityLockAcquired, + EntityId, + InstanceId, + ExecutionId, + CriticalSectionId, + LockSet, + Position, + AppName, + ExtensionVersion); + } + } + + [Event(EventIds.EntityLockReleased, Level = EventLevel.Informational, Version = 1)] + internal void EntityLockReleased( + string EntityId, + string InstanceId, + string Id, + string AppName, + string ExtensionVersion) + { + if (this.IsEnabled(EventLevel.Informational)) + { + // TODO: Use WriteEventCore for better performance + this.WriteEvent( + EventIds.EntityLockReleased, + EntityId, + InstanceId, + Id, + AppName, + ExtensionVersion); + } + } + [Event(EventIds.TaskActivityStarting, Level = EventLevel.Informational, Version = 1)] internal void TaskActivityStarting( string InstanceId, diff --git a/src/DurableTask.Core/NameObjectManager.cs b/src/DurableTask.Core/NameObjectManager.cs new file mode 100644 index 000000000..c8a5d5742 --- /dev/null +++ b/src/DurableTask.Core/NameObjectManager.cs @@ -0,0 +1,65 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.Core +{ + using System; + using System.Collections.Generic; + + internal class NameObjectManager : INameVersionObjectManager + { + readonly IDictionary> creators; + readonly object thisLock = new object(); + + public NameObjectManager() + { + this.creators = new Dictionary>(); + } + + public void Add(ObjectCreator creator) + { + lock (this.thisLock) + { + string key = GetKey(creator.Name, creator.Version); + + if (this.creators.ContainsKey(key)) + { + throw new InvalidOperationException("Duplicate entry detected: " + creator.Name + " " + + creator.Version); + } + + this.creators.Add(key, creator); + } + } + + public T GetObject(string name, string version) + { + string key = GetKey(name, version); + + lock (this.thisLock) + { + if (this.creators.TryGetValue(key, out ObjectCreator creator)) + { + return creator.Create(); + } + + return default(T); + } + } + + string GetKey(string name, string version) + { + return name; + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/OrchestrationContext.cs b/src/DurableTask.Core/OrchestrationContext.cs index 52238bbc2..dffc84df8 100644 --- a/src/DurableTask.Core/OrchestrationContext.cs +++ b/src/DurableTask.Core/OrchestrationContext.cs @@ -18,6 +18,9 @@ namespace DurableTask.Core using System.Threading; using System.Threading.Tasks; using Castle.DynamicProxy; + using DurableTask.Core.Entities; + using DurableTask.Core.Entities.OperationFormat; + using DurableTask.Core.Exceptions; using DurableTask.Core.Serializing; /// @@ -67,6 +70,11 @@ public abstract class OrchestrationContext /// internal ErrorPropagationMode ErrorPropagationMode { get; set; } + /// + /// Information about backend entity support, or null if the configured backend does not support entities. + /// + internal EntityBackendProperties EntityBackendProperties { get; set; } + /// /// Create a proxy client class to schedule remote TaskActivities via a strongly typed interface. /// @@ -368,6 +376,76 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve public abstract Task CreateSubOrchestrationInstance(string name, string version, string instanceId, object input, IDictionary tags); + /// + /// Calls an operation on an entity and returns the result asynchronously. + /// + /// The result type of the operation. + /// The target entity. + /// The name of the operation. + /// The input for the operation. + /// if the orchestration is inside a critical section and the lock for this entity is not available. + /// A task representing the result of the operation. + public virtual Task CallEntityAsync(Entities.EntityId entityId, string operationName, object operationInput = null) + => throw new NotImplementedException(); + + /// + /// Calls an operation on an entity and waits for it to complete. + /// + /// The target entity. + /// The name of the operation. + /// The input for the operation. + /// A task representing the completion of the operation on the entity. + /// if the orchestration is inside a critical section and the lock for this entity is not available. + public virtual Task CallEntityAsync(Entities.EntityId entityId, string operationName, object operationInput = null) + => throw new NotImplementedException(); + + /// + /// Signals an entity operation on the specified entity. + /// + /// The entity ID of the target. + /// The name of the operation. + /// The input for the operation. + /// if the orchestration is inside a critical section that locked this entity. + public virtual void SignalEntity(Entities.EntityId entityId, string operationName, object operationInput = null) + => throw new NotImplementedException(); + + /// + /// Signals an entity to perform an operation, at a specified time. Any result or exception is ignored (fire and forget). + /// + /// The entity ID of the target. + /// The time at which to start the operation. + /// The name of the operation. + /// if the orchestration is inside a critical section that locked this entity. + /// The input for the operation. + public virtual void SignalEntity(Entities.EntityId entityId, DateTime scheduledTimeUtc, string operationName, object operationInput = null) + => throw new NotImplementedException(); + + /// + /// Locks one or more entities for the duration of the critical section. + /// + /// + /// Locks can only be acquired if the current context is not already in a critical section. + /// + /// The entities whose locks should be acquired. + /// A task that must be awaited prior to entering the critical section, and which returns a IDisposable that must be disposed after exiting the critical section. + /// if the context is already in a critical section. + public virtual Task LockEntitiesAsync(params EntityId[] entities) + => throw new NotImplementedException(); + + /// + /// Whether this orchestration is currently inside a critical section. Critical sections are entered when calling + /// , and are exited when disposing the returned IDisposable. + /// + public virtual bool IsInsideCriticalSection { get { throw new NotImplementedException(); } } + + /// + /// Enumerates all the entities that can be called from within the current critical section. + /// This set contains all the entities that were locked prior to entering the critical section, + /// and for which there is not currently an operation call pending. + /// + /// An enumeration of all the currently available entities. + public virtual IEnumerable GetAvailableEntities() + => throw new NotImplementedException(); /// /// Raises an event for the specified orchestration instance, which eventually causes the OnEvent() method in the diff --git a/src/DurableTask.Core/OrchestratorExecutionResult.cs b/src/DurableTask.Core/OrchestratorExecutionResult.cs index 849692e3d..86c91c69e 100644 --- a/src/DurableTask.Core/OrchestratorExecutionResult.cs +++ b/src/DurableTask.Core/OrchestratorExecutionResult.cs @@ -15,6 +15,7 @@ namespace DurableTask.Core { using System; using System.Collections.Generic; + using System.Runtime.Serialization; using DurableTask.Core.Command; using DurableTask.Core.Common; using Newtonsoft.Json; @@ -22,6 +23,7 @@ namespace DurableTask.Core /// /// The result of an orchestration execution. /// + [DataContract] public class OrchestratorExecutionResult { /// diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs new file mode 100644 index 000000000..fadb39384 --- /dev/null +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -0,0 +1,896 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +namespace DurableTask.Core +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Threading; + using System.Threading.Tasks; + using DurableTask.Core.Common; + using DurableTask.Core.Entities; + using DurableTask.Core.Entities.EventFormat; + using DurableTask.Core.Entities.OperationFormat; + using DurableTask.Core.Entities.StateFormat; + using DurableTask.Core.Exceptions; + using DurableTask.Core.History; + using DurableTask.Core.Logging; + using DurableTask.Core.Middleware; + using DurableTask.Core.Tracing; + using Newtonsoft.Json; + + /// + /// Dispatcher for orchestrations and entities to handle processing and renewing, completion of orchestration events. + /// + public class TaskEntityDispatcher + { + readonly INameVersionObjectManager objectManager; + readonly IOrchestrationService orchestrationService; + readonly IEntityOrchestrationService entityOrchestrationService; + readonly WorkItemDispatcher dispatcher; + readonly DispatchMiddlewarePipeline dispatchPipeline; + readonly EntityBackendProperties entityBackendProperties; + readonly LogHelper logHelper; + readonly ErrorPropagationMode errorPropagationMode; + readonly TaskOrchestrationDispatcher.NonBlockingCountdownLock concurrentSessionLock; + + + internal TaskEntityDispatcher( + IOrchestrationService orchestrationService, + INameVersionObjectManager entityObjectManager, + DispatchMiddlewarePipeline entityDispatchPipeline, + LogHelper logHelper, + ErrorPropagationMode errorPropagationMode) + { + this.objectManager = entityObjectManager ?? throw new ArgumentNullException(nameof(entityObjectManager)); + this.orchestrationService = orchestrationService ?? throw new ArgumentNullException(nameof(orchestrationService)); + this.dispatchPipeline = entityDispatchPipeline ?? throw new ArgumentNullException(nameof(entityDispatchPipeline)); + this.logHelper = logHelper ?? throw new ArgumentNullException(nameof(logHelper)); + this.errorPropagationMode = errorPropagationMode; + this.entityOrchestrationService = (orchestrationService as IEntityOrchestrationService)!; + this.entityBackendProperties = entityOrchestrationService.GetEntityBackendProperties(); + + this.dispatcher = new WorkItemDispatcher( + "TaskEntityDispatcher", + item => item == null ? string.Empty : item.InstanceId, + this.OnFetchWorkItemAsync, + this.OnProcessWorkItemSessionAsync) + { + GetDelayInSecondsAfterOnFetchException = orchestrationService.GetDelayInSecondsAfterOnFetchException, + GetDelayInSecondsAfterOnProcessException = orchestrationService.GetDelayInSecondsAfterOnProcessException, + SafeReleaseWorkItem = orchestrationService.ReleaseTaskOrchestrationWorkItemAsync, + AbortWorkItem = orchestrationService.AbandonTaskOrchestrationWorkItemAsync, + DispatcherCount = orchestrationService.TaskOrchestrationDispatcherCount, + MaxConcurrentWorkItems = this.entityBackendProperties.MaxConcurrentTaskEntityWorkItems, + LogHelper = logHelper, + }; + + // To avoid starvation, we only allow half of all concurrently execution orchestrations to + // leverage extended sessions. + var maxConcurrentSessions = (int)Math.Ceiling(this.dispatcher.MaxConcurrentWorkItems / 2.0); + this.concurrentSessionLock = new TaskOrchestrationDispatcher.NonBlockingCountdownLock(maxConcurrentSessions); + } + + /// + /// The entity options configured, or null if the backend does not support entities. + /// + public EntityBackendProperties EntityBackendProperties => this.entityBackendProperties; + + /// + /// Starts the dispatcher to start getting and processing orchestration events + /// + public async Task StartAsync() + { + await this.dispatcher.StartAsync(); + } + + /// + /// Stops the dispatcher to stop getting and processing orchestration events + /// + /// Flag indicating whether to stop gracefully or immediately + public async Task StopAsync(bool forced) + { + await this.dispatcher.StopAsync(forced); + } + + + /// + /// Method to get the next work item to process within supplied timeout + /// + /// The max timeout to wait + /// A cancellation token used to cancel a fetch operation. + /// A new TaskOrchestrationWorkItem + protected Task OnFetchWorkItemAsync(TimeSpan receiveTimeout, CancellationToken cancellationToken) + { + return this.entityOrchestrationService.LockNextEntityWorkItemAsync(receiveTimeout, cancellationToken); + } + + async Task OnProcessWorkItemSessionAsync(TaskOrchestrationWorkItem workItem) + { + try + { + if (workItem.Session == null) + { + // Legacy behavior + await this.OnProcessWorkItemAsync(workItem); + return; + } + + var isExtendedSession = false; + + CorrelationTraceClient.Propagate( + () => + { + // Check if it is extended session. + isExtendedSession = this.concurrentSessionLock.Acquire(); + this.concurrentSessionLock.Release(); + workItem.IsExtendedSession = isExtendedSession; + }); + + var processCount = 0; + try + { + while (true) + { + // If the provider provided work items, execute them. + if (workItem.NewMessages?.Count > 0) + { + bool isCompletedOrInterrupted = await this.OnProcessWorkItemAsync(workItem); + if (isCompletedOrInterrupted) + { + break; + } + + processCount++; + } + + // Fetches beyond the first require getting an extended session lock, used to prevent starvation. + if (processCount > 0 && !isExtendedSession) + { + isExtendedSession = this.concurrentSessionLock.Acquire(); + if (!isExtendedSession) + { + TraceHelper.Trace(TraceEventType.Verbose, "OnProcessWorkItemSession-MaxOperations", "Failed to acquire concurrent session lock."); + break; + } + } + + TraceHelper.Trace(TraceEventType.Verbose, "OnProcessWorkItemSession-StartFetch", "Starting fetch of existing session."); + Stopwatch timer = Stopwatch.StartNew(); + + // Wait for new messages to arrive for the session. This call is expected to block (asynchronously) + // until either new messages are available or until a provider-specific timeout has expired. + workItem.NewMessages = await workItem.Session.FetchNewOrchestrationMessagesAsync(workItem); + if (workItem.NewMessages == null) + { + break; + } + + TraceHelper.Trace( + TraceEventType.Verbose, + "OnProcessWorkItemSession-EndFetch", + $"Fetched {workItem.NewMessages.Count} new message(s) after {timer.ElapsedMilliseconds} ms from existing session."); + workItem.OrchestrationRuntimeState.NewEvents.Clear(); + } + } + finally + { + if (isExtendedSession) + { + TraceHelper.Trace( + TraceEventType.Verbose, + "OnProcessWorkItemSession-Release", + $"Releasing extended session after {processCount} batch(es)."); + this.concurrentSessionLock.Release(); + } + } + } + catch (SessionAbortedException e) + { + // Either the orchestration or the orchestration service explicitly abandoned the session. + OrchestrationInstance instance = workItem.OrchestrationRuntimeState?.OrchestrationInstance ?? new OrchestrationInstance { InstanceId = workItem.InstanceId }; + this.logHelper.OrchestrationAborted(instance, e.Message); + TraceHelper.TraceInstance(TraceEventType.Warning, "TaskOrchestrationDispatcher-ExecutionAborted", instance, "{0}", e.Message); + await this.orchestrationService.AbandonTaskOrchestrationWorkItemAsync(workItem); + } + } + + class WorkItemEffects + { + public List ActivityMessages; + public List TimerMessages; + public List InstanceMessages; + public int taskIdCounter; + public string InstanceId; + public OrchestrationRuntimeState RuntimeState; + } + + /// + /// Method to process a new work item + /// + /// The work item to process + protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem workItem) + { + // correlation + CorrelationTraceClient.Propagate(() => CorrelationTraceContext.Current = workItem.TraceContext); + + OrchestrationRuntimeState originalOrchestrationRuntimeState = workItem.OrchestrationRuntimeState; + + OrchestrationRuntimeState runtimeState = workItem.OrchestrationRuntimeState; + runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); + + Task renewTask = null; + using var renewCancellationTokenSource = new CancellationTokenSource(); + if (workItem.LockedUntilUtc < DateTime.MaxValue) + { + // start a task to run RenewUntil + renewTask = Task.Factory.StartNew( + () => TaskOrchestrationDispatcher.RenewUntil(workItem, this.orchestrationService, this.logHelper, nameof(TaskEntityDispatcher), renewCancellationTokenSource.Token), + renewCancellationTokenSource.Token); + } + + WorkItemEffects effects = new WorkItemEffects() + { + ActivityMessages = new List(), + TimerMessages = new List(), + InstanceMessages = new List(), + taskIdCounter = 0, + InstanceId = workItem.InstanceId, + RuntimeState = runtimeState, + }; + + try + { + // Assumes that: if the batch contains a new "ExecutionStarted" event, it is the first message in the batch. + if (!TaskOrchestrationDispatcher.ReconcileMessagesWithState(workItem, "TaskEntityDispatcher", this.logHelper)) + { + // TODO : mark an orchestration as faulted if there is data corruption + this.logHelper.DroppingOrchestrationWorkItem(workItem, "Received work-item for an invalid orchestration"); + TraceHelper.TraceSession( + TraceEventType.Error, + "TaskEntityDispatcher-DeletedOrchestration", + runtimeState.OrchestrationInstance?.InstanceId, + "Received work-item for an invalid orchestration"); + } + else + { + + // we start with processing all the requests and figuring out which ones to execute now + // results can depend on whether the entity is locked, what the maximum batch size is, + // and whether the messages arrived out of order + + this.DetermineWork(workItem.OrchestrationRuntimeState, + out SchedulerState schedulerState, + out Work workToDoNow); + + if (workToDoNow.OperationCount > 0) + { + // execute the user-defined operations on this entity, via the middleware + var result = await this.ExecuteViaMiddlewareAsync(workToDoNow, runtimeState.OrchestrationInstance, schedulerState.EntityState); + + // go through all results + // for each operation that is not a signal, send a result message back to the calling orchestrator + for (int i = 0; i < result.Results!.Count; i++) + { + var req = workToDoNow.Operations[i]; + if (!req.IsSignal) + { + this.SendResultMessage(effects, req, result.Results[i]); + } + } + + if (result.Results.Count < workToDoNow.OperationCount) + { + // some operations were not processed + var deferred = workToDoNow.RemoveDeferredWork(result.Results.Count); + schedulerState.PutBack(deferred); + workToDoNow.ToBeContinued(schedulerState); + } + + // update the entity state based on the result + schedulerState.EntityState = result.EntityState; + schedulerState.EntityExists = result.EntityState != null; + + // perform the actions + foreach (var action in result.Actions!) + { + switch (action) + { + case (SendSignalOperationAction sendSignalAction): + this.SendSignalMessage(effects, schedulerState, sendSignalAction); + break; + case (StartNewOrchestrationOperationAction startAction): + this.ProcessSendStartMessage(effects, runtimeState, startAction); + break; + } + } + } + + // process the lock request, if any + if (workToDoNow.LockRequest != null) + { + this.ProcessLockRequest(effects, schedulerState, workToDoNow.LockRequest); + } + + if (workToDoNow.ToBeRescheduled != null) + { + foreach (var request in workToDoNow.ToBeRescheduled) + { + // Reschedule all signals that were received before their time + this.SendScheduledSelfMessage(effects, request); + } + } + + if (workToDoNow.SuspendAndContinue) + { + this.SendContinueSelfMessage(effects); + } + + // this batch is complete. Since this is an entity, we now + // (always) start a new execution, as in continue-as-new + + var serializedSchedulerState = this.SerializeSchedulerStateForNextExecution(schedulerState); + var nextExecutionStartedEvent = new ExecutionStartedEvent(-1, serializedSchedulerState) + { + OrchestrationInstance = new OrchestrationInstance + { + InstanceId = workItem.InstanceId, + ExecutionId = Guid.NewGuid().ToString("N") + }, + Tags = runtimeState.Tags, + ParentInstance = runtimeState.ParentInstance, + Name = runtimeState.Name, + Version = runtimeState.Version + }; + var entityStatus = new EntityStatus() + { + EntityExists = schedulerState.EntityExists, + QueueSize = schedulerState.Queue?.Count ?? 0, + LockedBy = schedulerState.LockedBy, + }; + var serializedEntityStatus = JsonConvert.SerializeObject(entityStatus, Serializer.InternalSerializerSettings); + + // create the new runtime state for the next execution + runtimeState = new OrchestrationRuntimeState(); + runtimeState.Status = serializedEntityStatus; + runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); + runtimeState.AddEvent(nextExecutionStartedEvent); + runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); + } + } + finally + { + if (renewTask != null) + { + try + { + renewCancellationTokenSource.Cancel(); + await renewTask; + } + catch (ObjectDisposedException) + { + // ignore + } + catch (OperationCanceledException) + { + // ignore + } + } + } + + OrchestrationState instanceState = (runtimeState.ExecutionStartedEvent != null) ? + instanceState = Utils.BuildOrchestrationState(runtimeState) : null; + + if (workItem.RestoreOriginalRuntimeStateDuringCompletion) + { + // some backends expect the original runtime state object + workItem.OrchestrationRuntimeState = originalOrchestrationRuntimeState; + } + else + { + workItem.OrchestrationRuntimeState = runtimeState; + } + + await this.orchestrationService.CompleteTaskOrchestrationWorkItemAsync( + workItem, + runtimeState, + effects.ActivityMessages, + effects.InstanceMessages, + effects.TimerMessages, + null, + instanceState); + + if (workItem.RestoreOriginalRuntimeStateDuringCompletion) + { + workItem.OrchestrationRuntimeState = runtimeState; + } + + return true; + } + + void ProcessLockRequest(WorkItemEffects effects, SchedulerState schedulerState, RequestMessage request) + { + this.logHelper.EntityLockAcquired(effects.InstanceId, request); + + // mark the entity state as locked + schedulerState.LockedBy = request.ParentInstanceId; + + request.Position++; + + if (request.Position < request.LockSet.Length) + { + // send lock request to next entity in the lock set + var target = new OrchestrationInstance() { InstanceId = request.LockSet[request.Position].ToString() }; + this.SendLockRequestMessage(effects, schedulerState, target, request); + } + else + { + // send lock acquisition completed response back to originating orchestration instance + var target = new OrchestrationInstance() { InstanceId = request.ParentInstanceId, ExecutionId = request.ParentExecutionId }; + this.SendLockResponseMessage(effects, target, request.Id); + } + } + + string SerializeSchedulerStateForNextExecution(SchedulerState schedulerState) + { + if (this.entityBackendProperties.SupportsImplicitEntityDeletion && schedulerState.IsEmpty && !schedulerState.Suspended) + { + // this entity scheduler is idle and the entity is deleted, so the instance and history can be removed from storage + // we convey this to the durability provider by issuing a continue-as-new with null input + return null; + } + else + { + // we persist the state of the entity scheduler and entity + return JsonConvert.SerializeObject(schedulerState, typeof(SchedulerState), Serializer.InternalSerializerSettings); + } + } + + #region Preprocess to determine work + + void DetermineWork(OrchestrationRuntimeState runtimeState, out SchedulerState schedulerState, out Work batch) + { + string instanceId = runtimeState.OrchestrationInstance.InstanceId; + schedulerState = new SchedulerState(); + batch = new Work(); + + Queue lockHolderMessages = null; + + foreach (HistoryEvent e in runtimeState.Events) + { + switch (e.EventType) + { + case EventType.ExecutionStarted: + + + if (runtimeState.Input != null) + { + try + { + // restore the scheduler state from the input + JsonConvert.PopulateObject(runtimeState.Input, schedulerState, Serializer.InternalSerializerSettings); + } + catch (Exception exception) + { + throw new EntitySchedulerException("Failed to deserialize entity scheduler state - may be corrupted or wrong version.", exception); + } + } + break; + + case EventType.EventRaised: + EventRaisedEvent eventRaisedEvent = (EventRaisedEvent)e; + + if (EntityMessageEventNames.IsRequestMessage(eventRaisedEvent.Name)) + { + // we are receiving an operation request or a lock request + var requestMessage = new RequestMessage(); + + try + { + JsonConvert.PopulateObject(eventRaisedEvent.Input, requestMessage, Serializer.InternalSerializerSettings); + } + catch (Exception exception) + { + throw new EntitySchedulerException("Failed to deserialize incoming request message - may be corrupted or wrong version.", exception); + } + + IEnumerable deliverNow; + + if (requestMessage.ScheduledTime.HasValue) + { + if ((requestMessage.ScheduledTime.Value - DateTime.UtcNow) > TimeSpan.FromMilliseconds(100)) + { + // message was delivered too early. This can happen e.g. if the orchestration service has limits on the delay times for messages. + // We handle this by rescheduling the message instead of processing it. + deliverNow = Array.Empty(); + batch.AddMessageToBeRescheduled(requestMessage); + } + else + { + // the message is scheduled to be delivered immediately. + // There are no FIFO guarantees for scheduled messages, so we skip the message sorter. + deliverNow = new RequestMessage[] { requestMessage }; + } + } + else + { + // run this through the message sorter to help with reordering and duplicate filtering + deliverNow = schedulerState.MessageSorter.ReceiveInOrder(requestMessage, this.entityBackendProperties.EntityMessageReorderWindow); + } + + foreach (var message in deliverNow) + { + if (schedulerState.LockedBy != null && schedulerState.LockedBy == message.ParentInstanceId) + { + if (lockHolderMessages == null) + { + lockHolderMessages = new Queue(); + } + + lockHolderMessages.Enqueue(message); + } + else + { + schedulerState.Enqueue(message); + } + } + } + else if (EntityMessageEventNames.IsReleaseMessage(eventRaisedEvent.Name)) + { + // we are receiving a lock release + var message = new ReleaseMessage(); + try + { + // restore the scheduler state from the input + JsonConvert.PopulateObject(eventRaisedEvent.Input, message, Serializer.InternalSerializerSettings); + } + catch (Exception exception) + { + throw new EntitySchedulerException("Failed to deserialize lock release message - may be corrupted or wrong version.", exception); + } + + if (schedulerState.LockedBy == message.ParentInstanceId) + { + this.logHelper.EntityLockReleased(instanceId, message); + schedulerState.LockedBy = null; + } + } + else + { + // this is a continue message. + // Resumes processing of previously queued operations, if any. + schedulerState.Suspended = false; + } + + break; + } + } + + // lock holder messages go to the front of the queue + if (lockHolderMessages != null) + { + schedulerState.PutBack(lockHolderMessages); + } + + if (!schedulerState.Suspended) + { + // 2. We add as many requests from the queue to the batch as possible, + // stopping at lock requests or when the maximum batch size is reached + while (schedulerState.MayDequeue()) + { + if (batch.OperationCount == this.entityBackendProperties.MaxEntityOperationBatchSize) + { + // we have reached the maximum batch size already + // insert a delay after this batch to ensure write back + batch.ToBeContinued(schedulerState); + break; + } + + var request = schedulerState.Dequeue(); + + if (request.IsLockRequest) + { + batch.AddLockRequest(request); + break; + } + else + { + batch.AddOperation(request); + } + } + } + } + + class Work + { + List operationBatch; // a (possibly empty) sequence of operations to be executed on the entity + RequestMessage lockRequest = null; // zero or one lock request to be executed after all the operations + List toBeRescheduled; // a (possibly empty) list of timed messages that were delivered too early and should be rescheduled + bool suspendAndContinue; // a flag telling as to send ourselves a continue signal + + public int OperationCount => this.operationBatch?.Count ?? 0; + public IReadOnlyList Operations => this.operationBatch; + public IReadOnlyList ToBeRescheduled => this.toBeRescheduled; + public RequestMessage LockRequest => this.lockRequest; + public bool SuspendAndContinue => this.suspendAndContinue; + + public void AddOperation(RequestMessage operationMessage) + { + if (this.operationBatch == null) + { + this.operationBatch = new List(); + } + this.operationBatch.Add(operationMessage); + } + + public void AddLockRequest(RequestMessage lockRequest) + { + Debug.Assert(this.lockRequest == null); + this.lockRequest = lockRequest; + } + + public void AddMessageToBeRescheduled(RequestMessage requestMessage) + { + if (this.toBeRescheduled == null) + { + this.toBeRescheduled = new List(); + } + this.toBeRescheduled.Add(requestMessage); + } + + public void ToBeContinued(SchedulerState schedulerState) + { + if (!schedulerState.Suspended) + { + this.suspendAndContinue = true; + } + } + + public List GetOperationRequests() + { + var operations = new List(this.operationBatch.Count); + for (int i = 0; i < this.operationBatch.Count; i++) + { + var request = this.operationBatch[i]; + operations.Add(new OperationRequest() + { + Operation = request.Operation, + Id = request.Id, + Input = request.Input, + }); + } + return operations; + } + + public Queue RemoveDeferredWork(int index) + { + var deferred = new Queue(); + for (int i = index; i < this.operationBatch.Count; i++) + { + deferred.Enqueue(this.operationBatch[i]); + } + this.operationBatch.RemoveRange(index, this.operationBatch.Count - index); + if (this.lockRequest != null) + { + deferred.Enqueue(this.lockRequest); + this.lockRequest = null; + } + return deferred; + } + } + + #endregion + + #region Send Messages + + void SendResultMessage(WorkItemEffects effects, RequestMessage request, OperationResult result) + { + var destination = new OrchestrationInstance() + { + InstanceId = request.ParentInstanceId, + ExecutionId = request.ParentExecutionId, + }; + var responseMessage = new ResponseMessage() + { + Result = result.Result, + ErrorMessage = result.ErrorMessage, + FailureDetails = result.FailureDetails, + }; + this.ProcessSendEventMessage(effects, destination, EntityMessageEventNames.ResponseMessageEventName(request.Id), responseMessage); + } + + void SendSignalMessage(WorkItemEffects effects, SchedulerState schedulerState, SendSignalOperationAction action) + { + OrchestrationInstance destination = new OrchestrationInstance() + { + InstanceId = action.InstanceId + }; + RequestMessage message = new RequestMessage() + { + ParentInstanceId = effects.InstanceId, + ParentExecutionId = null, // for entities, message sorter persists across executions + Id = Guid.NewGuid(), + IsSignal = true, + Operation = action.Name, + ScheduledTime = action.ScheduledTime, + }; + string eventName; + if (action.ScheduledTime.HasValue) + { + DateTime original = action.ScheduledTime.Value; + DateTime capped = this.entityBackendProperties.GetCappedScheduledTime(DateTime.UtcNow, original); + eventName = EntityMessageEventNames.ScheduledRequestMessageEventName(capped); + } + else + { + eventName = EntityMessageEventNames.RequestMessageEventName; + schedulerState.MessageSorter.LabelOutgoingMessage(message, action.InstanceId, DateTime.UtcNow, this.entityBackendProperties.EntityMessageReorderWindow); + } + this.ProcessSendEventMessage(effects, destination, eventName, message); + } + + void SendLockRequestMessage(WorkItemEffects effects, SchedulerState schedulerState, OrchestrationInstance target, RequestMessage message) + { + schedulerState.MessageSorter.LabelOutgoingMessage(message, target.InstanceId, DateTime.UtcNow, this.entityBackendProperties.EntityMessageReorderWindow); + this.ProcessSendEventMessage(effects, target, EntityMessageEventNames.RequestMessageEventName, message); + } + + void SendLockResponseMessage(WorkItemEffects effects, OrchestrationInstance target, Guid requestId) + { + var message = new ResponseMessage() + { + Result = "Lock Acquisition Completed", // ignored by receiver but shows up in traces + }; + this.ProcessSendEventMessage(effects, target, EntityMessageEventNames.ResponseMessageEventName(requestId), message); + } + + void SendScheduledSelfMessage(WorkItemEffects effects, RequestMessage request) + { + var self = new OrchestrationInstance() + { + InstanceId = effects.InstanceId, + }; + this.ProcessSendEventMessage(effects, self, EntityMessageEventNames.ScheduledRequestMessageEventName(request.ScheduledTime.Value), request); + } + + void SendContinueSelfMessage(WorkItemEffects effects) + { + var self = new OrchestrationInstance() + { + InstanceId = effects.InstanceId, + }; + this.ProcessSendEventMessage(effects, self, EntityMessageEventNames.ContinueMessageEventName, null); + } + + void ProcessSendEventMessage(WorkItemEffects effects, OrchestrationInstance destination, string eventName, object eventContent) + { + string serializedContent = null; + if (eventContent != null) + { + serializedContent = JsonConvert.SerializeObject(eventContent, Serializer.InternalSerializerSettings); + } + + var eventSentEvent = new EventSentEvent(effects.taskIdCounter++) + { + InstanceId = destination.InstanceId, + Name = eventName, + Input = serializedContent, + }; + this.logHelper.RaisingEvent(effects.RuntimeState.OrchestrationInstance, eventSentEvent); + + effects.InstanceMessages.Add(new TaskMessage + { + OrchestrationInstance = destination, + Event = new EventRaisedEvent(-1, serializedContent) + { + Name = eventName, + Input = serializedContent, + }, + }); + } + + void ProcessSendStartMessage(WorkItemEffects effects, OrchestrationRuntimeState runtimeState, StartNewOrchestrationOperationAction action) + { + OrchestrationInstance destination = new OrchestrationInstance() + { + InstanceId = action.InstanceId, + ExecutionId = Guid.NewGuid().ToString("N"), + }; + var executionStartedEvent = new ExecutionStartedEvent(-1, action.Input) + { + Tags = OrchestrationTags.MergeTags(action.Tags, runtimeState.Tags), + OrchestrationInstance = destination, + ParentInstance = new ParentInstance + { + OrchestrationInstance = runtimeState.OrchestrationInstance, + Name = runtimeState.Name, + Version = runtimeState.Version, + TaskScheduleId = effects.taskIdCounter++, + }, + Name = action.Name, + Version = action.Version, + }; + this.logHelper.SchedulingOrchestration(executionStartedEvent); + + effects.InstanceMessages.Add(new TaskMessage + { + OrchestrationInstance = destination, + Event = executionStartedEvent, + }); + } + + #endregion + + async Task ExecuteViaMiddlewareAsync(Work workToDoNow, OrchestrationInstance instance, string serializedEntityState) + { + // the request object that will be passed to the worker + var request = new OperationBatchRequest() + { + InstanceId = instance.InstanceId, + EntityState = serializedEntityState, + Operations = workToDoNow.GetOperationRequests(), + }; + + this.logHelper.EntityBatchExecuting(request); + + var entityId = EntityId.FromString(instance.InstanceId); + string entityName = entityId.Name; + string entityKey = entityId.Key; + + // Get the TaskOrchestration implementation. If it's not found, it either means that the developer never + // registered it (which is an error, and we'll throw for this further down) or it could be that some custom + // middleware (e.g. out-of-process execution middleware) is intended to implement the orchestration logic. + TaskEntity taskEntity = this.objectManager.GetObject(entityName, entityKey); + + var dispatchContext = new DispatchMiddlewareContext(); + dispatchContext.SetProperty(request); + + await this.dispatchPipeline.RunAsync(dispatchContext, async _ => + { + // Check to see if the custom middleware intercepted and substituted the orchestration execution + // with its own execution behavior, providing us with the end results. If so, we can terminate + // the dispatch pipeline here. + var resultFromMiddleware = dispatchContext.GetProperty(); + if (resultFromMiddleware != null) + { + return; + } + + if (taskEntity == null) + { + throw TraceHelper.TraceExceptionInstance( + TraceEventType.Error, + "TaskOrchestrationDispatcher-EntityTypeMissing", + instance, + new TypeMissingException($"Entity not found: {entityName}")); + } + + var options = new EntityExecutionOptions() + { + EntityBackendProperties = this.entityBackendProperties, + ErrorPropagationMode = this.errorPropagationMode, + }; + + var result = await taskEntity.ExecuteOperationBatchAsync(request, options); + + dispatchContext.SetProperty(result); + }); + + var result = dispatchContext.GetProperty(); + + this.logHelper.EntityBatchExecuted(request, result); + + return result; + } + } +} \ No newline at end of file diff --git a/src/DurableTask.Core/TaskHubClient.cs b/src/DurableTask.Core/TaskHubClient.cs index fdd27592e..c48736b63 100644 --- a/src/DurableTask.Core/TaskHubClient.cs +++ b/src/DurableTask.Core/TaskHubClient.cs @@ -19,6 +19,7 @@ namespace DurableTask.Core using System.Linq; using System.Threading; using System.Threading.Tasks; + using DurableTask.Core.Entities; using DurableTask.Core.History; using DurableTask.Core.Logging; using DurableTask.Core.Serializing; @@ -32,6 +33,9 @@ public sealed class TaskHubClient readonly DataConverter defaultConverter; readonly LogHelper logHelper; + internal LogHelper LogHelper => this.logHelper; + internal DataConverter DefaultConverter => this.defaultConverter; + /// /// The orchestration service client for this task hub client /// diff --git a/src/DurableTask.Core/TaskHubWorker.cs b/src/DurableTask.Core/TaskHubWorker.cs index 1b93bd62e..c7892a051 100644 --- a/src/DurableTask.Core/TaskHubWorker.cs +++ b/src/DurableTask.Core/TaskHubWorker.cs @@ -21,6 +21,7 @@ namespace DurableTask.Core using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; + using DurableTask.Core.Entities; using DurableTask.Core.Exceptions; using DurableTask.Core.Logging; using DurableTask.Core.Middleware; @@ -34,8 +35,12 @@ public sealed class TaskHubWorker : IDisposable { readonly INameVersionObjectManager activityManager; readonly INameVersionObjectManager orchestrationManager; + readonly INameVersionObjectManager entityManager; + + readonly IEntityOrchestrationService entityOrchestrationService; readonly DispatchMiddlewarePipeline orchestrationDispatchPipeline = new DispatchMiddlewarePipeline(); + readonly DispatchMiddlewarePipeline entityDispatchPipeline = new DispatchMiddlewarePipeline(); readonly DispatchMiddlewarePipeline activityDispatchPipeline = new DispatchMiddlewarePipeline(); readonly SemaphoreSlim slimLock = new SemaphoreSlim(1, 1); @@ -47,10 +52,16 @@ public sealed class TaskHubWorker : IDisposable // ReSharper disable once InconsistentNaming (avoid breaking change) public IOrchestrationService orchestrationService { get; } + /// + /// Indicates whether the configured backend supports entities. + /// + public bool SupportsEntities => this.entityOrchestrationService != null; + volatile bool isStarted; TaskActivityDispatcher activityDispatcher; TaskOrchestrationDispatcher orchestrationDispatcher; + TaskEntityDispatcher entityDispatcher; /// /// Create a new TaskHubWorker with given OrchestrationService @@ -60,7 +71,8 @@ public TaskHubWorker(IOrchestrationService orchestrationService) : this( orchestrationService, new NameVersionObjectManager(), - new NameVersionObjectManager()) + new NameVersionObjectManager(), + new NameObjectManager()) { } @@ -75,6 +87,7 @@ public TaskHubWorker(IOrchestrationService orchestrationService, ILoggerFactory orchestrationService, new NameVersionObjectManager(), new NameVersionObjectManager(), + new NameObjectManager(), loggerFactory) { } @@ -85,14 +98,18 @@ public TaskHubWorker(IOrchestrationService orchestrationService, ILoggerFactory /// Reference the orchestration service implementation /// NameVersionObjectManager for Orchestrations /// NameVersionObjectManager for Activities + /// The NameVersionObjectManager for entities. The version is the entity key. + /// public TaskHubWorker( IOrchestrationService orchestrationService, INameVersionObjectManager orchestrationObjectManager, - INameVersionObjectManager activityObjectManager) + INameVersionObjectManager activityObjectManager, + INameVersionObjectManager entityObjectManager) : this( orchestrationService, orchestrationObjectManager, activityObjectManager, + entityObjectManager, loggerFactory: null) { } @@ -104,17 +121,27 @@ public TaskHubWorker( /// The orchestration service implementation /// The for orchestrations /// The for activities + /// The for entities. The version is the entity key. /// The to use for logging public TaskHubWorker( IOrchestrationService orchestrationService, INameVersionObjectManager orchestrationObjectManager, INameVersionObjectManager activityObjectManager, + INameVersionObjectManager entityObjectManager, ILoggerFactory loggerFactory = null) { this.orchestrationManager = orchestrationObjectManager ?? throw new ArgumentException("orchestrationObjectManager"); this.activityManager = activityObjectManager ?? throw new ArgumentException("activityObjectManager"); + this.entityManager = entityObjectManager ?? throw new ArgumentException("entityObjectManager"); this.orchestrationService = orchestrationService ?? throw new ArgumentException("orchestrationService"); this.logHelper = new LogHelper(loggerFactory?.CreateLogger("DurableTask.Core")); + + // if the backend supports entities, configure it to collect entity work items in a separate queue. + if (orchestrationService is IEntityOrchestrationService entityOrchestrationService) + { + this.entityOrchestrationService = entityOrchestrationService; + entityOrchestrationService.ProcessEntitiesSeparately(); + } } /// @@ -127,6 +154,11 @@ public TaskHubWorker( /// public TaskActivityDispatcher TaskActivityDispatcher => this.activityDispatcher; + /// + /// Gets the entity dispatcher + /// + public TaskEntityDispatcher TaskEntityDispatcher => this.entityDispatcher; + /// /// Gets or sets the error propagation behavior when an activity or orchestration fails with an unhandled exception. /// @@ -153,6 +185,15 @@ public void AddOrchestrationDispatcherMiddleware(Func + /// Adds a middleware delegate to the entity dispatch pipeline. + /// + /// Delegate to invoke whenever a message is dispatched to an entity. + public void AddEntityDispatcherMiddleware(Func, Task> middleware) + { + this.entityDispatchPipeline.Add(middleware ?? throw new ArgumentNullException(nameof(middleware))); + } + /// /// Adds a middleware delegate to the activity dispatch pipeline. /// @@ -192,10 +233,25 @@ public async Task StartAsync() this.logHelper, this.ErrorPropagationMode); + if (this.SupportsEntities) + { + this.entityDispatcher = new TaskEntityDispatcher( + this.orchestrationService, + this.entityManager, + this.entityDispatchPipeline, + this.logHelper, + this.ErrorPropagationMode); + } + await this.orchestrationService.StartAsync(); await this.orchestrationDispatcher.StartAsync(); await this.activityDispatcher.StartAsync(); + if (this.SupportsEntities) + { + await this.entityDispatcher.StartAsync(); + } + this.logHelper.TaskHubWorkerStarted(sw.Elapsed); this.isStarted = true; } @@ -233,6 +289,7 @@ public async Task StopAsync(bool isForced) { this.orchestrationDispatcher.StopAsync(isForced), this.activityDispatcher.StopAsync(isForced), + this.SupportsEntities ? this.entityDispatcher.StopAsync(isForced) : Task.CompletedTask, }; await Task.WhenAll(dispatcherShutdowns); @@ -282,6 +339,43 @@ public TaskHubWorker AddTaskOrchestrations(params ObjectCreator + /// Loads user defined TaskEntity classes in the TaskHubWorker + /// + /// Types deriving from TaskEntity class + /// + public TaskHubWorker AddTaskEntities(params Type[] taskEntityTypes) + { + foreach (Type type in taskEntityTypes) + { + ObjectCreator creator = new NameValueObjectCreator( + type.Name, + string.Empty, + type); + + this.entityManager.Add(creator); + } + + return this; + } + + /// + /// Loads user defined TaskEntity classes in the TaskHubWorker + /// + /// + /// User specified ObjectCreators that will + /// create classes deriving TaskEntities with specific names and versions + /// + public TaskHubWorker AddTaskEntities(params ObjectCreator[] taskEntityCreators) + { + foreach (ObjectCreator creator in taskEntityCreators) + { + this.entityManager.Add(creator); + } + + return this; + } + /// /// Loads user defined TaskActivity objects in the TaskHubWorker /// diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index a9831ff47..62c027816 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -21,6 +21,8 @@ namespace DurableTask.Core using System.Threading.Tasks; using DurableTask.Core.Command; using DurableTask.Core.Common; + using DurableTask.Core.Entities; + using DurableTask.Core.Entities.OperationFormat; using DurableTask.Core.Exceptions; using DurableTask.Core.History; using DurableTask.Core.Serializing; @@ -31,6 +33,7 @@ internal class TaskOrchestrationContext : OrchestrationContext private readonly IDictionary openTasks; private readonly IDictionary orchestratorActionsMap; private OrchestrationCompleteOrchestratorAction continueAsNew; + private OrchestrationEntityContext entityContext; private bool executionCompletedOrTerminated; private int idCounter; private readonly Queue eventsWhileSuspended; @@ -44,9 +47,16 @@ public void AddEventToNextIteration(HistoryEvent he) continueAsNew.CarryoverEvents.Add(he); } + internal OrchestrationEntityContext EntityContext + => this.entityContext ?? (this.entityContext = new OrchestrationEntityContext( + this.OrchestrationInstance.InstanceId, + this.OrchestrationInstance.ExecutionId, + this)); + public TaskOrchestrationContext( OrchestrationInstance orchestrationInstance, TaskScheduler taskScheduler, + EntityBackendProperties entityBackendProperties = null, ErrorPropagationMode errorPropagationMode = ErrorPropagationMode.SerializeExceptions) { Utils.UnusedParameter(taskScheduler); @@ -58,6 +68,7 @@ public TaskOrchestrationContext( this.ErrorDataConverter = JsonDataConverter.Default; OrchestrationInstance = orchestrationInstance; IsReplaying = false; + this.EntityBackendProperties = entityBackendProperties; ErrorPropagationMode = errorPropagationMode; this.eventsWhileSuspended = new Queue(); } @@ -66,6 +77,28 @@ public TaskOrchestrationContext( public bool HasOpenTasks => this.openTasks.Count > 0; + public override bool IsInsideCriticalSection => this.entityContext?.IsInsideCriticalSection ?? false; + + private void CheckEntitySupport() + { + if (this.EntityBackendProperties == null) + { + throw new InvalidOperationException("Entities are not supported by the configured persistence backend."); + } + } + + public override IEnumerable GetAvailableEntities() + { + this.CheckEntitySupport(); + if (this.entityContext != null) + { + foreach(var (name, key) in this.entityContext.GetAvailableEntities()) + { + yield return new EntityId(name, key); + } + } + } + internal void ClearPendingActions() { this.orchestratorActionsMap.Clear(); @@ -188,6 +221,92 @@ async Task CreateSubOrchestrationInstanceCore( } } + public override Task CallEntityAsync(EntityId entityId, string operationName, object input = null) + { + this.CheckEntitySupport(); + (Guid operationId, string targetInstanceId, int taskId) = this.EntityOperationCore(entityId, operationName, input, false, null); + return this.entityContext.WaitForOperationResponseAsync(operationId, taskId, targetInstanceId); + } + + public override Task CallEntityAsync(EntityId entityId, string operationName, object input = null) + { + return this.CallEntityAsync(entityId, operationName, input); + } + + public override void SignalEntity(EntityId entityId, string operationName, object input = null) + { + this.CheckEntitySupport(); + this.EntityOperationCore(entityId, operationName, input, true, null); + } + + public override void SignalEntity(EntityId entityId, DateTime scheduledTimeUtc, string operationName, object input = null) + { + this.CheckEntitySupport(); + DateTime cappedTime = EntityBackendProperties.GetCappedScheduledTime(this.CurrentUtcDateTime, scheduledTimeUtc); + this.EntityOperationCore(entityId, operationName, input, true, (scheduledTimeUtc, cappedTime)); + } + + (Guid operationId, string targetInstanceId, int taskId) EntityOperationCore(EntityId entityId, string operationName, object input, bool oneWay, (DateTime original, DateTime capped)? scheduledTimeUtc = null) + { + OrchestrationInstance target = new OrchestrationInstance() + { + InstanceId = entityId.ToString(), + }; + + if (!this.EntityContext.ValidateOperationTransition(target.InstanceId, oneWay, out string errorMessage)) + { + throw new EntityLockingRulesViolationException(errorMessage); + } + + int taskId = this.idCounter; // we are not incrementing the counter here because it will be incremented when calling this.SendEvent below + Guid operationId = Utils.CreateGuidFromHash(string.Concat(OrchestrationInstance.ExecutionId, ":", taskId)); + + string serializedInput = this.MessageDataConverter.Serialize(input); + + var eventToSend = this.entityContext.EmitRequestMessage(target, operationName, oneWay, operationId, scheduledTimeUtc, serializedInput); + + this.SendEvent(eventToSend.TargetInstance, eventToSend.EventName, eventToSend.EventContent); + + return (operationId, target.InstanceId, taskId); + } + + public override Task LockEntitiesAsync(params EntityId[] entities) + { + this.CheckEntitySupport(); + + if (!this.EntityContext.ValidateAcquireTransition(out string errormsg)) + { + throw new EntityLockingRulesViolationException(errormsg); + } + + if (entities == null || entities.Length == 0) + { + throw new ArgumentException("The list of entities to lock must not be null or empty.", nameof(entities)); + } + + int taskId = this.idCounter; // we are not incrementing the counter here because it will be incremented when calling this.SendEvent below + Guid criticalSectionId = Utils.CreateGuidFromHash(string.Concat(OrchestrationInstance.ExecutionId, ":", taskId)); + + // send a message to the first entity to be acquired + EventToSend eventToSend = this.entityContext.EmitAcquireMessage(criticalSectionId, entities); + + this.SendEvent(eventToSend.TargetInstance, eventToSend.EventName, eventToSend.EventContent); + + return this.entityContext.WaitForLockResponseAsync(criticalSectionId, taskId); + } + + internal void ExitCriticalSection() + { + if (this.entityContext != null) + { + foreach (var releaseMessage in this.entityContext.EmitLockReleaseMessages()) + { + this.SendEvent(releaseMessage.TargetInstance, releaseMessage.EventName, releaseMessage.EventContent); + } + } + } + + public override void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData) { if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) @@ -406,6 +525,12 @@ public void HandleEventSentEvent(EventSentEvent eventSentEvent) public void HandleEventRaisedEvent(EventRaisedEvent eventRaisedEvent, bool skipCarryOverEvents, TaskOrchestration taskOrchestration) { + if (this.entityContext?.HandledAsEntityResponse(eventRaisedEvent, this) == true) + { + // this EventRaisedEvent was not an application-defined event, but an entity response + return; + } + if (skipCarryOverEvents || !this.HasContinueAsNew) { taskOrchestration.RaiseEvent(this, eventRaisedEvent.Name, eventRaisedEvent.Input); @@ -416,6 +541,47 @@ public void HandleEventRaisedEvent(EventRaisedEvent eventRaisedEvent, bool skipC } } + public void HandleEntityOperationCompletedEvent(EventRaisedEvent eventRaisedEvent, int taskId, string instanceId, OperationResult operationResult, TaskCompletionSource tcs) + { + T result; + if (operationResult.Result != null) + { + result = this.MessageDataConverter.Deserialize(operationResult.Result); + } + else + { + result = default; + } + tcs.SetResult(result); + } + + public void HandleEntityOperationFailedEvent(EventRaisedEvent eventRaisedEvent, int taskId, string instanceId, OperationResult operationResult, TaskCompletionSource tcs) + { + Exception cause = null; + + // When using ErrorPropagationMode.SerializeExceptions the "cause" is deserialized from history. + // This isn't fully reliable because not all exception types can be serialized/deserialized. + if (this.ErrorPropagationMode == ErrorPropagationMode.SerializeExceptions && !string.IsNullOrEmpty(operationResult.Result)) + { + try + { + cause = this.ErrorDataConverter.Deserialize(operationResult.Result); + } + catch (Exception converterException) when (!Utils.IsFatal(converterException)) + { + cause = new TaskFailedExceptionDeserializationException(operationResult.Result, converterException); + } + } + + var exceptionForCallingOrchestration = new OperationFailedException(eventRaisedEvent.EventId, taskId, instanceId, eventRaisedEvent.Name, operationResult.ErrorMessage, cause); + + if (this.ErrorPropagationMode == ErrorPropagationMode.UseFailureDetails) + { + exceptionForCallingOrchestration.FailureDetails = operationResult.FailureDetails; + } + + tcs.SetException(exceptionForCallingOrchestration); + } public void HandleTaskCompletedEvent(TaskCompletedEvent completedEvent) { @@ -497,8 +663,8 @@ public void HandleSubOrchestrationInstanceFailedEvent(SubOrchestrationInstanceFa // When using ErrorPropagationMode.UseFailureDetails we instead use FailureDetails to convey // error information, which doesn't involve any serialization at all. Exception cause = this.ErrorPropagationMode == ErrorPropagationMode.SerializeExceptions ? - Utils.RetrieveCause(failedEvent.Details, this.ErrorDataConverter) : - null; + Utils.RetrieveCause(failedEvent.Details, this.ErrorDataConverter) : + null; var failedException = new SubOrchestrationFailedException(failedEvent.EventId, taskId, info.Name, info.Version, @@ -608,7 +774,7 @@ public void FailOrchestration(Exception failure) details = orchestrationFailureException.Details; } } - else + else { if (this.ErrorPropagationMode == ErrorPropagationMode.UseFailureDetails) { @@ -625,6 +791,10 @@ public void FailOrchestration(Exception failure) public void CompleteOrchestration(string result, string details, OrchestrationStatus orchestrationStatus, FailureDetails failureDetails = null) { + // if we are still inside a critical section, we exit it before completing the orchestration + // so that the lock release messages are sent. + this.ExitCriticalSection(); + int id = this.idCounter++; OrchestrationCompleteOrchestratorAction completedOrchestratorAction; if (orchestrationStatus == OrchestrationStatus.Completed && this.continueAsNew != null) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index d7e3dcc98..f4cf20dbc 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -22,6 +22,7 @@ namespace DurableTask.Core using System.Threading.Tasks; using DurableTask.Core.Command; using DurableTask.Core.Common; + using DurableTask.Core.Entities; using DurableTask.Core.Exceptions; using DurableTask.Core.History; using DurableTask.Core.Logging; @@ -43,6 +44,8 @@ public class TaskOrchestrationDispatcher readonly LogHelper logHelper; ErrorPropagationMode errorPropagationMode; readonly NonBlockingCountdownLock concurrentSessionLock; + readonly IEntityOrchestrationService? entityOrchestrationService; + readonly EntityBackendProperties? entityBackendProperties; internal TaskOrchestrationDispatcher( IOrchestrationService orchestrationService, @@ -56,6 +59,8 @@ internal TaskOrchestrationDispatcher( this.dispatchPipeline = dispatchPipeline ?? throw new ArgumentNullException(nameof(dispatchPipeline)); this.logHelper = logHelper ?? throw new ArgumentNullException(nameof(logHelper)); this.errorPropagationMode = errorPropagationMode; + this.entityOrchestrationService = orchestrationService as IEntityOrchestrationService; + this.entityBackendProperties = this.entityOrchestrationService?.GetEntityBackendProperties(); this.dispatcher = new WorkItemDispatcher( "TaskOrchestrationDispatcher", @@ -113,7 +118,15 @@ public async Task StopAsync(bool forced) /// A new TaskOrchestrationWorkItem protected Task OnFetchWorkItemAsync(TimeSpan receiveTimeout, CancellationToken cancellationToken) { - return this.orchestrationService.LockNextTaskOrchestrationWorkItemAsync(receiveTimeout, cancellationToken); + if (this.entityOrchestrationService != null) + { + // we call the entity interface to make sure we are receiving only true orchestrations here + return this.entityOrchestrationService.LockNextOrchestrationWorkItemAsync(receiveTimeout, cancellationToken); + } + else + { + return this.orchestrationService.LockNextTaskOrchestrationWorkItemAsync(receiveTimeout, cancellationToken); + } } @@ -309,14 +322,14 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work { // start a task to run RenewUntil renewTask = Task.Factory.StartNew( - () => this.RenewUntil(workItem, renewCancellationTokenSource.Token), + () => RenewUntil(workItem, this.orchestrationService, this.logHelper, nameof(TaskOrchestrationDispatcher), renewCancellationTokenSource.Token), renewCancellationTokenSource.Token); } try { // Assumes that: if the batch contains a new "ExecutionStarted" event, it is the first message in the batch. - if (!this.ReconcileMessagesWithState(workItem)) + if (!ReconcileMessagesWithState(workItem, "TaskOrchestrationDispatcher", logHelper)) { // TODO : mark an orchestration as faulted if there is data corruption this.logHelper.DroppingOrchestrationWorkItem(workItem, "Received work-item for an invalid orchestration"); @@ -574,7 +587,6 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work instanceState.Status = runtimeState.Status; } - await this.orchestrationService.CompleteTaskOrchestrationWorkItemAsync( workItem, runtimeState, @@ -597,10 +609,10 @@ static OrchestrationExecutionContext GetOrchestrationExecutionContext(Orchestrat return new OrchestrationExecutionContext { OrchestrationTags = runtimeState.Tags ?? new Dictionary(capacity: 0) }; } - TimeSpan MinRenewalInterval = TimeSpan.FromSeconds(5); // prevents excessive retries if clocks are off - TimeSpan MaxRenewalInterval = TimeSpan.FromSeconds(30); + static TimeSpan MinRenewalInterval = TimeSpan.FromSeconds(5); // prevents excessive retries if clocks are off + static TimeSpan MaxRenewalInterval = TimeSpan.FromSeconds(30); - async Task RenewUntil(TaskOrchestrationWorkItem workItem, CancellationToken cancellationToken) + internal static async Task RenewUntil(TaskOrchestrationWorkItem workItem, IOrchestrationService orchestrationService, LogHelper logHelper, string dispatcher, CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { @@ -623,16 +635,16 @@ async Task RenewUntil(TaskOrchestrationWorkItem workItem, CancellationToken canc try { - this.logHelper.RenewOrchestrationWorkItemStarting(workItem); - TraceHelper.Trace(TraceEventType.Information, "TaskOrchestrationDispatcher-RenewWorkItemStarting", "Renewing work item for instance {0}", workItem.InstanceId); - await this.orchestrationService.RenewTaskOrchestrationWorkItemLockAsync(workItem); - this.logHelper.RenewOrchestrationWorkItemCompleted(workItem); - TraceHelper.Trace(TraceEventType.Information, "TaskOrchestrationDispatcher-RenewWorkItemCompleted", "Successfully renewed work item for instance {0}", workItem.InstanceId); + logHelper.RenewOrchestrationWorkItemStarting(workItem); + TraceHelper.Trace(TraceEventType.Information, $"{dispatcher}-RenewWorkItemStarting", "Renewing work item for instance {0}", workItem.InstanceId); + await orchestrationService.RenewTaskOrchestrationWorkItemLockAsync(workItem); + logHelper.RenewOrchestrationWorkItemCompleted(workItem); + TraceHelper.Trace(TraceEventType.Information, $"{dispatcher}-RenewWorkItemCompleted", "Successfully renewed work item for instance {0}", workItem.InstanceId); } catch (Exception exception) when (!Utils.IsFatal(exception)) { - this.logHelper.RenewOrchestrationWorkItemFailed(workItem, exception); - TraceHelper.TraceException(TraceEventType.Warning, "TaskOrchestrationDispatcher-RenewWorkItemFailed", exception, "Failed to renew work item for instance {0}", workItem.InstanceId); + logHelper.RenewOrchestrationWorkItemFailed(workItem, exception); + TraceHelper.TraceException(TraceEventType.Warning, $"{dispatcher}-RenewWorkItemFailed", exception, "Failed to renew work item for instance {0}", workItem.InstanceId); } } } @@ -677,7 +689,8 @@ await this.dispatchPipeline.RunAsync(dispatchContext, _ => runtimeState, taskOrchestration, this.orchestrationService.EventBehaviourForContinueAsNew, - this.errorPropagationMode); + this.entityBackendProperties, + this.errorPropagationMode); ; OrchestratorExecutionResult resultFromOrchestrator = executor.Execute(); dispatchContext.SetProperty(resultFromOrchestrator); return CompletedTask; @@ -719,8 +732,10 @@ await this.dispatchPipeline.RunAsync(dispatchContext, _ => /// Assumes that: if the batch contains a new "ExecutionStarted" event, it is the first message in the batch. /// /// A batch of work item messages. + /// The name of the dispatcher, used for tracing. + /// The log helper. /// True if workItem should be processed further. False otherwise. - bool ReconcileMessagesWithState(TaskOrchestrationWorkItem workItem) + internal static bool ReconcileMessagesWithState(TaskOrchestrationWorkItem workItem, string dispatcher, LogHelper logHelper) { foreach (TaskMessage message in workItem.NewMessages) { @@ -729,7 +744,7 @@ bool ReconcileMessagesWithState(TaskOrchestrationWorkItem workItem) { throw TraceHelper.TraceException( TraceEventType.Error, - "TaskOrchestrationDispatcher-OrchestrationInstanceMissing", + $"{dispatcher}-OrchestrationInstanceMissing", new InvalidOperationException("Message does not contain any OrchestrationInstance information")); } @@ -747,10 +762,10 @@ bool ReconcileMessagesWithState(TaskOrchestrationWorkItem workItem) return false; } - this.logHelper.ProcessingOrchestrationMessage(workItem, message); + logHelper.ProcessingOrchestrationMessage(workItem, message); TraceHelper.TraceInstance( TraceEventType.Information, - "TaskOrchestrationDispatcher-ProcessEvent", + $"{dispatcher}-ProcessEvent", orchestrationInstance, "Processing new event with Id {0} and type {1}", message.Event.EventId, @@ -761,10 +776,10 @@ bool ReconcileMessagesWithState(TaskOrchestrationWorkItem workItem) if (workItem.OrchestrationRuntimeState.ExecutionStartedEvent != null) { // this was caused due to a dupe execution started event, swallow this one - this.logHelper.DroppingOrchestrationMessage(workItem, message, "Duplicate start event"); + logHelper.DroppingOrchestrationMessage(workItem, message, "Duplicate start event"); TraceHelper.TraceInstance( TraceEventType.Warning, - "TaskOrchestrationDispatcher-DuplicateStartEvent", + $"{dispatcher}-DuplicateStartEvent", orchestrationInstance, "Duplicate start event. Ignoring event with Id {0} and type {1} ", message.Event.EventId, @@ -778,13 +793,13 @@ bool ReconcileMessagesWithState(TaskOrchestrationWorkItem workItem) workItem.OrchestrationRuntimeState.OrchestrationInstance?.ExecutionId)) { // eat up any events for previous executions - this.logHelper.DroppingOrchestrationMessage( + logHelper.DroppingOrchestrationMessage( workItem, message, $"ExecutionId of event ({orchestrationInstance.ExecutionId}) does not match current executionId"); TraceHelper.TraceInstance( TraceEventType.Warning, - "TaskOrchestrationDispatcher-ExecutionIdMismatch", + $"{dispatcher}-ExecutionIdMismatch", orchestrationInstance, "ExecutionId of event does not match current executionId. Ignoring event with Id {0} and type {1} ", message.Event.EventId, @@ -1036,7 +1051,7 @@ TaskMessage ProcessSendEventDecision( }; } - class NonBlockingCountdownLock + internal class NonBlockingCountdownLock { int available; diff --git a/src/DurableTask.Core/TaskOrchestrationExecutor.cs b/src/DurableTask.Core/TaskOrchestrationExecutor.cs index b0ca99976..c9fc3f026 100644 --- a/src/DurableTask.Core/TaskOrchestrationExecutor.cs +++ b/src/DurableTask.Core/TaskOrchestrationExecutor.cs @@ -22,6 +22,7 @@ namespace DurableTask.Core using System.Threading; using System.Threading.Tasks; using DurableTask.Core.Common; + using DurableTask.Core.Entities; using DurableTask.Core.Exceptions; using DurableTask.Core.History; @@ -43,23 +44,44 @@ public class TaskOrchestrationExecutor /// /// /// + /// /// public TaskOrchestrationExecutor( OrchestrationRuntimeState orchestrationRuntimeState, TaskOrchestration taskOrchestration, BehaviorOnContinueAsNew eventBehaviourForContinueAsNew, + EntityBackendProperties? entityBackendProperties, ErrorPropagationMode errorPropagationMode = ErrorPropagationMode.SerializeExceptions) { this.decisionScheduler = new SynchronousTaskScheduler(); this.context = new TaskOrchestrationContext( orchestrationRuntimeState.OrchestrationInstance, this.decisionScheduler, - errorPropagationMode); + entityBackendProperties, + errorPropagationMode + ); this.orchestrationRuntimeState = orchestrationRuntimeState; this.taskOrchestration = taskOrchestration; this.skipCarryOverEvents = eventBehaviourForContinueAsNew == BehaviorOnContinueAsNew.Ignore; } + /// + /// Initializes a new instance of the class. + /// This overload is needed only to avoid breaking changes because this is a public constructor. + /// + /// + /// + /// + /// + public TaskOrchestrationExecutor( + OrchestrationRuntimeState orchestrationRuntimeState, + TaskOrchestration taskOrchestration, + BehaviorOnContinueAsNew eventBehaviourForContinueAsNew, + ErrorPropagationMode errorPropagationMode = ErrorPropagationMode.SerializeExceptions) + : this(orchestrationRuntimeState, taskOrchestration, eventBehaviourForContinueAsNew, new EntityBackendProperties(), errorPropagationMode) + { + } + internal bool IsCompleted => this.result != null && (this.result.IsCompleted || this.result.IsFaulted); /// diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index 5d9ca75b9..d1b32eaaa 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -19,14 +19,18 @@ namespace DurableTask.AzureStorage.Tests using System.Diagnostics; using System.IO; using System.Linq; + using System.Reflection; using System.Runtime.Serialization; using System.Text; using System.Threading; using System.Threading.Tasks; using DurableTask.AzureStorage.Tracking; using DurableTask.Core; + using DurableTask.Core.Entities; + using DurableTask.Core.Entities.OperationFormat; using DurableTask.Core.Exceptions; using DurableTask.Core.History; + using DurableTask.Core.Serializing; using Microsoft.Practices.EnterpriseLibrary.SemanticLogging.Utility; using Microsoft.VisualStudio.TestTools.UnitTesting; using Microsoft.WindowsAzure.Storage; @@ -348,2331 +352,3493 @@ public async Task ContinueAsNewThenTimer(bool enableExtendedSessions) } } - [TestMethod] - public async Task PurgeInstanceHistoryForSingleInstanceWithoutLargeMessageBlobs() + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task CallCounterEntityFromOrchestration(bool enableExtendedSessions) { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) { - string instanceId = Guid.NewGuid().ToString(); await host.StartAsync(); - TestOrchestrationClient client = await host.StartOrchestrationAsync(typeof(Orchestrations.Factorial), 110, instanceId); - await client.WaitForCompletionAsync(TimeSpan.FromSeconds(60)); - - List historyEvents = await client.GetOrchestrationHistoryAsync(instanceId); - Assert.IsTrue(historyEvents.Count > 0); - - IList orchestrationStateList = await client.GetStateAsync(instanceId); - Assert.AreEqual(1, orchestrationStateList.Count); - Assert.AreEqual(instanceId, orchestrationStateList.First().OrchestrationInstance.InstanceId); - - await client.PurgeInstanceHistory(); - List historyEventsAfterPurging = await client.GetOrchestrationHistoryAsync(instanceId); - Assert.AreEqual(0, historyEventsAfterPurging.Count); + var entityId = new EntityId(nameof(Entities.Counter), Guid.NewGuid().ToString()); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.CallCounterEntity), entityId); + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - orchestrationStateList = await client.GetStateAsync(instanceId); - Assert.AreEqual(1, orchestrationStateList.Count); - Assert.IsNull(orchestrationStateList.First()); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual("OK", JToken.Parse(status?.Output)); await host.StopAsync(); } } - [TestMethod] - public async Task ValidateCustomStatusPersists() + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task BatchedEntitySignals(bool enableExtendedSessions) { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(false)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) { await host.StartAsync(); + + int numIterations = 100; + var entityId = new EntityId(nameof(Entities.BatchEntity), Guid.NewGuid().ToString()); + TestEntityClient client = await host.GetEntityClientAsync(typeof(Entities.BatchEntity), entityId); + + // send a number of signals immediately after each other + List tasks = new List(); + for (int i = 0; i < numIterations; i++) + { + tasks.Add(client.SignalEntity(i.ToString())); + } - string customStatus = "custom_status"; - var client = await host.StartOrchestrationAsync( - typeof(Test.Orchestrations.ChangeStatusOrchestration), - new string[] { customStatus }); - var state = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + await Task.WhenAll(tasks); - Assert.AreEqual(OrchestrationStatus.Completed, state?.OrchestrationStatus); - Assert.AreEqual(customStatus, JToken.Parse(state?.Status)); + var result = await client.WaitForEntityState>( + timeout: Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(20), + list => list.Count == numIterations ? null : $"waiting for {numIterations - list.Count} signals"); - await host.StopAsync(); - } - } + // validate the batching positions and sizes + int? cursize = null; + int curpos = 0; + int numBatches = 0; + foreach (var (position, size) in result) + { + if (cursize == null) + { + cursize = size; + curpos = 0; + numBatches++; + } - [TestMethod] - public async Task ValidateNullCustomStatusPersists() - { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(false)) - { - await host.StartAsync(); + Assert.AreEqual(curpos, position); - var client = await host.StartOrchestrationAsync( - typeof(Test.Orchestrations.ChangeStatusOrchestration), - // First set "custom_status", then set null and make sure it persists - new string[] { "custom_status", null }); - var state = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + if (++curpos == cursize) + { + cursize = null; + } + } - Assert.AreEqual(OrchestrationStatus.Completed, state?.OrchestrationStatus); - Assert.AreEqual(null, JToken.Parse(state?.Status).Value()); + // there should always be some batching going on + Assert.IsTrue(numBatches < numIterations); await host.StopAsync(); } } - [TestMethod] - public async Task PurgeInstanceHistoryForSingleInstanceWithLargeMessageBlobs() + [DataTestMethod] + public async Task CleanEntityStorage_OrphanedLock() { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost( + enableExtendedSessions: false, + modifySettingsAction: (settings) => settings.EntityMessageReorderWindowInMinutes = 0)) { await host.StartAsync(); - string instanceId = Guid.NewGuid().ToString(); - string message = this.GenerateMediumRandomStringPayload().ToString(); - TestOrchestrationClient client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message, instanceId); - OrchestrationState status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - - List historyEvents = await client.GetOrchestrationHistoryAsync(instanceId); - Assert.IsTrue(historyEvents.Count > 0); - - IList orchestrationStateList = await client.GetStateAsync(instanceId); - Assert.AreEqual(1, orchestrationStateList.Count); - Assert.AreEqual(instanceId, orchestrationStateList.First().OrchestrationInstance.InstanceId); - - int blobCount = await this.GetBlobCount("test-largemessages", instanceId); - Assert.IsTrue(blobCount > 0); - - IList results = await host.GetAllOrchestrationInstancesAsync(); - Assert.AreEqual(1, results.Count); + // construct unique names for this test + string prefix = Guid.NewGuid().ToString("N").Substring(0, 6); + var orphanedEntityId = new EntityId(nameof(Entities.Counter), $"{prefix}-orphaned"); + var orchestrationA = $"{prefix}-A"; + var orchestrationB = $"{prefix}-B"; - string result = JToken.Parse(results.First(x => x.OrchestrationInstance.InstanceId == instanceId).Output).ToString(); - Assert.AreEqual(message, result); + // run an orchestration A that leaves an orphaned lock + var clientA = await host.StartOrchestrationAsync(typeof(Orchestrations.LockThenFailReplay), (orphanedEntityId, true), orchestrationA); + var status = await clientA.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - await client.PurgeInstanceHistory(); + // run an orchestration B that queues behind A for the lock (and thus gets stuck) + var clientB = await host.StartOrchestrationAsync(typeof(Orchestrations.LockThenFailReplay), (orphanedEntityId, false), orchestrationB); + // remove empty entity and release orphaned lock + var entityClient = new TaskHubEntityClient(clientA.InnerClient); + var response = await entityClient.CleanEntityStorageAsync(true, true, CancellationToken.None); + Assert.AreEqual(1, response.NumberOfOrphanedLocksRemoved); + Assert.AreEqual(0, response.NumberOfEmptyEntitiesRemoved); - List historyEventsAfterPurging = await client.GetOrchestrationHistoryAsync(instanceId); - Assert.AreEqual(0, historyEventsAfterPurging.Count); + // wait for orchestration B to complete, now that the lock has been released + status = await clientB.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + Assert.IsTrue(status.OrchestrationStatus == OrchestrationStatus.Completed); - orchestrationStateList = await client.GetStateAsync(instanceId); - Assert.AreEqual(1, orchestrationStateList.Count); - Assert.IsNull(orchestrationStateList.First()); + // give the orchestration status time to be updated + await Task.Delay(TimeSpan.FromSeconds(20)); - blobCount = await this.GetBlobCount("test-largemessages", instanceId); - Assert.AreEqual(0, blobCount); + // clean again to remove the orphaned entity which is now empty also + response = await entityClient.CleanEntityStorageAsync(true, true, CancellationToken.None); + Assert.AreEqual(0, response.NumberOfOrphanedLocksRemoved); + Assert.AreEqual(1, response.NumberOfEmptyEntitiesRemoved); await host.StopAsync(); } } - [TestMethod] - public async Task PurgeInstanceHistoryForTimePeriodDeleteAll() + [DataTestMethod] + [DataRow(1)] + [DataRow(120)] + public async Task CleanEntityStorage_EmptyEntities(int numReps) { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost( + enableExtendedSessions: false, + modifySettingsAction: (settings) => settings.EntityMessageReorderWindowInMinutes = 0)) { await host.StartAsync(); - DateTime startDateTime = DateTime.Now; - string firstInstanceId = "instance1"; - TestOrchestrationClient client = await host.StartOrchestrationAsync(typeof(Orchestrations.FanOutFanIn), 50, firstInstanceId); - await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - string secondInstanceId = "instance2"; - client = await host.StartOrchestrationAsync(typeof(Orchestrations.FanOutFanIn), 50, secondInstanceId); - await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - string thirdInstanceId = "instance3"; - client = await host.StartOrchestrationAsync(typeof(Orchestrations.FanOutFanIn), 50, thirdInstanceId); - await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - string fourthInstanceId = "instance4"; - string message = this.GenerateMediumRandomStringPayload().ToString(); - client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message, fourthInstanceId); - var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + // construct unique names for this test + string prefix = Guid.NewGuid().ToString("N").Substring(0, 6); + EntityId[] entityIds = new EntityId[numReps]; + for (int i = 0; i < entityIds.Length; i++) + { + entityIds[i] = new EntityId("Counter", $"{prefix}-{i:D3}"); + } - IList results = await host.GetAllOrchestrationInstancesAsync(); - Assert.AreEqual(4, results.Count); - Assert.AreEqual("\"Done\"", results.First(x => x.OrchestrationInstance.InstanceId == firstInstanceId).Output); - Assert.AreEqual("\"Done\"", results.First(x => x.OrchestrationInstance.InstanceId == secondInstanceId).Output); - Assert.AreEqual("\"Done\"", results.First(x => x.OrchestrationInstance.InstanceId == thirdInstanceId).Output); - string result = JToken.Parse(results.First(x => x.OrchestrationInstance.InstanceId == fourthInstanceId).Output).ToString(); - Assert.AreEqual(message, result); + // create the empty entities + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.CreateEmptyEntities), entityIds); + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - List firstHistoryEvents = await client.GetOrchestrationHistoryAsync(firstInstanceId); - Assert.IsTrue(firstHistoryEvents.Count > 0); + // account for delay in updating instance tables + await Task.Delay(TimeSpan.FromSeconds(20)); - List secondHistoryEvents = await client.GetOrchestrationHistoryAsync(secondInstanceId); - Assert.IsTrue(secondHistoryEvents.Count > 0); + // remove all empty entities + var entityClient = new TaskHubEntityClient(client.InnerClient); + var response = await entityClient.CleanEntityStorageAsync(true, true, CancellationToken.None); + Assert.AreEqual(0, response.NumberOfOrphanedLocksRemoved); + Assert.AreEqual(numReps, response.NumberOfEmptyEntitiesRemoved); - List thirdHistoryEvents = await client.GetOrchestrationHistoryAsync(thirdInstanceId); - Assert.IsTrue(thirdHistoryEvents.Count > 0); + await host.StopAsync(); + } + } - List fourthHistoryEvents = await client.GetOrchestrationHistoryAsync(thirdInstanceId); - Assert.IsTrue(fourthHistoryEvents.Count > 0); + [DataTestMethod] + public async Task EntityQueries() + { + var yesterday = DateTime.UtcNow.Subtract(TimeSpan.FromDays(1)); + var tomorrow = DateTime.UtcNow.Add(TimeSpan.FromDays(1)); - IList firstOrchestrationStateList = await client.GetStateAsync(firstInstanceId); - Assert.AreEqual(1, firstOrchestrationStateList.Count); - Assert.AreEqual(firstInstanceId, firstOrchestrationStateList.First().OrchestrationInstance.InstanceId); + List entityIds = new List() + { + new EntityId("StringStore", "foo"), + new EntityId("StringStore", "bar"), + new EntityId("StringStore", "baz"), + new EntityId("StringStore2", "foo"), + }; - IList secondOrchestrationStateList = await client.GetStateAsync(secondInstanceId); - Assert.AreEqual(1, secondOrchestrationStateList.Count); - Assert.AreEqual(secondInstanceId, secondOrchestrationStateList.First().OrchestrationInstance.InstanceId); + var queries = new (TaskHubEntityClient.Query,Action>)[] + { + (new TaskHubEntityClient.Query + { + }, + result => + { + Assert.AreEqual(4, result.Count); + }), - IList thirdOrchestrationStateList = await client.GetStateAsync(thirdInstanceId); - Assert.AreEqual(1, thirdOrchestrationStateList.Count); - Assert.AreEqual(thirdInstanceId, thirdOrchestrationStateList.First().OrchestrationInstance.InstanceId); + (new TaskHubEntityClient.Query + { + EntityName = "StringStore", + LastOperationFrom = yesterday, + LastOperationTo = tomorrow, + FetchState = false, + }, + result => + { + Assert.AreEqual(3, result.Count); + Assert.IsTrue(result[0].State == null); + }), - IList fourthOrchestrationStateList = await client.GetStateAsync(fourthInstanceId); - Assert.AreEqual(1, fourthOrchestrationStateList.Count); - Assert.AreEqual(fourthInstanceId, fourthOrchestrationStateList.First().OrchestrationInstance.InstanceId); + (new TaskHubEntityClient.Query + { + EntityName = "StringStore", + LastOperationFrom = yesterday, + LastOperationTo = tomorrow, + FetchState = true, + }, + result => + { + Assert.AreEqual(3, result.Count); + Assert.IsTrue(result[0].State != null); + }), - int blobCount = await this.GetBlobCount("test-largemessages", fourthInstanceId); - Assert.AreEqual(6, blobCount); + (new TaskHubEntityClient.Query + { + EntityName = "StringStore", + PageSize = 1, + }, + result => + { + Assert.AreEqual(3, result.Count()); + }), - await client.PurgeInstanceHistoryByTimePeriod( - startDateTime, - DateTime.UtcNow, - new List - { - OrchestrationStatus.Completed, - OrchestrationStatus.Terminated, - OrchestrationStatus.Failed, - OrchestrationStatus.Running - }); + (new TaskHubEntityClient.Query + { + EntityName = "StringStore", + PageSize = 2, + }, + result => + { + Assert.AreEqual(3, result.Count()); + }), - List firstHistoryEventsAfterPurging = await client.GetOrchestrationHistoryAsync(firstInstanceId); - Assert.AreEqual(0, firstHistoryEventsAfterPurging.Count); + (new TaskHubEntityClient.Query + { + EntityName = "noResult", + }, + result => + { + Assert.AreEqual(0, result.Count()); + }), - List secondHistoryEventsAfterPurging = await client.GetOrchestrationHistoryAsync(secondInstanceId); - Assert.AreEqual(0, secondHistoryEventsAfterPurging.Count); + (new TaskHubEntityClient.Query + { + EntityName = "noResult", + LastOperationFrom = yesterday, + LastOperationTo = tomorrow, + }, + result => + { + Assert.AreEqual(0, result.Count()); + }), - List thirdHistoryEventsAfterPurging = await client.GetOrchestrationHistoryAsync(thirdInstanceId); - Assert.AreEqual(0, thirdHistoryEventsAfterPurging.Count); + (new TaskHubEntityClient.Query + { + LastOperationFrom = tomorrow, + }, + result => + { + Assert.AreEqual(0, result.Count()); + }), - ListfourthHistoryEventsAfterPurging = await client.GetOrchestrationHistoryAsync(fourthInstanceId); - Assert.AreEqual(0, fourthHistoryEventsAfterPurging.Count); + (new TaskHubEntityClient.Query + { + LastOperationTo = yesterday, + }, + result => + { + Assert.AreEqual(0, result.Count()); + }), - firstOrchestrationStateList = await client.GetStateAsync(firstInstanceId); - Assert.AreEqual(1, firstOrchestrationStateList.Count); - Assert.IsNull(firstOrchestrationStateList.First()); + (new TaskHubEntityClient.Query + { + InstanceIdPrefix = "StringStore", + }, + result => + { + Assert.AreEqual(0, result.Count()); + }), - secondOrchestrationStateList = await client.GetStateAsync(secondInstanceId); - Assert.AreEqual(1, secondOrchestrationStateList.Count); - Assert.IsNull(secondOrchestrationStateList.First()); + (new TaskHubEntityClient.Query + { + InstanceIdPrefix = "@StringStore", + }, + result => + { + Assert.AreEqual(4, result.Count()); + }), - thirdOrchestrationStateList = await client.GetStateAsync(thirdInstanceId); - Assert.AreEqual(1, thirdOrchestrationStateList.Count); - Assert.IsNull(thirdOrchestrationStateList.First()); + (new TaskHubEntityClient.Query + { + InstanceIdPrefix = "@stringstore", + }, + result => + { + Assert.AreEqual(0, result.Count()); + }), - fourthOrchestrationStateList = await client.GetStateAsync(fourthInstanceId); - Assert.AreEqual(1, fourthOrchestrationStateList.Count); - Assert.IsNull(fourthOrchestrationStateList.First()); + (new TaskHubEntityClient.Query + { + InstanceIdPrefix = "@StringStore@", + }, + result => + { + Assert.AreEqual(3, result.Count()); + }), - blobCount = await this.GetBlobCount("test-largemessages", fourthInstanceId); - Assert.AreEqual(0, blobCount); + (new TaskHubEntityClient.Query + { + InstanceIdPrefix = "@StringStore", + EntityName = "StringStore", + }, + result => + { + Assert.AreEqual(3, result.Count()); + }), - await host.StopAsync(); - } + (new TaskHubEntityClient.Query + { + InstanceIdPrefix = "@StringStore@", + EntityName = "StringStore", + }, + result => + { + Assert.AreEqual(3, result.Count()); + }), + + (new TaskHubEntityClient.Query + { + InstanceIdPrefix = "@StringStore@b", + EntityName = "StringStore", + }, + result => + { + Assert.AreEqual(2, result.Count()); + }), + }; + + await this.RunEntityQueries(queries, entityIds); } - private async Task GetBlobCount(string containerName, string directoryName) + [DataTestMethod] + public async Task EntityQueries_Deleted() { - string storageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(); - CloudStorageAccount storageAccount; - if (!CloudStorageAccount.TryParse(storageConnectionString, out storageAccount)) + var queries = new (TaskHubEntityClient.Query, Action>)[] { - Assert.Fail("Couldn't find the connection string to use to look up blobs!"); - return 0; - } - - CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient(); + (new TaskHubEntityClient.Query() + { + IncludeDeleted = false, + }, + result => + { + Assert.AreEqual(4, result.Count); + }), - CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(containerName); - await cloudBlobContainer.CreateIfNotExistsAsync(); - CloudBlobDirectory instanceDirectory = cloudBlobContainer.GetDirectoryReference(directoryName); - var blobs = new List(); - BlobContinuationToken blobContinuationToken = null; - do - { - BlobResultSegment results = await TimeoutHandler.ExecuteWithTimeout("GetBlobCount", "dummyAccount", new AzureStorageOrchestrationServiceSettings(), (context, timeoutToken) => + (new TaskHubEntityClient.Query() { - return instanceDirectory.ListBlobsSegmentedAsync( - useFlatBlobListing: true, - blobListingDetails: BlobListingDetails.Metadata, - maxResults: null, - currentToken: blobContinuationToken, - options: null, - operationContext: context, - cancellationToken: timeoutToken); - }); - - blobContinuationToken = results.ContinuationToken; - blobs.AddRange(results.Results); - } while (blobContinuationToken != null); + IncludeDeleted = true, + }, + result => + { + Assert.AreEqual(8, result.Count); + }), - Trace.TraceInformation( - "Found {0} blobs: {1}{2}", - blobs.Count, - Environment.NewLine, - string.Join(Environment.NewLine, blobs.Select(b => b.Uri))); - return blobs.Count; - } + (new TaskHubEntityClient.Query() + { + IncludeDeleted = false, + PageSize = 3, + }, + result => + { + Assert.AreEqual(4, result.Count); + }), + (new TaskHubEntityClient.Query() + { + IncludeDeleted = true, + PageSize = 3, + }, + result => + { + Assert.AreEqual(8, result.Count); + }), + }; - [TestMethod] - public async Task PurgeInstanceHistoryForTimePeriodDeletePartially() - { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false)) + List entityIds = new List() { - // Execute the orchestrator twice. Orchestrator will be replied. However instances might be two. - await host.StartAsync(); - DateTime startDateTime = DateTime.Now; - string firstInstanceId = Guid.NewGuid().ToString(); - TestOrchestrationClient client = await host.StartOrchestrationAsync(typeof(Orchestrations.FanOutFanIn), 50, firstInstanceId); - await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - DateTime endDateTime = DateTime.Now; - await Task.Delay(5000); - string secondInstanceId = Guid.NewGuid().ToString(); - client = await host.StartOrchestrationAsync(typeof(Orchestrations.FanOutFanIn), 50, secondInstanceId); - await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - string thirdInstanceId = Guid.NewGuid().ToString(); - client = await host.StartOrchestrationAsync(typeof(Orchestrations.FanOutFanIn), 50, thirdInstanceId); - await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - - IList results = await host.GetAllOrchestrationInstancesAsync(); - Assert.AreEqual(3, results.Count); - Assert.IsNotNull(results[0].Output.Equals("\"Done\"")); - Assert.IsNotNull(results[1].Output.Equals("\"Done\"")); - Assert.IsNotNull(results[2].Output.Equals("\"Done\"")); - - - List firstHistoryEvents = await client.GetOrchestrationHistoryAsync(firstInstanceId); - Assert.IsTrue(firstHistoryEvents.Count > 0); + new EntityId("StringStore", "foo"), + new EntityId("StringStore2", "bar"), + new EntityId("StringStore2", "baz"), + new EntityId("StringStore2", "foo"), + new EntityId("StringStore2", "ffo"), + new EntityId("StringStore2", "zzz"), + new EntityId("StringStore2", "aaa"), + new EntityId("StringStore2", "bbb"), + }; - List secondHistoryEvents = await client.GetOrchestrationHistoryAsync(secondInstanceId); - Assert.IsTrue(secondHistoryEvents.Count > 0); + List orchestrations = new List() + { + typeof(Orchestrations.SignalAndCallStringStore), + typeof(Orchestrations.CallAndDeleteStringStore), + typeof(Orchestrations.SignalAndCallStringStore), + typeof(Orchestrations.CallAndDeleteStringStore), + typeof(Orchestrations.SignalAndCallStringStore), + typeof(Orchestrations.CallAndDeleteStringStore), + typeof(Orchestrations.SignalAndCallStringStore), + typeof(Orchestrations.CallAndDeleteStringStore), + }; - List thirdHistoryEvents = await client.GetOrchestrationHistoryAsync(thirdInstanceId); - Assert.IsTrue(secondHistoryEvents.Count > 0); + await this.RunEntityQueries(queries, entityIds, orchestrations); + } - IList firstOrchestrationStateList = await client.GetStateAsync(firstInstanceId); - Assert.AreEqual(1, firstOrchestrationStateList.Count); - Assert.AreEqual(firstInstanceId, firstOrchestrationStateList.First().OrchestrationInstance.InstanceId); + private async Task RunEntityQueries( + (TaskHubEntityClient.Query, Action>)[] queries, + IList entitiyIds, + IList orchestrations = null) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false)) + { + await host.StartAsync(); - IList secondOrchestrationStateList = await client.GetStateAsync(secondInstanceId); - Assert.AreEqual(1, secondOrchestrationStateList.Count); - Assert.AreEqual(secondInstanceId, secondOrchestrationStateList.First().OrchestrationInstance.InstanceId); + TestOrchestrationClient client = null; - IList thirdOrchestrationStateList = await client.GetStateAsync(thirdInstanceId); - Assert.AreEqual(1, thirdOrchestrationStateList.Count); - Assert.AreEqual(thirdInstanceId, thirdOrchestrationStateList.First().OrchestrationInstance.InstanceId); + for (int i = 0; i < entitiyIds.Count; i++) + { + EntityId id = entitiyIds[i]; + Type orchestration = orchestrations == null ? typeof(Orchestrations.SignalAndCallStringStore) : orchestrations[i]; + client = await host.StartOrchestrationAsync(orchestration, id); + await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + } - await client.PurgeInstanceHistoryByTimePeriod(startDateTime, endDateTime, new List { OrchestrationStatus.Completed, OrchestrationStatus.Terminated, OrchestrationStatus.Failed, OrchestrationStatus.Running }); + // account for delay in updating instance tables + await Task.Delay(TimeSpan.FromSeconds(10)); - List firstHistoryEventsAfterPurging = await client.GetOrchestrationHistoryAsync(firstInstanceId); - Assert.AreEqual(0, firstHistoryEventsAfterPurging.Count); + for (int i = 0; i < queries.Count(); i++) + { + var query = queries[i].Item1; + var test = queries[i].Item2; + var results = new List(); + var entityClient = new TaskHubEntityClient(client.InnerClient); - List secondHistoryEventsAfterPurging = await client.GetOrchestrationHistoryAsync(secondInstanceId); - Assert.IsTrue(secondHistoryEventsAfterPurging.Count > 0); + do + { + var result = await entityClient.ListEntitiesAsync(query, CancellationToken.None); - List thirdHistoryEventsAfterPurging = await client.GetOrchestrationHistoryAsync(thirdInstanceId); - Assert.IsTrue(thirdHistoryEventsAfterPurging.Count > 0); + // The result may return fewer records than the page size, but never more + Assert.IsTrue(result.Entities.Count() <= query.PageSize); - firstOrchestrationStateList = await client.GetStateAsync(firstInstanceId); - Assert.AreEqual(1, firstOrchestrationStateList.Count); - Assert.IsNull(firstOrchestrationStateList.First()); + foreach (var element in result.Entities) + { + results.Add(element); + } - secondOrchestrationStateList = await client.GetStateAsync(secondInstanceId); - Assert.AreEqual(1, secondOrchestrationStateList.Count); - Assert.AreEqual(secondInstanceId, secondOrchestrationStateList.First().OrchestrationInstance.InstanceId); + query.ContinuationToken = result.ContinuationToken; + } + while (query.ContinuationToken != null); - thirdOrchestrationStateList = await client.GetStateAsync(thirdInstanceId); - Assert.AreEqual(1, thirdOrchestrationStateList.Count); - Assert.AreEqual(thirdInstanceId, thirdOrchestrationStateList.First().OrchestrationInstance.InstanceId); + test(results); + } await host.StopAsync(); } } /// - /// End-to-end test which validates parallel function execution by enumerating all files in the current directory - /// in parallel and getting the sum total of all file sizes. + /// End-to-end test which validates launching orchestrations from entities. /// [DataTestMethod] - [DataRow(true)] [DataRow(false)] - public async Task ParallelOrchestration(bool enableExtendedSessions) + [DataRow(true)] + public async Task EntityFireAndForget(bool extendedSessions) { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: extendedSessions)) { await host.StartAsync(); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.DiskUsage), Environment.CurrentDirectory); - var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(90)); + var entityId = new EntityId(nameof(Entities.Launcher), Guid.NewGuid().ToString()); + + var client = await host.StartOrchestrationAsync( + typeof(Orchestrations.LaunchOrchestrationFromEntity), + entityId); + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(90)); Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.AreEqual(Environment.CurrentDirectory, JToken.Parse(status?.Input)); - Assert.IsTrue(long.Parse(status?.Output) > 0L); + + var instanceId = (string) JToken.Parse(status?.Output); + Assert.IsTrue(instanceId != null); + var orchestrationState = (await client.InnerClient.GetOrchestrationStateAsync(instanceId, false)).FirstOrDefault(); + Assert.AreEqual(OrchestrationStatus.Completed, orchestrationState.OrchestrationStatus); await host.StopAsync(); } } + /// + /// End-to-end test which validates two simple entity scenarios: + /// a) send a signal to a counter, then poll until the signal is delivered. + /// b) same, but send the signal indirectly via a relay entity. + /// [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task LargeFanOutOrchestration(bool enableExtendedSessions) + [DataRow(false, false)] + [DataRow(true, false)] + [DataRow(false, true)] + [DataRow(true, true)] + public async Task DurableEntity_SignalThenPoll(bool extendedSessions, bool viaRelay) { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: extendedSessions)) { await host.StartAsync(); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.FanOutFanIn), 1000); - var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(5)); + var relayEntityId = new EntityId("Relay", ""); + var counterEntityId = new EntityId("Counter", Guid.NewGuid().ToString()); + + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.PollCounterEntity), counterEntityId); + var entityClient = new TaskHubEntityClient(client.InnerClient); + + if (viaRelay) + { + await entityClient.SignalEntityAsync(relayEntityId, "", (counterEntityId, "increment")); + } + else + { + await entityClient.SignalEntityAsync(counterEntityId, "increment"); + } + + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual("ok", (string)JToken.Parse(status?.Output)); await host.StopAsync(); } } - [TestMethod] - public async Task FanOutOrchestration_LargeHistoryBatches() + /// + /// End-to-end test which validates an entity scenario where a "LockedTransfer" orchestration locks + /// two "Counter" entities, and then in parallel increments/decrements them, respectively, using + /// a read-modify-write pattern. + /// + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task DurableEntity_SingleLockedTransfer(bool extendedSessions) { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: true)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: extendedSessions)) { await host.StartAsync(); - // This test creates history payloads that exceed the 4 MB limit imposed by Azure Storage - // when 100 entities are uploaded at a time. - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.SemiLargePayloadFanOutFanIn), 90); - var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(5)); + var counter1 = new EntityId("Counter", Guid.NewGuid().ToString()); + var counter2 = new EntityId("Counter", Guid.NewGuid().ToString()); + + var client = await host.StartOrchestrationAsync( + typeof(Orchestrations.LockedTransfer), + (counter1, counter2)); + + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + // validate the state of the counters + var entityClient = new TaskHubEntityClient(client.InnerClient); + var response1 = await entityClient.ReadEntityStateAsync(counter1); + var response2 = await entityClient.ReadEntityStateAsync(counter2); + Assert.IsTrue(response1.EntityExists); + Assert.IsTrue(response2.EntityExists); + Assert.AreEqual(-1, response1.EntityState); + Assert.AreEqual(1, response2.EntityState); + await host.StopAsync(); } } /// - /// End-to-end test which validates the ContinueAsNew functionality by implementing a counter actor pattern. + /// End-to-end test which validates an entity scenario where a a number of "LockedTransfer" orchestrations + /// concurrently operate on a number of entities, in a classical dining-philosophers configuration. + /// This showcases the deadlock prevention mechanism achieved by the sequential, ordered lock acquisition. /// [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task ActorOrchestration(bool enableExtendedSessions) + [DataRow(false, 5)] + [DataRow(true, 5)] + public async Task DurableEntity_MultipleLockedTransfers(bool extendedSessions, int numberEntities) { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: extendedSessions)) { await host.StartAsync(); - int initialValue = 0; - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Counter), initialValue); + // create specified number of entities + var counters = new EntityId[numberEntities]; + for (int i = 0; i < numberEntities; i++) + { + counters[i] = new EntityId("Counter", Guid.NewGuid().ToString()); + } - // Need to wait for the instance to start before sending events to it. - // TODO: This requirement may not be ideal and should be revisited. - await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); + // in parallel, start one transfer per counter, each decrementing a counter and incrementing + // its successor (where the last one wraps around to the first) + // This is a pattern that would deadlock if we didn't order the lock acquisition. + var clients = new Task[numberEntities]; + for (int i = 0; i < numberEntities; i++) + { + clients[i] = host.StartOrchestrationAsync( + typeof(Orchestrations.LockedTransfer), + (counters[i], counters[(i + 1) % numberEntities])); + } - // Perform some operations - await client.RaiseEventAsync("operation", "incr"); - await client.RaiseEventAsync("operation", "incr"); - await client.RaiseEventAsync("operation", "incr"); - await client.RaiseEventAsync("operation", "decr"); - await client.RaiseEventAsync("operation", "incr"); - await Task.Delay(2000); + await Task.WhenAll(clients); - // Make sure it's still running and didn't complete early (or fail). - var status = await client.GetStatusAsync(); - Assert.IsTrue( - status?.OrchestrationStatus == OrchestrationStatus.Running || - status?.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew); + // in parallel, wait for all transfers to complete + var stati = new Task[numberEntities]; + for (int i = 0; i < numberEntities; i++) + { + stati[i] = clients[i].Result.WaitForCompletionAsync(TimeSpan.FromSeconds(60)); + } - // The end message will cause the actor to complete itself. - await client.RaiseEventAsync("operation", "end"); + await Task.WhenAll(stati); - status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(10)); + // check that they all completed + for (int i = 0; i < numberEntities; i++) + { + Assert.AreEqual(OrchestrationStatus.Completed, stati[i].Result.OrchestrationStatus); + } - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.AreEqual(3, JToken.Parse(status?.Output)); + // in parallel, read all the entity states + var entityClient = new TaskHubEntityClient(clients[0].Result.InnerClient); + var entityStates = new Task>[numberEntities]; + for (int i = 0; i < numberEntities; i++) + { + entityStates[i] = entityClient.ReadEntityStateAsync(counters[i]); + } + await Task.WhenAll(entityStates); - // When using ContinueAsNew, the original input is discarded and replaced with the most recent state. - Assert.AreNotEqual(initialValue, JToken.Parse(status?.Input)); + // check that the counter states are all back to 0 + // (since each participated in 2 transfers, one incrementing and one decrementing) + for (int i = 0; i < numberEntities; i++) + { + Assert.IsTrue(entityStates[i].Result.EntityExists); + Assert.AreEqual(0, entityStates[i].Result.EntityState); + } await host.StopAsync(); } } /// - /// End-to-end test which validates the ContinueAsNew functionality by implementing character counter actor pattern. + /// End-to-end test which validates exception handling in entity operations. /// [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task ActorOrchestrationForLargeInput(bool enableExtendedSessions) + [DataRow(false, false, false)] + [DataRow(false, true, false)] + [DataRow(true, false, false)] + [DataRow(true, true, false)] + [DataRow(false, false, true)] + public async Task CallFaultyEntity(bool extendedSessions, bool rollbackOnExceptions, bool useFailureDetails) { - await this.ValidateCharacterCounterIntegrationTest(enableExtendedSessions); + var errorPropagationMode = useFailureDetails ? ErrorPropagationMode.UseFailureDetails : ErrorPropagationMode.SerializeExceptions; + + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost( + enableExtendedSessions: extendedSessions, + errorPropagationMode: errorPropagationMode)) + { + await host.StartAsync(); + + var entityName = rollbackOnExceptions ? "FaultyEntityWithRollback" : "FaultyEntityWithoutRollback"; + var entityId = new EntityId(entityName, Guid.NewGuid().ToString()); + + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.CallFaultyEntity), (entityId, rollbackOnExceptions, errorPropagationMode)); + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(90)); + + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual("\"ok\"", status?.Output); + + await host.StopAsync(); + } } - /// - /// End-to-end test which validates the deletion of all data generated by the ContinueAsNew functionality in the character counter actor pattern. - /// - [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task ActorOrchestrationDeleteAllLargeMessageBlobs(bool enableExtendedSessions) + [TestMethod] + public async Task PurgeInstanceHistoryForSingleInstanceWithoutLargeMessageBlobs() { - DateTime startDateTime = DateTime.UtcNow; + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false)) + { + string instanceId = Guid.NewGuid().ToString(); + await host.StartAsync(); + TestOrchestrationClient client = await host.StartOrchestrationAsync(typeof(Orchestrations.Factorial), 110, instanceId); + await client.WaitForCompletionAsync(TimeSpan.FromSeconds(60)); - Tuple resultTuple = await this.ValidateCharacterCounterIntegrationTest(enableExtendedSessions); - string instanceId = resultTuple.Item1; - TestOrchestrationClient client = resultTuple.Item2; + List historyEvents = await client.GetOrchestrationHistoryAsync(instanceId); + Assert.IsTrue(historyEvents.Count > 0); - List historyEvents = await client.GetOrchestrationHistoryAsync(instanceId); - Assert.IsTrue(historyEvents.Count > 0); + IList orchestrationStateList = await client.GetStateAsync(instanceId); + Assert.AreEqual(1, orchestrationStateList.Count); + Assert.AreEqual(instanceId, orchestrationStateList.First().OrchestrationInstance.InstanceId); - IList orchestrationStateList = await client.GetStateAsync(instanceId); - Assert.AreEqual(1, orchestrationStateList.Count); - Assert.AreEqual(instanceId, orchestrationStateList.First().OrchestrationInstance.InstanceId); + await client.PurgeInstanceHistory(); - int blobCount = await this.GetBlobCount("test-largemessages", instanceId); + List historyEventsAfterPurging = await client.GetOrchestrationHistoryAsync(instanceId); + Assert.AreEqual(0, historyEventsAfterPurging.Count); - Assert.AreEqual(3, blobCount); + orchestrationStateList = await client.GetStateAsync(instanceId); + Assert.AreEqual(1, orchestrationStateList.Count); + Assert.IsNull(orchestrationStateList.First()); - await client.PurgeInstanceHistoryByTimePeriod( - startDateTime, - DateTime.UtcNow, - new List - { - OrchestrationStatus.Completed, - OrchestrationStatus.Terminated, - OrchestrationStatus.Failed, - OrchestrationStatus.Running - }); - - historyEvents = await client.GetOrchestrationHistoryAsync(instanceId); - Assert.AreEqual(0, historyEvents.Count); - - orchestrationStateList = await client.GetStateAsync(instanceId); - Assert.AreEqual(1, orchestrationStateList.Count); - Assert.IsNull(orchestrationStateList.First()); - - blobCount = await this.GetBlobCount("test-largemessages", instanceId); - Assert.AreEqual(0, blobCount); + await host.StopAsync(); + } } - private async Task> ValidateCharacterCounterIntegrationTest(bool enableExtendedSessions) + [TestMethod] + public async Task ValidateCustomStatusPersists() { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(false)) { await host.StartAsync(); - string initialMessage = this.GenerateMediumRandomStringPayload().ToString(); - string finalMessage = initialMessage; - int counter = initialMessage.Length; - var initialValue = new Tuple(initialMessage, counter); - TestOrchestrationClient client = - await host.StartOrchestrationAsync(typeof(Orchestrations.CharacterCounter), initialValue); - - // Need to wait for the instance to start before sending events to it. - // TODO: This requirement may not be ideal and should be revisited. - OrchestrationState orchestrationState = - await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); - - // Perform some operations - await client.RaiseEventAsync("operation", "double"); - finalMessage = finalMessage + new string(finalMessage.Reverse().ToArray()); - counter *= 2; - - // TODO: Sleeping to avoid a race condition where multiple ContinueAsNew messages - // are processed by the same instance at the same time, resulting in a corrupt - // storage failure in DTFx. - await Task.Delay(10000); - await client.RaiseEventAsync("operation", "double"); - finalMessage = finalMessage + new string(finalMessage.Reverse().ToArray()); - counter *= 2; - await Task.Delay(10000); - await client.RaiseEventAsync("operation", "double"); - finalMessage = finalMessage + new string(finalMessage.Reverse().ToArray()); - counter *= 2; - await Task.Delay(10000); - - // Make sure it's still running and didn't complete early (or fail). - var status = await client.GetStatusAsync(); - Assert.IsTrue( - status?.OrchestrationStatus == OrchestrationStatus.Running || - status?.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew); - - // The end message will cause the actor to complete itself. - await client.RaiseEventAsync("operation", "end"); - - status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(10)); - - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - var result = JObject.Parse(status?.Output); - Assert.IsNotNull(result); + string customStatus = "custom_status"; + var client = await host.StartOrchestrationAsync( + typeof(Test.Orchestrations.ChangeStatusOrchestration), + new string[] { customStatus }); + var state = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - var input = JObject.Parse(status?.Input); - Assert.AreEqual(finalMessage, input["Item1"].Value()); - Assert.AreEqual(finalMessage.Length, input["Item2"].Value()); - Assert.AreEqual(finalMessage, result["Item1"].Value()); - Assert.AreEqual(counter, result["Item2"].Value()); + Assert.AreEqual(OrchestrationStatus.Completed, state?.OrchestrationStatus); + Assert.AreEqual(customStatus, JToken.Parse(state?.Status)); await host.StopAsync(); - - return new Tuple( - orchestrationState.OrchestrationInstance.InstanceId, - client); } } - - - /// - /// End-to-end test which validates the Terminate functionality. - /// - [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task TerminateOrchestration(bool enableExtendedSessions) + [TestMethod] + public async Task ValidateNullCustomStatusPersists() { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(false)) { await host.StartAsync(); - // Using the counter orchestration because it will wait indefinitely for input. - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Counter), 0); - - // Need to wait for the instance to start before we can terminate it. - // TODO: This requirement may not be ideal and should be revisited. - await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); - - await client.TerminateAsync("sayōnara"); - - var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(10)); + var client = await host.StartOrchestrationAsync( + typeof(Test.Orchestrations.ChangeStatusOrchestration), + // First set "custom_status", then set null and make sure it persists + new string[] { "custom_status", null }); + var state = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - Assert.AreEqual(OrchestrationStatus.Terminated, status?.OrchestrationStatus); - Assert.AreEqual("sayōnara", status?.Output); + Assert.AreEqual(OrchestrationStatus.Completed, state?.OrchestrationStatus); + Assert.AreEqual(null, JToken.Parse(state?.Status).Value()); await host.StopAsync(); } } - /// - /// End-to-end test which validates the Suspend-Resume functionality. - /// [TestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task SuspendResumeOrchestration(bool enableExtendedSessions) + public async Task PurgeInstanceHistoryForSingleInstanceWithLargeMessageBlobs() { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false)) { - string originalStatus = "OGstatus"; - string suspendReason = "sleepyOrch"; - string changedStatus = "newStatus"; - await host.StartAsync(); - var client = await host.StartOrchestrationAsync(typeof(Test.Orchestrations.NextExecution), originalStatus); - await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); - // Test case 1: Suspend changes the status Running->Suspended - await client.SuspendAsync(suspendReason); - var status = await client.WaitForStatusChange(TimeSpan.FromSeconds(10), OrchestrationStatus.Suspended); - Assert.AreEqual(OrchestrationStatus.Suspended, status?.OrchestrationStatus); - Assert.AreEqual(suspendReason, status?.Output); + string instanceId = Guid.NewGuid().ToString(); + string message = this.GenerateMediumRandomStringPayload().ToString(); + TestOrchestrationClient client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message, instanceId); + OrchestrationState status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - // Test case 2: external event does not go through - await client.RaiseEventAsync("changeStatusNow", changedStatus); - status = await client.GetStatusAsync(); - Assert.AreEqual(originalStatus, JToken.Parse(status?.Status)); + List historyEvents = await client.GetOrchestrationHistoryAsync(instanceId); + Assert.IsTrue(historyEvents.Count > 0); - // Test case 3: external event now goes through - await client.ResumeAsync("wakeUp"); - status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(10)); - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.AreEqual(changedStatus, JToken.Parse(status?.Status)); + IList orchestrationStateList = await client.GetStateAsync(instanceId); + Assert.AreEqual(1, orchestrationStateList.Count); + Assert.AreEqual(instanceId, orchestrationStateList.First().OrchestrationInstance.InstanceId); - await host.StopAsync(); - } - } + int blobCount = await this.GetBlobCount("test-largemessages", instanceId); + Assert.IsTrue(blobCount > 0); - /// - /// Test that a suspended orchestration can be terminated. - /// - [TestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task TerminateSuspendedOrchestration(bool enableExtendedSessions) - { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) - { - await host.StartAsync(); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Counter), 0); - await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); + IList results = await host.GetAllOrchestrationInstancesAsync(); + Assert.AreEqual(1, results.Count); - await client.SuspendAsync("suspend"); - await client.WaitForStatusChange(TimeSpan.FromSeconds(10), OrchestrationStatus.Suspended); + string result = JToken.Parse(results.First(x => x.OrchestrationInstance.InstanceId == instanceId).Output).ToString(); + Assert.AreEqual(message, result); - await client.TerminateAsync("terminate"); + await client.PurgeInstanceHistory(); - var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(10)); - Assert.AreEqual(OrchestrationStatus.Terminated, status?.OrchestrationStatus); - Assert.AreEqual("terminate", status?.Output); + List historyEventsAfterPurging = await client.GetOrchestrationHistoryAsync(instanceId); + Assert.AreEqual(0, historyEventsAfterPurging.Count); + + orchestrationStateList = await client.GetStateAsync(instanceId); + Assert.AreEqual(1, orchestrationStateList.Count); + Assert.IsNull(orchestrationStateList.First()); + + blobCount = await this.GetBlobCount("test-largemessages", instanceId); + Assert.AreEqual(0, blobCount); await host.StopAsync(); } } - /// - /// End-to-end test which validates the Rewind functionality on more than one orchestration. - /// [TestMethod] - public async Task RewindOrchestrationsFail() + public async Task PurgeInstanceHistoryForTimePeriodDeleteAll() { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: true)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false)) { - Orchestrations.FactorialOrchestratorFail.ShouldFail = true; await host.StartAsync(); + DateTime startDateTime = DateTime.Now; + string firstInstanceId = "instance1"; + TestOrchestrationClient client = await host.StartOrchestrationAsync(typeof(Orchestrations.FanOutFanIn), 50, firstInstanceId); + await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + string secondInstanceId = "instance2"; + client = await host.StartOrchestrationAsync(typeof(Orchestrations.FanOutFanIn), 50, secondInstanceId); + await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + string thirdInstanceId = "instance3"; + client = await host.StartOrchestrationAsync(typeof(Orchestrations.FanOutFanIn), 50, thirdInstanceId); + await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - string singletonInstanceId1 = $"1_Test_{Guid.NewGuid():N}"; - string singletonInstanceId2 = $"2_Test_{Guid.NewGuid():N}"; + string fourthInstanceId = "instance4"; + string message = this.GenerateMediumRandomStringPayload().ToString(); + client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message, fourthInstanceId); + var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - var client1 = await host.StartOrchestrationAsync( - typeof(Orchestrations.FactorialOrchestratorFail), - input: 3, - instanceId: singletonInstanceId1); + IList results = await host.GetAllOrchestrationInstancesAsync(); + Assert.AreEqual(4, results.Count); + Assert.AreEqual("\"Done\"", results.First(x => x.OrchestrationInstance.InstanceId == firstInstanceId).Output); + Assert.AreEqual("\"Done\"", results.First(x => x.OrchestrationInstance.InstanceId == secondInstanceId).Output); + Assert.AreEqual("\"Done\"", results.First(x => x.OrchestrationInstance.InstanceId == thirdInstanceId).Output); + string result = JToken.Parse(results.First(x => x.OrchestrationInstance.InstanceId == fourthInstanceId).Output).ToString(); + Assert.AreEqual(message, result); - var statusFail = await client1.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + List firstHistoryEvents = await client.GetOrchestrationHistoryAsync(firstInstanceId); + Assert.IsTrue(firstHistoryEvents.Count > 0); - Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); + List secondHistoryEvents = await client.GetOrchestrationHistoryAsync(secondInstanceId); + Assert.IsTrue(secondHistoryEvents.Count > 0); - Orchestrations.FactorialOrchestratorFail.ShouldFail = false; + List thirdHistoryEvents = await client.GetOrchestrationHistoryAsync(thirdInstanceId); + Assert.IsTrue(thirdHistoryEvents.Count > 0); - var client2 = await host.StartOrchestrationAsync( - typeof(Orchestrations.SayHelloWithActivity), - input: "Catherine", - instanceId: singletonInstanceId2); + List fourthHistoryEvents = await client.GetOrchestrationHistoryAsync(thirdInstanceId); + Assert.IsTrue(fourthHistoryEvents.Count > 0); - await client1.RewindAsync("Rewind failed orchestration only"); + IList firstOrchestrationStateList = await client.GetStateAsync(firstInstanceId); + Assert.AreEqual(1, firstOrchestrationStateList.Count); + Assert.AreEqual(firstInstanceId, firstOrchestrationStateList.First().OrchestrationInstance.InstanceId); - var statusRewind = await client1.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + IList secondOrchestrationStateList = await client.GetStateAsync(secondInstanceId); + Assert.AreEqual(1, secondOrchestrationStateList.Count); + Assert.AreEqual(secondInstanceId, secondOrchestrationStateList.First().OrchestrationInstance.InstanceId); - Assert.AreEqual(OrchestrationStatus.Completed, statusRewind?.OrchestrationStatus); - Assert.AreEqual("6", statusRewind?.Output); + IList thirdOrchestrationStateList = await client.GetStateAsync(thirdInstanceId); + Assert.AreEqual(1, thirdOrchestrationStateList.Count); + Assert.AreEqual(thirdInstanceId, thirdOrchestrationStateList.First().OrchestrationInstance.InstanceId); - await host.StopAsync(); - } - } + IList fourthOrchestrationStateList = await client.GetStateAsync(fourthInstanceId); + Assert.AreEqual(1, fourthOrchestrationStateList.Count); + Assert.AreEqual(fourthInstanceId, fourthOrchestrationStateList.First().OrchestrationInstance.InstanceId); - /// - /// End-to-end test which validates the Rewind functionality with fan in fan out pattern. - /// - [TestMethod] - public async Task RewindActivityFailFanOut() - { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: true)) - { - Activities.HelloFailFanOut.ShouldFail1 = false; - await host.StartAsync(); + int blobCount = await this.GetBlobCount("test-largemessages", fourthInstanceId); + Assert.AreEqual(6, blobCount); - string singletonInstanceId = $"Test_{Guid.NewGuid():N}"; + await client.PurgeInstanceHistoryByTimePeriod( + startDateTime, + DateTime.UtcNow, + new List + { + OrchestrationStatus.Completed, + OrchestrationStatus.Terminated, + OrchestrationStatus.Failed, + OrchestrationStatus.Running + }); - var client = await host.StartOrchestrationAsync( - typeof(Orchestrations.FanOutFanInRewind), - input: 3, - instanceId: singletonInstanceId); + List firstHistoryEventsAfterPurging = await client.GetOrchestrationHistoryAsync(firstInstanceId); + Assert.AreEqual(0, firstHistoryEventsAfterPurging.Count); - var statusFail = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + List secondHistoryEventsAfterPurging = await client.GetOrchestrationHistoryAsync(secondInstanceId); + Assert.AreEqual(0, secondHistoryEventsAfterPurging.Count); - Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); + List thirdHistoryEventsAfterPurging = await client.GetOrchestrationHistoryAsync(thirdInstanceId); + Assert.AreEqual(0, thirdHistoryEventsAfterPurging.Count); - Activities.HelloFailFanOut.ShouldFail2 = false; + ListfourthHistoryEventsAfterPurging = await client.GetOrchestrationHistoryAsync(fourthInstanceId); + Assert.AreEqual(0, fourthHistoryEventsAfterPurging.Count); - await client.RewindAsync("Rewind orchestrator with failed parallel activity."); + firstOrchestrationStateList = await client.GetStateAsync(firstInstanceId); + Assert.AreEqual(1, firstOrchestrationStateList.Count); + Assert.IsNull(firstOrchestrationStateList.First()); - var statusRewind = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + secondOrchestrationStateList = await client.GetStateAsync(secondInstanceId); + Assert.AreEqual(1, secondOrchestrationStateList.Count); + Assert.IsNull(secondOrchestrationStateList.First()); - Assert.AreEqual(OrchestrationStatus.Completed, statusRewind?.OrchestrationStatus); - Assert.AreEqual("\"Done\"", statusRewind?.Output); + thirdOrchestrationStateList = await client.GetStateAsync(thirdInstanceId); + Assert.AreEqual(1, thirdOrchestrationStateList.Count); + Assert.IsNull(thirdOrchestrationStateList.First()); + + fourthOrchestrationStateList = await client.GetStateAsync(fourthInstanceId); + Assert.AreEqual(1, fourthOrchestrationStateList.Count); + Assert.IsNull(fourthOrchestrationStateList.First()); + + blobCount = await this.GetBlobCount("test-largemessages", fourthInstanceId); + Assert.AreEqual(0, blobCount); await host.StopAsync(); } } - - /// - /// End-to-end test which validates the Rewind functionality on an activity function failure - /// with modified (to fail initially) SayHelloWithActivity orchestrator. - /// - [TestMethod] - public async Task RewindActivityFail() + private async Task GetBlobCount(string containerName, string directoryName) { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: true)) + string storageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(); + CloudStorageAccount storageAccount; + if (!CloudStorageAccount.TryParse(storageConnectionString, out storageAccount)) { - await host.StartAsync(); - - string singletonInstanceId = $"{Guid.NewGuid():N}"; - - var client = await host.StartOrchestrationAsync( - typeof(Orchestrations.SayHelloWithActivityFail), - input: "World", - instanceId: singletonInstanceId); - - var statusFail = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - - Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); - - Activities.HelloFailActivity.ShouldFail = false; - - await client.RewindAsync("Activity failure test."); + Assert.Fail("Couldn't find the connection string to use to look up blobs!"); + return 0; + } - var statusRewind = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient(); - Assert.AreEqual(OrchestrationStatus.Completed, statusRewind?.OrchestrationStatus); - Assert.AreEqual("\"Hello, World!\"", statusRewind?.Output); + CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(containerName); + await cloudBlobContainer.CreateIfNotExistsAsync(); + CloudBlobDirectory instanceDirectory = cloudBlobContainer.GetDirectoryReference(directoryName); + var blobs = new List(); + BlobContinuationToken blobContinuationToken = null; + do + { + BlobResultSegment results = await TimeoutHandler.ExecuteWithTimeout("GetBlobCount", "dummyAccount", new AzureStorageOrchestrationServiceSettings(), (context, timeoutToken) => + { + return instanceDirectory.ListBlobsSegmentedAsync( + useFlatBlobListing: true, + blobListingDetails: BlobListingDetails.Metadata, + maxResults: null, + currentToken: blobContinuationToken, + options: null, + operationContext: context, + cancellationToken: timeoutToken); + }); + + blobContinuationToken = results.ContinuationToken; + blobs.AddRange(results.Results); + } while (blobContinuationToken != null); - await host.StopAsync(); - } + Trace.TraceInformation( + "Found {0} blobs: {1}{2}", + blobs.Count, + Environment.NewLine, + string.Join(Environment.NewLine, blobs.Select(b => b.Uri))); + return blobs.Count; } + [TestMethod] - public async Task RewindMultipleActivityFail() + public async Task PurgeInstanceHistoryForTimePeriodDeletePartially() { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: true)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false)) { + // Execute the orchestrator twice. Orchestrator will be replied. However instances might be two. await host.StartAsync(); + DateTime startDateTime = DateTime.Now; + string firstInstanceId = Guid.NewGuid().ToString(); + TestOrchestrationClient client = await host.StartOrchestrationAsync(typeof(Orchestrations.FanOutFanIn), 50, firstInstanceId); + await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + DateTime endDateTime = DateTime.Now; + await Task.Delay(5000); + string secondInstanceId = Guid.NewGuid().ToString(); + client = await host.StartOrchestrationAsync(typeof(Orchestrations.FanOutFanIn), 50, secondInstanceId); + await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + string thirdInstanceId = Guid.NewGuid().ToString(); + client = await host.StartOrchestrationAsync(typeof(Orchestrations.FanOutFanIn), 50, thirdInstanceId); + await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - string singletonInstanceId = $"Test_{Guid.NewGuid():N}"; + IList results = await host.GetAllOrchestrationInstancesAsync(); + Assert.AreEqual(3, results.Count); + Assert.IsNotNull(results[0].Output.Equals("\"Done\"")); + Assert.IsNotNull(results[1].Output.Equals("\"Done\"")); + Assert.IsNotNull(results[2].Output.Equals("\"Done\"")); - var client = await host.StartOrchestrationAsync( - typeof(Orchestrations.FactorialMultipleActivityFail), - input: 4, - instanceId: singletonInstanceId); - var statusFail = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + List firstHistoryEvents = await client.GetOrchestrationHistoryAsync(firstInstanceId); + Assert.IsTrue(firstHistoryEvents.Count > 0); - Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); + List secondHistoryEvents = await client.GetOrchestrationHistoryAsync(secondInstanceId); + Assert.IsTrue(secondHistoryEvents.Count > 0); - Activities.MultiplyMultipleActivityFail.ShouldFail1 = false; + List thirdHistoryEvents = await client.GetOrchestrationHistoryAsync(thirdInstanceId); + Assert.IsTrue(secondHistoryEvents.Count > 0); - await client.RewindAsync("Rewind for activity failure 1."); + IList firstOrchestrationStateList = await client.GetStateAsync(firstInstanceId); + Assert.AreEqual(1, firstOrchestrationStateList.Count); + Assert.AreEqual(firstInstanceId, firstOrchestrationStateList.First().OrchestrationInstance.InstanceId); - statusFail = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + IList secondOrchestrationStateList = await client.GetStateAsync(secondInstanceId); + Assert.AreEqual(1, secondOrchestrationStateList.Count); + Assert.AreEqual(secondInstanceId, secondOrchestrationStateList.First().OrchestrationInstance.InstanceId); - Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); + IList thirdOrchestrationStateList = await client.GetStateAsync(thirdInstanceId); + Assert.AreEqual(1, thirdOrchestrationStateList.Count); + Assert.AreEqual(thirdInstanceId, thirdOrchestrationStateList.First().OrchestrationInstance.InstanceId); - Activities.MultiplyMultipleActivityFail.ShouldFail2 = false; + await client.PurgeInstanceHistoryByTimePeriod(startDateTime, endDateTime, new List { OrchestrationStatus.Completed, OrchestrationStatus.Terminated, OrchestrationStatus.Failed, OrchestrationStatus.Running }); - await client.RewindAsync("Rewind for activity failure 2."); + List firstHistoryEventsAfterPurging = await client.GetOrchestrationHistoryAsync(firstInstanceId); + Assert.AreEqual(0, firstHistoryEventsAfterPurging.Count); - var statusRewind = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + List secondHistoryEventsAfterPurging = await client.GetOrchestrationHistoryAsync(secondInstanceId); + Assert.IsTrue(secondHistoryEventsAfterPurging.Count > 0); - Assert.AreEqual(OrchestrationStatus.Completed, statusRewind?.OrchestrationStatus); - Assert.AreEqual("24", statusRewind?.Output); + List thirdHistoryEventsAfterPurging = await client.GetOrchestrationHistoryAsync(thirdInstanceId); + Assert.IsTrue(thirdHistoryEventsAfterPurging.Count > 0); + + firstOrchestrationStateList = await client.GetStateAsync(firstInstanceId); + Assert.AreEqual(1, firstOrchestrationStateList.Count); + Assert.IsNull(firstOrchestrationStateList.First()); + + secondOrchestrationStateList = await client.GetStateAsync(secondInstanceId); + Assert.AreEqual(1, secondOrchestrationStateList.Count); + Assert.AreEqual(secondInstanceId, secondOrchestrationStateList.First().OrchestrationInstance.InstanceId); + + thirdOrchestrationStateList = await client.GetStateAsync(thirdInstanceId); + Assert.AreEqual(1, thirdOrchestrationStateList.Count); + Assert.AreEqual(thirdInstanceId, thirdOrchestrationStateList.First().OrchestrationInstance.InstanceId); await host.StopAsync(); } } - [TestMethod] - public async Task RewindSubOrchestrationsTest() + /// + /// End-to-end test which validates parallel function execution by enumerating all files in the current directory + /// in parallel and getting the sum total of all file sizes. + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task ParallelOrchestration(bool enableExtendedSessions) { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: true)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) { await host.StartAsync(); - string ParentInstanceId = $"Parent_{Guid.NewGuid():N}"; - string ChildInstanceId = $"Child_{Guid.NewGuid():N}"; - - var clientParent = await host.StartOrchestrationAsync( - typeof(Orchestrations.ParentWorkflowSubOrchestrationFail), - input: true, - instanceId: ParentInstanceId); - - var statusFail = await clientParent.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - - Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); - - Orchestrations.ChildWorkflowSubOrchestrationFail.ShouldFail1 = false; - - await clientParent.RewindAsync("Rewind first suborchestration failure."); - - statusFail = await clientParent.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - - Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); - - Orchestrations.ChildWorkflowSubOrchestrationFail.ShouldFail2 = false; - - await clientParent.RewindAsync("Rewind second suborchestration failure."); - - var statusRewind = await clientParent.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.DiskUsage), Environment.CurrentDirectory); + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(90)); - Assert.AreEqual(OrchestrationStatus.Completed, statusRewind?.OrchestrationStatus); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual(Environment.CurrentDirectory, JToken.Parse(status?.Input)); + Assert.IsTrue(long.Parse(status?.Output) > 0L); await host.StopAsync(); } } - [TestMethod] - public async Task RewindSubOrchestrationActivityTest() + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task LargeFanOutOrchestration(bool enableExtendedSessions) { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: true)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) { await host.StartAsync(); - string ParentInstanceId = $"Parent_{Guid.NewGuid():N}"; - string ChildInstanceId = $"Child_{Guid.NewGuid():N}"; - - var clientParent = await host.StartOrchestrationAsync( - typeof(Orchestrations.ParentWorkflowSubOrchestrationActivityFail), - input: true, - instanceId: ParentInstanceId); - - var statusFail = await clientParent.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - - Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); - - Activities.HelloFailSubOrchestrationActivity.ShouldFail1 = false; - - await clientParent.RewindAsync("Rewinding 1: child should still fail."); - - statusFail = await clientParent.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - - Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); - - Activities.HelloFailSubOrchestrationActivity.ShouldFail2 = false; - - await clientParent.RewindAsync("Rewinding 2: child should complete."); - - var statusRewind = await clientParent.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.FanOutFanIn), 1000); + var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(5)); - Assert.AreEqual(OrchestrationStatus.Completed, statusRewind?.OrchestrationStatus); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); await host.StopAsync(); } } [TestMethod] - public async Task RewindNestedSubOrchestrationTest() + public async Task FanOutOrchestration_LargeHistoryBatches() { using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: true)) { await host.StartAsync(); - string GrandparentInstanceId = $"Grandparent_{Guid.NewGuid():N}"; - string ChildInstanceId = $"Child_{Guid.NewGuid():N}"; - - var clientGrandparent = await host.StartOrchestrationAsync( - typeof(Orchestrations.GrandparentWorkflowNestedActivityFail), - input: true, - instanceId: GrandparentInstanceId); - - var statusFail = await clientGrandparent.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - - Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); - - Activities.HelloFailNestedSuborchestration.ShouldFail1 = false; - - await clientGrandparent.RewindAsync("Rewind 1: Nested child activity still fails."); - - Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); - - Activities.HelloFailNestedSuborchestration.ShouldFail2 = false; - - await clientGrandparent.RewindAsync("Rewind 2: Nested child activity completes."); - - var statusRewind = await clientGrandparent.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + // This test creates history payloads that exceed the 4 MB limit imposed by Azure Storage + // when 100 entities are uploaded at a time. + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.SemiLargePayloadFanOutFanIn), 90); + var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(5)); - Assert.AreEqual(OrchestrationStatus.Completed, statusRewind?.OrchestrationStatus); - //Assert.AreEqual("\"Hello, Catherine!\"", statusRewind?.Output); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); await host.StopAsync(); } } + /// + /// End-to-end test which validates the ContinueAsNew functionality by implementing a counter actor pattern. + /// [DataTestMethod] [DataRow(true)] [DataRow(false)] - public async Task TimerCancellation(bool enableExtendedSessions) + public async Task ActorOrchestration(bool enableExtendedSessions) { using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) { await host.StartAsync(); - var timeout = TimeSpan.FromSeconds(10); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Approval), timeout); + int initialValue = 0; + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Counter), initialValue); // Need to wait for the instance to start before sending events to it. // TODO: This requirement may not be ideal and should be revisited. await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); - await client.RaiseEventAsync("approval", eventData: true); - var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + // Perform some operations + await client.RaiseEventAsync("operation", "incr"); + await client.RaiseEventAsync("operation", "incr"); + await client.RaiseEventAsync("operation", "incr"); + await client.RaiseEventAsync("operation", "decr"); + await client.RaiseEventAsync("operation", "incr"); + await Task.Delay(2000); + + // Make sure it's still running and didn't complete early (or fail). + var status = await client.GetStatusAsync(); + Assert.IsTrue( + status?.OrchestrationStatus == OrchestrationStatus.Running || + status?.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew); + + // The end message will cause the actor to complete itself. + await client.RaiseEventAsync("operation", "end"); + + status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(10)); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.AreEqual("Approved", JToken.Parse(status?.Output)); + Assert.AreEqual(3, JToken.Parse(status?.Output)); + + // When using ContinueAsNew, the original input is discarded and replaced with the most recent state. + Assert.AreNotEqual(initialValue, JToken.Parse(status?.Input)); await host.StopAsync(); } } /// - /// End-to-end test which validates the handling of durable timer expiration. + /// End-to-end test which validates the ContinueAsNew functionality by implementing character counter actor pattern. /// [DataTestMethod] [DataRow(true)] [DataRow(false)] - public async Task TimerExpiration(bool enableExtendedSessions) + public async Task ActorOrchestrationForLargeInput(bool enableExtendedSessions) + { + await this.ValidateCharacterCounterIntegrationTest(enableExtendedSessions); + } + + /// + /// End-to-end test which validates the deletion of all data generated by the ContinueAsNew functionality in the character counter actor pattern. + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task ActorOrchestrationDeleteAllLargeMessageBlobs(bool enableExtendedSessions) + { + DateTime startDateTime = DateTime.UtcNow; + + Tuple resultTuple = await this.ValidateCharacterCounterIntegrationTest(enableExtendedSessions); + string instanceId = resultTuple.Item1; + TestOrchestrationClient client = resultTuple.Item2; + + List historyEvents = await client.GetOrchestrationHistoryAsync(instanceId); + Assert.IsTrue(historyEvents.Count > 0); + + IList orchestrationStateList = await client.GetStateAsync(instanceId); + Assert.AreEqual(1, orchestrationStateList.Count); + Assert.AreEqual(instanceId, orchestrationStateList.First().OrchestrationInstance.InstanceId); + + int blobCount = await this.GetBlobCount("test-largemessages", instanceId); + + Assert.AreEqual(3, blobCount); + + await client.PurgeInstanceHistoryByTimePeriod( + startDateTime, + DateTime.UtcNow, + new List + { + OrchestrationStatus.Completed, + OrchestrationStatus.Terminated, + OrchestrationStatus.Failed, + OrchestrationStatus.Running + }); + + historyEvents = await client.GetOrchestrationHistoryAsync(instanceId); + Assert.AreEqual(0, historyEvents.Count); + + orchestrationStateList = await client.GetStateAsync(instanceId); + Assert.AreEqual(1, orchestrationStateList.Count); + Assert.IsNull(orchestrationStateList.First()); + + blobCount = await this.GetBlobCount("test-largemessages", instanceId); + Assert.AreEqual(0, blobCount); + } + + private async Task> ValidateCharacterCounterIntegrationTest(bool enableExtendedSessions) { using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) { await host.StartAsync(); - var timeout = TimeSpan.FromSeconds(10); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Approval), timeout); + string initialMessage = this.GenerateMediumRandomStringPayload().ToString(); + string finalMessage = initialMessage; + int counter = initialMessage.Length; + var initialValue = new Tuple(initialMessage, counter); + TestOrchestrationClient client = + await host.StartOrchestrationAsync(typeof(Orchestrations.CharacterCounter), initialValue); // Need to wait for the instance to start before sending events to it. // TODO: This requirement may not be ideal and should be revisited. - await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); + OrchestrationState orchestrationState = + await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); - // Don't send any notification - let the internal timeout expire + // Perform some operations + await client.RaiseEventAsync("operation", "double"); + finalMessage = finalMessage + new string(finalMessage.Reverse().ToArray()); + counter *= 2; + + // TODO: Sleeping to avoid a race condition where multiple ContinueAsNew messages + // are processed by the same instance at the same time, resulting in a corrupt + // storage failure in DTFx. + await Task.Delay(10000); + await client.RaiseEventAsync("operation", "double"); + finalMessage = finalMessage + new string(finalMessage.Reverse().ToArray()); + counter *= 2; + await Task.Delay(10000); + await client.RaiseEventAsync("operation", "double"); + finalMessage = finalMessage + new string(finalMessage.Reverse().ToArray()); + counter *= 2; + await Task.Delay(10000); + + // Make sure it's still running and didn't complete early (or fail). + var status = await client.GetStatusAsync(); + Assert.IsTrue( + status?.OrchestrationStatus == OrchestrationStatus.Running || + status?.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew); + + // The end message will cause the actor to complete itself. + await client.RaiseEventAsync("operation", "end"); + + status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(10)); - var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(20)); Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.AreEqual("Expired", JToken.Parse(status?.Output)); + var result = JObject.Parse(status?.Output); + Assert.IsNotNull(result); + + var input = JObject.Parse(status?.Input); + Assert.AreEqual(finalMessage, input["Item1"].Value()); + Assert.AreEqual(finalMessage.Length, input["Item2"].Value()); + Assert.AreEqual(finalMessage, result["Item1"].Value()); + Assert.AreEqual(counter, result["Item2"].Value()); await host.StopAsync(); + + return new Tuple( + orchestrationState.OrchestrationInstance.InstanceId, + client); } } + + /// - /// End-to-end test which validates that orchestrations run concurrently of each other (up to 100 by default). + /// End-to-end test which validates the Terminate functionality. /// [DataTestMethod] [DataRow(true)] [DataRow(false)] - public async Task OrchestrationConcurrency(bool enableExtendedSessions) + public async Task TerminateOrchestration(bool enableExtendedSessions) { using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) { await host.StartAsync(); - Func orchestrationStarter = async delegate () - { - var timeout = TimeSpan.FromSeconds(10); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Approval), timeout); - await client.WaitForCompletionAsync(TimeSpan.FromSeconds(60)); + // Using the counter orchestration because it will wait indefinitely for input. + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Counter), 0); - // Don't send any notification - let the internal timeout expire - }; + // Need to wait for the instance to start before we can terminate it. + // TODO: This requirement may not be ideal and should be revisited. + await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); - int iterations = 10; - var tasks = new Task[iterations]; - for (int i = 0; i < iterations; i++) - { - tasks[i] = orchestrationStarter(); - } + await client.TerminateAsync("sayōnara"); - // The 10 orchestrations above (which each delay for 10 seconds) should all complete in less than 60 seconds. - Task parallelOrchestrations = Task.WhenAll(tasks); - Task timeoutTask = Task.Delay(TimeSpan.FromSeconds(60)); + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(10)); - Task winner = await Task.WhenAny(parallelOrchestrations, timeoutTask); - Assert.AreEqual(parallelOrchestrations, winner); + Assert.AreEqual(OrchestrationStatus.Terminated, status?.OrchestrationStatus); + Assert.AreEqual("sayōnara", status?.Output); await host.StopAsync(); } } /// - /// End-to-end test which validates the orchestrator's exception handling behavior. + /// End-to-end test which validates the Suspend-Resume functionality. /// - [DataTestMethod] + [TestMethod] [DataRow(true)] [DataRow(false)] - public async Task HandledActivityException(bool enableExtendedSessions) + public async Task SuspendResumeOrchestration(bool enableExtendedSessions) { using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) { - await host.StartAsync(); + string originalStatus = "OGstatus"; + string suspendReason = "sleepyOrch"; + string changedStatus = "newStatus"; - // Empty string input should result in ArgumentNullException in the orchestration code. - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.TryCatchLoop), 5); - var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(15)); + await host.StartAsync(); + var client = await host.StartOrchestrationAsync(typeof(Test.Orchestrations.NextExecution), originalStatus); + await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.AreEqual(5, JToken.Parse(status?.Output)); + // Test case 1: Suspend changes the status Running->Suspended + await client.SuspendAsync(suspendReason); + var status = await client.WaitForStatusChange(TimeSpan.FromSeconds(10), OrchestrationStatus.Suspended); + Assert.AreEqual(OrchestrationStatus.Suspended, status?.OrchestrationStatus); + Assert.AreEqual(suspendReason, status?.Output); + + // Test case 2: external event does not go through + await client.RaiseEventAsync("changeStatusNow", changedStatus); + status = await client.GetStatusAsync(); + Assert.AreEqual(originalStatus, JToken.Parse(status?.Status)); + + // Test case 3: external event now goes through + await client.ResumeAsync("wakeUp"); + status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(10)); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual(changedStatus, JToken.Parse(status?.Status)); await host.StopAsync(); } } /// - /// End-to-end test which validates the handling of unhandled exceptions generated from orchestrator code. + /// Test that a suspended orchestration can be terminated. /// - [DataTestMethod] + [TestMethod] [DataRow(true)] [DataRow(false)] - public async Task UnhandledOrchestrationException(bool enableExtendedSessions) + public async Task TerminateSuspendedOrchestration(bool enableExtendedSessions) { using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) { await host.StartAsync(); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Counter), 0); + await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); - // Empty string input should result in ArgumentNullException in the orchestration code. - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Throw), ""); - var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + await client.SuspendAsync("suspend"); + await client.WaitForStatusChange(TimeSpan.FromSeconds(10), OrchestrationStatus.Suspended); - Assert.AreEqual(OrchestrationStatus.Failed, status?.OrchestrationStatus); - Assert.IsTrue(status?.Output.Contains("null") == true); + await client.TerminateAsync("terminate"); + + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(10)); + + Assert.AreEqual(OrchestrationStatus.Terminated, status?.OrchestrationStatus); + Assert.AreEqual("terminate", status?.Output); await host.StopAsync(); } } /// - /// End-to-end test which validates the handling of unhandled exceptions generated from activity code. + /// End-to-end test which validates the Rewind functionality on more than one orchestration. /// - [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task UnhandledActivityException(bool enableExtendedSessions) + [TestMethod] + public async Task RewindOrchestrationsFail() { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: true)) { + Orchestrations.FactorialOrchestratorFail.ShouldFail = true; await host.StartAsync(); - string message = "Kah-BOOOOM!!!"; - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Throw), message); - var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + string singletonInstanceId1 = $"1_Test_{Guid.NewGuid():N}"; + string singletonInstanceId2 = $"2_Test_{Guid.NewGuid():N}"; - Assert.AreEqual(OrchestrationStatus.Failed, status?.OrchestrationStatus); - Assert.IsTrue(status?.Output.Contains(message) == true); + var client1 = await host.StartOrchestrationAsync( + typeof(Orchestrations.FactorialOrchestratorFail), + input: 3, + instanceId: singletonInstanceId1); - await host.StopAsync(); - } - } + var statusFail = await client1.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - /// - /// Fan-out/fan-in test which ensures each operation is run only once. - /// - [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task FanOutToTableStorage(bool enableExtendedSessions) - { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) - { - await host.StartAsync(); + Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); - int iterations = 100; + Orchestrations.FactorialOrchestratorFail.ShouldFail = false; - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.MapReduceTableStorage), iterations); - var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(120)); + var client2 = await host.StartOrchestrationAsync( + typeof(Orchestrations.SayHelloWithActivity), + input: "Catherine", + instanceId: singletonInstanceId2); - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.AreEqual(iterations, int.Parse(status?.Output)); + await client1.RewindAsync("Rewind failed orchestration only"); + + var statusRewind = await client1.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Completed, statusRewind?.OrchestrationStatus); + Assert.AreEqual("6", statusRewind?.Output); await host.StopAsync(); } } /// - /// Test which validates the ETW event source. + /// End-to-end test which validates the Rewind functionality with fan in fan out pattern. /// [TestMethod] - public void ValidateEventSource() - { -#if NETCOREAPP - EventSourceAnalyzer.InspectAll(AnalyticsEventSource.Log); -#else - try - { - EventSourceAnalyzer.InspectAll(AnalyticsEventSource.Log); - } - catch (FormatException) - { - Assert.Inconclusive("Known issue with .NET Framework, EventSourceAnalyzer, and DateTime parameters"); - } -#endif - } - - /// - /// End-to-end test which validates that orchestrations with <=60KB text message sizes can run successfully. - /// - [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task SmallTextMessagePayloads(bool enableExtendedSessions) + public async Task RewindActivityFailFanOut() { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: true)) { + Activities.HelloFailFanOut.ShouldFail1 = false; await host.StartAsync(); - // Generate a small random string payload - const int TargetPayloadSize = 1 * 1024; // 1 KB - const string Chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 {}/<>.-"; - var sb = new StringBuilder(); - var random = new Random(); - while (Encoding.Unicode.GetByteCount(sb.ToString()) < TargetPayloadSize) - { - for (int i = 0; i < 1000; i++) - { - sb.Append(Chars[random.Next(Chars.Length)]); - } - } + string singletonInstanceId = $"Test_{Guid.NewGuid():N}"; - string message = sb.ToString(); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); - var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(60)); + var client = await host.StartOrchestrationAsync( + typeof(Orchestrations.FanOutFanInRewind), + input: 3, + instanceId: singletonInstanceId); - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.AreEqual(message, JToken.Parse(status?.Output)); + var statusFail = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - await host.StopAsync(); - } - } + Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); - /// - /// End-to-end test which validates that orchestrations with > 60KB text message sizes can run successfully. - /// - [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task LargeQueueTextMessagePayloads_BlobUrl(bool enableExtendedSessions) - { - // Small enough to be a small table message, but a large queue message - const int largeMessageSize = 25 * 1024; + Activities.HelloFailFanOut.ShouldFail2 = false; - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions, fetchLargeMessages: false)) - { - await host.StartAsync(); + await client.RewindAsync("Rewind orchestrator with failed parallel activity."); - string message = this.GenerateMediumRandomStringPayload(largeMessageSize, utf8ByteSize: 3, utf16ByteSize: 2).ToString(); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); - var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + var statusRewind = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.AreEqual(message, JToken.Parse(status?.Output)); - Assert.AreEqual(message, JToken.Parse(status.Input)); + Assert.AreEqual(OrchestrationStatus.Completed, statusRewind?.OrchestrationStatus); + Assert.AreEqual("\"Done\"", statusRewind?.Output); await host.StopAsync(); } } + /// - /// End-to-end test which validates that orchestrations with > 60KB text message sizes can run successfully. + /// End-to-end test which validates the Rewind functionality on an activity function failure + /// with modified (to fail initially) SayHelloWithActivity orchestrator. /// - [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task LargeTableTextMessagePayloads_SizeViolation_BlobUrl(bool enableExtendedSessions) + [TestMethod] + public async Task RewindActivityFail() { - // Small enough to be a small queue message, but a large table message due to UTF encoding differences of ASCII characters - const int largeMessageSize = 32 * 1024; - - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions, fetchLargeMessages: false)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: true)) { await host.StartAsync(); - string message = this.GenerateMediumRandomStringPayload(largeMessageSize, utf8ByteSize: 1, utf16ByteSize: 2).ToString(); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); - var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + string singletonInstanceId = $"{Guid.NewGuid():N}"; - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - await ValidateBlobUrlAsync( - host.TaskHub, - client.InstanceId, - status?.Input, - Encoding.UTF8.GetByteCount(message)); - await ValidateBlobUrlAsync( - host.TaskHub, - client.InstanceId, - status?.Output, - Encoding.UTF8.GetByteCount(message)); - await host.StopAsync(); - } - } + var client = await host.StartOrchestrationAsync( + typeof(Orchestrations.SayHelloWithActivityFail), + input: "World", + instanceId: singletonInstanceId); - /// - /// End-to-end test which validates that orchestrations with > 60KB text message sizes can run successfully. - /// - [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task LargeOverallTextMessagePayloads_BlobUrl(bool enableExtendedSessions) - { - const int largeMessageSize = 80 * 1024; + var statusFail = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions, fetchLargeMessages: false)) - { - await host.StartAsync(); + Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); - string message = this.GenerateMediumRandomStringPayload(numChars: largeMessageSize).ToString(); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); - var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + Activities.HelloFailActivity.ShouldFail = false; - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - await ValidateBlobUrlAsync( - host.TaskHub, - client.InstanceId, - status?.Input, - Encoding.UTF8.GetByteCount(message)); - await ValidateBlobUrlAsync( - host.TaskHub, - client.InstanceId, - status?.Output, - Encoding.UTF8.GetByteCount(message)); + await client.RewindAsync("Activity failure test."); - Assert.IsTrue(status.Output.EndsWith("-Result.json.gz")); - Assert.IsTrue(status.Input.EndsWith("-Input.json.gz")); + var statusRewind = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Completed, statusRewind?.OrchestrationStatus); + Assert.AreEqual("\"Hello, World!\"", statusRewind?.Output); await host.StopAsync(); } } - /// - /// End-to-end test which validates that orchestrations with > 60KB text message sizes can run successfully. - /// - [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task LargeTextMessagePayloads_FetchLargeMessages(bool enableExtendedSessions) + [TestMethod] + public async Task RewindMultipleActivityFail() { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions, fetchLargeMessages: true)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: true)) { await host.StartAsync(); - string message = this.GenerateMediumRandomStringPayload().ToString(); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); - var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + string singletonInstanceId = $"Test_{Guid.NewGuid():N}"; - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.AreEqual(message, JToken.Parse(status?.Input)); - Assert.AreEqual(message, JToken.Parse(status?.Output)); + var client = await host.StartOrchestrationAsync( + typeof(Orchestrations.FactorialMultipleActivityFail), + input: 4, + instanceId: singletonInstanceId); + + var statusFail = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); + + Activities.MultiplyMultipleActivityFail.ShouldFail1 = false; + + await client.RewindAsync("Rewind for activity failure 1."); + + statusFail = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); + + Activities.MultiplyMultipleActivityFail.ShouldFail2 = false; + + await client.RewindAsync("Rewind for activity failure 2."); + + var statusRewind = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Completed, statusRewind?.OrchestrationStatus); + Assert.AreEqual("24", statusRewind?.Output); await host.StopAsync(); } } - /// - /// End-to-end test which validates that orchestrations with > 60KB text message sizes can run successfully. - /// - [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task LargeTableTextMessagePayloads_FetchLargeMessages(bool enableExtendedSessions) + [TestMethod] + public async Task RewindSubOrchestrationsTest() { - // Small enough to be a small queue message, but a large table message due to UTF encoding differences of ASCII characters - const int largeMessageSize = 32 * 1024; - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions, fetchLargeMessages: true)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: true)) { await host.StartAsync(); - string message = this.GenerateMediumRandomStringPayload(largeMessageSize, utf8ByteSize: 1, utf16ByteSize: 2).ToString(); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); - var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + string ParentInstanceId = $"Parent_{Guid.NewGuid():N}"; + string ChildInstanceId = $"Child_{Guid.NewGuid():N}"; - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.AreEqual(message, JToken.Parse(status?.Input)); - Assert.AreEqual(message, JToken.Parse(status?.Output)); + var clientParent = await host.StartOrchestrationAsync( + typeof(Orchestrations.ParentWorkflowSubOrchestrationFail), + input: true, + instanceId: ParentInstanceId); + + var statusFail = await clientParent.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); + + Orchestrations.ChildWorkflowSubOrchestrationFail.ShouldFail1 = false; + + await clientParent.RewindAsync("Rewind first suborchestration failure."); + + statusFail = await clientParent.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); + + Orchestrations.ChildWorkflowSubOrchestrationFail.ShouldFail2 = false; + + await clientParent.RewindAsync("Rewind second suborchestration failure."); + + var statusRewind = await clientParent.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Completed, statusRewind?.OrchestrationStatus); await host.StopAsync(); } } - /// - /// End-to-end test which validates that orchestrations with > 60KB of tag data can be run successfully. - /// [TestMethod] - public async Task LargeOrchestrationTags() + public async Task RewindSubOrchestrationActivityTest() { - const int largeMessageSize = 64 * 1024; - using TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false, fetchLargeMessages: true); - await host.StartAsync(); + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: true)) + { + await host.StartAsync(); - string bigMessage = this.GenerateMediumRandomStringPayload(largeMessageSize, utf8ByteSize: 1, utf16ByteSize: 2).ToString(); - var bigTags = new Dictionary { { "BigTag", bigMessage } }; - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), "Hello, world!", tags: bigTags); - var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + string ParentInstanceId = $"Parent_{Guid.NewGuid():N}"; + string ChildInstanceId = $"Child_{Guid.NewGuid():N}"; - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + var clientParent = await host.StartOrchestrationAsync( + typeof(Orchestrations.ParentWorkflowSubOrchestrationActivityFail), + input: true, + instanceId: ParentInstanceId); - // TODO: Uncomment these assertions as part of https://github.com/Azure/durabletask/issues/840. - ////Assert.IsNotNull(status?.Tags); - ////Assert.AreEqual(bigTags.Count, status.Tags.Count); - ////Assert.IsTrue(bigTags.TryGetValue("BigTag", out string actualMessage)); - ////Assert.AreEqual(bigMessage, actualMessage); + var statusFail = await clientParent.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - await host.StopAsync(); + Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); + + Activities.HelloFailSubOrchestrationActivity.ShouldFail1 = false; + + await clientParent.RewindAsync("Rewinding 1: child should still fail."); + + statusFail = await clientParent.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); + + Activities.HelloFailSubOrchestrationActivity.ShouldFail2 = false; + + await clientParent.RewindAsync("Rewinding 2: child should complete."); + + var statusRewind = await clientParent.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Completed, statusRewind?.OrchestrationStatus); + + await host.StopAsync(); + } } - /// - /// End-to-end test which validates that orchestrations with > 60KB text message sizes can run successfully. - /// - [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task NonBlobUriPayload_FetchLargeMessages_RetainsOriginalPayload(bool enableExtendedSessions) + [TestMethod] + public async Task RewindNestedSubOrchestrationTest() { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions, fetchLargeMessages: true)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: true)) { await host.StartAsync(); - string message = "https://anygivenurl.azurewebsites.net"; - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); - var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + string GrandparentInstanceId = $"Grandparent_{Guid.NewGuid():N}"; + string ChildInstanceId = $"Child_{Guid.NewGuid():N}"; - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.AreEqual(message, JToken.Parse(status?.Input)); - Assert.AreEqual(message, JToken.Parse(status?.Output)); + var clientGrandparent = await host.StartOrchestrationAsync( + typeof(Orchestrations.GrandparentWorkflowNestedActivityFail), + input: true, + instanceId: GrandparentInstanceId); + + var statusFail = await clientGrandparent.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); + + Activities.HelloFailNestedSuborchestration.ShouldFail1 = false; + + await clientGrandparent.RewindAsync("Rewind 1: Nested child activity still fails."); + + Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); + + Activities.HelloFailNestedSuborchestration.ShouldFail2 = false; + + await clientGrandparent.RewindAsync("Rewind 2: Nested child activity completes."); + + var statusRewind = await clientGrandparent.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Completed, statusRewind?.OrchestrationStatus); + //Assert.AreEqual("\"Hello, Catherine!\"", statusRewind?.Output); await host.StopAsync(); } } - /// - /// End-to-end test which validates that orchestrations with > 60KB text message sizes can run successfully. - /// [DataTestMethod] [DataRow(true)] [DataRow(false)] - public async Task LargeTextMessagePayloads_FetchLargeMessages_QueryState(bool enableExtendedSessions) + public async Task TimerCancellation(bool enableExtendedSessions) { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions, fetchLargeMessages: true)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) { await host.StartAsync(); - string message = this.GenerateMediumRandomStringPayload().ToString(); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); - var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + var timeout = TimeSpan.FromSeconds(10); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Approval), timeout); - //Ensure that orchestration state querying also retrieves messages - status = (await client.GetStateAsync(status.OrchestrationInstance.InstanceId)).First(); + // Need to wait for the instance to start before sending events to it. + // TODO: This requirement may not be ideal and should be revisited. + await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); + await client.RaiseEventAsync("approval", eventData: true); + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.AreEqual(message, JToken.Parse(status?.Input)); - Assert.AreEqual(message, JToken.Parse(status?.Output)); + Assert.AreEqual("Approved", JToken.Parse(status?.Output)); await host.StopAsync(); } } /// - /// End-to-end test which validates that exception messages that are considered valid Urls in the Uri.TryCreate() method - /// are handled with an additional Uri format check + /// End-to-end test which validates the handling of durable timer expiration. /// [DataTestMethod] [DataRow(true)] [DataRow(false)] - public async Task LargeTextMessagePayloads_URIFormatCheck(bool enableExtendedSessions) + public async Task TimerExpiration(bool enableExtendedSessions) { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions, fetchLargeMessages: true)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) { await host.StartAsync(); - string message = this.GenerateMediumRandomStringPayload().ToString(); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.ThrowException), "durabletask.core.exceptions.taskfailedexception: Task failed with an unhandled exception: This is an invalid operation.)"); - var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); - - //Ensure that orchestration state querying also retrieves messages - status = (await client.GetStateAsync(status.OrchestrationInstance.InstanceId)).First(); + var timeout = TimeSpan.FromSeconds(10); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Approval), timeout); - Assert.AreEqual(OrchestrationStatus.Failed, status?.OrchestrationStatus); - Assert.IsTrue(status?.Output.Contains("invalid operation") == true); + // Need to wait for the instance to start before sending events to it. + // TODO: This requirement may not be ideal and should be revisited. + await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); - await host.StopAsync(); - } - } + // Don't send any notification - let the internal timeout expire - private StringBuilder GenerateMediumRandomStringPayload(int numChars = 128*1024, short utf8ByteSize = 1, short utf16ByteSize = 2) - { - string Chars; - if (utf16ByteSize != 2 && utf16ByteSize != 4) - { - throw new InvalidOperationException($"No characters have byte size {utf16ByteSize} for UTF16"); - } - else if (utf8ByteSize < 1 || utf8ByteSize > 4) - { - throw new InvalidOperationException($"No characters have byte size {utf8ByteSize} for UTF8."); - } - else if (utf8ByteSize == 1 && utf16ByteSize == 2) - { - // Use a character set that is small for UTF8 and large for UTF16 - // This allows us to produce a smaller string for UTF8 than UTF16. - Chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 {}/<>."; - } - else if (utf16ByteSize == 2 && utf8ByteSize == 3) - { - // Use a character set that is small for UTF16 and large for UTF8 - // This allows us to produce a smaller string for UTF16 than UTF8. - Chars = "มันสนุกพี่บุ๋มมันโจ๊ะ"; - } - else - { - throw new InvalidOperationException($"This method has not yet added support for characters of utf8 size {utf8ByteSize} and utf16 size {utf16ByteSize}"); - } + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(20)); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual("Expired", JToken.Parse(status?.Output)); - var random = new Random(); - var sb = new StringBuilder(); - for (int i = 0; i < numChars; i++) - { - sb.Append(Chars[random.Next(Chars.Length)]); + await host.StopAsync(); } - - return sb; } /// - /// End-to-end test which validates that orchestrations with > 60KB binary bytes message sizes can run successfully. + /// End-to-end test which validates that orchestrations run concurrently of each other (up to 100 by default). /// [DataTestMethod] [DataRow(true)] [DataRow(false)] - public async Task LargeBinaryByteMessagePayloads(bool enableExtendedSessions) + public async Task OrchestrationConcurrency(bool enableExtendedSessions) { using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) { await host.StartAsync(); - // Construct byte array from large binary file of size 826KB - string originalFileName = "large.jpeg"; - string currentDirectory = Directory.GetCurrentDirectory(); - string originalFilePath = Path.Combine(currentDirectory, originalFileName); - byte[] readBytes = File.ReadAllBytes(originalFilePath); + Func orchestrationStarter = async delegate () + { + var timeout = TimeSpan.FromSeconds(10); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Approval), timeout); + await client.WaitForCompletionAsync(TimeSpan.FromSeconds(60)); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.EchoBytes), readBytes); - var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(1)); + // Don't send any notification - let the internal timeout expire + }; - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + int iterations = 10; + var tasks = new Task[iterations]; + for (int i = 0; i < iterations; i++) + { + tasks[i] = orchestrationStarter(); + } - byte[] resultBytes = JObject.Parse(status?.Output).ToObject(); - Assert.IsTrue(readBytes.SequenceEqual(resultBytes)); + // The 10 orchestrations above (which each delay for 10 seconds) should all complete in less than 60 seconds. + Task parallelOrchestrations = Task.WhenAll(tasks); + Task timeoutTask = Task.Delay(TimeSpan.FromSeconds(60)); + + Task winner = await Task.WhenAny(parallelOrchestrations, timeoutTask); + Assert.AreEqual(parallelOrchestrations, winner); await host.StopAsync(); } } /// - /// End-to-end test which validates that orchestrations with > 60KB binary string message sizes can run successfully. + /// End-to-end test which validates the orchestrator's exception handling behavior. /// [DataTestMethod] [DataRow(true)] [DataRow(false)] - public async Task LargeBinaryStringMessagePayloads(bool enableExtendedSessions) + public async Task HandledActivityException(bool enableExtendedSessions) { using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) { await host.StartAsync(); - // Construct string message from large binary file of size 826KB - string originalFileName = "large.jpeg"; - string currentDirectory = Directory.GetCurrentDirectory(); - string originalFilePath = Path.Combine(currentDirectory, originalFileName); - byte[] readBytes = File.ReadAllBytes(originalFilePath); - string message = Convert.ToBase64String(readBytes); - - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); - var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(1)); + // Empty string input should result in ArgumentNullException in the orchestration code. + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.TryCatchLoop), 5); + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(15)); Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - - // Large message payloads may actually get bigger when stored in blob storage. - string result = JToken.Parse(status?.Output).ToString(); - Assert.AreEqual(message, result); + Assert.AreEqual(5, JToken.Parse(status?.Output)); await host.StopAsync(); } } /// - /// End-to-end test which validates that a completed singleton instance can be recreated. + /// End-to-end test which validates the handling of unhandled exceptions generated from orchestrator code. /// [DataTestMethod] [DataRow(true)] [DataRow(false)] - public async Task RecreateCompletedInstance(bool enableExtendedSessions) + public async Task UnhandledOrchestrationException(bool enableExtendedSessions) { using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) { await host.StartAsync(); - string singletonInstanceId = $"HelloSingleton_{Guid.NewGuid():N}"; - - var client = await host.StartOrchestrationAsync( - typeof(Orchestrations.SayHelloWithActivity), - input: "One", - instanceId: singletonInstanceId); + // Empty string input should result in ArgumentNullException in the orchestration code. + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Throw), ""); var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.AreEqual("One", JToken.Parse(status?.Input)); - Assert.AreEqual("Hello, One!", JToken.Parse(status?.Output)); - - client = await host.StartOrchestrationAsync( - typeof(Orchestrations.SayHelloWithActivity), - input: "Two", - instanceId: singletonInstanceId); - status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.AreEqual("Two", JToken.Parse(status?.Input)); - Assert.AreEqual("Hello, Two!", JToken.Parse(status?.Output)); + Assert.AreEqual(OrchestrationStatus.Failed, status?.OrchestrationStatus); + Assert.IsTrue(status?.Output.Contains("null") == true); await host.StopAsync(); } } /// - /// End-to-end test which validates that a failed singleton instance can be recreated. + /// End-to-end test which validates the handling of unhandled exceptions generated from activity code. /// [DataTestMethod] [DataRow(true)] [DataRow(false)] - public async Task RecreateFailedInstance(bool enableExtendedSessions) + public async Task UnhandledActivityException(bool enableExtendedSessions) { using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) { await host.StartAsync(); - string singletonInstanceId = $"HelloSingleton_{Guid.NewGuid():N}"; - - var client = await host.StartOrchestrationAsync( - typeof(Orchestrations.SayHelloWithActivity), - input: null, // this will cause the orchestration to fail - instanceId: singletonInstanceId); + string message = "Kah-BOOOOM!!!"; + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Throw), message); var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); Assert.AreEqual(OrchestrationStatus.Failed, status?.OrchestrationStatus); - - client = await host.StartOrchestrationAsync( - typeof(Orchestrations.SayHelloWithActivity), - input: "NotNull", - instanceId: singletonInstanceId); - status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - - Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.AreEqual("Hello, NotNull!", JToken.Parse(status?.Output)); + Assert.IsTrue(status?.Output.Contains(message) == true); await host.StopAsync(); } } /// - /// End-to-end test which validates that a terminated orchestration can be recreated. + /// Fan-out/fan-in test which ensures each operation is run only once. /// [DataTestMethod] [DataRow(true)] [DataRow(false)] - public async Task RecreateTerminatedInstance(bool enableExtendedSessions) + public async Task FanOutToTableStorage(bool enableExtendedSessions) { using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) { await host.StartAsync(); - string singletonInstanceId = $"SingletonCounter_{Guid.NewGuid():N}"; - - // Using the counter orchestration because it will wait indefinitely for input. - var client = await host.StartOrchestrationAsync( - typeof(Orchestrations.Counter), - input: -1, - instanceId: singletonInstanceId); - - // Need to wait for the instance to start before we can terminate it. - await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); - - await client.TerminateAsync("sayōnara"); - - var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(10)); - - Assert.AreEqual(OrchestrationStatus.Terminated, status?.OrchestrationStatus); - Assert.AreEqual("-1", status?.Input); - Assert.AreEqual("sayōnara", status?.Output); + int iterations = 100; - client = await host.StartOrchestrationAsync( - typeof(Orchestrations.Counter), - input: 0, - instanceId: singletonInstanceId); - status = await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.MapReduceTableStorage), iterations); + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(120)); - Assert.AreEqual(OrchestrationStatus.Running, status?.OrchestrationStatus); - Assert.AreEqual("0", status?.Input); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual(iterations, int.Parse(status?.Output)); await host.StopAsync(); } } /// - /// End-to-end test which validates that a running orchestration can be recreated. + /// Test which validates the ETW event source. /// - [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task RecreateRunningInstance(bool enableExtendedSessions) + [TestMethod] + public void ValidateEventSource() { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost( - enableExtendedSessions, - extendedSessionTimeoutInSeconds: 15)) +#if NETCOREAPP + EventSourceAnalyzer.InspectAll(AnalyticsEventSource.Log); +#else + try { - await host.StartAsync(); - - string singletonInstanceId = $"SingletonCounter_{DateTime.Now:o}"; - - // Using the counter orchestration because it will wait indefinitely for input. - var client = await host.StartOrchestrationAsync( - typeof(Orchestrations.Counter), - input: 0, - instanceId: singletonInstanceId); - - var status = await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); - - Assert.AreEqual(OrchestrationStatus.Running, status?.OrchestrationStatus); - Assert.AreEqual("0", status?.Input); - Assert.AreEqual(null, status?.Output); - - client = await host.StartOrchestrationAsync( - typeof(Orchestrations.Counter), - input: 99, - instanceId: singletonInstanceId); - - // Note that with extended sessions, the startup time may take longer because the dispatcher - // will wait for the current extended session to expire before the new create message is accepted. - status = await client.WaitForStartupAsync(TimeSpan.FromSeconds(20)); - - Assert.AreEqual(OrchestrationStatus.Running, status?.OrchestrationStatus); - Assert.AreEqual("99", status?.Input); - - await host.StopAsync(); + EventSourceAnalyzer.InspectAll(AnalyticsEventSource.Log); + } + catch (FormatException) + { + Assert.Inconclusive("Known issue with .NET Framework, EventSourceAnalyzer, and DateTime parameters"); } +#endif } /// - /// End-to-end test which validates that an orchestration can continue processing - /// even after its extended session has expired. + /// End-to-end test which validates that orchestrations with <=60KB text message sizes can run successfully. /// - [TestMethod] - public async Task ExtendedSessions_SessionTimeout() + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task SmallTextMessagePayloads(bool enableExtendedSessions) { - const int SessionTimeoutInseconds = 5; - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost( - enableExtendedSessions: true, - extendedSessionTimeoutInSeconds: SessionTimeoutInseconds)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) { await host.StartAsync(); - string singletonInstanceId = $"SingletonCounter_{DateTime.Now:o}"; - - // Using the counter orchestration because it will wait indefinitely for input. - var client = await host.StartOrchestrationAsync( - typeof(Orchestrations.Counter), - input: 0, - instanceId: singletonInstanceId); - - var status = await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); - - Assert.AreEqual(OrchestrationStatus.Running, status?.OrchestrationStatus); - Assert.AreEqual("0", status?.Input); - Assert.AreEqual(null, status?.Output); - - // Delay long enough for the session to expire - await Task.Delay(TimeSpan.FromSeconds(SessionTimeoutInseconds + 1)); - - await client.RaiseEventAsync("operation", "incr"); - await Task.Delay(TimeSpan.FromSeconds(2)); - - // Make sure it's still running and didn't complete early (or fail). - status = await client.GetStatusAsync(); - Assert.IsTrue( - status?.OrchestrationStatus == OrchestrationStatus.Running || - status?.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew); - - // The end message will cause the actor to complete itself. - await client.RaiseEventAsync("operation", "end"); + // Generate a small random string payload + const int TargetPayloadSize = 1 * 1024; // 1 KB + const string Chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 {}/<>.-"; + var sb = new StringBuilder(); + var random = new Random(); + while (Encoding.Unicode.GetByteCount(sb.ToString()) < TargetPayloadSize) + { + for (int i = 0; i < 1000; i++) + { + sb.Append(Chars[random.Next(Chars.Length)]); + } + } - status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(10)); + string message = sb.ToString(); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(60)); Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.AreEqual(1, JToken.Parse(status?.Output)); + Assert.AreEqual(message, JToken.Parse(status?.Output)); await host.StopAsync(); } } /// - /// Tests an orchestration that does two consecutive fan-out, fan-ins. - /// This is a regression test for https://github.com/Azure/durabletask/issues/241. + /// End-to-end test which validates that orchestrations with > 60KB text message sizes can run successfully. /// [DataTestMethod] [DataRow(true)] [DataRow(false)] - public async Task DoubleFanOut(bool enableExtendedSessions) + public async Task LargeQueueTextMessagePayloads_BlobUrl(bool enableExtendedSessions) { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + // Small enough to be a small table message, but a large queue message + const int largeMessageSize = 25 * 1024; + + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions, fetchLargeMessages: false)) { await host.StartAsync(); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.DoubleFanOut), null); - var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - + string message = this.GenerateMediumRandomStringPayload(largeMessageSize, utf8ByteSize: 3, utf16ByteSize: 2).ToString(); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); + var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual(message, JToken.Parse(status?.Output)); + Assert.AreEqual(message, JToken.Parse(status.Input)); + await host.StopAsync(); } } - private static async Task ValidateBlobUrlAsync(string taskHubName, string instanceId, string value, int originalPayloadSize = 0) + /// + /// End-to-end test which validates that orchestrations with > 60KB text message sizes can run successfully. + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task LargeTableTextMessagePayloads_SizeViolation_BlobUrl(bool enableExtendedSessions) { - string sanitizedInstanceId = KeySanitation.EscapePartitionKey(instanceId); + // Small enough to be a small queue message, but a large table message due to UTF encoding differences of ASCII characters + const int largeMessageSize = 32 * 1024; - CloudStorageAccount account = CloudStorageAccount.Parse(TestHelpers.GetTestStorageAccountConnectionString()); - Assert.IsTrue(value.StartsWith(account.BlobStorageUri.PrimaryUri.OriginalString)); - Assert.IsTrue(value.Contains("/" + sanitizedInstanceId + "/")); - Assert.IsTrue(value.EndsWith(".json.gz")); + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions, fetchLargeMessages: false)) + { + await host.StartAsync(); - string containerName = $"{taskHubName.ToLowerInvariant()}-largemessages"; - CloudBlobClient client = account.CreateCloudBlobClient(); - CloudBlobContainer container = client.GetContainerReference(containerName); - Assert.IsTrue(await container.ExistsAsync(), $"Blob container {containerName} is expected to exist."); + string message = this.GenerateMediumRandomStringPayload(largeMessageSize, utf8ByteSize: 1, utf16ByteSize: 2).ToString(); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); + var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); - await client.GetBlobReferenceFromServerAsync(new Uri(value)); - CloudBlobDirectory instanceDirectory = container.GetDirectoryReference(sanitizedInstanceId); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + await ValidateBlobUrlAsync( + host.TaskHub, + client.InstanceId, + status?.Input, + Encoding.UTF8.GetByteCount(message)); + await ValidateBlobUrlAsync( + host.TaskHub, + client.InstanceId, + status?.Output, + Encoding.UTF8.GetByteCount(message)); + await host.StopAsync(); + } + } - string blobName = value.Split('/').Last(); - CloudBlob blob = instanceDirectory.GetBlobReference(blobName); - Assert.IsTrue(await blob.ExistsAsync(), $"Blob named {blob.Uri} is expected to exist."); + /// + /// End-to-end test which validates that orchestrations with > 60KB text message sizes can run successfully. + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task LargeOverallTextMessagePayloads_BlobUrl(bool enableExtendedSessions) + { + const int largeMessageSize = 80 * 1024; - if (originalPayloadSize > 0) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions, fetchLargeMessages: false)) { - await blob.FetchAttributesAsync(); - Assert.IsTrue(blob.Properties.Length < originalPayloadSize, "Blob is expected to be compressed"); + await host.StartAsync(); + + string message = this.GenerateMediumRandomStringPayload(numChars: largeMessageSize).ToString(); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); + var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + await ValidateBlobUrlAsync( + host.TaskHub, + client.InstanceId, + status?.Input, + Encoding.UTF8.GetByteCount(message)); + await ValidateBlobUrlAsync( + host.TaskHub, + client.InstanceId, + status?.Output, + Encoding.UTF8.GetByteCount(message)); + + Assert.IsTrue(status.Output.EndsWith("-Result.json.gz")); + Assert.IsTrue(status.Input.EndsWith("-Input.json.gz")); + + await host.StopAsync(); } } /// - /// Tests the behavior of from orchestrations and activities. + /// End-to-end test which validates that orchestrations with > 60KB text message sizes can run successfully. /// [DataTestMethod] [DataRow(true)] [DataRow(false)] - public async Task AbortOrchestrationAndActivity(bool enableExtendedSessions) + public async Task LargeTextMessagePayloads_FetchLargeMessages(bool enableExtendedSessions) { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions, fetchLargeMessages: true)) { await host.StartAsync(); - string input = Guid.NewGuid().ToString(); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.AbortSessionOrchestration), input); - var status = await client.WaitForCompletionAsync(StandardTimeout); + string message = this.GenerateMediumRandomStringPayload().ToString(); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); + var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.IsNotNull(status.Output); - Assert.AreEqual("True", JToken.Parse(status.Output)); + Assert.AreEqual(message, JToken.Parse(status?.Input)); + Assert.AreEqual(message, JToken.Parse(status?.Output)); + await host.StopAsync(); } } - /// - /// Validates scheduled starts, ensuring they are executed according to defined start date time + /// + /// End-to-end test which validates that orchestrations with > 60KB text message sizes can run successfully. /// - /// - /// [DataTestMethod] [DataRow(true)] [DataRow(false)] - public async Task ScheduledStart_Inline(bool enableExtendedSessions) + public async Task LargeTableTextMessagePayloads_FetchLargeMessages(bool enableExtendedSessions) { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + // Small enough to be a small queue message, but a large table message due to UTF encoding differences of ASCII characters + const int largeMessageSize = 32 * 1024; + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions, fetchLargeMessages: true)) { await host.StartAsync(); - var expectedStartTime = DateTime.UtcNow.AddSeconds(30); - var clientStartingIn30Seconds = await host.StartOrchestrationAsync(typeof(Orchestrations.CurrentTimeInline), "Current Time!", startAt: expectedStartTime); - var clientStartingNow = await host.StartOrchestrationAsync(typeof(Orchestrations.CurrentTimeInline), "Current Time!"); + string message = this.GenerateMediumRandomStringPayload(largeMessageSize, utf8ByteSize: 1, utf16ByteSize: 2).ToString(); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); + var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); - var statusStartingNow = clientStartingNow.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - var statusStartingIn30Seconds = clientStartingIn30Seconds.WaitForCompletionAsync(TimeSpan.FromSeconds(60)); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual(message, JToken.Parse(status?.Input)); + Assert.AreEqual(message, JToken.Parse(status?.Output)); - await Task.WhenAll(statusStartingNow, statusStartingIn30Seconds); + await host.StopAsync(); + } + } - Assert.AreEqual(OrchestrationStatus.Completed, statusStartingNow.Result?.OrchestrationStatus); - Assert.AreEqual("Current Time!", JToken.Parse(statusStartingNow.Result?.Input)); - Assert.IsNull(statusStartingNow.Result.ScheduledStartTime); + /// + /// End-to-end test which validates that orchestrations with > 60KB of tag data can be run successfully. + /// + [TestMethod] + public async Task LargeOrchestrationTags() + { + const int largeMessageSize = 64 * 1024; + using TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false, fetchLargeMessages: true); + await host.StartAsync(); - Assert.AreEqual(OrchestrationStatus.Completed, statusStartingIn30Seconds.Result?.OrchestrationStatus); - Assert.AreEqual("Current Time!", JToken.Parse(statusStartingIn30Seconds.Result?.Input)); - Assert.AreEqual(expectedStartTime, statusStartingIn30Seconds.Result.ScheduledStartTime); + string bigMessage = this.GenerateMediumRandomStringPayload(largeMessageSize, utf8ByteSize: 1, utf16ByteSize: 2).ToString(); + var bigTags = new Dictionary { { "BigTag", bigMessage } }; + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), "Hello, world!", tags: bigTags); + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - var startNowResult = (DateTime)JToken.Parse(statusStartingNow.Result?.Output); - var startIn30SecondsResult = (DateTime)JToken.Parse(statusStartingIn30Seconds.Result?.Output); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - Assert.IsTrue(startIn30SecondsResult > startNowResult); - Assert.IsTrue(startIn30SecondsResult >= expectedStartTime); + // TODO: Uncomment these assertions as part of https://github.com/Azure/durabletask/issues/840. + ////Assert.IsNotNull(status?.Tags); + ////Assert.AreEqual(bigTags.Count, status.Tags.Count); + ////Assert.IsTrue(bigTags.TryGetValue("BigTag", out string actualMessage)); + ////Assert.AreEqual(bigMessage, actualMessage); - await host.StopAsync(); - } + await host.StopAsync(); } /// - /// Validates scheduled starts, ensuring they are executed according to defined start date time + /// End-to-end test which validates that orchestrations with > 60KB text message sizes can run successfully. /// - /// - /// [DataTestMethod] [DataRow(true)] [DataRow(false)] - public async Task ScheduledStart_Activity(bool enableExtendedSessions) + public async Task NonBlobUriPayload_FetchLargeMessages_RetainsOriginalPayload(bool enableExtendedSessions) { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions, fetchLargeMessages: true)) { await host.StartAsync(); - var expectedStartTime = DateTime.UtcNow.AddSeconds(30); - var clientStartingIn30Seconds = await host.StartOrchestrationAsync(typeof(Orchestrations.CurrentTimeActivity), "Current Time!", startAt: expectedStartTime); - var clientStartingNow = await host.StartOrchestrationAsync(typeof(Orchestrations.CurrentTimeActivity), "Current Time!"); + string message = "https://anygivenurl.azurewebsites.net"; + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); + var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); - var statusStartingNow = clientStartingNow.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - var statusStartingIn30Seconds = clientStartingIn30Seconds.WaitForCompletionAsync(TimeSpan.FromSeconds(60)); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual(message, JToken.Parse(status?.Input)); + Assert.AreEqual(message, JToken.Parse(status?.Output)); - await Task.WhenAll(statusStartingNow, statusStartingIn30Seconds); + await host.StopAsync(); + } + } - Assert.AreEqual(OrchestrationStatus.Completed, statusStartingNow.Result?.OrchestrationStatus); - Assert.AreEqual("Current Time!", JToken.Parse(statusStartingNow.Result?.Input)); - Assert.IsNull(statusStartingNow.Result.ScheduledStartTime); + /// + /// End-to-end test which validates that orchestrations with > 60KB text message sizes can run successfully. + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task LargeTextMessagePayloads_FetchLargeMessages_QueryState(bool enableExtendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions, fetchLargeMessages: true)) + { + await host.StartAsync(); - Assert.AreEqual(OrchestrationStatus.Completed, statusStartingIn30Seconds.Result?.OrchestrationStatus); - Assert.AreEqual("Current Time!", JToken.Parse(statusStartingIn30Seconds.Result?.Input)); - Assert.AreEqual(expectedStartTime, statusStartingIn30Seconds.Result.ScheduledStartTime); + string message = this.GenerateMediumRandomStringPayload().ToString(); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); + var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); - var startNowResult = (DateTime)JToken.Parse(statusStartingNow.Result?.Output); - var startIn30SecondsResult = (DateTime)JToken.Parse(statusStartingIn30Seconds.Result?.Output); + //Ensure that orchestration state querying also retrieves messages + status = (await client.GetStateAsync(status.OrchestrationInstance.InstanceId)).First(); - Assert.IsTrue(startIn30SecondsResult > startNowResult); - Assert.IsTrue(startIn30SecondsResult >= expectedStartTime); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual(message, JToken.Parse(status?.Input)); + Assert.AreEqual(message, JToken.Parse(status?.Output)); await host.StopAsync(); } } /// - /// Validates scheduled starts, ensuring they are executed according to defined start date time + /// End-to-end test which validates that exception messages that are considered valid Urls in the Uri.TryCreate() method + /// are handled with an additional Uri format check /// - /// - /// [DataTestMethod] [DataRow(true)] [DataRow(false)] - public async Task ScheduledStart_Activity_GetStatus_Returns_ScheduledStart(bool enableExtendedSessions) + public async Task LargeTextMessagePayloads_URIFormatCheck(bool enableExtendedSessions) { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions, fetchLargeMessages: true)) { await host.StartAsync(); - var expectedStartTime = DateTime.UtcNow.AddSeconds(30); - var clientStartingIn30Seconds = await host.StartOrchestrationAsync(typeof(Orchestrations.DelayedCurrentTimeActivity), "Delayed Current Time!", startAt: expectedStartTime); - var clientStartingNow = await host.StartOrchestrationAsync(typeof(Orchestrations.DelayedCurrentTimeActivity), "Delayed Current Time!"); - - var statusStartingIn30Seconds = await clientStartingIn30Seconds.GetStatusAsync(); - Assert.IsNotNull(statusStartingIn30Seconds.ScheduledStartTime); - Assert.AreEqual(expectedStartTime, statusStartingIn30Seconds.ScheduledStartTime); + string message = this.GenerateMediumRandomStringPayload().ToString(); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.ThrowException), "durabletask.core.exceptions.taskfailedexception: Task failed with an unhandled exception: This is an invalid operation.)"); + var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); - var statusStartingNow = await clientStartingNow.GetStatusAsync(); - Assert.IsNull(statusStartingNow.ScheduledStartTime); + //Ensure that orchestration state querying also retrieves messages + status = (await client.GetStateAsync(status.OrchestrationInstance.InstanceId)).First(); - await Task.WhenAll( - clientStartingNow.WaitForCompletionAsync(TimeSpan.FromSeconds(35)), - clientStartingIn30Seconds.WaitForCompletionAsync(TimeSpan.FromSeconds(65)) - ); + Assert.AreEqual(OrchestrationStatus.Failed, status?.OrchestrationStatus); + Assert.IsTrue(status?.Output.Contains("invalid operation") == true); await host.StopAsync(); } } - static class Orchestrations - { - internal class SayHelloInline : TaskOrchestration - { - public override Task RunTask(OrchestrationContext context, string input) - { - return Task.FromResult($"Hello, {input}!"); - } - } + private StringBuilder GenerateMediumRandomStringPayload(int numChars = 128*1024, short utf8ByteSize = 1, short utf16ByteSize = 2) + { + string Chars; + if (utf16ByteSize != 2 && utf16ByteSize != 4) + { + throw new InvalidOperationException($"No characters have byte size {utf16ByteSize} for UTF16"); + } + else if (utf8ByteSize < 1 || utf8ByteSize > 4) + { + throw new InvalidOperationException($"No characters have byte size {utf8ByteSize} for UTF8."); + } + else if (utf8ByteSize == 1 && utf16ByteSize == 2) + { + // Use a character set that is small for UTF8 and large for UTF16 + // This allows us to produce a smaller string for UTF8 than UTF16. + Chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 {}/<>."; + } + else if (utf16ByteSize == 2 && utf8ByteSize == 3) + { + // Use a character set that is small for UTF16 and large for UTF8 + // This allows us to produce a smaller string for UTF16 than UTF8. + Chars = "มันสนุกพี่บุ๋มมันโจ๊ะ"; + } + else + { + throw new InvalidOperationException($"This method has not yet added support for characters of utf8 size {utf8ByteSize} and utf16 size {utf16ByteSize}"); + } + + var random = new Random(); + var sb = new StringBuilder(); + for (int i = 0; i < numChars; i++) + { + sb.Append(Chars[random.Next(Chars.Length)]); + } + + return sb; + } + + /// + /// End-to-end test which validates that orchestrations with > 60KB binary bytes message sizes can run successfully. + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task LargeBinaryByteMessagePayloads(bool enableExtendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + { + await host.StartAsync(); + + // Construct byte array from large binary file of size 826KB + string originalFileName = "large.jpeg"; + string currentDirectory = Directory.GetCurrentDirectory(); + string originalFilePath = Path.Combine(currentDirectory, originalFileName); + byte[] readBytes = File.ReadAllBytes(originalFilePath); + + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.EchoBytes), readBytes); + var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(1)); + + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + + byte[] resultBytes = JObject.Parse(status?.Output).ToObject(); + Assert.IsTrue(readBytes.SequenceEqual(resultBytes)); + + await host.StopAsync(); + } + } + + /// + /// End-to-end test which validates that orchestrations with > 60KB binary string message sizes can run successfully. + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task LargeBinaryStringMessagePayloads(bool enableExtendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + { + await host.StartAsync(); + + // Construct string message from large binary file of size 826KB + string originalFileName = "large.jpeg"; + string currentDirectory = Directory.GetCurrentDirectory(); + string originalFilePath = Path.Combine(currentDirectory, originalFileName); + byte[] readBytes = File.ReadAllBytes(originalFilePath); + string message = Convert.ToBase64String(readBytes); + + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), message); + var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(1)); + + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + + // Large message payloads may actually get bigger when stored in blob storage. + string result = JToken.Parse(status?.Output).ToString(); + Assert.AreEqual(message, result); + + await host.StopAsync(); + } + } + + /// + /// End-to-end test which validates that a completed singleton instance can be recreated. + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task RecreateCompletedInstance(bool enableExtendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + { + await host.StartAsync(); + + string singletonInstanceId = $"HelloSingleton_{Guid.NewGuid():N}"; + + var client = await host.StartOrchestrationAsync( + typeof(Orchestrations.SayHelloWithActivity), + input: "One", + instanceId: singletonInstanceId); + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual("One", JToken.Parse(status?.Input)); + Assert.AreEqual("Hello, One!", JToken.Parse(status?.Output)); + + client = await host.StartOrchestrationAsync( + typeof(Orchestrations.SayHelloWithActivity), + input: "Two", + instanceId: singletonInstanceId); + status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual("Two", JToken.Parse(status?.Input)); + Assert.AreEqual("Hello, Two!", JToken.Parse(status?.Output)); + + await host.StopAsync(); + } + } + + /// + /// End-to-end test which validates that a failed singleton instance can be recreated. + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task RecreateFailedInstance(bool enableExtendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + { + await host.StartAsync(); + + string singletonInstanceId = $"HelloSingleton_{Guid.NewGuid():N}"; + + var client = await host.StartOrchestrationAsync( + typeof(Orchestrations.SayHelloWithActivity), + input: null, // this will cause the orchestration to fail + instanceId: singletonInstanceId); + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Failed, status?.OrchestrationStatus); + + client = await host.StartOrchestrationAsync( + typeof(Orchestrations.SayHelloWithActivity), + input: "NotNull", + instanceId: singletonInstanceId); + status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual("Hello, NotNull!", JToken.Parse(status?.Output)); + + await host.StopAsync(); + } + } + + /// + /// End-to-end test which validates that a terminated orchestration can be recreated. + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task RecreateTerminatedInstance(bool enableExtendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + { + await host.StartAsync(); + + string singletonInstanceId = $"SingletonCounter_{Guid.NewGuid():N}"; + + // Using the counter orchestration because it will wait indefinitely for input. + var client = await host.StartOrchestrationAsync( + typeof(Orchestrations.Counter), + input: -1, + instanceId: singletonInstanceId); + + // Need to wait for the instance to start before we can terminate it. + await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); + + await client.TerminateAsync("sayōnara"); + + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(10)); + + Assert.AreEqual(OrchestrationStatus.Terminated, status?.OrchestrationStatus); + Assert.AreEqual("-1", status?.Input); + Assert.AreEqual("sayōnara", status?.Output); + + client = await host.StartOrchestrationAsync( + typeof(Orchestrations.Counter), + input: 0, + instanceId: singletonInstanceId); + status = await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); + + Assert.AreEqual(OrchestrationStatus.Running, status?.OrchestrationStatus); + Assert.AreEqual("0", status?.Input); + + await host.StopAsync(); + } + } + + /// + /// End-to-end test which validates that a running orchestration can be recreated. + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task RecreateRunningInstance(bool enableExtendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost( + enableExtendedSessions, + extendedSessionTimeoutInSeconds: 15)) + { + await host.StartAsync(); + + string singletonInstanceId = $"SingletonCounter_{DateTime.Now:o}"; + + // Using the counter orchestration because it will wait indefinitely for input. + var client = await host.StartOrchestrationAsync( + typeof(Orchestrations.Counter), + input: 0, + instanceId: singletonInstanceId); + + var status = await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); + + Assert.AreEqual(OrchestrationStatus.Running, status?.OrchestrationStatus); + Assert.AreEqual("0", status?.Input); + Assert.AreEqual(null, status?.Output); + + client = await host.StartOrchestrationAsync( + typeof(Orchestrations.Counter), + input: 99, + instanceId: singletonInstanceId); + + // Note that with extended sessions, the startup time may take longer because the dispatcher + // will wait for the current extended session to expire before the new create message is accepted. + status = await client.WaitForStartupAsync(TimeSpan.FromSeconds(20)); + + Assert.AreEqual(OrchestrationStatus.Running, status?.OrchestrationStatus); + Assert.AreEqual("99", status?.Input); + + await host.StopAsync(); + } + } + + /// + /// End-to-end test which validates that an orchestration can continue processing + /// even after its extended session has expired. + /// + [TestMethod] + public async Task ExtendedSessions_SessionTimeout() + { + const int SessionTimeoutInseconds = 5; + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost( + enableExtendedSessions: true, + extendedSessionTimeoutInSeconds: SessionTimeoutInseconds)) + { + await host.StartAsync(); + + string singletonInstanceId = $"SingletonCounter_{DateTime.Now:o}"; + + // Using the counter orchestration because it will wait indefinitely for input. + var client = await host.StartOrchestrationAsync( + typeof(Orchestrations.Counter), + input: 0, + instanceId: singletonInstanceId); + + var status = await client.WaitForStartupAsync(TimeSpan.FromSeconds(10)); + + Assert.AreEqual(OrchestrationStatus.Running, status?.OrchestrationStatus); + Assert.AreEqual("0", status?.Input); + Assert.AreEqual(null, status?.Output); + + // Delay long enough for the session to expire + await Task.Delay(TimeSpan.FromSeconds(SessionTimeoutInseconds + 1)); + + await client.RaiseEventAsync("operation", "incr"); + await Task.Delay(TimeSpan.FromSeconds(2)); + + // Make sure it's still running and didn't complete early (or fail). + status = await client.GetStatusAsync(); + Assert.IsTrue( + status?.OrchestrationStatus == OrchestrationStatus.Running || + status?.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew); + + // The end message will cause the actor to complete itself. + await client.RaiseEventAsync("operation", "end"); + + status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(10)); + + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual(1, JToken.Parse(status?.Output)); + + await host.StopAsync(); + } + } + + /// + /// Tests an orchestration that does two consecutive fan-out, fan-ins. + /// This is a regression test for https://github.com/Azure/durabletask/issues/241. + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task DoubleFanOut(bool enableExtendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + { + await host.StartAsync(); + + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.DoubleFanOut), null); + var status = await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + await host.StopAsync(); + } + } + + private static async Task ValidateBlobUrlAsync(string taskHubName, string instanceId, string value, int originalPayloadSize = 0) + { + string sanitizedInstanceId = KeySanitation.EscapePartitionKey(instanceId); + + CloudStorageAccount account = CloudStorageAccount.Parse(TestHelpers.GetTestStorageAccountConnectionString()); + Assert.IsTrue(value.StartsWith(account.BlobStorageUri.PrimaryUri.OriginalString)); + Assert.IsTrue(value.Contains("/" + sanitizedInstanceId + "/")); + Assert.IsTrue(value.EndsWith(".json.gz")); + + string containerName = $"{taskHubName.ToLowerInvariant()}-largemessages"; + CloudBlobClient client = account.CreateCloudBlobClient(); + CloudBlobContainer container = client.GetContainerReference(containerName); + Assert.IsTrue(await container.ExistsAsync(), $"Blob container {containerName} is expected to exist."); + + await client.GetBlobReferenceFromServerAsync(new Uri(value)); + CloudBlobDirectory instanceDirectory = container.GetDirectoryReference(sanitizedInstanceId); + + string blobName = value.Split('/').Last(); + CloudBlob blob = instanceDirectory.GetBlobReference(blobName); + Assert.IsTrue(await blob.ExistsAsync(), $"Blob named {blob.Uri} is expected to exist."); + + if (originalPayloadSize > 0) + { + await blob.FetchAttributesAsync(); + Assert.IsTrue(blob.Properties.Length < originalPayloadSize, "Blob is expected to be compressed"); + } + } + + /// + /// Tests the behavior of from orchestrations and activities. + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task AbortOrchestrationAndActivity(bool enableExtendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + { + await host.StartAsync(); + + string input = Guid.NewGuid().ToString(); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.AbortSessionOrchestration), input); + var status = await client.WaitForCompletionAsync(StandardTimeout); + + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.IsNotNull(status.Output); + Assert.AreEqual("True", JToken.Parse(status.Output)); + await host.StopAsync(); + } + } + + /// + /// Validates scheduled starts, ensuring they are executed according to defined start date time + /// + /// + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task ScheduledStart_Inline(bool enableExtendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + { + await host.StartAsync(); + + var expectedStartTime = DateTime.UtcNow.AddSeconds(30); + var clientStartingIn30Seconds = await host.StartOrchestrationAsync(typeof(Orchestrations.CurrentTimeInline), "Current Time!", startAt: expectedStartTime); + var clientStartingNow = await host.StartOrchestrationAsync(typeof(Orchestrations.CurrentTimeInline), "Current Time!"); + + var statusStartingNow = clientStartingNow.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + var statusStartingIn30Seconds = clientStartingIn30Seconds.WaitForCompletionAsync(TimeSpan.FromSeconds(60)); + + await Task.WhenAll(statusStartingNow, statusStartingIn30Seconds); + + Assert.AreEqual(OrchestrationStatus.Completed, statusStartingNow.Result?.OrchestrationStatus); + Assert.AreEqual("Current Time!", JToken.Parse(statusStartingNow.Result?.Input)); + Assert.IsNull(statusStartingNow.Result.ScheduledStartTime); + + Assert.AreEqual(OrchestrationStatus.Completed, statusStartingIn30Seconds.Result?.OrchestrationStatus); + Assert.AreEqual("Current Time!", JToken.Parse(statusStartingIn30Seconds.Result?.Input)); + Assert.AreEqual(expectedStartTime, statusStartingIn30Seconds.Result.ScheduledStartTime); + + var startNowResult = (DateTime)JToken.Parse(statusStartingNow.Result?.Output); + var startIn30SecondsResult = (DateTime)JToken.Parse(statusStartingIn30Seconds.Result?.Output); + + Assert.IsTrue(startIn30SecondsResult > startNowResult); + Assert.IsTrue(startIn30SecondsResult >= expectedStartTime); + + await host.StopAsync(); + } + } + + /// + /// Validates scheduled starts, ensuring they are executed according to defined start date time + /// + /// + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task ScheduledStart_Activity(bool enableExtendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + { + await host.StartAsync(); + + var expectedStartTime = DateTime.UtcNow.AddSeconds(30); + var clientStartingIn30Seconds = await host.StartOrchestrationAsync(typeof(Orchestrations.CurrentTimeActivity), "Current Time!", startAt: expectedStartTime); + var clientStartingNow = await host.StartOrchestrationAsync(typeof(Orchestrations.CurrentTimeActivity), "Current Time!"); + + var statusStartingNow = clientStartingNow.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); + var statusStartingIn30Seconds = clientStartingIn30Seconds.WaitForCompletionAsync(TimeSpan.FromSeconds(60)); + + await Task.WhenAll(statusStartingNow, statusStartingIn30Seconds); + + Assert.AreEqual(OrchestrationStatus.Completed, statusStartingNow.Result?.OrchestrationStatus); + Assert.AreEqual("Current Time!", JToken.Parse(statusStartingNow.Result?.Input)); + Assert.IsNull(statusStartingNow.Result.ScheduledStartTime); + + Assert.AreEqual(OrchestrationStatus.Completed, statusStartingIn30Seconds.Result?.OrchestrationStatus); + Assert.AreEqual("Current Time!", JToken.Parse(statusStartingIn30Seconds.Result?.Input)); + Assert.AreEqual(expectedStartTime, statusStartingIn30Seconds.Result.ScheduledStartTime); + + var startNowResult = (DateTime)JToken.Parse(statusStartingNow.Result?.Output); + var startIn30SecondsResult = (DateTime)JToken.Parse(statusStartingIn30Seconds.Result?.Output); + + Assert.IsTrue(startIn30SecondsResult > startNowResult); + Assert.IsTrue(startIn30SecondsResult >= expectedStartTime); + + await host.StopAsync(); + } + } + + /// + /// Validates scheduled starts, ensuring they are executed according to defined start date time + /// + /// + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task ScheduledStart_Activity_GetStatus_Returns_ScheduledStart(bool enableExtendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions)) + { + await host.StartAsync(); + + var expectedStartTime = DateTime.UtcNow.AddSeconds(30); + var clientStartingIn30Seconds = await host.StartOrchestrationAsync(typeof(Orchestrations.DelayedCurrentTimeActivity), "Delayed Current Time!", startAt: expectedStartTime); + var clientStartingNow = await host.StartOrchestrationAsync(typeof(Orchestrations.DelayedCurrentTimeActivity), "Delayed Current Time!"); + + var statusStartingIn30Seconds = await clientStartingIn30Seconds.GetStatusAsync(); + Assert.IsNotNull(statusStartingIn30Seconds.ScheduledStartTime); + Assert.AreEqual(expectedStartTime, statusStartingIn30Seconds.ScheduledStartTime); + + var statusStartingNow = await clientStartingNow.GetStatusAsync(); + Assert.IsNull(statusStartingNow.ScheduledStartTime); + + await Task.WhenAll( + clientStartingNow.WaitForCompletionAsync(TimeSpan.FromSeconds(35)), + clientStartingIn30Seconds.WaitForCompletionAsync(TimeSpan.FromSeconds(65)) + ); + + await host.StopAsync(); + } + } + + static class Orchestrations + { + internal class SayHelloInline : TaskOrchestration + { + public override Task RunTask(OrchestrationContext context, string input) + { + return Task.FromResult($"Hello, {input}!"); + } + } + + [KnownType(typeof(Activities.Hello))] + internal class SayHelloWithActivity : TaskOrchestration + { + public override Task RunTask(OrchestrationContext context, string input) + { + return context.ScheduleTask(typeof(Activities.Hello), input); + } + } + + [KnownType(typeof(Activities.HelloFailActivity))] + internal class SayHelloWithActivityFail : TaskOrchestration + { + public override Task RunTask(OrchestrationContext context, string input) + { + return context.ScheduleTask(typeof(Activities.HelloFailActivity), input); + } + } + + [KnownType(typeof(Activities.Multiply))] + internal class Factorial : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext context, int n) + { + long result = 1; + for (int i = 1; i <= n; i++) + { + result = await (context.ScheduleTask(typeof(Activities.Multiply), new[] { result, i })); + } + return result; + } + } + + [KnownType(typeof(Activities.Multiply))] + internal class FactorialFail : TaskOrchestration + { + public static bool ShouldFail = true; + public override async Task RunTask(OrchestrationContext context, int n) + { + long result = 1; + for (int i = 1; i <= n; i++) + { + result = await (context.ScheduleTask(typeof(Activities.Multiply), new[] { result, i })); + } + if (ShouldFail) + { + throw new Exception("Simulating a transient, unhandled exception"); + } + return result; + } + } + + [KnownType(typeof(Activities.Multiply))] + internal class FactorialOrchestratorFail : TaskOrchestration + { + public static bool ShouldFail = true; + public override async Task RunTask(OrchestrationContext context, int n) + { + long result = 1; + for (int i = 1; i <= n; i++) + { + result = await (context.ScheduleTask(typeof(Activities.Multiply), new[] { result, i })); + } + if (ShouldFail) + { + throw new Exception("Simulating a transient, unhandled exception"); + } + return result; + } + } + + [KnownType(typeof(Activities.MultiplyMultipleActivityFail))] + internal class FactorialMultipleActivityFail : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext context, int n) + { + long result = 1; + for (int i = 1; i <= n; i++) + { + result = await (context.ScheduleTask(typeof(Activities.MultiplyMultipleActivityFail), new[] { result, i })); + } + + return result; + } + } + + [KnownType(typeof(Activities.Multiply))] + internal class FactorialNoReplay : Factorial + { + public override Task RunTask(OrchestrationContext context, int n) + { + if (context.IsReplaying) + { + throw new Exception("Replaying is forbidden in this test."); + } + + return base.RunTask(context, n); + } + } + + internal class LongRunningOrchestrator : TaskOrchestration + { + public override Task RunTask(OrchestrationContext context, string input) + { + Thread.Sleep(TimeSpan.FromSeconds(10)); + if (input == "0") + { + context.ContinueAsNew("1"); + return Task.FromResult(""); + } + else + { + return Task.FromResult("ok"); + } + } + } + + [KnownType(typeof(Activities.GetFileList))] + [KnownType(typeof(Activities.GetFileSize))] + internal class DiskUsage : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext context, string directory) + { + string[] files = await context.ScheduleTask(typeof(Activities.GetFileList), directory); + + var tasks = new Task[files.Length]; + for (int i = 0; i < files.Length; i++) + { + tasks[i] = context.ScheduleTask(typeof(Activities.GetFileSize), files[i]); + } + + await Task.WhenAll(tasks); + + long totalBytes = tasks.Sum(t => t.Result); + return totalBytes; + } + } + + [KnownType(typeof(Activities.Hello))] + internal class FanOutFanIn : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext context, int parallelTasks) + { + var tasks = new Task[parallelTasks]; + for (int i = 0; i < tasks.Length; i++) + { + tasks[i] = context.ScheduleTask(typeof(Activities.Hello), i.ToString("000")); + } + + await Task.WhenAll(tasks); + + return "Done"; + } + } + + [KnownType(typeof(Activities.HelloFailFanOut))] + internal class FanOutFanInRewind : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext context, int parallelTasks) + { + var tasks = new Task[parallelTasks]; + for (int i = 0; i < tasks.Length; i++) + { + tasks[i] = context.ScheduleTask(typeof(Activities.HelloFailFanOut), i.ToString("000")); + } + + await Task.WhenAll(tasks); + + return "Done"; + } + } + + [KnownType(typeof(Activities.Echo))] + internal class SemiLargePayloadFanOutFanIn : TaskOrchestration + { + static readonly string Some50KBPayload = new string('x', 25 * 1024); // Assumes UTF-16 encoding + static readonly string Some16KBPayload = new string('x', 8 * 1024); // Assumes UTF-16 encoding + + public override async Task RunTask(OrchestrationContext context, int parallelTasks) + { + var tasks = new Task[parallelTasks]; + for (int i = 0; i < tasks.Length; i++) + { + tasks[i] = context.ScheduleTask(typeof(Activities.Echo), Some50KBPayload); + } + + await Task.WhenAll(tasks); + + return "Done"; + } + + public override string GetStatus() + { + return Some16KBPayload; + } + } + + [KnownType(typeof(Orchestrations.ParentWorkflowSubOrchestrationFail))] + [KnownType(typeof(Activities.Hello))] + public class ChildWorkflowSubOrchestrationFail : TaskOrchestration + { + public static bool ShouldFail1 = true; + public static bool ShouldFail2 = true; + public override async Task RunTask(OrchestrationContext context, int input) + { + if (ShouldFail1 || ShouldFail2) + { + throw new Exception("Simulating sub-orchestration failure..."); + } + var result = await context.ScheduleTask(typeof(Activities.Hello), input); + return result; + } + } + + [KnownType(typeof(Orchestrations.ParentWorkflowSubOrchestrationActivityFail))] + [KnownType(typeof(Activities.HelloFailSubOrchestrationActivity))] + public class ChildWorkflowSubOrchestrationActivityFail : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext context, int input) + { + var result = await context.ScheduleTask(typeof(Activities.HelloFailSubOrchestrationActivity), input); + return result; + } + } + + [KnownType(typeof(Orchestrations.GrandparentWorkflowNestedActivityFail))] + [KnownType(typeof(Orchestrations.ParentWorkflowNestedActivityFail))] + [KnownType(typeof(Activities.HelloFailNestedSuborchestration))] + public class ChildWorkflowNestedActivityFail : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext context, int input) + { + var result = await context.ScheduleTask(typeof(Activities.HelloFailNestedSuborchestration), input); + return result; + } + } + + [KnownType(typeof(Orchestrations.ChildWorkflowSubOrchestrationFail))] + [KnownType(typeof(Activities.Hello))] + public class ParentWorkflowSubOrchestrationFail : TaskOrchestration + { + public static string Result; + public override async Task RunTask(OrchestrationContext context, bool waitForCompletion) + { + var results = new Task[2]; + for (int i = 0; i < 2; i++) + { + Task r = context.CreateSubOrchestrationInstance(typeof(Orchestrations.ChildWorkflowSubOrchestrationFail), i); + if (waitForCompletion) + { + await r; + } + results[i] = r; + } + + string[] data = await Task.WhenAll(results); + Result = string.Concat(data); + return Result; + } + } + + + [KnownType(typeof(Orchestrations.GrandparentWorkflowNestedActivityFail))] + [KnownType(typeof(Orchestrations.ChildWorkflowNestedActivityFail))] + [KnownType(typeof(Activities.HelloFailNestedSuborchestration))] + public class ParentWorkflowNestedActivityFail : TaskOrchestration + { + public static string Result; + public override async Task RunTask(OrchestrationContext context, bool waitForCompletion) + { + var results = new Task[2]; + for (int i = 0; i < 2; i++) + { + Task r = context.CreateSubOrchestrationInstance(typeof(Orchestrations.ChildWorkflowNestedActivityFail), i); + if (waitForCompletion) + { + await r; + } + results[i] = r; + } + + string[] data = await Task.WhenAll(results); + Result = string.Concat(data); + return Result; + } + } + + [KnownType(typeof(Orchestrations.ChildWorkflowSubOrchestrationActivityFail))] + [KnownType(typeof(Activities.HelloFailSubOrchestrationActivity))] + public class ParentWorkflowSubOrchestrationActivityFail : TaskOrchestration + { + public static string Result; + public override async Task RunTask(OrchestrationContext context, bool waitForCompletion) + { + var results = new Task[2]; + for (int i = 0; i < 2; i++) + { + Task r = context.CreateSubOrchestrationInstance(typeof(Orchestrations.ChildWorkflowSubOrchestrationActivityFail), i); + if (waitForCompletion) + { + await r; + } + results[i] = r; + } + + string[] data = await Task.WhenAll(results); + Result = string.Concat(data); + return Result; + } + } + + [KnownType(typeof(Orchestrations.ParentWorkflowNestedActivityFail))] + [KnownType(typeof(Orchestrations.ChildWorkflowNestedActivityFail))] + [KnownType(typeof(Activities.HelloFailNestedSuborchestration))] + public class GrandparentWorkflowNestedActivityFail : TaskOrchestration + { + public static string Result; + public override async Task RunTask(OrchestrationContext context, bool waitForCompletion) + { + var results = new Task[2]; + for (int i = 0; i < 2; i++) + { + Task r = context.CreateSubOrchestrationInstance(typeof(Orchestrations.ParentWorkflowNestedActivityFail), i); + if (waitForCompletion) + { + await r; + } + results[i] = r; + } + + string[] data = await Task.WhenAll(results); + Result = string.Concat(data); + return Result; + } + } + + internal class Counter : TaskOrchestration + { + TaskCompletionSource waitForOperationHandle; + + public override async Task RunTask(OrchestrationContext context, int currentValue) + { + string operation = await this.WaitForOperation(); + + bool done = false; + switch (operation?.ToLowerInvariant()) + { + case "incr": + currentValue++; + break; + case "decr": + currentValue--; + break; + case "end": + done = true; + break; + } + + if (!done) + { + context.ContinueAsNew(currentValue); + } + + return currentValue; - [KnownType(typeof(Activities.Hello))] - internal class SayHelloWithActivity : TaskOrchestration - { - public override Task RunTask(OrchestrationContext context, string input) - { - return context.ScheduleTask(typeof(Activities.Hello), input); } - } - [KnownType(typeof(Activities.HelloFailActivity))] - internal class SayHelloWithActivityFail : TaskOrchestration - { - public override Task RunTask(OrchestrationContext context, string input) + async Task WaitForOperation() { - return context.ScheduleTask(typeof(Activities.HelloFailActivity), input); + this.waitForOperationHandle = new TaskCompletionSource(); + string operation = await this.waitForOperationHandle.Task; + this.waitForOperationHandle = null; + return operation; } - } - [KnownType(typeof(Activities.Multiply))] - internal class Factorial : TaskOrchestration - { - public override async Task RunTask(OrchestrationContext context, int n) + public override void OnEvent(OrchestrationContext context, string name, string input) { - long result = 1; - for (int i = 1; i <= n; i++) + Assert.AreEqual("operation", name, true, "Unknown signal recieved..."); + if (this.waitForOperationHandle != null) { - result = await (context.ScheduleTask(typeof(Activities.Multiply), new[] { result, i })); + this.waitForOperationHandle.SetResult(input); } - return result; } - } + } - [KnownType(typeof(Activities.Multiply))] - internal class FactorialFail : TaskOrchestration + [KnownType(typeof(Entities.Counter))] + public sealed class CallCounterEntity : TaskOrchestration { - public static bool ShouldFail = true; - public override async Task RunTask(OrchestrationContext context, int n) + public override async Task RunTask(OrchestrationContext context, EntityId entityId) { - long result = 1; - for (int i = 1; i <= n; i++) + await context.CallEntityAsync(entityId, "set", 33); + int result = await context.CallEntityAsync(entityId, "get"); + + if (result == 33) { - result = await (context.ScheduleTask(typeof(Activities.Multiply), new[] { result, i })); + return "OK"; } - if (ShouldFail) + else { - throw new Exception("Simulating a transient, unhandled exception"); + return $"wrong result: {result} instead of 33"; } - return result; } } - [KnownType(typeof(Activities.Multiply))] - internal class FactorialOrchestratorFail : TaskOrchestration + [KnownType(typeof(Entities.Counter))] + [KnownType(typeof(Entities.Relay))] + public sealed class PollCounterEntity : TaskOrchestration { - public static bool ShouldFail = true; - public override async Task RunTask(OrchestrationContext context, int n) + public override async Task RunTask(OrchestrationContext ctx, EntityId entityId) { - long result = 1; - for (int i = 1; i <= n; i++) - { - result = await (context.ScheduleTask(typeof(Activities.Multiply), new[] { result, i })); - } - if (ShouldFail) + while (true) { - throw new Exception("Simulating a transient, unhandled exception"); + var result = await ctx.CallEntityAsync(entityId, "get"); + + if (result != 0) + { + if (result == 1) + { + return "ok"; + } + else + { + return $"fail: wrong entity state: expected 1, got {result}"; + } + } + + await ctx.CreateTimer(DateTime.UtcNow + TimeSpan.FromSeconds(1), CancellationToken.None); } - return result; } } - [KnownType(typeof(Activities.MultiplyMultipleActivityFail))] - internal class FactorialMultipleActivityFail : TaskOrchestration + [KnownType(typeof(Entities.Counter))] + public sealed class CreateEmptyEntities : TaskOrchestration { - public override async Task RunTask(OrchestrationContext context, int n) + public override async Task RunTask(OrchestrationContext context, EntityId[] entityIds) { - long result = 1; - for (int i = 1; i <= n; i++) + var tasks = new List(); + for (int i = 0; i < entityIds.Length; i++) { - result = await (context.ScheduleTask(typeof(Activities.MultiplyMultipleActivityFail), new[] { result, i })); + tasks.Add(context.CallEntityAsync(entityIds[i], "delete")); } - return result; + await Task.WhenAll(tasks); + return "ok"; } } - [KnownType(typeof(Activities.Multiply))] - internal class FactorialNoReplay : Factorial + [KnownType(typeof(Entities.Counter))] + [KnownType(typeof(Activities.Hello))] + public sealed class LockThenFailReplay : TaskOrchestration { - public override Task RunTask(OrchestrationContext context, int n) + public override async Task RunTask(OrchestrationContext context, (EntityId entityId, bool createNondeterminismFailure) input) { - if (context.IsReplaying) + if (!(input.createNondeterminismFailure && context.IsReplaying)) { - throw new Exception("Replaying is forbidden in this test."); + await context.LockEntitiesAsync(input.entityId); + + await context.ScheduleTask(typeof(Activities.Hello), "Tokyo"); } - return base.RunTask(context, n); + return "ok"; } } - internal class LongRunningOrchestrator : TaskOrchestration + [KnownType(typeof(Entities.Counter))] + public sealed class LockedTransfer : TaskOrchestration<(int, int), (EntityId, EntityId)> { - public override Task RunTask(OrchestrationContext context, string input) + public override async Task<(int, int)> RunTask(OrchestrationContext ctx, (EntityId, EntityId) input) { - Thread.Sleep(TimeSpan.FromSeconds(10)); - if (input == "0") + var (from, to) = input; + + if (from.Equals(to)) { - context.ContinueAsNew("1"); - return Task.FromResult(""); + throw new ArgumentException("from and to must be distinct"); } - else + + if (ctx.IsInsideCriticalSection) { - return Task.FromResult("ok"); + throw new Exception("test failed: lock context is incorrect"); } - } - } - [KnownType(typeof(Activities.GetFileList))] - [KnownType(typeof(Activities.GetFileSize))] - internal class DiskUsage : TaskOrchestration - { - public override async Task RunTask(OrchestrationContext context, string directory) - { - string[] files = await context.ScheduleTask(typeof(Activities.GetFileList), directory); + int fromBalance; + int toBalance; - var tasks = new Task[files.Length]; - for (int i = 0; i < files.Length; i++) + using (await ctx.LockEntitiesAsync(from, to)) { - tasks[i] = context.ScheduleTask(typeof(Activities.GetFileSize), files[i]); - } + if (!ctx.IsInsideCriticalSection) + { + throw new Exception("test failed: lock context is incorrect, must be in critical section"); + } - await Task.WhenAll(tasks); + var availableEntities = ctx.GetAvailableEntities().ToList(); - long totalBytes = tasks.Sum(t => t.Result); - return totalBytes; - } - } + if (availableEntities.Count != 2 || !availableEntities.Contains(from) || !availableEntities.Contains(to)) + { + throw new Exception("test failed: lock context is incorrect: available locks do not match"); + } - [KnownType(typeof(Activities.Hello))] - internal class FanOutFanIn : TaskOrchestration - { - public override async Task RunTask(OrchestrationContext context, int parallelTasks) - { - var tasks = new Task[parallelTasks]; - for (int i = 0; i < tasks.Length; i++) - { - tasks[i] = context.ScheduleTask(typeof(Activities.Hello), i.ToString("000")); + // read balances in parallel + var t1 = ctx.CallEntityAsync(from, "get"); + + availableEntities = ctx.GetAvailableEntities().ToList(); + if (availableEntities.Count != 1 || !availableEntities.Contains(to)) + { + throw new Exception("test failed: lock context is incorrect: available locks wrong"); + } + + var t2 = ctx.CallEntityAsync(to, "get"); + + availableEntities = ctx.GetAvailableEntities().ToList(); + if (availableEntities.Count != 0) + { + throw new Exception("test failed: lock context is incorrect: available locks wrong"); + } + + fromBalance = await t1; + toBalance = await t2; + + availableEntities = ctx.GetAvailableEntities().ToList(); + if (availableEntities.Count != 2 || !availableEntities.Contains(from) || !availableEntities.Contains(to)) + { + throw new Exception("test failed: lock context is incorrect: available locks do not match"); + } + + // modify + fromBalance--; + toBalance++; + + // write balances in parallel + var t3 = ctx.CallEntityAsync(from, "set", fromBalance); + var t4 = ctx.CallEntityAsync(to, "set", toBalance); + await t3; + await t4; } - await Task.WhenAll(tasks); + if (ctx.IsInsideCriticalSection) + { + throw new Exception("test failed: lock context is incorrect"); + } - return "Done"; + return (fromBalance, toBalance); } } - [KnownType(typeof(Activities.HelloFailFanOut))] - internal class FanOutFanInRewind : TaskOrchestration + + [KnownType(typeof(Entities.StringStore))] + [KnownType(typeof(Entities.StringStore2))] + public sealed class SignalAndCallStringStore : TaskOrchestration { - public override async Task RunTask(OrchestrationContext context, int parallelTasks) + public override async Task RunTask(OrchestrationContext ctx, EntityId entity) { - var tasks = new Task[parallelTasks]; - for (int i = 0; i < tasks.Length; i++) + ctx.SignalEntity(entity, "set", "333"); + + var result = await ctx.CallEntityAsync(entity, "get"); + + if (result != "333") { - tasks[i] = context.ScheduleTask(typeof(Activities.HelloFailFanOut), i.ToString("000")); + return $"fail: wrong entity state: expected 333, got {result}"; } - await Task.WhenAll(tasks); - - return "Done"; + return "ok"; } } - [KnownType(typeof(Activities.Echo))] - internal class SemiLargePayloadFanOutFanIn : TaskOrchestration + [KnownType(typeof(Entities.StringStore))] + [KnownType(typeof(Entities.StringStore2))] + public sealed class CallAndDeleteStringStore : TaskOrchestration { - static readonly string Some50KBPayload = new string('x', 25 * 1024); // Assumes UTF-16 encoding - static readonly string Some16KBPayload = new string('x', 8 * 1024); // Assumes UTF-16 encoding - - public override async Task RunTask(OrchestrationContext context, int parallelTasks) + public override async Task RunTask(OrchestrationContext ctx, EntityId entity) { - var tasks = new Task[parallelTasks]; - for (int i = 0; i < tasks.Length; i++) - { - tasks[i] = context.ScheduleTask(typeof(Activities.Echo), Some50KBPayload); - } + await ctx.CallEntityAsync(entity, "set", "333"); - await Task.WhenAll(tasks); - - return "Done"; - } + await ctx.CallEntityAsync(entity, "delete"); - public override string GetStatus() - { - return Some16KBPayload; + return "ok"; } } - [KnownType(typeof(Orchestrations.ParentWorkflowSubOrchestrationFail))] - [KnownType(typeof(Activities.Hello))] - public class ChildWorkflowSubOrchestrationFail : TaskOrchestration + [KnownType(typeof(Orchestrations.DelayedSignal))] + [KnownType(typeof(Entities.Launcher))] + public sealed class LaunchOrchestrationFromEntity : TaskOrchestration { - public static bool ShouldFail1 = true; - public static bool ShouldFail2 = true; - public override async Task RunTask(OrchestrationContext context, int input) + public override async Task RunTask(OrchestrationContext ctx, EntityId entityId) { - if (ShouldFail1 || ShouldFail2) + await ctx.CallEntityAsync(entityId, "launch", "hello"); + + while (true) { - throw new Exception("Simulating sub-orchestration failure..."); + var orchestrationId = await ctx.CallEntityAsync(entityId, "get"); + + if (orchestrationId != null) + { + return orchestrationId; + } + + await ctx.CreateTimer(DateTime.UtcNow + TimeSpan.FromSeconds(1), CancellationToken.None); } - var result = await context.ScheduleTask(typeof(Activities.Hello), input); - return result; } } - [KnownType(typeof(Orchestrations.ParentWorkflowSubOrchestrationActivityFail))] - [KnownType(typeof(Activities.HelloFailSubOrchestrationActivity))] - public class ChildWorkflowSubOrchestrationActivityFail : TaskOrchestration + public sealed class DelayedSignal : TaskOrchestration { - public override async Task RunTask(OrchestrationContext context, int input) + public override async Task RunTask(OrchestrationContext ctx, EntityId entityId) { - var result = await context.ScheduleTask(typeof(Activities.HelloFailSubOrchestrationActivity), input); - return result; - } - } + await ctx.CreateTimer(ctx.CurrentUtcDateTime + TimeSpan.FromSeconds(.2), CancellationToken.None); - [KnownType(typeof(Orchestrations.GrandparentWorkflowNestedActivityFail))] - [KnownType(typeof(Orchestrations.ParentWorkflowNestedActivityFail))] - [KnownType(typeof(Activities.HelloFailNestedSuborchestration))] - public class ChildWorkflowNestedActivityFail : TaskOrchestration - { - public override async Task RunTask(OrchestrationContext context, int input) - { - var result = await context.ScheduleTask(typeof(Activities.HelloFailNestedSuborchestration), input); - return result; + ctx.SignalEntity(entityId, "done"); + + return ""; } } - [KnownType(typeof(Orchestrations.ChildWorkflowSubOrchestrationFail))] - [KnownType(typeof(Activities.Hello))] - public class ParentWorkflowSubOrchestrationFail : TaskOrchestration - { - public static string Result; - public override async Task RunTask(OrchestrationContext context, bool waitForCompletion) + [KnownType(typeof(Entities.FaultyEntityWithRollback))] + [KnownType(typeof(Entities.FaultyEntityWithoutRollback))] + public sealed class CallFaultyEntity : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext ctx, (EntityId, bool, ErrorPropagationMode) input) { - var results = new Task[2]; - for (int i = 0; i < 2; i++) + (EntityId entityId, bool rollbackOnException, ErrorPropagationMode mode) = input; + + async Task ExpectException(Task t) { - Task r = context.CreateSubOrchestrationInstance(typeof(Orchestrations.ChildWorkflowSubOrchestrationFail), i); - if (waitForCompletion) + try { - await r; + await t; + Assert.IsTrue(false, "expected exception"); + } + catch (OperationFailedException e) + { + if (mode == ErrorPropagationMode.SerializeExceptions) + { + Assert.IsTrue(e.InnerException != null && e.InnerException is TInner); + Assert.IsTrue(e.FailureDetails == null); + } + else // check that we received the failure details instead of the exception + { + Assert.IsTrue(e.InnerException == null); + Assert.IsTrue(e.FailureDetails != null); + Assert.IsTrue(e.FailureDetails.ErrorType == typeof(TInner).FullName); + } + } + catch (Exception e) + { + Assert.IsTrue(false, $"wrong exception: {e}"); + } + } + + Task Get() => ctx.CallEntityAsync(entityId, "Get"); + Task Set(int val) => ctx.CallEntityAsync(entityId, "Set", val); + Task SetThenThrow(int val) => ctx.CallEntityAsync(entityId, "SetThenThrow", val); + Task SetToUnserializable() => ctx.CallEntityAsync(entityId, "SetToUnserializable"); + Task SetToUnDeserializable() => ctx.CallEntityAsync(entityId, "SetToUnDeserializable"); + Task DeleteThenThrow () => ctx.CallEntityAsync(entityId, "DeleteThenThrow"); + Task Delete() => ctx.CallEntityAsync(entityId, "Delete"); + + + async Task TestRollbackToNonexistent() + { + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); + await ExpectException(SetToUnserializable()); + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); + } + + async Task TestNondeserializableState() + { + await SetToUnDeserializable(); + Assert.IsTrue(await ctx.CallEntityAsync(entityId, "exists")); + await ExpectException(Get()); + await ctx.CallEntityAsync(entityId, "deletewithoutreading"); + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); + } + + async Task TestSecondOperationRollback() + { + await Set(3); + Assert.AreEqual(3, await Get()); + await ExpectException(SetThenThrow(333)); + if (rollbackOnException) + { + Assert.AreEqual(3, await Get()); + } + else + { + Assert.AreEqual(333, await Get()); + } + await Delete(); + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); + } + + async Task TestDeleteOperationRollback() + { + await Set(3); + Assert.AreEqual(3, await Get()); + await ExpectException(SetThenThrow(333)); + if (rollbackOnException) + { + Assert.AreEqual(3, await Get()); + } + else + { + Assert.AreEqual(333, await Get()); + } + await ExpectException(DeleteThenThrow()); + if (rollbackOnException) + { + Assert.AreEqual(3, await Get()); + } + else + { + Assert.AreEqual(0, await Get()); + } + await Delete(); + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); + } + + async Task TestFirstOperationRollback() + { + await ExpectException(SetThenThrow(333)); + + if (!rollbackOnException) + { + Assert.IsTrue(await ctx.CallEntityAsync(entityId, "exists")); + await Delete(); } - results[i] = r; + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); } - string[] data = await Task.WhenAll(results); - Result = string.Concat(data); - return Result; - } - } + // we use this utility function to try to enforce that a bunch of signals is delivered as a single batch. + // This is required for some of the tests here to work, since the batching affects the entity state management. + // The "enforcement" mechanism we use is not 100% failsafe (it still makes timing assumptions about the provider) + // but it should be more reliable than the original version of this test which failed quite frequently, as it was + // simply assuming that signals that are sent at the same time are always processed as a batch. + async Task ProcessAllSignalsInSingleBatch(Action sendSignals) + { + // first issue a signal that, when delivered, keeps the entity busy for a second + ctx.SignalEntity(entityId, "delay", 1); + // we now need to yield briefly so that the delay signal is sent before the others + await ctx.CreateTimer(ctx.CurrentUtcDateTime + TimeSpan.FromMilliseconds(1), CancellationToken.None); - [KnownType(typeof(Orchestrations.GrandparentWorkflowNestedActivityFail))] - [KnownType(typeof(Orchestrations.ChildWorkflowNestedActivityFail))] - [KnownType(typeof(Activities.HelloFailNestedSuborchestration))] - public class ParentWorkflowNestedActivityFail : TaskOrchestration - { - public static string Result; - public override async Task RunTask(OrchestrationContext context, bool waitForCompletion) - { - var results = new Task[2]; - for (int i = 0; i < 2; i++) + // now send the signals in the batch. These should all arrive and get queued (inside the storage provider) + // while the entity is executing the delay operation. Therefore, after the delay operation finishes, + // all of the signals are processed in a single batch. + sendSignals(); + } + + async Task TestRollbackInBatch1() { - Task r = context.CreateSubOrchestrationInstance(typeof(Orchestrations.ChildWorkflowNestedActivityFail), i); - if (waitForCompletion) + await ProcessAllSignalsInSingleBatch(() => { - await r; + ctx.SignalEntity(entityId, "Set", 42); + ctx.SignalEntity(entityId, "SetThenThrow", 333); + ctx.SignalEntity(entityId, "DeleteThenThrow"); + }); + + if (rollbackOnException) + { + Assert.AreEqual(42, await Get()); + } + else + { + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); + ctx.SignalEntity(entityId, "Set", 42); } - results[i] = r; } - string[] data = await Task.WhenAll(results); - Result = string.Concat(data); - return Result; - } - } - - [KnownType(typeof(Orchestrations.ChildWorkflowSubOrchestrationActivityFail))] - [KnownType(typeof(Activities.HelloFailSubOrchestrationActivity))] - public class ParentWorkflowSubOrchestrationActivityFail : TaskOrchestration - { - public static string Result; - public override async Task RunTask(OrchestrationContext context, bool waitForCompletion) - { - var results = new Task[2]; - for (int i = 0; i < 2; i++) + async Task TestRollbackInBatch2() { - Task r = context.CreateSubOrchestrationInstance(typeof(Orchestrations.ChildWorkflowSubOrchestrationActivityFail), i); - if (waitForCompletion) + await ProcessAllSignalsInSingleBatch(() => { - await r; + ctx.SignalEntity(entityId, "Get"); + ctx.SignalEntity(entityId, "Set", 42); + ctx.SignalEntity(entityId, "Delete"); + ctx.SignalEntity(entityId, "Set", 43); + ctx.SignalEntity(entityId, "DeleteThenThrow"); + }); + + if (rollbackOnException) + { + Assert.AreEqual(43, await Get()); + } + else + { + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); } - results[i] = r; } - string[] data = await Task.WhenAll(results); - Result = string.Concat(data); - return Result; - } - } - - [KnownType(typeof(Orchestrations.ParentWorkflowNestedActivityFail))] - [KnownType(typeof(Orchestrations.ChildWorkflowNestedActivityFail))] - [KnownType(typeof(Activities.HelloFailNestedSuborchestration))] - public class GrandparentWorkflowNestedActivityFail : TaskOrchestration - { - public static string Result; - public override async Task RunTask(OrchestrationContext context, bool waitForCompletion) - { - var results = new Task[2]; - for (int i = 0; i < 2; i++) + async Task TestRollbackInBatch3() { - Task r = context.CreateSubOrchestrationInstance(typeof(Orchestrations.ParentWorkflowNestedActivityFail), i); - if (waitForCompletion) + Task getTask = null; + + await ProcessAllSignalsInSingleBatch(() => { - await r; + ctx.SignalEntity(entityId, "Set", 55); + ctx.SignalEntity(entityId, "SetToUnserializable"); + getTask = ctx.CallEntityAsync(entityId, "Get"); + }); + + if (rollbackOnException) + { + Assert.AreEqual(55, await getTask); + await Delete(); } - results[i] = r; + else + { + await ExpectException(getTask); + await ctx.CallEntityAsync(entityId, "deletewithoutreading"); + } + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); } - string[] data = await Task.WhenAll(results); - Result = string.Concat(data); - return Result; - } - } + async Task TestRollbackInBatch4() + { + Task getTask = null; + + await ProcessAllSignalsInSingleBatch(() => + { + ctx.SignalEntity(entityId, "Set", 56); + ctx.SignalEntity(entityId, "SetToUnDeserializable"); + ctx.SignalEntity(entityId, "SetThenThrow", 999); + getTask = ctx.CallEntityAsync(entityId, "Get"); + }); - internal class Counter : TaskOrchestration - { - TaskCompletionSource waitForOperationHandle; + if (rollbackOnException) + { + // we rolled back to an un-deserializable state + await ExpectException(getTask); + } + else + { + // we don't roll back the 999 when throwing + Assert.AreEqual(999, await getTask); + } - public override async Task RunTask(OrchestrationContext context, int currentValue) - { - string operation = await this.WaitForOperation(); + await ctx.CallEntityAsync(entityId, "deletewithoutreading"); + } - bool done = false; - switch (operation?.ToLowerInvariant()) - { - case "incr": - currentValue++; - break; - case "decr": - currentValue--; - break; - case "end": - done = true; - break; + + async Task TestRollbackInBatch5() + { + await ProcessAllSignalsInSingleBatch(() => + { + ctx.SignalEntity(entityId, "Set", 1); + ctx.SignalEntity(entityId, "Delete"); + ctx.SignalEntity(entityId, "Set", 2); + ctx.SignalEntity(entityId, "Delete"); + ctx.SignalEntity(entityId, "SetThenThrow", 3); + }); + + if (rollbackOnException) + { + Assert.IsFalse(await ctx.CallEntityAsync(entityId, "exists")); + } + else + { + Assert.AreEqual(3, await Get()); // not rolled back + } } - if (!done) + try { - context.ContinueAsNew(currentValue); + await TestRollbackToNonexistent(); + await TestNondeserializableState(); + await TestSecondOperationRollback(); + await TestDeleteOperationRollback(); + await TestFirstOperationRollback(); + + await TestRollbackInBatch1(); + await TestRollbackInBatch2(); + await TestRollbackInBatch3(); + await TestRollbackInBatch4(); + await TestRollbackInBatch5(); + + return "ok"; } - - return currentValue; - - } - - async Task WaitForOperation() - { - this.waitForOperationHandle = new TaskCompletionSource(); - string operation = await this.waitForOperationHandle.Task; - this.waitForOperationHandle = null; - return operation; - } - - public override void OnEvent(OrchestrationContext context, string name, string input) - { - Assert.AreEqual("operation", name, true, "Unknown signal recieved..."); - if (this.waitForOperationHandle != null) + catch (Exception e) { - this.waitForOperationHandle.SetResult(input); + return e.ToString(); } } } @@ -3074,6 +4240,344 @@ public override Task RunTask(OrchestrationContext context, string inpu } } + static class Entities + { + internal class Counter : TaskEntity + { + public override int CreateInitialState(EntityContext ctx) + { + return 0; + } + + public override ValueTask ExecuteOperationAsync(EntityContext ctx) + { + switch (ctx.OperationName) + { + case "get": + return new ValueTask(ctx.State); + + case "increment": + ctx.State++; + return new ValueTask(ctx.State); + + case "add": + ctx.State += ctx.GetInput(); + return new ValueTask(ctx.State); + + case "set": + ctx.State = ctx.GetInput(); + return default; + + case "delete": + ctx.DeleteState(); + return default; + + default: + throw new InvalidOperationException($"unknown operation: {ctx.OperationName}"); + } + } + } + + //-------------- An entity that forwards a signal ----------------- + + internal class Relay : TaskEntity + { + public override ValueTask ExecuteOperationAsync(EntityContext context) + { + var (destination, operation) = context.GetInput<(EntityId, string)>(); + + context.SignalEntity(destination, operation); + + return default; + } + } + + // -------------- An entity that records all batch positions and batch sizes ----------------- + internal class BatchEntity : TaskEntity> + { + public override List<(int, int)> CreateInitialState(EntityContext> context) + => new List<(int, int)>(); + + public override ValueTask ExecuteOperationAsync(EntityContext> context) + { + context.State.Add((context.BatchPosition, context.BatchSize)); + return default; + } + } + + //-------------- a very simple entity that stores a string ----------------- + // it offers two operations: + // "set" (takes a string, assigns it to the current state, does not return anything) + // "get" (returns a string containing the current state)// An entity that records all batch positions and batch sizes + internal class StringStore : TaskEntity + { + public override ValueTask ExecuteOperationAsync(EntityContext context) + { + switch (context.OperationName) + { + case "set": + context.State = context.GetInput(); + return default; + + case "get": + return new ValueTask(context.State); + + default: + throw new NotImplementedException("no such operation"); + } + } + } + + //-------------- a slightly less trivial version of the same ----------------- + // as before with two differences: + // - "get" throws an exception if the entity does not already exist, i.e. state was not set to anything + // - a new operation "delete" deletes the entity, i.e. clears all state + internal class StringStore2 : TaskEntity + { + public override ValueTask ExecuteOperationAsync(EntityContext context) + { + switch (context.OperationName) + { + case "delete": + context.DeleteState(); + return default; + + case "set": + context.State = context.GetInput(); + return default; + + case "get": + if (!context.HasState) + { + throw new InvalidOperationException("this entity does not like 'get' when it does not have state yet"); + } + + return new ValueTask(context.State); + + default: + throw new NotImplementedException("no such operation"); + } + } + } + + //-------------- An entity that launches an orchestration ----------------- + + internal class Launcher : TaskEntity + { + public class State + { + public string Id; + public bool Done; + } + + public override State CreateInitialState(EntityContext context) => new State(); + + public override ValueTask ExecuteOperationAsync(EntityContext context) + { + switch (context.OperationName) + { + case "launch": + { + context.State.Id = context.StartNewOrchestration(typeof(Orchestrations.DelayedSignal), input: context.EntityId); + return default; + } + + case "done": + { + context.State.Done = true; + return default; + } + + case "get": + { + return new ValueTask(context.State.Done ? context.State.Id : null); + } + + default: + throw new NotImplementedException("no such entity operation"); + } + } + } + + //-------------- an entity that is designed to throw certain exceptions during operations, serialization, or deserialization ----------------- + + internal class FaultyEntityWithRollback : FaultyEntity + { + public override ValueTask ExecuteOperationAsync(EntityContext context) + { + context.EntityExecutionOptions.RollbackOnExceptions = true; + return base.ExecuteOperationAsync(context); + } + public override FaultyEntity CreateInitialState(EntityContext context) => new FaultyEntityWithRollback(); + } + + internal class FaultyEntityWithoutRollback : FaultyEntity + { + public override ValueTask ExecuteOperationAsync(EntityContext context) + { + context.EntityExecutionOptions.RollbackOnExceptions = false; + return base.ExecuteOperationAsync(context); + } + public override FaultyEntity CreateInitialState(EntityContext context) => new FaultyEntityWithoutRollback(); + } + + [JsonObject(MemberSerialization.OptIn)] + internal abstract class FaultyEntity : TaskEntity + { + [JsonProperty] + public int Value { get; set; } + + [JsonProperty()] + [JsonConverter(typeof(CustomJsonConverter))] + public object ObjectWithFaultySerialization { get; set; } + + [JsonProperty] + public int NumberIncrementsSent { get; set; } + + DataConverter ErrorDataConverter = new JsonDataConverter(new JsonSerializerSettings + { + TypeNameHandling = TypeNameHandling.Objects, + ReferenceLoopHandling = ReferenceLoopHandling.Ignore, + }); + + public override ValueTask ExecuteOperationAsync(EntityContext context) + { + context.EntityExecutionOptions.ErrorDataConverter = this.ErrorDataConverter; + + switch (context.OperationName) + { + case "exists": + return new ValueTask(context.HasState); + + case "deletewithoutreading": + context.DeleteState(); + return default; + + case "Get": + if (!context.HasState) + { + return new ValueTask(0); + } + else + { + return new ValueTask(context.State.Value); + } + + case "GetNumberIncrementsSent": + return new ValueTask(context.State.NumberIncrementsSent); + + case "Set": + context.State.Value = context.GetInput(); + return default; + + case "SetToUnserializable": + context.State.ObjectWithFaultySerialization = new UnserializableKaboom(); + return default; + + case "SetToUnDeserializable": + context.State.ObjectWithFaultySerialization = new UnDeserializableKaboom(); + return default; + + case "SetThenThrow": + context.State.Value = context.GetInput(); + throw new FaultyEntity.SerializableKaboom(); + + case "Send": + Send(); + return default; + + case "SendThenThrow": + Send(); + throw new FaultyEntity.SerializableKaboom(); + + case "SendThenMakeUnserializable": + Send(); + context.State.ObjectWithFaultySerialization = new UnserializableKaboom(); + return default; + + case "Delete": + context.DeleteState(); + return default; + + case "DeleteThenThrow": + context.DeleteState(); + throw new FaultyEntity.SerializableKaboom(); + + case "Throw": + throw new FaultyEntity.SerializableKaboom(); + + case "ThrowUnserializable": + throw new FaultyEntity.UnserializableKaboom(); + + case "ThrowUnDeserializable": + throw new FaultyEntity.UnDeserializableKaboom(); + + default: + throw new NotImplementedException("no such entity operation"); + } + + void Send() + { + var desc = $"{++context.State.NumberIncrementsSent}:{context.State.Value}"; + context.SignalEntity(context.GetInput(), desc); + } + } + + public class CustomJsonConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(SerializableKaboom) + || objectType == typeof(UnserializableKaboom) + || objectType == typeof(UnDeserializableKaboom); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.Null) + { + return null; + } + + var typename = serializer.Deserialize(reader); + + if (typename != nameof(SerializableKaboom)) + { + throw new JsonSerializationException("purposefully designed to not be deserializable"); + } + + return new SerializableKaboom(); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + if (value is UnserializableKaboom) + { + throw new JsonSerializationException("purposefully designed to not be serializable"); + } + + serializer.Serialize(writer, value.GetType().Name); + } + } + + [JsonObject] + public class UnserializableKaboom : Exception + { + } + + [JsonObject] + public class SerializableKaboom : Exception + { + } + + [JsonObject] + public class UnDeserializableKaboom : Exception + { + } + } + } + + static class Activities { internal class HelloFailActivity : TaskActivity diff --git a/test/DurableTask.AzureStorage.Tests/TestHelpers.cs b/test/DurableTask.AzureStorage.Tests/TestHelpers.cs index 37073cbb3..d4e2e8b05 100644 --- a/test/DurableTask.AzureStorage.Tests/TestHelpers.cs +++ b/test/DurableTask.AzureStorage.Tests/TestHelpers.cs @@ -18,6 +18,7 @@ namespace DurableTask.AzureStorage.Tests using System.Diagnostics; using System.Diagnostics.Tracing; using System.Threading.Tasks; + using DurableTask.Core; using DurableTask.Core.Logging; using Microsoft.Extensions.Logging; @@ -28,6 +29,7 @@ public static TestOrchestrationHost GetTestOrchestrationHost( bool enableExtendedSessions, int extendedSessionTimeoutInSeconds = 30, bool fetchLargeMessages = true, + ErrorPropagationMode errorPropagationMode = ErrorPropagationMode.SerializeExceptions, Action? modifySettingsAction = null) { string storageConnectionString = GetTestStorageAccountConnectionString(); @@ -48,7 +50,7 @@ public static TestOrchestrationHost GetTestOrchestrationHost( // Give the caller a chance to make test-specific changes to the settings modifySettingsAction?.Invoke(settings); - return new TestOrchestrationHost(settings); + return new TestOrchestrationHost(settings, errorPropagationMode); } public static string GetTestStorageAccountConnectionString() diff --git a/test/DurableTask.AzureStorage.Tests/TestOrchestrationClient.cs b/test/DurableTask.AzureStorage.Tests/TestOrchestrationClient.cs index 92d018eed..236d0a231 100644 --- a/test/DurableTask.AzureStorage.Tests/TestOrchestrationClient.cs +++ b/test/DurableTask.AzureStorage.Tests/TestOrchestrationClient.cs @@ -45,6 +45,8 @@ public TestOrchestrationClient( public string InstanceId => this.instanceId; + public TaskHubClient InnerClient => client; + public async Task WaitForCompletionAsync(TimeSpan timeout) { timeout = AdjustTimeout(timeout); diff --git a/test/DurableTask.AzureStorage.Tests/TestOrchestrationHost.cs b/test/DurableTask.AzureStorage.Tests/TestOrchestrationHost.cs index b88e0dfcf..ae4ca843d 100644 --- a/test/DurableTask.AzureStorage.Tests/TestOrchestrationHost.cs +++ b/test/DurableTask.AzureStorage.Tests/TestOrchestrationHost.cs @@ -20,6 +20,7 @@ namespace DurableTask.AzureStorage.Tests using System.Runtime.Serialization; using System.Threading.Tasks; using DurableTask.Core; + using DurableTask.Core.Entities; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -32,8 +33,9 @@ internal sealed class TestOrchestrationHost : IDisposable readonly TaskHubClient client; readonly HashSet addedOrchestrationTypes; readonly HashSet addedActivityTypes; + readonly HashSet addedEntityTypes; - public TestOrchestrationHost(AzureStorageOrchestrationServiceSettings settings) + public TestOrchestrationHost(AzureStorageOrchestrationServiceSettings settings, ErrorPropagationMode errorPropagationMode = ErrorPropagationMode.SerializeExceptions) { this.service = new AzureStorageOrchestrationService(settings); this.service.CreateAsync().GetAwaiter().GetResult(); @@ -43,6 +45,9 @@ public TestOrchestrationHost(AzureStorageOrchestrationServiceSettings settings) this.client = new TaskHubClient(service, loggerFactory: settings.LoggerFactory); this.addedOrchestrationTypes = new HashSet(); this.addedActivityTypes = new HashSet(); + this.addedEntityTypes = new HashSet(); + + worker.ErrorPropagationMode = errorPropagationMode; } public string TaskHub => this.settings.TaskHubName; @@ -86,7 +91,7 @@ public async Task StartOrchestrationAsync( this.addedOrchestrationTypes.Add(orchestrationType); } - // Allow orchestration types to declare which activity types they depend on. + // Allow orchestration types to declare which activity and entity types they depend on. // CONSIDER: Make this a supported pattern in DTFx? KnownTypeAttribute[] knownTypes = (KnownTypeAttribute[])orchestrationType.GetCustomAttributes(typeof(KnownTypeAttribute), false); @@ -95,6 +100,8 @@ public async Task StartOrchestrationAsync( { bool orch = referencedKnownType.Type.IsSubclassOf(typeof(TaskOrchestration)); bool activ = referencedKnownType.Type.IsSubclassOf(typeof(TaskActivity)); + bool entit = referencedKnownType.Type.IsSubclassOf(typeof(TaskEntity)); + if (orch && !this.addedOrchestrationTypes.Contains(referencedKnownType.Type)) { this.worker.AddTaskOrchestrations(referencedKnownType.Type); @@ -106,6 +113,12 @@ public async Task StartOrchestrationAsync( this.worker.AddTaskActivities(referencedKnownType.Type); this.addedActivityTypes.Add(referencedKnownType.Type); } + + else if (entit && !this.addedEntityTypes.Contains(referencedKnownType.Type)) + { + this.worker.AddTaskEntities(referencedKnownType.Type); + this.addedEntityTypes.Add(referencedKnownType.Type); + } } DateTime creationTime = DateTime.UtcNow; @@ -139,6 +152,17 @@ public async Task StartOrchestrationAsync( return new TestOrchestrationClient(this.client, orchestrationType, instance.InstanceId, creationTime); } + public Task GetEntityClientAsync(Type entityType, EntityId entityId) + { + if (!this.addedEntityTypes.Contains(entityType)) + { + this.worker.AddTaskEntities(entityType); + this.addedEntityTypes.Add(entityType); + } + + return Task.FromResult(new TestEntityClient(new TaskHubEntityClient(this.client), entityId)); + } + public Task> StartInlineOrchestration( TInput input, string orchestrationName,