build: Add Components functional test scenarios to daily pipelines

tests/functional/SkillFunctionalTests.sln → tests/functional/ComponentsFunctionalTests.sln

@@ -1,9 +1,9 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 16
-VisualStudioVersion = 16.0.29806.167
+# Visual Studio Version 17
+VisualStudioVersion = 17.0.31423.177
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SkillFunctionalTests", "Tests\SkillFunctionalTests\SkillFunctionalTests.csproj", "{DE0231D9-A9BE-4D4C-A830-7ADCF28EA5D6}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ComponentsFunctionalTests", "Tests\ComponentsFunctionalTests\ComponentsFunctionalTests.csproj", "{DE0231D9-A9BE-4D4C-A830-7ADCF28EA5D6}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TranscriptTestRunner", "Libraries\TranscriptTestRunner\TranscriptTestRunner.csproj", "{65C3A2E0-FDC5-4132-9980-3BAE230E9F2E}"
 EndProject

tests/functional/Libraries/TranscriptConverter/README.md

@@ -0,0 +1,18 @@
+# Transcript Converter
+
+## User Step-by-step Guide
+This step-by-step guide shows how to run the TranscriptConverter project to convert a transcript to a test script to be used in TranscriptTestRunner.
+
+## Generate a test script
+1- Create a transcript file, follow [these steps](https://docs.microsoft.com/en-us/azure/bot-service/bot-builder-debug-transcript?view=azure-bot-service-4.0#creatingstoring-a-bot-transcript-file).
+
+2- Build the TranscriptConverter project and navigate to its executable.
+
+3- The command to convert a transcript to a new test script can be executed like this:
+```PS
+btc convert "path-to-source-transcript"
+```
+You can convert a transcript to an existing test script like this:
+```PS
+btc convert "path-to-source-transcript" "path-to-target-test-script"
+```

tests/functional/Libraries/TranscriptConverter/TranscriptConverter.csproj

@@ -8,7 +8,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Bot.Schema" Version="4.13.2" />
+    <PackageReference Include="Microsoft.Bot.Schema" Version="4.14.0" />
     <PackageReference Include="System.CommandLine" Version="2.0.0-beta1.20574.7" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
   </ItemGroup>

tests/functional/Tests/ComponentsFunctionalTests/Common/HostBot.cs

@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+namespace ComponentsFunctionalTests.Common
+{
+    public enum HostBot
+    {
+        /// <summary>
+        /// Empty bot implemented using DotNet and Web App.
+        /// </summary>
+        EmptyBotDotNetWebApp,
+
+        /// <summary>
+        /// Empty bot implemented using JS and Web App.
+        /// </summary>
+        EmptyBotJSWebApp
+    }
+}

tests/functional/Tests/SkillFunctionalTests/Common/TestCase.cs → tests/functional/Tests/ComponentsFunctionalTests/Common/TestCase.cs

@@ -4,7 +4,7 @@
 using Microsoft.Bot.Connector;
 using TranscriptTestRunner;
 
-namespace SkillFunctionalTests.Common
+namespace ComponentsFunctionalTests.Common
 {
     public class TestCase
     {
@@ -16,8 +16,6 @@ public class TestCase
 
         public HostBot HostBot { get; set; }
 
-        public string TargetSkill { get; set; }
-
         public string Script { get; set; }
     }
 }

tests/functional/Tests/SkillFunctionalTests/Common/TestCaseBuilder.cs → tests/functional/Tests/ComponentsFunctionalTests/Common/TestCaseBuilder.cs

@@ -5,11 +5,11 @@
 using System.Collections.Generic;
 using TranscriptTestRunner;
 
-namespace SkillFunctionalTests.Common
+namespace ComponentsFunctionalTests.Common
 {
     public class TestCaseBuilder
     {
-        public IEnumerable<object[]> BuildTestCases(List<string> channelIds, List<string> deliveryModes, List<HostBot> hostBots, List<string> targetSkills, List<string> scripts, Func<TestCase, bool> shouldExclude = null)
+        public IEnumerable<object[]> BuildTestCases(List<string> channelIds, List<string> deliveryModes, List<HostBot> hostBots, List<string> scripts, Func<TestCase, bool> shouldExclude = null)
         {
             var testCases = new List<object[]>();
             var count = 1;
@@ -21,23 +21,19 @@ public IEnumerable<object[]> BuildTestCases(List<string> channelIds, List<string
                     {
                         foreach (var hostBot in hostBots)
                         {
-                            foreach (var targetSkill in targetSkills)
+                            var testCase = new TestCase
                             {
-                                var testCase = new TestCase
-                                {
-                                    Description = $"{script}, {hostBot}, {targetSkill}, {channelId}, {deliveryMode}",
-                                    ChannelId = channelId,
-                                    DeliveryMode = deliveryMode,
-                                    HostBot = hostBot,
-                                    TargetSkill = targetSkill,
-                                    Script = script
-                                };
+                                Description = $"{script}, {hostBot}, {channelId}, {deliveryMode}",
+                                ChannelId = channelId,
+                                DeliveryMode = deliveryMode,
+                                HostBot = hostBot,
+                                Script = script
+                            };
 
-                                if (!ExcludeTestCase(shouldExclude, testCase))
-                                {
-                                    testCases.Add(new object[] { new TestCaseDataObject(testCase) });
-                                    count++;
-                                }
+                            if (!ExcludeTestCase(shouldExclude, testCase))
+                            {
+                                testCases.Add(new object[] { new TestCaseDataObject(testCase) });
+                                count++;
                             }
                         }
                     }

tests/functional/Tests/SkillFunctionalTests/Common/TestCaseDataObject.cs → tests/functional/Tests/ComponentsFunctionalTests/Common/TestCaseDataObject.cs

@@ -5,7 +5,7 @@
 using Newtonsoft.Json;
 using Xunit.Abstractions;
 
-namespace SkillFunctionalTests.Common
+namespace ComponentsFunctionalTests.Common
 {
     public class TestCaseDataObject : IXunitSerializable
     {

tests/functional/Tests/SkillFunctionalTests/MessageWithAttachment/MessageWithAttachmentTests.cs → tests/functional/Tests/ComponentsFunctionalTests/EmptyBot/EmptyBotTests.cs

@@ -4,65 +4,47 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Threading.Tasks;
+using ComponentsFunctionalTests.Common;
 using Microsoft.Bot.Connector;
 using Microsoft.Bot.Schema;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json;
-using SkillFunctionalTests.Common;
 using TranscriptTestRunner;
 using TranscriptTestRunner.XUnit;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace SkillFunctionalTests.MessageWithAttachment
+namespace ComponentsFunctionalTests.EmptyBot
 {
-    [Trait("TestCategory", "Attachments")]
-    public class MessageWithAttachmentTests : ScriptTestBase
+    [Trait("TestCategory", "EmptyBot")]
+    public class EmptyBotTests : ScriptTestBase
     {
-        private readonly string _testScriptsFolder = Directory.GetCurrentDirectory() + @"/MessageWithAttachment/TestScripts";
+        private readonly string _testScriptsFolder = Directory.GetCurrentDirectory() + @"/EmptyBot/TestScripts";
 
-        public MessageWithAttachmentTests(ITestOutputHelper output)
+        public EmptyBotTests(ITestOutputHelper output)
             : base(output)
         {
         }
 
         public static IEnumerable<object[]> TestCases()
         {
             var channelIds = new List<string> { Channels.Directline };
-            var deliverModes = new List<string>
+            var deliveryModes = new List<string>
             {
-                DeliveryModes.Normal,
-                DeliveryModes.ExpectReplies
+                DeliveryModes.Normal
             };
 
             var hostBots = new List<HostBot>
             {
-                HostBot.WaterfallHostBotDotNet,
-                HostBot.WaterfallHostBotJS,
-                HostBot.WaterfallHostBotPython,
-
-                // TODO: Enable this when the port to composer is ready
-                //HostBot.ComposerHostBotDotNet
-            };
-
-            var targetSkills = new List<string>
-            {
-                SkillBotNames.WaterfallSkillBotDotNet,
-                SkillBotNames.WaterfallSkillBotJS,
-                SkillBotNames.WaterfallSkillBotPython,
-
-                // TODO: Enable this when the port to composer is ready
-                //SkillBotNames.ComposerSkillBotDotNet
+                HostBot.EmptyBotDotNetWebApp,
+                HostBot.EmptyBotJSWebApp
             };
 
-            var scripts = new List<string>
-            {
-                "MessageWithAttachment.json",
-            };
+            var scripts = new List<string> { "EmptyBot.json" };
 
             var testCaseBuilder = new TestCaseBuilder();
 
-            var testCases = testCaseBuilder.BuildTestCases(channelIds, deliverModes, hostBots, targetSkills, scripts);
+            var testCases = testCaseBuilder.BuildTestCases(channelIds, deliveryModes, hostBots, scripts);
             foreach (var testCase in testCases)
             {
                 yield return testCase;
@@ -77,12 +59,12 @@ public async Task RunTestCases(TestCaseDataObject testData)
             Logger.LogInformation(JsonConvert.SerializeObject(testCase, Formatting.Indented));
 
             var options = TestClientOptions[testCase.HostBot];
+
             var runner = new XUnitTestRunner(new TestClientFactory(testCase.ChannelId, options, Logger).GetTestClient(), TestRequestTimeout, Logger);
 
             var testParams = new Dictionary<string, string>
             {
-                { "DeliveryMode", testCase.DeliveryMode },
-                { "TargetSkill", testCase.TargetSkill }
+                { "DeliveryMode", testCase.DeliveryMode }
             };
 
             await runner.RunTestAsync(Path.Combine(_testScriptsFolder, testCase.Script), testParams);

tests/functional/Tests/ComponentsFunctionalTests/EmptyBot/TestScripts/EmptyBot.json

@@ -0,0 +1,61 @@
+{
+  "items": [
+    {
+      "type": "conversationUpdate",
+      "role": "user"
+    },
+    {
+      "type": "message",
+      "role": "bot",
+      "text": "Welcome to your bot.",
+      "assertions": [
+        "type == 'message'",
+        "from.role == 'bot'",
+        "recipient.role == 'user'",
+        "text == 'Welcome to your bot.'",
+        "speak == 'Welcome to your bot.'"
+      ]
+    },
+    {
+      "type": "trace",
+      "role": "bot",
+      "assertions": [
+        "type == 'trace'",
+        "from.role == 'bot'",
+        "recipient.role == 'user'",
+        "label == 'Bot State'",
+        "valueType == 'https://www.botframework.com/schemas/botState'",
+        "name == 'BotState'"
+      ]
+    },
+    {
+      "type": "message",
+      "role": "user",
+      "text": "test"
+    },
+    {
+      "type": "message",
+      "role": "bot",
+      "text": "Sorry, I didn't get that.",
+      "assertions": [
+        "type == 'message'",
+        "from.role == 'bot'",
+        "recipient.role == 'user'",
+        "text == 'Sorry, I didn\\'t get that.'",
+        "speak == 'Sorry, I didn\\'t get that.'"
+      ]
+    },
+    {
+      "type": "trace",
+      "role": "bot",
+      "assertions": [
+        "type == 'trace'",
+        "from.role == 'bot'",
+        "recipient.role == 'user'",
+        "label == 'Bot State'",
+        "valueType == 'https://www.botframework.com/schemas/botState'",
+        "name == 'BotState'"
+      ]
+    }
+  ]
+}