From 4df4fd44b5226f751241316ec0d099fc1061a1f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 13 Jan 2026 12:31:00 +0100 Subject: [PATCH 1/4] edits --- docs/Overview.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/Overview.md b/docs/Overview.md index 411f0bb4b4..2375db6639 100644 --- a/docs/Overview.md +++ b/docs/Overview.md @@ -243,7 +243,7 @@ The version is negotiated between the components at the beginning of every workf The server supports processing only a single request at a time. Unless the request is [Cancel](#cancel) or [Abort](#abort) request. -All notifications are sent before as response is sent. +All notifications are sent before a response is sent. #### Message documentation @@ -286,7 +286,7 @@ The version is determined by choosing the highest common supported version. When The receiving side should remember the agreed value, and use it as the highest supported version for any downstream component. In the case above runner should send 6 to testhost, even though the runner supports versions up to 7. -The request was introduced in TestPlatform version `16.0.0`. Runners before this version are not allowed. Testhosts before this version are allowed, the version of testhost if figured out by scanning the assembly, and the request is not sent to them. Version 0 is used for communication. +The request was introduced in TestPlatform version `16.0.0`. Runners before this version are not allowed. Testhosts before this version are allowed, the version of testhost is figured out by scanning the assembly, and the request is not sent to them. Version 0 is used for communication. Versions: @@ -590,7 +590,7 @@ public class DiscoveryCompletePayload // If true TotalCount is also set to -1. public bool IsAborted { get; set; } - // Metrics. + // Telemetry metrics that are reported to client. public IDictionary? Metrics { get; set; } // Sources which were fully discovered. @@ -710,11 +710,12 @@ public class DiscoveryCriteria // Discovered test event will be raised after discovering at minimum this number of tests, - // or when DiscoveredTestEventTimeout is passed. + // or when DiscoveredTestEventTimeout is passed. This helps batching the results, making communication between processes faster. public long FrequencyOfDiscoveredTestsEvent { get; private set; } // Discovered test event will be raised after this much time since last test - // when FrequencyOfDiscoveredTestsEvent is passed. + // when FrequencyOfDiscoveredTestsEvent is passed. This helps sending the batches more frequently, + // when the batch is large, but there is small amount of tests, or the discovery is slow. Giving more frequent feedback to user. public TimeSpan DiscoveredTestEventTimeout { get; private set; } // Run settings for the discovery. From caf374809ca1f808af8b1fe71b8ad0ef52d21ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 13 Jan 2026 16:30:09 +0100 Subject: [PATCH 2/4] edits --- docs/Overview.md | 54 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/docs/Overview.md b/docs/Overview.md index 2375db6639..53d5054608 100644 --- a/docs/Overview.md +++ b/docs/Overview.md @@ -1,10 +1,10 @@ -# Test Platform messaging +# Test Platform -- [Test Platform messaging](#test-platform-messaging) +- [Test Platform](#test-platform) - [What is TestPlatform?](#what-is-testplatform) - [How it works?](#how-it-works) - [Workflows](#workflows) - - [Specification](#specification) + - [Communication Protocol](#communication-protocol) - [Base Protocol](#base-protocol) - [Header Part](#header-part) - [Content Part](#content-part) @@ -49,9 +49,15 @@ - [TestExecution.StatsChange notification (Runner)](#testexecutionstatschange-notification-runner) - [TestExecution.StatsChange notification (Client)](#testexecutionstatschange-notification-client) - [Datacollection](#datacollection) + - [Extensibility](#extensibility) + - [DLL Extension points](#dll-extension-points) + - [Test Adapter](#test-adapter) + - [Test Logger](#test-logger) + - [Runtime Provider](#runtime-provider) + - [TranslationLayer extension points](#translationlayer-extension-points) - [.NET Implementation](#net-implementation) - [Architecture](#architecture) - - [Extension points](#extension-points) + ## What is TestPlatform? @@ -128,7 +134,7 @@ The Run workflow described above is very common in command line tools, and proba - *TestSession* - starts a set of testhosts for given test sources, to make the ready to run. - *AttachmentProcessing* - processes a given set of attachments that were produced during a previous test run, e.g. merges code coverage files. -## Specification +## Communication Protocol ### Base Protocol @@ -382,7 +388,7 @@ sequenceDiagram participant c as Client
(Visual Studio) participant r as Runner
(vstest.console.exe) c->>r: Run vstest.console -port X -r->>r: Connect to port Y +r->>r: Connect to port X r->>c: TestSession.Connected ``` @@ -590,7 +596,7 @@ public class DiscoveryCompletePayload // If true TotalCount is also set to -1. public bool IsAborted { get; set; } - // Telemetry metrics that are reported to client. + // Telemetry. public IDictionary? Metrics { get; set; } // Sources which were fully discovered. @@ -765,7 +771,7 @@ public class DiscoveryCompletePayload // If true TotalCount is also set to -1. public bool IsAborted { get; set; } - // Metrics. + // Telemetry. public IDictionary? Metrics { get; set; } // Sources which were fully discovered. @@ -1109,7 +1115,7 @@ public class TestRunCompleteEventArgs // Value is set to TimeSpan.Zero in case of any error. public TimeSpan ElapsedTimeInRunningTests { get; private set; } - // Metrics. + // Telemetry. public IDictionary? Metrics { get; set; } // Extensions that were discovered in this run. @@ -1230,10 +1236,10 @@ public class TestRunCriteriaWithTests public class TestExecutionContext { - // Gets or sets the frequency of run stats event. + // Gets or sets the batch size for sending test results. public long FrequencyOfRunStatsChangeEvent { get; set; } - // Gets or sets the timeout that triggers sending results regardless of cache size. + // Gets or sets the timeout that triggers sending results regardless of batch size. public TimeSpan RunStatsChangeEventTimeout { get; set; } // Gets or sets a value indicating whether execution is out of process. @@ -1242,7 +1248,7 @@ public class TestExecutionContext // Gets or sets a value indicating whether testhost process should be kept running after test run completion. public bool KeepAlive { get; set; } - // Gets or sets a value indicating whether test case level events need to be sent or not. + // Gets or sets a value indicating whether test case level events need to be sent or not. TODO: what is it? Is there since first commit, no usages on grep.app. public bool AreTestCaseLevelEventsRequired { get; set; } // Gets or sets a value indicating whether execution is in debug mode. @@ -1359,7 +1365,7 @@ public class TestRunCompleteEventArgs // Value is set to TimeSpan.Zero in case of any error. public TimeSpan ElapsedTimeInRunningTests { get; private set; } - // Metrics. + // Telemetry. public IDictionary? Metrics { get; set; } // Extensions that were discovered in this run. @@ -1718,10 +1724,30 @@ Same as above [TestExecution.StatsChange notification (Runner)](#testexecutionst ### Datacollection +## Extensibility + +Test Platform offers multiple points for extensibility. These extensibility points allow test framework authors to integrate their test frameworks with VSTest by providing a test adapter. Similarly a dataCollector extension can be provided that observes the run, or a logger extension can be provided that can observe the run and logs (reports) on the tests that were executed. + +IDEs (and other clients) can integrate with VSTest using the TranslationLayer package, which manages communication between the client and the runner. + + +### DLL Extension points + +Extensions are looked up by Reflection from the dlls provided to the run, based on a naming convention. Dlls named `*TestAdapter.dll`, `*TestLogger.dll`, `*Collector.dll`, `*RuntimeProvider.dll` are considered by the plugin loader. The types of extensions that each of those dlls can hold are **NOT** restricted to what the naming pattern suggests. A `TestAdapter.dll` can contain also different types of extensions. + +#### Test Adapter + +#### Test Logger + +#### Runtime Provider + + +### TranslationLayer extension points + TODO ### .NET Implementation #### Architecture -#### Extension points + From 96a6da4d2b4ca28bd079cb26013b543709b9a0eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 14 Jan 2026 17:06:04 +0100 Subject: [PATCH 3/4] Adapter extension --- docs/Overview.md | 110 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 109 insertions(+), 1 deletion(-) diff --git a/docs/Overview.md b/docs/Overview.md index 53d5054608..c51b66c2b7 100644 --- a/docs/Overview.md +++ b/docs/Overview.md @@ -51,6 +51,7 @@ - [Datacollection](#datacollection) - [Extensibility](#extensibility) - [DLL Extension points](#dll-extension-points) + - [ObjectModel](#objectmodel) - [Test Adapter](#test-adapter) - [Test Logger](#test-logger) - [Runtime Provider](#runtime-provider) @@ -1733,10 +1734,117 @@ IDEs (and other clients) can integrate with VSTest using the TranslationLayer pa ### DLL Extension points -Extensions are looked up by Reflection from the dlls provided to the run, based on a naming convention. Dlls named `*TestAdapter.dll`, `*TestLogger.dll`, `*Collector.dll`, `*RuntimeProvider.dll` are considered by the plugin loader. The types of extensions that each of those dlls can hold are **NOT** restricted to what the naming pattern suggests. A `TestAdapter.dll` can contain also different types of extensions. +Extensions are looked up by Reflection from the dlls provided to the run, based on a naming convention. Dlls named `*TestAdapter.dll`, `*TestLogger.dll`, `*Collector.dll`, `*RuntimeProvider.dll` are considered by the plugin loader. + +The extensions should target netstandard2.0, and not have any dll dependencies. The no-dependency rule is more lose for the TestAdapter extension point, because the tests target the framework that they will eventually run as. While the other extensions (data collector, and logger) get loaded into a process that might not align with the project target framework, e.g. in case where .NET Core project is ran from Visual Studio, where vstest.console.exe is using .NET Framework, and datacollector.exe is also using .NET Framework. + +#### ObjectModel + +ObjectModel is Test platform dll that holds all types and abstractions needed for extensibility. It is often used as reference dll and it is not typically shipped together with the extension. The runner / testhost / datacollector provides its own version of ObjectModel. + +```xml + +``` + +This reference uses `PrivateAssets="All"` to avoid copying the dll into the output folder. #### Test Adapter +Test adapter is the most used extension point that allows you to write your own test framework integration. This extension point is used by MSTest, NUnit, XUnit.net and other, to implement a `test adapter`, an adaptation layer that communicates with the particular test framework. + +The test adapter implements two interfaces: `ITestExecutor` or `ITestExecutor2` and `ITestDiscoverer`, both need to be implemented. + +The class implementing the discoverer has to be decorated with `DefaultExecutorUri`, that links it to the executor for which this discoverer discovers tests. + +Optionally the discoverer can be decorated with additional attributes: +- `FileExtensionAttribute` to filter the sources which it is able to inspect typically .exe and .dll are specified, but test frameworks don't have to limit themselves to just that. +- `CategoryAttribute` to specify `managed` or `native` assembly type. This only applies to adapters that specify .exe or .dll file extensions. https://github.com/microsoft/vstest/blob/main/docs/RFCs/0020-Improving-Logic-To-Pass-Sources-To-Adapters.md +- `DirectoryBasedTestDiscovererAttribute` - specifies a discoverer that is able to handle directories, rather than files. Currently used by PythonPyTestAdapter. Does not work in commandline - providing directory fails validation, but works via TranslationLayer. + +The class implementing the executor has to be decorated with `ExtensionUriAttribute`, specifying the name of the executor in Uri form. Such as `executor://mstestadapter/v2`. There is no specified schema or validation, but using `executor:///` is the norm. This is used for identification in telemetry, for logging and reporting, and for linking to the discoverer. + +An example of a test adapter: + +```cs +[ExtensionUri(Id)] +[DefaultExecutorUri(Id)] +public class Perfy : ITestDiscoverer, ITestExecutor +{ + public const string Id = "executor://ExampleTestAdapter/v1"; + public static readonly Uri Uri = new Uri(Id); + + public void DiscoverTests(IEnumerable sources, IDiscoveryContext context, + IMessageLogger logger, ITestCaseDiscoverySink discoverySink) + { + // Sources are the files (one ore more) to be discovered, typically those are dlls that we need to inspect for tests. But can be any file. + // The test framework somehow finds the tests, and will report them back by creating a TestCase object and reporting it to the sink. + var tests = ... + + foreach (var test in tests) + { + // TestCase sends back at least: + // - test case name, this should be unique, but stable across multiple discoveries and runs + // - uri of the executor. + // - the source (dll) in which the test case was found. + // It can also populate the additional information on the object, such as in which code file and on which line the test case is defined. + var tc = new TestCase(test.Name, Uri, source); + + // Send test case sends it to local cache, which batches the updates and sends them to the runner. + discoverySink.SendTestCase(tc); + } + } + + public void Cancel() + { + // Cancel the work that is happening. This is called "asynchronously", during running or discovering tests. + } + + public void RunTests(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle) + { + // Runs all tests specified in the collection of tests, those tests are typically discovered by discovery, sent to IDE. + // And then the IDE sends them back to a new instance of the the test runner. + // The test framework will do internal discovery of the tests (for MSTest that means scanning all methods in classes to find the ones decorated with [TestMethod]), + // finding the names of those tests, and filtering them down to the provided list. + // This method is used primarily by IDEs such as Visual Studio. + + // Ultimately is does not differ much from RunTests below. See that for details. + } + + public void RunTests(IEnumerable sources, IRunContext runContext, IFrameworkHandle frameworkHandle) + { + // The test framework inspects all the sources (files / dlls) that are provided, and finds all tests in them. + // It runs all tests, and when they finish it reports each testcase, and test result. These results are batched and sent to the runner. + // Typically this callback would be injected into the framework, so results can be sent as soon as the test finishes. For simplicity we + // show pseudo-code with synchronous loop. + + foreach (var source in sources) + { + // In MSTest this means finding all methods decorated with [TestMethod], and returning an object with test information, and the related method. + var tests = testFramework.FindTests(source); + + foreach (var test in tests) + { + var tc = new TestCase(test.Name, Uri, source); + // Notify datacollectors that test started + frameworkHandle.RecordStart(test); + TestOutcome result = testFramework.RunTest(test); + // Notify datacollectors that test ended + frameworkHandle.RecordEnd(test, result); + + frameworkHandle.RecordResult(new TestResult(tc) + { + Outcome = result; // e.g. Passed + }); + } + } + } +} +``` + +Additional information can be sourced from the context parameters that contain the run configuration. Additional information can be reported to the platform by using the `IMessageLogger` which is provided to discovery, and from which `IFrameworkHandle` also derives. + +Additional example of a toy test framework and adapter can be found in . Or in the respective implementations of MSTest, NUnit, XUnit and similar. + #### Test Logger #### Runtime Provider From 38fa05c4ec6c6817c16094dd3a446a6aa71bf0cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 14 Jan 2026 17:15:24 +0100 Subject: [PATCH 4/4] info about versions of interfaces --- docs/Overview.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/Overview.md b/docs/Overview.md index c51b66c2b7..b75d2a68b8 100644 --- a/docs/Overview.md +++ b/docs/Overview.md @@ -1843,6 +1843,10 @@ public class Perfy : ITestDiscoverer, ITestExecutor Additional information can be sourced from the context parameters that contain the run configuration. Additional information can be reported to the platform by using the `IMessageLogger` which is provided to discovery, and from which `IFrameworkHandle` also derives. +`ITestExecutor2` provides additional methods `ShouldAttachToTestHost` that return true or false. When false is returned, the debugger does not attache to the testhost process. This is used when execution is delegated to a sub-process, e.g. a .py file is provided, the run is routed via .NET Framework testhost, but eventually delegated to python executable. In that case the adapter returns false from the callback to avoid attaching Visual Studio to testhost.exe. + +The instance of `IFrameworkHandle` provided to the extension point, can be upcast to `IFrameworkHandle2` to use additional capabilities that allow attaching debugger to child processes. The attaching assumes that the child process uses the same TFM as the current process. + Additional example of a toy test framework and adapter can be found in . Or in the respective implementations of MSTest, NUnit, XUnit and similar. #### Test Logger