From 866df217354000628ef1c48ae98ed3058ef865fb Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Fri, 4 Aug 2023 15:11:30 -0700 Subject: [PATCH 1/3] EUDB compliance recommendations and example --- docs/EUDB-compliance.md | 65 ++++++++++++++++++++++++ examples/cpp/EventSender/EventSender.cpp | 64 ++++++++++++++++++++--- lib/api/IRuntimeConfig.hpp | 6 +++ lib/config/RuntimeConfig_Default.hpp | 6 +++ lib/tpm/TransmissionPolicyManager.cpp | 5 ++ 5 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 docs/EUDB-compliance.md diff --git a/docs/EUDB-compliance.md b/docs/EUDB-compliance.md new file mode 100644 index 000000000..1e110d297 --- /dev/null +++ b/docs/EUDB-compliance.md @@ -0,0 +1,65 @@ +# EUDB Guidance for 1DS C++ SDK + +In order to satisfy the Microsoft commitment to ensuring Privacy and Compliance, specifically +EUDB compliance with respect to EU 'Schrems II' decision, it is imperative for certain +commercial products to perform EUDB URL upload determination during application launch. + +1DS Collector service accepts data from all One Observability client SDKs. By default, traffic +simply flows to whichever region can best handle the traffic. This approach works well for +system required metadata. However some client scenarios require that data is sent to a specific +geographic location only. + +1DS C++ SDK supports ability to specify / adjust the upload URL at runtime. + +Two approaches could be applied to implement EUDB-compliant data upload. + +## Option 1: Create two instances of 1DS C++ SDK - one for US collector, another for EU collector + +See [Multiple Log Managers Example](https://github.com/microsoft/cpp_client_telemetry/tree/main/examples/cpp/SampleCppLogManagers) +that illustrates how to create multiple instances, each acting as a separate vertical pillar with +their own data collection URL. Two instances `LogManagerUS` and `LogManagerEU` may be configured +each with their own data collection URL, for example: + +- For US customers: `https://us-mobile.events.data.microsoft.com/OneCollector/1.0/` +- For EU customers: `https://eu-mobile.events.data.microsoft.com/OneCollector/1.0/` + +Depending on data requirements and outcome of dynamic EUDB determination, i.e. organization / +M365 Commercial Tenant is located in EU, the app decides to use `LogManagerEU` instance for +telemetry. Default `LogManager` instance can still be used for region-agnostic "global" +collection of required system diagnostics data. Remember to use the proper compliant instance +depending on event type. + +## Option 2: Autodetect the corresponding data collection URL on app start + +EventSender example has been modified to illustrate the concept: + +- Application starts. + +- `LogManager::Initialize(...)` is called with `ILogConfiguration[CFG_STR_COLLECTOR_URL]` set to +empty value `""`. This configuration instructs the SDK to run in offline mode. All data gets +logged to offline storage and not uploaded. This setting has the same effect as running in +paused state. Key difference is that irrespective of upload timer cadence - even for immediate +priority events, 1DS SDK never attempts to trigger the upload. This spetial configuration option +is safer than simply issuing `PauseTransmission` on app start. + +Then application must perform asynchronous EUDB URL detection in its own asynchronous task / +thread. URL detection process is asynchronous and may take significant amount of time from hundred +milliseconds to seconds. In order to avoid affecting application launch startup performance, +application may perform other startup and logging actions concurrently. All events get logged +in offline cache. + +- As part of the configuration update process - application calls `LogManager::PauseTransmission()` +done to ensure exclusive access to uploader configuration. + +- Once the EUDB URL is obtained from remote configuration provisioning service (ECS, MSGraph, +OneSettings, etc.), or read cached value from local app configuration storage, the value is supplied +to 1DS SDK: + +`ILogConfiguration[CFG_STR_COLLECTOR_URL] = eudb_url` + +Note that 1DS SDK itself does not provide a feature to store the cached URL value. It is up to the +product owners to decide what caching mechanism they would like to use: registry, ECS cache, Unity +player settings, mobile app settings provider, etc. + +- Finally the app code could call `LogManager::ResumeTransmission()` - to apply the new configuration +settings and enable the data upload to compliant destination. diff --git a/examples/cpp/EventSender/EventSender.cpp b/examples/cpp/EventSender/EventSender.cpp index 46555d2ad..23cb419f5 100644 --- a/examples/cpp/EventSender/EventSender.cpp +++ b/examples/cpp/EventSender/EventSender.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "LogManager.hpp" @@ -71,6 +72,38 @@ const char* defaultConfig = static_cast JSON_CONFIG } ); +// Mock function that performs random selection of destination URL. 1DS SDK does not define how the app needs to perform +// the region determination. Products should use MSGraph API, OCPS, or other remote config provisioning sources, such as +// ECS: https://learn.microsoft.com/en-us/deployedge/edge-configuration-and-experiments - in order to identify what 1DS +// collector to use for specific Enterprise or Consumer end-user telemetry uploads. Note that the EUDB URL determination +// is performed asynchronously and could take a few seconds. EUDB URL for Enterprise applications may be cached +// in app-specific configuration storage. 1DS SDK does not provide a feature to cache the data collection URL used for +// a previous session. +std::string GetEudbCollectorUrl() +{ + const auto randSeed = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + srand(static_cast(randSeed)); + return (rand() % 2) ? "https://us-mobile.events.data.microsoft.com/OneCollector/1.0/" : "https://eu-mobile.events.data.microsoft.com/OneCollector/1.0/"; +} + +void UpdateUploadUrl() +{ + printf("Performing collector URL detection...\n"); + // Transmissions must be paused prior to adjusting the URL. + LogManager::PauseTransmission(); + + // Obtain a reference to current configuration. + auto& config = LogManager::GetLogConfiguration(); + + // Update configuration in-place. + config[CFG_STR_COLLECTOR_URL] = GetEudbCollectorUrl(); + + // Resume transmission once EUDB collector URL detection is obtained. In case if EUDB collector determination fails, only required + // system diagnostics data containing no EUPI MAY be uploaded to global data collection endpoint. It is up to product teams to + // decide what strategy works best for their product. + LogManager::ResumeTransmission(); +} + int main(int argc, char *argv[]) { // 2nd (optional) parameter - path to custom SDK configuration @@ -87,24 +120,43 @@ int main(int argc, char *argv[]) // LogManager configuration auto& config = LogManager::GetLogConfiguration(); - config = MAT::FromJSON(jsonConfig); + auto customLogConfig = MAT::FromJSON(jsonConfig); + config = customLogConfig; // Assignment operation COLLATES the default + custom config // LogManager initialization ILogger *logger = LogManager::Initialize(); - bool utcActive = (bool)(config[CFG_STR_UTC][CFG_BOOL_UTC_ACTIVE]); + const bool utcActive = (bool)(config[CFG_STR_UTC][CFG_BOOL_UTC_ACTIVE]); printf("Running in %s mode...\n", (utcActive) ? "UTC" : "direct upload"); if (utcActive) { - printf("UTC provider group Id: %s\n", (const char *)(config[CFG_STR_UTC][CFG_STR_PROVIDER_GROUP_ID])); - printf("UTC large payloads: %s\n", ((bool)(config[CFG_STR_UTC][CFG_BOOL_UTC_LARGE_PAYLOADS])) ? "supported" : "not supported"); + printf("UTC provider group Id: %s\n", static_cast(config[CFG_STR_UTC][CFG_STR_PROVIDER_GROUP_ID])); + printf("UTC large payloads: %s\n", static_cast(config[CFG_STR_UTC][CFG_BOOL_UTC_LARGE_PAYLOADS]) ? "supported" : "not supported"); } else { - printf("Collector URL: %s\n", (const char *)(config[CFG_STR_COLLECTOR_URL])); + // LogManager::ILogConfiguration[CFG_STR_COLLECTOR_URL] defaults to global URL. + // + // If app-provided JSON config is empty on start, means the app intended to asynchronously + // obtain the data collection URL for EUDB compliance. App subsequently sets an empty URL - + // by assigning an empty value to the log manager instance CFG_STR_COLLECTOR_URL. At this + // point the Uploads are not performed until EUDB-compliant endpoint URL is obtained. + // + // Note that since ILogConfiguration configuration tree does not provide a thread-safety + // guarantee between the main thread and SDK uploader thread(s), adjusting the upload + // parameters, e.g. URL or timers, requires the app to pause transmission, adjust params, + // then resume transmission. + // + if (!customLogConfig.HasConfig(CFG_STR_COLLECTOR_URL)) + { + // If configuration provided as a parameter does not contain the URL + UpdateUploadUrl(); + } + const std::string url = config[CFG_STR_COLLECTOR_URL]; + printf("Collector URL: %s\n", url.c_str()); } - printf("Token (iKey): %s\n", (const char *)(config[CFG_STR_PRIMARY_TOKEN])); + printf("Token (iKey): %s\n", static_cast(config[CFG_STR_PRIMARY_TOKEN])); #if 0 // Code example that shows how to convert ILogConfiguration to JSON diff --git a/lib/api/IRuntimeConfig.hpp b/lib/api/IRuntimeConfig.hpp index d5788da80..aa6fcd8ba 100644 --- a/lib/api/IRuntimeConfig.hpp +++ b/lib/api/IRuntimeConfig.hpp @@ -34,6 +34,12 @@ namespace MAT_NS_BEGIN /// A string that contains the collector URI. virtual std::string GetCollectorUrl() = 0; + /// + /// Check used by uploader sequence to verify if URL is defined. + /// + /// true if URL is set, false otherwise. + virtual bool IsCollectorUrlSet() = 0; + /// /// Adds extension fields (created by the configuration provider) to an /// event. diff --git a/lib/config/RuntimeConfig_Default.hpp b/lib/config/RuntimeConfig_Default.hpp index 70cca8946..e919f67ca 100644 --- a/lib/config/RuntimeConfig_Default.hpp +++ b/lib/config/RuntimeConfig_Default.hpp @@ -104,6 +104,12 @@ namespace MAT_NS_BEGIN return std::string(url); } + virtual bool IsCollectorUrlSet() override + { + const char* url = config[CFG_STR_COLLECTOR_URL]; + return (url != nullptr) && (url[0] != '\0'); + } + virtual void DecorateEvent(std::map& extension, std::string const& experimentationProject, std::string const& eventName) override { UNREFERENCED_PARAMETER(extension); diff --git a/lib/tpm/TransmissionPolicyManager.cpp b/lib/tpm/TransmissionPolicyManager.cpp index 551ebf3e8..ed44829f5 100644 --- a/lib/tpm/TransmissionPolicyManager.cpp +++ b/lib/tpm/TransmissionPolicyManager.cpp @@ -107,6 +107,11 @@ namespace MAT_NS_BEGIN { return; } LOCKGUARD(m_scheduledUploadMutex); + if (!m_config.IsCollectorUrlSet()) + { + LOG_TRACE("Collector URL is not set, no upload."); + return; + } if (delay.count() < 0 || m_timerdelay.count() < 0) { LOG_TRACE("Negative delay(%d) or m_timerdelay(%d), no upload", delay.count(), m_timerdelay.count()); From dc55a61408276ae33beaeefd50d6d7cc201487bb Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Fri, 4 Aug 2023 15:23:07 -0700 Subject: [PATCH 2/3] Fix spelling issue --- docs/EUDB-compliance.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/EUDB-compliance.md b/docs/EUDB-compliance.md index 1e110d297..afe35c73e 100644 --- a/docs/EUDB-compliance.md +++ b/docs/EUDB-compliance.md @@ -39,7 +39,7 @@ EventSender example has been modified to illustrate the concept: empty value `""`. This configuration instructs the SDK to run in offline mode. All data gets logged to offline storage and not uploaded. This setting has the same effect as running in paused state. Key difference is that irrespective of upload timer cadence - even for immediate -priority events, 1DS SDK never attempts to trigger the upload. This spetial configuration option +priority events, 1DS SDK never attempts to trigger the upload. This special configuration option is safer than simply issuing `PauseTransmission` on app start. Then application must perform asynchronous EUDB URL detection in its own asynchronous task / From 3c92a5c28035af1acf1606f1b7a33dae1553fb7f Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Tue, 8 Aug 2023 12:43:13 -0700 Subject: [PATCH 3/3] Addressing code review comments. --- docs/EUDB-compliance.md | 5 ++++- examples/cpp/EventSender/EventSender.cpp | 4 +++- lib/tpm/TransmissionPolicyManager.cpp | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/EUDB-compliance.md b/docs/EUDB-compliance.md index afe35c73e..e12f3ffed 100644 --- a/docs/EUDB-compliance.md +++ b/docs/EUDB-compliance.md @@ -42,7 +42,7 @@ paused state. Key difference is that irrespective of upload timer cadence - even priority events, 1DS SDK never attempts to trigger the upload. This special configuration option is safer than simply issuing `PauseTransmission` on app start. -Then application must perform asynchronous EUDB URL detection in its own asynchronous task / +Then application must perform asynchronous EUDB URL detection once in its own asynchronous task / thread. URL detection process is asynchronous and may take significant amount of time from hundred milliseconds to seconds. In order to avoid affecting application launch startup performance, application may perform other startup and logging actions concurrently. All events get logged @@ -57,6 +57,9 @@ to 1DS SDK: `ILogConfiguration[CFG_STR_COLLECTOR_URL] = eudb_url` +This assignment of URL is done once during application start. Application does not need to change the +data collection URL after that. + Note that 1DS SDK itself does not provide a feature to store the cached URL value. It is up to the product owners to decide what caching mechanism they would like to use: registry, ECS cache, Unity player settings, mobile app settings provider, etc. diff --git a/examples/cpp/EventSender/EventSender.cpp b/examples/cpp/EventSender/EventSender.cpp index 23cb419f5..a5ebf2355 100644 --- a/examples/cpp/EventSender/EventSender.cpp +++ b/examples/cpp/EventSender/EventSender.cpp @@ -79,6 +79,8 @@ const char* defaultConfig = static_cast JSON_CONFIG // is performed asynchronously and could take a few seconds. EUDB URL for Enterprise applications may be cached // in app-specific configuration storage. 1DS SDK does not provide a feature to cache the data collection URL used for // a previous session. +// +// Note that this function to determine the URL is called once, early at boot. std::string GetEudbCollectorUrl() { const auto randSeed = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); @@ -95,7 +97,7 @@ void UpdateUploadUrl() // Obtain a reference to current configuration. auto& config = LogManager::GetLogConfiguration(); - // Update configuration in-place. + // Update configuration in-place. This is done once after the regional data collection URL is determined. config[CFG_STR_COLLECTOR_URL] = GetEudbCollectorUrl(); // Resume transmission once EUDB collector URL detection is obtained. In case if EUDB collector determination fails, only required diff --git a/lib/tpm/TransmissionPolicyManager.cpp b/lib/tpm/TransmissionPolicyManager.cpp index ed44829f5..efdb27ea9 100644 --- a/lib/tpm/TransmissionPolicyManager.cpp +++ b/lib/tpm/TransmissionPolicyManager.cpp @@ -106,12 +106,12 @@ namespace MAT_NS_BEGIN { if (guard.isPaused()) { return; } - LOCKGUARD(m_scheduledUploadMutex); if (!m_config.IsCollectorUrlSet()) { LOG_TRACE("Collector URL is not set, no upload."); return; } + LOCKGUARD(m_scheduledUploadMutex); if (delay.count() < 0 || m_timerdelay.count() < 0) { LOG_TRACE("Negative delay(%d) or m_timerdelay(%d), no upload", delay.count(), m_timerdelay.count());