From ad16298e247480c6ed034c957cc3649c75ae6a8c Mon Sep 17 00:00:00 2001 From: andretoyama-msft <106272532+andretoyama-msft@users.noreply.github.com> Date: Thu, 5 Jan 2023 15:54:00 -0300 Subject: [PATCH 01/25] Initial stage of DoClient Snap/Agent app and ubuntu core only flag (#149) * Initial stage of DoClient Snap/Agent app and ubuntu core only flag --- CMakeLists.txt | 1 + build/build.py | 8 +++ client-lite/CMakeLists.txt | 3 + client-lite/src/exe/docs.cpp | 5 +- client-lite/src/util/proc_launch_helper.h | 2 + snap/snapcraft.yaml | 86 +++++++++++++++++++++++ 6 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 snap/snapcraft.yaml diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c76281b..4dacba27 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,7 @@ option (DO_INCLUDE_PLUGINS "Build subproject plugins" OFF) option (DO_INCLUDE_SDK "Build subproject sdk-cpp" OFF) option (DO_BUILD_TESTS "Set DO_BUILD_TESTS to OFF to skip building tests." ON) +option (DO_BUILD_FOR_SNAP "Enable DO Snap build option" OFF) set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED ON) diff --git a/build/build.py b/build/build.py index 786eaa2b..189d9768 100644 --- a/build/build.py +++ b/build/build.py @@ -94,6 +94,10 @@ def __init__(self): '--package-for', dest='package_type', type=str, help='Supply package type. e.g. deb, or rpm' ) + self.parser.add_argument( + '--build-for-snap', dest='build_for_snap', action='store_true', + help='Only build components and features those required for the Ubuntu Core snap' + ) '''Agent only''' self.parser.add_argument( @@ -412,6 +416,7 @@ def __init__(self, script_args): self.package_type = self.script_args.package_type.lower() self.static_analysis = self.script_args.static_analysis + self.build_for_snap = self.script_args.build_for_snap @property def platform(self): @@ -436,6 +441,9 @@ def generate_options(self): if self.static_analysis: generate_options.extend(["-DCMAKE_CXX_CPPLINT=cpplint"]) + if self.build_for_snap: + generate_options.extend(["-DDO_BUILD_FOR_SNAP=1"]) + return generate_options def run(self): diff --git a/client-lite/CMakeLists.txt b/client-lite/CMakeLists.txt index 621f81f4..d0a72836 100644 --- a/client-lite/CMakeLists.txt +++ b/client-lite/CMakeLists.txt @@ -99,6 +99,9 @@ if (DO_PROXY_SUPPORT) target_compile_definitions(docs_common PRIVATE DO_PROXY_SUPPORT) target_link_libraries(docs_common PUBLIC libproxy::proxy) endif () +if (DO_BUILD_FOR_SNAP) + target_compile_definitions(docs_common PUBLIC DO_BUILD_FOR_SNAP) +endif () target_link_dl_lib(docs_common) diff --git a/client-lite/src/exe/docs.cpp b/client-lite/src/exe/docs.cpp index 53aa5cf8..23fc5a56 100644 --- a/client-lite/src/exe/docs.cpp +++ b/client-lite/src/exe/docs.cpp @@ -139,8 +139,11 @@ HRESULT Run() try RestPortAdvertiser portAdvertiser(controller.Port()); DoLogInfo("Port number written to %s", portAdvertiser.OutFilePath().data()); - + +#ifndef DO_BUILD_FOR_SNAP DropPermissions(); +#endif + DOLog::Init(docli::GetLogDirectory(), DOLog::Level::Verbose); diff --git a/client-lite/src/util/proc_launch_helper.h b/client-lite/src/util/proc_launch_helper.h index 73cebc33..2917dd64 100644 --- a/client-lite/src/util/proc_launch_helper.h +++ b/client-lite/src/util/proc_launch_helper.h @@ -62,7 +62,9 @@ inline void InitializePath(const std::string& path, mode_t mode = 0) try if (mode != 0) { +#ifndef DO_BUILD_FOR_SNAP SetDOPathPermissions(path, mode); +#endif } } } CATCH_LOG() diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml new file mode 100644 index 00000000..768e730a --- /dev/null +++ b/snap/snapcraft.yaml @@ -0,0 +1,86 @@ +name: deliveryoptimization-client +base: core20 # the base snap is the execution environment for this snap +version: '0.1' +summary: Ubuntu Core 20.04 DO +description: | + A proof-of-concept for the Delivery Optimization Client Ubuntu Core snap. + +grade: devel # must be 'stable' to release into candidate/stable channels +confinement: devmode # use 'strict' once you have the right plugs and slots + +##### +# +# Keywords +# +# after - Specifies part's dependencies. +# See https://snapcraft.io/docs/parts-lifecycle#heading--step-dependencies +# https://snapcraft.io/docs/parts-lifecycle +# plugin - Specifies a plugin required for building the part. +# +# slot - Specifies a code section to be shared with other snaps using Content Interface. [PROVIDER] +# See https://snapcraft.io/docs/content-interface +# plug - Specifies a target folder to access the files shared by the provider (or providers). [CONSUMER] +# See https://snapcraft.io/docs/content-interface +# +#### + +parts: + installdeps: + plugin: nil + source: . + override-build: | + ./build/scripts/bootstrap.sh --platform ubuntu2004 --install build + + agent: + plugin: python + source: . + override-build: | + python3 ./build/build.py --project agent --build-for-snap + mkdir -p ../install/bin + cp /tmp/build-deliveryoptimization-agent/linux-debug/client-lite/deliveryoptimization-agent ../install/bin/deliveryoptimization-agent + + after: + - installdeps + + stage-packages: + - libasn1-8-heimdal + - libboost-filesystem1.71.0 + - libbrotli1 + - libcurl4 + - libgssapi3-heimdal + - libhcrypto4-heimdal + - libheimbase1-heimdal + - libheimntlm0-heimdal + - libhx509-5-heimdal + - libkrb5-26-heimdal + - libldap-2.4-2 + - libnghttp2-14 + - libproxy1v5 + - libpsl5 + - libroken18-heimdal + - librtmp1 + - libsasl2-2 + - libssh-4 + - libwind0-heimdal + +apps: + deliveryoptimization-client: + command: bin/deliveryoptimization-agent + daemon: simple + refresh-mode: restart + restart-condition: always + restart-delay: 10s + plugs: + - network + - network-bind + +slots: + port-number: + interface: content + content: port-number + read: [ $SNAP_DATA/var/run/deliveryoptimization-agent ] + + config-file: + interface: content + content: config-file + write: [ $SNAP_DATA/etc/deliveryoptimization-agent/sdk-config.json ] From 98919b269e375f2ee317f0f1d91e655b91800a04 Mon Sep 17 00:00:00 2001 From: andretoyama-msft <106272532+andretoyama-msft@users.noreply.github.com> Date: Mon, 30 Jan 2023 15:36:13 -0300 Subject: [PATCH 02/25] Allow overriding cmake search path (#152) --- CMakeLists.txt | 4 +++- build/build.py | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4dacba27..1e453045 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,7 +83,9 @@ if (DO_PLATFORM_LINUX) # (Enable --trace-expand in cmake and look at lines from boost_headers-config.cmake and its callers.) # Workaround: force cmake to look in "/usr" since the /lib and /bin paths are either # not relevant for our usage or they are symlinks to /usr/lib and /usr/bin. - set(CMAKE_PREFIX_PATH "/usr") + if (NOT CMAKE_PREFIX_PATH) + set(CMAKE_PREFIX_PATH "/usr") + endif () endif (DO_PLATFORM_LINUX) if (DO_PLATFORM_WINDOWS AND DO_INCLUDE_SDK) diff --git a/build/build.py b/build/build.py index 189d9768..996f7b9e 100644 --- a/build/build.py +++ b/build/build.py @@ -98,6 +98,10 @@ def __init__(self): '--build-for-snap', dest='build_for_snap', action='store_true', help='Only build components and features those required for the Ubuntu Core snap' ) + self.parser.add_argument( + '--search-prefix', dest='search_prefix', type=str, + help='search prefix to pass to CMake' + ) '''Agent only''' self.parser.add_argument( @@ -417,6 +421,7 @@ def __init__(self, script_args): self.static_analysis = self.script_args.static_analysis self.build_for_snap = self.script_args.build_for_snap + self.search_prefix = self.script_args.search_prefix @property def platform(self): @@ -444,6 +449,9 @@ def generate_options(self): if self.build_for_snap: generate_options.extend(["-DDO_BUILD_FOR_SNAP=1"]) + if self.search_prefix: + generate_options.extend(["-DCMAKE_PREFIX_PATH={}".format(self.search_prefix)]) + return generate_options def run(self): From 11e67b8e14f015d55f84aa290ea6f1c8174a0b5a Mon Sep 17 00:00:00 2001 From: GeorgeSiggins <77402445+GeorgeSiggins@users.noreply.github.com> Date: Tue, 7 Feb 2023 03:18:14 +1000 Subject: [PATCH 03/25] Fix small typo in examples in README.md (#155) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 263e6bc1..d239345e 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ This will handle installing to the appropriate paths, and also the necessary set > cd /tmp/build-deliveryoptimization-agent/linux-debug/ > sudo apt-get install ./deliveryoptimization-agent*.deb > cd /tmp/build-deliveryoptimization-plugin-apt/linux-debug/ -> sudo apt get install ./deliveryoptimization-plugin-apt*.deb +> sudo apt-get install ./deliveryoptimization-plugin-apt*.deb ``` 2. If you build and install using cmake, or through some other custom means, be sure to setup the DO user/groups correctly in your installation. From 17b19971834dbead5f164aafc44a9dab05c5af3d Mon Sep 17 00:00:00 2001 From: JeffreySaathoff <107890382+JeffreySaathoff@users.noreply.github.com> Date: Wed, 8 Feb 2023 12:27:37 -0800 Subject: [PATCH 04/25] SDK: Add get_property support, remove callbacks and vectors from properties (#153) --- azure-pipelines/build/windows/dosdkcpp.yml | 2 +- build/build.py | 2 +- sdk-cpp/include/do_download.h | 41 +- sdk-cpp/include/do_download_property.h | 66 +- sdk-cpp/include/do_download_status.h | 4 + sdk-cpp/src/do_download.cpp | 20 +- sdk-cpp/src/do_download_property.cpp | 25 +- .../deliveryoptimization.h | 576 ++++++++++++++++++ .../internal/com/deliveryoptimization/do.hpp | 132 ---- .../com/do_download_property_internal.cpp | 185 +++--- sdk-cpp/src/internal/com/download_impl.cpp | 282 +++------ .../internal/do_download_property_internal.h | 45 +- sdk-cpp/src/internal/do_error_helpers.h | 10 + sdk-cpp/src/internal/download_impl.h | 9 +- sdk-cpp/src/internal/download_interface.h | 3 +- .../rest/do_download_property_internal.cpp | 52 +- sdk-cpp/src/internal/rest/download_impl.cpp | 2 +- sdk-cpp/tests/download_properties_tests.cpp | 159 ++--- sdk-cpp/tests/test_helpers.h | 20 +- 19 files changed, 956 insertions(+), 679 deletions(-) create mode 100644 sdk-cpp/src/internal/com/deliveryoptimization/deliveryoptimization.h delete mode 100644 sdk-cpp/src/internal/com/deliveryoptimization/do.hpp diff --git a/azure-pipelines/build/windows/dosdkcpp.yml b/azure-pipelines/build/windows/dosdkcpp.yml index 92492a72..24ace13a 100644 --- a/azure-pipelines/build/windows/dosdkcpp.yml +++ b/azure-pipelines/build/windows/dosdkcpp.yml @@ -32,7 +32,7 @@ pr: - sdk-cpp/build/cleanup-install.sh pool: - vmImage: "windows-2019" + vmImage: "windows-2022" jobs: - job: Debug diff --git a/build/build.py b/build/build.py index 996f7b9e..9f64a245 100644 --- a/build/build.py +++ b/build/build.py @@ -495,7 +495,7 @@ def compiler(self): def generator(self): # No need to specify architecture here as the default target platform name (architecture) is that of the host and is provided in the CMAKE_VS_PLATFORM_NAME_DEFAULT variable # https://cmake.org/cmake/help/latest/generator/Visual%20Studio%2016%202019.html - return super().generator or 'Visual Studio 16 2019' + return super().generator or 'Visual Studio 17 2022' @property def generate_options(self): diff --git a/sdk-cpp/include/do_download.h b/sdk-cpp/include/do_download.h index 694e0fe8..bddd7eda 100644 --- a/sdk-cpp/include/do_download.h +++ b/sdk-cpp/include/do_download.h @@ -34,6 +34,7 @@ class download std::error_code finalize() noexcept; std::error_code abort() noexcept; std::error_code get_status(download_status& status) noexcept; + std::error_code set_status_callback(status_callback_t callback) noexcept; std::error_code start_and_wait_until_completion(std::chrono::seconds timeoutSecs = std::chrono::hours(24)) noexcept; std::error_code start_and_wait_until_completion(const std::atomic_bool& isCancelled, std::chrono::seconds timeoutSecs = std::chrono::hours(24)) noexcept; @@ -41,13 +42,41 @@ class download static std::error_code download_url_to_path(const std::string& uri, const std::string& downloadFilePath, const std::atomic_bool& isCancelled, std::chrono::seconds timeoutSecs = std::chrono::hours(24)) noexcept; /* - For devices running windows before 20H1, dosvc exposed a now-deprecated com interface for setting certain download properties. - After 20H1, these properties were added to newer com interface, which this SDK is using. - Attempting to set a download property on a version of windows earlier than 20H1 will not set the property and throw an exception with - error code msdo::errc::do_e_unknown_property_id. + Certain properties are not supported on older versions of Windows, resulting in + msdo::errc::do_e_unknown_property_id from the following methods. See do_download_property.h. */ - std::error_code set_property(download_property key, const download_property_value& value) noexcept; - std::error_code get_property(download_property key, download_property_value& value) noexcept; + std::error_code set_property(download_property prop, const download_property_value& value) noexcept; + std::error_code get_property(download_property prop, download_property_value& value) noexcept; + + template + std::error_code set_property(download_property prop, const T& value) noexcept + { + download_property_value propVal; + std::error_code ec = download_property_value::make(value, propVal); + if (!ec) + { + ec = set_property(prop, propVal); + } + return ec; + } + + template + std::error_code get_property(download_property prop, T& value) noexcept + { + value = {}; + download_property_value propVal; + std::error_code ec = get_property(prop, propVal); + if (!ec) + { + ec = propVal.as(value); + } + return ec; + } + + std::error_code set_cost_policy(download_cost_policy value) noexcept + { + return set_property(download_property::cost_policy, static_cast(value)); + } private: download(); diff --git a/sdk-cpp/include/do_download_property.h b/sdk-cpp/include/do_download_property.h index 355f7673..e34f1195 100644 --- a/sdk-cpp/include/do_download_property.h +++ b/sdk-cpp/include/do_download_property.h @@ -4,7 +4,6 @@ #ifndef _DELIVERY_OPTIMIZATION_DO_DOWNLOAD_PROPERTY_H #define _DELIVERY_OPTIMIZATION_DO_DOWNLOAD_PROPERTY_H -#include #include #include #include @@ -23,69 +22,70 @@ class CDownloadPropertyValueInternal; class download; class download_status; -/* -For REST interface, these download properties are not yet supported -SDK will throw/return msdo::errc::e_notimpl if attempting to set/get a property -*/ enum class download_property { - id, // std::string - uri, // std::string - catalog_id, // std::string - caller_name, // std::string - download_file_path, // std::string - http_custom_headers, // std::string - cost_policy, // uint32 + id = 0, // string (readonly) + uri, // string + catalog_id, // string + caller_name, // string + download_file_path, // string + http_custom_headers, // string + cost_policy, // uint32 (see download_cost_policy enum) security_flags, // uint32 callback_freq_percent, // uint32 callback_freq_seconds, // uint32 no_progress_timeout_seconds, // uint32 use_foreground_priority, // bool blocking_mode, // bool - callback_interface, // void*, but used for storing lambda expressions - stream_interface, // void* - security_context, // byte array network_token, // bool - correlation_vector, // std::string - decryption_info, // std::string - integrity_check_info, // std::string - integrity_check_mandatory, // boolean + + // Available beginning in Windows 20H1 (build 19041) + correlation_vector, // string + decryption_info, // string + integrity_check_info, // string + integrity_check_mandatory, // bool total_size_bytes, // uint64 - // For the COM interface, the following properties are available only in Windows 21H2 (Build Number 22000) and beyond + // Available beginning in Windows 21H2 (build 22000) disallow_on_cellular, // bool - http_custom_auth_headers, // std::string + http_custom_auth_headers, // string + + // Available beginning in Windows 22H2 (build 22621) + allow_http_to_https_redirect, // bool + non_volatile, // bool }; -class download_property_value +// Values for download_property::cost_policy +enum class download_cost_policy : uint32_t { + always = 0, // download regardless of cost (foreground default) + unrestricted_network, // pause download on any metered network + standard, // pause download if over or near data limit (background default) + no_roaming, // pause download if roaming + no_surcharge, // pause download if over data limit +}; -/* -CDownloadImpl is declared as a friend class because it needs to access the platform-specific native value for download_property_value -The type of the native value is defined in CDownloadPropertyValueInternal, because DO header files are platform agnostic -This is so any user of the SDK does not have to worry about supplying platform specific compile definitions to use the SDK -*/ +class download_property_value +{ friend class details::CDownloadImpl; public: - using status_callback_t = std::function; - download_property_value(); ~download_property_value() = default; static std::error_code make(const std::string& val, download_property_value& out); + static std::error_code make(const std::wstring& val, download_property_value& out); + static std::error_code make(const char* val, download_property_value& out) { return make(std::string(val), out); } + static std::error_code make(const wchar_t* val, download_property_value& out) { return make(std::wstring(val), out); } static std::error_code make(uint32_t val, download_property_value& out); static std::error_code make(uint64_t val, download_property_value& out); static std::error_code make(bool val, download_property_value& out); - static std::error_code make(std::vector& val, download_property_value& out); - static std::error_code make(const status_callback_t& val, download_property_value& out); std::error_code as(bool& val) const noexcept; std::error_code as(uint32_t& val) const noexcept; std::error_code as(uint64_t& val) const noexcept; std::error_code as(std::string& val) const noexcept; - std::error_code as(std::vector& val) const noexcept; - std::error_code as(status_callback_t& val) const noexcept; + std::error_code as(std::wstring& val) const noexcept; private: std::shared_ptr _val; diff --git a/sdk-cpp/include/do_download_status.h b/sdk-cpp/include/do_download_status.h index 5813fb2b..f2d83f65 100644 --- a/sdk-cpp/include/do_download_status.h +++ b/sdk-cpp/include/do_download_status.h @@ -5,6 +5,7 @@ #define _DELIVERY_OPTIMIZATION_DO_DOWNLOAD_STATUS_H #include +#include #include #include "do_errors.h" @@ -71,6 +72,9 @@ class download_status }; +class download; +using status_callback_t = std::function; + } // namespace deliveryoptimization } // namespace microsoft diff --git a/sdk-cpp/src/do_download.cpp b/sdk-cpp/src/do_download.cpp index 966d9ebe..94934a76 100644 --- a/sdk-cpp/src/do_download.cpp +++ b/sdk-cpp/src/do_download.cpp @@ -30,7 +30,6 @@ download::~download() = default; std::error_code download::make(const std::string& uri, const std::string& downloadFilePath, std::unique_ptr& out) noexcept { out.reset(); - // using 'new' to access non-public constructor std::unique_ptr tmp(new download()); tmp->_download = std::make_shared(); std::error_code code = tmp->_download->Init(uri, downloadFilePath); @@ -69,6 +68,11 @@ std::error_code download::get_status(download_status& status) noexcept return _download->GetStatus(status); } +std::error_code download::set_status_callback(status_callback_t callback) noexcept +{ + return _download->SetStatusCallback(callback, *this); +} + std::error_code download::start_and_wait_until_completion(std::chrono::seconds timeOut) noexcept { std::atomic_bool isCancelled{ false }; @@ -157,18 +161,8 @@ static std::error_code g_TryOverrideDownlevelOsSetPropertyError(download_propert std::error_code download::set_property(download_property prop, const download_property_value& val) noexcept { - if (prop == download_property::callback_interface) - { - download_property_value::status_callback_t userCallback; - DO_RETURN_IF_FAILED(val.as(userCallback)); - - return _download->SetCallback(userCallback, *this); - } - else - { - auto ec = _download->SetProperty(prop, val); - return g_TryOverrideDownlevelOsSetPropertyError(prop, ec); - } + auto ec = _download->SetProperty(prop, val); + return g_TryOverrideDownlevelOsSetPropertyError(prop, ec); } std::error_code download::get_property(download_property prop, download_property_value& val) noexcept diff --git a/sdk-cpp/src/do_download_property.cpp b/sdk-cpp/src/do_download_property.cpp index bd1043aa..ba20ed31 100644 --- a/sdk-cpp/src/do_download_property.cpp +++ b/sdk-cpp/src/do_download_property.cpp @@ -28,7 +28,7 @@ std::error_code download_property_value::make(const std::string& val, download_p return DO_OK; } -std::error_code download_property_value::make(uint32_t val, download_property_value& out) +std::error_code download_property_value::make(const std::wstring& val, download_property_value& out) { download_property_value temp; std::error_code code = temp._val->Init(val); @@ -38,17 +38,7 @@ std::error_code download_property_value::make(uint32_t val, download_property_va return DO_OK; } -std::error_code download_property_value::make(uint64_t val, download_property_value& out) -{ - download_property_value temp; - std::error_code code = temp._val->Init(val); - DO_RETURN_IF_FAILED(code); - - out = temp; - return DO_OK; -} - -std::error_code download_property_value::make(bool val, download_property_value& out) +std::error_code download_property_value::make(uint32_t val, download_property_value& out) { download_property_value temp; std::error_code code = temp._val->Init(val); @@ -58,7 +48,7 @@ std::error_code download_property_value::make(bool val, download_property_value& return DO_OK; } -std::error_code download_property_value::make(std::vector& val, download_property_value& out) +std::error_code download_property_value::make(uint64_t val, download_property_value& out) { download_property_value temp; std::error_code code = temp._val->Init(val); @@ -68,7 +58,7 @@ std::error_code download_property_value::make(std::vector& val, d return DO_OK; } -std::error_code download_property_value::make(const status_callback_t& val, download_property_value& out) +std::error_code download_property_value::make(bool val, download_property_value& out) { download_property_value temp; std::error_code code = temp._val->Init(val); @@ -98,12 +88,7 @@ std::error_code download_property_value::as(std::string& val) const noexcept return _val->As(val); } -std::error_code download_property_value::as(std::vector& val) const noexcept -{ - return _val->As(val); -} - -std::error_code download_property_value::as(status_callback_t& val) const noexcept +std::error_code download_property_value::as(std::wstring& val) const noexcept { return _val->As(val); } diff --git a/sdk-cpp/src/internal/com/deliveryoptimization/deliveryoptimization.h b/sdk-cpp/src/internal/com/deliveryoptimization/deliveryoptimization.h new file mode 100644 index 00000000..6e63dd25 --- /dev/null +++ b/sdk-cpp/src/internal/com/deliveryoptimization/deliveryoptimization.h @@ -0,0 +1,576 @@ + + +/* this ALWAYS GENERATED file contains the definitions for the interfaces */ + + + /* File created by MIDL compiler version 8.01.0628 */ +/* @@MIDL_FILE_HEADING( ) */ + + + +/* verify that the version is high enough to compile this file*/ +#ifndef __REQUIRED_RPCNDR_H_VERSION__ +#define __REQUIRED_RPCNDR_H_VERSION__ 501 +#endif + +/* verify that the version is high enough to compile this file*/ +#ifndef __REQUIRED_RPCSAL_H_VERSION__ +#define __REQUIRED_RPCSAL_H_VERSION__ 100 +#endif + +#include "rpc.h" +#include "rpcndr.h" + +#ifndef __RPCNDR_H_VERSION__ +#error this stub requires an updated version of +#endif /* __RPCNDR_H_VERSION__ */ + +#ifndef COM_NO_WINDOWS_H +#include "windows.h" +#include "ole2.h" +#endif /*COM_NO_WINDOWS_H*/ + +#ifndef __deliveryoptimization_h__ +#define __deliveryoptimization_h__ + +#if defined(_MSC_VER) && (_MSC_VER >= 1020) +#pragma once +#endif + +#ifndef DECLSPEC_XFGVIRT +#if defined(_CONTROL_FLOW_GUARD_XFG) +#define DECLSPEC_XFGVIRT(base, func) __declspec(xfg_virtual(base, func)) +#else +#define DECLSPEC_XFGVIRT(base, func) +#endif +#endif + +/* Forward Declarations */ + +#ifndef __IDODownload_FWD_DEFINED__ +#define __IDODownload_FWD_DEFINED__ +typedef interface IDODownload IDODownload; + +#endif /* __IDODownload_FWD_DEFINED__ */ + + +#ifndef __IDODownloadStatusCallback_FWD_DEFINED__ +#define __IDODownloadStatusCallback_FWD_DEFINED__ +typedef interface IDODownloadStatusCallback IDODownloadStatusCallback; + +#endif /* __IDODownloadStatusCallback_FWD_DEFINED__ */ + + +#ifndef __IDOManager_FWD_DEFINED__ +#define __IDOManager_FWD_DEFINED__ +typedef interface IDOManager IDOManager; + +#endif /* __IDOManager_FWD_DEFINED__ */ + + +#ifndef __DeliveryOptimization_FWD_DEFINED__ +#define __DeliveryOptimization_FWD_DEFINED__ + +#ifdef __cplusplus +typedef class DeliveryOptimization DeliveryOptimization; +#else +typedef struct DeliveryOptimization DeliveryOptimization; +#endif /* __cplusplus */ + +#endif /* __DeliveryOptimization_FWD_DEFINED__ */ + + +/* header files for imported files */ +#include "oaidl.h" + +#ifdef __cplusplus +extern "C"{ +#endif + + +/* interface __MIDL_itf_deliveryoptimization_0000_0000 */ +/* [local] */ + +#include + +#pragma region Desktop Family +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + +typedef struct _DO_DOWNLOAD_RANGE + { + UINT64 Offset; + UINT64 Length; + } DO_DOWNLOAD_RANGE; + +typedef struct _DO_DOWNLOAD_RANGES_INFO + { + UINT RangeCount; + /* [size_is] */ DO_DOWNLOAD_RANGE Ranges[ 1 ]; + } DO_DOWNLOAD_RANGES_INFO; + +typedef +enum _DODownloadState + { + DODownloadState_Created = 0, + DODownloadState_Transferring = ( DODownloadState_Created + 1 ) , + DODownloadState_Transferred = ( DODownloadState_Transferring + 1 ) , + DODownloadState_Finalized = ( DODownloadState_Transferred + 1 ) , + DODownloadState_Aborted = ( DODownloadState_Finalized + 1 ) , + DODownloadState_Paused = ( DODownloadState_Aborted + 1 ) + } DODownloadState; + +typedef struct _DO_DOWNLOAD_STATUS + { + UINT64 BytesTotal; + UINT64 BytesTransferred; + DODownloadState State; + HRESULT Error; + HRESULT ExtendedError; + } DO_DOWNLOAD_STATUS; + +typedef +enum _DODownloadCostPolicy + { + DODownloadCostPolicy_Always = 0, + DODownloadCostPolicy_Unrestricted = ( DODownloadCostPolicy_Always + 1 ) , + DODownloadCostPolicy_Standard = ( DODownloadCostPolicy_Unrestricted + 1 ) , + DODownloadCostPolicy_NoRoaming = ( DODownloadCostPolicy_Standard + 1 ) , + DODownloadCostPolicy_NoSurcharge = ( DODownloadCostPolicy_NoRoaming + 1 ) , + DODownloadCostPolicy_NoCellular = ( DODownloadCostPolicy_NoSurcharge + 1 ) + } DODownloadCostPolicy; + +typedef +enum _DODownloadProperty + { + DODownloadProperty_Id = 0, + DODownloadProperty_Uri = ( DODownloadProperty_Id + 1 ) , + DODownloadProperty_ContentId = ( DODownloadProperty_Uri + 1 ) , + DODownloadProperty_DisplayName = ( DODownloadProperty_ContentId + 1 ) , + DODownloadProperty_LocalPath = ( DODownloadProperty_DisplayName + 1 ) , + DODownloadProperty_HttpCustomHeaders = ( DODownloadProperty_LocalPath + 1 ) , + DODownloadProperty_CostPolicy = ( DODownloadProperty_HttpCustomHeaders + 1 ) , + DODownloadProperty_SecurityFlags = ( DODownloadProperty_CostPolicy + 1 ) , + DODownloadProperty_CallbackFreqPercent = ( DODownloadProperty_SecurityFlags + 1 ) , + DODownloadProperty_CallbackFreqSeconds = ( DODownloadProperty_CallbackFreqPercent + 1 ) , + DODownloadProperty_NoProgressTimeoutSeconds = ( DODownloadProperty_CallbackFreqSeconds + 1 ) , + DODownloadProperty_ForegroundPriority = ( DODownloadProperty_NoProgressTimeoutSeconds + 1 ) , + DODownloadProperty_BlockingMode = ( DODownloadProperty_ForegroundPriority + 1 ) , + DODownloadProperty_CallbackInterface = ( DODownloadProperty_BlockingMode + 1 ) , + DODownloadProperty_StreamInterface = ( DODownloadProperty_CallbackInterface + 1 ) , + DODownloadProperty_SecurityContext = ( DODownloadProperty_StreamInterface + 1 ) , + DODownloadProperty_NetworkToken = ( DODownloadProperty_SecurityContext + 1 ) , + DODownloadProperty_CorrelationVector = ( DODownloadProperty_NetworkToken + 1 ) , + DODownloadProperty_DecryptionInfo = ( DODownloadProperty_CorrelationVector + 1 ) , + DODownloadProperty_IntegrityCheckInfo = ( DODownloadProperty_DecryptionInfo + 1 ) , + DODownloadProperty_IntegrityCheckMandatory = ( DODownloadProperty_IntegrityCheckInfo + 1 ) , + DODownloadProperty_TotalSizeBytes = ( DODownloadProperty_IntegrityCheckMandatory + 1 ) , + DODownloadProperty_DisallowOnCellular = ( DODownloadProperty_TotalSizeBytes + 1 ) , + DODownloadProperty_HttpCustomAuthHeaders = ( DODownloadProperty_DisallowOnCellular + 1 ) , + DODownloadProperty_HttpAllowSecureToNonSecureRedirect = ( DODownloadProperty_HttpCustomAuthHeaders + 1 ) , + DODownloadProperty_NonVolatile = ( DODownloadProperty_HttpAllowSecureToNonSecureRedirect + 1 ) + } DODownloadProperty; + +typedef struct _DO_DOWNLOAD_ENUM_CATEGORY + { + DODownloadProperty Property; + LPCWSTR Value; + } DO_DOWNLOAD_ENUM_CATEGORY; + + + +extern RPC_IF_HANDLE __MIDL_itf_deliveryoptimization_0000_0000_v0_0_c_ifspec; +extern RPC_IF_HANDLE __MIDL_itf_deliveryoptimization_0000_0000_v0_0_s_ifspec; + +#ifndef __IDODownload_INTERFACE_DEFINED__ +#define __IDODownload_INTERFACE_DEFINED__ + +/* interface IDODownload */ +/* [uuid][object] */ + + +EXTERN_C const IID IID_IDODownload; + +#if defined(__cplusplus) && !defined(CINTERFACE) + + MIDL_INTERFACE("FBBD7FC0-C147-4727-A38D-827EF071EE77") + IDODownload : public IUnknown + { + public: + virtual HRESULT STDMETHODCALLTYPE Start( + /* [unique][in] */ __RPC__in_opt const DO_DOWNLOAD_RANGES_INFO *ranges) = 0; + + virtual HRESULT STDMETHODCALLTYPE Pause( void) = 0; + + virtual HRESULT STDMETHODCALLTYPE Abort( void) = 0; + + virtual HRESULT STDMETHODCALLTYPE Finalize( void) = 0; + + virtual HRESULT STDMETHODCALLTYPE GetStatus( + /* [out] */ __RPC__out DO_DOWNLOAD_STATUS *status) = 0; + + virtual HRESULT STDMETHODCALLTYPE GetProperty( + /* [in] */ DODownloadProperty propId, + /* [out] */ __RPC__out VARIANT *propVal) = 0; + + virtual HRESULT STDMETHODCALLTYPE SetProperty( + /* [in] */ DODownloadProperty propId, + /* [in] */ __RPC__in const VARIANT *propVal) = 0; + + }; + + +#else /* C style interface */ + + typedef struct IDODownloadVtbl + { + BEGIN_INTERFACE + + DECLSPEC_XFGVIRT(IUnknown, QueryInterface) + HRESULT ( STDMETHODCALLTYPE *QueryInterface )( + __RPC__in IDODownload * This, + /* [in] */ __RPC__in REFIID riid, + /* [annotation][iid_is][out] */ + _COM_Outptr_ void **ppvObject); + + DECLSPEC_XFGVIRT(IUnknown, AddRef) + ULONG ( STDMETHODCALLTYPE *AddRef )( + __RPC__in IDODownload * This); + + DECLSPEC_XFGVIRT(IUnknown, Release) + ULONG ( STDMETHODCALLTYPE *Release )( + __RPC__in IDODownload * This); + + DECLSPEC_XFGVIRT(IDODownload, Start) + HRESULT ( STDMETHODCALLTYPE *Start )( + __RPC__in IDODownload * This, + /* [unique][in] */ __RPC__in_opt const DO_DOWNLOAD_RANGES_INFO *ranges); + + DECLSPEC_XFGVIRT(IDODownload, Pause) + HRESULT ( STDMETHODCALLTYPE *Pause )( + __RPC__in IDODownload * This); + + DECLSPEC_XFGVIRT(IDODownload, Abort) + HRESULT ( STDMETHODCALLTYPE *Abort )( + __RPC__in IDODownload * This); + + DECLSPEC_XFGVIRT(IDODownload, Finalize) + HRESULT ( STDMETHODCALLTYPE *Finalize )( + __RPC__in IDODownload * This); + + DECLSPEC_XFGVIRT(IDODownload, GetStatus) + HRESULT ( STDMETHODCALLTYPE *GetStatus )( + __RPC__in IDODownload * This, + /* [out] */ __RPC__out DO_DOWNLOAD_STATUS *status); + + DECLSPEC_XFGVIRT(IDODownload, GetProperty) + HRESULT ( STDMETHODCALLTYPE *GetProperty )( + __RPC__in IDODownload * This, + /* [in] */ DODownloadProperty propId, + /* [out] */ __RPC__out VARIANT *propVal); + + DECLSPEC_XFGVIRT(IDODownload, SetProperty) + HRESULT ( STDMETHODCALLTYPE *SetProperty )( + __RPC__in IDODownload * This, + /* [in] */ DODownloadProperty propId, + /* [in] */ __RPC__in const VARIANT *propVal); + + END_INTERFACE + } IDODownloadVtbl; + + interface IDODownload + { + CONST_VTBL struct IDODownloadVtbl *lpVtbl; + }; + + + +#ifdef COBJMACROS + + +#define IDODownload_QueryInterface(This,riid,ppvObject) \ + ( (This)->lpVtbl -> QueryInterface(This,riid,ppvObject) ) + +#define IDODownload_AddRef(This) \ + ( (This)->lpVtbl -> AddRef(This) ) + +#define IDODownload_Release(This) \ + ( (This)->lpVtbl -> Release(This) ) + + +#define IDODownload_Start(This,ranges) \ + ( (This)->lpVtbl -> Start(This,ranges) ) + +#define IDODownload_Pause(This) \ + ( (This)->lpVtbl -> Pause(This) ) + +#define IDODownload_Abort(This) \ + ( (This)->lpVtbl -> Abort(This) ) + +#define IDODownload_Finalize(This) \ + ( (This)->lpVtbl -> Finalize(This) ) + +#define IDODownload_GetStatus(This,status) \ + ( (This)->lpVtbl -> GetStatus(This,status) ) + +#define IDODownload_GetProperty(This,propId,propVal) \ + ( (This)->lpVtbl -> GetProperty(This,propId,propVal) ) + +#define IDODownload_SetProperty(This,propId,propVal) \ + ( (This)->lpVtbl -> SetProperty(This,propId,propVal) ) + +#endif /* COBJMACROS */ + + +#endif /* C style interface */ + + + + +#endif /* __IDODownload_INTERFACE_DEFINED__ */ + + +#ifndef __IDODownloadStatusCallback_INTERFACE_DEFINED__ +#define __IDODownloadStatusCallback_INTERFACE_DEFINED__ + +/* interface IDODownloadStatusCallback */ +/* [uuid][object] */ + + +EXTERN_C const IID IID_IDODownloadStatusCallback; + +#if defined(__cplusplus) && !defined(CINTERFACE) + + MIDL_INTERFACE("D166E8E3-A90E-4392-8E87-05E996D3747D") + IDODownloadStatusCallback : public IUnknown + { + public: + virtual HRESULT STDMETHODCALLTYPE OnStatusChange( + /* [in] */ __RPC__in_opt IDODownload *download, + /* [in] */ __RPC__in const DO_DOWNLOAD_STATUS *status) = 0; + + }; + + +#else /* C style interface */ + + typedef struct IDODownloadStatusCallbackVtbl + { + BEGIN_INTERFACE + + DECLSPEC_XFGVIRT(IUnknown, QueryInterface) + HRESULT ( STDMETHODCALLTYPE *QueryInterface )( + __RPC__in IDODownloadStatusCallback * This, + /* [in] */ __RPC__in REFIID riid, + /* [annotation][iid_is][out] */ + _COM_Outptr_ void **ppvObject); + + DECLSPEC_XFGVIRT(IUnknown, AddRef) + ULONG ( STDMETHODCALLTYPE *AddRef )( + __RPC__in IDODownloadStatusCallback * This); + + DECLSPEC_XFGVIRT(IUnknown, Release) + ULONG ( STDMETHODCALLTYPE *Release )( + __RPC__in IDODownloadStatusCallback * This); + + DECLSPEC_XFGVIRT(IDODownloadStatusCallback, OnStatusChange) + HRESULT ( STDMETHODCALLTYPE *OnStatusChange )( + __RPC__in IDODownloadStatusCallback * This, + /* [in] */ __RPC__in_opt IDODownload *download, + /* [in] */ __RPC__in const DO_DOWNLOAD_STATUS *status); + + END_INTERFACE + } IDODownloadStatusCallbackVtbl; + + interface IDODownloadStatusCallback + { + CONST_VTBL struct IDODownloadStatusCallbackVtbl *lpVtbl; + }; + + + +#ifdef COBJMACROS + + +#define IDODownloadStatusCallback_QueryInterface(This,riid,ppvObject) \ + ( (This)->lpVtbl -> QueryInterface(This,riid,ppvObject) ) + +#define IDODownloadStatusCallback_AddRef(This) \ + ( (This)->lpVtbl -> AddRef(This) ) + +#define IDODownloadStatusCallback_Release(This) \ + ( (This)->lpVtbl -> Release(This) ) + + +#define IDODownloadStatusCallback_OnStatusChange(This,download,status) \ + ( (This)->lpVtbl -> OnStatusChange(This,download,status) ) + +#endif /* COBJMACROS */ + + +#endif /* C style interface */ + + + + +#endif /* __IDODownloadStatusCallback_INTERFACE_DEFINED__ */ + + +#ifndef __IDOManager_INTERFACE_DEFINED__ +#define __IDOManager_INTERFACE_DEFINED__ + +/* interface IDOManager */ +/* [uuid][object] */ + + +EXTERN_C const IID IID_IDOManager; + +#if defined(__cplusplus) && !defined(CINTERFACE) + + MIDL_INTERFACE("400E2D4A-1431-4C1A-A748-39CA472CFDB1") + IDOManager : public IUnknown + { + public: + virtual HRESULT STDMETHODCALLTYPE CreateDownload( + /* [out] */ __RPC__deref_out_opt IDODownload **download) = 0; + + virtual HRESULT STDMETHODCALLTYPE EnumDownloads( + /* [unique][in] */ __RPC__in_opt const DO_DOWNLOAD_ENUM_CATEGORY *category, + /* [out] */ __RPC__deref_out_opt IEnumUnknown **ppEnum) = 0; + + }; + + +#else /* C style interface */ + + typedef struct IDOManagerVtbl + { + BEGIN_INTERFACE + + DECLSPEC_XFGVIRT(IUnknown, QueryInterface) + HRESULT ( STDMETHODCALLTYPE *QueryInterface )( + __RPC__in IDOManager * This, + /* [in] */ __RPC__in REFIID riid, + /* [annotation][iid_is][out] */ + _COM_Outptr_ void **ppvObject); + + DECLSPEC_XFGVIRT(IUnknown, AddRef) + ULONG ( STDMETHODCALLTYPE *AddRef )( + __RPC__in IDOManager * This); + + DECLSPEC_XFGVIRT(IUnknown, Release) + ULONG ( STDMETHODCALLTYPE *Release )( + __RPC__in IDOManager * This); + + DECLSPEC_XFGVIRT(IDOManager, CreateDownload) + HRESULT ( STDMETHODCALLTYPE *CreateDownload )( + __RPC__in IDOManager * This, + /* [out] */ __RPC__deref_out_opt IDODownload **download); + + DECLSPEC_XFGVIRT(IDOManager, EnumDownloads) + HRESULT ( STDMETHODCALLTYPE *EnumDownloads )( + __RPC__in IDOManager * This, + /* [unique][in] */ __RPC__in_opt const DO_DOWNLOAD_ENUM_CATEGORY *category, + /* [out] */ __RPC__deref_out_opt IEnumUnknown **ppEnum); + + END_INTERFACE + } IDOManagerVtbl; + + interface IDOManager + { + CONST_VTBL struct IDOManagerVtbl *lpVtbl; + }; + + + +#ifdef COBJMACROS + + +#define IDOManager_QueryInterface(This,riid,ppvObject) \ + ( (This)->lpVtbl -> QueryInterface(This,riid,ppvObject) ) + +#define IDOManager_AddRef(This) \ + ( (This)->lpVtbl -> AddRef(This) ) + +#define IDOManager_Release(This) \ + ( (This)->lpVtbl -> Release(This) ) + + +#define IDOManager_CreateDownload(This,download) \ + ( (This)->lpVtbl -> CreateDownload(This,download) ) + +#define IDOManager_EnumDownloads(This,category,ppEnum) \ + ( (This)->lpVtbl -> EnumDownloads(This,category,ppEnum) ) + +#endif /* COBJMACROS */ + + +#endif /* C style interface */ + + + + +#endif /* __IDOManager_INTERFACE_DEFINED__ */ + + + +#ifndef __DeliveryOptimization_LIBRARY_DEFINED__ +#define __DeliveryOptimization_LIBRARY_DEFINED__ + +/* library DeliveryOptimization */ +/* [uuid] */ + + +EXTERN_C const IID LIBID_DeliveryOptimization; + +EXTERN_C const CLSID CLSID_DeliveryOptimization; + +#ifdef __cplusplus + +class DECLSPEC_UUID("5b99fa76-721c-423c-adac-56d03c8a8007") +DeliveryOptimization; +#endif +#endif /* __DeliveryOptimization_LIBRARY_DEFINED__ */ + +/* interface __MIDL_itf_deliveryoptimization_0000_0004 */ +/* [local] */ + +#define DO_LENGTH_TO_EOF (UINT64)(-1) + +#define DecryptionInfo_KeyData L"KeyData" +#define DecryptionInfo_EncryptionBufferSize L"EncryptionBufferSize" +#define DecryptionInfo_AlgorithmName L"AlgorithmName" +#define DecryptionInfo_ChainingMode L"ChainingMode" + +#define IntegrityCheckInfo_PiecesHashFileUrl L"PiecesHashFileUrl" +#define IntegrityCheckInfo_PiecesHashFileDigest L"PiecesHashFileDigest" +#define IntegrityCheckInfo_PiecesHashFileDigestAlgorithm L"PiecesHashFileDigestAlgorithm" +#define IntegrityCheckInfo_HashOfHashes L"HashOfHashes" + +#endif /* WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) */ +#pragma endregion + + +extern RPC_IF_HANDLE __MIDL_itf_deliveryoptimization_0000_0004_v0_0_c_ifspec; +extern RPC_IF_HANDLE __MIDL_itf_deliveryoptimization_0000_0004_v0_0_s_ifspec; + +/* Additional Prototypes for ALL interfaces */ + +unsigned long __RPC_USER VARIANT_UserSize( __RPC__in unsigned long *, unsigned long , __RPC__in VARIANT * ); +unsigned char * __RPC_USER VARIANT_UserMarshal( __RPC__in unsigned long *, __RPC__inout_xcount(0) unsigned char *, __RPC__in VARIANT * ); +unsigned char * __RPC_USER VARIANT_UserUnmarshal(__RPC__in unsigned long *, __RPC__in_xcount(0) unsigned char *, __RPC__out VARIANT * ); +void __RPC_USER VARIANT_UserFree( __RPC__in unsigned long *, __RPC__in VARIANT * ); + +unsigned long __RPC_USER VARIANT_UserSize64( __RPC__in unsigned long *, unsigned long , __RPC__in VARIANT * ); +unsigned char * __RPC_USER VARIANT_UserMarshal64( __RPC__in unsigned long *, __RPC__inout_xcount(0) unsigned char *, __RPC__in VARIANT * ); +unsigned char * __RPC_USER VARIANT_UserUnmarshal64(__RPC__in unsigned long *, __RPC__in_xcount(0) unsigned char *, __RPC__out VARIANT * ); +void __RPC_USER VARIANT_UserFree64( __RPC__in unsigned long *, __RPC__in VARIANT * ); + +/* end of Additional Prototypes */ + +#ifdef __cplusplus +} +#endif + +#endif + + diff --git a/sdk-cpp/src/internal/com/deliveryoptimization/do.hpp b/sdk-cpp/src/internal/com/deliveryoptimization/do.hpp deleted file mode 100644 index 0beed91c..00000000 --- a/sdk-cpp/src/internal/com/deliveryoptimization/do.hpp +++ /dev/null @@ -1,132 +0,0 @@ -#ifndef _do_hpp_ -#define _do_hpp_ -extern "C" { - -// Prerequisites. These may already be included via a precompiled header, possibly via other headers such as windows.h or ole.h. -//#include "rpc.h" -//#include "rpcndr.h" -//#include "oaidl.h" - -// DO reference: https://docs.microsoft.com/en-us/windows/win32/delivery_optimization/do-reference - -// https://docs.microsoft.com/en-us/windows/win32/delivery_optimization/deliveryoptimizationdownloadtypes/ns-deliveryoptimizationdownloadtypes-do_download_range -typedef struct _DO_DOWNLOAD_RANGE -{ - UINT64 Offset; - UINT64 Length; -} DO_DOWNLOAD_RANGE; - -// https://docs.microsoft.com/en-us/windows/win32/delivery_optimization/do/ns-do-do_download_range_info -typedef struct _DO_DOWNLOAD_RANGES_INFO -{ - UINT RangeCount; - DO_DOWNLOAD_RANGE Ranges[1]; -} DO_DOWNLOAD_RANGES_INFO; - -// https://docs.microsoft.com/en-us/windows/win32/delivery_optimization/deliveryoptimizationdownloadtypes/ne-deliveryoptimizationdownloadtypes-dodownloadstate -typedef enum _DODownloadState -{ - DODownloadState_Created = 0, - DODownloadState_Transferring, - DODownloadState_Transferred, - DODownloadState_Finalized, - DODownloadState_Aborted, - DODownloadState_Paused -} DODownloadState; - -// https://docs.microsoft.com/en-us/windows/win32/delivery_optimization/do/ns-do-do_download_status -typedef struct _DO_DOWNLOAD_STATUS -{ - UINT64 BytesTotal; - UINT64 BytesTransferred; - DODownloadState State; - HRESULT Error; - HRESULT ExtendedError; -} DO_DOWNLOAD_STATUS; - -// https://docs.microsoft.com/en-us/windows/win32/delivery_optimization/deliveryoptimizationdownloadtypes/ne-deliveryoptimizationdownloadtypes-dodownloadcostpolicy -typedef enum _DODownloadCostPolicy -{ - DODownloadCostPolicy_Always = 0, - DODownloadCostPolicy_Unrestricted, - DODownloadCostPolicy_Standard, - DODownloadCostPolicy_NoRoaming, - DODownloadCostPolicy_NoSurcharge, - DODownloadCostPolicy_NoCellular -} DODownloadCostPolicy; - -// https://docs.microsoft.com/en-us/windows/win32/delivery_optimization/deliveryoptimizationdownloadtypes/ne-deliveryoptimizationdownloadtypes-dodownloadproperty -typedef enum _DODownloadProperty -{ - DODownloadProperty_Id = 0, - DODownloadProperty_Uri, - DODownloadProperty_ContentId, - DODownloadProperty_DisplayName, - DODownloadProperty_LocalPath, - DODownloadProperty_HttpCustomHeaders, - DODownloadProperty_CostPolicy, - DODownloadProperty_SecurityFlags, - DODownloadProperty_CallbackFreqPercent, - DODownloadProperty_CallbackFreqSeconds, - DODownloadProperty_NoProgressTimeoutSeconds, - DODownloadProperty_ForegroundPriority, - DODownloadProperty_BlockingMode, - DODownloadProperty_CallbackInterface, - DODownloadProperty_StreamInterface, - DODownloadProperty_SecurityContext, - DODownloadProperty_NetworkToken, - DODownloadProperty_CorrelationVector, - DODownloadProperty_DecryptionInfo, - DODownloadProperty_IntegrityCheckInfo, - DODownloadProperty_IntegrityCheckMandatory, - DODownloadProperty_TotalSizeBytes, - DODownloadProperty_DisallowOnCellular, // Windows 21H2 (Build Number 22000) - DODownloadProperty_HttpCustomAuthHeaders, // Windows 21H2 (Build Number 22000) -} DODownloadProperty; - -// https://docs.microsoft.com/en-us/windows/win32/delivery_optimization/do/ns-do-do_download_enum_category -typedef struct _DO_DOWNLOAD_ENUM_CATEGORY -{ - DODownloadProperty Property; - LPCWSTR Value; -} DO_DOWNLOAD_ENUM_CATEGORY; - -// https://docs.microsoft.com/en-us/windows/win32/delivery_optimization/do/nn-do-idodownload -interface DECLSPEC_UUID("FBBD7FC0-C147-4727-A38D-827EF071EE77") DECLSPEC_NOVTABLE -IDODownload : public IUnknown -{ -public: - IFACEMETHOD(Start)(const DO_DOWNLOAD_RANGES_INFO *ranges) = 0; - IFACEMETHOD(Pause)() = 0; - IFACEMETHOD(Abort)() = 0; - IFACEMETHOD(Finalize)() = 0; - IFACEMETHOD(GetStatus)(DO_DOWNLOAD_STATUS* status) = 0; - IFACEMETHOD(GetProperty)(DODownloadProperty propId, VARIANT* propVal) = 0; - IFACEMETHOD(SetProperty)(DODownloadProperty propId, const VARIANT* propVal) = 0; -}; -DEFINE_GUID(IID_IDODownload, 0xFBBD7FC0, 0xC147, 0x4727, 0xA3, 0x8D, 0x82, 0x7E, 0xF0, 0x71, 0xEE, 0x77); - -// https://docs.microsoft.com/en-us/windows/win32/delivery_optimization/do/nn-do-idodownloadstatuscallback -interface DECLSPEC_UUID("D166E8E3-A90E-4392-8E87-05E996D3747D") DECLSPEC_NOVTABLE -IDODownloadStatusCallback : public IUnknown -{ -public: - IFACEMETHOD(OnStatusChange)(IDODownload* download, const DO_DOWNLOAD_STATUS* status) = 0; -}; -DEFINE_GUID(IID_IDODownloadStatusCallback, 0xD166E8E3, 0xA90E, 0x4392, 0x8E, 0x87, 0x05, 0xE9, 0x96, 0xD3, 0x74, 0x7D); - -// https://docs.microsoft.com/en-us/windows/win32/delivery_optimization/do/nn-do-idomanager -interface DECLSPEC_UUID("400E2D4A-1431-4C1A-A748-39CA472CFDB1") DECLSPEC_NOVTABLE -IDOManager : public IUnknown -{ -public: - IFACEMETHOD(CreateDownload)(IDODownload** download) = 0; - IFACEMETHOD(EnumDownloads)(const DO_DOWNLOAD_ENUM_CATEGORY* category, IEnumUnknown** ppEnum) = 0; -}; -DEFINE_GUID(IID_IDOManager, 0x400E2D4A, 0x1431, 0x4C1A, 0xA7, 0x48, 0x39, 0xCA, 0x47, 0x2C, 0xFD, 0xB1); - -DEFINE_GUID(CLSID_DeliveryOptimization, 0x5b99fa76, 0x721c, 0x423c, 0xad, 0xac, 0x56, 0xd0, 0x3c, 0x8a, 0x80, 0x07); -class DECLSPEC_UUID("5b99fa76-721c-423c-adac-56d03c8a8007") DeliveryOptimization; - -} // extern "C" -#endif // _do_hpp_ diff --git a/sdk-cpp/src/internal/com/do_download_property_internal.cpp b/sdk-cpp/src/internal/com/do_download_property_internal.cpp index 233c5a89..c5cb639e 100644 --- a/sdk-cpp/src/internal/com/do_download_property_internal.cpp +++ b/sdk-cpp/src/internal/com/do_download_property_internal.cpp @@ -19,150 +19,161 @@ namespace deliveryoptimization namespace details { -static std::error_code UTF8toWstr(const char* str, std::wstring& wstr) +static std::error_code UTF8toWstr(const std::string& str, std::wstring& wstr) { - size_t cch = strlen(str); - if (cch == 0) + wstr.clear(); + size_t cch = str.size(); + if (cch != 0) { - wstr = std::wstring(); + std::vector dest(cch * 4); + const int result = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), static_cast(cch), dest.data(), static_cast(dest.size())); + if (result == 0) + { + return make_error_code(HRESULT_FROM_WIN32(::GetLastError())); + } + wstr = std::wstring(dest.data(), static_cast(result)); } + return DO_OK; +} - std::vector dest(cch * 4); - const int result = MultiByteToWideChar(CP_UTF8, 0, str, static_cast(cch), dest.data(), static_cast(dest.size())); - if (result == 0) +static std::error_code WstrToUTF8(const std::wstring& wstr, std::string& str) +{ + str.clear(); + size_t cch = wstr.size(); + if (cch != 0) { - return make_error_code(HRESULT_FROM_WIN32(::GetLastError())); + std::vector dest(cch * 4); + const int result = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), static_cast(cch), dest.data(), static_cast(dest.size()), 0, 0); + if (result == 0) + { + return make_error_code(HRESULT_FROM_WIN32(::GetLastError())); + } + str = std::string(dest.data(), static_cast(result)); } - wstr = std::wstring(dest.data(), static_cast(result)); return DO_OK; } -CDownloadPropertyValueInternal::CDownloadPropertyValueInternal() +unique_variant::unique_variant() { - VariantInit(&_var); + VariantInit(this); } -std::error_code CDownloadPropertyValueInternal::Init(const std::string& val) noexcept +unique_variant::unique_variant(const VARIANT& other) noexcept : + VARIANT(other) { - V_VT(&_var) = VT_BSTR; +} +unique_variant::unique_variant(unique_variant&& other) noexcept : + VARIANT(other) +{ + VariantInit(&other); +} + +unique_variant& unique_variant::operator=(unique_variant&& other) noexcept +{ + VariantClear(this); + VARIANT::operator=(other); + VariantInit(&other); + return *this; +} + +unique_variant::~unique_variant() +{ + VariantClear(this); +} + +std::error_code CDownloadPropertyValueInternal::Init(const std::string& val) noexcept +{ std::wstring wval; - auto hr = UTF8toWstr(val.c_str(), wval); - DO_RETURN_IF_FAILED(hr); + DO_RETURN_IF_FAILED(UTF8toWstr(val, wval)); + return Init(wval); +} - BSTR bstr = SysAllocString(wval.c_str()); +std::error_code CDownloadPropertyValueInternal::Init(const std::wstring& val) noexcept +{ + BSTR bstr = SysAllocString(val.c_str()); if (bstr == nullptr) { return msdo::details::make_error_code(std::errc::not_enough_memory); } - V_BSTR(&_var) = bstr; + V_VT(&_var) = VT_BSTR; + V_BSTR(&_var) = bstr; return DO_OK; -}; +} std::error_code CDownloadPropertyValueInternal::Init(uint32_t val) noexcept { V_VT(&_var) = VT_UI4; V_UI4(&_var) = val; return DO_OK; -}; +} std::error_code CDownloadPropertyValueInternal::Init(uint64_t val) noexcept { V_VT(&_var) = VT_UI8; V_UI8(&_var) = val; return DO_OK; -}; +} std::error_code CDownloadPropertyValueInternal::Init(bool val) noexcept { V_VT(&_var) = VT_BOOL; V_BOOL(&_var) = val ? VARIANT_TRUE : VARIANT_FALSE; return DO_OK; -}; - -std::error_code CDownloadPropertyValueInternal::Init(std::vector& val) noexcept -{ - return make_error_code(errc::e_not_impl); -}; - -std::error_code CDownloadPropertyValueInternal::Init(const download_property_value::status_callback_t& val) noexcept -{ - _callback = val; - return DO_OK; } -CDownloadPropertyValueInternal::~CDownloadPropertyValueInternal() -{ -#ifdef DEBUG - assert(SUCCEEDED(VariantClear(&_var))); -#else - (void)VariantClear(&_var); -#endif -}; - -CDownloadPropertyValueInternal::CDownloadPropertyValueInternal(const CDownloadPropertyValueInternal& rhs) -{ - HRESULT res = VariantCopy(&_var, &rhs._var); -#if DEBUG - assert(SUCCEEDED(res)); -#endif - if (FAILED(res)) - { - std::terminate(); - } - _callback = rhs._callback; -}; - -CDownloadPropertyValueInternal& CDownloadPropertyValueInternal::operator=(CDownloadPropertyValueInternal copy) -{ - swap(*this, copy); - return *this; -}; - -CDownloadPropertyValueInternal::CDownloadPropertyValueInternal(CDownloadPropertyValueInternal&& rhs) noexcept -{ - _var = rhs._var; - rhs._var = {}; - V_VT(&rhs._var) = VT_EMPTY; - _callback = std::move(rhs._callback); -}; - -const CDownloadPropertyValueInternal::native_type& CDownloadPropertyValueInternal::native_value() const noexcept -{ - return _var; -}; - std::error_code CDownloadPropertyValueInternal::As(bool& val) const noexcept { - return make_error_code(errc::e_not_impl); -}; + val = false; + unique_variant v2; + RETURN_IF_FAILED(VariantChangeType(&v2, &_var, 0, VT_BOOL)); + val = (V_BOOL(&v2) != VARIANT_FALSE); + return DO_OK; +} std::error_code CDownloadPropertyValueInternal::As(uint32_t& val) const noexcept { - return make_error_code(errc::e_not_impl); -}; + val = 0; + unique_variant v2; + RETURN_IF_FAILED(VariantChangeType(&v2, &_var, 0, VT_UI4)); + val = V_UI4(&v2); + return DO_OK; +} std::error_code CDownloadPropertyValueInternal::As(uint64_t& val) const noexcept { - return make_error_code(errc::e_not_impl); -}; + val = 0; + unique_variant v2; + RETURN_IF_FAILED(VariantChangeType(&v2, &_var, 0, VT_UI8)); + val = V_UI8(&v2); + return DO_OK; +} std::error_code CDownloadPropertyValueInternal::As(std::string& val) const noexcept { - return make_error_code(errc::e_not_impl); -}; - -std::error_code CDownloadPropertyValueInternal::As(std::vector& val) const noexcept -{ - return make_error_code(errc::e_not_impl); + val.clear(); + std::wstring wstr; + DO_RETURN_IF_FAILED(As(wstr)); + DO_RETURN_IF_FAILED(WstrToUTF8(wstr, val)); + return DO_OK; } -std::error_code CDownloadPropertyValueInternal::As(download_property_value::status_callback_t& val) const noexcept +std::error_code CDownloadPropertyValueInternal::As(std::wstring& val) const noexcept { - val = _callback; + val.clear(); + if (V_VT(&_var) == VT_BSTR) + { + val = V_BSTR(&_var); // avoid the extra string copy from VariantChangeType + } + else + { + unique_variant v2; + RETURN_IF_FAILED(VariantChangeType(&v2, &_var, 0, VT_BSTR)); + val = V_BSTR(&v2); + } return DO_OK; -}; +} } // namespace details } // namespace deliveryoptimization diff --git a/sdk-cpp/src/internal/com/download_impl.cpp b/sdk-cpp/src/internal/com/download_impl.cpp index b06e8f35..fb4421d8 100644 --- a/sdk-cpp/src/internal/com/download_impl.cpp +++ b/sdk-cpp/src/internal/com/download_impl.cpp @@ -3,7 +3,6 @@ #include "download_impl.h" -#include #include #include #include @@ -16,16 +15,6 @@ namespace msdo = microsoft::deliveryoptimization; using namespace Microsoft::WRL; -#ifndef FAILED -#define FAILED(hr) (((int32_t)(hr)) < 0) -#endif - -#ifndef RETURN_IF_FAILED -#define RETURN_IF_FAILED(hr) { \ - int32_t __hr = (hr); \ - if (FAILED(__hr)) return std::error_code(__hr, msdo::details::do_category()); } -#endif - namespace microsoft { namespace deliveryoptimization @@ -35,41 +24,17 @@ namespace details static msdo::download_state ConvertFromComState(DODownloadState platformState) { - msdo::download_state state; - switch (platformState) - { - case DODownloadState_Created: - { - state = msdo::download_state::created; - break; - } - case DODownloadState_Transferring: - { - state = msdo::download_state::transferring; - break; - } - case DODownloadState_Transferred: - { - state = msdo::download_state::transferred; - break; - } - case DODownloadState_Finalized: - { - state = msdo::download_state::finalized; - break; - } - case DODownloadState_Aborted: - { - state = msdo::download_state::aborted; - break; - } - case DODownloadState_Paused: - { - state = msdo::download_state::paused; - break; - } - } - return state; + static const msdo::download_state c_stateMap[] = + { + msdo::download_state::created, // DODownloadState_Created + msdo::download_state::transferring, // DODownloadState_Transferring + msdo::download_state::transferred, // DODownloadState_Transferred + msdo::download_state::finalized, // DODownloadState_Finalized + msdo::download_state::aborted, // DODownloadState_Aborted + msdo::download_state::paused, // DODownloadState_Paused + }; + auto index = static_cast(platformState); + return (index < ARRAYSIZE(c_stateMap)) ? c_stateMap[index] : msdo::download_state::paused; } static msdo::download_status ConvertFromComStatus(const DO_DOWNLOAD_STATUS& platformStatus) @@ -78,127 +43,48 @@ static msdo::download_status ConvertFromComStatus(const DO_DOWNLOAD_STATUS& plat platformStatus.ExtendedError, ConvertFromComState(platformStatus.State)); } -static std::error_code ConvertToComProperty(msdo::download_property key, DODownloadProperty& comProperty) -{ - switch (key) - { - case msdo::download_property::blocking_mode: - { - comProperty = DODownloadProperty_BlockingMode; - return DO_OK; - } - case msdo::download_property::callback_interface: - { - comProperty = DODownloadProperty_CallbackInterface; - return DO_OK; - } - case msdo::download_property::disallow_on_cellular: - { - comProperty = DODownloadProperty_DisallowOnCellular; - return DO_OK; - } - case msdo::download_property::caller_name: - { - comProperty = DODownloadProperty_DisplayName; - return DO_OK; - } - case msdo::download_property::catalog_id: - { - comProperty = DODownloadProperty_ContentId; - return DO_OK; - } - case msdo::download_property::correlation_vector: - { - comProperty = DODownloadProperty_CorrelationVector; - return DO_OK; - } - case msdo::download_property::cost_policy: - { - comProperty = DODownloadProperty_CostPolicy; - return DO_OK; - } - case msdo::download_property::decryption_info: - { - comProperty = DODownloadProperty_DecryptionInfo; - return DO_OK; - } - case msdo::download_property::download_file_path: - { - comProperty = DODownloadProperty_LocalPath; - return DO_OK; - } - case msdo::download_property::http_custom_auth_headers: - { - comProperty = DODownloadProperty::DODownloadProperty_HttpCustomAuthHeaders; - return DO_OK; - } - case msdo::download_property::http_custom_headers: - { - comProperty = DODownloadProperty_HttpCustomHeaders; - return DO_OK; - } - case msdo::download_property::id: - { - comProperty = DODownloadProperty_Id; - return DO_OK; - } - case msdo::download_property::integrity_check_info: - { - comProperty = DODownloadProperty_IntegrityCheckInfo; - return DO_OK; - } - case msdo::download_property::integrity_check_mandatory: - { - comProperty = DODownloadProperty_IntegrityCheckMandatory; - return DO_OK; - } - case msdo::download_property::network_token: - { - comProperty = DODownloadProperty_NetworkToken; - return DO_OK; - } - case msdo::download_property::no_progress_timeout_seconds: - { - comProperty = DODownloadProperty_NoProgressTimeoutSeconds; - return DO_OK; - } - case msdo::download_property::stream_interface: - { - comProperty = DODownloadProperty_StreamInterface; - return DO_OK; - } - case msdo::download_property::security_context: - { - comProperty = DODownloadProperty_SecurityContext; - return DO_OK; - } - case msdo::download_property::total_size_bytes: - { - comProperty = DODownloadProperty_TotalSizeBytes; - return DO_OK; - } - case msdo::download_property::uri: - { - comProperty = DODownloadProperty_Uri; - return DO_OK; - } - case msdo::download_property::use_foreground_priority: - { - comProperty = DODownloadProperty_ForegroundPriority; - return DO_OK; - } - default: - { - return make_error_code(E_INVALIDARG); - } - } +static std::error_code ConvertToComProperty(msdo::download_property prop, DODownloadProperty& comProperty) +{ + static const DODownloadProperty c_propMap[] = + { + DODownloadProperty_Id, // id + DODownloadProperty_Uri, // uri + DODownloadProperty_ContentId, // catalog_id + DODownloadProperty_DisplayName, // caller_name + DODownloadProperty_LocalPath, // download_file_path + DODownloadProperty_HttpCustomHeaders, // http_custom_headers + DODownloadProperty_CostPolicy, // cost_policy + DODownloadProperty_SecurityFlags, // security_flags + DODownloadProperty_CallbackFreqPercent, // callback_freq_percent + DODownloadProperty_CallbackFreqSeconds, // callback_freq_seconds + DODownloadProperty_NoProgressTimeoutSeconds, // no_progress_timeout_seconds + DODownloadProperty_ForegroundPriority, // use_foreground_priority + DODownloadProperty_BlockingMode, // blocking_mode + DODownloadProperty_NetworkToken, // network_token + DODownloadProperty_CorrelationVector, // correlation_vector + DODownloadProperty_DecryptionInfo, // decryption_info + DODownloadProperty_IntegrityCheckInfo, // integrity_check_info + DODownloadProperty_IntegrityCheckMandatory, // integrity_check_mandatory + DODownloadProperty_TotalSizeBytes, // total_size_bytes + DODownloadProperty_DisallowOnCellular, // disallow_on_cellular + DODownloadProperty_HttpCustomAuthHeaders, // http_custom_auth_headers + DODownloadProperty_HttpAllowSecureToNonSecureRedirect, // allow_http_to_https_redirect + DODownloadProperty_NonVolatile, // non_volatile + }; + auto index = static_cast(prop); + if (index >= ARRAYSIZE(c_propMap)) + { + return make_error_code(errc::invalid_arg); + } + comProperty = c_propMap[index]; + return DO_OK; } class DOStatusCallback : public RuntimeClass, IDODownloadStatusCallback> { public: - HRESULT RuntimeClassInitialize(const msdo::download_property_value::status_callback_t& callback, msdo::download& download) + HRESULT RuntimeClassInitialize(const msdo::status_callback_t& callback, msdo::download& download) { _download = &download; _callback = callback; @@ -213,7 +99,7 @@ class DOStatusCallback : } private: - msdo::download_property_value::status_callback_t _callback; + msdo::status_callback_t _callback; msdo::download* _download; }; @@ -228,16 +114,17 @@ std::error_code CDownloadImpl::Init(const std::string& uri, const std::string& d RPC_C_AUTHZ_NONE, COLE_DEFAULT_PRINCIPAL, RPC_C_AUTHN_LEVEL_DEFAULT, RPC_C_IMP_LEVEL_IMPERSONATE, nullptr, EOAC_STATIC_CLOAKING)); + _spDownload = std::move(spDownload); + download_property_value propUri; download_property_value propDownloadFilePath; DO_RETURN_IF_FAILED(download_property_value::make(uri, propUri)); DO_RETURN_IF_FAILED(download_property_value::make(downloadFilePath, propDownloadFilePath)); - DO_RETURN_IF_FAILED(_SetPropertyHelper(*spDownload.Get(), download_property::uri, propUri)); - DO_RETURN_IF_FAILED(_SetPropertyHelper(*spDownload.Get(), download_property::download_file_path, propDownloadFilePath)); + DO_RETURN_IF_FAILED(SetProperty(download_property::uri, propUri)); + DO_RETURN_IF_FAILED(SetProperty(download_property::download_file_path, propDownloadFilePath)); - _spDownload = std::move(spDownload); - return DO_OK;; + return DO_OK; } // Support only full file downloads for now @@ -276,45 +163,56 @@ std::error_code CDownloadImpl::GetStatus(msdo::download_status& status) noexcept return DO_OK; } -std::error_code CDownloadImpl::GetProperty(msdo::download_property key, msdo::download_property_value& value) noexcept -{ - return _GetPropertyHelper(key, value); -} - -std::error_code CDownloadImpl::SetProperty(msdo::download_property key, const msdo::download_property_value& val) noexcept -{ - assert(key != msdo::download_property::callback_interface); - return _SetPropertyHelper(*_spDownload.Get(), key, val); -} - -std::error_code CDownloadImpl::SetCallback(const download_property_value::status_callback_t& callback, download& download) noexcept +std::error_code CDownloadImpl::SetStatusCallback(const msdo::status_callback_t& callback, msdo::download& download) noexcept { Microsoft::WRL::ComPtr spCallback; RETURN_IF_FAILED(MakeAndInitialize(&spCallback, callback, download)); - VARIANT vtCallback; - VariantInit(&vtCallback); + unique_variant vtCallback; V_VT(&vtCallback) = VT_UNKNOWN; - V_UNKNOWN(&vtCallback) = spCallback.Get(); - spCallback.Get()->AddRef(); - DODownloadProperty prop; - ConvertToComProperty(msdo::download_property::callback_interface, prop); - const auto hr = _spDownload->SetProperty(prop, &vtCallback); - VariantClear(&vtCallback); - return make_error_code(hr); + V_UNKNOWN(&vtCallback) = spCallback.Detach(); + RETURN_IF_FAILED(_spDownload->SetProperty(DODownloadProperty_CallbackInterface, &vtCallback)); + return DO_OK; } -std::error_code CDownloadImpl::_SetPropertyHelper(IDODownload& download, msdo::download_property key, const msdo::download_property_value& val) noexcept +std::error_code CDownloadImpl::SetProperty(msdo::download_property key, const msdo::download_property_value& val) noexcept { DODownloadProperty prop; DO_RETURN_IF_FAILED(ConvertToComProperty(key, prop)); - - return make_error_code(download.SetProperty(prop, &val._val->native_value())); + RETURN_IF_FAILED(_spDownload->SetProperty(prop, &val._val->native_value())); + return DO_OK; } -std::error_code CDownloadImpl::_GetPropertyHelper(msdo::download_property key, msdo::download_property_value& value) noexcept +std::error_code CDownloadImpl::GetProperty(msdo::download_property key, msdo::download_property_value& value) noexcept { - return make_error_code(errc::e_not_impl); + DODownloadProperty prop; + DO_RETURN_IF_FAILED(ConvertToComProperty(key, prop)); + + unique_variant var; + RETURN_IF_FAILED(_spDownload->GetProperty(prop, &var)); + + switch (V_VT(&var)) + { + case VT_BOOL: + DO_RETURN_IF_FAILED(download_property_value::make(var.boolVal != VARIANT_FALSE, value)); + break; + + case VT_UI4: + DO_RETURN_IF_FAILED(download_property_value::make(var.uintVal, value)); + break; + + case VT_UI8: + DO_RETURN_IF_FAILED(download_property_value::make(var.ullVal, value)); + break; + + case VT_BSTR: + DO_RETURN_IF_FAILED(download_property_value::make(var.bstrVal, value)); + break; + + default: + return make_error_code(E_UNEXPECTED); + } + return DO_OK; } } // namespace details diff --git a/sdk-cpp/src/internal/do_download_property_internal.h b/sdk-cpp/src/internal/do_download_property_internal.h index e7959451..20fd1b92 100644 --- a/sdk-cpp/src/internal/do_download_property_internal.h +++ b/sdk-cpp/src/internal/do_download_property_internal.h @@ -10,7 +10,6 @@ #include #endif -#include #include #include @@ -24,47 +23,49 @@ namespace deliveryoptimization namespace details { +#if defined(DO_INTERFACE_COM) +struct unique_variant : VARIANT +{ + unique_variant(); + explicit unique_variant(const VARIANT& other) noexcept; // takes ownership via shallow copy + unique_variant(unique_variant&& other) noexcept; + unique_variant& operator=(unique_variant&& other) noexcept; + ~unique_variant(); + + unique_variant(const unique_variant& other) = delete; + unique_variant& operator=(const unique_variant&) = delete; + unique_variant& operator=(const VARIANT&) = delete; +}; +#endif + class CDownloadPropertyValueInternal { public: #if defined(DO_INTERFACE_COM) - using native_type = VARIANT; + using native_type = unique_variant; #else - using native_type = boost::variant>; + using native_type = boost::variant; #endif - CDownloadPropertyValueInternal(); + + CDownloadPropertyValueInternal() = default; + ~CDownloadPropertyValueInternal() = default; std::error_code Init(const std::string& val) noexcept; + std::error_code Init(const std::wstring& val) noexcept; std::error_code Init(uint32_t val) noexcept; std::error_code Init(uint64_t val) noexcept; std::error_code Init(bool val) noexcept; - std::error_code Init(std::vector& val) noexcept; - std::error_code Init(const download_property_value::status_callback_t& val) noexcept; - - ~CDownloadPropertyValueInternal(); - - CDownloadPropertyValueInternal(const CDownloadPropertyValueInternal& rhs); - CDownloadPropertyValueInternal& operator=(CDownloadPropertyValueInternal copy); - CDownloadPropertyValueInternal(CDownloadPropertyValueInternal&& rhs) noexcept; - - friend void swap(CDownloadPropertyValueInternal& first, CDownloadPropertyValueInternal& second) noexcept - { - std::swap(first._var, second._var); - std::swap(first._callback, second._callback); - } std::error_code As(bool& val) const noexcept; std::error_code As(uint32_t& val) const noexcept; std::error_code As(uint64_t& val) const noexcept; std::error_code As(std::string& val) const noexcept; - std::error_code As(download_property_value::status_callback_t& val) const noexcept; - std::error_code As(std::vector& val) const noexcept; + std::error_code As(std::wstring& val) const noexcept; - const native_type& native_value() const noexcept; + const native_type& native_value() const noexcept { return _var; } private: native_type _var; - download_property_value::status_callback_t _callback; }; } // namespace details diff --git a/sdk-cpp/src/internal/do_error_helpers.h b/sdk-cpp/src/internal/do_error_helpers.h index db5055fb..bf4307d9 100644 --- a/sdk-cpp/src/internal/do_error_helpers.h +++ b/sdk-cpp/src/internal/do_error_helpers.h @@ -53,6 +53,16 @@ inline std::error_code make_error_code(errc e) return std::error_code(static_cast(e), do_category()); } +#ifndef FAILED +#define FAILED(hr) (((int32_t)(hr)) < 0) +#endif + +#ifndef RETURN_IF_FAILED +#define RETURN_IF_FAILED(hr) { \ + int32_t __hr = (hr); \ + if (FAILED(__hr)) return std::error_code(__hr, do_category()); } +#endif + #ifdef DO_ENABLE_EXCEPTIONS class exception : public std::exception diff --git a/sdk-cpp/src/internal/download_impl.h b/sdk-cpp/src/internal/download_impl.h index c1858ffb..ab058d1c 100644 --- a/sdk-cpp/src/internal/download_impl.h +++ b/sdk-cpp/src/internal/download_impl.h @@ -14,7 +14,8 @@ #if defined(DO_INTERFACE_COM) #include -#include "do.hpp" // Fwd declaration of IDODownload doesn't work well w/ all build systems +// Future: delete the local copy of deliveryoptimization.h and require Windows SDK 22621+ +#include // IDODownload, etc. #endif namespace microsoft @@ -37,16 +38,12 @@ class CDownloadImpl : public IDownload std::error_code Abort() noexcept override; std::error_code GetStatus(download_status& status) noexcept override; + std::error_code SetStatusCallback(const status_callback_t& callback, download& download) noexcept override; std::error_code GetProperty(download_property key, download_property_value& value) noexcept override; std::error_code SetProperty(download_property key, const download_property_value& val) noexcept override; - std::error_code SetCallback(const download_property_value::status_callback_t& callback, download& download) noexcept override; private: - #if defined(DO_INTERFACE_COM) - static std::error_code _SetPropertyHelper(IDODownload& download, download_property key, const download_property_value& val) noexcept; - std::error_code _GetPropertyHelper(download_property key, download_property_value& value) noexcept; - Microsoft::WRL::ComPtr _spDownload; #elif defined(DO_INTERFACE_REST) std::error_code _DownloadOperationCall(const std::string& type) noexcept; diff --git a/sdk-cpp/src/internal/download_interface.h b/sdk-cpp/src/internal/download_interface.h index d00a4eeb..4f661fec 100644 --- a/sdk-cpp/src/internal/download_interface.h +++ b/sdk-cpp/src/internal/download_interface.h @@ -5,7 +5,6 @@ #define _DELIVERY_OPTIMIZATION_DOWNLOAD_INTERFACE_H #include "do_download_status.h" - #include "do_download_property.h" class download; @@ -31,10 +30,10 @@ class IDownload virtual std::error_code Abort() noexcept = 0; virtual std::error_code GetStatus(download_status& status) noexcept = 0; + virtual std::error_code SetStatusCallback(const status_callback_t& callback, download& download) noexcept = 0; virtual std::error_code GetProperty(download_property key, download_property_value& value) noexcept = 0; virtual std::error_code SetProperty(download_property key, const download_property_value& val) noexcept = 0; - virtual std::error_code SetCallback(const download_property_value::status_callback_t& callback, download& download) noexcept = 0; }; } // namespace details } // namespace deliveryoptimization diff --git a/sdk-cpp/src/internal/rest/do_download_property_internal.cpp b/sdk-cpp/src/internal/rest/do_download_property_internal.cpp index b53d2716..2df79aca 100644 --- a/sdk-cpp/src/internal/rest/do_download_property_internal.cpp +++ b/sdk-cpp/src/internal/rest/do_download_property_internal.cpp @@ -5,7 +5,6 @@ #include #include -#include #include #include "do_errors.h" @@ -18,66 +17,31 @@ namespace deliveryoptimization namespace details { -CDownloadPropertyValueInternal::CDownloadPropertyValueInternal() = default; - std::error_code CDownloadPropertyValueInternal::Init(const std::string& val) noexcept { return make_error_code(errc::e_not_impl); } -std::error_code CDownloadPropertyValueInternal::Init(uint32_t val) noexcept -{ - return make_error_code(errc::e_not_impl); -} - -std::error_code CDownloadPropertyValueInternal::Init(uint64_t val) noexcept +std::error_code CDownloadPropertyValueInternal::Init(const std::wstring& val) noexcept { return make_error_code(errc::e_not_impl); } -std::error_code CDownloadPropertyValueInternal::Init(bool val) noexcept +std::error_code CDownloadPropertyValueInternal::Init(uint32_t val) noexcept { return make_error_code(errc::e_not_impl); } -std::error_code CDownloadPropertyValueInternal::Init(std::vector& val) noexcept +std::error_code CDownloadPropertyValueInternal::Init(uint64_t val) noexcept { return make_error_code(errc::e_not_impl); } -std::error_code CDownloadPropertyValueInternal::Init(const download_property_value::status_callback_t& val) noexcept +std::error_code CDownloadPropertyValueInternal::Init(bool val) noexcept { return make_error_code(errc::e_not_impl); } -CDownloadPropertyValueInternal::~CDownloadPropertyValueInternal() -{ -} - -CDownloadPropertyValueInternal::CDownloadPropertyValueInternal(const CDownloadPropertyValueInternal& rhs) -{ - _var = rhs._var; - _callback = rhs._callback; -} - -CDownloadPropertyValueInternal& CDownloadPropertyValueInternal::operator=(CDownloadPropertyValueInternal copy) -{ - swap(*this, copy); - return *this; -} - -CDownloadPropertyValueInternal::CDownloadPropertyValueInternal(CDownloadPropertyValueInternal&& rhs) noexcept -{ - _var = rhs._var; - rhs._var = {}; - _callback = std::move(rhs._callback); -} - -const CDownloadPropertyValueInternal::native_type& CDownloadPropertyValueInternal::native_value() const noexcept -{ - return _var; -} - std::error_code CDownloadPropertyValueInternal::As(bool& val) const noexcept { return make_error_code(errc::e_not_impl); @@ -98,17 +62,11 @@ std::error_code CDownloadPropertyValueInternal::As(std::string& val) const noexc return make_error_code(errc::e_not_impl); } -std::error_code CDownloadPropertyValueInternal::As(std::vector& val) const noexcept +std::error_code CDownloadPropertyValueInternal::As(std::wstring& val) const noexcept { return make_error_code(errc::e_not_impl); } -std::error_code CDownloadPropertyValueInternal::As(download_property_value::status_callback_t& val) const noexcept -{ - val = _callback; - return DO_OK; -} - } // namespace details } // namespace deliveryoptimization } // namespace microsoft diff --git a/sdk-cpp/src/internal/rest/download_impl.cpp b/sdk-cpp/src/internal/rest/download_impl.cpp index 41b726b0..786815ed 100644 --- a/sdk-cpp/src/internal/rest/download_impl.cpp +++ b/sdk-cpp/src/internal/rest/download_impl.cpp @@ -142,7 +142,7 @@ std::error_code CDownloadImpl::SetProperty(msdo::download_property key, const ms return make_error_code(msdo::errc::e_not_impl); } -std::error_code CDownloadImpl::SetCallback(const download_property_value::status_callback_t& callback, download& download) noexcept +std::error_code CDownloadImpl::SetStatusCallback(const status_callback_t& callback, download& download) noexcept { return make_error_code(msdo::errc::e_not_impl); } diff --git a/sdk-cpp/tests/download_properties_tests.cpp b/sdk-cpp/tests/download_properties_tests.cpp index 3afe3df2..37788af1 100644 --- a/sdk-cpp/tests/download_properties_tests.cpp +++ b/sdk-cpp/tests/download_properties_tests.cpp @@ -5,7 +5,6 @@ #include #include -#include #include #include "do_download.h" @@ -14,6 +13,7 @@ #include "test_helpers.h" namespace msdo = microsoft::deliveryoptimization; +namespace msdot = microsoft::deliveryoptimization::test; using namespace std::chrono_literals; class DownloadPropertyTests : public ::testing::Test @@ -36,12 +36,6 @@ class DownloadPropertyTests : public ::testing::Test { ASSERT_TRUE(std::find(expectedErrors.begin(), expectedErrors.end(), code) != expectedErrors.end()); } - - static void VerifyCallWithExpectedErrors(const std::function& op, const std::vector& expectedErrors) - { - auto ec = op(); - VerifyError(ec.value(), expectedErrors); - } #endif }; @@ -56,68 +50,35 @@ static uint32_t TimeOperation(const std::function& op) return static_cast(std::chrono::duration_cast(end - start).count()); } -static std::unique_ptr g_MakeDownload(const std::string& url, const std::string& destPath) -{ - std::unique_ptr downloadObj; - auto ec = msdo::download::make(url, destPath, downloadObj); - if (ec) throw std::exception(); - - // msdo::download_property_value propUri; - // auto ec = msdo::download_property_value::make(url, propUri); - // if (ec) throw std::exception(); - // ec = obj.set_property(msdo::download_property::uri, propUri); - // if (ec) throw std::exception(); - - // msdo::download_property_value propFilePath; - // ec = msdo::download_property_value::make(destPath, propFilePath); - // ec = obj.set_property(msdo::download_property::uri, propFilePath); - // if (ec) throw std::exception(); - - return downloadObj; -} - -template -static msdo::download_property_value g_MakePropertyValue(T value) -{ - msdo::download_property_value propValue; - auto ec = msdo::download_property_value::make(value, propValue); - if (ec) - { - throw std::exception(); - } - return propValue; -} - -//TODO: Not sure how much value these tests are, functional tests utilize parsing log lines to verify these properties were set, could be useful here -//At the moment, these tests are essentially just verifying that these properties can be set and download succeeds TEST_F(DownloadPropertyTests, SmallDownloadSetCallerNameTest) { - auto simpleDownload = g_MakeDownload(g_smallFileUrl, g_tmpFileName); + auto simpleDownload = msdot::download::make(g_smallFileUrl, g_tmpFileName); std::string strCallerName("dosdkcpp_tests"); - msdo::download_property_value callerName = g_MakePropertyValue(strCallerName); - ASSERT_EQ(simpleDownload->set_property(msdo::download_property::caller_name, callerName).value(), 0); - ASSERT_EQ(simpleDownload->start_and_wait_until_completion().value(), 0); + simpleDownload->set_property(msdo::download_property::caller_name, strCallerName); + + std::string outCallerName; + simpleDownload->get_property(msdo::download_property::caller_name, outCallerName); + ASSERT_EQ(strCallerName, outCallerName); + + simpleDownload->start_and_wait_until_completion(); ASSERT_TRUE(boost::filesystem::exists(g_tmpFileName)); } TEST_F(DownloadPropertyTests, SmallDownloadWithPhfDigestandCvTest) { - auto simpleDownload = g_MakeDownload(g_smallFileUrl, g_tmpFileName); + std::unique_ptr simpleDownload; + ASSERT_EQ(msdo::download::make(g_smallFileUrl, g_tmpFileName, simpleDownload).value(), 0); std::vector expectedErrors = { 0, static_cast(msdo::errc::do_e_unknown_property_id) }; - msdo::download_property_value integrityCheckMandatory = g_MakePropertyValue(false); - int32_t code = simpleDownload->set_property(msdo::download_property::integrity_check_mandatory, integrityCheckMandatory).value(); + int32_t code = simpleDownload->set_property(msdo::download_property::integrity_check_mandatory, false).value(); VerifyError(code, expectedErrors); - msdo::download_property_value integrityCheckInfo = g_MakePropertyValue(g_smallFilePhfInfoJson); - code = simpleDownload->set_property(msdo::download_property::integrity_check_info, integrityCheckInfo).value(); + code = simpleDownload->set_property(msdo::download_property::integrity_check_info, g_smallFilePhfInfoJson).value(); VerifyError(code, expectedErrors); - std::string strCorrelationVector("g+Vo71JZwkmJdYfF.0"); - msdo::download_property_value correlationVector = g_MakePropertyValue(strCorrelationVector); - code = simpleDownload->set_property(msdo::download_property::correlation_vector, correlationVector).value(); + code = simpleDownload->set_property(msdo::download_property::correlation_vector, "g+Vo71JZwkmJdYfF.0").value(); VerifyError(code, expectedErrors); ASSERT_EQ(simpleDownload->start_and_wait_until_completion().value(), 0); @@ -127,36 +88,29 @@ TEST_F(DownloadPropertyTests, SmallDownloadWithPhfDigestandCvTest) TEST_F(DownloadPropertyTests, InvalidPhfDigestTest) { - auto simpleDownload = g_MakeDownload(g_smallFileUrl, g_tmpFileName); + std::unique_ptr simpleDownload; + ASSERT_EQ(msdo::download::make(g_smallFileUrl, g_tmpFileName, simpleDownload).value(), 0); - msdo::download_property_value integrityCheckInfo = g_MakePropertyValue("blah"); std::vector expectedErrors = { 0, static_cast(msdo::errc::invalid_arg) }; - VerifyCallWithExpectedErrors([&]() - { - return simpleDownload->set_property(msdo::download_property::integrity_check_info, integrityCheckInfo); - }, expectedErrors); + + int32_t code = simpleDownload->set_property(msdo::download_property::integrity_check_info, "blah").value(); + VerifyError(code, expectedErrors); } -// For some reason, custom headers are getting rejected and returning E_INVALIDARG now, disabling test for now -TEST_F(DownloadPropertyTests, DISABLED_SmallDownloadWithCustomHeaders) +TEST_F(DownloadPropertyTests, SmallDownloadWithCustomHeaders) { - auto simpleDownload = g_MakeDownload(g_smallFileUrl, g_tmpFileName); - - std::string strHttpCustomHeaders("XCustom1=someData\nXCustom2=moreData"); - msdo::download_property_value httpCustomHeaders = g_MakePropertyValue(strHttpCustomHeaders); - simpleDownload->set_property(msdo::download_property::http_custom_headers, httpCustomHeaders); - + auto simpleDownload = msdot::download::make(g_smallFileUrl, g_tmpFileName); + simpleDownload->set_property(msdo::download_property::http_custom_headers, "XCustom1=someData\nXCustom2=moreData\n"); simpleDownload->start_and_wait_until_completion(); - ASSERT_TRUE(boost::filesystem::exists(g_tmpFileName)); } TEST_F(DownloadPropertyTests, CallbackTestUseDownload) { - auto simpleDownload = g_MakeDownload(g_largeFileUrl, g_tmpFileName); - bool fPauseDownload = false; + auto simpleDownload = msdot::download::make(g_largeFileUrl, g_tmpFileName); - msdo::download_property_value callback = g_MakePropertyValue([&fPauseDownload](msdo::download& download, msdo::download_status& status) + bool fPauseDownload = false; + auto callback = [&fPauseDownload](msdo::download& download, msdo::download_status& status) { char msgBuf[1024]; snprintf(msgBuf, sizeof(msgBuf), "Received status callback: %" PRIu64 "/%" PRIu64 ", 0x%x, 0x%x, %u", @@ -170,23 +124,22 @@ TEST_F(DownloadPropertyTests, CallbackTestUseDownload) { ASSERT_EQ(download.pause().value(), 0); } - }); + }; - ASSERT_EQ(simpleDownload->set_property(msdo::download_property::callback_interface, callback).value(), 0); - ASSERT_EQ(simpleDownload->start().value(), 0); + simpleDownload->set_status_callback(callback); + simpleDownload->start(); std::this_thread::sleep_for(5s); fPauseDownload = true; - microsoft::deliveryoptimization::test::download testDownload{std::move(simpleDownload)}; - TestHelpers::WaitForState(testDownload, msdo::download_state::paused); + TestHelpers::WaitForState(*simpleDownload, msdo::download_state::paused); } TEST_F(DownloadPropertyTests, SetCallbackTest) { - auto simpleDownload = g_MakeDownload(g_smallFileUrl, g_tmpFileName); + auto simpleDownload = msdot::download::make(g_smallFileUrl, g_tmpFileName); int i= 0; - msdo::download_property_value callback = g_MakePropertyValue([&i](msdo::download&, msdo::download_status& status) + auto callback = [&i](msdo::download&, msdo::download_status& status) { char msgBuf[1024]; snprintf(msgBuf, sizeof(msgBuf), "Received status callback: %" PRIu64 "/%" PRIu64 ", 0x%x, 0x%x, %u", @@ -194,32 +147,21 @@ TEST_F(DownloadPropertyTests, SetCallbackTest) static_cast(status.state())); std::cout << msgBuf << std::endl; i += 1; - }); - ASSERT_EQ(simpleDownload->set_property(msdo::download_property::callback_interface, callback).value(), 0); + }; + simpleDownload->set_status_callback(callback); - ASSERT_EQ(simpleDownload->start_and_wait_until_completion().value(), 0); + simpleDownload->start_and_wait_until_completion(); ASSERT_GE(i, 0); } TEST_F(DownloadPropertyTests, OverrideCallbackTest) { - auto simpleDownload = g_MakeDownload(g_smallFileUrl, g_tmpFileName); - + auto simpleDownload = msdot::download::make(g_smallFileUrl, g_tmpFileName); int i = 0; - msdo::download_property_value callback = g_MakePropertyValue([&i](msdo::download&, msdo::download_status&) - { - i += 1; - }); - ASSERT_EQ(simpleDownload->set_property(msdo::download_property::callback_interface, callback).value(), 0); - - msdo::download_property_value::status_callback_t cb2 = [](msdo::download&, msdo::download_status&) {}; - - callback = g_MakePropertyValue(cb2); - ASSERT_EQ(simpleDownload->set_property(msdo::download_property::callback_interface, callback).value(), 0); - - ASSERT_EQ(simpleDownload->start_and_wait_until_completion().value(), 0); - + simpleDownload->set_status_callback([&i](msdo::download&, msdo::download_status&){ i += 1; }); + simpleDownload->set_status_callback([](msdo::download&, msdo::download_status&){}); + simpleDownload->start_and_wait_until_completion(); ASSERT_EQ(i, 0); } @@ -227,23 +169,29 @@ TEST_F(DownloadPropertyTests, ForegroundBackgroundRace) { uint32_t backgroundDuration = TimeOperation([&]() { - auto simpleDownload = g_MakeDownload(g_largeFileUrl, g_tmpFileName); + auto simpleDownload = msdot::download::make(g_largeFileUrl, g_tmpFileName); + + simpleDownload->set_property(msdo::download_property::use_foreground_priority, false); - msdo::download_property_value foregroundPriority = g_MakePropertyValue(false); - ASSERT_EQ(simpleDownload->set_property(msdo::download_property::use_foreground_priority, foregroundPriority).value(), 0); + bool outVal = true; + simpleDownload->get_property(msdo::download_property::use_foreground_priority, outVal); + ASSERT_FALSE(outVal); - ASSERT_EQ(simpleDownload->start_and_wait_until_completion().value(), 0); + simpleDownload->start_and_wait_until_completion(); }); printf("Time for background download: %u ms\n", backgroundDuration); uint32_t foregroundDuration = TimeOperation([&]() { - auto simpleDownload = g_MakeDownload(g_largeFileUrl, g_tmpFileName2); + auto simpleDownload = msdot::download::make(g_largeFileUrl, g_tmpFileName2); - msdo::download_property_value foregroundPriority = g_MakePropertyValue(true); - ASSERT_EQ(simpleDownload->set_property(msdo::download_property::use_foreground_priority, foregroundPriority).value(), 0); + simpleDownload->set_property(msdo::download_property::use_foreground_priority, true); - ASSERT_EQ(simpleDownload->start_and_wait_until_completion().value(), 0); + bool outVal = false; + simpleDownload->get_property(msdo::download_property::use_foreground_priority, outVal); + ASSERT_TRUE(outVal); + + simpleDownload->start_and_wait_until_completion(); }); printf("Time for foreground download: %u ms\n", foregroundDuration); @@ -254,9 +202,8 @@ TEST_F(DownloadPropertyTests, ForegroundBackgroundRace) TEST_F(DownloadPropertyTests, SmallDownloadSetCallerNameFailureTest) { - std::string strCallerName("dosdkcpp_tests"); msdo::download_property_value callerName; - auto ec = msdo::download_property_value::make(strCallerName, callerName); + auto ec = msdo::download_property_value::make("dosdkcpp_tests", callerName); ASSERT_EQ(ec.value(), static_cast(msdo::errc::e_not_impl)); } diff --git a/sdk-cpp/tests/test_helpers.h b/sdk-cpp/tests/test_helpers.h index 3736d115..ef1a6187 100644 --- a/sdk-cpp/tests/test_helpers.h +++ b/sdk-cpp/tests/test_helpers.h @@ -75,6 +75,11 @@ class download msdod::throw_if_fail(ec); return status; } + void set_status_callback(status_callback_t callback) + { + std::error_code ec = _downloadImpl->set_status_callback(callback); + msdod::throw_if_fail(ec); + } void start_and_wait_until_completion(std::chrono::seconds timeoutSecs = std::chrono::hours(24)) { @@ -100,22 +105,17 @@ class download msdod::throw_if_fail(ec); } - /* - For devices running windows before 20H1, dosvc exposed a now-deprecated com interface for setting certain download properties. - After 20H1, these properties were added to newer com interface, which this SDK is using. - Attempting to set a download property on a version of windows earlier than 20H1 will not set the property and throw an exception with error code msdo::errc::do_e_unknown_property_id - */ - void set_property(msdo::download_property key, const msdo::download_property_value& value) + template + void set_property(msdo::download_property key, const T& value) { std::error_code ec = _downloadImpl->set_property(key, value); msdod::throw_if_fail(ec); } - msdo::download_property_value get_property(msdo::download_property key) + template + void get_property(msdo::download_property key, T& value) { - msdo::download_property_value propValue; - std::error_code ec = _downloadImpl->get_property(key, propValue); + std::error_code ec = _downloadImpl->get_property(key, value); msdod::throw_if_fail(ec); - return propValue; } private: From 8b3bb68d098d0730cbc8172721f9e1c0ceaaa213 Mon Sep 17 00:00:00 2001 From: JeffreySaathoff <107890382+JeffreySaathoff@users.noreply.github.com> Date: Thu, 9 Feb 2023 15:28:23 -0800 Subject: [PATCH 05/25] Add support for streamed output (#156) --- sdk-cpp/include/do_download.h | 2 + sdk-cpp/include/do_download_property.h | 3 - sdk-cpp/include/do_download_status.h | 4 +- sdk-cpp/include/do_errors.h | 16 ++--- sdk-cpp/src/do_download.cpp | 17 +++-- sdk-cpp/src/internal/com/download_impl.cpp | 64 ++++++++++++++++--- .../internal/do_download_property_internal.h | 1 - sdk-cpp/src/internal/do_error_helpers.h | 6 +- sdk-cpp/src/internal/download_impl.h | 6 +- sdk-cpp/src/internal/download_interface.h | 7 +- sdk-cpp/src/internal/rest/download_impl.cpp | 10 ++- sdk-cpp/tests/download_properties_tests.cpp | 19 ++++++ sdk-cpp/tests/test_helpers.h | 14 ++++ 13 files changed, 135 insertions(+), 34 deletions(-) diff --git a/sdk-cpp/include/do_download.h b/sdk-cpp/include/do_download.h index bddd7eda..9d4381f3 100644 --- a/sdk-cpp/include/do_download.h +++ b/sdk-cpp/include/do_download.h @@ -26,6 +26,7 @@ class download public: ~download(); + static std::error_code make(const std::string& uri, std::unique_ptr& out) noexcept; static std::error_code make(const std::string& uri, const std::string& downloadFilePath, std::unique_ptr& out) noexcept; std::error_code start() noexcept; @@ -35,6 +36,7 @@ class download std::error_code abort() noexcept; std::error_code get_status(download_status& status) noexcept; std::error_code set_status_callback(status_callback_t callback) noexcept; + std::error_code set_output_stream(output_stream_callback_t callback) noexcept; std::error_code start_and_wait_until_completion(std::chrono::seconds timeoutSecs = std::chrono::hours(24)) noexcept; std::error_code start_and_wait_until_completion(const std::atomic_bool& isCancelled, std::chrono::seconds timeoutSecs = std::chrono::hours(24)) noexcept; diff --git a/sdk-cpp/include/do_download_property.h b/sdk-cpp/include/do_download_property.h index e34f1195..f394799c 100644 --- a/sdk-cpp/include/do_download_property.h +++ b/sdk-cpp/include/do_download_property.h @@ -6,7 +6,6 @@ #include #include -#include #include "do_errors.h" @@ -19,8 +18,6 @@ namespace details class CDownloadImpl; class CDownloadPropertyValueInternal; } -class download; -class download_status; enum class download_property { diff --git a/sdk-cpp/include/do_download_status.h b/sdk-cpp/include/do_download_status.h index f2d83f65..5243b9e9 100644 --- a/sdk-cpp/include/do_download_status.h +++ b/sdk-cpp/include/do_download_status.h @@ -69,12 +69,14 @@ class download_status int32_t _errorCode { 0 }; int32_t _extendedErrorCode { 0 }; download_state _state { download_state::created }; - }; class download; using status_callback_t = std::function; +// Note: std::span might be nice here, but it requires C++20 +using output_stream_callback_t = std::function; + } // namespace deliveryoptimization } // namespace microsoft diff --git a/sdk-cpp/include/do_errors.h b/sdk-cpp/include/do_errors.h index 08e1a0f9..d5933e1a 100644 --- a/sdk-cpp/include/do_errors.h +++ b/sdk-cpp/include/do_errors.h @@ -14,14 +14,14 @@ namespace deliveryoptimization enum class errc : int32_t { - e_not_impl = -2063400958, - unexpected = -2147418113, - invalid_arg = -2147024809, - not_found = -2147023728, - no_service = -2133848063, - download_no_progress = -2133843966, - do_e_invalid_state = -2133843949, // TODO: Revisit convention here - should separate error code enum be used for do_e* errors? - do_e_unknown_property_id = -2133843951 + e_not_impl = static_cast(0x80004001), // E_NOTIMPL + unexpected = static_cast(0x8000FFFF), // E_UNEXPECTED + invalid_arg = static_cast(0x80070057), // E_INVALIDARG + not_found = static_cast(0x80070490), // E_NOT_SET (ERROR_NOT_FOUND) + no_service = static_cast(0x80D01001), // DO_E_NO_SERVICE + download_no_progress = static_cast(0x80D02002), // DO_E_DOWNLOAD_NO_PROGRESS + do_e_unknown_property_id = static_cast(0x80D02011), // DO_E_UNKNOWN_PROPERTY_ID + do_e_invalid_state = static_cast(0x80D02013), // DO_E_INVALID_STATE }; } //namespace deliveryoptimization diff --git a/sdk-cpp/src/do_download.cpp b/sdk-cpp/src/do_download.cpp index 94934a76..da47bcbc 100644 --- a/sdk-cpp/src/do_download.cpp +++ b/sdk-cpp/src/do_download.cpp @@ -7,10 +7,9 @@ #include #include +#include "download_impl.h" #include "do_errors.h" #include "do_error_helpers.h" -#include "download_interface.h" -#include "download_impl.h" namespace msdod = microsoft::deliveryoptimization::details; using namespace std::chrono_literals; // NOLINT(build/namespaces) @@ -27,13 +26,18 @@ download::download() download::~download() = default; +std::error_code download::make(const std::string& uri, std::unique_ptr& out) noexcept +{ + std::string emptyPath; + return make(uri, emptyPath, out); +} + std::error_code download::make(const std::string& uri, const std::string& downloadFilePath, std::unique_ptr& out) noexcept { out.reset(); std::unique_ptr tmp(new download()); tmp->_download = std::make_shared(); - std::error_code code = tmp->_download->Init(uri, downloadFilePath); - DO_RETURN_IF_FAILED(code); + DO_RETURN_IF_FAILED(tmp->_download->Init(uri, downloadFilePath)); out = std::move(tmp); return DO_OK; } @@ -73,6 +77,11 @@ std::error_code download::set_status_callback(status_callback_t callback) noexce return _download->SetStatusCallback(callback, *this); } +std::error_code download::set_output_stream(output_stream_callback_t callback) noexcept +{ + return _download->SetStreamCallback(callback); +} + std::error_code download::start_and_wait_until_completion(std::chrono::seconds timeOut) noexcept { std::atomic_bool isCancelled{ false }; diff --git a/sdk-cpp/src/internal/com/download_impl.cpp b/sdk-cpp/src/internal/com/download_impl.cpp index fb4421d8..10113e62 100644 --- a/sdk-cpp/src/internal/com/download_impl.cpp +++ b/sdk-cpp/src/internal/com/download_impl.cpp @@ -103,6 +103,38 @@ class DOStatusCallback : msdo::download* _download; }; +class DOStreamCallback : + public RuntimeClass, ChainInterfaces> +{ +public: + HRESULT RuntimeClassInitialize(const msdo::output_stream_callback_t& callback) + { + _callback = callback; + return S_OK; + } + + // ISequentialStream + IFACEMETHODIMP Read(void*, ULONG, ULONG*) override { return E_NOTIMPL; } + IFACEMETHODIMP Write(const void* pv, ULONG cb, ULONG* /* pcbWritten */) override + { + return HRESULT_FROM_WIN32(_callback(static_cast(pv), cb).value()); + } + + // IStream (ISequentialStream would be sufficient but DO QI's for IStream) + IFACEMETHODIMP Seek(LARGE_INTEGER, DWORD, ULARGE_INTEGER*) override { return E_NOTIMPL; } + IFACEMETHODIMP SetSize(ULARGE_INTEGER) override { return E_NOTIMPL; } + IFACEMETHODIMP CopyTo(IStream*, ULARGE_INTEGER, ULARGE_INTEGER*, ULARGE_INTEGER*) override { return E_NOTIMPL; } + IFACEMETHODIMP Commit(DWORD) override { return E_NOTIMPL; } + IFACEMETHODIMP Revert() override { return E_NOTIMPL; } + IFACEMETHODIMP LockRegion(ULARGE_INTEGER, ULARGE_INTEGER, DWORD) override { return E_NOTIMPL; } + IFACEMETHODIMP UnlockRegion(ULARGE_INTEGER, ULARGE_INTEGER, DWORD) override { return E_NOTIMPL; } + IFACEMETHODIMP Stat(STATSTG*, DWORD) override { return E_NOTIMPL; } + IFACEMETHODIMP Clone(IStream**) override { return E_NOTIMPL; } + +private: + msdo::output_stream_callback_t _callback; +}; + std::error_code CDownloadImpl::Init(const std::string& uri, const std::string& downloadFilePath) noexcept { Microsoft::WRL::ComPtr manager; @@ -117,12 +149,15 @@ std::error_code CDownloadImpl::Init(const std::string& uri, const std::string& d _spDownload = std::move(spDownload); download_property_value propUri; - download_property_value propDownloadFilePath; DO_RETURN_IF_FAILED(download_property_value::make(uri, propUri)); - DO_RETURN_IF_FAILED(download_property_value::make(downloadFilePath, propDownloadFilePath)); - DO_RETURN_IF_FAILED(SetProperty(download_property::uri, propUri)); - DO_RETURN_IF_FAILED(SetProperty(download_property::download_file_path, propDownloadFilePath)); + + if (!downloadFilePath.empty()) + { + download_property_value propDownloadFilePath; + DO_RETURN_IF_FAILED(download_property_value::make(downloadFilePath, propDownloadFilePath)); + DO_RETURN_IF_FAILED(SetProperty(download_property::download_file_path, propDownloadFilePath)); + } return DO_OK; } @@ -130,9 +165,11 @@ std::error_code CDownloadImpl::Init(const std::string& uri, const std::string& d // Support only full file downloads for now std::error_code CDownloadImpl::Start() noexcept { - _DO_DOWNLOAD_RANGES_INFO emptyRanges = {}; - emptyRanges.RangeCount = 0; - return make_error_code(_spDownload->Start(&emptyRanges)); + DO_DOWNLOAD_RANGES_INFO ranges = {}; + ranges.RangeCount = 1; + ranges.Ranges[0].Offset = 0; + ranges.Ranges[0].Length = DO_LENGTH_TO_EOF; + return make_error_code(_spDownload->Start(&ranges)); } std::error_code CDownloadImpl::Pause() noexcept @@ -174,6 +211,17 @@ std::error_code CDownloadImpl::SetStatusCallback(const msdo::status_callback_t& RETURN_IF_FAILED(_spDownload->SetProperty(DODownloadProperty_CallbackInterface, &vtCallback)); return DO_OK; } +std::error_code CDownloadImpl::SetStreamCallback(const msdo::output_stream_callback_t& callback) noexcept +{ + Microsoft::WRL::ComPtr spCallback; + RETURN_IF_FAILED(MakeAndInitialize(&spCallback, callback)); + + unique_variant vtCallback; + V_VT(&vtCallback) = VT_UNKNOWN; + V_UNKNOWN(&vtCallback) = spCallback.Detach(); + RETURN_IF_FAILED(_spDownload->SetProperty(DODownloadProperty_StreamInterface, &vtCallback)); + return DO_OK; +} std::error_code CDownloadImpl::SetProperty(msdo::download_property key, const msdo::download_property_value& val) noexcept { @@ -210,7 +258,7 @@ std::error_code CDownloadImpl::GetProperty(msdo::download_property key, msdo::do break; default: - return make_error_code(E_UNEXPECTED); + return make_error_code(errc::unexpected); } return DO_OK; } diff --git a/sdk-cpp/src/internal/do_download_property_internal.h b/sdk-cpp/src/internal/do_download_property_internal.h index 20fd1b92..3085e07e 100644 --- a/sdk-cpp/src/internal/do_download_property_internal.h +++ b/sdk-cpp/src/internal/do_download_property_internal.h @@ -11,7 +11,6 @@ #endif #include -#include #include "do_download_property.h" #include "do_errors.h" diff --git a/sdk-cpp/src/internal/do_error_helpers.h b/sdk-cpp/src/internal/do_error_helpers.h index bf4307d9..1e5da012 100644 --- a/sdk-cpp/src/internal/do_error_helpers.h +++ b/sdk-cpp/src/internal/do_error_helpers.h @@ -5,8 +5,12 @@ #define _DELIVERY_OPTIMIZATION_DO_ERROR_HELPERS_H #include -#include #include + +#ifdef DO_ENABLE_EXCEPTIONS +#include +#endif + #include "do_errors.h" namespace microsoft diff --git a/sdk-cpp/src/internal/download_impl.h b/sdk-cpp/src/internal/download_impl.h index ab058d1c..997a357f 100644 --- a/sdk-cpp/src/internal/download_impl.h +++ b/sdk-cpp/src/internal/download_impl.h @@ -4,12 +4,7 @@ #ifndef _DELIVERY_OPTIMIZATION_DOWNLOAD_IMPL_H #define _DELIVERY_OPTIMIZATION_DOWNLOAD_IMPL_H -#include - #include "download_interface.h" -#include "do_download_property.h" -#include "do_download_status.h" -#include "do_errors.h" #if defined(DO_INTERFACE_COM) #include @@ -39,6 +34,7 @@ class CDownloadImpl : public IDownload std::error_code GetStatus(download_status& status) noexcept override; std::error_code SetStatusCallback(const status_callback_t& callback, download& download) noexcept override; + std::error_code SetStreamCallback(const output_stream_callback_t& callback) noexcept override; std::error_code GetProperty(download_property key, download_property_value& value) noexcept override; std::error_code SetProperty(download_property key, const download_property_value& val) noexcept override; diff --git a/sdk-cpp/src/internal/download_interface.h b/sdk-cpp/src/internal/download_interface.h index 4f661fec..7770c209 100644 --- a/sdk-cpp/src/internal/download_interface.h +++ b/sdk-cpp/src/internal/download_interface.h @@ -4,15 +4,17 @@ #ifndef _DELIVERY_OPTIMIZATION_DOWNLOAD_INTERFACE_H #define _DELIVERY_OPTIMIZATION_DOWNLOAD_INTERFACE_H +#include + #include "do_download_status.h" #include "do_download_property.h" -class download; - namespace microsoft { namespace deliveryoptimization { +class download; + namespace details { @@ -31,6 +33,7 @@ class IDownload virtual std::error_code GetStatus(download_status& status) noexcept = 0; virtual std::error_code SetStatusCallback(const status_callback_t& callback, download& download) noexcept = 0; + virtual std::error_code SetStreamCallback(const output_stream_callback_t& callback) noexcept = 0; virtual std::error_code GetProperty(download_property key, download_property_value& value) noexcept = 0; virtual std::error_code SetProperty(download_property key, const download_property_value& val) noexcept = 0; diff --git a/sdk-cpp/src/internal/rest/download_impl.cpp b/sdk-cpp/src/internal/rest/download_impl.cpp index 786815ed..3b870f91 100644 --- a/sdk-cpp/src/internal/rest/download_impl.cpp +++ b/sdk-cpp/src/internal/rest/download_impl.cpp @@ -30,7 +30,10 @@ std::error_code CDownloadImpl::Init(const std::string& uri, const std::string& d cpprest_web::uri_builder builder(g_downloadUriPart); builder.append_path("create"); builder.append_query("Uri", uri); - builder.append_query("DownloadFilePath", downloadFilePath); + if (!downloadFilePath.empty()) + { + builder.append_query("DownloadFilePath", downloadFilePath); + } for (int retryAttempts = 0; retryAttempts < g_maxNumRetryAttempts; retryAttempts++) { @@ -147,6 +150,11 @@ std::error_code CDownloadImpl::SetStatusCallback(const status_callback_t& callba return make_error_code(msdo::errc::e_not_impl); } +std::error_code CDownloadImpl::SetStreamCallback(const output_stream_callback_t& callback) noexcept +{ + return make_error_code(msdo::errc::e_not_impl); +} + std::error_code CDownloadImpl::_DownloadOperationCall(const std::string& type) noexcept { try diff --git a/sdk-cpp/tests/download_properties_tests.cpp b/sdk-cpp/tests/download_properties_tests.cpp index 37788af1..88914286 100644 --- a/sdk-cpp/tests/download_properties_tests.cpp +++ b/sdk-cpp/tests/download_properties_tests.cpp @@ -198,6 +198,25 @@ TEST_F(DownloadPropertyTests, ForegroundBackgroundRace) ASSERT_LT(foregroundDuration, backgroundDuration); } +TEST_F(DownloadPropertyTests, BasicStreamTest) +{ + auto simpleDownload = msdot::download::make(g_smallFileUrl); + + uint64_t downloadedBytes = 0; + simpleDownload->set_output_stream([&downloadedBytes](const unsigned char*, size_t cb) -> std::error_code { downloadedBytes += cb; return DO_OK; }); + + simpleDownload->start(); + TestHelpers::WaitForState(*simpleDownload, msdo::download_state::transferred); + + ASSERT_GT(downloadedBytes, 0); + + uint64_t fileSize = 0; + simpleDownload->get_property(msdo::download_property::total_size_bytes, fileSize); + ASSERT_EQ(downloadedBytes, fileSize); + + simpleDownload->finalize(); +} + #elif defined(DO_CLIENT_AGENT) TEST_F(DownloadPropertyTests, SmallDownloadSetCallerNameFailureTest) diff --git a/sdk-cpp/tests/test_helpers.h b/sdk-cpp/tests/test_helpers.h index ef1a6187..4e1c732b 100644 --- a/sdk-cpp/tests/test_helpers.h +++ b/sdk-cpp/tests/test_helpers.h @@ -29,6 +29,15 @@ namespace test class download { public: + static std::unique_ptr make(const std::string& uri) + { + std::unique_ptr newDownload; + std::error_code ec = msdo::download::make(uri, newDownload); + msdod::throw_if_fail(ec); + auto returnVal = std::make_unique(std::move(newDownload)); + return returnVal; + } + static std::unique_ptr make(const std::string& uri, const std::string& downloadFilePath) { std::unique_ptr newDownload; @@ -80,6 +89,11 @@ class download std::error_code ec = _downloadImpl->set_status_callback(callback); msdod::throw_if_fail(ec); } + void set_output_stream(output_stream_callback_t callback) + { + std::error_code ec = _downloadImpl->set_output_stream(callback); + msdod::throw_if_fail(ec); + } void start_and_wait_until_completion(std::chrono::seconds timeoutSecs = std::chrono::hours(24)) { From 4561954177db3b4130d7b400b6433c2bd800abb6 Mon Sep 17 00:00:00 2001 From: JeffreySaathoff <107890382+JeffreySaathoff@users.noreply.github.com> Date: Mon, 13 Feb 2023 13:01:03 -0800 Subject: [PATCH 06/25] Add support for partial downloads (ranges) (#157) --- sdk-cpp/include/do_download.h | 10 +-- sdk-cpp/include/do_download_status.h | 10 ++- sdk-cpp/src/do_download.cpp | 9 +++ .../com/do_download_property_internal.cpp | 3 - sdk-cpp/src/internal/com/download_impl.cpp | 64 ++++++++++++++----- sdk-cpp/src/internal/download_impl.h | 2 + sdk-cpp/src/internal/download_interface.h | 2 + .../rest/do_download_property_internal.cpp | 5 -- sdk-cpp/src/internal/rest/download_impl.cpp | 5 ++ sdk-cpp/tests/download_properties_tests.cpp | 19 ++++++ sdk-cpp/tests/test_helpers.h | 5 ++ 11 files changed, 105 insertions(+), 29 deletions(-) diff --git a/sdk-cpp/include/do_download.h b/sdk-cpp/include/do_download.h index 9d4381f3..374f158c 100644 --- a/sdk-cpp/include/do_download.h +++ b/sdk-cpp/include/do_download.h @@ -38,15 +38,17 @@ class download std::error_code set_status_callback(status_callback_t callback) noexcept; std::error_code set_output_stream(output_stream_callback_t callback) noexcept; + // start() and set_ranges() are not thread safe. Simultaneous calls to these + // methods may have unpredictable results. + std::error_code set_ranges(const download_range* ranges, size_t count) noexcept; // Future: use std::span (requires C++20) + std::error_code start_and_wait_until_completion(std::chrono::seconds timeoutSecs = std::chrono::hours(24)) noexcept; std::error_code start_and_wait_until_completion(const std::atomic_bool& isCancelled, std::chrono::seconds timeoutSecs = std::chrono::hours(24)) noexcept; static std::error_code download_url_to_path(const std::string& uri, const std::string& downloadFilePath, std::chrono::seconds timeoutSecs = std::chrono::hours(24)) noexcept; static std::error_code download_url_to_path(const std::string& uri, const std::string& downloadFilePath, const std::atomic_bool& isCancelled, std::chrono::seconds timeoutSecs = std::chrono::hours(24)) noexcept; - /* - Certain properties are not supported on older versions of Windows, resulting in - msdo::errc::do_e_unknown_property_id from the following methods. See do_download_property.h. - */ + // Certain properties are not supported on older versions of Windows, resulting in + // msdo::errc::do_e_unknown_property_id from the following methods. See do_download_property.h. std::error_code set_property(download_property prop, const download_property_value& value) noexcept; std::error_code get_property(download_property prop, download_property_value& value) noexcept; diff --git a/sdk-cpp/include/do_download_status.h b/sdk-cpp/include/do_download_status.h index 5243b9e9..dcaafba2 100644 --- a/sdk-cpp/include/do_download_status.h +++ b/sdk-cpp/include/do_download_status.h @@ -74,9 +74,17 @@ class download_status class download; using status_callback_t = std::function; -// Note: std::span might be nice here, but it requires C++20 +// Future: use std::span (requires C++20) using output_stream_callback_t = std::function; +// TODO: create a new header for things that aren't 'status' related, such as output_stream_callback_t and ranges +struct download_range +{ + uint64_t offset; + uint64_t length; +}; +static constexpr uint64_t length_to_eof = static_cast(-1); + } // namespace deliveryoptimization } // namespace microsoft diff --git a/sdk-cpp/src/do_download.cpp b/sdk-cpp/src/do_download.cpp index da47bcbc..43737343 100644 --- a/sdk-cpp/src/do_download.cpp +++ b/sdk-cpp/src/do_download.cpp @@ -82,6 +82,15 @@ std::error_code download::set_output_stream(output_stream_callback_t callback) n return _download->SetStreamCallback(callback); } +std::error_code download::set_ranges(const download_range* ranges, size_t count) noexcept +{ + if ((ranges == nullptr) || (count == 0)) + { + return details::make_error_code(errc::invalid_arg); + } + return _download->SetRanges(ranges, count); +} + std::error_code download::start_and_wait_until_completion(std::chrono::seconds timeOut) noexcept { std::atomic_bool isCancelled{ false }; diff --git a/sdk-cpp/src/internal/com/do_download_property_internal.cpp b/sdk-cpp/src/internal/com/do_download_property_internal.cpp index c5cb639e..a6942d78 100644 --- a/sdk-cpp/src/internal/com/do_download_property_internal.cpp +++ b/sdk-cpp/src/internal/com/do_download_property_internal.cpp @@ -3,11 +3,8 @@ #include "do_download_property_internal.h" -#include #include -#include -#include "do_errors.h" #include "do_error_helpers.h" namespace msdo = microsoft::deliveryoptimization; diff --git a/sdk-cpp/src/internal/com/download_impl.cpp b/sdk-cpp/src/internal/com/download_impl.cpp index 10113e62..c0ee60aa 100644 --- a/sdk-cpp/src/internal/com/download_impl.cpp +++ b/sdk-cpp/src/internal/com/download_impl.cpp @@ -3,13 +3,7 @@ #include "download_impl.h" -#include -#include -#include - -#include "do_download_property.h" #include "do_download_property_internal.h" -#include "do_errors.h" #include "do_error_helpers.h" namespace msdo = microsoft::deliveryoptimization; @@ -137,9 +131,9 @@ class DOStreamCallback : std::error_code CDownloadImpl::Init(const std::string& uri, const std::string& downloadFilePath) noexcept { - Microsoft::WRL::ComPtr manager; + ComPtr manager; RETURN_IF_FAILED(CoCreateInstance(__uuidof(DeliveryOptimization), nullptr, CLSCTX_LOCAL_SERVER, IID_PPV_ARGS(&manager))); - Microsoft::WRL::ComPtr spDownload; + ComPtr spDownload; RETURN_IF_FAILED(manager->CreateDownload(&spDownload)); RETURN_IF_FAILED(CoSetProxyBlanket(static_cast(spDownload.Get()), RPC_C_AUTHN_DEFAULT, @@ -148,6 +142,10 @@ std::error_code CDownloadImpl::Init(const std::string& uri, const std::string& d _spDownload = std::move(spDownload); + // Assume full file until told otherwise + _spRanges = std::make_unique(); + _spRanges->RangeCount = 0; // empty == full file + download_property_value propUri; DO_RETURN_IF_FAILED(download_property_value::make(uri, propUri)); DO_RETURN_IF_FAILED(SetProperty(download_property::uri, propUri)); @@ -162,14 +160,11 @@ std::error_code CDownloadImpl::Init(const std::string& uri, const std::string& d return DO_OK; } -// Support only full file downloads for now std::error_code CDownloadImpl::Start() noexcept { - DO_DOWNLOAD_RANGES_INFO ranges = {}; - ranges.RangeCount = 1; - ranges.Ranges[0].Offset = 0; - ranges.Ranges[0].Length = DO_LENGTH_TO_EOF; - return make_error_code(_spDownload->Start(&ranges)); + // Note: ranges are reset on Start. It is correct to pass nullptr for subsequent Start calls (i.e. "Resume"). + std::unique_ptr spRanges = std::move(_spRanges); + return make_error_code(_spDownload->Start(spRanges.get())); } std::error_code CDownloadImpl::Pause() noexcept @@ -202,7 +197,7 @@ std::error_code CDownloadImpl::GetStatus(msdo::download_status& status) noexcept std::error_code CDownloadImpl::SetStatusCallback(const msdo::status_callback_t& callback, msdo::download& download) noexcept { - Microsoft::WRL::ComPtr spCallback; + ComPtr spCallback; RETURN_IF_FAILED(MakeAndInitialize(&spCallback, callback, download)); unique_variant vtCallback; @@ -213,13 +208,22 @@ std::error_code CDownloadImpl::SetStatusCallback(const msdo::status_callback_t& } std::error_code CDownloadImpl::SetStreamCallback(const msdo::output_stream_callback_t& callback) noexcept { - Microsoft::WRL::ComPtr spCallback; + ComPtr spCallback; RETURN_IF_FAILED(MakeAndInitialize(&spCallback, callback)); unique_variant vtCallback; V_VT(&vtCallback) = VT_UNKNOWN; V_UNKNOWN(&vtCallback) = spCallback.Detach(); RETURN_IF_FAILED(_spDownload->SetProperty(DODownloadProperty_StreamInterface, &vtCallback)); + + // Streamed output requires ranges. Convert empty ranges into a full file range. + if (_spRanges && (_spRanges->RangeCount == 0)) + { + _spRanges->RangeCount = 1; + _spRanges->Ranges[0].Offset = 0; // [0]: Minimum struct size includes 1 array entry. No need to reallocate. + _spRanges->Ranges[0].Length = DO_LENGTH_TO_EOF; + } + return DO_OK; } @@ -263,6 +267,34 @@ std::error_code CDownloadImpl::GetProperty(msdo::download_property key, msdo::do return DO_OK; } +// msdo::download_range and DO_DOWNLOAD_RANGE are equivalent -- safe to typecast/copy from one to the other +static_assert(sizeof(download_range) == sizeof(DO_DOWNLOAD_RANGE)); +static_assert(FIELD_OFFSET(download_range, offset) == FIELD_OFFSET(DO_DOWNLOAD_RANGE, Offset)); +static_assert(FIELD_OFFSET(download_range, length) == FIELD_OFFSET(DO_DOWNLOAD_RANGE, Length)); +static_assert(length_to_eof == DO_LENGTH_TO_EOF); + +std::error_code CDownloadImpl::SetRanges(const download_range* ranges, size_t count) noexcept +{ + size_t cbBufferSize = sizeof(DO_DOWNLOAD_RANGES_INFO); // includes 1 DO_DOWNLOAD_RANGE + if (count > 1) + { + cbBufferSize += ((count - 1) * sizeof(DO_DOWNLOAD_RANGE)); + } + + void *pvBuffer = ::operator new(cbBufferSize, std::nothrow); + if (pvBuffer == nullptr) + { + return make_error_code(E_OUTOFMEMORY); + } + + std::unique_ptr spRanges(new(pvBuffer) DO_DOWNLOAD_RANGES_INFO()); + spRanges->RangeCount = static_cast(count); + memcpy(&spRanges->Ranges[0], ranges, count * sizeof(ranges[0])); + + _spRanges = std::move(spRanges); + return DO_OK; +} + } // namespace details } // namespace deliveryoptimization } // namespace microsoft diff --git a/sdk-cpp/src/internal/download_impl.h b/sdk-cpp/src/internal/download_impl.h index 997a357f..1ca2d4fa 100644 --- a/sdk-cpp/src/internal/download_impl.h +++ b/sdk-cpp/src/internal/download_impl.h @@ -37,10 +37,12 @@ class CDownloadImpl : public IDownload std::error_code SetStreamCallback(const output_stream_callback_t& callback) noexcept override; std::error_code GetProperty(download_property key, download_property_value& value) noexcept override; std::error_code SetProperty(download_property key, const download_property_value& val) noexcept override; + std::error_code SetRanges(const download_range* ranges, size_t count) noexcept override; private: #if defined(DO_INTERFACE_COM) Microsoft::WRL::ComPtr _spDownload; + std::unique_ptr _spRanges; #elif defined(DO_INTERFACE_REST) std::error_code _DownloadOperationCall(const std::string& type) noexcept; diff --git a/sdk-cpp/src/internal/download_interface.h b/sdk-cpp/src/internal/download_interface.h index 7770c209..f00d23fd 100644 --- a/sdk-cpp/src/internal/download_interface.h +++ b/sdk-cpp/src/internal/download_interface.h @@ -37,6 +37,8 @@ class IDownload virtual std::error_code GetProperty(download_property key, download_property_value& value) noexcept = 0; virtual std::error_code SetProperty(download_property key, const download_property_value& val) noexcept = 0; + + virtual std::error_code SetRanges(const download_range* ranges, size_t count) noexcept = 0; }; } // namespace details } // namespace deliveryoptimization diff --git a/sdk-cpp/src/internal/rest/do_download_property_internal.cpp b/sdk-cpp/src/internal/rest/do_download_property_internal.cpp index 2df79aca..daba8795 100644 --- a/sdk-cpp/src/internal/rest/do_download_property_internal.cpp +++ b/sdk-cpp/src/internal/rest/do_download_property_internal.cpp @@ -3,11 +3,6 @@ #include "do_download_property_internal.h" -#include -#include -#include - -#include "do_errors.h" #include "do_error_helpers.h" namespace microsoft diff --git a/sdk-cpp/src/internal/rest/download_impl.cpp b/sdk-cpp/src/internal/rest/download_impl.cpp index 3b870f91..2161e358 100644 --- a/sdk-cpp/src/internal/rest/download_impl.cpp +++ b/sdk-cpp/src/internal/rest/download_impl.cpp @@ -155,6 +155,11 @@ std::error_code CDownloadImpl::SetStreamCallback(const output_stream_callback_t& return make_error_code(msdo::errc::e_not_impl); } +std::error_code CDownloadImpl::SetRanges(const download_range* ranges, size_t count) noexcept +{ + return make_error_code(errc::e_not_impl); +} + std::error_code CDownloadImpl::_DownloadOperationCall(const std::string& type) noexcept { try diff --git a/sdk-cpp/tests/download_properties_tests.cpp b/sdk-cpp/tests/download_properties_tests.cpp index 88914286..d49b041c 100644 --- a/sdk-cpp/tests/download_properties_tests.cpp +++ b/sdk-cpp/tests/download_properties_tests.cpp @@ -217,6 +217,25 @@ TEST_F(DownloadPropertyTests, BasicStreamTest) simpleDownload->finalize(); } +TEST_F(DownloadPropertyTests, BasicRangesTest) +{ + auto simpleDownload = msdot::download::make(g_largeFileUrl); + + uint64_t downloadedBytes = 0; + simpleDownload->set_output_stream([&downloadedBytes](const unsigned char*, size_t cb) -> std::error_code { downloadedBytes += cb; return DO_OK; }); + + msdo::download_range ranges[2] = + { + { 0, 100 }, + { 200, 1000 }, + }; + simpleDownload->set_ranges(ranges, 2); + + simpleDownload->start_and_wait_until_completion(); + + ASSERT_EQ(downloadedBytes, 1100); +} + #elif defined(DO_CLIENT_AGENT) TEST_F(DownloadPropertyTests, SmallDownloadSetCallerNameFailureTest) diff --git a/sdk-cpp/tests/test_helpers.h b/sdk-cpp/tests/test_helpers.h index 4e1c732b..40f23b27 100644 --- a/sdk-cpp/tests/test_helpers.h +++ b/sdk-cpp/tests/test_helpers.h @@ -94,6 +94,11 @@ class download std::error_code ec = _downloadImpl->set_output_stream(callback); msdod::throw_if_fail(ec); } + void set_ranges(const download_range* ranges, size_t count) + { + std::error_code ec = _downloadImpl->set_ranges(ranges, count); + msdod::throw_if_fail(ec); + } void start_and_wait_until_completion(std::chrono::seconds timeoutSecs = std::chrono::hours(24)) { From 8df51fc107568b6c3cc2ea205676efe726ea7ddb Mon Sep 17 00:00:00 2001 From: JeffreySaathoff <107890382+JeffreySaathoff@users.noreply.github.com> Date: Wed, 15 Feb 2023 10:35:11 -0800 Subject: [PATCH 07/25] Add implementation for EnumDownloads (#158) --- sdk-cpp/include/do_config.h | 2 +- sdk-cpp/include/do_download.h | 12 +++- sdk-cpp/include/do_errors.h | 23 ++++--- sdk-cpp/src/do_download.cpp | 47 ++++++++++++- .../src/internal/com/do_config_internal.cpp | 2 +- .../com/do_download_property_internal.cpp | 4 +- sdk-cpp/src/internal/com/download_impl.cpp | 68 ++++++++++++++++++- .../internal/do_download_property_internal.h | 3 + sdk-cpp/src/internal/do_error_helpers.h | 21 +----- sdk-cpp/src/internal/download_impl.h | 8 +++ .../src/internal/rest/do_config_internal.cpp | 4 +- .../rest/do_download_property_internal.cpp | 20 +++--- sdk-cpp/src/internal/rest/download_impl.cpp | 27 ++++++-- sdk-cpp/tests/download_properties_tests.cpp | 30 +++++++- sdk-cpp/tests/download_tests_common.cpp | 18 ++--- 15 files changed, 218 insertions(+), 71 deletions(-) diff --git a/sdk-cpp/include/do_config.h b/sdk-cpp/include/do_config.h index b69b595d..f96610e7 100644 --- a/sdk-cpp/include/do_config.h +++ b/sdk-cpp/include/do_config.h @@ -6,7 +6,7 @@ /* This file exposes apis for helping set an iot_connection_string for the DO agent, so that an application can supply a Microsoft Connected Cache device's hostname -While this file will compile for all platforms, they only serve a purpose for the DeliveryOptimization Agent on linux devices, all other usage will fail and return e_not_impl +While this file will compile for all platforms, they only serve a purpose for the DeliveryOptimization Agent on linux devices, all other usage will fail and return errc::not_impl */ #ifdef __cplusplus diff --git a/sdk-cpp/include/do_download.h b/sdk-cpp/include/do_download.h index 374f158c..fb23cd6b 100644 --- a/sdk-cpp/include/do_download.h +++ b/sdk-cpp/include/do_download.h @@ -8,6 +8,7 @@ #include #include #include +#include #include "do_download_status.h" #include "do_download_property.h" @@ -48,7 +49,7 @@ class download static std::error_code download_url_to_path(const std::string& uri, const std::string& downloadFilePath, const std::atomic_bool& isCancelled, std::chrono::seconds timeoutSecs = std::chrono::hours(24)) noexcept; // Certain properties are not supported on older versions of Windows, resulting in - // msdo::errc::do_e_unknown_property_id from the following methods. See do_download_property.h. + // msdo::errc::unknown_property_id from the following methods. See do_download_property.h. std::error_code set_property(download_property prop, const download_property_value& value) noexcept; std::error_code get_property(download_property prop, download_property_value& value) noexcept; @@ -82,10 +83,17 @@ class download return set_property(download_property::cost_policy, static_cast(value)); } + // Returns existing downloads + static std::error_code get_downloads(std::vector>& out) noexcept; + + // Returns existing downloads, filtered by download_property::id, uri, catalog_id, caller_name or download_file_path + static std::error_code get_downloads(download_property prop, const std::string& value, std::vector>& out) noexcept; + static std::error_code get_downloads(download_property prop, const std::wstring& value, std::vector>& out) noexcept; + private: download(); - std::shared_ptr _download; + std::unique_ptr _download; }; } // namespace deliveryoptimization diff --git a/sdk-cpp/include/do_errors.h b/sdk-cpp/include/do_errors.h index d5933e1a..d3c3be9f 100644 --- a/sdk-cpp/include/do_errors.h +++ b/sdk-cpp/include/do_errors.h @@ -11,19 +11,20 @@ namespace microsoft { namespace deliveryoptimization { - -enum class errc : int32_t +namespace errc { - e_not_impl = static_cast(0x80004001), // E_NOTIMPL - unexpected = static_cast(0x8000FFFF), // E_UNEXPECTED - invalid_arg = static_cast(0x80070057), // E_INVALIDARG - not_found = static_cast(0x80070490), // E_NOT_SET (ERROR_NOT_FOUND) - no_service = static_cast(0x80D01001), // DO_E_NO_SERVICE - download_no_progress = static_cast(0x80D02002), // DO_E_DOWNLOAD_NO_PROGRESS - do_e_unknown_property_id = static_cast(0x80D02011), // DO_E_UNKNOWN_PROPERTY_ID - do_e_invalid_state = static_cast(0x80D02013), // DO_E_INVALID_STATE -}; +constexpr auto not_impl = static_cast(0x80004001); // E_NOTIMPL +constexpr auto unexpected = static_cast(0x8000FFFF); // E_UNEXPECTED +constexpr auto invalid_arg = static_cast(0x80070057); // E_INVALIDARG +constexpr auto not_found = static_cast(0x80070490); // E_NOT_SET (ERROR_NOT_FOUND) +constexpr auto no_service = static_cast(0x80D01001); // DO_E_NO_SERVICE +constexpr auto download_no_progress = static_cast(0x80D02002); // DO_E_DOWNLOAD_NO_PROGRESS +constexpr auto no_downloads = static_cast(0x80D02005); // DO_E_NO_DOWNLOADS +constexpr auto unknown_property_id = static_cast(0x80D02011); // DO_E_UNKNOWN_PROPERTY_ID +constexpr auto invalid_state = static_cast(0x80D02013); // DO_E_INVALID_STATE + +} //namespace errc } //namespace deliveryoptimization } //namespace microsoft diff --git a/sdk-cpp/src/do_download.cpp b/sdk-cpp/src/do_download.cpp index 43737343..e6a4c422 100644 --- a/sdk-cpp/src/do_download.cpp +++ b/sdk-cpp/src/do_download.cpp @@ -21,7 +21,7 @@ namespace deliveryoptimization download::download() { - _download = std::make_shared(); + _download = std::make_unique(); } download::~download() = default; @@ -36,7 +36,6 @@ std::error_code download::make(const std::string& uri, const std::string& downlo { out.reset(); std::unique_ptr tmp(new download()); - tmp->_download = std::make_shared(); DO_RETURN_IF_FAILED(tmp->_download->Init(uri, downloadFilePath)); out = std::move(tmp); return DO_OK; @@ -166,7 +165,7 @@ static std::error_code g_TryOverrideDownlevelOsSetPropertyError(download_propert { // Temporary backward-compatibility for MSEdge. // These properties were not supported in IDODownload interface until build 19041. - if ((ec.value() == static_cast(errc::do_e_unknown_property_id)) && + if ((ec.value() == errc::unknown_property_id) && ((prop == download_property::correlation_vector) || (prop == download_property::integrity_check_info))) { return DO_OK; @@ -188,5 +187,47 @@ std::error_code download::get_property(download_property prop, download_property return _download->GetProperty(prop, val); } +std::error_code download::get_downloads(std::vector>& out) noexcept +{ + out.clear(); + std::vector> results; + DO_RETURN_IF_FAILED(msdod::CDownloadImpl::EnumDownloads(results)); + for (auto& result : results) + { + std::unique_ptr tmp(new download()); + tmp->_download = std::move(result); + out.push_back(std::move(tmp)); + } + return DO_OK; +} + +std::error_code download::get_downloads(download_property prop, const std::string& value, std::vector>& out) noexcept +{ + out.clear(); + std::vector> results; + DO_RETURN_IF_FAILED(msdod::CDownloadImpl::EnumDownloads(prop, value, results)); + for (auto& result : results) + { + std::unique_ptr tmp(new download()); + tmp->_download = std::move(result); + out.push_back(std::move(tmp)); + } + return DO_OK; +} + +std::error_code download::get_downloads(download_property prop, const std::wstring& value, std::vector>& out) noexcept +{ + out.clear(); + std::vector> results; + DO_RETURN_IF_FAILED(msdod::CDownloadImpl::EnumDownloads(prop, value, results)); + for (auto& result : results) + { + std::unique_ptr tmp(new download()); + tmp->_download = std::move(result); + out.push_back(std::move(tmp)); + } + return DO_OK; +} + } // namespace deliveryoptimization } // namespace microsoft diff --git a/sdk-cpp/src/internal/com/do_config_internal.cpp b/sdk-cpp/src/internal/com/do_config_internal.cpp index af2b95c6..e98ceadb 100644 --- a/sdk-cpp/src/internal/com/do_config_internal.cpp +++ b/sdk-cpp/src/internal/com/do_config_internal.cpp @@ -7,7 +7,7 @@ namespace msdo = microsoft::deliveryoptimization; int internal_set_iot_connection_string(const char* value) { - return static_cast(msdo::errc::e_not_impl); + return msdo::errc::not_impl; } char* internal_get_components_version() diff --git a/sdk-cpp/src/internal/com/do_download_property_internal.cpp b/sdk-cpp/src/internal/com/do_download_property_internal.cpp index a6942d78..9055c2c6 100644 --- a/sdk-cpp/src/internal/com/do_download_property_internal.cpp +++ b/sdk-cpp/src/internal/com/do_download_property_internal.cpp @@ -16,7 +16,7 @@ namespace deliveryoptimization namespace details { -static std::error_code UTF8toWstr(const std::string& str, std::wstring& wstr) +std::error_code UTF8toWstr(const std::string& str, std::wstring& wstr) { wstr.clear(); size_t cch = str.size(); @@ -33,7 +33,7 @@ static std::error_code UTF8toWstr(const std::string& str, std::wstring& wstr) return DO_OK; } -static std::error_code WstrToUTF8(const std::wstring& wstr, std::string& str) +std::error_code WstrToUTF8(const std::wstring& wstr, std::string& str) { str.clear(); size_t cch = wstr.size(); diff --git a/sdk-cpp/src/internal/com/download_impl.cpp b/sdk-cpp/src/internal/com/download_impl.cpp index c0ee60aa..b7ff6080 100644 --- a/sdk-cpp/src/internal/com/download_impl.cpp +++ b/sdk-cpp/src/internal/com/download_impl.cpp @@ -136,7 +136,7 @@ std::error_code CDownloadImpl::Init(const std::string& uri, const std::string& d ComPtr spDownload; RETURN_IF_FAILED(manager->CreateDownload(&spDownload)); - RETURN_IF_FAILED(CoSetProxyBlanket(static_cast(spDownload.Get()), RPC_C_AUTHN_DEFAULT, + RETURN_IF_FAILED(CoSetProxyBlanket(spDownload.Get(), RPC_C_AUTHN_DEFAULT, RPC_C_AUTHZ_NONE, COLE_DEFAULT_PRINCIPAL, RPC_C_AUTHN_LEVEL_DEFAULT, RPC_C_IMP_LEVEL_IMPERSONATE, nullptr, EOAC_STATIC_CLOAKING)); @@ -295,6 +295,72 @@ std::error_code CDownloadImpl::SetRanges(const download_range* ranges, size_t co return DO_OK; } +std::error_code CDownloadImpl::EnumDownloads(std::vector>& out) noexcept +{ + out.clear(); + return _EnumDownloads(nullptr, out); +} + +std::error_code CDownloadImpl::EnumDownloads(download_property prop, const std::string& value, std::vector>& out) noexcept +{ + out.clear(); + std::wstring wval; + DO_RETURN_IF_FAILED(UTF8toWstr(value, wval)); + return EnumDownloads(prop, wval, out); +} + +std::error_code CDownloadImpl::EnumDownloads(download_property prop, const std::wstring& value, std::vector>& out) noexcept +{ + out.clear(); + DO_DOWNLOAD_ENUM_CATEGORY category; + DO_RETURN_IF_FAILED(ConvertToComProperty(prop, category.Property)); + category.Value = value.c_str(); + return _EnumDownloads(&category, out); +} + +std::error_code CDownloadImpl::_EnumDownloads(const DO_DOWNLOAD_ENUM_CATEGORY* pCategory, std::vector>& out) noexcept +{ + out.clear(); + ComPtr manager; + RETURN_IF_FAILED(CoCreateInstance(__uuidof(DeliveryOptimization), nullptr, CLSCTX_LOCAL_SERVER, IID_PPV_ARGS(&manager))); + ComPtr spEnum; + RETURN_IF_FAILED(manager->EnumDownloads(pCategory, &spEnum)); + + std::vector> results; + ULONG cFetched = 0; + do + { + ComPtr spunk; + RETURN_IF_FAILED(spEnum->Next(1, &spunk, &cFetched)); + if (cFetched == 1) + { + ComPtr spDownload; + RETURN_IF_FAILED(spunk->QueryInterface(IID_PPV_ARGS(&spDownload))); + + RETURN_IF_FAILED(CoSetProxyBlanket(spDownload.Get(), RPC_C_AUTHN_DEFAULT, + RPC_C_AUTHZ_NONE, COLE_DEFAULT_PRINCIPAL, RPC_C_AUTHN_LEVEL_DEFAULT, RPC_C_IMP_LEVEL_IMPERSONATE, + nullptr, EOAC_STATIC_CLOAKING)); + + auto tmp = std::make_unique(); + tmp->_spDownload = std::move(spDownload); + + // Assume full file in created state, else leave ranges null + msdo::download_status status; + DO_RETURN_IF_FAILED(tmp->GetStatus(status)); + if (status.state() == msdo::download_state::created) + { + tmp->_spRanges = std::make_unique(); + tmp->_spRanges->RangeCount = 0; // empty == full file + } + + results.push_back(std::move(tmp)); + } + } while (cFetched > 0); + + out = std::move(results); + return DO_OK; +} + } // namespace details } // namespace deliveryoptimization } // namespace microsoft diff --git a/sdk-cpp/src/internal/do_download_property_internal.h b/sdk-cpp/src/internal/do_download_property_internal.h index 3085e07e..a984e441 100644 --- a/sdk-cpp/src/internal/do_download_property_internal.h +++ b/sdk-cpp/src/internal/do_download_property_internal.h @@ -23,6 +23,9 @@ namespace details { #if defined(DO_INTERFACE_COM) +std::error_code UTF8toWstr(const std::string& str, std::wstring& wstr); +std::error_code WstrToUTF8(const std::wstring& wstr, std::string& str); + struct unique_variant : VARIANT { unique_variant(); diff --git a/sdk-cpp/src/internal/do_error_helpers.h b/sdk-cpp/src/internal/do_error_helpers.h index 1e5da012..849a4668 100644 --- a/sdk-cpp/src/internal/do_error_helpers.h +++ b/sdk-cpp/src/internal/do_error_helpers.h @@ -52,19 +52,10 @@ inline std::error_code make_error_code(int32_t e) return std::error_code(e, do_category()); } -inline std::error_code make_error_code(errc e) -{ - return std::error_code(static_cast(e), do_category()); -} - -#ifndef FAILED -#define FAILED(hr) (((int32_t)(hr)) < 0) -#endif - #ifndef RETURN_IF_FAILED #define RETURN_IF_FAILED(hr) { \ int32_t __hr = (hr); \ - if (FAILED(__hr)) return std::error_code(__hr, do_category()); } + if (__hr < 0) return std::error_code(__hr, do_category()); } #endif #ifdef DO_ENABLE_EXCEPTIONS @@ -83,11 +74,6 @@ class exception : public std::exception { } - exception(errc code) : - exception(std::error_code(static_cast(code), do_category())) - { - } - const char* what() const noexcept override { return _msg.c_str(); @@ -128,11 +114,6 @@ inline void ThrowException(int32_t errorCode) throw exception(errorCode); } -inline void ThrowException(errc errorCode) -{ - throw exception(errorCode); -} - #endif // DO_ENABLE_EXCEPTIONS } // namespace details diff --git a/sdk-cpp/src/internal/download_impl.h b/sdk-cpp/src/internal/download_impl.h index 1ca2d4fa..cc12edc6 100644 --- a/sdk-cpp/src/internal/download_impl.h +++ b/sdk-cpp/src/internal/download_impl.h @@ -4,6 +4,8 @@ #ifndef _DELIVERY_OPTIMIZATION_DOWNLOAD_IMPL_H #define _DELIVERY_OPTIMIZATION_DOWNLOAD_IMPL_H +#include + #include "download_interface.h" #if defined(DO_INTERFACE_COM) @@ -39,8 +41,14 @@ class CDownloadImpl : public IDownload std::error_code SetProperty(download_property key, const download_property_value& val) noexcept override; std::error_code SetRanges(const download_range* ranges, size_t count) noexcept override; + static std::error_code EnumDownloads(std::vector>& out) noexcept; + static std::error_code EnumDownloads(download_property prop, const std::string& value, std::vector>& out) noexcept; + static std::error_code EnumDownloads(download_property prop, const std::wstring& value, std::vector>& out) noexcept; + private: #if defined(DO_INTERFACE_COM) + static std::error_code CDownloadImpl::_EnumDownloads(const DO_DOWNLOAD_ENUM_CATEGORY* pCategory, std::vector>& out) noexcept; + Microsoft::WRL::ComPtr _spDownload; std::unique_ptr _spRanges; #elif defined(DO_INTERFACE_REST) diff --git a/sdk-cpp/src/internal/rest/do_config_internal.cpp b/sdk-cpp/src/internal/rest/do_config_internal.cpp index d2410dd8..0a27f628 100644 --- a/sdk-cpp/src/internal/rest/do_config_internal.cpp +++ b/sdk-cpp/src/internal/rest/do_config_internal.cpp @@ -15,7 +15,7 @@ #include "do_persistence.h" #include "do_version.h" #elif defined(DO_CLIENT_DOSVC) -#include "do_errors.h" // msdo::errc::e_not_impl +#include "do_errors.h" // msdo::errc::not_impl #endif namespace msdo = microsoft::deliveryoptimization; @@ -190,7 +190,7 @@ void internal_free_version_buf(char** ppBuffer) int internal_set_iot_connection_string(const char* value) { - return static_cast(msdo::errc::e_not_impl); + return msdo::errc::not_impl; } char* internal_get_components_version() diff --git a/sdk-cpp/src/internal/rest/do_download_property_internal.cpp b/sdk-cpp/src/internal/rest/do_download_property_internal.cpp index daba8795..4be88d30 100644 --- a/sdk-cpp/src/internal/rest/do_download_property_internal.cpp +++ b/sdk-cpp/src/internal/rest/do_download_property_internal.cpp @@ -14,52 +14,52 @@ namespace details std::error_code CDownloadPropertyValueInternal::Init(const std::string& val) noexcept { - return make_error_code(errc::e_not_impl); + return make_error_code(errc::not_impl); } std::error_code CDownloadPropertyValueInternal::Init(const std::wstring& val) noexcept { - return make_error_code(errc::e_not_impl); + return make_error_code(errc::not_impl); } std::error_code CDownloadPropertyValueInternal::Init(uint32_t val) noexcept { - return make_error_code(errc::e_not_impl); + return make_error_code(errc::not_impl); } std::error_code CDownloadPropertyValueInternal::Init(uint64_t val) noexcept { - return make_error_code(errc::e_not_impl); + return make_error_code(errc::not_impl); } std::error_code CDownloadPropertyValueInternal::Init(bool val) noexcept { - return make_error_code(errc::e_not_impl); + return make_error_code(errc::not_impl); } std::error_code CDownloadPropertyValueInternal::As(bool& val) const noexcept { - return make_error_code(errc::e_not_impl); + return make_error_code(errc::not_impl); } std::error_code CDownloadPropertyValueInternal::As(uint32_t& val) const noexcept { - return make_error_code(errc::e_not_impl); + return make_error_code(errc::not_impl); } std::error_code CDownloadPropertyValueInternal::As(uint64_t& val) const noexcept { - return make_error_code(errc::e_not_impl); + return make_error_code(errc::not_impl); } std::error_code CDownloadPropertyValueInternal::As(std::string& val) const noexcept { - return make_error_code(errc::e_not_impl); + return make_error_code(errc::not_impl); } std::error_code CDownloadPropertyValueInternal::As(std::wstring& val) const noexcept { - return make_error_code(errc::e_not_impl); + return make_error_code(errc::not_impl); } } // namespace details diff --git a/sdk-cpp/src/internal/rest/download_impl.cpp b/sdk-cpp/src/internal/rest/download_impl.cpp index 2161e358..a1755c3d 100644 --- a/sdk-cpp/src/internal/rest/download_impl.cpp +++ b/sdk-cpp/src/internal/rest/download_impl.cpp @@ -48,7 +48,7 @@ std::error_code CDownloadImpl::Init(const std::string& uri, const std::string& d // Handle DOCS in Shutdown state while Create request was issued, client will restart and this loop will break // TODO(jimson): SDK doesn't have the capability to start do-client-lite.service if not running already. Test this // code path when it is implemented. - if (e.error_code().value() != static_cast(msdo::errc::no_service)) + if (e.error_code().value() != msdo::errc::no_service) { return e.error_code(); } @@ -137,27 +137,42 @@ std::error_code CDownloadImpl::GetStatus(msdo::download_status& outStatus) noexc std::error_code CDownloadImpl::GetProperty(msdo::download_property key, msdo::download_property_value& value) noexcept { - return make_error_code(msdo::errc::e_not_impl); + return make_error_code(msdo::errc::not_impl); } std::error_code CDownloadImpl::SetProperty(msdo::download_property key, const msdo::download_property_value& val) noexcept { - return make_error_code(msdo::errc::e_not_impl); + return make_error_code(msdo::errc::not_impl); } std::error_code CDownloadImpl::SetStatusCallback(const status_callback_t& callback, download& download) noexcept { - return make_error_code(msdo::errc::e_not_impl); + return make_error_code(msdo::errc::not_impl); } std::error_code CDownloadImpl::SetStreamCallback(const output_stream_callback_t& callback) noexcept { - return make_error_code(msdo::errc::e_not_impl); + return make_error_code(msdo::errc::not_impl); } std::error_code CDownloadImpl::SetRanges(const download_range* ranges, size_t count) noexcept { - return make_error_code(errc::e_not_impl); + return make_error_code(errc::not_impl); +} + +std::error_code CDownloadImpl::EnumDownloads(std::vector>& out) noexcept +{ + return make_error_code(errc::not_impl); +} + +std::error_code CDownloadImpl::EnumDownloads(download_property prop, const std::string& value, std::vector>& out) noexcept +{ + return make_error_code(errc::not_impl); +} + +std::error_code CDownloadImpl::EnumDownloads(download_property prop, const std::wstring& value, std::vector>& out) noexcept +{ + return make_error_code(errc::not_impl); } std::error_code CDownloadImpl::_DownloadOperationCall(const std::string& type) noexcept diff --git a/sdk-cpp/tests/download_properties_tests.cpp b/sdk-cpp/tests/download_properties_tests.cpp index d49b041c..02427424 100644 --- a/sdk-cpp/tests/download_properties_tests.cpp +++ b/sdk-cpp/tests/download_properties_tests.cpp @@ -70,7 +70,7 @@ TEST_F(DownloadPropertyTests, SmallDownloadWithPhfDigestandCvTest) std::unique_ptr simpleDownload; ASSERT_EQ(msdo::download::make(g_smallFileUrl, g_tmpFileName, simpleDownload).value(), 0); - std::vector expectedErrors = { 0, static_cast(msdo::errc::do_e_unknown_property_id) }; + std::vector expectedErrors = { 0, msdo::errc::unknown_property_id }; int32_t code = simpleDownload->set_property(msdo::download_property::integrity_check_mandatory, false).value(); VerifyError(code, expectedErrors); @@ -91,7 +91,7 @@ TEST_F(DownloadPropertyTests, InvalidPhfDigestTest) std::unique_ptr simpleDownload; ASSERT_EQ(msdo::download::make(g_smallFileUrl, g_tmpFileName, simpleDownload).value(), 0); - std::vector expectedErrors = { 0, static_cast(msdo::errc::invalid_arg) }; + std::vector expectedErrors = { 0, msdo::errc::invalid_arg }; int32_t code = simpleDownload->set_property(msdo::download_property::integrity_check_info, "blah").value(); VerifyError(code, expectedErrors); @@ -236,13 +236,37 @@ TEST_F(DownloadPropertyTests, BasicRangesTest) ASSERT_EQ(downloadedBytes, 1100); } +TEST_F(DownloadPropertyTests, BasicEnumDownloadsTest) +{ + auto download1 = msdot::download::make(g_smallFileUrl, g_tmpFileName); + + std::string downloadId1; + download1->get_property(msdo::download_property::id, downloadId1); + + std::vector> downloads; + ASSERT_EQ(msdo::download::get_downloads(msdo::download_property::id, downloadId1, downloads).value(), 0); + ASSERT_EQ(downloads.size(), 1); + + auto download2 = std::make_unique(std::move(downloads[0])); + std::string downloadId2; + download2->get_property(msdo::download_property::id, downloadId2); + ASSERT_EQ(downloadId1, downloadId2); + + download2->start_and_wait_until_completion(); + + std::this_thread::sleep_for(1s); + + ASSERT_EQ(msdo::download::get_downloads(msdo::download_property::id, downloadId1, downloads).value(), msdo::errc::no_downloads); + ASSERT_EQ(downloads.size(), 0); +} + #elif defined(DO_CLIENT_AGENT) TEST_F(DownloadPropertyTests, SmallDownloadSetCallerNameFailureTest) { msdo::download_property_value callerName; auto ec = msdo::download_property_value::make("dosdkcpp_tests", callerName); - ASSERT_EQ(ec.value(), static_cast(msdo::errc::e_not_impl)); + ASSERT_EQ(ec.value(), msdo::errc::not_impl); } #else diff --git a/sdk-cpp/tests/download_tests_common.cpp b/sdk-cpp/tests/download_tests_common.cpp index 3e8fab55..e8b9b1ca 100644 --- a/sdk-cpp/tests/download_tests_common.cpp +++ b/sdk-cpp/tests/download_tests_common.cpp @@ -296,7 +296,7 @@ TEST_F(DownloadTests, Download1NeverStartedDownload2CancelledSameFileTest) } catch (const msdod::exception& e) { - ASSERT_EQ(e.error_code().value(), static_cast(msdo::errc::not_found)); + ASSERT_EQ(e.error_code().value(), msdo::errc::not_found); } ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); } @@ -325,9 +325,9 @@ TEST_F(DownloadTests, ResumeOnAlreadyDownloadedFileTest) catch (const msdod::exception& e) { #if defined(DO_INTERFACE_COM) - ASSERT_EQ(e.error_code().value(), static_cast(msdo::errc::do_e_invalid_state)); + ASSERT_EQ(e.error_code().value(), msdo::errc::invalid_state); #elif defined(DO_INTERFACE_REST) - ASSERT_EQ(e.error_code().value(), static_cast(msdo::errc::not_found)); + ASSERT_EQ(e.error_code().value(), msdo::errc::not_found); #endif } } @@ -357,9 +357,9 @@ TEST_F(DownloadTests, CancelDownloadOnCompletedState) catch (const msdod::exception& e) { #if defined(DO_INTERFACE_COM) - ASSERT_EQ(e.error_code().value(), static_cast(msdo::errc::do_e_invalid_state)); + ASSERT_EQ(e.error_code().value(), msdo::errc::invalid_state); #elif defined(DO_INTERFACE_REST) - ASSERT_EQ(e.error_code().value(), static_cast(msdo::errc::not_found)); + ASSERT_EQ(e.error_code().value(), msdo::errc::not_found); #endif }; } @@ -389,9 +389,9 @@ TEST_F(DownloadTests, CancelDownloadInTransferredState) catch (const msdod::exception& e) { #if defined(DO_INTERFACE_COM) - ASSERT_EQ(e.error_code().value(), static_cast(msdo::errc::do_e_invalid_state)); + ASSERT_EQ(e.error_code().value(), msdo::errc::invalid_state); #elif defined(DO_INTERFACE_REST) - ASSERT_EQ(e.error_code().value(), static_cast(msdo::errc::not_found)); + ASSERT_EQ(e.error_code().value(), msdo::errc::not_found); #endif } } @@ -607,7 +607,7 @@ TEST_F(DownloadTests, SimpleBlockingDownloadTest_ClientNotRunning) } catch (const msdod::exception& e) { - ASSERT_EQ(e.error_code().value(), static_cast(msdo::errc::no_service)); + ASSERT_EQ(e.error_code().value(), msdo::errc::no_service); } ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); } @@ -631,7 +631,7 @@ TEST_F(DownloadTests, SimpleBlockingDownloadTest_ClientNotRunningPortFilePresent } catch (const msdod::exception& e) { - ASSERT_EQ(e.error_code().value(), static_cast(msdo::errc::no_service)); + ASSERT_EQ(e.error_code().value(), msdo::errc::no_service); } ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); } From 9ab56986e669c5ad05f327d3985947bffe5d6458 Mon Sep 17 00:00:00 2001 From: andretoyama-msft <106272532+andretoyama-msft@users.noreply.github.com> Date: Tue, 7 Mar 2023 03:08:58 -0300 Subject: [PATCH 08/25] Ubuntu snap: Create sdk-tests snap to test integration with the agent snap (#159) Co-authored-by: Shishir Bhat --- .gitignore | 4 + build/build-snaps.sh | 29 +++++++ sdk-cpp/tests/CMakeLists.txt | 4 + sdk-cpp/tests/test_data.cpp | 12 ++- .../snapcraft-agent.yaml | 15 +++- snapcraft-options/snapcraft-sdk.yaml | 75 +++++++++++++++++++ 6 files changed, 134 insertions(+), 5 deletions(-) create mode 100755 build/build-snaps.sh rename snap/snapcraft.yaml => snapcraft-options/snapcraft-agent.yaml (87%) create mode 100644 snapcraft-options/snapcraft-sdk.yaml diff --git a/.gitignore b/.gitignore index f75b3862..9d6e843b 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,7 @@ _deps *_CPack_Packages/ *.deb out/ + +# Ubuntu Snap files +*.snap +snap/ diff --git a/build/build-snaps.sh b/build/build-snaps.sh new file mode 100755 index 00000000..c0d6de34 --- /dev/null +++ b/build/build-snaps.sh @@ -0,0 +1,29 @@ +#! /bin/bash + +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +set -e + +if [ $1 == "agent" -o $1 == "sdk-tests" ] +then + echo "Building $1 snap..." + + if [ ! -d "./snap" ] + then + mkdir ./snap + fi + + if [ $1 == "agent" ] + then + cp ./snapcraft-options/snapcraft-agent.yaml ./snap/snapcraft.yaml + + else + cp ./snapcraft-options/snapcraft-sdk.yaml ./snap/snapcraft.yaml + fi + + snapcraft + +else + echo "$1 is not a valid snap option, valid options are 'agent' or 'sdk-tests'" +fi \ No newline at end of file diff --git a/sdk-cpp/tests/CMakeLists.txt b/sdk-cpp/tests/CMakeLists.txt index e5d3fccc..7d1978cd 100644 --- a/sdk-cpp/tests/CMakeLists.txt +++ b/sdk-cpp/tests/CMakeLists.txt @@ -69,6 +69,10 @@ add_executable(deliveryoptimization-sdk-tests ${test_source}) target_compile_definitions(deliveryoptimization-sdk-tests PRIVATE DO_ENABLE_EXCEPTIONS) add_platform_interface_definitions(deliveryoptimization-sdk-tests) +if (DO_BUILD_FOR_SNAP) + target_compile_definitions(deliveryoptimization-sdk-tests PRIVATE DO_BUILD_FOR_SNAP) +endif () + target_include_directories(deliveryoptimization-sdk-tests PRIVATE ${dosdkcpp_private_includes} diff --git a/sdk-cpp/tests/test_data.cpp b/sdk-cpp/tests/test_data.cpp index e0604e13..577c2b0b 100644 --- a/sdk-cpp/tests/test_data.cpp +++ b/sdk-cpp/tests/test_data.cpp @@ -10,11 +10,17 @@ const uint64_t g_smallFileSizeBytes = 1837u; const uint64_t g_largeFileSizeBytes = 536870440u; const uint64_t g_prodFileSizeBytes = 25006511u; +#ifdef DO_BUILD_FOR_SNAP + std::string downloadPath = "/var/lib/deliveryoptimization-snap-downloads-root"; +#else + std::string downloadPath = (boost::filesystem::temp_directory_path()).string(); +#endif + const std::string g_largeFileUrl = "http://main.oremdl.microsoft.com.nsatc.net/dotc/ReplacementDCATFile.txt"; const std::string g_malformedFilePath = "?f309adfasdf///dfasdfj39fjasldfjasdf/// ///.1"; -const std::string g_tmpFileName = (boost::filesystem::temp_directory_path() / boost::filesystem::path("docsdk_testfile.txt")).string(); -const std::string g_tmpFileName2 = (boost::filesystem::temp_directory_path() / boost::filesystem::path("docsdk_testfile2.txt")).string(); -const std::string g_tmpFileName3 = (boost::filesystem::temp_directory_path() / boost::filesystem::path("docsdk_testfile3.txt")).string(); +const std::string g_tmpFileName = (downloadPath / boost::filesystem::path("docsdk_testfile.txt")).string(); +const std::string g_tmpFileName2 = (downloadPath / boost::filesystem::path("docsdk_testfile2.txt")).string(); +const std::string g_tmpFileName3 = (downloadPath / boost::filesystem::path("docsdk_testfile3.txt")).string(); const std::string g_smallFileUrl = "http://main.oremdl.microsoft.com.nsatc.net/dotc/49c591d405d307e25e72a19f7e79b53d69f19954/43A54FC03C6A979E9AAEAE2493757D1429A5C8A8D093FB7B8103E8CC8DF7B6B6"; const std::string g_smallFilePhfInfoJson = R"( { diff --git a/snap/snapcraft.yaml b/snapcraft-options/snapcraft-agent.yaml similarity index 87% rename from snap/snapcraft.yaml rename to snapcraft-options/snapcraft-agent.yaml index 768e730a..ba61fcd0 100644 --- a/snap/snapcraft.yaml +++ b/snapcraft-options/snapcraft-agent.yaml @@ -1,7 +1,7 @@ name: deliveryoptimization-client base: core20 # the base snap is the execution environment for this snap version: '0.1' -summary: Ubuntu Core 20.04 DO +summary: Ubuntu Core 20.04 DO Agent description: | A proof-of-concept for the Delivery Optimization Client Ubuntu Core snap. @@ -64,7 +64,7 @@ parts: - libwind0-heimdal apps: - deliveryoptimization-client: + agent: command: bin/deliveryoptimization-agent daemon: simple refresh-mode: restart @@ -84,3 +84,14 @@ slots: interface: content content: config-file write: [ $SNAP_DATA/etc/deliveryoptimization-agent/sdk-config.json ] + +plugs: + client-downloads-folder: + interface: content + content: client-downloads-folder + target: $SNAP_DATA/deliveryoptimization-snap-downloads-root + +layout: +# adu_data_dir + /var/lib/deliveryoptimization-snap-downloads-root: + symlink: $SNAP_DATA/deliveryoptimization-snap-downloads-root \ No newline at end of file diff --git a/snapcraft-options/snapcraft-sdk.yaml b/snapcraft-options/snapcraft-sdk.yaml new file mode 100644 index 00000000..a2ee89d0 --- /dev/null +++ b/snapcraft-options/snapcraft-sdk.yaml @@ -0,0 +1,75 @@ +name: deliveryoptimization-sdk-tests +base: core20 # the base snap is the execution environment for this snap +version: '0.1' +summary: Ubuntu Core 20.04 DO SDK Tests +description: | + SDK snap (consumer) to test communication with the DO agent snap (producer). + +grade: devel # must be 'stable' to release into candidate/stable channels +confinement: devmode # use 'strict' once you have the right plugs and slots + +##### +# +# Keywords +# +# after - Specifies part's dependencies. +# See https://snapcraft.io/docs/parts-lifecycle#heading--step-dependencies +# https://snapcraft.io/docs/parts-lifecycle +# plugin - Specifies a plugin required for building the part. +# +# slot - Specifies a code section to be shared with other snaps using Content Interface. [PROVIDER] +# See https://snapcraft.io/docs/content-interface +# plug - Specifies a target folder to access the files shared by the provider (or providers). [CONSUMER] +# See https://snapcraft.io/docs/content-interface +# +#### + +parts: + installdeps: + plugin: nil + source: . + override-build: | + ./build/scripts/bootstrap.sh --platform ubuntu2004 --install build + + sdk: + plugin: python + source: . + override-build: | + python3 ./build/build.py --project sdk --build-for-snap + mkdir -p ../install/bin + mkdir -p ../install/lib + cp /tmp/build-deliveryoptimization-sdk/linux-debug/sdk-cpp/tests/deliveryoptimization-sdk-tests ../install/bin/deliveryoptimization-sdk-tests + cp /tmp/build-deliveryoptimization-sdk/linux-debug/sdk-cpp/libdeliveryoptimization* ../install/lib + + after: + - installdeps + + stage-packages: + - libboost-program-options1.71.0 + - libboost-filesystem1.71.0 + +apps: + sdk-tests: + command: bin/deliveryoptimization-sdk-tests + +plugs: + do-port-number: + interface: content + content: do-port-number + target: $SNAP_DATA/do-port-number + + do-config-file: + interface: content + content: do-config-file + target: $SNAP_DATA/do-config-file + +slots: + downloads-folder: + interface: content + content: downloads-folder + write: + - $SNAP_DATA/var/lib/deliveryoptimization-snap-downloads-root + +layout: + /var/lib/deliveryoptimization-snap-downloads-root: + symlink: $SNAP_DATA/var/lib/deliveryoptimization-snap-downloads-root \ No newline at end of file From ef69e59efb9b084fc18fa4c5fd14b3076cd7cd61 Mon Sep 17 00:00:00 2001 From: shishirb-MSFT <50385517+shishirb-MSFT@users.noreply.github.com> Date: Wed, 8 Mar 2023 17:16:25 -0800 Subject: [PATCH 09/25] Ubuntu snap: Use SNAP_DATA env var. Add documentation. (#160) * Agent: Use SNAP_DATA environment variable to ensure agent is writing to directories within the snap filesystem. Without this, the agent was writing to the host filesystem (in devmode). * SDK: Corresponding change required to use SNAP_DATA so that the plugs are used. * Update slot name for the downloads to be unique to DU agent. * Add documentation about snap commands and general info. --- client-lite/CMakeLists.txt | 12 +++- client-lite/src/exe/docs.cpp | 6 +- client-lite/src/util/do_persistence.cpp | 34 +++++++++-- client-lite/src/util/proc_launch_helper.h | 8 +-- sdk-cpp/CMakeLists.txt | 3 + .../src/internal/rest/util/do_persistence.cpp | 38 ++++++++++-- sdk-cpp/tests/download_tests_common.cpp | 16 ++--- sdk-cpp/tests/rest/port_discovery_tests.cpp | 5 ++ sdk-cpp/tests/rest/test_helpers.cpp | 9 ++- sdk-cpp/tests/test_data.cpp | 12 +++- sdk-cpp/tests/test_data.h | 9 ++- snapcraft-options/connect-snaps.sh | 15 +++++ snapcraft-options/snapcraft-agent.yaml | 30 ++++++---- snapcraft-options/snapcraft-sdk.yaml | 26 +++++---- snapcraft-options/ubuntu-snap.md | 58 +++++++++++++++++++ 15 files changed, 227 insertions(+), 54 deletions(-) create mode 100755 snapcraft-options/connect-snaps.sh create mode 100644 snapcraft-options/ubuntu-snap.md diff --git a/client-lite/CMakeLists.txt b/client-lite/CMakeLists.txt index d0a72836..35d95007 100644 --- a/client-lite/CMakeLists.txt +++ b/client-lite/CMakeLists.txt @@ -68,8 +68,16 @@ file (GLOB files_docs_common src/util/*.cpp ) -# Enable easy debugging in devdebug mode by not requiring running as root -if (DO_DEV_DEBUG) +if (DO_BUILD_FOR_SNAP) + # Use relative paths here. Agent will prefix them with $SNAP_DATA environment variable at runtime. + # This should match the slots declared in snapcraft-agent.yaml. + # These cmake vars are used in Debian maintainer scripts like postrm/postinst but snap does not use those scripts. + message("Agent: Build to create an Ubuntu Snap package") + set(docs_svc_config_dir_path "etc") + set(docs_svc_log_dir_path "log") + set(docs_svc_run_dir_path "run") +elseif (DO_DEV_DEBUG) + # Enable easy debugging in devdebug mode by not requiring running as root message("Agent: Dev debug mode") set(docs_svc_config_dir_path "/tmp/etc/${DOSVC_BIN_NAME}") set(docs_svc_log_dir_path "/tmp/log/${DOSVC_BIN_NAME}") diff --git a/client-lite/src/exe/docs.cpp b/client-lite/src/exe/docs.cpp index 23fc5a56..8360ce00 100644 --- a/client-lite/src/exe/docs.cpp +++ b/client-lite/src/exe/docs.cpp @@ -139,15 +139,17 @@ HRESULT Run() try RestPortAdvertiser portAdvertiser(controller.Port()); DoLogInfo("Port number written to %s", portAdvertiser.OutFilePath().data()); - + #ifndef DO_BUILD_FOR_SNAP DropPermissions(); #endif - DOLog::Init(docli::GetLogDirectory(), DOLog::Level::Verbose); DoLogInfo("Started, %s", msdoutil::ComponentVersion().c_str()); + DoLogInfo("**Paths**\nLog: %s\nRun: %s\nConfig: %s\nSdkConfig: %s\nAdminConfig: %s", + docli::GetLogDirectory().c_str(), docli::GetRuntimeDirectory().c_str(), docli::GetConfigDirectory().c_str(), + docli::GetSDKConfigFilePath().c_str(), docli::GetAdminConfigFilePath().c_str()); ProcessController procController([&downloadManager]() { diff --git a/client-lite/src/util/do_persistence.cpp b/client-lite/src/util/do_persistence.cpp index 68cac515..9e677e9d 100644 --- a/client-lite/src/util/do_persistence.cpp +++ b/client-lite/src/util/do_persistence.cpp @@ -4,36 +4,60 @@ #include "do_common.h" #include "do_persistence.h" +#include // std::getenv + namespace docli { +static std::string ConstructPath(const char* suffix) +{ + std::string outputPath; +#ifdef DO_BUILD_FOR_SNAP + const char* snapDataRoot = std::getenv("SNAP_DATA"); + THROW_HR_IF(E_UNEXPECTED, (snapDataRoot == nullptr) || (*snapDataRoot == '\0')); + outputPath = snapDataRoot; + if (outputPath.back() != '/') + { + outputPath.push_back('/'); + } + if (*suffix == '/') + { + ++suffix; + } + outputPath += suffix; +#else + outputPath = suffix; +#endif + return outputPath; +} + const std::string& GetLogDirectory() { - static std::string logDirectory(DO_AGENT_LOG_DIRECTORY_PATH); + static std::string logDirectory(ConstructPath(DO_AGENT_LOG_DIRECTORY_PATH)); return logDirectory; } const std::string& GetRuntimeDirectory() { - static std::string runDirectory(DO_RUN_DIRECTORY_PATH); + static std::string runDirectory(ConstructPath(DO_RUN_DIRECTORY_PATH)); return runDirectory; } const std::string& GetConfigDirectory() { - static std::string configDirectory(DO_CONFIG_DIRECTORY_PATH); + static std::string configDirectory(ConstructPath(DO_CONFIG_DIRECTORY_PATH)); return configDirectory; } const std::string& GetSDKConfigFilePath() { - static std::string configFilePath(DO_CONFIG_DIRECTORY_PATH "/sdk-config.json"); + static std::string configFilePath(ConstructPath(DO_CONFIG_DIRECTORY_PATH "/sdk-config.json")); return configFilePath; } const std::string& GetAdminConfigFilePath() { - static std::string configFilePath(DO_CONFIG_DIRECTORY_PATH "/admin-config.json"); + static std::string configFilePath(ConstructPath(DO_CONFIG_DIRECTORY_PATH "/admin-config.json")); return configFilePath; } diff --git a/client-lite/src/util/proc_launch_helper.h b/client-lite/src/util/proc_launch_helper.h index 2917dd64..6c4abfde 100644 --- a/client-lite/src/util/proc_launch_helper.h +++ b/client-lite/src/util/proc_launch_helper.h @@ -57,8 +57,8 @@ inline void InitializePath(const std::string& path, mode_t mode = 0) try boost::filesystem::path dirPath(path); if (!boost::filesystem::exists(dirPath)) { - DoLogInfo("Creating directory at %s", path.c_str()); - boost::filesystem::create_directory(dirPath); + DoLogInfo("Creating directories for %s", path.c_str()); + boost::filesystem::create_directories(dirPath); if (mode != 0) { @@ -80,8 +80,6 @@ inline void InitializeDOPaths() inline void DropPermissions() { - uid_t userid = GetUserIdByName("do"); - // process is running as root, drop privileges if (getuid() == 0) { @@ -94,6 +92,8 @@ inline void DropPermissions() { THROW_HR_MSG(E_FAIL, "setgid: Unable to drop group privileges: %u, errno: %d", groupid, errno); } + + uid_t userid = GetUserIdByName("do"); if (setuid(userid) != 0) { THROW_HR_MSG(E_FAIL, "setuid: Unable to drop user privileges: %u, errno: %d", userid, errno); diff --git a/sdk-cpp/CMakeLists.txt b/sdk-cpp/CMakeLists.txt index f43227d5..8c3070a1 100644 --- a/sdk-cpp/CMakeLists.txt +++ b/sdk-cpp/CMakeLists.txt @@ -124,6 +124,9 @@ target_compile_definitions(${DO_SDK_LIB_NAME} if (DO_DEV_DEBUG) target_compile_definitions(${DO_SDK_LIB_NAME} PRIVATE DO_DEV_DEBUG) endif () +if (DO_BUILD_FOR_SNAP) + target_compile_definitions(${DO_SDK_LIB_NAME} PRIVATE DO_BUILD_FOR_SNAP) +endif () if (DO_PLATFORM_LINUX OR DO_PLATFORM_MAC) # TODO(shishirb) Remove internal use of exceptions on these platforms target_compile_definitions(${DO_SDK_LIB_NAME} PRIVATE DO_ENABLE_EXCEPTIONS) diff --git a/sdk-cpp/src/internal/rest/util/do_persistence.cpp b/sdk-cpp/src/internal/rest/util/do_persistence.cpp index 0ca80df3..025dd655 100644 --- a/sdk-cpp/src/internal/rest/util/do_persistence.cpp +++ b/sdk-cpp/src/internal/rest/util/do_persistence.cpp @@ -2,9 +2,11 @@ // Licensed under the MIT License. #include - #include "do_persistence.h" +#include "do_errors.h" +#include "do_error_helpers.h" + namespace microsoft { namespace deliveryoptimization @@ -12,9 +14,33 @@ namespace deliveryoptimization namespace details { +#ifdef DO_BUILD_FOR_SNAP +static std::string ConstructPath(const char* suffix) +{ + const char* snapDataRoot = std::getenv("SNAP_DATA"); + if ((snapDataRoot == nullptr) || (*snapDataRoot == '\0')) + { + ThrowException(microsoft::deliveryoptimization::errc::unexpected); + } + std::string outputPath = snapDataRoot; + if (outputPath.back() != '/') + { + outputPath.push_back('/'); + } + if (*suffix == '/') + { + ++suffix; + } + outputPath += suffix; + return outputPath; +} +#endif + const std::string& GetRuntimeDirectory() { -#ifdef DO_DEV_DEBUG +#ifdef DO_BUILD_FOR_SNAP + static std::string runDirectory(ConstructPath("do-port-numbers")); +#elif defined(DO_DEV_DEBUG) static std::string runDirectory("/tmp/run/deliveryoptimization-agent"); #else static std::string runDirectory("/var/run/deliveryoptimization-agent"); @@ -24,7 +50,9 @@ const std::string& GetRuntimeDirectory() const std::string& GetConfigFilePath() { -#ifdef DO_DEV_DEBUG +#ifdef DO_BUILD_FOR_SNAP + static std::string configFilePath(ConstructPath("do-configs/sdk-config.json")); +#elif defined(DO_DEV_DEBUG) static std::string configFilePath("/tmp/etc/deliveryoptimization-agent/sdk-config.json"); #else static std::string configFilePath("/etc/deliveryoptimization-agent/sdk-config.json"); @@ -35,7 +63,9 @@ const std::string& GetConfigFilePath() // TODO(shishirb): this is used only in test const std::string& GetAdminConfigFilePath() { -#ifdef DO_DEV_DEBUG +#ifdef DO_BUILD_FOR_SNAP + static std::string configFilePath(ConstructPath("do-configs/admin-config.json")); +#elif defined(DO_DEV_DEBUG) static std::string configFilePath("/tmp/etc/deliveryoptimization-agent/admin-config.json"); #else static std::string configFilePath("/etc/deliveryoptimization-agent/admin-config.json"); diff --git a/sdk-cpp/tests/download_tests_common.cpp b/sdk-cpp/tests/download_tests_common.cpp index e8b9b1ca..616b07ab 100644 --- a/sdk-cpp/tests/download_tests_common.cpp +++ b/sdk-cpp/tests/download_tests_common.cpp @@ -588,14 +588,15 @@ TEST_F(DownloadTests, FileDeletionAfterPause) #endif } -#if defined(DO_INTERFACE_REST) +// SNAP: tests only have read permissions into the 'port numbers' directory +#if defined(DO_INTERFACE_REST) and !defined(DO_BUILD_FOR_SNAP) TEST_F(DownloadTests, SimpleBlockingDownloadTest_ClientNotRunning) { - TestHelpers::StopService("deliveryoptimization-agent.service"); + TestHelpers::StopService(g_docsSvcName.c_str()); auto startService = dotest::util::scope_exit([]() { - TestHelpers::StartService("deliveryoptimization-agent.service"); + TestHelpers::StartService(g_docsSvcName.c_str()); }); TestHelpers::DeleteRestPortFiles(); // can be removed if docs deletes file on shutdown @@ -614,11 +615,10 @@ TEST_F(DownloadTests, SimpleBlockingDownloadTest_ClientNotRunning) TEST_F(DownloadTests, SimpleBlockingDownloadTest_ClientNotRunningPortFilePresent) { - // TODO(shishirb) Service name should come from cmake - TestHelpers::StopService("deliveryoptimization-agent.service"); + TestHelpers::StopService(g_docsSvcName.c_str()); auto startService = dotest::util::scope_exit([]() { - TestHelpers::StartService("deliveryoptimization-agent.service"); + TestHelpers::StartService(g_docsSvcName.c_str()); }); TestHelpers::DeleteRestPortFiles(); TestHelpers::CreateRestPortFiles(1); @@ -638,10 +638,10 @@ TEST_F(DownloadTests, SimpleBlockingDownloadTest_ClientNotRunningPortFilePresent TEST_F(DownloadTests, MultipleRestPortFileExists_Download) { - TestHelpers::StopService("deliveryoptimization-agent.service"); + TestHelpers::StopService(g_docsSvcName.c_str()); auto startService = dotest::util::scope_exit([]() { - TestHelpers::StartService("deliveryoptimization-agent.service"); + TestHelpers::StartService(g_docsSvcName.c_str()); }); TestHelpers::CreateRestPortFiles(5); ASSERT_GE(TestHelpers::CountRestPortFiles(), 5u); diff --git a/sdk-cpp/tests/rest/port_discovery_tests.cpp b/sdk-cpp/tests/rest/port_discovery_tests.cpp index 3ad0d690..6bd2214d 100644 --- a/sdk-cpp/tests/rest/port_discovery_tests.cpp +++ b/sdk-cpp/tests/rest/port_discovery_tests.cpp @@ -52,8 +52,13 @@ void PortDiscoveryTests::TearDown() TestHelpers::StartService(g_docsSvcName); } +// SNAP: tests only have read permissions into the 'port numbers' directory +#ifndef DO_BUILD_FOR_SNAP + TEST_F(PortDiscoveryTests, DiscoverPortTest) { std::string foundPort = msdod::CPortFinder::GetDOPort(false); ASSERT_EQ(foundPort, samplePortNumber); } + +#endif diff --git a/sdk-cpp/tests/rest/test_helpers.cpp b/sdk-cpp/tests/rest/test_helpers.cpp index 02d62c5b..62c58d11 100644 --- a/sdk-cpp/tests/rest/test_helpers.cpp +++ b/sdk-cpp/tests/rest/test_helpers.cpp @@ -46,21 +46,24 @@ int TestHelpers::ShutdownProcess(std::string procName) void TestHelpers::RestartService(const std::string& name) { +#ifndef DO_BUILD_FOR_SNAP + // 'snap' does not support reset-failed const auto resetCmd = dtu::FormatString("systemctl reset-failed %s", name.c_str()); dtu::ExecuteSystemCommand(resetCmd.data()); // avoids hitting start-limit-hit error in systemd +#endif - const auto restartCmd = dtu::FormatString("systemctl restart %s", name.c_str()); + const auto restartCmd = dtu::FormatString(DO_SERVICE_CONTROLLER " restart %s", name.c_str()); dtu::ExecuteSystemCommand(restartCmd.data()); } void TestHelpers::StartService(const std::string& name) { - dtu::ExecuteSystemCommand(dtu::FormatString("systemctl start %s", name.c_str()).data()); + dtu::ExecuteSystemCommand(dtu::FormatString(DO_SERVICE_CONTROLLER " start %s", name.c_str()).data()); } void TestHelpers::StopService(const std::string& name) { - dtu::ExecuteSystemCommand(dtu::FormatString("systemctl stop %s", name.c_str()).data()); + dtu::ExecuteSystemCommand(dtu::FormatString(DO_SERVICE_CONTROLLER " stop %s", name.c_str()).data()); } int TestHelpers::_KillProcess(int pid, int signal) diff --git a/sdk-cpp/tests/test_data.cpp b/sdk-cpp/tests/test_data.cpp index 577c2b0b..cbc01004 100644 --- a/sdk-cpp/tests/test_data.cpp +++ b/sdk-cpp/tests/test_data.cpp @@ -11,9 +11,10 @@ const uint64_t g_largeFileSizeBytes = 536870440u; const uint64_t g_prodFileSizeBytes = 25006511u; #ifdef DO_BUILD_FOR_SNAP - std::string downloadPath = "/var/lib/deliveryoptimization-snap-downloads-root"; +// For testing production scenario, use the same path DU agent will use +static const std::string downloadPath = "/var/lib/deviceupdate-agent-downloads"; #else - std::string downloadPath = (boost::filesystem::temp_directory_path()).string(); +static const std::string downloadPath = boost::filesystem::temp_directory_path().string(); #endif const std::string g_largeFileUrl = "http://main.oremdl.microsoft.com.nsatc.net/dotc/ReplacementDCATFile.txt"; @@ -35,9 +36,14 @@ const std::chrono::seconds g_smallFileWaitTime = 10s; const std::chrono::seconds g_largeFileWaitTime = 5min; #if defined(DO_INTERFACE_REST) -const std::string g_docsProcName = "deliveryoptimization-agent"; + +#ifdef DO_BUILD_FOR_SNAP +const std::string g_docsSvcName = "deliveryoptimization-client.agent"; +#else const std::string g_docsSvcName = "deliveryoptimization-agent.service"; #endif +#endif + // This MCC instance only works within our test lab azure VMs. Can be overriden via cmdline. std::string g_mccHostName = "10.1.0.70"; diff --git a/sdk-cpp/tests/test_data.h b/sdk-cpp/tests/test_data.h index da6eec6a..6ffeba93 100644 --- a/sdk-cpp/tests/test_data.h +++ b/sdk-cpp/tests/test_data.h @@ -7,12 +7,19 @@ #include #include +#ifdef DO_BUILD_FOR_SNAP +// Using systemctl within snap package results in "Running in chroot, ignoring request". +// Use the snap command instead. +#define DO_SERVICE_CONTROLLER "snap" +#else +#define DO_SERVICE_CONTROLLER "systemctl" +#endif + extern const uint64_t g_smallFileSizeBytes; extern const uint64_t g_largeFileSizeBytes; extern const uint64_t g_prodFileSizeBytes; #if defined(DO_INTERFACE_REST) -extern const std::string g_docsProcName; extern const std::string g_docsSvcName; #elif defined(DO_INTERFACE_COM) extern const std::string g_smallFilePhfInfoJson; diff --git a/snapcraft-options/connect-snaps.sh b/snapcraft-options/connect-snaps.sh new file mode 100755 index 00000000..3ba11251 --- /dev/null +++ b/snapcraft-options/connect-snaps.sh @@ -0,0 +1,15 @@ +#! /bin/bash + +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# $ snap connect : : + +set -e + +# Plug agent snap into sdk-tests snap downloads slot +sudo snap connect deliveryoptimization-client:deviceupdate-agent-downloads deliveryoptimization-sdk-tests:downloads-folder + +# Plug sdk-tests snap into agent's run and config slots +sudo snap connect deliveryoptimization-sdk-tests:do-port-numbers deliveryoptimization-client:do-port-numbers +sudo snap connect deliveryoptimization-sdk-tests:do-configs deliveryoptimization-client:do-configs diff --git a/snapcraft-options/snapcraft-agent.yaml b/snapcraft-options/snapcraft-agent.yaml index ba61fcd0..83874071 100644 --- a/snapcraft-options/snapcraft-agent.yaml +++ b/snapcraft-options/snapcraft-agent.yaml @@ -75,23 +75,29 @@ apps: - network-bind slots: - port-number: + do-port-numbers: interface: content - content: port-number - read: [ $SNAP_DATA/var/run/deliveryoptimization-agent ] + content: do-port-numbers + read: [ $SNAP_DATA/run ] - config-file: + do-configs: interface: content - content: config-file - write: [ $SNAP_DATA/etc/deliveryoptimization-agent/sdk-config.json ] + content: do-configs + write: [ $SNAP_DATA/etc ] plugs: - client-downloads-folder: + deviceupdate-agent-downloads: interface: content - content: client-downloads-folder - target: $SNAP_DATA/deliveryoptimization-snap-downloads-root + content: deviceupdate-agent-downloads + target: $SNAP_DATA/deviceupdate-agent-downloads layout: -# adu_data_dir - /var/lib/deliveryoptimization-snap-downloads-root: - symlink: $SNAP_DATA/deliveryoptimization-snap-downloads-root \ No newline at end of file + # DU agent will provide the /var/lib path as download location. + # Map it to the correct path within this snap. + /var/lib/deviceupdate-agent-downloads: + symlink: $SNAP_DATA/deviceupdate-agent-downloads + + # Not using layout for /var/run because /var/run is already a symlink in Ubuntu 20.04, + # and package startup fails if we want another symlink there. + # Not using /var/log and /etc just for consistency: snap adn non-snap versions do not use + # the same directory paths. diff --git a/snapcraft-options/snapcraft-sdk.yaml b/snapcraft-options/snapcraft-sdk.yaml index a2ee89d0..95fa8cd5 100644 --- a/snapcraft-options/snapcraft-sdk.yaml +++ b/snapcraft-options/snapcraft-sdk.yaml @@ -53,23 +53,29 @@ apps: command: bin/deliveryoptimization-sdk-tests plugs: - do-port-number: + do-port-numbers: interface: content - content: do-port-number - target: $SNAP_DATA/do-port-number + content: do-port-numbers + target: $SNAP_DATA/do-port-numbers - do-config-file: + do-configs: interface: content - content: do-config-file - target: $SNAP_DATA/do-config-file + content: do-configs + target: $SNAP_DATA/do-configs slots: downloads-folder: interface: content content: downloads-folder - write: - - $SNAP_DATA/var/lib/deliveryoptimization-snap-downloads-root + write: [ $SNAP_DATA/var/lib/deviceupdate-agent-downloads ] layout: - /var/lib/deliveryoptimization-snap-downloads-root: - symlink: $SNAP_DATA/var/lib/deliveryoptimization-snap-downloads-root \ No newline at end of file + # For testing production scenario, use the same path DU agent will use. + # The path is referred to in sdk-cpp/tests/test_data.cpp. + /var/lib/deviceupdate-agent-downloads: + symlink: $SNAP_DATA/var/lib/deviceupdate-agent-downloads + + # Not using layout for agent's restport files because /var/run is already a symlink in Ubuntu 20.04, + # and package startup fails if we want another symlink there. + # Not using layout for the configs, the /etc directory, just for consistency: snap and non-snap versions use + # different directory paths. diff --git a/snapcraft-options/ubuntu-snap.md b/snapcraft-options/ubuntu-snap.md new file mode 100644 index 00000000..a256bb48 --- /dev/null +++ b/snapcraft-options/ubuntu-snap.md @@ -0,0 +1,58 @@ +# Ubuntu Snap packages - What is it, how to build, and how to test + +## What +https://snapcraft.io/docs + +## Building +Run the commands in the root of the repo. + +- First time build or after modifying corresponding yaml file: + - `$ ./build/build-snaps.sh agent` + - `$ ./build/build-snaps.sh sdk-tests` + +- Subsequent builds of the same component without modifying the corresponding yaml file: + - `$ snapcraft` + +- The build will generate *.snap files in the current working directory. Example: **deliveryoptimization-sdk-tests_0.1_amd64.snap**. + +## Installing +- `$ sudo snap install --devmode ./deliveryoptimization-sdk-tests_0.1_amd64.snap` +- `$ sudo snap install --devmode ./deliveryoptimization-client_0.1_amd64.snap` + +## The snap environment +It is a fruitful exercise to look around in the host file system and see how snap structures the installed snaps. Look at these paths to start with: +- /snap/ +- /var/snap/ + +## Connecting plugs and slots +- For the SDK test snap to work, the plugs and slots must be connected. Run the **connect-snaps.sh** script after both snaps are installed. +- The connections can be listed using this command: + - `$ snap connections | grep delivery` + +## Running or executing +- Agent + - The agent is declared as a daemon/service, so it starts running immediately after successful installation. + - Stop service: `$ sudo snap stop deliveryoptimization-client.agent` + - Start service: `$ sudo snap start deliveryoptimization-client.agent` + - Read service journal: `$ sudo snap logs deliveryoptimization-client.agent` + - systemctl can also be used with the correct service unit name: `$ systemctl status snap.deliveryoptimization-client.agent.service` + - **journalctl** can also be used to follow logs: `$ journalctl -f -u snap.deliveryoptimization-client.agent.service` + +- SDK tests + - This snap declares the **deliveryoptimization-sdk-tests** binary as an app. + - One way to run the tests is: `$ sudo snap run deliveryoptimization-sdk-tests.sdk-tests --gtest_filter=*SimpleDownloadTest` + - Advanced usage: get a shell into the snap and invoke the binary directly. This also allows us to inspect the runtime environment + more easily, like listing environment variables, listing files/dirs, etc.
+ Use `$ sudo snap run --shell deliveryoptimization-sdk-tests.sdk-tests` to open a shell into the snap.
+ Then we can use regular shell commands like `printenv | grep SNAP`, `cd $SNAP`, `ls -l $SNAP_DATA/do-port-numbers/`
+ We can also run the test binary like so: `cd $SNAP` and then `bin/deliveryoptimization-sdk-tests --gtest_filter=*SimpleDownloadTest` + +## Uninstalling or cleaning up +- Uninstall installed snap packages + - `$ sudo snap remove deliveryoptimization-client` + - `$ sudo snap remove deliveryoptimization-sdk-tests` + +- Cleaning up build environment: Building sometimes fails with strange errors from **multipass**. Cleaning up the multipass instance helps. + - List the available instances: `$ multipass list` + - Delete an instance, example: `$ multipass delete snapcraft-deliveryoptimization-client` + - Purge deleted instances: `$ multipass purge` From 1e1468e2cd356a072522234dc0ec1668c35700da Mon Sep 17 00:00:00 2001 From: shishirb-MSFT <50385517+shishirb-MSFT@users.noreply.github.com> Date: Fri, 10 Mar 2023 08:16:12 -0800 Subject: [PATCH 10/25] SDK: Remove unused test helpers (#161) --- sdk-cpp/tests/rest/test_helpers.cpp | 71 ----------------------------- sdk-cpp/tests/test_helpers.h | 7 --- 2 files changed, 78 deletions(-) diff --git a/sdk-cpp/tests/rest/test_helpers.cpp b/sdk-cpp/tests/rest/test_helpers.cpp index 62c58d11..c2c2c235 100644 --- a/sdk-cpp/tests/rest/test_helpers.cpp +++ b/sdk-cpp/tests/rest/test_helpers.cpp @@ -29,21 +29,6 @@ namespace dtu = dotest::util; namespace msdod = microsoft::deliveryoptimization::details; -bool TestHelpers::IsActiveProcess(std::string procName) -{ - return _GetPidFromProcName(procName) != -1; -} - -// TODO(jimson): Enable starting/shutting down docs as service -int TestHelpers::ShutdownProcess(std::string procName) -{ - if (IsActiveProcess(procName)) - { - return _KillProcess(_GetPidFromProcName(procName), SIGINT); - } - return -1; -} - void TestHelpers::RestartService(const std::string& name) { #ifndef DO_BUILD_FOR_SNAP @@ -66,62 +51,6 @@ void TestHelpers::StopService(const std::string& name) dtu::ExecuteSystemCommand(dtu::FormatString(DO_SERVICE_CONTROLLER " stop %s", name.c_str()).data()); } -int TestHelpers::_KillProcess(int pid, int signal) -{ - try - { - return kill(pid, signal); - } - catch (...) - { - // TODO(jimson): Implement Logging for Tests - return -1; - } -} - -int TestHelpers::_GetPidFromProcName(std::string procName) -{ - int pid = -1; - - // Open the /proc directory - DIR *dp = opendir("/proc"); - if (dp != NULL) - { - // Enumerate all entries in directory until process found - struct dirent *dirp; - while (pid < 0 && (dirp = readdir(dp))) - { - // Skip non-numeric entries - int id = atoi(dirp->d_name); - if (id > 0) - { - // Read contents of virtual /proc/{pid}/cmdline file - std::string cmdPath = std::string("/proc/") + dirp->d_name + "/cmdline"; - std::ifstream cmdFile(cmdPath.c_str()); - std::string cmdLine; - getline(cmdFile, cmdLine); - if (!cmdLine.empty()) - { - // Keep first cmdline item which contains the program path - size_t pos = cmdLine.find('\0'); - if (pos != std::string::npos) - cmdLine = cmdLine.substr(0, pos); - // Keep program name only, removing the path - pos = cmdLine.rfind('/'); - if (pos != std::string::npos) - cmdLine = cmdLine.substr(pos + 1); - // Compare against requested process name - if (procName == cmdLine) - pid = id; - } - } - } - } - closedir(dp); - - return pid; -} - void TestHelpers::CreateRestPortFiles(int numFiles) { std::string portFile; diff --git a/sdk-cpp/tests/test_helpers.h b/sdk-cpp/tests/test_helpers.h index 40f23b27..f638a1c8 100644 --- a/sdk-cpp/tests/test_helpers.h +++ b/sdk-cpp/tests/test_helpers.h @@ -212,8 +212,6 @@ class TestHelpers } #ifdef DO_INTERFACE_REST - static bool IsActiveProcess(std::string name); - static int ShutdownProcess(std::string name); static void RestartService(const std::string& name); static void StartService(const std::string& name); static void StopService(const std::string& name); @@ -227,11 +225,6 @@ class TestHelpers #endif // DO_INTERFACE_REST private: -#ifdef DO_INTERFACE_REST - static int _GetPidFromProcName(std::string name); - static int _KillProcess(int pid, int signal); -#endif // DO_INTERFACE_REST - // Disallow creating an instance of this object TestHelpers() {} From 62d76bc0fd0dadd10c7524ffd29fe3d4de151740 Mon Sep 17 00:00:00 2001 From: shishirb-MSFT <50385517+shishirb-MSFT@users.noreply.github.com> Date: Fri, 10 Mar 2023 16:13:25 -0800 Subject: [PATCH 11/25] Enable bootstrap.sh to determine OS and version on its own (#162) --- build/docker/debian10/amd64/Dockerfile | 25 ++- build/docker/debian10/arm32/Dockerfile | 2 +- build/docker/debian10/arm64/Dockerfile | 2 +- build/docker/ubuntu1804/arm64/Dockerfile | 2 +- build/docker/ubuntu2004/amd64/Dockerfile | 2 +- build/docker/ubuntu2004/arm64/Dockerfile | 2 +- build/scripts/bootstrap.sh | 184 +++++++++++++++++------ snapcraft-options/snapcraft-agent.yaml | 2 +- snapcraft-options/snapcraft-sdk.yaml | 2 +- 9 files changed, 152 insertions(+), 71 deletions(-) diff --git a/build/docker/debian10/amd64/Dockerfile b/build/docker/debian10/amd64/Dockerfile index 965f7550..915c7b19 100644 --- a/build/docker/debian10/amd64/Dockerfile +++ b/build/docker/debian10/amd64/Dockerfile @@ -2,35 +2,28 @@ # First, install the docker extension for VSCode. Then you can right-click on this file # and choose Build Image. Give it a name and it will build the image. # +# Alternatively, use the command line: +# Copy the build script to the build directory: +# cp /build/scripts/bootstrap.sh /build/docker/debian10/amd64 +# +# After running the above, you can build the image by running in the current dockerfile directory: +# sudo docker build -t debian10_amd64 . --no-cache --network=host +# # Open interactive terminal into the image in a container: # docker run -ti --rm --entrypoint=/bin/bash -v :/code -v :/build # Example: -# docker run -ti --rm --entrypoint=/bin/bash -v D:\do-client-lite:/code -v D:\temp\build_client_lite\arm-linux-debug:/build custom-debian10-amd64 +# sudo docker run -ti --rm --entrypoint=/bin/bash -v ~/code/do-client/:/code debian10_amd64 FROM mcr.microsoft.com/mirror/docker/library/debian:buster@sha256:0e328f15f5cb0f93bff52a37d98f0bbfc40f97fcaf250096fd2b2c6871a497eb SHELL [ "/bin/bash", "-c"] -# QEMU is a Linux emulator which enables cross-arch support in docker -# In order to build this image on a Linux host, need to install QEMU: -# -# sudo apt-get install qemu-user -# update-binfmts --display -# sudo apt install qemu binfmt-support qemu-user-static -# cp /usr/bin/qemu-arm-static /build/docker/debian10/amd64 -# -# Then copy the build script to the build directory -# cp /build/scripts/bootstrap.sh /build/docker/debian10/amd64 -# -# After running the above, you can build the image by running in the current dockerfile directory -# sudo docker build -t debian10_amd64 . --no-cache --network=host - COPY bootstrap.sh /tmp/bootstrap.sh WORKDIR /tmp/ RUN chmod +x bootstrap.sh -RUN ./bootstrap.sh --platform debian10 --install build +RUN ./bootstrap.sh --install build VOLUME /code WORKDIR /code diff --git a/build/docker/debian10/arm32/Dockerfile b/build/docker/debian10/arm32/Dockerfile index c3f7c424..11954ed1 100644 --- a/build/docker/debian10/arm32/Dockerfile +++ b/build/docker/debian10/arm32/Dockerfile @@ -30,7 +30,7 @@ COPY bootstrap.sh /tmp/bootstrap.sh WORKDIR /tmp/ RUN chmod +x bootstrap.sh -RUN ./bootstrap.sh --platform debian10 --install build +RUN ./bootstrap.sh --install build VOLUME /code WORKDIR /code diff --git a/build/docker/debian10/arm64/Dockerfile b/build/docker/debian10/arm64/Dockerfile index 25687e58..dc770aac 100644 --- a/build/docker/debian10/arm64/Dockerfile +++ b/build/docker/debian10/arm64/Dockerfile @@ -43,7 +43,7 @@ COPY bootstrap.sh /tmp/bootstrap.sh WORKDIR /tmp/ RUN chmod +x bootstrap.sh -RUN ./bootstrap.sh --platform debian10 --install build +RUN ./bootstrap.sh --install build VOLUME /code WORKDIR /code diff --git a/build/docker/ubuntu1804/arm64/Dockerfile b/build/docker/ubuntu1804/arm64/Dockerfile index b6314d01..36ff2019 100644 --- a/build/docker/ubuntu1804/arm64/Dockerfile +++ b/build/docker/ubuntu1804/arm64/Dockerfile @@ -30,7 +30,7 @@ COPY bootstrap.sh /tmp/bootstrap.sh WORKDIR /tmp/ RUN chmod +x bootstrap.sh -RUN ./bootstrap.sh --platform ubuntu1804 --install build +RUN ./bootstrap.sh --install build VOLUME /code WORKDIR /code diff --git a/build/docker/ubuntu2004/amd64/Dockerfile b/build/docker/ubuntu2004/amd64/Dockerfile index 9366005d..0bcac670 100644 --- a/build/docker/ubuntu2004/amd64/Dockerfile +++ b/build/docker/ubuntu2004/amd64/Dockerfile @@ -21,7 +21,7 @@ COPY bootstrap.sh /tmp/bootstrap.sh WORKDIR /tmp/ RUN chmod +x bootstrap.sh -RUN ./bootstrap.sh --platform ubuntu2004 --install build +RUN ./bootstrap.sh --install build VOLUME /code WORKDIR /code diff --git a/build/docker/ubuntu2004/arm64/Dockerfile b/build/docker/ubuntu2004/arm64/Dockerfile index 9b04bc88..a47701e1 100644 --- a/build/docker/ubuntu2004/arm64/Dockerfile +++ b/build/docker/ubuntu2004/arm64/Dockerfile @@ -33,7 +33,7 @@ COPY bootstrap.sh /tmp/bootstrap.sh WORKDIR /tmp/ RUN chmod +x bootstrap.sh -RUN ./bootstrap.sh --platform ubuntu2004 --install build +RUN ./bootstrap.sh --install build VOLUME /code WORKDIR /code diff --git a/build/scripts/bootstrap.sh b/build/scripts/bootstrap.sh index 1f8fe4a8..c42441ad 100755 --- a/build/scripts/bootstrap.sh +++ b/build/scripts/bootstrap.sh @@ -16,16 +16,18 @@ set -e # Defaults INSTALL=all -PLATFORM=unknown + +OS="" +DISTRO="" +VER="" +ARCH="" function usage { cat < --install - --platform # Platform to provision, supported platforms: ubuntu1804, ubuntu2004, debian 10, osx. Required +$(basename "$0") - Script to setup development environments for Delivery Optimization +Usage: $(basename "$0") --install --install # Which command to run, supported commands: build, developertools, containertools, qemu, all. Default is all EOM - exit 1 } function parseArgs { @@ -37,18 +39,6 @@ function parseArgs { shift exit 0 ;; - --platform | -p) - PLATFORM="${2,,}" - if [[ "$PLATFORM" == "debian10" || "$PLATFORM" == "ubuntu1804" || "$PLATFORM" == "ubuntu2004" || "$PLATFORM" == "osx" ]]; - then - echo -e "[INFO] Platform set to: ${PLATFORM}" - else - echo -e "[ERROR] Unsupported platform: ${PLATFORM}" - exit 1 - fi - - shift - ;; --install | -i) INSTALL="${2,,}" echo -e "[INFO] Install command to run set to: ${INSTALL}" @@ -64,24 +54,16 @@ function parseArgs { function installBuildDependencies { - if [[ "$PLATFORM" != "unknown" ]] - then - echo "[INFO] Platform check succesful" - else - echo "[WARNING] No platform supplied, please supply a platform." - exit 1 - fi - echo "[INFO] Installing build dependencies" - if [[ "$PLATFORM" == "osx" ]]; + if [ $OS == "macos" ]; then # Need to install homebrew to get all the above stuff /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" brew install cmake ninja g++ gdb gdbserver python3 git curl unzip tar pkg-config ./install-vcpkg-deps.sh ~/deliveryoptimization_tools/ - elif isSupportedLinux + elif [ $OS == "linux" ]; then apt-get install -y make build-essential g++ gdb gdbserver gcc git wget @@ -94,7 +76,7 @@ function installBuildDependencies mkdir /tmp/gtest pushd /tmp/gtest - if [[ "$PLATFORM" == "ubuntu2004" || "$PLATFORM" == "debian10" ]]; + if [[ ($DISTRO == "ubuntu" && $VER == "20.04") || ($DISTRO == "debian" && $VER == "10") ]]; then # The latest native-version of gtest on ubuntu2004 and debian10 currently has a bug where # CMakeLists doesn't declare an install target, causing 'make install' to fail. @@ -125,10 +107,10 @@ function installBuildDependencies function installDeveloperTools { echo "[INFO] Installing developer tools" - if [[ "$PLATFORM" == "osx" ]] + if [ $OS == "macos" ]; then brew install cpplint - elif isSupportedLinux + elif [ $OS == "linux" ]; then apt-get install -y python-pip pip install cpplint @@ -136,16 +118,14 @@ function installDeveloperTools # Installs to a non-standard location so add to PATH manually export PATH=$PATH:~/.local/bin else - "[INFO] Developer tools not supported on this platform" + echo "[INFO] Developer tools not supported on this platform" fi } function installContainerTools { - if !isSupportedLinux + if [ $OS == "linux" ]; then - echo "[INFO] Container builds not supported on this platform" - else apt-get install -y curl echo "[INFO] Installing Docker" @@ -153,41 +133,39 @@ function installContainerTools # Instructions located at: https://docs.docker.com/engine/install/ubuntu/ curl -fsSL https://get.docker.com -o get-docker.sh sh get-docker.sh + else + echo "[INFO] Container builds not supported on this platform" fi } function installQemu { - if !isSupportedLinux + if [ $OS == "linux" ]; then - echo "[INFO] Emulated builds not supported on this platform" - else echo "[INFO] Installing Qemu for cross-arch support" # Install qemu for cross-arch support apt-get -y install qemu binfmt-support qemu-user-static # Register qemu with docker to more easily run cross-arch containers docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + else + echo "[INFO] Emulated builds not supported on this platform" fi } function installAll { echo "Setting up development environment for do-client" - installBuildDependencies - - if isSupportedLinux - then - installDeveloperTools - installContainerTools - installQemu - fi + installDeveloperTools + installContainerTools + installQemu } function isSupportedLinux() { - if [[ "$PLATFORM" == "debian10" || "$PLATFORM" == "ubuntu1804" || "$PLATFORM" == "ubuntu2004" ]]; + if [[ ($DISTRO == "ubuntu" && ($VER == "18.04" || $VER == "20.04")) + || ($DISTRO == "debian" && ($VER == "10" || $VER == "11")) ]]; then return 0 else @@ -195,12 +173,122 @@ function isSupportedLinux() fi } +function isSupportedMacOS() +{ + # No specific version check needed at present + return 0 +} + +# From DU project: https://github.com/Azure/iot-hub-device-update/blob/main/scripts/install-deps.sh +function determine_machine_architecture() +{ + local arch='' + arch="$(uname -m)" + local ret_val=$? + if [[ $ret_val != 0 ]]; then + echo "Failed to get cpu architecture." + return 1 + else + if [[ $arch == aarch64* || $arch == armv8* ]]; then + ARCH="arm64" + elif [[ $arch == armv7* || $arch == 'arm' ]]; then + ARCH="arm32" + elif [[ $arch == 'x86_64' || $arch == 'amd64' ]]; then + ARCH="amd64" + else + echo "Machine architecture '$arch' is not supported." + return 1 + fi + fi +} + +function determine_linux_distro() +{ + # Checking distro name and version + if [ -r /etc/os-release ]; then + # freedesktop.org and systemd + DISTRO=$(grep "^ID\s*=\s*" /etc/os-release | sed -e "s/^ID\s*=\s*//") + VER=$(grep "^VERSION_ID=" /etc/os-release | sed -e "s/^VERSION_ID=//") + VER=$(sed -e 's/^"//' -e 's/"$//' <<< "$VER") + elif type lsb_release > /dev/null 2>&1; then + # linuxbase.org + DISTRO=$(lsb_release -si) + VER=$(lsb_release -sr) + elif [ -f /etc/lsb-release ]; then + # For some versions of Debian/Ubuntu without lsb_release command + DISTRO=$(grep DISTRIB_ID /etc/lsb-release | awk -F'=' '{ print $2 }') + VER=$(grep DISTRIB_RELEASE /etc/lsb-release | awk -F'=' '{ print $2 }') + elif [ -f /etc/debian_version ]; then + # Older Debian/Ubuntu/etc. + DISTRO=Debian + VER=$(cat /etc/debian_version) + else + # Fall back to uname, e.g. "Linux ", also works for BSD, etc. + DISTRO=$(uname -s) + VER=$(uname -r) + fi + + # Convert to lowercase + DISTRO="$(echo "$DISTRO" | tr '[:upper:]' '[:lower:]')" +} + +function determine_osx_ver() +{ + echo "Not implemented yet" + return 1 +} + +function determine_os_and_arch() +{ + if [[ $OSTYPE == linux* ]]; then + echo "Running on a linux-based OS" + OS="linux" + determine_linux_distro || return 1 + elif [[ $OSTYPE == darwin* ]]; then + echo "Running on MacOS" + OS="macos" + determine_osx_ver || return 1 + else + echo "Unsupported OS '$OSTYPE'" + return 1 + fi + + determine_machine_architecture || return 1 +} + +function isSupportedOS() +{ + if [ $OS == "linux" ]; then + if ! isSupportedLinux; then + echo "Unsupported Linux distro/version" + return 1 + fi + elif [ $OS == "macos" ]; then + if ! isSupportedMacOS; then + echo "Unsupported MacOS version" + return 1 + fi + else + echo "Unsupported OS and/or Version" + return 1 + fi + return 0 +} + main() { + determine_os_and_arch || return 1 + + echo "OS = $OS" + echo "DISTRO = $DISTRO" + echo "VER = $VER" + echo "ARCH = $ARCH" + + isSupportedOS || return 1 + parseArgs "$@" - if isSupportedLinux - then + if [ $OS == "linux" ]; then echo "[INFO] Updating package manager" apt-get update -y --fix-missing fi diff --git a/snapcraft-options/snapcraft-agent.yaml b/snapcraft-options/snapcraft-agent.yaml index 83874071..c552919f 100644 --- a/snapcraft-options/snapcraft-agent.yaml +++ b/snapcraft-options/snapcraft-agent.yaml @@ -29,7 +29,7 @@ parts: plugin: nil source: . override-build: | - ./build/scripts/bootstrap.sh --platform ubuntu2004 --install build + ./build/scripts/bootstrap.sh --install build agent: plugin: python diff --git a/snapcraft-options/snapcraft-sdk.yaml b/snapcraft-options/snapcraft-sdk.yaml index 95fa8cd5..7ddda9eb 100644 --- a/snapcraft-options/snapcraft-sdk.yaml +++ b/snapcraft-options/snapcraft-sdk.yaml @@ -29,7 +29,7 @@ parts: plugin: nil source: . override-build: | - ./build/scripts/bootstrap.sh --platform ubuntu2004 --install build + ./build/scripts/bootstrap.sh --install build sdk: plugin: python From d6f4e40e392dc065af733561dfcdd53936397fe8 Mon Sep 17 00:00:00 2001 From: JeffreySaathoff <107890382+JeffreySaathoff@users.noreply.github.com> Date: Mon, 20 Mar 2023 13:32:41 -0700 Subject: [PATCH 12/25] Add support for setting client certificate (#164) --- sdk-cpp/include/do_download.h | 7 ++++++- sdk-cpp/src/do_download.cpp | 9 +++++++++ sdk-cpp/src/internal/com/download_impl.cpp | 16 ++++++++++++++++ sdk-cpp/src/internal/download_impl.h | 1 + sdk-cpp/src/internal/download_interface.h | 2 ++ sdk-cpp/src/internal/rest/download_impl.cpp | 5 +++++ sdk-cpp/tests/download_properties_tests.cpp | 18 ++++++++++++++++++ 7 files changed, 57 insertions(+), 1 deletion(-) diff --git a/sdk-cpp/include/do_download.h b/sdk-cpp/include/do_download.h index fb23cd6b..a9248a1c 100644 --- a/sdk-cpp/include/do_download.h +++ b/sdk-cpp/include/do_download.h @@ -27,6 +27,7 @@ class download public: ~download(); + // Future: use std::string_view for all input string params (requires C++17) static std::error_code make(const std::string& uri, std::unique_ptr& out) noexcept; static std::error_code make(const std::string& uri, const std::string& downloadFilePath, std::unique_ptr& out) noexcept; @@ -49,7 +50,7 @@ class download static std::error_code download_url_to_path(const std::string& uri, const std::string& downloadFilePath, const std::atomic_bool& isCancelled, std::chrono::seconds timeoutSecs = std::chrono::hours(24)) noexcept; // Certain properties are not supported on older versions of Windows, resulting in - // msdo::errc::unknown_property_id from the following methods. See do_download_property.h. + // errc::unknown_property_id from the following methods. See do_download_property.h. std::error_code set_property(download_property prop, const download_property_value& value) noexcept; std::error_code get_property(download_property prop, download_property_value& value) noexcept; @@ -83,6 +84,10 @@ class download return set_property(download_property::cost_policy, static_cast(value)); } + // Serialized certificate, for use with http requests + // Future: use std::span (requires C++20) + std::error_code set_client_cert(const unsigned char* data, size_t size) noexcept; + // Returns existing downloads static std::error_code get_downloads(std::vector>& out) noexcept; diff --git a/sdk-cpp/src/do_download.cpp b/sdk-cpp/src/do_download.cpp index e6a4c422..b7d503be 100644 --- a/sdk-cpp/src/do_download.cpp +++ b/sdk-cpp/src/do_download.cpp @@ -90,6 +90,15 @@ std::error_code download::set_ranges(const download_range* ranges, size_t count) return _download->SetRanges(ranges, count); } +std::error_code download::set_client_cert(const unsigned char* data, size_t size) noexcept +{ + if ((data == nullptr) || (size == 0)) + { + return details::make_error_code(errc::invalid_arg); + } + return _download->SetClientCert(data, size); +} + std::error_code download::start_and_wait_until_completion(std::chrono::seconds timeOut) noexcept { std::atomic_bool isCancelled{ false }; diff --git a/sdk-cpp/src/internal/com/download_impl.cpp b/sdk-cpp/src/internal/com/download_impl.cpp index b7ff6080..8aa0c193 100644 --- a/sdk-cpp/src/internal/com/download_impl.cpp +++ b/sdk-cpp/src/internal/com/download_impl.cpp @@ -295,6 +295,22 @@ std::error_code CDownloadImpl::SetRanges(const download_range* ranges, size_t co return DO_OK; } +std::error_code CDownloadImpl::SetClientCert(const unsigned char* data, size_t size) noexcept +{ + unique_variant var; + V_ARRAY(&var) = SafeArrayCreateVector(VT_UI1, 0, static_cast(size)); + if (V_ARRAY(&var) == nullptr) + { + return make_error_code(E_OUTOFMEMORY); + } + V_VT(&var) = VT_ARRAY | VT_UI1; + + memcpy(V_ARRAY(&var)->pvData, data, size); + + RETURN_IF_FAILED(_spDownload->SetProperty(DODownloadProperty_SecurityContext, &var)); + return DO_OK; +} + std::error_code CDownloadImpl::EnumDownloads(std::vector>& out) noexcept { out.clear(); diff --git a/sdk-cpp/src/internal/download_impl.h b/sdk-cpp/src/internal/download_impl.h index cc12edc6..c9e12d27 100644 --- a/sdk-cpp/src/internal/download_impl.h +++ b/sdk-cpp/src/internal/download_impl.h @@ -40,6 +40,7 @@ class CDownloadImpl : public IDownload std::error_code GetProperty(download_property key, download_property_value& value) noexcept override; std::error_code SetProperty(download_property key, const download_property_value& val) noexcept override; std::error_code SetRanges(const download_range* ranges, size_t count) noexcept override; + std::error_code SetClientCert(const unsigned char* data, size_t size) noexcept override; static std::error_code EnumDownloads(std::vector>& out) noexcept; static std::error_code EnumDownloads(download_property prop, const std::string& value, std::vector>& out) noexcept; diff --git a/sdk-cpp/src/internal/download_interface.h b/sdk-cpp/src/internal/download_interface.h index f00d23fd..d77dc715 100644 --- a/sdk-cpp/src/internal/download_interface.h +++ b/sdk-cpp/src/internal/download_interface.h @@ -39,6 +39,8 @@ class IDownload virtual std::error_code SetProperty(download_property key, const download_property_value& val) noexcept = 0; virtual std::error_code SetRanges(const download_range* ranges, size_t count) noexcept = 0; + virtual std::error_code SetClientCert(const unsigned char* data, size_t size) noexcept = 0; + }; } // namespace details } // namespace deliveryoptimization diff --git a/sdk-cpp/src/internal/rest/download_impl.cpp b/sdk-cpp/src/internal/rest/download_impl.cpp index a1755c3d..d7d6b306 100644 --- a/sdk-cpp/src/internal/rest/download_impl.cpp +++ b/sdk-cpp/src/internal/rest/download_impl.cpp @@ -160,6 +160,11 @@ std::error_code CDownloadImpl::SetRanges(const download_range* ranges, size_t co return make_error_code(errc::not_impl); } +std::error_code CDownloadImpl::SetClientCert(const unsigned char* data, size_t size) noexcept +{ + return make_error_code(errc::not_impl); +} + std::error_code CDownloadImpl::EnumDownloads(std::vector>& out) noexcept { return make_error_code(errc::not_impl); diff --git a/sdk-cpp/tests/download_properties_tests.cpp b/sdk-cpp/tests/download_properties_tests.cpp index 02427424..e49b32b6 100644 --- a/sdk-cpp/tests/download_properties_tests.cpp +++ b/sdk-cpp/tests/download_properties_tests.cpp @@ -260,6 +260,24 @@ TEST_F(DownloadPropertyTests, BasicEnumDownloadsTest) ASSERT_EQ(downloads.size(), 0); } +TEST_F(DownloadPropertyTests, InvalidClientCertTest) +{ + std::unique_ptr download; + ASSERT_EQ(msdo::download::make(g_smallFileUrl, g_tmpFileName, download).value(), 0); + + // A convenient piece of data that isn't a valid cert + auto data = reinterpret_cast(g_largeFileUrl.data()); + auto size = g_largeFileUrl.size(); + + ASSERT_EQ(download->set_client_cert(nullptr, 0).value(), msdo::errc::invalid_arg); + ASSERT_EQ(download->set_client_cert(nullptr, size).value(), msdo::errc::invalid_arg); + ASSERT_EQ(download->set_client_cert(data, 0).value(), msdo::errc::invalid_arg); + + ASSERT_EQ(download->set_client_cert(data, size).value(), static_cast(0x80092003)); // CRYPT_E_FILE_ERROR + + ASSERT_EQ(download->abort().value(), 0); +} + #elif defined(DO_CLIENT_AGENT) TEST_F(DownloadPropertyTests, SmallDownloadSetCallerNameFailureTest) From 70c5e52d007448d197d40cfa8a0e2d59053f186c Mon Sep 17 00:00:00 2001 From: andretoyama-msft <106272532+andretoyama-msft@users.noreply.github.com> Date: Tue, 21 Mar 2023 10:42:43 -0300 Subject: [PATCH 13/25] Change confinement level to strict (#163) --- snapcraft-options/snapcraft-agent.yaml | 2 +- snapcraft-options/snapcraft-sdk.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/snapcraft-options/snapcraft-agent.yaml b/snapcraft-options/snapcraft-agent.yaml index c552919f..f8826526 100644 --- a/snapcraft-options/snapcraft-agent.yaml +++ b/snapcraft-options/snapcraft-agent.yaml @@ -6,7 +6,7 @@ description: | A proof-of-concept for the Delivery Optimization Client Ubuntu Core snap. grade: devel # must be 'stable' to release into candidate/stable channels -confinement: devmode # use 'strict' once you have the right plugs and slots +confinement: strict # use 'strict' once you have the right plugs and slots ##### # diff --git a/snapcraft-options/snapcraft-sdk.yaml b/snapcraft-options/snapcraft-sdk.yaml index 7ddda9eb..5ab07bee 100644 --- a/snapcraft-options/snapcraft-sdk.yaml +++ b/snapcraft-options/snapcraft-sdk.yaml @@ -6,7 +6,7 @@ description: | SDK snap (consumer) to test communication with the DO agent snap (producer). grade: devel # must be 'stable' to release into candidate/stable channels -confinement: devmode # use 'strict' once you have the right plugs and slots +confinement: strict # use 'strict' once you have the right plugs and slots ##### # From 952fc4822e1e17666ae729a701150c19480bf48d Mon Sep 17 00:00:00 2001 From: andretoyama-msft <106272532+andretoyama-msft@users.noreply.github.com> Date: Tue, 21 Mar 2023 14:40:30 -0300 Subject: [PATCH 14/25] Update ubuntu-snap.md (#165) * Update ubuntu-snap.md --- snapcraft-options/ubuntu-snap.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/snapcraft-options/ubuntu-snap.md b/snapcraft-options/ubuntu-snap.md index a256bb48..46ecbdfa 100644 --- a/snapcraft-options/ubuntu-snap.md +++ b/snapcraft-options/ubuntu-snap.md @@ -10,11 +10,21 @@ Run the commands in the root of the repo. - `$ ./build/build-snaps.sh agent` - `$ ./build/build-snaps.sh sdk-tests` +- By default snapcraft doesn't allow multiple snaps definition in the same code base, so this build-snaps.sh file was created so that +we can copy the different snap files into the main snapcraft.yaml file and then build it. + - Subsequent builds of the same component without modifying the corresponding yaml file: - `$ snapcraft` - The build will generate *.snap files in the current working directory. Example: **deliveryoptimization-sdk-tests_0.1_amd64.snap**. +- In the snapcraft.yaml the regular script to build for both the delivery optimization agent and sdk is called (`build.py`), but since +there are some different behaviors the code should have when inside a snap, for that we use a flag `--build-for-snap` on the build. +Below you can see the usage of this flag, but **is not** necessary to execute them, those lines will be executed inside the snapcraft.yaml +file when you call the `build-snaps.sh` file. + - `$ python3 ./build/build.py --project agent --build-for-snap` + - `$ python3 ./build/build.py --project sdk --build-for-snap` + ## Installing - `$ sudo snap install --devmode ./deliveryoptimization-sdk-tests_0.1_amd64.snap` - `$ sudo snap install --devmode ./deliveryoptimization-client_0.1_amd64.snap` @@ -24,11 +34,26 @@ It is a fruitful exercise to look around in the host file system and see how sna - /snap/ - /var/snap/ -## Connecting plugs and slots +## Connecting to Other Snaps + +- Snapcraft allows communication between different snaps by using a content interface to handle connections with the purpose +of sharing code and data between them. These connections are composed by plugs and slots, where a slot is the producer snap +and the plug is the consumer snap. + - For the SDK test snap to work, the plugs and slots must be connected. Run the **connect-snaps.sh** script after both snaps are installed. - The connections can be listed using this command: - `$ snap connections | grep delivery` +```shell +Interface Plug Slot Notes +content - deliveryoptimization-client:do-configs - +content - deliveryoptimization-client:do-port-numbers - +content deliveryoptimization-client:deviceupdate-agent-downloads - - +network deliveryoptimization-client:network :network - +network-bind deliveryoptimization-client:network-bind :network-bind - + +``` + ## Running or executing - Agent - The agent is declared as a daemon/service, so it starts running immediately after successful installation. From 7dc133697d0170b60762e45f668240a08bd62caa Mon Sep 17 00:00:00 2001 From: shishirb-MSFT <50385517+shishirb-MSFT@users.noreply.github.com> Date: Wed, 22 Mar 2023 22:55:10 -0700 Subject: [PATCH 15/25] Fix minsizerel binary size checks (#168) * Agent binary size is much lower than the limit we were checking against. The limit was probably not updated after the earlier size reductions. * SDK size is lower but only by a small margin. Updated that one too. * Also update the script to fail if size has reduced too much in order to catch this early next time. --- .../linux/du/templates/doclient-lite-native-steps.yml | 2 +- .../build/linux/du/templates/dosdkcpp-native-steps.yml | 2 +- build/scripts/check_binary_size.sh | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) mode change 100644 => 100755 build/scripts/check_binary_size.sh diff --git a/azure-pipelines/build/linux/du/templates/doclient-lite-native-steps.yml b/azure-pipelines/build/linux/du/templates/doclient-lite-native-steps.yml index e6da6636..1c3480f5 100644 --- a/azure-pipelines/build/linux/du/templates/doclient-lite-native-steps.yml +++ b/azure-pipelines/build/linux/du/templates/doclient-lite-native-steps.yml @@ -24,7 +24,7 @@ steps: inputs: targetType: 'filePath' filePath: 'build/scripts/check_binary_size.sh' - arguments: '1251928 /tmp/build-deliveryoptimization-agent/linux-${{parameters.config}}/client-lite/deliveryoptimization-agent' + arguments: '363928 /tmp/build-deliveryoptimization-agent/linux-${{parameters.config}}/client-lite/deliveryoptimization-agent' displayName: 'Limit binary size increase' - task: CmdLine@2 diff --git a/azure-pipelines/build/linux/du/templates/dosdkcpp-native-steps.yml b/azure-pipelines/build/linux/du/templates/dosdkcpp-native-steps.yml index 3b96bf14..318c0bc7 100644 --- a/azure-pipelines/build/linux/du/templates/dosdkcpp-native-steps.yml +++ b/azure-pipelines/build/linux/du/templates/dosdkcpp-native-steps.yml @@ -38,7 +38,7 @@ steps: inputs: targetType: 'filePath' filePath: 'build/scripts/check_binary_size.sh' - arguments: '443632 /tmp/build-deliveryoptimization-sdk/linux-${{parameters.config}}/sdk-cpp/libdeliveryoptimization.so.*.*.*' + arguments: '431184 /tmp/build-deliveryoptimization-sdk/linux-${{parameters.config}}/sdk-cpp/libdeliveryoptimization.so.*.*.?' displayName: 'Limit binary size increase' - task: CmdLine@2 diff --git a/build/scripts/check_binary_size.sh b/build/scripts/check_binary_size.sh old mode 100644 new mode 100755 index 6da467c8..fc09238f --- a/build/scripts/check_binary_size.sh +++ b/build/scripts/check_binary_size.sh @@ -22,8 +22,11 @@ do change_pct=`echo "scale=2; ($actual_size - $expected_size) / $expected_size * 100" | bc` echo Size check, expected = $expected_size, actual = $actual_size, change pct = $change_pct% change_pct=$( printf "%.0f" $change_pct ) - if [ $change_pct -ge 5 ]; then - echo "Size check FAIL. Adjust expected_size if this increase is expected." - exit 3 + if [ $change_pct -ge 2 ]; then + echo "[FAIL] Size increased beyond threshold. Adjust expected_size if this is expected." + exit 3 + elif [ $change_pct -le -5 ]; then + echo "[WARNING] Size reduced beyond threshold. Adjust expected_size if this is expected." + exit 4 fi done From 211cfcebfe943a5876ecc0da44862ba50141552d Mon Sep 17 00:00:00 2001 From: shishirb-MSFT <50385517+shishirb-MSFT@users.noreply.github.com> Date: Thu, 23 Mar 2023 10:12:06 -0700 Subject: [PATCH 16/25] Replace boost::filesystem usage with std::filesystem (#169) * Always wanted to move to the C++17 filesystem implementation. What prompted the change now: The FindBoost issue on Ubuntu 20.04 involving symlinking /lib to /usr/lib but /include being non-existent. * This change removes public dependency on Boost libs entirely, leaving only the tests using boost.program_options. So, our cmake config file will no longer have to deal with the paths for boost. * GCC 8, which is the default at least on Ubuntu 18.04, requires use of std::experimental::filesystem. Code has been changed appropriately to support building with both flavors of filesystem. --- CMakeLists.txt | 16 ++- .../templates/doclient-lite-native-steps.yml | 2 +- .../du/templates/dosdkcpp-native-steps.yml | 2 +- build/scripts/bootstrap.sh | 2 +- build/scripts/install-vcpkg-deps.ps1 | 1 - build/scripts/install-vcpkg-deps.sh | 3 - client-lite/CMakeLists.txt | 18 +-- client-lite/src/include/do_filesystem.h | 30 +++++ client-lite/src/ipc/rest_port_advertiser.h | 8 +- client-lite/src/util/do_json_parser.cpp | 6 +- client-lite/src/util/proc_launch_helper.h | 9 +- client-lite/test/CMakeLists.txt | 10 +- client-lite/test/do_log_tests.cpp | 4 +- client-lite/test/docs_tests.cpp | 4 +- client-lite/test/download_manager_tests.cpp | 2 +- client-lite/test/json_parser_tests.cpp | 12 +- client-lite/test/mcc_manager.tests.cpp | 24 ++-- client-lite/test/test_common.h | 12 +- client-lite/test/test_verifiers.h | 4 +- common/cmake/do-build-helpers.cmake | 31 ++++++ sdk-cpp/CMakeLists.txt | 18 +-- .../deliveryoptimization_sdk-config.cmake | 7 -- sdk-cpp/src/internal/do_filesystem.h | 27 +++++ .../src/internal/rest/do_config_internal.cpp | 15 ++- .../src/internal/rest/util/do_port_finder.cpp | 16 +-- sdk-cpp/tests/CMakeLists.txt | 10 +- sdk-cpp/tests/download_properties_tests.cpp | 8 +- sdk-cpp/tests/download_tests_common.cpp | 104 +++++++++--------- sdk-cpp/tests/rest/config_tests.cpp | 6 +- sdk-cpp/tests/rest/mcc_download_tests.cpp | 6 +- .../tests/rest/network_connectivity_tests.cpp | 4 +- sdk-cpp/tests/rest/port_discovery_tests.cpp | 10 +- sdk-cpp/tests/rest/rest_interface_tests.cpp | 6 +- sdk-cpp/tests/rest/test_helpers.cpp | 8 +- sdk-cpp/tests/test_data.cpp | 10 +- sdk-cpp/tests/test_helpers.h | 26 ++--- sdk-cpp/tests/tests_common.h | 2 +- snapcraft-options/snapcraft-agent.yaml | 1 - snapcraft-options/snapcraft-sdk.yaml | 1 - 39 files changed, 267 insertions(+), 218 deletions(-) create mode 100644 client-lite/src/include/do_filesystem.h create mode 100644 sdk-cpp/src/internal/do_filesystem.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 1e453045..c279e92d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,10 +17,6 @@ option (DO_INCLUDE_SDK "Build subproject sdk-cpp" OFF) option (DO_BUILD_TESTS "Set DO_BUILD_TESTS to OFF to skip building tests." ON) option (DO_BUILD_FOR_SNAP "Enable DO Snap build option" OFF) -set(CMAKE_CXX_STANDARD 14) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) - # Get verbose output from cmake generation and build steps set(CMAKE_VERBOSE_MAKEFILE ON) @@ -62,6 +58,16 @@ else() message(FATAL_ERROR "Unknown platform for client") endif() +# C++17 is required to use std::filesystem but Edge builds on Windows is not ready to move up yet. +# Luckily, we do not need std::filesystem on Windows. +if (DO_INCLUDE_SDK AND DO_PLATFORM_WINDOWS) + set(CMAKE_CXX_STANDARD 14) +else () + set(CMAKE_CXX_STANDARD 17) +endif () +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + # PIC (Position Independent Code) ensures .a files can be linked to executables that have PIE enabled set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -85,7 +91,7 @@ if (DO_PLATFORM_LINUX) # not relevant for our usage or they are symlinks to /usr/lib and /usr/bin. if (NOT CMAKE_PREFIX_PATH) set(CMAKE_PREFIX_PATH "/usr") - endif () + endif () endif (DO_PLATFORM_LINUX) if (DO_PLATFORM_WINDOWS AND DO_INCLUDE_SDK) diff --git a/azure-pipelines/build/linux/du/templates/doclient-lite-native-steps.yml b/azure-pipelines/build/linux/du/templates/doclient-lite-native-steps.yml index 1c3480f5..a630bfe7 100644 --- a/azure-pipelines/build/linux/du/templates/doclient-lite-native-steps.yml +++ b/azure-pipelines/build/linux/du/templates/doclient-lite-native-steps.yml @@ -24,7 +24,7 @@ steps: inputs: targetType: 'filePath' filePath: 'build/scripts/check_binary_size.sh' - arguments: '363928 /tmp/build-deliveryoptimization-agent/linux-${{parameters.config}}/client-lite/deliveryoptimization-agent' + arguments: '523672 /tmp/build-deliveryoptimization-agent/linux-${{parameters.config}}/client-lite/deliveryoptimization-agent' displayName: 'Limit binary size increase' - task: CmdLine@2 diff --git a/azure-pipelines/build/linux/du/templates/dosdkcpp-native-steps.yml b/azure-pipelines/build/linux/du/templates/dosdkcpp-native-steps.yml index 318c0bc7..3b699b4c 100644 --- a/azure-pipelines/build/linux/du/templates/dosdkcpp-native-steps.yml +++ b/azure-pipelines/build/linux/du/templates/dosdkcpp-native-steps.yml @@ -38,7 +38,7 @@ steps: inputs: targetType: 'filePath' filePath: 'build/scripts/check_binary_size.sh' - arguments: '431184 /tmp/build-deliveryoptimization-sdk/linux-${{parameters.config}}/sdk-cpp/libdeliveryoptimization.so.*.*.?' + arguments: '596080 /tmp/build-deliveryoptimization-sdk/linux-${{parameters.config}}/sdk-cpp/libdeliveryoptimization.so.*.*.?' displayName: 'Limit binary size increase' - task: CmdLine@2 diff --git a/build/scripts/bootstrap.sh b/build/scripts/bootstrap.sh index c42441ad..43d35584 100755 --- a/build/scripts/bootstrap.sh +++ b/build/scripts/bootstrap.sh @@ -69,7 +69,7 @@ function installBuildDependencies apt-get install -y make build-essential g++ gdb gdbserver gcc git wget apt-get install -y python3 ninja-build apt-get install -y cmake libmsgsl-dev - apt-get install -y libboost-system-dev libboost-filesystem-dev libboost-program-options-dev + apt-get install -y libboost-program-options-dev apt-get install -y libproxy-dev libssl-dev uuid-dev libcurl4-openssl-dev rm -rf /tmp/gtest diff --git a/build/scripts/install-vcpkg-deps.ps1 b/build/scripts/install-vcpkg-deps.ps1 index 8a56e683..af43b7cc 100644 --- a/build/scripts/install-vcpkg-deps.ps1 +++ b/build/scripts/install-vcpkg-deps.ps1 @@ -13,5 +13,4 @@ git checkout 2021.05.12 .\vcpkg integrate install .\vcpkg update .\vcpkg install gtest:x64-windows -.\vcpkg install boost-filesystem:x64-windows .\vcpkg install boost-program-options:x64-windows diff --git a/build/scripts/install-vcpkg-deps.sh b/build/scripts/install-vcpkg-deps.sh index af4213fe..2a6a71f7 100644 --- a/build/scripts/install-vcpkg-deps.sh +++ b/build/scripts/install-vcpkg-deps.sh @@ -17,9 +17,6 @@ git checkout 2021.05.12 ./vcpkg integrate install ./vcpkg install ms-gsl -./vcpkg install boost-log -./vcpkg install boost-beast -./vcpkg install boost-iostreams ./vcpkg install boost-uuid ./vcpkg install boost-program-options ./vcpkg install gtest diff --git a/client-lite/CMakeLists.txt b/client-lite/CMakeLists.txt index 35d95007..ee6bca08 100644 --- a/client-lite/CMakeLists.txt +++ b/client-lite/CMakeLists.txt @@ -9,8 +9,6 @@ project (${DOSVC_BIN_NAME} VERSION 1.0.0) option (DO_PROXY_SUPPORT "Set DO_PROXY_SUPPORT to OFF to turn off proxy support for downloads and thus remove dependency on libproxy." ON) -add_definitions(-DBOOST_ALL_DYN_LINK=1) - # -Wno-noexcept-type, the offending function is SetResultLoggingCallback, this warning is fixed in C++17 because exception specification # is part of a function type. Since the offending function is not public when compiled into docs_common just add the compiler flag here # to disable the warning. @@ -29,7 +27,7 @@ function (target_link_dl_lib target) endfunction () # Include external libraries here: -find_package(Boost COMPONENTS filesystem REQUIRED) +find_package(Boost REQUIRED) find_package(CURL REQUIRED) # g++ requires explicit specification of the thread library to be used find_package(Threads REQUIRED) @@ -46,6 +44,8 @@ if (GSL_INCLUDE_DIR STREQUAL "GSL_INCLUDE_DIR-NOTFOUND") message(FATAL_ERROR "Could not find MS Guidelines Support Library.") endif() +try_set_filesystem_lib() + set (docs_common_includes ${include_directories_for_arm} ${GSL_INCLUDE_DIR} @@ -90,13 +90,15 @@ endif () add_do_version_lib(${PROJECT_NAME} ${PROJECT_VERSION}) -# Build product files into a lib for use by other targets +# Build product files into a lib for use by other targets. add_library(docs_common STATIC ${files_docs_common}) target_compile_definitions(docs_common - PRIVATE DO_CONFIG_DIRECTORY_PATH="${docs_svc_config_dir_path}" - DO_AGENT_LOG_DIRECTORY_PATH="${docs_svc_log_dir_path}" - DO_RUN_DIRECTORY_PATH="${docs_svc_run_dir_path}" + PRIVATE + DO_CONFIG_DIRECTORY_PATH="${docs_svc_config_dir_path}" + DO_AGENT_LOG_DIRECTORY_PATH="${docs_svc_log_dir_path}" + DO_RUN_DIRECTORY_PATH="${docs_svc_run_dir_path}" ) +add_boost_definitions(docs_common PUBLIC) if (DO_DEV_DEBUG) target_compile_definitions(docs_common PUBLIC DO_DEV_DEBUG) endif () @@ -124,8 +126,8 @@ add_platform_interface_definitions(${DOSVC_BIN_NAME}) target_include_directories(${DOSVC_BIN_NAME} PRIVATE ${docs_common_includes}) target_link_libraries(${DOSVC_BIN_NAME} docs_common - ${Boost_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} + ${CXX_FILESYSTEM_LIBS} ) if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") diff --git a/client-lite/src/include/do_filesystem.h b/client-lite/src/include/do_filesystem.h new file mode 100644 index 00000000..773bf536 --- /dev/null +++ b/client-lite/src/include/do_filesystem.h @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#ifndef _DELIVERY_OPTIMIZATION_DO_FILESYSTEM_H +#define _DELIVERY_OPTIMIZATION_DO_FILESYSTEM_H + +#if defined(__cpp_lib_filesystem) +#define INCLUDE_STD_FILESYSTEM_EXPERIMENTAL 0 +#elif defined(__cpp_lib_experimental_filesystem) +#define INCLUDE_STD_FILESYSTEM_EXPERIMENTAL 1 +#elif !defined(__has_include) +// Cannot check if headers exist, assume experimental +#define INCLUDE_STD_FILESYSTEM_EXPERIMENTAL 1 +#elif __has_include() +#define INCLUDE_STD_FILESYSTEM_EXPERIMENTAL 0 +#elif __has_include() +#define INCLUDE_STD_FILESYSTEM_EXPERIMENTAL 1 +#else +#error Could not find system header "" or "" +#endif + +#if INCLUDE_STD_FILESYSTEM_EXPERIMENTAL +#include +namespace fs = std::experimental::filesystem; +#else +#include +namespace fs = std::filesystem; +#endif + +#endif // _DELIVERY_OPTIMIZATION_DO_FILESYSTEM_H diff --git a/client-lite/src/ipc/rest_port_advertiser.h b/client-lite/src/ipc/rest_port_advertiser.h index 9780f3aa..c137f0d3 100644 --- a/client-lite/src/ipc/rest_port_advertiser.h +++ b/client-lite/src/ipc/rest_port_advertiser.h @@ -7,8 +7,8 @@ #include // open, write #include // getpid #include -#include #include +#include "do_filesystem.h" #include "do_persistence.h" #include "error_macros.h" @@ -60,13 +60,13 @@ class RestPortAdvertiser void _DeleteOlderPortFiles() try { auto& runtimeDirectory = docli::GetRuntimeDirectory(); - for (boost::filesystem::directory_iterator itr(runtimeDirectory); itr != boost::filesystem::directory_iterator(); ++itr) + for (fs::directory_iterator itr(runtimeDirectory); itr != fs::directory_iterator(); ++itr) { auto& dirEntry = itr->path(); if (dirEntry.filename().string().find(_restPortFileNamePrefix) != std::string::npos) { - boost::system::error_code ec; - boost::filesystem::remove(dirEntry, ec); + std::error_code ec; + fs::remove(dirEntry, ec); if (ec) { DoLogWarning("Failed to delete old port file (%d, %s) %s", ec.value(), ec.message().data(), dirEntry.string().data()); diff --git a/client-lite/src/util/do_json_parser.cpp b/client-lite/src/util/do_json_parser.cpp index 5ff42980..11d1d962 100644 --- a/client-lite/src/util/do_json_parser.cpp +++ b/client-lite/src/util/do_json_parser.cpp @@ -4,7 +4,7 @@ #include "do_common.h" #include "do_json_parser.h" -#include +#include "do_filesystem.h" #include std::chrono::seconds JsonParser::RefreshInterval = std::chrono::seconds(60); @@ -13,7 +13,7 @@ std::chrono::seconds JsonParser::RefreshInterval = std::chrono::seconds(60); JsonParser::JsonParser(const std::string& jsonFilePath, bool alwaysCreateFile) : _jsonFilePath(jsonFilePath) { - if (alwaysCreateFile && !(boost::filesystem::exists(_jsonFilePath))) + if (alwaysCreateFile && !(fs::exists(_jsonFilePath))) { DoLogInfo("json file not found at %s, creating file", _jsonFilePath.data()); boost::property_tree::ptree json; @@ -33,7 +33,7 @@ void JsonParser::_TryRefresh(bool force) return; } - if (boost::filesystem::exists(_jsonFilePath)) + if (fs::exists(_jsonFilePath)) { try { diff --git a/client-lite/src/util/proc_launch_helper.h b/client-lite/src/util/proc_launch_helper.h index 6c4abfde..312324de 100644 --- a/client-lite/src/util/proc_launch_helper.h +++ b/client-lite/src/util/proc_launch_helper.h @@ -9,9 +9,8 @@ #include #include -#include - #include "do_common.h" +#include "do_filesystem.h" #include "do_persistence.h" inline gid_t GetGroupIdByName(const char *name) @@ -54,11 +53,11 @@ inline void SetDOPathPermissions(const std::string& path, mode_t mode) inline void InitializePath(const std::string& path, mode_t mode = 0) try { - boost::filesystem::path dirPath(path); - if (!boost::filesystem::exists(dirPath)) + fs::path dirPath(path); + if (!fs::exists(dirPath)) { DoLogInfo("Creating directories for %s", path.c_str()); - boost::filesystem::create_directories(dirPath); + fs::create_directories(dirPath); if (mode != 0) { diff --git a/client-lite/test/CMakeLists.txt b/client-lite/test/CMakeLists.txt index ea9db484..3249ba35 100644 --- a/client-lite/test/CMakeLists.txt +++ b/client-lite/test/CMakeLists.txt @@ -1,21 +1,17 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -# Unit tests for DOCS - -# Only need program_options here but have to include others required by docs_common. -# TODO: docs_common should declare what it needs via target_link_libraries(PUBLIC). -find_package(Boost COMPONENTS filesystem program_options REQUIRED) +find_package(Boost COMPONENTS program_options REQUIRED) find_package(GTest REQUIRED) file (GLOB files_docs_tests *.cpp) -add_executable (deliveryoptimization-agent-tests ${files_docs_tests}) +add_executable(deliveryoptimization-agent-tests ${files_docs_tests}) add_platform_interface_definitions(deliveryoptimization-agent-tests) target_link_libraries(deliveryoptimization-agent-tests docs_common dotestutil - stdc++fs ${Boost_LIBRARIES} GTest::GTest + ${CXX_FILESYSTEM_LIBS} ) diff --git a/client-lite/test/do_log_tests.cpp b/client-lite/test/do_log_tests.cpp index 2d3dfbad..d5ab0606 100644 --- a/client-lite/test/do_log_tests.cpp +++ b/client-lite/test/do_log_tests.cpp @@ -26,12 +26,12 @@ TEST_F(DOLoggerTests, BasicWriteToFile) DOLog::Close(); UINT nLogFilesFound = 0; - for (cppfs::recursive_directory_iterator itr(g_testTempDir); itr != cppfs::recursive_directory_iterator{}; ++itr) + for (fs::recursive_directory_iterator itr(g_testTempDir); itr != fs::recursive_directory_iterator{}; ++itr) { ++nLogFilesFound; const auto filePath = itr->path(); ASSERT_NE(strstr(filePath.c_str(), "do-agent."), nullptr); - ASSERT_GT(cppfs::file_size(filePath), 0); + ASSERT_GT(fs::file_size(filePath), 0); std::ifstream fs; fs.open(filePath.string()); diff --git a/client-lite/test/docs_tests.cpp b/client-lite/test/docs_tests.cpp index 7911af2c..c531ec56 100644 --- a/client-lite/test/docs_tests.cpp +++ b/client-lite/test/docs_tests.cpp @@ -6,7 +6,7 @@ #include #include "test_data.h" -const cppfs::path g_testTempDir = "/tmp/docs_test_scratch"; +const fs::path g_testTempDir = "/tmp/docs_test_scratch"; int main(int argc, char** argv) { @@ -15,7 +15,7 @@ int main(int argc, char** argv) // TODO(shishirb) enable console only logging std::error_code ec; - cppfs::create_directories(g_testTempDir, ec); + fs::create_directories(g_testTempDir, ec); if (ec) { printf("Failed to create test dir: %s\n", g_testTempDir.string().data()); diff --git a/client-lite/test/download_manager_tests.cpp b/client-lite/test/download_manager_tests.cpp index cbb21c0e..90e6d0a5 100644 --- a/client-lite/test/download_manager_tests.cpp +++ b/client-lite/test/download_manager_tests.cpp @@ -136,7 +136,7 @@ TEST_F(DownloadManagerTests, FileDownloadFatal404) ASSERT_EQ(status.BytesTotal, 0); VerifyError(status, HTTP_E_STATUS_NOT_FOUND); VerifyDownloadHttpStatus(*DownloadForId(manager, id), 404); - ASSERT_TRUE(cppfs::exists(destFile)); + ASSERT_TRUE(fs::exists(destFile)); manager.AbortDownload(id); VerifyFileNotFound(destFile); } diff --git a/client-lite/test/json_parser_tests.cpp b/client-lite/test/json_parser_tests.cpp index e0b01daa..946d55a9 100644 --- a/client-lite/test/json_parser_tests.cpp +++ b/client-lite/test/json_parser_tests.cpp @@ -12,7 +12,7 @@ #include #include -const cppfs::path g_jsonTestFilePath = g_testTempDir / "docs_config.json"; +const fs::path g_jsonTestFilePath = g_testTempDir / "docs_config.json"; const std::map g_testData = { @@ -70,9 +70,9 @@ class JsonParserTests : public ::testing::Test void SetUp() override { JsonParser::RefreshInterval = std::chrono::seconds(10); // for faster test times - if (cppfs::exists(g_jsonTestFilePath)) + if (fs::exists(g_jsonTestFilePath)) { - cppfs::remove(g_jsonTestFilePath); + fs::remove(g_jsonTestFilePath); } } }; @@ -111,7 +111,7 @@ TEST_F(JsonParserTests, ReadConfigsWithUpdates) VerifyItemFound(reader, "newkey", "newvalue"); // Delete config file and verify - cppfs::remove(g_jsonTestFilePath); + fs::remove(g_jsonTestFilePath); VerifyItemFound(reader, "newkey", "newvalue"); // within refresh interval VerifyTestData(reader, true); @@ -122,9 +122,9 @@ TEST_F(JsonParserTests, ReadConfigsWithUpdates) TEST_F(JsonParserTests, ReadConfigsWithFileCreatedLater) { - if (cppfs::exists(g_jsonTestFilePath)) + if (fs::exists(g_jsonTestFilePath)) { - cppfs::remove(g_jsonTestFilePath); + fs::remove(g_jsonTestFilePath); } JsonParser reader(g_jsonTestFilePath); diff --git a/client-lite/test/mcc_manager.tests.cpp b/client-lite/test/mcc_manager.tests.cpp index 3769df46..c9523a2e 100644 --- a/client-lite/test/mcc_manager.tests.cpp +++ b/client-lite/test/mcc_manager.tests.cpp @@ -28,13 +28,13 @@ using btcp_t = boost::asio::ip::tcp; #define TEST_MOCK_MCC_HOST "10.130.48.179" #define TEST_MOCK_MCC_HOST2 "10.130.48.180" -const cppfs::path g_adminConfigFilePath = g_testTempDir / "admin-config.json"; -const cppfs::path g_sdkConfigFilePath = g_testTempDir / "sdk-config.json"; +const fs::path g_adminConfigFilePath = g_testTempDir / "admin-config.json"; +const fs::path g_sdkConfigFilePath = g_testTempDir / "sdk-config.json"; static void SetIoTConnectionString(const char* connectionString) { boost::property_tree::ptree json; - if (cppfs::exists(g_sdkConfigFilePath)) + if (fs::exists(g_sdkConfigFilePath)) { boost::property_tree::read_json(g_sdkConfigFilePath, json); } @@ -45,7 +45,7 @@ static void SetIoTConnectionString(const char* connectionString) static void SetDOCacheHostConfig(const char* server) { boost::property_tree::ptree json; - if (cppfs::exists(g_adminConfigFilePath)) + if (fs::exists(g_adminConfigFilePath)) { boost::property_tree::read_json(g_adminConfigFilePath, json); } @@ -56,7 +56,7 @@ static void SetDOCacheHostConfig(const char* server) static void SetFallbackDelayConfig(std::chrono::seconds delay) { boost::property_tree::ptree json; - if (cppfs::exists(g_adminConfigFilePath)) + if (fs::exists(g_adminConfigFilePath)) { boost::property_tree::read_json(g_adminConfigFilePath, json); } @@ -71,13 +71,13 @@ class MCCManagerTests : public ::testing::Test { ClearTestTempDir(); - if (cppfs::exists(g_adminConfigFilePath)) + if (fs::exists(g_adminConfigFilePath)) { - cppfs::remove(g_adminConfigFilePath); + fs::remove(g_adminConfigFilePath); } - if (cppfs::exists(g_sdkConfigFilePath)) + if (fs::exists(g_sdkConfigFilePath)) { - cppfs::remove(g_sdkConfigFilePath); + fs::remove(g_sdkConfigFilePath); } } @@ -107,7 +107,7 @@ TEST_F(MCCManagerTests, ParseIoTConnectionString) _VerifyExpectedCacheHost(TEST_MOCK_MCC_HOST); // No gateway specified - cppfs::remove(g_sdkConfigFilePath); + fs::remove(g_sdkConfigFilePath); _VerifyExpectedCacheHost(""); } @@ -117,7 +117,7 @@ TEST_F(MCCManagerTests, AdminConfigOverride) SetDOCacheHostConfig(TEST_MOCK_MCC_HOST2); _VerifyExpectedCacheHost(TEST_MOCK_MCC_HOST2); - cppfs::remove(g_adminConfigFilePath); + fs::remove(g_adminConfigFilePath); _VerifyExpectedCacheHost(TEST_MOCK_MCC_HOST); } @@ -127,7 +127,7 @@ TEST_F(MCCManagerTests, AdminConfigEmptyString) SetDOCacheHostConfig(""); _VerifyExpectedCacheHost(""); - cppfs::remove(g_adminConfigFilePath); + fs::remove(g_adminConfigFilePath); _VerifyExpectedCacheHost(TEST_MOCK_MCC_HOST); } diff --git a/client-lite/test/test_common.h b/client-lite/test/test_common.h index 45ea5716..96dd9c51 100644 --- a/client-lite/test/test_common.h +++ b/client-lite/test/test_common.h @@ -4,21 +4,19 @@ #pragma once #include "do_common.h" +#include "do_filesystem.h" #include -#include #include #include -namespace cppfs = std::experimental::filesystem; - -extern const cppfs::path g_testTempDir; +extern const fs::path g_testTempDir; inline void ClearTestTempDir() { - if (cppfs::exists(g_testTempDir)) + if (fs::exists(g_testTempDir)) { - cppfs::remove_all(g_testTempDir); + fs::remove_all(g_testTempDir); } - cppfs::create_directories(g_testTempDir); + fs::create_directories(g_testTempDir); } diff --git a/client-lite/test/test_verifiers.h b/client-lite/test/test_verifiers.h index 6dbccb40..e05d550e 100644 --- a/client-lite/test/test_verifiers.h +++ b/client-lite/test/test_verifiers.h @@ -10,12 +10,12 @@ inline void VerifyFileSize(const std::string& path, UINT64 cbExpectedSize) { - ASSERT_EQ(cppfs::file_size(path), cbExpectedSize) << "File size check: " << path; + ASSERT_EQ(fs::file_size(path), cbExpectedSize) << "File size check: " << path; } inline void VerifyFileNotFound(const std::string& path) { - ASSERT_FALSE(cppfs::exists(path)) << "File absent check: " << path; + ASSERT_FALSE(fs::exists(path)) << "File absent check: " << path; } inline void VerifyNoError(const DownloadStatus& status) diff --git a/common/cmake/do-build-helpers.cmake b/common/cmake/do-build-helpers.cmake index f7e2bd03..e7178ec2 100644 --- a/common/cmake/do-build-helpers.cmake +++ b/common/cmake/do-build-helpers.cmake @@ -160,3 +160,34 @@ function (add_platform_interface_definitions target_name) endif () endfunction () + +function (add_boost_definitions target_name scope) + if (DO_PLATFORM_LINUX OR DO_PLATFORM_MAC) + # BOOST_ERROR_CODE_HEADER_ONLY and BOOST_SYSTEM_NO_DEPRECATED are required to use header-only components + # like boost.asio without linking to the boost.system shared library. + target_compile_definitions(${target_name} ${scope} BOOST_ERROR_CODE_HEADER_ONLY BOOST_SYSTEM_NO_DEPRECATED) + endif () +endfunction () + +function (try_set_filesystem_lib) + + # Sets the variable CXX_FILESYSTEM_LIBS if an extra lib is required for c++ filesystem support. + # C++17 std::filesystem support is implemented in std::experimental::filesystem prior to GCC 9. + # Only the experimental version requires linking to stdc++fs library (so troublesome!). + # There is no built-in support in cmake to handle this difference (like a FindFileSystem.cmake module). + # See also: Portable linking for C++17 std::filesystem (https://gitlab.kitware.com/cmake/cmake/-/issues/17834) + + if (DO_PLATFORM_WINDOWS) + # std::filesystem not required for Windows builds + return () + endif () + + message ("Compiler identified: ${CMAKE_CXX_COMPILER_ID} - ${CMAKE_CXX_COMPILER_VERSION}") + if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") + if (${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 9.0.0) + message (STATUS "Using std::experimental filesystem library") + set(CXX_FILESYSTEM_LIBS stdc++fs PARENT_SCOPE) + endif () + endif () + +endfunction () diff --git a/sdk-cpp/CMakeLists.txt b/sdk-cpp/CMakeLists.txt index 8c3070a1..e58850c5 100644 --- a/sdk-cpp/CMakeLists.txt +++ b/sdk-cpp/CMakeLists.txt @@ -4,11 +4,7 @@ project (deliveryoptimization_sdk VERSION 1.0.0) if (DO_PLATFORM_LINUX OR DO_PLATFORM_MAC) - message("DO configured to use shared Boost libraries") - add_definitions(-DBOOST_ALL_DYN_LINK=1) - - # Include external libraries here - find_package(Boost COMPONENTS filesystem REQUIRED) + find_package(Boost REQUIRED) endif() if (DO_PLATFORM_LINUX) @@ -21,6 +17,7 @@ endif() set(DO_SDK_LIB_NAME "deliveryoptimization") add_do_version_lib("${DO_SDK_LIB_NAME}-lib" ${PROJECT_VERSION}) +try_set_filesystem_lib() ### Region CMake Build ### @@ -65,10 +62,6 @@ if(DO_PLATFORM_LINUX) dohttp ) - set(sdk_public_linked_libs - ${Boost_LIBRARIES} - ) - strip_symbols(${DO_SDK_LIB_NAME}) elseif(DO_PLATFORM_MAC) @@ -90,9 +83,6 @@ elseif(DO_PLATFORM_MAC) dohttp ) - set(sdk_public_linked_libs - ${Boost_LIBRARIES} - ) elseif(DO_PLATFORM_WINDOWS) file(GLOB sdk_source ${sdk_source_common} @@ -121,6 +111,7 @@ target_compile_definitions(${DO_SDK_LIB_NAME} PRIVATE ${sdk_compile_definitions} ) +add_boost_definitions(${DO_SDK_LIB_NAME} PRIVATE) if (DO_DEV_DEBUG) target_compile_definitions(${DO_SDK_LIB_NAME} PRIVATE DO_DEV_DEBUG) endif () @@ -134,10 +125,9 @@ endif () add_platform_interface_definitions(${DO_SDK_LIB_NAME}) target_link_libraries(${DO_SDK_LIB_NAME} - PUBLIC - ${sdk_public_linked_libs} PRIVATE ${sdk_private_linked_libs} + ${CXX_FILESYSTEM_LIBS} ) ### Endregion CMake Build ### diff --git a/sdk-cpp/build/cmake/deliveryoptimization_sdk-config.cmake b/sdk-cpp/build/cmake/deliveryoptimization_sdk-config.cmake index 63e550b8..45122fe8 100644 --- a/sdk-cpp/build/cmake/deliveryoptimization_sdk-config.cmake +++ b/sdk-cpp/build/cmake/deliveryoptimization_sdk-config.cmake @@ -1,11 +1,4 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -include(CMakeFindDependencyMacro) -if (${CMAKE_VERSION} VERSION_GREATER "3.9.0") - find_dependency(Boost COMPONENTS filesystem system) -else () - # Old cmake versions do not support extra args to find_dependency - find_dependency(Boost) -endif () include("${CMAKE_CURRENT_LIST_DIR}/deliveryoptimization_sdk-targets.cmake") diff --git a/sdk-cpp/src/internal/do_filesystem.h b/sdk-cpp/src/internal/do_filesystem.h new file mode 100644 index 00000000..ec4f1fd0 --- /dev/null +++ b/sdk-cpp/src/internal/do_filesystem.h @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#if defined(__cpp_lib_filesystem) +#define INCLUDE_STD_FILESYSTEM_EXPERIMENTAL 0 +#elif defined(__cpp_lib_experimental_filesystem) +#define INCLUDE_STD_FILESYSTEM_EXPERIMENTAL 1 +#elif !defined(__has_include) +// Cannot check if headers exist, assume experimental +#define INCLUDE_STD_FILESYSTEM_EXPERIMENTAL 1 +#elif __has_include() +#define INCLUDE_STD_FILESYSTEM_EXPERIMENTAL 0 +#elif __has_include() +#define INCLUDE_STD_FILESYSTEM_EXPERIMENTAL 1 +#else +#error Could not find system header "" or "" +#endif + +#if INCLUDE_STD_FILESYSTEM_EXPERIMENTAL +#include +namespace fs = std::experimental::filesystem; +#else +#include +namespace fs = std::filesystem; +#endif diff --git a/sdk-cpp/src/internal/rest/do_config_internal.cpp b/sdk-cpp/src/internal/rest/do_config_internal.cpp index 0a27f628..3018e676 100644 --- a/sdk-cpp/src/internal/rest/do_config_internal.cpp +++ b/sdk-cpp/src/internal/rest/do_config_internal.cpp @@ -8,10 +8,9 @@ #include #include -#include - #include #include +#include "do_filesystem.h" #include "do_persistence.h" #include "do_version.h" #elif defined(DO_CLIENT_DOSVC) @@ -31,9 +30,9 @@ static int WriteIoTConnectionStringToConfigFile(const char* value) noexcept int returnValue = 0; // ptree's exceptions do not provide an error code, and SDK has no logging of its own. // Return specific errors as a workaround. - boost::filesystem::path filePath{microsoft::deliveryoptimization::details::GetConfigFilePath()}; - boost::system::error_code ec; - if (boost::filesystem::exists(filePath.parent_path(), ec)) + fs::path filePath{microsoft::deliveryoptimization::details::GetConfigFilePath()}; + std::error_code ec; + if (fs::exists(filePath.parent_path(), ec)) { try { @@ -74,10 +73,10 @@ static std::string GetSdkVersion() return msdoutil::ComponentVersion(false); } -static std::string GetBinaryVersion(const boost::filesystem::path& binaryFilePath) +static std::string GetBinaryVersion(const fs::path& binaryFilePath) { std::string version; - if (boost::filesystem::exists(binaryFilePath)) + if (fs::exists(binaryFilePath)) { FILE* fp = nullptr; try @@ -118,7 +117,7 @@ static void AppendBinaryVersion(const char* binFileName, std::stringstream& allV // to cover special cases (for example, installed not via our deb/rpm packages). // We do not use bp::search_path() in order to avoid executing a malicious program with the // same name and which appears earlier in the PATH environment variable. - boost::filesystem::path binFilePath("/usr/local/bin"); + fs::path binFilePath("/usr/local/bin"); binFilePath /= binFileName; std::string version = GetBinaryVersion(binFilePath); if (version.empty()) diff --git a/sdk-cpp/src/internal/rest/util/do_port_finder.cpp b/sdk-cpp/src/internal/rest/util/do_port_finder.cpp index 7835d19a..3ba369a6 100644 --- a/sdk-cpp/src/internal/rest/util/do_port_finder.cpp +++ b/sdk-cpp/src/internal/rest/util/do_port_finder.cpp @@ -2,13 +2,13 @@ #include #include +#include #include #include -#include - #include "do_errors.h" #include "do_error_helpers.h" +#include "do_filesystem.h" #include "do_persistence.h" using namespace std::chrono_literals; // NOLINT(build/namespaces) @@ -25,19 +25,19 @@ namespace details static std::string g_DiscoverDOPort() { const std::string runtimeDirectory = GetRuntimeDirectory(); - if (!boost::filesystem::exists(runtimeDirectory)) + if (!fs::exists(runtimeDirectory)) { return std::string(); } - boost::filesystem::path mostRecentFile(""); - std::time_t mostRecentTime = 0; - for (boost::filesystem::directory_iterator itr(runtimeDirectory); itr != boost::filesystem::directory_iterator(); ++itr) + fs::path mostRecentFile(""); + auto mostRecentTime = fs::file_time_type::min(); + for (fs::directory_iterator itr(runtimeDirectory); itr != fs::directory_iterator(); ++itr) { - const boost::filesystem::path& dirEntry = itr->path(); + const fs::path& dirEntry = itr->path(); if (dirEntry.filename().string().find("restport") != std::string::npos) { - std::time_t currTime = boost::filesystem::last_write_time(dirEntry); + auto currTime = fs::last_write_time(dirEntry); if (currTime > mostRecentTime) { mostRecentTime = currTime; diff --git a/sdk-cpp/tests/CMakeLists.txt b/sdk-cpp/tests/CMakeLists.txt index 7d1978cd..d22d1623 100644 --- a/sdk-cpp/tests/CMakeLists.txt +++ b/sdk-cpp/tests/CMakeLists.txt @@ -3,7 +3,7 @@ ## Tests for DO SDK -find_package(Boost COMPONENTS filesystem program_options REQUIRED) +find_package(Boost COMPONENTS program_options REQUIRED) find_package(GTest REQUIRED) set(sdk_tests_linked_libs_common @@ -30,10 +30,6 @@ if(DO_PLATFORM_LINUX) "../src/internal/rest/util" ) - set(sdk_tests_linked_libs - stdc++fs - ) - set(test_source_private "rest/*.cpp" ) @@ -67,6 +63,7 @@ file (GLOB test_source add_executable(deliveryoptimization-sdk-tests ${test_source}) # Tests make use of C++ exceptions. MSEdge build on Windows disables C++ exceptions but also does not build our tests. target_compile_definitions(deliveryoptimization-sdk-tests PRIVATE DO_ENABLE_EXCEPTIONS) +add_boost_definitions(deliveryoptimization-sdk-tests PRIVATE) add_platform_interface_definitions(deliveryoptimization-sdk-tests) if (DO_BUILD_FOR_SNAP) @@ -77,9 +74,10 @@ target_include_directories(deliveryoptimization-sdk-tests PRIVATE ${dosdkcpp_private_includes} ${dosdkcpp_private_includes_common} + ${Boost_INCLUDE_DIRS} ) target_link_libraries(deliveryoptimization-sdk-tests - ${sdk_tests_linked_libs} ${sdk_tests_linked_libs_common} + ${CXX_FILESYSTEM_LIBS} ) \ No newline at end of file diff --git a/sdk-cpp/tests/download_properties_tests.cpp b/sdk-cpp/tests/download_properties_tests.cpp index e49b32b6..16637949 100644 --- a/sdk-cpp/tests/download_properties_tests.cpp +++ b/sdk-cpp/tests/download_properties_tests.cpp @@ -22,7 +22,7 @@ class DownloadPropertyTests : public ::testing::Test void SetUp() override { TestHelpers::CleanTestDir(); - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); } void TearDown() override { @@ -62,7 +62,7 @@ TEST_F(DownloadPropertyTests, SmallDownloadSetCallerNameTest) ASSERT_EQ(strCallerName, outCallerName); simpleDownload->start_and_wait_until_completion(); - ASSERT_TRUE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_TRUE(fs::exists(g_tmpFileName)); } TEST_F(DownloadPropertyTests, SmallDownloadWithPhfDigestandCvTest) @@ -83,7 +83,7 @@ TEST_F(DownloadPropertyTests, SmallDownloadWithPhfDigestandCvTest) ASSERT_EQ(simpleDownload->start_and_wait_until_completion().value(), 0); - ASSERT_TRUE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_TRUE(fs::exists(g_tmpFileName)); } TEST_F(DownloadPropertyTests, InvalidPhfDigestTest) @@ -102,7 +102,7 @@ TEST_F(DownloadPropertyTests, SmallDownloadWithCustomHeaders) auto simpleDownload = msdot::download::make(g_smallFileUrl, g_tmpFileName); simpleDownload->set_property(msdo::download_property::http_custom_headers, "XCustom1=someData\nXCustom2=moreData\n"); simpleDownload->start_and_wait_until_completion(); - ASSERT_TRUE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_TRUE(fs::exists(g_tmpFileName)); } TEST_F(DownloadPropertyTests, CallbackTestUseDownload) diff --git a/sdk-cpp/tests/download_tests_common.cpp b/sdk-cpp/tests/download_tests_common.cpp index 616b07ab..3bd02da2 100644 --- a/sdk-cpp/tests/download_tests_common.cpp +++ b/sdk-cpp/tests/download_tests_common.cpp @@ -11,8 +11,6 @@ #include #include -#include - #include "do_download.h" #include "do_download_status.h" #include "do_errors.h" @@ -80,21 +78,21 @@ TEST_F(DownloadTests, SimpleDownloadTest) ASSERT_EQ(status.bytes_total(), g_smallFileSizeBytes); simpleDownload->finalize(); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName)), g_smallFileSizeBytes); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName)), g_smallFileSizeBytes); } TEST_F(DownloadTests, SimpleBlockingDownloadTest) { - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); msdot::download::download_url_to_path(g_smallFileUrl, g_tmpFileName); - ASSERT_TRUE(boost::filesystem::exists(g_tmpFileName)); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName)), g_smallFileSizeBytes); + ASSERT_TRUE(fs::exists(g_tmpFileName)); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName)), g_smallFileSizeBytes); } TEST_F(DownloadTests, CancelBlockingDownloadTest) { std::atomic_bool cancelToken { false }; - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); std::thread downloadThread([&]() { try @@ -110,7 +108,7 @@ TEST_F(DownloadTests, CancelBlockingDownloadTest) std::this_thread::sleep_for(1s); cancelToken = true; downloadThread.join(); - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); } TEST_F(DownloadTests, BlockingDownloadTimeout) @@ -128,13 +126,13 @@ TEST_F(DownloadTests, BlockingDownloadTimeout) ASSERT_GE(elapsedTime, std::chrono::seconds(2)); ASSERT_LE(elapsedTime, std::chrono::seconds(5)); } - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); } // Note: This test takes a long time to execute due to 30 retry intervals from DOCS TEST_F(DownloadTests, SimpleDownloadTest_With404Url) { - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); try { @@ -165,13 +163,13 @@ TEST_F(DownloadTests, SimpleDownloadTest_WithMalformedPath) #elif defined(DO_INTERFACE_REST) ASSERT_EQ(e.error_code().value(), DO_ERROR_FROM_SYSTEM_ERROR(ENOENT)); #endif - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); } } TEST_F(DownloadTests, SimpleDownloadTest_With404UrlAndMalformedPath) { - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); try { @@ -189,7 +187,7 @@ TEST_F(DownloadTests, SimpleDownloadTest_With404UrlAndMalformedPath) #elif defined(DO_INTERFACE_REST) ASSERT_EQ(e.error_code().value(), DO_ERROR_FROM_SYSTEM_ERROR(ENOENT)); #endif - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); } catch (const std::exception& se) { @@ -200,7 +198,7 @@ TEST_F(DownloadTests, SimpleDownloadTest_With404UrlAndMalformedPath) TEST_F(DownloadTests, Download1PausedDownload2SameDestTest) { - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); auto simpleDownload = msdot::download::make(g_largeFileUrl, g_tmpFileName); msdo::download_status status = simpleDownload->get_status(); ASSERT_EQ(status.state(), msdo::download_state::created); @@ -214,7 +212,7 @@ TEST_F(DownloadTests, Download1PausedDownload2SameDestTest) #ifdef DO_CLIENT_AGENT // Only DO Agent creates the output file upon calling start() - ASSERT_TRUE(boost::filesystem::exists(g_tmpFileName)) << "Verify output file created for first download"; + ASSERT_TRUE(fs::exists(g_tmpFileName)) << "Verify output file created for first download"; // DO Agent does not support downloading to an already existing path (yet?). // Verify second download fails and the first download resumes successfully. @@ -229,23 +227,23 @@ TEST_F(DownloadTests, Download1PausedDownload2SameDestTest) ASSERT_EQ(e.error_code().value(), DO_ERROR_FROM_SYSTEM_ERROR(EEXIST)); } simpleDownload2->abort(); - ASSERT_TRUE(boost::filesystem::exists(g_tmpFileName)); // not deleted, the earlier download is still active + ASSERT_TRUE(fs::exists(g_tmpFileName)); // not deleted, the earlier download is still active simpleDownload->abort(); - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); // download2 should now succeed simpleDownload2 = msdot::download::make(g_smallFileUrl, g_tmpFileName); simpleDownload2->start(); WaitForDownloadCompletion(*simpleDownload2); - ASSERT_EQ(boost::filesystem::file_size(g_tmpFileName), g_smallFileSizeBytes); + ASSERT_EQ(fs::file_size(g_tmpFileName), g_smallFileSizeBytes); #elif defined(DO_CLIENT_DOSVC) // DoSvc does support downloading to an already existing path via // aborting the first download. Verify this and that the second download succeeds. msdo::test::download::download_url_to_path(g_smallFileUrl, g_tmpFileName); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName)), g_smallFileSizeBytes); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName)), g_smallFileSizeBytes); status = simpleDownload->get_status(); ASSERT_EQ(status.state(), msdo::download_state::aborted); @@ -257,7 +255,7 @@ TEST_F(DownloadTests, Download1PausedDownload2SameDestTest) TEST_F(DownloadTests, Download1PausedDownload2SameFileDownload1Resume) { - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); auto simpleDownload = msdot::download::make(g_largeFileUrl, g_tmpFileName); msdo::download_status status = simpleDownload->get_status(); ASSERT_EQ(status.state(), msdo::download_state::created); @@ -271,19 +269,19 @@ TEST_F(DownloadTests, Download1PausedDownload2SameFileDownload1Resume) std::cout << "Downloading the same file with a second download" << std::endl; msdot::download::download_url_to_path(g_largeFileUrl, g_tmpFileName2); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName2)), g_largeFileSizeBytes); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName2)), g_largeFileSizeBytes); std::cout << "Resuming and waiting for completion of first download" << std::endl; simpleDownload->resume(); TestHelpers::WaitForState(*simpleDownload, msdo::download_state::transferred, g_largeFileWaitTime); simpleDownload->finalize(); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName)), g_largeFileSizeBytes); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName)), g_largeFileSizeBytes); } TEST_F(DownloadTests, Download1NeverStartedDownload2CancelledSameFileTest) { - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); auto simpleDownload = msdot::download::make(g_largeFileUrl, g_tmpFileName); msdo::download_status status = simpleDownload->get_status(); ASSERT_EQ(status.state(), msdo::download_state::created); @@ -298,7 +296,7 @@ TEST_F(DownloadTests, Download1NeverStartedDownload2CancelledSameFileTest) { ASSERT_EQ(e.error_code().value(), msdo::errc::not_found); } - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); } TEST_F(DownloadTests, ResumeOnAlreadyDownloadedFileTest) @@ -316,7 +314,7 @@ TEST_F(DownloadTests, ResumeOnAlreadyDownloadedFileTest) ASSERT_EQ(status.bytes_total(), g_smallFileSizeBytes); simpleDownload->finalize(); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName)), g_smallFileSizeBytes); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName)), g_smallFileSizeBytes); try { @@ -348,7 +346,7 @@ TEST_F(DownloadTests, CancelDownloadOnCompletedState) ASSERT_EQ(status.bytes_total(), g_smallFileSizeBytes); simpleDownload->finalize(); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName)), g_smallFileSizeBytes); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName)), g_smallFileSizeBytes); try { @@ -380,7 +378,7 @@ TEST_F(DownloadTests, CancelDownloadInTransferredState) #if defined(DO_INTERFACE_REST) // On Windows need to finalize in order for file to show up on disk, so exclude check here - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName)), g_smallFileSizeBytes); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName)), g_smallFileSizeBytes); #endif try { @@ -422,7 +420,7 @@ static void _PauseResumeTest(bool delayAfterStart = false) ASSERT_EQ(status.state(), msdo::download_state::transferred); ASSERT_EQ(status.bytes_total(), status.bytes_transferred()); simpleDownload->finalize(); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName)), g_largeFileSizeBytes); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName)), g_largeFileSizeBytes); } TEST_F(DownloadTests, PauseResumeTest) @@ -437,25 +435,25 @@ TEST_F(DownloadTests, PauseResumeTestWithDelayAfterStart) TEST_F(DownloadTests, MultipleConsecutiveDownloadTest) { - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); msdot::download::download_url_to_path(g_smallFileUrl, g_tmpFileName); - ASSERT_TRUE(boost::filesystem::exists(g_tmpFileName)); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName)), g_smallFileSizeBytes); + ASSERT_TRUE(fs::exists(g_tmpFileName)); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName)), g_smallFileSizeBytes); - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName2)); + ASSERT_FALSE(fs::exists(g_tmpFileName2)); msdot::download::download_url_to_path(g_smallFileUrl, g_tmpFileName2); - ASSERT_TRUE(boost::filesystem::exists(g_tmpFileName2)); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName2)), g_smallFileSizeBytes); + ASSERT_TRUE(fs::exists(g_tmpFileName2)); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName2)), g_smallFileSizeBytes); - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName3)); + ASSERT_FALSE(fs::exists(g_tmpFileName3)); msdot::download::download_url_to_path(g_smallFileUrl, g_tmpFileName3); - ASSERT_TRUE(boost::filesystem::exists(g_tmpFileName3)); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName3)), g_smallFileSizeBytes); + ASSERT_TRUE(fs::exists(g_tmpFileName3)); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName3)), g_smallFileSizeBytes); } TEST_F(DownloadTests, MultipleConcurrentDownloadTest) { - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); std::thread downloadThread([&]() { try @@ -495,9 +493,9 @@ TEST_F(DownloadTests, MultipleConcurrentDownloadTest) downloadThread2.join(); downloadThread3.join(); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName)), g_smallFileSizeBytes); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName2)), g_smallFileSizeBytes); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName3)), g_smallFileSizeBytes); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName)), g_smallFileSizeBytes); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName2)), g_smallFileSizeBytes); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName3)), g_smallFileSizeBytes); } TEST_F(DownloadTests, MultipleConcurrentDownloadTest_WithCancels) @@ -546,9 +544,9 @@ TEST_F(DownloadTests, MultipleConcurrentDownloadTest_WithCancels) downloadThread2.join(); downloadThread3.join(); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName)), g_smallFileSizeBytes); - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName2)); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName3)), g_smallFileSizeBytes); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName)), g_smallFileSizeBytes); + ASSERT_FALSE(fs::exists(g_tmpFileName2)); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName3)), g_smallFileSizeBytes); } TEST_F(DownloadTests, FileDeletionAfterPause) @@ -566,8 +564,8 @@ TEST_F(DownloadTests, FileDeletionAfterPause) auto status = largeDownload->get_status(); ASSERT_EQ(status.state(), msdo::download_state::paused) << "Download is paused"; - boost::filesystem::remove(g_tmpFileName2); - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName2)) << "Output file deleted"; + fs::remove(g_tmpFileName2); + ASSERT_FALSE(fs::exists(g_tmpFileName2)) << "Output file deleted"; #if defined(DO_CLIENT_AGENT) try @@ -600,7 +598,7 @@ TEST_F(DownloadTests, SimpleBlockingDownloadTest_ClientNotRunning) }); TestHelpers::DeleteRestPortFiles(); // can be removed if docs deletes file on shutdown - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); try { msdot::download::download_url_to_path(g_smallFileUrl, g_tmpFileName); @@ -610,7 +608,7 @@ TEST_F(DownloadTests, SimpleBlockingDownloadTest_ClientNotRunning) { ASSERT_EQ(e.error_code().value(), msdo::errc::no_service); } - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); } TEST_F(DownloadTests, SimpleBlockingDownloadTest_ClientNotRunningPortFilePresent) @@ -623,7 +621,7 @@ TEST_F(DownloadTests, SimpleBlockingDownloadTest_ClientNotRunningPortFilePresent TestHelpers::DeleteRestPortFiles(); TestHelpers::CreateRestPortFiles(1); - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); try { msdot::download::download_url_to_path(g_smallFileUrl, g_tmpFileName); @@ -633,7 +631,7 @@ TEST_F(DownloadTests, SimpleBlockingDownloadTest_ClientNotRunningPortFilePresent { ASSERT_EQ(e.error_code().value(), msdo::errc::no_service); } - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); } TEST_F(DownloadTests, MultipleRestPortFileExists_Download) @@ -649,10 +647,10 @@ TEST_F(DownloadTests, MultipleRestPortFileExists_Download) std::this_thread::sleep_for(std::chrono::seconds(2)); ASSERT_EQ(TestHelpers::CountRestPortFiles(), 1u) << "All other restport files must be deleted by the agent"; - ASSERT_FALSE(boost::filesystem::exists(g_tmpFileName)); + ASSERT_FALSE(fs::exists(g_tmpFileName)); msdot::download::download_url_to_path(g_smallFileUrl, g_tmpFileName); - ASSERT_TRUE(boost::filesystem::exists(g_tmpFileName)); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName)), g_smallFileSizeBytes); + ASSERT_TRUE(fs::exists(g_tmpFileName)); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName)), g_smallFileSizeBytes); } #endif // DO_INTERFACE_REST diff --git a/sdk-cpp/tests/rest/config_tests.cpp b/sdk-cpp/tests/rest/config_tests.cpp index 181e7431..8ffa7057 100644 --- a/sdk-cpp/tests/rest/config_tests.cpp +++ b/sdk-cpp/tests/rest/config_tests.cpp @@ -4,7 +4,6 @@ #include "tests_common.h" #include -#include #include #include #include "do_config.h" @@ -12,7 +11,6 @@ #include "test_data.h" #include "test_helpers.h" -namespace cppfs = std::experimental::filesystem; namespace msdo = microsoft::deliveryoptimization; namespace msdod = microsoft::deliveryoptimization::details; @@ -22,9 +20,9 @@ class ConfigTests : public ::testing::Test void SetUp() override { TestHelpers::CleanTestDir(); - if (cppfs::exists(msdod::GetConfigFilePath())) + if (fs::exists(msdod::GetConfigFilePath())) { - cppfs::remove(msdod::GetConfigFilePath()); + fs::remove(msdod::GetConfigFilePath()); } } diff --git a/sdk-cpp/tests/rest/mcc_download_tests.cpp b/sdk-cpp/tests/rest/mcc_download_tests.cpp index ccd03ae9..64aa4bcf 100644 --- a/sdk-cpp/tests/rest/mcc_download_tests.cpp +++ b/sdk-cpp/tests/rest/mcc_download_tests.cpp @@ -5,7 +5,6 @@ #include #include -#include #include "do_config.h" #include "do_download.h" #include "do_download_status.h" @@ -14,7 +13,6 @@ #include "test_data.h" #include "test_helpers.h" -namespace cppfs = std::experimental::filesystem; namespace msdo = microsoft::deliveryoptimization; namespace msdod = microsoft::deliveryoptimization::details; namespace msdot = microsoft::deliveryoptimization::test; @@ -25,9 +23,9 @@ class MCCDownloadTests : public ::testing::Test void SetUp() override { TestHelpers::CleanTestDir(); - if (cppfs::exists(msdod::GetConfigFilePath())) + if (fs::exists(msdod::GetConfigFilePath())) { - cppfs::remove(msdod::GetConfigFilePath()); + fs::remove(msdod::GetConfigFilePath()); } } diff --git a/sdk-cpp/tests/rest/network_connectivity_tests.cpp b/sdk-cpp/tests/rest/network_connectivity_tests.cpp index 093ce228..f5daff7c 100644 --- a/sdk-cpp/tests/rest/network_connectivity_tests.cpp +++ b/sdk-cpp/tests/rest/network_connectivity_tests.cpp @@ -6,8 +6,6 @@ #include #include -#include - #include "do_download.h" #include "do_download_status.h" #include "do_errors.h" @@ -65,7 +63,7 @@ TEST_F(NetworkConnectivityTests, DISABLED_SimpleBlockingDownloadNetworkReconnect try { msdo::download::download_url_to_path(g_largeFileUrl, g_tmpFileName); - ASSERT_EQ(boost::filesystem::file_size(boost::filesystem::path(g_tmpFileName)), g_largeFileSizeBytes); + ASSERT_EQ(fs::file_size(fs::path(g_tmpFileName)), g_largeFileSizeBytes); } catch (const msdod::exception& e) { diff --git a/sdk-cpp/tests/rest/port_discovery_tests.cpp b/sdk-cpp/tests/rest/port_discovery_tests.cpp index 6bd2214d..220c982d 100644 --- a/sdk-cpp/tests/rest/port_discovery_tests.cpp +++ b/sdk-cpp/tests/rest/port_discovery_tests.cpp @@ -2,8 +2,6 @@ #include -#include - #include "do_persistence.h" #include "do_port_finder.h" #include "test_data.h" @@ -34,9 +32,9 @@ void PortDiscoveryTests::SetUp() { TestHelpers::StopService(g_docsSvcName); // ensure agent does not write to port file TestHelpers::DeleteRestPortFiles(); - if (!boost::filesystem::exists(msdod::GetRuntimeDirectory())) + if (!fs::exists(msdod::GetRuntimeDirectory())) { - boost::filesystem::create_directories(msdod::GetRuntimeDirectory()); + fs::create_directories(msdod::GetRuntimeDirectory()); } std::ofstream file(_testFilePath); @@ -45,9 +43,9 @@ void PortDiscoveryTests::SetUp() void PortDiscoveryTests::TearDown() { - if (boost::filesystem::exists(_testFilePath)) + if (fs::exists(_testFilePath)) { - boost::filesystem::remove(_testFilePath); + fs::remove(_testFilePath); } TestHelpers::StartService(g_docsSvcName); } diff --git a/sdk-cpp/tests/rest/rest_interface_tests.cpp b/sdk-cpp/tests/rest/rest_interface_tests.cpp index 3570de6b..42856ca9 100644 --- a/sdk-cpp/tests/rest/rest_interface_tests.cpp +++ b/sdk-cpp/tests/rest/rest_interface_tests.cpp @@ -4,8 +4,6 @@ #include "tests_common.h" #include -#include -namespace cppfs = std::experimental::filesystem; #include using btcp_t = boost::asio::ip::tcp; @@ -23,9 +21,9 @@ class RestInterfaceTests : public ::testing::Test public: void SetUp() override { - if (cppfs::exists(msdod::GetAdminConfigFilePath())) + if (fs::exists(msdod::GetAdminConfigFilePath())) { - cppfs::remove(msdod::GetAdminConfigFilePath()); + fs::remove(msdod::GetAdminConfigFilePath()); } } diff --git a/sdk-cpp/tests/rest/test_helpers.cpp b/sdk-cpp/tests/rest/test_helpers.cpp index c2c2c235..34cf6900 100644 --- a/sdk-cpp/tests/rest/test_helpers.cpp +++ b/sdk-cpp/tests/rest/test_helpers.cpp @@ -17,8 +17,6 @@ #include #include -#include - #include "do_persistence.h" #include "do_port_finder.h" @@ -64,12 +62,12 @@ void TestHelpers::CreateRestPortFiles(int numFiles) void TestHelpers::DeleteRestPortFiles() { - for (boost::filesystem::directory_iterator itr(msdod::GetRuntimeDirectory()); itr != boost::filesystem::directory_iterator(); ++itr) + for (fs::directory_iterator itr(msdod::GetRuntimeDirectory()); itr != fs::directory_iterator(); ++itr) { auto& dirEntry = itr->path(); if (dirEntry.filename().string().find("restport") != std::string::npos) { - boost::filesystem::remove(dirEntry); + fs::remove(dirEntry); } } } @@ -77,7 +75,7 @@ void TestHelpers::DeleteRestPortFiles() unsigned int TestHelpers::CountRestPortFiles() { unsigned int count = 0; - for (boost::filesystem::directory_iterator itr(msdod::GetRuntimeDirectory()); itr != boost::filesystem::directory_iterator(); ++itr) + for (fs::directory_iterator itr(msdod::GetRuntimeDirectory()); itr != fs::directory_iterator(); ++itr) { auto& dirEntry = itr->path(); if (dirEntry.filename().string().find("restport") != std::string::npos) diff --git a/sdk-cpp/tests/test_data.cpp b/sdk-cpp/tests/test_data.cpp index cbc01004..c0fbc30e 100644 --- a/sdk-cpp/tests/test_data.cpp +++ b/sdk-cpp/tests/test_data.cpp @@ -2,7 +2,7 @@ // Licensed under the MIT License. #include "test_data.h" -#include "boost/filesystem.hpp" +#include "do_filesystem.h" using namespace std::chrono_literals; // NOLINT(build/namespaces) @@ -14,14 +14,14 @@ const uint64_t g_prodFileSizeBytes = 25006511u; // For testing production scenario, use the same path DU agent will use static const std::string downloadPath = "/var/lib/deviceupdate-agent-downloads"; #else -static const std::string downloadPath = boost::filesystem::temp_directory_path().string(); +static const std::string downloadPath = fs::temp_directory_path().string(); #endif const std::string g_largeFileUrl = "http://main.oremdl.microsoft.com.nsatc.net/dotc/ReplacementDCATFile.txt"; const std::string g_malformedFilePath = "?f309adfasdf///dfasdfj39fjasldfjasdf/// ///.1"; -const std::string g_tmpFileName = (downloadPath / boost::filesystem::path("docsdk_testfile.txt")).string(); -const std::string g_tmpFileName2 = (downloadPath / boost::filesystem::path("docsdk_testfile2.txt")).string(); -const std::string g_tmpFileName3 = (downloadPath / boost::filesystem::path("docsdk_testfile3.txt")).string(); +const std::string g_tmpFileName = (downloadPath / fs::path("docsdk_testfile.txt")).string(); +const std::string g_tmpFileName2 = (downloadPath / fs::path("docsdk_testfile2.txt")).string(); +const std::string g_tmpFileName3 = (downloadPath / fs::path("docsdk_testfile3.txt")).string(); const std::string g_smallFileUrl = "http://main.oremdl.microsoft.com.nsatc.net/dotc/49c591d405d307e25e72a19f7e79b53d69f19954/43A54FC03C6A979E9AAEAE2493757D1429A5C8A8D093FB7B8103E8CC8DF7B6B6"; const std::string g_smallFilePhfInfoJson = R"( { diff --git a/sdk-cpp/tests/test_helpers.h b/sdk-cpp/tests/test_helpers.h index f638a1c8..264327a6 100644 --- a/sdk-cpp/tests/test_helpers.h +++ b/sdk-cpp/tests/test_helpers.h @@ -150,34 +150,34 @@ class TestHelpers public: static void CleanTestDir() { - if (boost::filesystem::exists(g_tmpFileName)) + if (fs::exists(g_tmpFileName)) { - boost::filesystem::remove(g_tmpFileName); + fs::remove(g_tmpFileName); } - if (boost::filesystem::exists(g_tmpFileName2)) + if (fs::exists(g_tmpFileName2)) { - boost::filesystem::remove(g_tmpFileName2); + fs::remove(g_tmpFileName2); } - if (boost::filesystem::exists(g_tmpFileName3)) + if (fs::exists(g_tmpFileName3)) { - boost::filesystem::remove(g_tmpFileName3); + fs::remove(g_tmpFileName3); } } // DoSvc creates temporary files with a unique name in the same directory as the output file - static void DeleteDoSvcTemporaryFiles(const boost::filesystem::path& outputFilePath) + static void DeleteDoSvcTemporaryFiles(const fs::path& outputFilePath) { - const boost::filesystem::path parentDir = outputFilePath.parent_path(); - for (boost::filesystem::directory_iterator itr(parentDir); itr != boost::filesystem::directory_iterator(); ++itr) + const fs::path parentDir = outputFilePath.parent_path(); + for (fs::directory_iterator itr(parentDir); itr != fs::directory_iterator(); ++itr) { - const boost::filesystem::directory_entry& dirEntry = *itr; + const fs::directory_entry& dirEntry = *itr; // Remove all files with names that match DO*.tmp - if (boost::filesystem::is_regular_file(dirEntry) + if (fs::is_regular_file(dirEntry) && (dirEntry.path().filename().string().find("DO") == 0) && (dirEntry.path().extension() == ".tmp")) { - boost::system::error_code ec; - boost::filesystem::remove(dirEntry, ec); + std::error_code ec; + fs::remove(dirEntry, ec); if (ec) { std::cout << "Temp file deletion error: " << ec.message() << ", " << dirEntry.path() << '\n'; diff --git a/sdk-cpp/tests/tests_common.h b/sdk-cpp/tests/tests_common.h index e47dc26d..7a278253 100644 --- a/sdk-cpp/tests/tests_common.h +++ b/sdk-cpp/tests/tests_common.h @@ -4,10 +4,10 @@ #ifndef _DELIVERY_OPTIMIZATION_TESTS_COMMON_H #define _DELIVERY_OPTIMIZATION_TESTS_COMMON_H -#include #include #include #include "do_error_helpers.h" +#include "do_filesystem.h" #endif diff --git a/snapcraft-options/snapcraft-agent.yaml b/snapcraft-options/snapcraft-agent.yaml index f8826526..0ab5000c 100644 --- a/snapcraft-options/snapcraft-agent.yaml +++ b/snapcraft-options/snapcraft-agent.yaml @@ -44,7 +44,6 @@ parts: stage-packages: - libasn1-8-heimdal - - libboost-filesystem1.71.0 - libbrotli1 - libcurl4 - libgssapi3-heimdal diff --git a/snapcraft-options/snapcraft-sdk.yaml b/snapcraft-options/snapcraft-sdk.yaml index 5ab07bee..ca375e5d 100644 --- a/snapcraft-options/snapcraft-sdk.yaml +++ b/snapcraft-options/snapcraft-sdk.yaml @@ -46,7 +46,6 @@ parts: stage-packages: - libboost-program-options1.71.0 - - libboost-filesystem1.71.0 apps: sdk-tests: From 02c9ae2c7484182903c66ad986a834762fc569e6 Mon Sep 17 00:00:00 2001 From: shishirb-MSFT <50385517+shishirb-MSFT@users.noreply.github.com> Date: Thu, 23 Mar 2023 12:44:54 -0700 Subject: [PATCH 17/25] A more complete fix for non-existent /include path (#167) * A prior PR (#145) fixed the issue when building DO itself. However, consumers of DO still face the issue because DO's cmake config file was not updated. This is now fixed. * DO's own CMakeLists.txt checks for /lib being a symlink before updating CMAKE_PREFIX_PATH (update to fix in #145). * Also enable exec permission for the sdk cleanup install script. --- CMakeLists.txt | 9 ++++++-- sdk-cpp/build/cleanup-install.sh | 0 .../deliveryoptimization_sdk-config.cmake | 23 ++++++++++++++++++- 3 files changed, 29 insertions(+), 3 deletions(-) mode change 100644 => 100755 sdk-cpp/build/cleanup-install.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index c279e92d..d23aa6fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,8 +89,13 @@ if (DO_PLATFORM_LINUX) # (Enable --trace-expand in cmake and look at lines from boost_headers-config.cmake and its callers.) # Workaround: force cmake to look in "/usr" since the /lib and /bin paths are either # not relevant for our usage or they are symlinks to /usr/lib and /usr/bin. - if (NOT CMAKE_PREFIX_PATH) - set(CMAKE_PREFIX_PATH "/usr") + if (IS_SYMLINK "/lib" AND NOT EXISTS "/include") + if (CMAKE_PREFIX_PATH) + message (WARNING "/lib is a symlink and /include does not exist. CMAKE_PREFIX_PATH is already set, not overriding it.") + else () + message (WARNING "/lib is a symlink and /include does not exist. Updating CMAKE_PREFIX_PATH to /usr.") + set(CMAKE_PREFIX_PATH "/usr") + endif () endif () endif (DO_PLATFORM_LINUX) diff --git a/sdk-cpp/build/cleanup-install.sh b/sdk-cpp/build/cleanup-install.sh old mode 100644 new mode 100755 diff --git a/sdk-cpp/build/cmake/deliveryoptimization_sdk-config.cmake b/sdk-cpp/build/cmake/deliveryoptimization_sdk-config.cmake index 45122fe8..72f75d0b 100644 --- a/sdk-cpp/build/cmake/deliveryoptimization_sdk-config.cmake +++ b/sdk-cpp/build/cmake/deliveryoptimization_sdk-config.cmake @@ -1,4 +1,25 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -include("${CMAKE_CURRENT_LIST_DIR}/deliveryoptimization_sdk-targets.cmake") +if (NOT TARGET Microsoft::deliveryoptimization) + include("${CMAKE_CURRENT_LIST_DIR}/deliveryoptimization_sdk-targets.cmake") +endif () + +if (TARGET Microsoft::deliveryoptimization) + if (CMAKE_SYSTEM_NAME MATCHES Linux) + + # Ubuntu 20.04 symlinks /lib to /usr/lib but there is no equivalent /include path. + # This makes cmake's auto-generated targets file set include dirs to the non-existent + # /include path. Fix it by updating the INTERFACE_INCLUDE_DIRECTORIES property directly. + + get_target_property(DOSDK_INC_DIR_PRIV Microsoft::deliveryoptimization INTERFACE_INCLUDE_DIRECTORIES) + if (${DOSDK_INC_DIR_PRIV} MATCHES "^/include" AND NOT EXISTS "/include") + message(WARNING "/include does not exist. Updating include dirs from ${DOSDK_INC_DIR_PRIV} to /usr${DOSDK_INC_DIR_PRIV}.") + set_target_properties(Microsoft::deliveryoptimization PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "/usr${DOSDK_INC_DIR_PRIV}") + endif () + + # Cleanup temporary variable + set(DOSDK_INC_DIR_PRIV) + + endif () +endif () From 7fdc10b7c7b0dd46005498d894d18c7ddddba432 Mon Sep 17 00:00:00 2001 From: andretoyama-msft <106272532+andretoyama-msft@users.noreply.github.com> Date: Tue, 2 May 2023 12:47:59 -0300 Subject: [PATCH 18/25] Adding support for building DO snaps for arm64 architecture (#170) --- build/build-snaps.sh | 7 ++++++- snapcraft-options/snapcraft-agent.yaml | 4 ++++ snapcraft-options/snapcraft-sdk.yaml | 7 +++++++ snapcraft-options/ubuntu-snap.md | 20 +++++++++++++++++--- 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/build/build-snaps.sh b/build/build-snaps.sh index c0d6de34..fcc23c6e 100755 --- a/build/build-snaps.sh +++ b/build/build-snaps.sh @@ -22,7 +22,12 @@ then cp ./snapcraft-options/snapcraft-sdk.yaml ./snap/snapcraft.yaml fi - snapcraft + if [ $2 -a $2 == "arm64" ] + then + snapcraft --target-arch=arm64 --destructive-mode --enable-experimental-target-arch + else + snapcraft + fi else echo "$1 is not a valid snap option, valid options are 'agent' or 'sdk-tests'" diff --git a/snapcraft-options/snapcraft-agent.yaml b/snapcraft-options/snapcraft-agent.yaml index 0ab5000c..3beed190 100644 --- a/snapcraft-options/snapcraft-agent.yaml +++ b/snapcraft-options/snapcraft-agent.yaml @@ -8,6 +8,10 @@ description: | grade: devel # must be 'stable' to release into candidate/stable channels confinement: strict # use 'strict' once you have the right plugs and slots +architectures: + - build-on: [amd64, arm64] + run-on: [amd64, armhf, arm64] + ##### # # Keywords diff --git a/snapcraft-options/snapcraft-sdk.yaml b/snapcraft-options/snapcraft-sdk.yaml index ca375e5d..26b78eae 100644 --- a/snapcraft-options/snapcraft-sdk.yaml +++ b/snapcraft-options/snapcraft-sdk.yaml @@ -8,6 +8,10 @@ description: | grade: devel # must be 'stable' to release into candidate/stable channels confinement: strict # use 'strict' once you have the right plugs and slots +architectures: + - build-on: [amd64, arm64] + run-on: [amd64, armhf, arm64] + ##### # # Keywords @@ -50,6 +54,9 @@ parts: apps: sdk-tests: command: bin/deliveryoptimization-sdk-tests + plugs: + - network + - network-bind plugs: do-port-numbers: diff --git a/snapcraft-options/ubuntu-snap.md b/snapcraft-options/ubuntu-snap.md index 46ecbdfa..6fa78385 100644 --- a/snapcraft-options/ubuntu-snap.md +++ b/snapcraft-options/ubuntu-snap.md @@ -10,13 +10,19 @@ Run the commands in the root of the repo. - `$ ./build/build-snaps.sh agent` - `$ ./build/build-snaps.sh sdk-tests` +- **Building on ARM64 architecture** + - `$ ./build/build-snaps.sh agent arm64` + - `$ ./build/build-snaps.sh sdk-tests arm64` + +If the `arm64` argument is not provided then by default snapcraft will build the snap for `amd64` architecture. + - By default snapcraft doesn't allow multiple snaps definition in the same code base, so this build-snaps.sh file was created so that we can copy the different snap files into the main snapcraft.yaml file and then build it. - Subsequent builds of the same component without modifying the corresponding yaml file: - `$ snapcraft` -- The build will generate *.snap files in the current working directory. Example: **deliveryoptimization-sdk-tests_0.1_amd64.snap**. +- The build will generate *.snap files in the current working directory. Example: **deliveryoptimization-sdk-tests_0.1_multi.snap**. - In the snapcraft.yaml the regular script to build for both the delivery optimization agent and sdk is called (`build.py`), but since there are some different behaviors the code should have when inside a snap, for that we use a flag `--build-for-snap` on the build. @@ -26,8 +32,16 @@ file when you call the `build-snaps.sh` file. - `$ python3 ./build/build.py --project sdk --build-for-snap` ## Installing -- `$ sudo snap install --devmode ./deliveryoptimization-sdk-tests_0.1_amd64.snap` -- `$ sudo snap install --devmode ./deliveryoptimization-client_0.1_amd64.snap` +- `$ sudo snap install --devmode ./deliveryoptimization-sdk-tests_0.1_multi.snap` +- `$ sudo snap install --devmode ./deliveryoptimization-client_0.1_multi.snap` + +Note that the `--devmode` flag is being used in the install command, this should be used in devmode confinment only. If the snap is in strict confinment then you can remove the flag. + +In case of metadata signatures error when running the install command, the `--dangerous` flag should be used. +The --dangerous flag in snapcraft bypasses the signature check, allowing you to install an unsigned Snap package. + +- `$ sudo snap install --dangerous ./deliveryoptimization-sdk-tests_0.1_multi.snap` +- `$ sudo snap install --dangerous ./deliveryoptimization-client_0.1_multi.snap` ## The snap environment It is a fruitful exercise to look around in the host file system and see how snap structures the installed snaps. Look at these paths to start with: From c86438fc48d24b4c1290fdb82b0cd002df58735d Mon Sep 17 00:00:00 2001 From: andretoyama-msft <106272532+andretoyama-msft@users.noreply.github.com> Date: Wed, 24 May 2023 18:55:54 -0300 Subject: [PATCH 19/25] Adjust agent snapcraft yaml before first publish (#171) --- snapcraft-options/snapcraft-agent.yaml | 4 ++-- snapcraft-options/ubuntu-snap.md | 28 +++++++++++++------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/snapcraft-options/snapcraft-agent.yaml b/snapcraft-options/snapcraft-agent.yaml index 3beed190..a1ae6fb9 100644 --- a/snapcraft-options/snapcraft-agent.yaml +++ b/snapcraft-options/snapcraft-agent.yaml @@ -1,6 +1,6 @@ -name: deliveryoptimization-client +name: deliveryoptimization-agent base: core20 # the base snap is the execution environment for this snap -version: '0.1' +version: '1.0.1' summary: Ubuntu Core 20.04 DO Agent description: | A proof-of-concept for the Delivery Optimization Client Ubuntu Core snap. diff --git a/snapcraft-options/ubuntu-snap.md b/snapcraft-options/ubuntu-snap.md index 6fa78385..2088a7ab 100644 --- a/snapcraft-options/ubuntu-snap.md +++ b/snapcraft-options/ubuntu-snap.md @@ -33,7 +33,7 @@ file when you call the `build-snaps.sh` file. ## Installing - `$ sudo snap install --devmode ./deliveryoptimization-sdk-tests_0.1_multi.snap` -- `$ sudo snap install --devmode ./deliveryoptimization-client_0.1_multi.snap` +- `$ sudo snap install --devmode ./deliveryoptimization-agent_1.0.1_multi.snap` Note that the `--devmode` flag is being used in the install command, this should be used in devmode confinment only. If the snap is in strict confinment then you can remove the flag. @@ -41,7 +41,7 @@ In case of metadata signatures error when running the install command, the `--da The --dangerous flag in snapcraft bypasses the signature check, allowing you to install an unsigned Snap package. - `$ sudo snap install --dangerous ./deliveryoptimization-sdk-tests_0.1_multi.snap` -- `$ sudo snap install --dangerous ./deliveryoptimization-client_0.1_multi.snap` +- `$ sudo snap install --dangerous ./deliveryoptimization-agent_1.0.1_multi.snap` ## The snap environment It is a fruitful exercise to look around in the host file system and see how snap structures the installed snaps. Look at these paths to start with: @@ -60,22 +60,22 @@ and the plug is the consumer snap. ```shell Interface Plug Slot Notes -content - deliveryoptimization-client:do-configs - -content - deliveryoptimization-client:do-port-numbers - -content deliveryoptimization-client:deviceupdate-agent-downloads - - -network deliveryoptimization-client:network :network - -network-bind deliveryoptimization-client:network-bind :network-bind - +content - deliveryoptimization-agent:do-configs - +content - deliveryoptimization-agent:do-port-numbers - +content deliveryoptimization-agent:deviceupdate-agent-downloads - - +network deliveryoptimization-agent:network :network - +network-bind deliveryoptimization-agent:network-bind :network-bind - ``` ## Running or executing - Agent - The agent is declared as a daemon/service, so it starts running immediately after successful installation. - - Stop service: `$ sudo snap stop deliveryoptimization-client.agent` - - Start service: `$ sudo snap start deliveryoptimization-client.agent` - - Read service journal: `$ sudo snap logs deliveryoptimization-client.agent` - - systemctl can also be used with the correct service unit name: `$ systemctl status snap.deliveryoptimization-client.agent.service` - - **journalctl** can also be used to follow logs: `$ journalctl -f -u snap.deliveryoptimization-client.agent.service` + - Stop service: `$ sudo snap stop deliveryoptimization-agent.agent` + - Start service: `$ sudo snap start deliveryoptimization-agent.agent` + - Read service journal: `$ sudo snap logs deliveryoptimization-agent.agent` + - systemctl can also be used with the correct service unit name: `$ systemctl status snap.deliveryoptimization-agent.agent.service` + - **journalctl** can also be used to follow logs: `$ journalctl -f -u snap.deliveryoptimization-agent.agent.service` - SDK tests - This snap declares the **deliveryoptimization-sdk-tests** binary as an app. @@ -88,10 +88,10 @@ network-bind deliveryoptimization-client:network-bind :network ## Uninstalling or cleaning up - Uninstall installed snap packages - - `$ sudo snap remove deliveryoptimization-client` + - `$ sudo snap remove deliveryoptimization-agent` - `$ sudo snap remove deliveryoptimization-sdk-tests` - Cleaning up build environment: Building sometimes fails with strange errors from **multipass**. Cleaning up the multipass instance helps. - List the available instances: `$ multipass list` - - Delete an instance, example: `$ multipass delete snapcraft-deliveryoptimization-client` + - Delete an instance, example: `$ multipass delete snapcraft-deliveryoptimization-agent` - Purge deleted instances: `$ multipass purge` From d0e7d7f886f3b62c78baca2298d425080c4fe4e0 Mon Sep 17 00:00:00 2001 From: andretoyama-msft <106272532+andretoyama-msft@users.noreply.github.com> Date: Thu, 15 Jun 2023 11:26:59 -0300 Subject: [PATCH 20/25] User/andretoyama msft/support ubuntu2204 (#172) * Add pipeline suport ubuntu 22.04 * Add Dockerfiles to build ubuntu 22.04 images * fix gtest release * fix gsl include for ubuntu 22.04 * update pip install for ubuntu 22.04 * install file utility dep for CPackDeb --- .../linux/du/docker/doclient-lite-docker.yml | 11 +++++ .../build/linux/du/docker/dopapt-docker.yml | 11 +++++ .../build/linux/du/docker/dosdkcpp-docker.yml | 12 +++++ azure-pipelines/publishing/github-release.yml | 8 ++++ build/docker/ubuntu2204/amd64/Dockerfile | 36 ++++++++++++++ build/docker/ubuntu2204/arm64/Dockerfile | 47 +++++++++++++++++++ build/scripts/bootstrap.sh | 15 +++--- client-lite/src/ipc/rest_port_advertiser.h | 2 +- common/lib-dohttp/do_http_parser.cpp | 2 +- .../src/internal/rest/util/do_http_client.cpp | 2 +- 10 files changed, 137 insertions(+), 9 deletions(-) create mode 100644 build/docker/ubuntu2204/amd64/Dockerfile create mode 100644 build/docker/ubuntu2204/arm64/Dockerfile diff --git a/azure-pipelines/build/linux/du/docker/doclient-lite-docker.yml b/azure-pipelines/build/linux/du/docker/doclient-lite-docker.yml index 52c65b73..38db4f67 100644 --- a/azure-pipelines/build/linux/du/docker/doclient-lite-docker.yml +++ b/azure-pipelines/build/linux/du/docker/doclient-lite-docker.yml @@ -61,3 +61,14 @@ jobs: imageVersion: ${{variables.containerImageVersion}} stepsTemplate: 'doclient-lite-docker-steps.yml' +- template: ../templates/do-docker-jobs.yml + parameters: + targetOsArch: 'ubuntu2204_arm64' # azure pipelines does not support '.' in display names + imageVersion: ${{variables.containerImageVersion}} + stepsTemplate: 'doclient-lite-docker-steps.yml' + +- template: ../templates/do-docker-jobs.yml + parameters: + targetOsArch: 'ubuntu2204_x64' # azure pipelines does not support '.' in display names + imageVersion: ${{variables.containerImageVersion}} + stepsTemplate: 'doclient-lite-docker-steps.yml' diff --git a/azure-pipelines/build/linux/du/docker/dopapt-docker.yml b/azure-pipelines/build/linux/du/docker/dopapt-docker.yml index bbda6c76..10be67e9 100644 --- a/azure-pipelines/build/linux/du/docker/dopapt-docker.yml +++ b/azure-pipelines/build/linux/du/docker/dopapt-docker.yml @@ -61,3 +61,14 @@ jobs: imageVersion: ${{variables.containerImageVersion}} stepsTemplate: 'dopapt-docker-steps.yml' +- template: ../templates/do-docker-jobs.yml + parameters: + targetOsArch: 'ubuntu2204_arm64' + imageVersion: ${{variables.containerImageVersion}} + stepsTemplate: 'dopapt-docker-steps.yml' + +- template: ../templates/do-docker-jobs.yml + parameters: + targetOsArch: 'ubuntu2204_x64' + imageVersion: ${{variables.containerImageVersion}} + stepsTemplate: 'dopapt-docker-steps.yml' diff --git a/azure-pipelines/build/linux/du/docker/dosdkcpp-docker.yml b/azure-pipelines/build/linux/du/docker/dosdkcpp-docker.yml index 8c644f60..a8a2d44b 100644 --- a/azure-pipelines/build/linux/du/docker/dosdkcpp-docker.yml +++ b/azure-pipelines/build/linux/du/docker/dosdkcpp-docker.yml @@ -60,3 +60,15 @@ jobs: targetOsArch: 'ubuntu2004_x64' imageVersion: ${{variables.containerImageVersion}} stepsTemplate: 'dosdkcpp-docker-steps.yml' + +- template: ../templates/do-docker-jobs.yml + parameters: + targetOsArch: 'ubuntu2204_arm64' + imageVersion: ${{variables.containerImageVersion}} + stepsTemplate: 'dosdkcpp-docker-steps.yml' + +- template: ../templates/do-docker-jobs.yml + parameters: + targetOsArch: 'ubuntu2204_x64' + imageVersion: ${{variables.containerImageVersion}} + stepsTemplate: 'dosdkcpp-docker-steps.yml' diff --git a/azure-pipelines/publishing/github-release.yml b/azure-pipelines/publishing/github-release.yml index 03d0eb39..d065e4eb 100644 --- a/azure-pipelines/publishing/github-release.yml +++ b/azure-pipelines/publishing/github-release.yml @@ -60,6 +60,14 @@ stages: parameters: targetOsArch: 'ubuntu2004_arm64' imageVersion: ${{variables.containerImageVersion}} + - template: templates/release-docker-build-steps.yml + parameters: + targetOsArch: 'ubuntu2204_x64' + imageVersion: ${{variables.containerImageVersion}} + - template: templates/release-docker-build-steps.yml + parameters: + targetOsArch: 'ubuntu2204_arm64' + imageVersion: ${{variables.containerImageVersion}} - template: templates/release-docker-build-steps.yml parameters: targetOsArch: 'debian10_arm32' diff --git a/build/docker/ubuntu2204/amd64/Dockerfile b/build/docker/ubuntu2204/amd64/Dockerfile new file mode 100644 index 00000000..5977d074 --- /dev/null +++ b/build/docker/ubuntu2204/amd64/Dockerfile @@ -0,0 +1,36 @@ +# Dockerfile for building DO client components for Ubuntu 22.04 amd64 +# First, install the docker extension for VSCode. Then you can right-click on this file +# and choose Build Image. Give it a name and it will build the image. +# +# Open interactive terminal into the image in a container: +# docker run -ti --rm --entrypoint=/bin/bash -v :/code -v :/build +# Example: +# docker run -ti --rm --entrypoint=/bin/bash -v D:\do-client-lite:/code -v D:\temp\build_client_lite\arm-linux-debug:/build custom-ubuntu2204-arm64 + +FROM mcr.microsoft.com/mirror/docker/library/ubuntu:22.04@sha256:2fdb1cf4995abb74c035e5f520c0f3a46f12b3377a59e86ecca66d8606ad64f9 + +SHELL [ "/bin/bash", "-c"] + +# You can build the image by running in the current dockerfile directory +# sudo docker build -t . --no-cache --network=host + +# Ubuntu 22.04 requires user prompt for apt-get update command, docker has issues handling this input +# ENV DEBIAN_FRONTEND=noninteractive + +COPY bootstrap.sh /tmp/bootstrap.sh + +WORKDIR /tmp/ +RUN chmod +x bootstrap.sh +RUN ./bootstrap.sh --install build + +VOLUME /code +WORKDIR /code + +ENTRYPOINT [ "/bin/bash", "-c"] + +# We specify an empty command so that we can pass options to the ENTRYPOINT command. +# This is a bit of a Dockerfile quirk where if the ENTRYPOINT value is defined, +# then CMD becomes the default options passed to ENTRYPOINT. +# In this case we don't have any desired default arguments. +# However, we have to specify CMD to enable passing of command line parameters to ENTRYPOINT in the first place. +CMD [ ] diff --git a/build/docker/ubuntu2204/arm64/Dockerfile b/build/docker/ubuntu2204/arm64/Dockerfile new file mode 100644 index 00000000..85677ae7 --- /dev/null +++ b/build/docker/ubuntu2204/arm64/Dockerfile @@ -0,0 +1,47 @@ +# Dockerfile for building DO client components for Ubuntu 22.04 arm64 +# First, install the docker extension for VSCode. Then you can right-click on this file +# and choose Build Image. Give it a name and it will build the image. +# +# Open interactive terminal into the image in a container: +# docker run -ti --rm --entrypoint=/bin/bash -v :/code -v :/build +# Example: +# docker run -ti --rm --entrypoint=/bin/bash -v D:\do-client-lite:/code -v D:\temp\build_client_lite\arm-linux-debug:/build custom-ubuntu2204-arm64 + +FROM mcr.microsoft.com/mirror/docker/library/ubuntu:22.04@sha256:77bdd217935d10f0e753ed84118e9b11d3ab0a66a82bdf322087354ccd833733 +SHELL [ "/bin/bash", "-c"] + +# QEMU is a Linux emulator which enables cross-arch support in docker +# In order to build this image on a Linux host, need to install QEMU: +# +# sudo apt-get install qemu-user +# update-binfmts --display +# sudo apt install qemu binfmt-support qemu-user-static +# cp /usr/bin/qemu-aarch64-static /build/docker/ubuntu2204/arm64 +# +# Then copy the build script to the build directory +# cp /build/bootstrap.sh build/docker/ubuntu2204/arm64 +# +# After running the above, you can build the image by running in the current dockerfile directory +# sudo docker build -t . --no-cache --network=host + +# Ubuntu 22.04 requires user prompt for apt-get update command, docker has issues handling this input +ENV DEBIAN_FRONTEND=noninteractive + +COPY qemu-aarch64-static /usr/bin/qemu-aarch64-static +COPY bootstrap.sh /tmp/bootstrap.sh + +WORKDIR /tmp/ +RUN chmod +x bootstrap.sh +RUN ./bootstrap.sh --install build + +VOLUME /code +WORKDIR /code + +ENTRYPOINT [ "/bin/bash", "-c"] + +# We specify an empty command so that we can pass options to the ENTRYPOINT command. +# This is a bit of a Dockerfile quirk where if the ENTRYPOINT value is defined, +# then CMD becomes the default options passed to ENTRYPOINT. +# In this case we don't have any desired default arguments. +# However, we have to specify CMD to enable passing of command line parameters to ENTRYPOINT in the first place. +CMD [ ] diff --git a/build/scripts/bootstrap.sh b/build/scripts/bootstrap.sh index 43d35584..f9453bd5 100755 --- a/build/scripts/bootstrap.sh +++ b/build/scripts/bootstrap.sh @@ -68,21 +68,25 @@ function installBuildDependencies apt-get install -y make build-essential g++ gdb gdbserver gcc git wget apt-get install -y python3 ninja-build - apt-get install -y cmake libmsgsl-dev apt-get install -y libboost-program-options-dev apt-get install -y libproxy-dev libssl-dev uuid-dev libcurl4-openssl-dev + apt-get install -y cmake libmsgsl-dev + apt-get install -y file # Required by CPack for packaging and by default is not installed on Ubuntu 22.04 + rm -rf /tmp/gtest mkdir /tmp/gtest pushd /tmp/gtest - if [[ ($DISTRO == "ubuntu" && $VER == "20.04") || ($DISTRO == "debian" && $VER == "10") ]]; + if [[ ($DISTRO == "ubuntu" && ($VER == "20.04" || $VER == "22.04")) || ($DISTRO == "debian" && $VER == "10") ]]; then + if [[ $VER == "22.04" ]]; then release="v1.13.0"; else release="release-1.10.0"; fi; + # The latest native-version of gtest on ubuntu2004 and debian10 currently has a bug where # CMakeLists doesn't declare an install target, causing 'make install' to fail. # Clone from source and use release-1.10.0 instead, since gtest is a source package anyways. git clone https://github.com/google/googletest.git . - git checkout release-1.10.0 + git checkout $release mkdir cmake cd cmake cmake /tmp/gtest @@ -112,8 +116,7 @@ function installDeveloperTools brew install cpplint elif [ $OS == "linux" ]; then - apt-get install -y python-pip - pip install cpplint + apt install python3-pip # Installs to a non-standard location so add to PATH manually export PATH=$PATH:~/.local/bin @@ -164,7 +167,7 @@ function installAll function isSupportedLinux() { - if [[ ($DISTRO == "ubuntu" && ($VER == "18.04" || $VER == "20.04")) + if [[ ($DISTRO == "ubuntu" && ($VER == "18.04" || $VER == "20.04" || $VER == "22.04")) || ($DISTRO == "debian" && ($VER == "10" || $VER == "11")) ]]; then return 0 diff --git a/client-lite/src/ipc/rest_port_advertiser.h b/client-lite/src/ipc/rest_port_advertiser.h index c137f0d3..e0635c77 100644 --- a/client-lite/src/ipc/rest_port_advertiser.h +++ b/client-lite/src/ipc/rest_port_advertiser.h @@ -7,7 +7,7 @@ #include // open, write #include // getpid #include -#include +#include #include "do_filesystem.h" #include "do_persistence.h" #include "error_macros.h" diff --git a/common/lib-dohttp/do_http_parser.cpp b/common/lib-dohttp/do_http_parser.cpp index 12740d7b..785aa177 100644 --- a/common/lib-dohttp/do_http_parser.cpp +++ b/common/lib-dohttp/do_http_parser.cpp @@ -7,7 +7,7 @@ #include #endif #include -#include +#include namespace microsoft { diff --git a/sdk-cpp/src/internal/rest/util/do_http_client.cpp b/sdk-cpp/src/internal/rest/util/do_http_client.cpp index ec58d6e9..f044f2a4 100644 --- a/sdk-cpp/src/internal/rest/util/do_http_client.cpp +++ b/sdk-cpp/src/internal/rest/util/do_http_client.cpp @@ -7,7 +7,7 @@ // Include this header explicitly to get it regardless of which boost version is installed. #include #include -#include +#include #include "do_errors.h" #include "do_error_helpers.h" From bd9883ccfbc80e3bd58b2cf1dfbbebd0f074500f Mon Sep 17 00:00:00 2001 From: andretoyama-msft <106272532+andretoyama-msft@users.noreply.github.com> Date: Thu, 6 Jul 2023 10:56:44 -0300 Subject: [PATCH 21/25] add parallel flag into build options (#173) * add parallel flag into build options * add RPi recognizition in the build --- build/build.py | 14 ++++++++++++++ sdk-cpp/tests/test_data.cpp | 2 +- snapcraft-options/connect-snaps.sh | 6 +++--- snapcraft-options/snapcraft-agent.yaml | 10 +++++----- snapcraft-options/snapcraft-sdk.yaml | 10 +++++----- 5 files changed, 28 insertions(+), 14 deletions(-) diff --git a/build/build.py b/build/build.py index 9f64a245..e514696a 100644 --- a/build/build.py +++ b/build/build.py @@ -102,6 +102,12 @@ def __init__(self): '--search-prefix', dest='search_prefix', type=str, help='search prefix to pass to CMake' ) + + '''Option to control resource usage when running on a Raspberry Pi''' + self.parser.add_argument( + '--parallel', dest='parallel', type=int, + help='Control the number of parallel build processes or threads' + ) '''Agent only''' self.parser.add_argument( @@ -422,6 +428,7 @@ def __init__(self, script_args): self.static_analysis = self.script_args.static_analysis self.build_for_snap = self.script_args.build_for_snap self.search_prefix = self.script_args.search_prefix + self.parallel = self.script_args.parallel @property def platform(self): @@ -462,6 +469,13 @@ def run(self): def package(self): subprocess.call(['/bin/bash', '-c', 'cd {} && cpack .'.format(self.build_path)]) + @property + def build_options(self): + build_options_linux = super().build_options + if (self.parallel): + build_options_linux += ['--parallel', str(self.parallel)] + return build_options_linux + class WindowsBuildRunner(BuildRunnerBase): """Windows BuildRunner class.""" diff --git a/sdk-cpp/tests/test_data.cpp b/sdk-cpp/tests/test_data.cpp index c0fbc30e..3ecbd1ef 100644 --- a/sdk-cpp/tests/test_data.cpp +++ b/sdk-cpp/tests/test_data.cpp @@ -38,7 +38,7 @@ const std::chrono::seconds g_largeFileWaitTime = 5min; #if defined(DO_INTERFACE_REST) #ifdef DO_BUILD_FOR_SNAP -const std::string g_docsSvcName = "deliveryoptimization-client.agent"; +const std::string g_docsSvcName = "deliveryoptimization-agent.agent"; #else const std::string g_docsSvcName = "deliveryoptimization-agent.service"; #endif diff --git a/snapcraft-options/connect-snaps.sh b/snapcraft-options/connect-snaps.sh index 3ba11251..a5159909 100755 --- a/snapcraft-options/connect-snaps.sh +++ b/snapcraft-options/connect-snaps.sh @@ -8,8 +8,8 @@ set -e # Plug agent snap into sdk-tests snap downloads slot -sudo snap connect deliveryoptimization-client:deviceupdate-agent-downloads deliveryoptimization-sdk-tests:downloads-folder +sudo snap connect deliveryoptimization-agent:deviceupdate-agent-downloads deliveryoptimization-sdk-tests:downloads-folder # Plug sdk-tests snap into agent's run and config slots -sudo snap connect deliveryoptimization-sdk-tests:do-port-numbers deliveryoptimization-client:do-port-numbers -sudo snap connect deliveryoptimization-sdk-tests:do-configs deliveryoptimization-client:do-configs +sudo snap connect deliveryoptimization-sdk-tests:do-port-numbers deliveryoptimization-agent:do-port-numbers +sudo snap connect deliveryoptimization-sdk-tests:do-configs deliveryoptimization-agent:do-configs diff --git a/snapcraft-options/snapcraft-agent.yaml b/snapcraft-options/snapcraft-agent.yaml index a1ae6fb9..547b53ef 100644 --- a/snapcraft-options/snapcraft-agent.yaml +++ b/snapcraft-options/snapcraft-agent.yaml @@ -8,10 +8,6 @@ description: | grade: devel # must be 'stable' to release into candidate/stable channels confinement: strict # use 'strict' once you have the right plugs and slots -architectures: - - build-on: [amd64, arm64] - run-on: [amd64, armhf, arm64] - ##### # # Keywords @@ -39,7 +35,11 @@ parts: plugin: python source: . override-build: | - python3 ./build/build.py --project agent --build-for-snap + if grep -q "Raspberry Pi" /proc/cpuinfo; then + python3 ./build/build.py --project agent --build-for-snap --parallel 1 + else + python3 ./build/build.py --project agent --build-for-snap + fi mkdir -p ../install/bin cp /tmp/build-deliveryoptimization-agent/linux-debug/client-lite/deliveryoptimization-agent ../install/bin/deliveryoptimization-agent diff --git a/snapcraft-options/snapcraft-sdk.yaml b/snapcraft-options/snapcraft-sdk.yaml index 26b78eae..6ebc7249 100644 --- a/snapcraft-options/snapcraft-sdk.yaml +++ b/snapcraft-options/snapcraft-sdk.yaml @@ -8,10 +8,6 @@ description: | grade: devel # must be 'stable' to release into candidate/stable channels confinement: strict # use 'strict' once you have the right plugs and slots -architectures: - - build-on: [amd64, arm64] - run-on: [amd64, armhf, arm64] - ##### # # Keywords @@ -39,7 +35,11 @@ parts: plugin: python source: . override-build: | - python3 ./build/build.py --project sdk --build-for-snap + if grep -q "Raspberry Pi" /proc/cpuinfo; then + python3 ./build/build.py --project sdk --build-for-snap --parallel 1 + else + python3 ./build/build.py --project sdk --build-for-snap + fi mkdir -p ../install/bin mkdir -p ../install/lib cp /tmp/build-deliveryoptimization-sdk/linux-debug/sdk-cpp/tests/deliveryoptimization-sdk-tests ../install/bin/deliveryoptimization-sdk-tests From 6459cc59426d38990ed75a6f103460ae148a95db Mon Sep 17 00:00:00 2001 From: andretoyama-msft <106272532+andretoyama-msft@users.noreply.github.com> Date: Wed, 9 Aug 2023 17:52:58 -0300 Subject: [PATCH 22/25] User/andretoyama/add debian11 support (#174) * Add pipeline config and Dockerfiles for Debian11 * test new ACR service connection * add gtest support to debian11 --- .../linux/du/docker/doclient-lite-docker.yml | 22 ++++++++- .../build/linux/du/docker/dopapt-docker.yml | 18 ++++++++ .../build/linux/du/docker/dosdkcpp-docker.yml | 18 ++++++++ .../templates/doclient-lite-docker-steps.yml | 4 +- .../du/templates/dopapt-docker-steps.yml | 4 +- .../du/templates/dosdkcpp-docker-steps.yml | 4 +- azure-pipelines/publishing/github-release.yml | 12 +++++ build/docker/debian11/amd64/Dockerfile | 39 ++++++++++++++++ build/docker/debian11/arm32/Dockerfile | 45 +++++++++++++++++++ build/docker/debian11/arm64/Dockerfile | 44 ++++++++++++++++++ build/scripts/bootstrap.sh | 4 +- 11 files changed, 204 insertions(+), 10 deletions(-) create mode 100644 build/docker/debian11/amd64/Dockerfile create mode 100644 build/docker/debian11/arm32/Dockerfile create mode 100644 build/docker/debian11/arm64/Dockerfile diff --git a/azure-pipelines/build/linux/du/docker/doclient-lite-docker.yml b/azure-pipelines/build/linux/du/docker/doclient-lite-docker.yml index 38db4f67..cc2ce0db 100644 --- a/azure-pipelines/build/linux/du/docker/doclient-lite-docker.yml +++ b/azure-pipelines/build/linux/du/docker/doclient-lite-docker.yml @@ -33,13 +33,31 @@ jobs: - template: ../templates/do-docker-jobs.yml parameters: - targetOsArch: 'debian10_arm64' + targetOsArch: 'debian10_arm64' # azure pipelines does not support '.' in display names imageVersion: ${{variables.containerImageVersion}} stepsTemplate: 'doclient-lite-docker-steps.yml' - template: ../templates/do-docker-jobs.yml parameters: - targetOsArch: 'debian10_x64' + targetOsArch: 'debian10_x64' # azure pipelines does not support '.' in display names + imageVersion: ${{variables.containerImageVersion}} + stepsTemplate: 'doclient-lite-docker-steps.yml' + +- template: ../templates/do-docker-jobs.yml + parameters: + targetOsArch: 'debian11_arm32' # azure pipelines does not support '.' in display names + imageVersion: ${{variables.containerImageVersion}} + stepsTemplate: 'doclient-lite-docker-steps.yml' + +- template: ../templates/do-docker-jobs.yml + parameters: + targetOsArch: 'debian11_arm64' # azure pipelines does not support '.' in display names + imageVersion: ${{variables.containerImageVersion}} + stepsTemplate: 'doclient-lite-docker-steps.yml' + +- template: ../templates/do-docker-jobs.yml + parameters: + targetOsArch: 'debian11_x64' # azure pipelines does not support '.' in display names imageVersion: ${{variables.containerImageVersion}} stepsTemplate: 'doclient-lite-docker-steps.yml' diff --git a/azure-pipelines/build/linux/du/docker/dopapt-docker.yml b/azure-pipelines/build/linux/du/docker/dopapt-docker.yml index 10be67e9..f48bbf05 100644 --- a/azure-pipelines/build/linux/du/docker/dopapt-docker.yml +++ b/azure-pipelines/build/linux/du/docker/dopapt-docker.yml @@ -43,6 +43,24 @@ jobs: imageVersion: ${{variables.containerImageVersion}} stepsTemplate: 'dopapt-docker-steps.yml' +- template: ../templates/do-docker-jobs.yml + parameters: + targetOsArch: 'debian11_arm32' + imageVersion: ${{variables.containerImageVersion}} + stepsTemplate: 'dopapt-docker-steps.yml' + +- template: ../templates/do-docker-jobs.yml + parameters: + targetOsArch: 'debian11_arm64' + imageVersion: ${{variables.containerImageVersion}} + stepsTemplate: 'dopapt-docker-steps.yml' + +- template: ../templates/do-docker-jobs.yml + parameters: + targetOsArch: 'debian11_x64' + imageVersion: ${{variables.containerImageVersion}} + stepsTemplate: 'dopapt-docker-steps.yml' + - template: ../templates/do-docker-jobs.yml parameters: targetOsArch: 'ubuntu1804_arm64' diff --git a/azure-pipelines/build/linux/du/docker/dosdkcpp-docker.yml b/azure-pipelines/build/linux/du/docker/dosdkcpp-docker.yml index a8a2d44b..12e51b25 100644 --- a/azure-pipelines/build/linux/du/docker/dosdkcpp-docker.yml +++ b/azure-pipelines/build/linux/du/docker/dosdkcpp-docker.yml @@ -43,6 +43,24 @@ jobs: imageVersion: ${{variables.containerImageVersion}} stepsTemplate: 'dosdkcpp-docker-steps.yml' +- template: ../templates/do-docker-jobs.yml + parameters: + targetOsArch: 'debian11_arm32' + imageVersion: ${{variables.containerImageVersion}} + stepsTemplate: 'dosdkcpp-docker-steps.yml' + +- template: ../templates/do-docker-jobs.yml + parameters: + targetOsArch: 'debian11_arm64' + imageVersion: ${{variables.containerImageVersion}} + stepsTemplate: 'dosdkcpp-docker-steps.yml' + +- template: ../templates/do-docker-jobs.yml + parameters: + targetOsArch: 'debian11_x64' + imageVersion: ${{variables.containerImageVersion}} + stepsTemplate: 'dosdkcpp-docker-steps.yml' + - template: ../templates/do-docker-jobs.yml parameters: targetOsArch: 'ubuntu1804_arm64' diff --git a/azure-pipelines/build/linux/du/templates/doclient-lite-docker-steps.yml b/azure-pipelines/build/linux/du/templates/doclient-lite-docker-steps.yml index f03bf8c1..dac13467 100644 --- a/azure-pipelines/build/linux/du/templates/doclient-lite-docker-steps.yml +++ b/azure-pipelines/build/linux/du/templates/doclient-lite-docker-steps.yml @@ -14,14 +14,14 @@ steps: displayName: Login to ACR inputs: command: login - containerRegistry: doclientcontainerregistry + containerRegistry: doclient-dockercontainerregistry-buildpipeline # name of the service connection that connect the pipeline to the ACR repository: $(parameters.targetOsArch) - task: Docker@2 displayName: Pull latest build image inputs: command: pull - containerRegistry: doclientcontainerregistry + containerRegistry: doclient-dockercontainerregistry-buildpipeline # name of the service connection that connect the pipeline to the ACR arguments: 'doclientcontainerregistry.azurecr.io/${{parameters.targetOsArch}}:${{parameters.imageVersion}}' - task: CmdLine@2 diff --git a/azure-pipelines/build/linux/du/templates/dopapt-docker-steps.yml b/azure-pipelines/build/linux/du/templates/dopapt-docker-steps.yml index be74d4a3..6be79d12 100644 --- a/azure-pipelines/build/linux/du/templates/dopapt-docker-steps.yml +++ b/azure-pipelines/build/linux/du/templates/dopapt-docker-steps.yml @@ -14,14 +14,14 @@ steps: displayName: Login to ACR inputs: command: login - containerRegistry: doclientcontainerregistry + containerRegistry: doclient-dockercontainerregistry-buildpipeline # name of the service connection that connect the pipeline to the ACR repository: $(parameters.targetOsArch) - task: Docker@2 displayName: Pull latest build image inputs: command: pull - containerRegistry: doclientcontainerregistry + containerRegistry: doclient-dockercontainerregistry-buildpipeline # name of the service connection that connect the pipeline to the ACR arguments: 'doclientcontainerregistry.azurecr.io/${{parameters.targetOsArch}}:${{parameters.imageVersion}}' - task: CmdLine@2 diff --git a/azure-pipelines/build/linux/du/templates/dosdkcpp-docker-steps.yml b/azure-pipelines/build/linux/du/templates/dosdkcpp-docker-steps.yml index 80f4f309..583e426f 100644 --- a/azure-pipelines/build/linux/du/templates/dosdkcpp-docker-steps.yml +++ b/azure-pipelines/build/linux/du/templates/dosdkcpp-docker-steps.yml @@ -14,14 +14,14 @@ steps: displayName: Login to ACR inputs: command: login - containerRegistry: doclientcontainerregistry + containerRegistry: doclient-dockercontainerregistry-buildpipeline # name of the service connection that connect the pipeline to the ACR repository: $(parameters.targetOsArch) - task: Docker@2 displayName: Pull latest build image inputs: command: pull - containerRegistry: doclientcontainerregistry + containerRegistry: doclient-dockercontainerregistry-buildpipeline # name of the service connection that connect the pipeline to the ACR arguments: 'doclientcontainerregistry.azurecr.io/${{parameters.targetOsArch}}:${{parameters.imageVersion}}' - task: CmdLine@2 diff --git a/azure-pipelines/publishing/github-release.yml b/azure-pipelines/publishing/github-release.yml index d065e4eb..65872e5a 100644 --- a/azure-pipelines/publishing/github-release.yml +++ b/azure-pipelines/publishing/github-release.yml @@ -80,6 +80,18 @@ stages: parameters: targetOsArch: 'debian10_x64' imageVersion: ${{variables.containerImageVersion}} + - template: templates/release-docker-build-steps.yml + parameters: + targetOsArch: 'debian11_arm32' + imageVersion: ${{variables.containerImageVersion}} + - template: templates/release-docker-build-steps.yml + parameters: + targetOsArch: 'debian11_arm64' + imageVersion: ${{variables.containerImageVersion}} + - template: templates/release-docker-build-steps.yml + parameters: + targetOsArch: 'debian11_x64' + imageVersion: ${{variables.containerImageVersion}} - stage: release condition: succeeded() diff --git a/build/docker/debian11/amd64/Dockerfile b/build/docker/debian11/amd64/Dockerfile new file mode 100644 index 00000000..b00d97f0 --- /dev/null +++ b/build/docker/debian11/amd64/Dockerfile @@ -0,0 +1,39 @@ +# Dockerfile for building DO client components for Debian 11 amd64 +# First, install the docker extension for VSCode. Then you can right-click on this file +# and choose Build Image. Give it a name and it will build the image. +# +# Alternatively, use the command line: +# Copy the build script to the build directory: +# cp /build/scripts/bootstrap.sh /build/docker/debian11/amd64 +# +# After running the above, you can build the image by running in the current dockerfile directory: +# sudo docker build -t debian11_amd64 . --no-cache --network=host +# +# Open interactive terminal into the image in a container: +# docker run -ti --rm --entrypoint=/bin/bash -v :/code -v :/build +# Example: +# sudo docker run -ti --rm --entrypoint=/bin/bash -v ~/code/do-client/:/code debian11_amd64 + +FROM mcr.microsoft.com/mirror/docker/library/debian:buster@sha256:3b6053ca925336c804e2d3f080af177efcdc9f51198a627569bfc7c7e730ef7e + +SHELL [ "/bin/bash", "-c"] + +COPY bootstrap.sh /tmp/bootstrap.sh + +WORKDIR /tmp/ + +RUN chmod +x bootstrap.sh +RUN ./bootstrap.sh --install build + + +VOLUME /code +WORKDIR /code + +ENTRYPOINT [ "/bin/bash", "-c" ] + +# We specify an empty command so that we can pass options to the ENTRYPOINT command. +# This is a bit of a Dockerfile quirk where if the ENTRYPOINT value is defined, +# then CMD becomes the default options passed to ENTRYPOINT. +# In this case we don't have any desired default arguments. +# However, we have to specify CMD to enable passing of command line parameters to ENTRYPOINT in the first place. +CMD [ ] diff --git a/build/docker/debian11/arm32/Dockerfile b/build/docker/debian11/arm32/Dockerfile new file mode 100644 index 00000000..2772a437 --- /dev/null +++ b/build/docker/debian11/arm32/Dockerfile @@ -0,0 +1,45 @@ +# Dockerfile for building DO client components for Debian 11 arm32 +# First, install the docker extension for VSCode. Then you can right-click on this file +# and choose Build Image. Give it a name and it will build the image. +# +# Open interactive terminal into the image in a container: +# docker run -ti --rm --entrypoint=/bin/bash -v :/code -v :/build +# Example: +# docker run -ti --rm --entrypoint=/bin/bash -v D:\do-client-lite:/code -v D:\temp\build_client_lite\arm-linux-debug:/build custom-debian11-arm32 + +FROM mcr.microsoft.com/mirror/docker/library/debian:buster@sha256:819b11bd0ade30fbc72f4b83593d0d126b2f3329b053495833219213fe37714d + +SHELL [ "/bin/bash", "-c"] + +# QEMU is a Linux emulator which enables cross-arch support in docker +# In order to build this image on a Linux host, need to install QEMU: +# +# sudo apt-get install qemu-user +# update-binfmts --display +# sudo apt install qemu binfmt-support qemu-user-static +# cp /usr/bin/qemu-arm-static /build/docker/debian11/arm32 +# +# Then copy the build script to the build directory +# cp /build/scripts/bootstrap.sh /build/docker/debian11/arm32 +# +# After running the above, you can build the image by running in the current dockerfile directory +# sudo docker build -t . --no-cache --network=host + +COPY qemu-arm-static /usr/bin/qemu-arm-static +COPY bootstrap.sh /tmp/bootstrap.sh + +WORKDIR /tmp/ +RUN chmod +x bootstrap.sh +RUN ./bootstrap.sh --install build + +VOLUME /code +WORKDIR /code + +ENTRYPOINT [ "/bin/bash", "-c" ] + +# We specify an empty command so that we can pass options to the ENTRYPOINT command. +# This is a bit of a Dockerfile quirk where if the ENTRYPOINT value is defined, +# then CMD becomes the default options passed to ENTRYPOINT. +# In this case we don't have any desired default arguments. +# However, we have to specify CMD to enable passing of command line parameters to ENTRYPOINT in the first place. +CMD [ ] diff --git a/build/docker/debian11/arm64/Dockerfile b/build/docker/debian11/arm64/Dockerfile new file mode 100644 index 00000000..2511a64c --- /dev/null +++ b/build/docker/debian11/arm64/Dockerfile @@ -0,0 +1,44 @@ +# Dockerfile for building DO client components for Debian 11 arm64 +# First, install the docker extension for VSCode. Then you can right-click on this file +# and choose Build Image. Give it a name and it will build the image. +# +# Open interactive terminal into the image in a container: +# docker run -ti --rm --entrypoint=/bin/bash -v :/code -v :/build +# Example: +# docker run -ti --rm --entrypoint=/bin/bash -v D:\do-client-lite:/code -v D:\temp\build_client_lite\arm-linux-debug:/build custom-debian11-arm64 + +FROM mcr.microsoft.com/mirror/docker/library/debian:buster@sha256:de3c0d12dd75f1a47595ff0ce78f2d30d6ca95c3ad66af06c8815d1f9b8e208d + +SHELL [ "/bin/bash", "-c"] + +# QEMU is a Linux emulator which enables cross-arch support in docker +# In order to build this image on a Linux host, need to install QEMU: +# +# sudo apt-get install qemu-user +# update-binfmts --display +# sudo apt install qemu binfmt-support qemu-user-static +# cp /usr/bin/qemu-arm-static /build/docker/debian11/arm64 +# +# Then copy the build script to the build directory +# cp /build/scripts/bootstrap.sh /build/docker/debian11/arm64 +# After running the above, you can build the image by running in the current dockerfile directory +# sudo docker build -t debian11_arm64 . --no-cache --network=host + +COPY qemu-arm-static /usr/bin/qemu-arm-static +COPY bootstrap.sh /tmp/bootstrap.sh + +WORKDIR /tmp/ +RUN chmod +x bootstrap.sh +RUN ./bootstrap.sh --install build + +VOLUME /code +WORKDIR /code + +ENTRYPOINT [ "/bin/bash", "-c" ] + +# We specify an empty command so that we can pass options to the ENTRYPOINT command. +# This is a bit of a Dockerfile quirk where if the ENTRYPOINT value is defined, +# then CMD becomes the default options passed to ENTRYPOINT. +# In this case we don't have any desired default arguments. +# However, we have to specify CMD to enable passing of command line parameters to ENTRYPOINT in the first place. +CMD [ ] \ No newline at end of file diff --git a/build/scripts/bootstrap.sh b/build/scripts/bootstrap.sh index f9453bd5..c86e5ba9 100755 --- a/build/scripts/bootstrap.sh +++ b/build/scripts/bootstrap.sh @@ -78,11 +78,11 @@ function installBuildDependencies mkdir /tmp/gtest pushd /tmp/gtest - if [[ ($DISTRO == "ubuntu" && ($VER == "20.04" || $VER == "22.04")) || ($DISTRO == "debian" && $VER == "10") ]]; + if [[ ($DISTRO == "ubuntu" && ($VER == "20.04" || $VER == "22.04")) || ($DISTRO == "debian" && ($VER == "10" || $VER == "11")) ]]; then if [[ $VER == "22.04" ]]; then release="v1.13.0"; else release="release-1.10.0"; fi; - # The latest native-version of gtest on ubuntu2004 and debian10 currently has a bug where + # The latest native-version of gtest on the latest versions of ubuntu and debian currently has a bug where # CMakeLists doesn't declare an install target, causing 'make install' to fail. # Clone from source and use release-1.10.0 instead, since gtest is a source package anyways. git clone https://github.com/google/googletest.git . From 5777cc43ccde4b6838063999f88613edbcf3ecf0 Mon Sep 17 00:00:00 2001 From: Shishir Bhat Date: Thu, 7 Sep 2023 13:18:24 -0700 Subject: [PATCH 23/25] Increment Agent, SDK version to 1.1.0 --- client-lite/CMakeLists.txt | 2 +- sdk-cpp/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client-lite/CMakeLists.txt b/client-lite/CMakeLists.txt index ee6bca08..a34b1743 100644 --- a/client-lite/CMakeLists.txt +++ b/client-lite/CMakeLists.txt @@ -5,7 +5,7 @@ if (NOT DOSVC_BIN_NAME) message (FATAL_ERROR "Agent daemon name not defined") endif () -project (${DOSVC_BIN_NAME} VERSION 1.0.0) +project (${DOSVC_BIN_NAME} VERSION 1.1.0) option (DO_PROXY_SUPPORT "Set DO_PROXY_SUPPORT to OFF to turn off proxy support for downloads and thus remove dependency on libproxy." ON) diff --git a/sdk-cpp/CMakeLists.txt b/sdk-cpp/CMakeLists.txt index e58850c5..6160d615 100644 --- a/sdk-cpp/CMakeLists.txt +++ b/sdk-cpp/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -project (deliveryoptimization_sdk VERSION 1.0.0) +project (deliveryoptimization_sdk VERSION 1.1.0) if (DO_PLATFORM_LINUX OR DO_PLATFORM_MAC) find_package(Boost REQUIRED) From fcd73a6a4ea4d7c7674f4e1f2adeaf6ed89db4a2 Mon Sep 17 00:00:00 2001 From: shishirb-MSFT <50385517+shishirb-MSFT@users.noreply.github.com> Date: Thu, 7 Sep 2023 13:56:06 -0700 Subject: [PATCH 24/25] Update test file URLs (#176) * There was a change to test files hosting location recently. --- client-lite/test/test_data.cpp | 6 +++--- sdk-cpp/tests/test_data.cpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/client-lite/test/test_data.cpp b/client-lite/test/test_data.cpp index a02f59ee..d440ba9f 100644 --- a/client-lite/test/test_data.cpp +++ b/client-lite/test/test_data.cpp @@ -5,10 +5,10 @@ const uint64_t g_prodFileSizeBytes = 25006511u; -const std::string g_smallFileUrl = "http://main.oremdl.microsoft.com.nsatc.net/dotc/49c591d405d307e25e72a19f7e79b53d69f19954/43A54FC03C6A979E9AAEAE2493757D1429A5C8A8D093FB7B8103E8CC8DF7B6B6"; +const std::string g_smallFileUrl = "http://download.windowsupdate.com/phf/dotc/49c591d405d307e25e72a19f7e79b53d69f19954/43A54FC03C6A979E9AAEAE2493757D1429A5C8A8D093FB7B8103E8CC8DF7B6B6"; const std::string g_smallFile2Url = "http://extorigin-int.dcat.dsp.mp.microsoft.com/filestreamingservice/files/81701079-fa80-48b4-825a-27af227e4192"; -const std::string g_largeFileUrl = "http://main.oremdl.microsoft.com.nsatc.net/dotc/ReplacementDCATFile.txt"; -const std::string g_404Url = "http://main.oremdl.microsoft.com.nsatc.net/dotc/49c591d405d307e25e72a19f7e79b53d69f19954/nonexistent"; +const std::string g_largeFileUrl = "http://download.windowsupdate.com/phf/dotc/ReplacementDCATFile.txt"; +const std::string g_404Url = "http://download.windowsupdate.com/phf/dotc/49c591d405d307e25e72a19f7e79b53d69f19954/nonexistent"; const std::string g_prodFileUrl = "http://dl.delivery.mp.microsoft.com/filestreamingservice/files/52fa8751-747d-479d-8f22-e32730cc0eb1"; // This MCC instance only works within our test lab azure VMs. Can be overriden via cmdline. diff --git a/sdk-cpp/tests/test_data.cpp b/sdk-cpp/tests/test_data.cpp index 3ecbd1ef..f243b744 100644 --- a/sdk-cpp/tests/test_data.cpp +++ b/sdk-cpp/tests/test_data.cpp @@ -17,18 +17,18 @@ static const std::string downloadPath = "/var/lib/deviceupdate-agent-downloads"; static const std::string downloadPath = fs::temp_directory_path().string(); #endif -const std::string g_largeFileUrl = "http://main.oremdl.microsoft.com.nsatc.net/dotc/ReplacementDCATFile.txt"; +const std::string g_largeFileUrl = "http://download.windowsupdate.com/phf/dotc/ReplacementDCATFile.txt"; const std::string g_malformedFilePath = "?f309adfasdf///dfasdfj39fjasldfjasdf/// ///.1"; const std::string g_tmpFileName = (downloadPath / fs::path("docsdk_testfile.txt")).string(); const std::string g_tmpFileName2 = (downloadPath / fs::path("docsdk_testfile2.txt")).string(); const std::string g_tmpFileName3 = (downloadPath / fs::path("docsdk_testfile3.txt")).string(); -const std::string g_smallFileUrl = "http://main.oremdl.microsoft.com.nsatc.net/dotc/49c591d405d307e25e72a19f7e79b53d69f19954/43A54FC03C6A979E9AAEAE2493757D1429A5C8A8D093FB7B8103E8CC8DF7B6B6"; +const std::string g_smallFileUrl = "http://download.windowsupdate.com/phf/dotc/49c591d405d307e25e72a19f7e79b53d69f19954/43A54FC03C6A979E9AAEAE2493757D1429A5C8A8D093FB7B8103E8CC8DF7B6B6"; const std::string g_smallFilePhfInfoJson = R"( { "PiecesHashFileUrl":"https://storage4do.blob.core.windows.net/testdata/SmallFile.phf", "HashOfHashes":"Q6VPwDxql56arq4kk3V9FCmlyKjQk/t7gQPozI33trY=" })"; -const std::string g_404Url = "http://main.oremdl.microsoft.com.nsatc.net/dotc/49c591d405d307e25e72a19f7e79b53d69f19954/nonexistent"; +const std::string g_404Url = "http://download.windowsupdate.com/phf/dotc/49c591d405d307e25e72a19f7e79b53d69f19954/nonexistent"; const std::string g_prodFileUrl = "http://dl.delivery.mp.microsoft.com/filestreamingservice/files/52fa8751-747d-479d-8f22-e32730cc0eb1"; const std::chrono::seconds g_smallFileWaitTime = 10s; From 5c090eeadd0a315d434749e6ea7bfd0ef057e4b2 Mon Sep 17 00:00:00 2001 From: shishirb-MSFT <50385517+shishirb-MSFT@users.noreply.github.com> Date: Thu, 7 Sep 2023 14:36:31 -0700 Subject: [PATCH 25/25] Increment Agent snap version also to 1.1.0 (#177) --- snapcraft-options/snapcraft-agent.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapcraft-options/snapcraft-agent.yaml b/snapcraft-options/snapcraft-agent.yaml index 547b53ef..7c39402c 100644 --- a/snapcraft-options/snapcraft-agent.yaml +++ b/snapcraft-options/snapcraft-agent.yaml @@ -1,6 +1,6 @@ name: deliveryoptimization-agent base: core20 # the base snap is the execution environment for this snap -version: '1.0.1' +version: '1.1.0' summary: Ubuntu Core 20.04 DO Agent description: | A proof-of-concept for the Delivery Optimization Client Ubuntu Core snap.