Merge branch develop into main [2022_10_28]

.gitignore

@@ -55,3 +55,4 @@ CTestTestfile.cmake
 _deps
 *_CPack_Packages/
 *.deb
+out/

CMakeLists.txt

@@ -71,13 +71,31 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 if (DO_PLATFORM_LINUX)
     # PIE (Position Independent Executable) ensures exe/.so can run when ASLR is enabled in the target OS
     set(COMPILER_HARDENING_FLAGS
-        "-fPIE -pie -D_FORTIFY_SOURCE=2 -fstack-protector-strong -Wformat -Werror=format-security")
+        "-fPIE -pie -D_FORTIFY_SOURCE=2 -fstack-protector-strong -Wformat -Werror=format-security -Wreturn-local-addr")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall ${COMPILER_HARDENING_FLAGS} -fmerge-all-constants")
 
     # relro+now thwarts some attack vectors by reordering some ELF data structures and also by making the GOT read-only
     set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pie -z relro -z now")
 endif (DO_PLATFORM_LINUX)
 
+if (DO_PLATFORM_WINDOWS AND DO_INCLUDE_SDK)
+    # Replicate some of MSEdge build config to catch issues before integration.
+    # Use Visual Studio 2019 and the x64-Clang-Debug-SDK cmake config to enable these configs.
+    if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+        set(DISABLE_WARNINGS "-Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-reserved-id-macro -Wno-global-constructors -Wno-exit-time-destructors -Wno-extra-semi-stmt")
+        # Note: Many warnings to fix before we can turn on -Werror
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wreturn-stack-address -Wno-string-conversion ${DISABLE_WARNINGS}")
+    endif ()
+    if (NOT DO_BUILD_TESTS)
+        # Disable C++ exceptions
+        string(REGEX REPLACE "/EH[a-z]+" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /D_HAS_EXCEPTIONS=0")
+        # Disable RTTI
+        string(REGEX REPLACE "/GR" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-")
+    endif ()
+endif ()
+
 # Define DEBUG macro in debug builds
 string(TOLOWER ${CMAKE_BUILD_TYPE} DO_BUILD_TYPE)
 if (DO_BUILD_TYPE MATCHES debug)

CMakeSettings.json

@@ -0,0 +1,38 @@
+{
+  "configurations": [
+    {
+      "name": "x64-Debug",
+      "generator": "Ninja",
+      "configurationType": "Debug",
+      "inheritEnvironments": [ "msvc_x64_x64" ],
+      "buildRoot": "${projectDir}\\out\\build\\${name}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": ""
+    },
+    {
+      "name": "x64-Clang-Debug-SDK",
+      "generator": "Ninja",
+      "configurationType": "Debug",
+      "buildRoot": "${projectDir}\\out\\build\\${name}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "inheritEnvironments": [ "clang_cl_x64_x64" ],
+      "variables": [
+        {
+          "name": "DO_INCLUDE_SDK",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "DO_BUILD_TESTS",
+          "value": "False",
+          "type": "BOOL"
+        }
+      ]
+    }
+  ]
+}

README.md

@@ -18,7 +18,8 @@ through native C++ code.
 ## Plug-ins
 
 Add-on that enables APT downloads to go through Delivery Optimization Agent.
-It is requires the SDK and Agent components.
+It is a required component only on devices that must download APT packages via a Microsoft Connected Cache instance.
+During install, it replaces itself as APT's HTTP(S) transport mechanism, thus receiving all APT downloads requests.
 
 ## Getting Started
 
@@ -39,8 +40,8 @@ Run the appropriate bootstrapper depending on development machine platform:
 > cd build/bootstrap
 ```
 
-### Building DO client components 
-**NOTICE:**  
+### Building DO client components
+**NOTICE:**
 **If you are modifying this project and distributing your own custom build, please modify the DO_BUILDER_IDENTIFIER cmake variable located in https://github.com/microsoft/do-client/blob/main/CMakeLists.txt**
 
 After setting up your development machine, navigate back into the project root
@@ -49,7 +50,7 @@ After setting up your development machine, navigate back into the project root
 > cd <project root>
 ```
 
-We provide an easy-to-use python script for building our client components from the project root, you can inspect build.py for additional build flags  
+We provide an easy-to-use python script for building our client components from the project root, you can inspect build.py for additional build flags
 On debian-based systems, run this command to build the client and package it as a .deb file
 
 ```markdown
@@ -71,7 +72,7 @@ Navigate to the build output directory for the agent and install the agent packa
 > sudo apt-get install ./deliveryoptimization-agent*.deb
 ```
 
-The sdk produces a runtime and development package, in this case you'll want to install both  
+The sdk produces a runtime and development package, in this case you'll want to install both
 Navigate to build output directory for the sdk and install both packages
 
 ```markdown
@@ -92,7 +93,7 @@ At this point, you should have built and packaged all components
 
 There are a couple ways for you to install the DO client components
 
-1. If you have built the component into a debian package, you can simply find the debian package and install like detailed above.  
+1. If you have built the component into a debian package, you can simply find the debian package and install like detailed above.
 This will handle installing to the appropriate paths, and also the necessary setup of DO user/group permissions needed for DO-agent.
 
 ```markdown
@@ -104,19 +105,19 @@ This will handle installing to the appropriate paths, and also the necessary set
 > sudo apt get install ./deliveryoptimization-plugin-apt*.deb
 ```
 
-2. If you build and install using cmake, or through some other custom means, be sure to setup the DO user/groups correctly in your installation.  
+2. If you build and install using cmake, or through some other custom means, be sure to setup the DO user/groups correctly in your installation.
 You can reference this [script](https://github.com/microsoft/do-client/blob/main/client-lite/build/postinst.in.sh) to see how to setup the DO user/group and install DO as a daemon.
 
 ### Testing DO Client components
 
-As guidance, please ensure proper code coverage for project contributions  
+As guidance, please ensure proper code coverage for project contributions
 Unit tests for the agent and sdk are produced as a part of the above build command, you can find them in the build output directory
 
 ```markdown
 > cd /tmp/build-deliveryoptimization-agent/linux-debug/client-lite/test
 ```
 
-Our tests utilize the [GTest](https://github.com/google/googletest) unit testing framework, which supports test filtering via command line  
+Our tests utilize the [GTest](https://github.com/google/googletest) unit testing framework, which supports test filtering via command line
 You can run all agent tests by running
 
 ```markdown
@@ -134,7 +135,7 @@ The test executable for the SDK is located in the sdk build output as well
 > cd /tmp/build-deliveryoptimization-sdk/linux-debug/sdk-cpp/tests
 ```
 
-The sdk tests expect a running do-agent, you can either manually run the agent executable from its build output or install the agent package as you may have done while building the plugin  
+The sdk tests expect a running do-agent, you can either manually run the agent executable from its build output or install the agent package as you may have done while building the plugin
 You can run the sdk tests just like the agent tests
 
 ```markdown
@@ -149,10 +150,10 @@ And filter them similarly
 
 ## Support
 
-This repository is currently in a **Public Preview** state.  During this phase, all DO components
-found in this repo will be supported for 90 days beyond the release date of a new release.  At
-the end of the 90 day window, we will not guarantee support for the previous version.  Please plan
-to migrate to the new DO components within that 90-day window to avoid any disruptions.
+The APT plugin component is currently in a **Public Preview** state.  During this phase, it will be
+supported for 90 days beyond the release date of a new release.  At the end of the 90 day window,
+we will not guarantee support for the previous version.  Please plan to migrate to a newer release
+within that 90-day window to avoid any disruptions.
 
 ## Filing a Bug
 
@@ -175,12 +176,6 @@ tracked appropriately.
 | x86-64 | [![Build Status](https://deliveryoptimization.visualstudio.com/client/_apis/build/status/DO%20Simple%20Client%20ARM%20Build?branchName=main)](https://deliveryoptimization.visualstudio.com/client/_build/latest?definitionId=25&branchName=main) | [![Build Status](https://deliveryoptimization.visualstudio.com/client/_apis/build/status/DO%20CPP-SDK%20ARM%20Build?branchName=main)](https://deliveryoptimization.visualstudio.com/client/_build/latest?definitionId=33&branchName=main) | [![Build Status](https://deliveryoptimization.visualstudio.com/client/_apis/build/status/DO%20Plugins%20ARM%20Build?branchName=main)](https://deliveryoptimization.visualstudio.com/client/_build/latest?definitionId=31&branchName=main) |
 | arm64 | [![Build Status](https://deliveryoptimization.visualstudio.com/client/_apis/build/status/DO%20Simple%20Client%20ARM%20Build?branchName=main)](https://deliveryoptimization.visualstudio.com/client/_build/latest?definitionId=25&branchName=main) | [![Build Status](https://deliveryoptimization.visualstudio.com/client/_apis/build/status/DO%20CPP-SDK%20ARM%20Build?branchName=main)](https://deliveryoptimization.visualstudio.com/client/_build/latest?definitionId=33&branchName=main) | [![Build Status](https://deliveryoptimization.visualstudio.com/client/_apis/build/status/DO%20Plugins%20ARM%20Build?branchName=main)](https://deliveryoptimization.visualstudio.com/client/_build/latest?definitionId=31&branchName=main) |
 
-#### Debian 9
-
-| Architecture | Agent | SDK | Plugin |
-|-----|--------|-----|--------|
-| arm32 | [![Build Status](https://deliveryoptimization.visualstudio.com/client/_apis/build/status/DO%20Simple%20Client%20ARM%20Build?branchName=main)](https://deliveryoptimization.visualstudio.com/client/_build/latest?definitionId=25&branchName=main) | [![Build Status](https://deliveryoptimization.visualstudio.com/client/_apis/build/status/DO%20CPP-SDK%20ARM%20Build?branchName=main)](https://deliveryoptimization.visualstudio.com/client/_build/latest?definitionId=33&branchName=main) | [![Build Status](https://deliveryoptimization.visualstudio.com/client/_apis/build/status/DO%20Plugins%20ARM%20Build?branchName=main)](https://deliveryoptimization.visualstudio.com/client/_build/latest?definitionId=31&branchName=main) |
-
 #### Debian 10
 
 | Architecture | Agent | SDK | Plugin |

azure-pipelines/build/linux/du/docker/doclient-lite-docker.yml

@@ -25,12 +25,6 @@ pool:
   demands: ImageOverride -equals do-adu-build-$(imageVersion)
 
 jobs:
-- template: ../templates/do-docker-jobs.yml
-  parameters:
-    targetOsArch: 'debian9_arm32'
-    imageVersion: '0.8.0' #hard coded version because debian9 support will soon be removed
-    stepsTemplate: 'doclient-lite-docker-steps.yml'
-
 - template: ../templates/do-docker-jobs.yml
   parameters:
     targetOsArch: 'debian10_arm32'

azure-pipelines/build/linux/du/docker/dopapt-docker.yml

@@ -25,12 +25,6 @@ pool:
   demands: ImageOverride -equals do-adu-build-$(imageVersion)
 
 jobs:
-- template: ../templates/do-docker-jobs.yml
-  parameters:
-    targetOsArch: 'debian9_arm32'
-    imageVersion: '0.8.0' #hard coded version because debian9 support will soon be removed
-    stepsTemplate: 'dopapt-docker-steps.yml'
-
 - template: ../templates/do-docker-jobs.yml
   parameters:
     targetOsArch: 'debian10_arm32'

azure-pipelines/build/linux/du/docker/dosdkcpp-docker.yml

@@ -43,12 +43,6 @@ jobs:
     imageVersion: ${{variables.containerImageVersion}}
     stepsTemplate: 'dosdkcpp-docker-steps.yml'
 
-- template: ../templates/do-docker-jobs.yml
-  parameters:
-    targetOsArch: 'debian9_arm32'
-    imageVersion: ${{variables.containerImageVersion}}
-    stepsTemplate: 'dosdkcpp-docker-steps.yml'
-
 - template: ../templates/do-docker-jobs.yml
   parameters:
     targetOsArch: 'ubuntu1804_arm64'

azure-pipelines/build/linux/du/templates/do-docker-jobs.yml

@@ -2,7 +2,7 @@
 # Consume this jobs template in a pipeline yaml by passing in parameter values.
 
 parameters:
-- name: targetOsArch    # example: debian9_arm32
+- name: targetOsArch    # example: ubuntu2004_arm64
   type: string
 - name: imageVersion
   type: string

azure-pipelines/build/linux/du/templates/doclient-lite-docker-steps.yml

@@ -2,7 +2,7 @@
 # Consume this steps template in one or more jobs by passing in parameter values.
 
 parameters:
-- name: targetOsArch    # example: debian9_arm32
+- name: targetOsArch    # example: ubuntu2004_arm64
   type: string
 - name: imageVersion
   type: string

azure-pipelines/build/linux/du/templates/dopapt-docker-steps.yml

@@ -2,7 +2,7 @@
 # Consume this steps template in one or more jobs by passing in parameter values.
 
 parameters:
-- name: targetOsArch    # example: debian9_arm32
+- name: targetOsArch    # example: ubuntu2004_arm64
   type: string
 - name: imageVersion
   type: string

azure-pipelines/build/linux/du/templates/dosdkcpp-docker-steps.yml

@@ -2,7 +2,7 @@
 # Consume this steps template in one or more jobs by passing in parameter values.
 
 parameters:
-- name: targetOsArch    # example: debian9_arm32
+- name: targetOsArch    # example: ubuntu2004_arm64
   type: string
 - name: imageVersion
   type: string

azure-pipelines/build/windows/templates/dosdkcpp-steps.yml

@@ -8,7 +8,7 @@ parameters:
   type: string
 - name: vcpkgDir
   type: string
-  default: $(Agent.TempDirectory)\deliveryoptimization_tools
+  default: $(Agent.TempDirectory)\deliveryoptimization_tools\vcpkg
 - name: dependencyScriptsLocation
   type: string 
   default: $(Build.SourcesDirectory)/build/scripts
@@ -17,27 +17,47 @@ parameters:
   default: false
 
 steps:
+- task: PowerShell@2
+  displayName: 'Find or install vcpkg'
+  inputs:
+    targetType: 'inline'
+    script: |
+      if (Test-Path "$env:VCPKG_ROOT\vcpkg.exe")
+      {
+          # Must be a version of pipeline agent that has vcpkg pre-installed
+          Write-Host "Found existing vcpkg in $env:VCPKG_ROOT"
+      }
+      else
+      {
+          $vcpkgInstallDir = ${{parameters.vcpkgDir}}
+          Write-Host "Installing vcpkg to $vcpkgInstallDir"
+          git clone https://github.com/microsoft/vcpkg $vcpkgInstallDir
+          # Build pipeline agents usually have both these defined
+          Write-Host "##vso[task.setvariable variable=VCPKG_ROOT;isoutput=true;isreadonly=true;]$vcpkgInstallDir"
+          Write-Host "##vso[task.setvariable variable=VCPKG_INSTALLATION_ROOT;isoutput=true;isreadonly=true;]$vcpkgInstallDir"
+      }
+
 # Key input is hashed to compute the cache key so when the contents of install-vcpkg-deps.ps1 changes, a new key will be generated
 # This way, anytime we modify dependencies from install-vcpkg-deps.ps1 script, we don't retrieve from the old cache
 - task: Cache@2
   inputs:
     key: '${{parameters.dependencyScriptsLocation}}/install-vcpkg-deps.ps1 | "${{parameters.targetOsArch}}"'
-    path: ${{parameters.vcpkgDir}}
+    path: $(VCPKG_ROOT)
     cacheHitVar: CACHE_RESTORED
 
 - task: PowerShell@2
   condition: and(succeeded(), ne(variables.CACHE_RESTORED, 'true'))
   inputs:
     targetType: 'filePath'
     filePath: ${{parameters.dependencyScriptsLocation}}\install-vcpkg-deps.ps1
-    arguments: ${{parameters.vcpkgDir}}
+    arguments: $(VCPKG_ROOT)
   displayName: 'Install vcpkg dependencies'
 
 - task: PythonScript@0
   inputs:
     scriptSource: 'filePath'
     scriptPath: 'build/build.py'
-    arguments: '--project sdk --config ${{parameters.config}} --vcpkgdir ${{parameters.vcpkgDir}}\vcpkg --clean'
+    arguments: '--project sdk --config ${{parameters.config}} --vcpkgdir $(VCPKG_ROOT) --clean'
   displayName: 'Build sdk-cpp ${{parameters.targetOsArch}}-${{parameters.config}}'
 
 - task: CmdLine@2

azure-pipelines/publishing/github-release.yml

@@ -60,10 +60,6 @@ stages:
     parameters:
       targetOsArch: 'ubuntu2004_arm64'
       imageVersion: ${{variables.containerImageVersion}}
-  - template: templates/release-docker-build-steps.yml
-    parameters:
-      targetOsArch: 'debian9_arm32'
-      imageVersion: '0.8.0' #hard coded version because debian9 support will soon be removed
   - template: templates/release-docker-build-steps.yml
     parameters:
       targetOsArch: 'debian10_arm32'

azure-pipelines/publishing/templates/release-docker-build-steps.yml

@@ -2,7 +2,7 @@
 # Consume this steps template in one or more jobs by passing in parameter values.
 
 parameters:
-- name: targetOsArch        # example: debian9_arm32
+- name: targetOsArch        # example: ubuntu2004_arm64
   type: string
 - name: imageVersion
   type: string
@@ -60,14 +60,14 @@ jobs:
     displayName: 'Copy plugin .deb file'
 
   - task: DeleteFiles@1
-    inputs: 
+    inputs:
       SourceFolder: $(Build.ArtifactStagingDirectory)
       Contents: |
         **/*
     displayName: 'Clean build folder before creating tar file folder'
 
   - task: ArchiveFiles@2
-    inputs: 
+    inputs:
       rootFolderOrFile: /tmp/${{parameters.targetOsArch}}
       includeRootFolder: False
       archiveType: tar

azure-pipelines/publishing/templates/release-native-build-steps.yml

@@ -1,5 +1,5 @@
 parameters:
-- name: targetOsArch        # example: debian9_arm32
+- name: targetOsArch        # example: ubuntu2004_arm64
   type: string
 
 jobs:
@@ -53,14 +53,14 @@ jobs:
     displayName: 'Copy plugin .deb file'
 
   - task: DeleteFiles@1
-    inputs: 
+    inputs:
       SourceFolder: $(Build.ArtifactStagingDirectory)
       Contents: |
         **/*
     displayName: 'Clean build folder before creating tar file folder'
 
   - task: ArchiveFiles@2
-    inputs: 
+    inputs:
       rootFolderOrFile: /tmp/${{parameters.targetOsArch}}
       includeRootFolder: False
       archiveType: tar