User/jimson/test hosted pool

azure-pipelines/build/doclient-lite-docker.yml

@@ -14,26 +14,28 @@ schedules:
     include:
     - main
 
-pool: dotestlab
+# TODO(jimson): Provision a 1ES managed image with artifacts that have all our dependencies preinstalled
+#pool: 1es_hosted_pool_ubuntu_1804
+
+# The microsoft-hosted agents come with docker preinstalled, the caveat being that the bootstrap script needs to be called to provision everything else
+pool:
+  vmImage: 'ubuntu-18.04'
 
 jobs:
 - template: templates/do-docker-jobs.yml
   parameters:
-    targetOS: 'debian10'
+    targetOS: 'debian9'
     targetArch: 'arm32'
-    dockerImageName: 'jimsonmsft/debian10-arm32:latest'
     stepsTemplate: 'doclient-lite-docker-steps.yml'
 
 - template: templates/do-docker-jobs.yml
   parameters:
-    targetOS: 'debian9'
+    targetOS: 'debian10'
     targetArch: 'arm32'
-    dockerImageName: 'jimsonmsft/debian9-arm32:latest'
     stepsTemplate: 'doclient-lite-docker-steps.yml'
 
 - template: templates/do-docker-jobs.yml
   parameters:
-    targetOS: 'ubuntu1804'
+    targetOS: 'ubuntu1804' # azure pipelines does not support '.' in display names
     targetArch: 'arm64'
-    dockerImageName: 'jimsonmsft/ubuntu18.04-arm64:latest'
     stepsTemplate: 'doclient-lite-docker-steps.yml'

azure-pipelines/build/doclient-lite-native.yml

@@ -37,20 +37,24 @@ pr:
     exclude:
       - azure-pipelines/*
 
-pool: dotestlab
+# TODO(jimson): This is a base Ubuntu 1804 Image right now, switch to an image that contains all our dependencies preinstalled
+# The advantage to using this over the Microsoft-hosted agent is parallel execution of jobs (infinite # of vms can be spun up at execution time)
+pool: 1es_hosted_pool_ubuntu_1804
 
 jobs:
 - job: Debug
   steps:
   - template: templates/doclient-lite-native-steps.yml
     parameters:
+      targetOS: 'ubuntu1804'
       targetOsArch: 'ubuntu1804-x64'
       config: debug
 
 - job: Release
   steps:
   - template: templates/doclient-lite-native-steps.yml
     parameters:
+      targetOS: 'ubuntu1804'
       targetOsArch: 'ubuntu1804-x64'
       config: minsizerel
       skipTests: true

azure-pipelines/build/dopapt-docker.yml

@@ -14,26 +14,28 @@ schedules:
     include:
     - main
 
-pool: dotestlab
+# TODO(jimson): Provision a 1ES managed image with artifacts that have all our dependencies preinstalled
+#pool: 1es_hosted_pool_ubuntu_1804
+
+# The microsoft-hosted agents come with docker preinstalled, the caveat being that the bootstrap script needs to be called to provision everything else
+pool:
+  vmImage: 'ubuntu-18.04'
 
 jobs:
 - template: templates/do-docker-jobs.yml
   parameters:
     targetOS: 'debian10'
     targetArch: 'arm32'
-    dockerImageName: 'jimsonmsft/debian10-arm32:latest'
     stepsTemplate: 'dopapt-docker-steps.yml'
 
 - template: templates/do-docker-jobs.yml
   parameters:
     targetOS: 'debian9'
     targetArch: 'arm32'
-    dockerImageName: 'jimsonmsft/debian9-arm32:latest'
     stepsTemplate: 'dopapt-docker-steps.yml'
 
 - template: templates/do-docker-jobs.yml
   parameters:
     targetOS: 'ubuntu1804'
     targetArch: 'arm64'
-    dockerImageName: 'jimsonmsft/ubuntu18.04-arm64:latest'
     stepsTemplate: 'dopapt-docker-steps.yml'

azure-pipelines/build/dopapt-native.yml

@@ -39,19 +39,23 @@ pr:
       - 'azure-pipelines/*'
       - 'plugins/linux-apt/scripts/configure-apt-method.sh'
 
-pool: dotestlab
+# TODO(jimson): This is a base Ubuntu 1804 Image right now, switch to an image that contains all our dependencies preinstalled
+# The advantage to using this over the Microsoft-hosted agent is parallel execution of jobs (infinite # of vms can be spun up at execution time)
+pool: 1es_hosted_pool_ubuntu_1804
 
 jobs:
 - job: Debug
   steps:
   - template: templates/dopapt-native-steps.yml
     parameters:
+      targetOs: 'ubuntu1804'
       targetOsArch: 'ubuntu1804-x64'
       config: debug
 
 - job: Release
   steps:
   - template: templates/dopapt-native-steps.yml
     parameters:
+      targetOs: 'ubuntu1804'
       targetOsArch: 'ubuntu1804-x64'
       config: minsizerel

azure-pipelines/build/dosdkcpp-docker.yml

@@ -14,26 +14,28 @@ schedules:
     include:
     - main
 
-pool: dotestlab
+# TODO(jimson): Provision a 1ES managed image with artifacts that have all our dependencies preinstalled
+#pool: 1es_hosted_pool_ubuntu_1804
+
+# The microsoft-hosted agents come with docker preinstalled, the caveat being that the bootstrap script needs to be called to provision everything else
+pool:
+  vmImage: 'ubuntu-18.04'
 
 jobs:
 - template: templates/do-docker-jobs.yml
   parameters:
     targetOS: 'debian10'
     targetArch: 'arm32'
-    dockerImageName: 'jimsonmsft/debian10-arm32:latest'
     stepsTemplate: 'dosdkcpp-docker-steps.yml'
 
 - template: templates/do-docker-jobs.yml
   parameters:
     targetOS: 'debian9'
     targetArch: 'arm32'
-    dockerImageName: 'jimsonmsft/debian9-arm32:latest'
     stepsTemplate: 'dosdkcpp-docker-steps.yml'
 
 - template: templates/do-docker-jobs.yml
   parameters:
     targetOS: 'ubuntu1804'
     targetArch: 'arm64'
-    dockerImageName: 'jimsonmsft/ubuntu18.04-arm64:latest'
     stepsTemplate: 'dosdkcpp-docker-steps.yml'

azure-pipelines/build/dosdkcpp-native.yml

@@ -40,20 +40,24 @@ pr:
       - azure-pipelines/*
       - sdk-cpp/build/cleanup-install.sh
 
-pool: dotestlab
+# TODO(jimson): This is a base Ubuntu 1804 Image right now, switch to an image that contains all our dependencies preinstalled
+# The advantage to using this over the Microsoft-hosted agent is parallel execution of jobs (infinite # of vms can be spun up at execution time)
+pool: 1es_hosted_pool_ubuntu_1804
 
 jobs:
 - job: Debug
   steps:
   - template: templates/dosdkcpp-native-steps.yml
     parameters:
+      targetOS: 'ubuntu1804'
       targetOsArch: 'ubuntu1804-x64'
       config: debug
 
 - job: Release
   steps:
   - template: templates/dosdkcpp-native-steps.yml
     parameters:
+      targetOS: 'ubuntu1804'
       targetOsArch: 'ubuntu1804-x64'
       config: minsizerel
       skipTests: true

azure-pipelines/build/templates/do-docker-jobs.yml

@@ -6,8 +6,6 @@ parameters:
   type: string
 - name: targetArch      # example: arm32
   type: string
-- name: dockerImageName # example: jimsonmsft/debian10-arm32:latest
-  type: string
 - name: stepsTemplate   # example: dopapt-docker-steps.yml
   type: string
 
@@ -16,14 +14,14 @@ jobs:
   steps:
   - template: ${{parameters.stepsTemplate}}
     parameters:
+      targetOS: ${{parameters.targetOS}}
       targetOsArch: ${{parameters.targetOS}}-${{parameters.targetArch}}
-      dockerImageName: ${{parameters.dockerImageName}}
       config: debug
 
 - job: ${{parameters.targetOS}}_${{parameters.targetArch}}_release
   steps:
   - template: ${{parameters.stepsTemplate}}
     parameters:
+      targetOS: ${{parameters.targetOS}}
       targetOsArch: ${{parameters.targetOS}}-${{parameters.targetArch}}
-      dockerImageName: ${{parameters.dockerImageName}}
       config: minsizerel

azure-pipelines/build/templates/doclient-lite-docker-steps.yml

@@ -2,18 +2,46 @@
 # Consume this steps template in one or more jobs by passing in parameter values.
 
 parameters:
-- name: targetOsArch    # example: debian10-arm32
+- name: targetOS        # example: ubuntu1804
   type: string
-- name: dockerImageName # example: jimsonmsft/debian10-arm32:latest
+- name: targetOsArch    # example: ubuntu1804-x64
   type: string
 - name: config          # debug/release/minsizerel/relwithdebuginfo
   type: string
 
 steps:
+- task: Docker@2
+  displayName: Login to ACR
+  inputs:
+    command: login
+    containerRegistry: doclientcontainerregistry
+    repository: $(parameters.targetOsArch)
+
+- task: Docker@2
+  displayName: Pull latest build image
+  inputs:
+    command: pull
+    containerRegistry: doclientcontainerregistry
+    arguments: 'doclientcontainerregistry.azurecr.io/${{parameters.targetOsArch}}:latest'
+
+# TODO(jimson): Once these are onboarded as an artifact to Image Factory, can remove the below two steps as well as the targetOS parameter
+# Another alternative is to just do the build from within a docker container, and then run the tests on the host
+- task: CmdLine@2
+  inputs:
+    script: 'chmod +x bootstrap.sh'
+    workingDirectory: '$(Build.SourcesDirectory)/build'
+  displayName: 'chmod bootstrap script'
+
+- task: CmdLine@2
+  inputs:
+    script: 'sudo ./bootstrap.sh --platform ${{parameters.targetOS}} --install qemu'
+    workingDirectory: '$(Build.SourcesDirectory)/build'
+  displayName: 'Install & register QEMU'
+
 - task: CmdLine@2
   inputs:
   # Unix Makefiles used in place of Ninja due to issues with 32-bit compatability on cross-arch builds
-    script: 'sudo docker run --rm --entrypoint=python3 -v $(Build.SourcesDirectory):/code -v /tmp/build-deliveryoptimization-agent-${{parameters.targetOsArch}}:/tmp/build-deliveryoptimization-agent ${{parameters.dockerImageName}} "/code/build/build.py" "--clean" "--package-for" "DEB" "--generator" "Unix Makefiles" "--config" "${{parameters.config}}" "--project" "agent"'
+    script: 'sudo docker run --rm --entrypoint=python3 -v $(Build.SourcesDirectory):/code -v /tmp/build-deliveryoptimization-agent-${{parameters.targetOsArch}}:/tmp/build-deliveryoptimization-agent doclientcontainerregistry.azurecr.io/${{parameters.targetOsArch}}:latest "/code/build/build.py" "--clean" "--package-for" "DEB" "--config" "${{parameters.config}}" "--project" "agent"'
   displayName: 'Build client-lite ${{parameters.targetOsArch}}-${{parameters.config}}'
 
 - task: CopyFiles@2

azure-pipelines/build/templates/doclient-lite-native-steps.yml

@@ -2,7 +2,9 @@
 # Consume this steps template in one or more jobs by passing in parameter values.
 
 parameters:
-- name: targetOsArch    # example: ubuntu18.04-x64
+- name: targetOS        # example: ubuntu1804
+  type: string
+- name: targetOsArch    # example: ubuntu1804-x64
   type: string
 - name: config          # debug/release
   type: string
@@ -11,12 +13,26 @@ parameters:
   default: false
 
 steps:
-# TODO(shishirb) Include --skip-tests build.py option when skipTests is true
-- task: PythonScript@0
+# TODO(jimson): Once these are onboarded as an artifact to Image Factory, can remove the below two steps
+# Another alternative is to just do the build from within a docker container, and then run the tests on the host
+- task: CmdLine@2
+  inputs:
+    script: 'chmod +x bootstrap.sh'
+    workingDirectory: '$(Build.SourcesDirectory)/build'
+  displayName: 'chmod bootstrap script'
+
+- task: CmdLine@2
+  inputs:
+    script: 'sudo ./bootstrap.sh --platform ${{parameters.targetOS}}'
+    workingDirectory: '$(Build.SourcesDirectory)/build'
+  displayName: 'Install Dependencies'
+
+# Have to use cmdline rather than built in python task, because the python env variable chooses python2
+# There is a pipeline task which allows you to specify version, but that requires configuring of the agent tools directory
+- task: CmdLine@2
   inputs:
-    scriptSource: 'filePath'
-    scriptPath: 'build/build.py'
-    arguments: '--project agent --config ${{parameters.config}} --package-for DEB --clean'
+    script: 'python3 ./build.py --project agent --config ${{parameters.config}} --package-for DEB --clean'
+    workingDirectory: '$(Build.SourcesDirectory)/build'
   displayName: 'Build agent ${{parameters.targetOsArch}}-${{parameters.config}}'
 
 - task: Bash@3

azure-pipelines/build/templates/dopapt-docker-steps.yml

@@ -2,19 +2,52 @@
 # Consume this steps template in one or more jobs by passing in parameter values.
 
 parameters:
-- name: targetOsArch    # example: debian10-arm32
+- name: targetOS        # example: ubuntu1804
   type: string
-- name: dockerImageName # example: jimsonmsft/debian10-arm32:latest
+- name: targetOsArch    # example: ubuntu1804-x64
   type: string
-- name: config          # debug/release
+- name: config          # debug/release/minsizerel/relwithdebuginfo
   type: string
 
 steps:
 - checkout: self
 
+# TODO(jimson): Once these are onboarded as an artifact to Image Factory, can remove all dependency install steps
 - task: CmdLine@2
   inputs:
-    script: 'sudo docker run --rm --entrypoint=/bin/bash -v $(Build.SourcesDirectory):/code -v /tmp/build-deliveryoptimization-plugin-apt-${{parameters.targetOsArch}}:/tmp/build-deliveryoptimization-plugin-apt ${{parameters.dockerImageName}} "/code/build/docker/docker-build-plugin.sh" "/code" "${{parameters.config}}"'
+    script: 'chmod +x bootstrap.sh'
+    workingDirectory: '$(Build.SourcesDirectory)/build'
+  displayName: 'chmod bootstrap script'
+
+- task: CmdLine@2
+  inputs:
+    script: 'sudo ./bootstrap.sh --platform ${{parameters.targetOS}} --install containertools'
+    workingDirectory: '$(Build.SourcesDirectory)/build'
+  displayName: 'Install Docker'
+
+- task: Docker@2
+  displayName: Login to ACR
+  inputs:
+    command: login
+    containerRegistry: doclientcontainerregistry
+    repository: $(parameters.targetOsArch)
+
+- task: Docker@2
+  displayName: Pull latest build image
+  inputs:
+    command: pull
+    containerRegistry: doclientcontainerregistry
+    arguments: 'doclientcontainerregistry.azurecr.io/${{parameters.targetOsArch}}:latest'
+
+- task: CmdLine@2
+  inputs:
+    script: 'sudo ./bootstrap.sh --platform ${{parameters.targetOS}} --install qemu'
+    workingDirectory: '$(Build.SourcesDirectory)/build'
+  displayName: 'Install & register QEMU'
+
+- task: CmdLine@2
+  inputs:
+    script: 'sudo docker run --rm --entrypoint=/bin/bash -v $(Build.SourcesDirectory):/code -v /tmp/build-deliveryoptimization-plugin-apt-${{parameters.targetOsArch}}:/tmp/build-deliveryoptimization-plugin-apt doclientcontainerregistry.azurecr.io/${{parameters.targetOsArch}}:latest "/code/build/docker/docker-build-plugin.sh" "/code" "${{parameters.config}}"'
   displayName: 'Build deliveryoptimization-plugin-apt ${{parameters.targetOsArch}}-${{parameters.config}}'
 
 - task: CopyFiles@2