on-demand buildkite agents

ansible/.gitignore

@@ -1,3 +1,5 @@
+library
+
 .cache/
 *.pyc
 

ansible/ci.yml

@@ -56,75 +56,9 @@
         ip6tables-save > /etc/iptables/rules.v6
 
 - hosts: buildkite_agents
-  tasks:
-    - name: test dependencies (1/2)
-      apt: { name: "{{ item }}", install_recommends: no, update_cache: yes, cache_valid_time: 3600 }
-      with_items:
-        - build-essential
-        - cmake
-        - curl
-        - gdb
-        - git
-        - jq
-        - libblas-dev
-        - libbz2-dev
-        - libcairo-dev
-        - libclang-4.0-dev # dstep
-        - libcurl4-gnutls-dev
-        - libevent-dev
-        - libgcrypt20-dev
-        - libgpg-error-dev
-        - libgtk-3-0
-        - liblapack-dev
-        - liblzo2-dev
-        - libopenblas-dev
-        - libreadline-dev
-        - libssl-dev
-        - libxml2-dev
-        - libxslt1-dev
-        - libzmq3-dev
-        - llvm-3.9-dev # ldc
-        - mongodb-server
-        - moreutils # sponge et.al.
-        - ninja-build # ldc
-        - pkg-config
-        - python-dev
-        - python-yaml
-        - python3-nose
-        - redis-server
-        - rsync
-        - unzip
-      tags: deps
-    - name: test dependencies (2/2)
-      apt: { deb: "{{ item }}", install_recommends: no }
-      with_items:
-        - https://bintray.com/sociomantic-tsunami/dlang/download_file?file_path=libebtree6_6.0.s7-rc5-xenial_amd64.deb
-        - https://bintray.com/sociomantic-tsunami/dlang/download_file?file_path=libebtree6-dev_6.0.s7-rc5-xenial_amd64.deb
-        - https://bintray.com/sociomantic-tsunami/dlang/download_file?file_path=d1to2fix_0.10.0-alpha1-xenial_amd64.deb
+  roles:
+    - role: test_deps
       tags: deps
-    - name: disable mongodb journaling
-      lineinfile: { dest: /etc/mongodb.conf, regexp: "^{{ item.key }} *=", line: "{{ item.key }} = {{ item.value }}" }
-      with_dict:
-        journal: 'false'
-        nojournal: 'true'
-      notify: restart mongodb
-    - name: use ld.gold
-      alternatives: { name: ld, link: /usr/bin/ld, path: /usr/bin/ld.gold }
-    - name: add load_travis_script command
-      copy:
-        dest: /usr/local/bin/get_travis_test_script
-        mode: 0555
-        content: |
-          #!/usr/bin/env python
-          import sys, yaml
-
-          script = yaml.load(open('.travis.yml', 'r')).get('script', '')
-          if isinstance(script, list):
-            script = '\n'.join(script)
-          print(script)
-  handlers:
-    - name: restart mongodb
-      service: { name: mongodb, state: restarted }
 
 - hosts: buildkite_agents
   vars_files:

ansible/gen_ci-agent_hcloud_image.yml

@@ -0,0 +1,122 @@
+---
+- hosts: localhost
+  become: False
+  vars_files:
+    - vars/passwords.yml # hcloud_dlang_ci_token
+  tasks:
+    - name: get server to build image
+      uri:
+        url: "https://api.hetzner.cloud/v1/servers?name=gen-ci-agent-image"
+        headers:
+          Authorization: "Bearer {{ hcloud_dlang_ci_token }}"
+        status_code: 200
+        return_content: yes
+      register: result
+      tags: create
+    - name: set server facts
+      set_fact:
+        server: "{{ result.json.servers[0] }}"
+      tags: create
+      when: result.json.servers | length
+    - name: create server to build image
+      uri:
+        url: "https://api.hetzner.cloud/v1/servers"
+        method: POST
+        headers:
+          Authorization: "Bearer {{ hcloud_dlang_ci_token }}"
+          Content-Type: application/json
+        body: |-
+          {
+            "name": "gen-ci-agent-image",
+            "server_type": "cx11",
+            "image": "ubuntu-16.04",
+            "ssh_keys": ["dawg@FedoraX64", "seb@wilzba.ch"]
+          }
+        status_code: 201
+        return_content: yes
+      register: result
+      tags: create
+      when: server is not defined
+    - name: set server facts
+      set_fact:
+        server: "{{ result.json.server }}"
+      tags: create
+      when: server is not defined
+    - name: wait for ssh to come up
+      wait_for:
+        port: 22
+        host: "{{ server.public_net.ipv4.ip }}"
+        search_regex: OpenSSH
+      tags: create
+    - name: add build server as new host
+      add_host:
+        hostname: gen-ci-agent-image
+        ansible_ssh_host: "{{ server.public_net.ipv4.ip }}"
+        groups:
+          - hcloud
+          - buildkite_demand_agents
+      tags: create
+
+- hosts: gen-ci-agent-image
+  user: root
+  gather_facts: False
+  pre_tasks:
+    - name: install python
+      raw: |
+        if [ ! -e /usr/bin/python ]; then
+            apt-get update -qq && apt-get install -qq --no-install-recommends python-minimal python-apt
+        fi
+    - name: gather facts
+      setup:
+
+- hosts: gen-ci-agent-image
+  gather_facts: False
+  vars_files:
+    - vars/passwords.yml # hcloud_dlang_ci_token
+  roles:
+    - role: users
+      tags: users
+      ansible_ssh_user: root
+      groups_for_roles:
+        - role: admin
+          groups: sudo,adm,lxd
+    - role: common
+      tags: common
+    - role: test_deps
+      tags: deps
+    - role: buildkite_agent
+      tags: buildkite_agent
+      priority: 2
+      num_agents: 8 # for CX51 server
+      agent_tags: '"queue=default,memory=high"'
+
+- hosts: localhost
+  become: False
+  vars_files:
+    - vars/passwords.yml # hcloud_dlang_ci_token
+  tasks:
+    - name: shutdown server
+      uri:
+        url: "https://api.hetzner.cloud/v1/servers/{{ server.id }}/actions/shutdown"
+        method: POST
+        headers:
+          Authorization: "Bearer {{ hcloud_dlang_ci_token }}"
+        status_code: 201
+      tags: image
+    - name: create snapshot
+      uri:
+        url: "https://api.hetzner.cloud/v1/servers/{{ server.id }}/actions/create_image"
+        method: POST
+        headers:
+          Authorization: "Bearer {{ hcloud_dlang_ci_token }}"
+          Content-Type: application/json
+        body: '{ "description": "ci-agent" }'
+        status_code: 201
+      tags: image
+    - name: delete server
+      uri:
+        url: "https://api.hetzner.cloud/v1/servers/{{ server.id }}"
+        method: DELETE
+        headers:
+          Authorization: "Bearer {{ hcloud_dlang_ci_token }}"
+      tags: image

ansible/host_vars/gen-buildkite-agent-image

@@ -0,0 +1 @@
+ansible_ssh_extra_args: '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'

ansible/inventories/static

@@ -40,10 +40,14 @@ lxd_containers
 [scaleway:children]
 scaleway_metal
 
+# running at Hetzner cloud
+[hcloud]
+
 # all non-containerized host
 [servers:children]
 lxd_hosts
 scaleway
+hcloud
 
 # admin setting
 [servers:vars]

ansible/roles/buildkite_agent/defaults/main.yml

@@ -2,5 +2,6 @@
 agent_version: 3.4.0
 priority: 5 # Higher priority agents are assigned work first
 stop_timeout: 5min # systemd TimeoutStopSec for graceful agent shutdown
+num_agents: 1
 agent_tags: '"queue=default,memory=high"'
 allowed_pipelines:

ansible/roles/buildkite_agent/tasks/main.yml

@@ -43,8 +43,12 @@
       TimeoutStopSec={{ stop_timeout }}
       KillMode=mixed
   notify: systemctl daemon-reload
-- name: enable buildkite-agent
-  service: { name: buildkite-agent, enabled: yes, state: started }
+
+- name: disable buildkite-agent
+  service: { name: buildkite-agent, enabled: no, state: stopped }
+- name: enable buildkite-agents
+  service: { name: "buildkite-agent@{{ item }}", enabled: yes, state: started }
+  with_items: "{{ range(1, num_agents + 1) | list }}"
 
 - include_tasks: agent_shutdown_check.yml
   when: inventory_hostname in groups.buildkite_demand_agents

ansible/roles/test_deps/handlers/main.yml

@@ -0,0 +1,3 @@
+---
+- name: restart mongodb
+  service: { name: mongodb, state: restarted }

ansible/roles/test_deps/tasks/main.yml

@@ -0,0 +1,64 @@
+---
+- name: test dependencies (1/2)
+  apt: { name: "{{ item }}", install_recommends: no, update_cache: yes, cache_valid_time: 3600 }
+  with_items:
+    - build-essential
+    - cmake
+    - curl
+    - gdb
+    - git
+    - jq
+    - libblas-dev
+    - libbz2-dev
+    - libcairo-dev
+    - libclang-4.0-dev # dstep
+    - libcurl4-gnutls-dev
+    - libevent-dev
+    - libgcrypt20-dev
+    - libgpg-error-dev
+    - libgtk-3-0
+    - liblapack-dev
+    - liblzo2-dev
+    - libopenblas-dev
+    - libreadline-dev
+    - libssl-dev
+    - libxml2-dev
+    - libxslt1-dev
+    - libzmq3-dev
+    - llvm-3.9-dev # ldc
+    - mongodb-server
+    - moreutils # sponge et.al.
+    - ninja-build # ldc
+    - pkg-config
+    - python-dev
+    - python-yaml
+    - python3-nose
+    - redis-server
+    - rsync
+    - unzip
+- name: test dependencies (2/2)
+  apt: { deb: "{{ item }}", install_recommends: no }
+  with_items:
+    - https://bintray.com/sociomantic-tsunami/dlang/download_file?file_path=libebtree6_6.0.s7-rc5-xenial_amd64.deb
+    - https://bintray.com/sociomantic-tsunami/dlang/download_file?file_path=libebtree6-dev_6.0.s7-rc5-xenial_amd64.deb
+    - https://bintray.com/sociomantic-tsunami/dlang/download_file?file_path=d1to2fix_0.10.0-alpha1-xenial_amd64.deb
+- name: disable mongodb journaling
+  lineinfile: { dest: /etc/mongodb.conf, regexp: "^{{ item.key }} *=", line: "{{ item.key }} = {{ item.value }}" }
+  with_dict:
+    journal: 'false'
+    nojournal: 'true'
+  notify: restart mongodb
+- name: use ld.gold
+  alternatives: { name: ld, link: /usr/bin/ld, path: /usr/bin/ld.gold }
+- name: add load_travis_script command
+  copy:
+    dest: /usr/local/bin/get_travis_test_script
+    mode: 0555
+    content: |
+      #!/usr/bin/env python
+      import sys, yaml
+
+      script = yaml.load(open('.travis.yml', 'r')).get('script', '')
+      if isinstance(script, list):
+        script = '\n'.join(script)
+      print(script)

ansible/vars/passwords.yml

@@ -1,32 +1,36 @@
 $ANSIBLE_VAULT;1.1;AES256
-37363036353564396261626166663836353137353237393630363339343633636133306134356166
-3365313335356335313230303766346139393835313532660a626335383366633636646563666166
-64633632643263656163666531613336336662613837373931396463373339363430666233373766
-6337613630343466310a626533356162363630666264643532363237396135623138653633336565
-34386336653665613334653433393836653336313538313561666465633762383364356236323037
-31376438323637383033666562656130613738303532333565383464393031336537386361333061
-34316335636264643435663564323566353038386632373235633630316234633936643936373761
-38356430356364663132346163343566646230333638663136636337383130653633653031396231
-37653566376235656164323035366132306330316531313861633131643232636166653066386233
-63333037363862613331363833626263623039343261396538623331633836306161646237393764
-65363537303538613466356363383665646264306237373961613231323933363536326164663165
-31656230376462633137653266323366663435333136323466643862613539316130386335396662
-62373630616538616335333639663433376435353837313335363231386665323761343265386266
-61343839336430313937386538646366663462373638656462623261346661656131623131383437
-30323733636462383537643232343632366434316637373338373230643334636239666631393235
-31643561376661343333623765643935643764383564383166393533343234393066373064356363
-64623166373130366430623866616238346131653530623262386239633735653262626566646332
-35646664323761663064653930336136306332336539623439623865346561323264663634323331
-30393538623836643266353036386461356338653439303431333634393533346531623839663735
-61623236613734616538636337336431633164333538356137353832663339356333653637666139
-65383561343837646235633832356162333031353764316632353330303964303266663637663062
-33396631393032663966646564373064333866633866653862626334386431353532396635316165
-38633438623638326164643761616237353235316134396234343339333335613530666665333839
-30316331356437373239333936626531353231346137396566613035643336643635623931343931
-61383232353539613039343061383830653730663261333335323135653432393238366431306538
-65343235326630643835306436623830663965336462653061643535653964326437343962646636
-33353263353861393133393266636132336535326665656161666366363930623339393638323533
-34616432366137386130616330323564653061633364396339616365666537383664643164383864
-65363435656435386663396139303037303736386539663563663165663837653061656434336435
-62366330653233326462393835323231613938623935623833333263366239663933646438356663
-313238373963343239303835626533613935
+31326432306536333237393931613062636632343838623335366433373166313962393134343435
+3365663566666232316466346461653130316639653263300a386331353136343665653733303865
+66316662303934363863306165393131626263336663386166666535373231623434396436323061
+6238323664373464390a323430346139656664386432383831313632366433656664366130616361
+61653132343031623466343765653039303635643737386635356538326262326662656666386633
+30333034313333393266616236353363316638303862623133613364373866313830383337653864
+66393734313764653866646330346238316338353937363163623565616133353739643532373363
+38373161336335663630663636356663343135663866623964363635353932323261653861373137
+62356131376465306564336361326437623136366132663536383734303862376162333463666339
+39626164306464326464666135633966646466653831313632393135353166656130653530653330
+33656161653562623338336332323062333931306436336337626539623836336464343163376466
+62366261613639313334663233343633386265663031326566323034323766643238636232376166
+32373563653730303232376261373732626537303466306337643930336264616631326232356261
+61643433353866333630323765393536306332363966373236333861633035326437613765343063
+62663233663838343961613830626663613733303631616131633465316637653366303263356439
+65303866643465333262366433643264613433623962666332396634666131333039386634613132
+37343933313765373638336134643961336536313630656362663239373035633539383737366439
+61383537373963643063636565336162626362376639333639366665323031313962346163643066
+35313335626566633634393335633539326235626230346637663431316233343665633033646530
+66626130663931663933353836343231393165343634396263313130663838323732333435313362
+61366638306565326636393830373038346339323439623230326632666331623965343935343036
+36633333326438666338333331663335653232643231363432653362656162643966313637386432
+34333230356266646335613137363434303266323561336363383465616637633764373938303965
+36633364656637353439373831326532353737666136393237386539363539663439623431303332
+39393935366263356161393266313934353134666530653232376433303239366165643432383733
+61663264626665363737343836373263646364646437653464623263336633303537666137313139
+65333663333465343339393630396230613165663631646633356665313037393633316266333265
+63363634303437653665393634663462356632623133323838646264383436633464613765326633
+38656430613335303538613863343235323936333139626431346330663130326165393065326666
+34356435656439386335326562626431656266666163363563336463663566613339383763623434
+33393038376630613031353330616165383330373837396136346466316333306463643966646262
+66386338343135353530613937373732313439316231663433653662626366346433313666623137
+39363333633132666434323162333562626161326332626133316237306530646565396465393165
+61363939663432633531643732633261323065303937653764633763616465393433373161666534
+356530393339323734393261643639666338