Login via Github

ansible/README.md

@@ -23,7 +23,10 @@ For less typing store vault and sudo passwords in [Pass: The Standard Unix Passw
 pass add dlangci/ansible_vault
 pass add dlangci/sudo
 ```
-
+At best also tell git how to diff encrypted files.
+```sh
+git config diff.ansible-vault.textconv 'ansible-vault view --vault-password-file=ansible/query_vault_pass.sh'
+```
 Alternatively comment out `vault_password_file` in [ansible.cfg](ansible.cfg) and `ansible_become_pass` in [group_vars/all](group_vars/all)
 and pass `-K, --ask-become-pass` and `--ask-vault-pass` to `ansible-playbook`.
 

ansible/ci.yml

@@ -14,6 +14,8 @@
   vars:
     - jenkins_http_port: 8080
     - jenkins_slave_agent_port: 55000
+    - jenkins_admin_username: MartinNowak
+    - jenkins_github_organizations: [dlang]
   vars_files:
     - vars/passwords.yml # jenkins_admin_password
   roles:
@@ -25,7 +27,7 @@
 
 - hosts: jenkins_slaves
   vars:
-    - jenkins_addr: ci.dlang.io
+    - jenkins_admin_username: MartinNowak
   vars_files:
     - vars/passwords.yml # jenkins_admin_password
   roles:

ansible/group_vars/vagrant

@@ -1 +1,2 @@
-ansible_ssh_host: 172.16.1.{{ host_id }}
+# pseudo backup URL
+backup_url: ftp://user:password@backup.example.com

ansible/group_vars/vagrant_hosts

@@ -0,0 +1 @@
+ansible_ssh_host: 172.16.1.{{ host_id }}

ansible/inventory

@@ -11,6 +11,10 @@ runner-01.ci.lxd
 runner-02.ci.lxd
 dub-registry.ci.lxd
 
+[ci_containers:vars]
+jenkins_url=https://ci.dlang.io/
+jenkins_use_github_auth=True
+
 [ci_containers:vars]
 # ssh proxy into container (running ssh itself)
 ansible_ssh_common_args=-o ProxyCommand="ssh -W %h:%p ci.dlang.io"

ansible/roles/jenkins/defaults/main.yml

@@ -1,6 +1,21 @@
 ---
 jenkins_version: 2.60.1-1 # LTS
-jenkins_admin_username: admin
-jenkins_admin_password:
 jenkins_http_port:
 jenkins_slave_agent_port:
+jenkins_plugins: []
+# public URL of the instance, e.g. https://ci.example.com
+jenkins_url:
+# also see https://plugins.jenkins.io/github-oauth
+# wether to use github or password auth
+jenkins_use_github_auth: false
+# either github username or admin username
+jenkins_admin_username: admin
+# personal github access token or password
+jenkins_admin_password:
+# optional admin email address
+jenkins_admin_email:
+# organizations with write access
+jenkins_github_organizations: []
+# client id and secret of registered github app
+jenkins_github_app_client_id:
+jenkins_github_app_client_secret:

ansible/roles/jenkins/tasks/main.yml

@@ -20,8 +20,17 @@
     # use internal hostname for slave listener http://stackoverflow.com/a/39965700/2371032
     line: JAVA_ARGS="$JAVA_ARGS -Djenkins.install.runSetupWizard=false -Dhudson.TcpSlaveAgentListener.hostName={{ inventory_hostname }} -Xms512m -Xmx1024m"
   notify: restart jenkins
-- name: configure jenkins
-  template: { src: init.groovy.j2, dest: /var/lib/jenkins/init.groovy, owner: jenkins, mode: 0400 }
+- set_fact:
+    jenkins_plugins: "{{ jenkins_plugins + ['thinBackup'] }}"
+- set_fact:
+    jenkins_plugins: "{{ jenkins_plugins + ['github-oauth'] }}"
+  when: jenkins_use_github_auth
+- name: add jenkins init.groovy.d folder
+  file: { dest: /var/lib/jenkins/init.groovy.d, state: directory, owner: jenkins, mode: 0500 }
+- name: add jenkins init scripts
+  template: { src: "{{ item.1 }}.groovy.j2", dest: "/var/lib/jenkins/init.groovy.d/{{ item.0 }}-{{ item.1 }}.groovy", owner: jenkins, mode: 0400 }
+  # manually delete scripts when changing order
+  with_indexed_items: [plugins, "{{ 'github_auth' if jenkins_use_github_auth else 'password_auth' }}", executors]
   notify: restart jenkins
 - name: compress logs
   cron:

ansible/roles/jenkins/templates/backup_partial.sh.j2

@@ -1,5 +1,5 @@
 log 'Running thinBackup'
-curl -fsS --data "script=$(cat backup.groovy)" --user 'admin:{{ jenkins_admin_password }}' http://localhost:{{ jenkins_http_port }}/scriptText
+curl -fsS --data "script=$(cat backup.groovy)" --user '{{ jenkins_admin_username }}:{{ jenkins_admin_password }}' http://localhost:{{ jenkins_http_port }}/scriptText
 
 folders=( FULL-* )
 log "Compressing and uploading thinBackup ${folders[0]}"

ansible/roles/jenkins/templates/executors.groovy.j2

@@ -0,0 +1,35 @@
+#!groovy
+import hudson.model.Node
+import hudson.slaves.DumbSlave
+import hudson.slaves.JNLPLauncher
+import hudson.slaves.RetentionStrategy
+import java.util.logging.Logger
+import jenkins.model.Jenkins
+import jenkins.model.JenkinsLocationConfiguration
+import jenkins.security.s2m.AdminWhitelistRule
+
+def logger = Logger.getLogger("")
+def instance = Jenkins.getInstance()
+
+logger.info('configuring executors and slaves')
+
+instance.setNumExecutors(0) // no builds on master
+instance.setSlaveAgentPort([{{ jenkins_slave_agent_port }}])
+instance.getInjector().getInstance(AdminWhitelistRule.class).setMasterKillSwitch(false);
+
+def loccfg = new jenkins.model.JenkinsLocationConfiguration();
+loccfg.setUrl('{{ jenkins_url }}')
+{% if jenkins_admin_email %}
+loccfg.setAdminAddress('{{ jenkins_admin_email }}');
+{% endif %}
+
+instance.save()
+
+{% for host in groups.jenkins_slaves %}
+instance.addNode(
+new DumbSlave("{{ host.split('.')[0] }}", "", '/var/lib/jenkins', '4',
+              Node.Mode.NORMAL, '',
+              new JNLPLauncher(), new RetentionStrategy.Always(),
+              new ArrayList())
+)
+{% endfor %}

ansible/roles/jenkins/templates/github_auth.groovy.j2

@@ -0,0 +1,61 @@
+import jenkins.model.Jenkins
+import java.util.logging.Logger
+
+def logger = Logger.getLogger("")
+def instance = Jenkins.getInstance()
+
+logger.info('configuring github authentication plugin')
+
+// https://plugins.jenkins.io/github-oauth
+import hudson.security.SecurityRealm
+import org.jenkinsci.plugins.GithubSecurityRealm
+String githubWebUri = 'https://github.com'
+String githubApiUri = 'https://api.github.com'
+String clientID = '{{ jenkins_github_app_client_id }}'
+String clientSecret = '{{ jenkins_github_app_client_secret }}'
+String oauthScopes = 'read:org,user:email'
+SecurityRealm github_realm = new GithubSecurityRealm(githubWebUri, githubApiUri, clientID, clientSecret, oauthScopes)
+//check for equality, no need to modify the runtime if no settings changed
+if(!github_realm.equals(instance.getSecurityRealm())) {
+    instance.setSecurityRealm(github_realm)
+    instance.save()
+}
+
+import org.jenkinsci.plugins.GithubAuthorizationStrategy
+import hudson.security.AuthorizationStrategy
+
+//permissions are ordered similar to web UI
+//Admin User Names
+String adminUserNames = '{{ jenkins_admin_username }}'
+//Participant in Organization
+String organizationNames = '{{ jenkins_github_organizations | join(',') }} '
+//Use Github repository permissions
+boolean useRepositoryPermissions = true
+//Grant READ permissions to all Authenticated Users
+boolean authenticatedUserReadPermission = false
+//Grant CREATE Job permissions to all Authenticated Users
+boolean authenticatedUserCreateJobPermission = false
+//Grant READ permissions for /github-webhook
+boolean allowGithubWebHookPermission = true
+//Grant READ permissions for /cc.xml
+boolean allowCcTrayPermission = false
+//Grant READ permissions for Anonymous Users
+boolean allowAnonymousReadPermission = true
+//Grant ViewStatus permissions for Anonymous Users
+boolean allowAnonymousJobStatusPermission = false
+
+AuthorizationStrategy github_authorization = new GithubAuthorizationStrategy(adminUserNames,
+    authenticatedUserReadPermission,
+    useRepositoryPermissions,
+    authenticatedUserCreateJobPermission,
+    organizationNames,
+    allowGithubWebHookPermission,
+    allowCcTrayPermission,
+    allowAnonymousReadPermission,
+    allowAnonymousJobStatusPermission)
+
+//check for equality, no need to modify the runtime if no settings changed
+if(!github_authorization.equals(instance.getAuthorizationStrategy())) {
+    instance.setAuthorizationStrategy(github_authorization)
+    instance.save()
+}

ansible/roles/jenkins/templates/password_auth.groovy.j2

@@ -0,0 +1,18 @@
+#!groovy
+import hudson.security.FullControlOnceLoggedInAuthorizationStrategy
+import hudson.security.HudsonPrivateSecurityRealm
+import java.util.logging.Logger
+import jenkins.model.Jenkins
+
+def logger = Logger.getLogger("")
+def instance = Jenkins.getInstance()
+
+logger.info('configuring password authentication')
+
+def hudsonRealm = new HudsonPrivateSecurityRealm(false)
+hudsonRealm.createAccount('{{ jenkins_admin_username }}', '{{ jenkins_admin_password }}')
+instance.setSecurityRealm(hudsonRealm)
+
+def strategy = new FullControlOnceLoggedInAuthorizationStrategy()
+instance.setAuthorizationStrategy(strategy)
+instance.save()

ansible/roles/jenkins/templates/plugins.groovy.j2

@@ -0,0 +1,40 @@
+#!groovy
+// https://github.com/blacklabelops/jenkins/blob/585cb66d8add2fc6b18996ad7d91bdd0034fbb74/imagescripts/initplugins.sh
+import jenkins.model.Jenkins
+import java.util.logging.Logger
+
+def logger = Logger.getLogger("")
+def instance = Jenkins.getInstance()
+
+def plugins = '{{ jenkins_plugins | join(' ') }}'
+logger.info("installing plugins: '${plugins}'")
+
+def installed = false
+def initialized = false
+logger.info("" + plugins)
+def pm = instance.getPluginManager()
+def uc = instance.getUpdateCenter()
+plugins.split().each {
+    if (!pm.getPlugin(it)) {
+        logger.info("Looking UpdateCenter for " + it)
+        if (!initialized) {
+            uc.updateAllSites()
+            initialized = true
+        }
+        def plugin = uc.getPlugin(it)
+        if (plugin) {
+            logger.info("Installing " + it)
+    	    def installFuture = plugin.deploy()
+            while(!installFuture.isDone()) {
+                logger.info("Waiting for plugin install: " + it)
+                sleep(3000)
+            }
+            installed = true
+        }
+    }
+}
+if (installed) {
+    logger.info("Plugins installed, initializing a restart!")
+    instance.save()
+    instance.restart()
+}

ansible/roles/jenkins_slave/tasks/main.yml

@@ -3,15 +3,30 @@
   apt: { name: default-jre, install_recommends: no, update_cache: yes, cache_valid_time: 3600 }
 - name: add jenkins slave user
   user: { name: jenkins, system: yes, home: /var/lib/jenkins }
+# see https://wiki.jenkins.io/display/JENKINS/Remote+access+API#RemoteaccessAPI-CSRFProtection
+# and https://support.cloudbees.com/hc/en-us/articles/222520647-How-to-find-slave-secret-key
+- set_fact:
+    script: |-
+      import hudson.model.Hudson;
+      def slave = Hudson.instance.slaves.find { it.name == '{{ inventory_hostname_short }}' }.getComputer()
+      println(slave.getJnlpMac())
+- name: get CSRF token
+  uri:
+    url: "{{ jenkins_url}}crumbIssuer/api/json"
+    user: "{{ jenkins_admin_username }}"
+    password: "{{ jenkins_admin_password }}"
+    force_basic_auth: yes
+    method: GET
+  register: csrf_token
 - name: get slave secret
   uri:
-    url: https://{{ jenkins_addr}}/scriptText
-    user: admin
+    url: "{{ jenkins_url}}scriptText"
+    user: "{{ jenkins_admin_username }}"
     password: "{{ jenkins_admin_password }}"
     force_basic_auth: yes
     method: POST
     body: >-
-      script=import hudson.model.Hudson; println(Hudson.instance.slaves.find { it.name == '{{ inventory_hostname_short }}' }.getComputer().getJnlpMac())
+      {{ csrf_token.json.crumbRequestField }}={{ csrf_token.json.crumb }}&script={{ script | urlencode() }}
     return_content: yes
   register: response
   check_mode: no

ansible/roles/jenkins_slave/templates/jenkins.service.j2

@@ -4,8 +4,8 @@ After=local-fs.target network.target
 
 [Service]
 Type=simple
-ExecStartPre=/usr/bin/wget https://{{ jenkins_addr }}/jnlpJars/slave.jar -O /var/lib/jenkins/slave.jar
-ExecStart=/usr/bin/java -Xms64m -Xmx256m -jar /var/lib/jenkins/slave.jar -jnlpUrl https://{{ jenkins_addr }}/computer/{{ inventory_hostname_short }}/slave-agent.jnlp -secret {{ secret }}
+ExecStartPre=/usr/bin/wget {{ jenkins_url }}jnlpJars/slave.jar -O /var/lib/jenkins/slave.jar
+ExecStart=/usr/bin/java -Xms64m -Xmx256m -jar /var/lib/jenkins/slave.jar -jnlpUrl {{ jenkins_url }}computer/{{ inventory_hostname_short }}/slave-agent.jnlp -secret {{ secret }}
 Restart=on-failure
 RestartSec=10
 

ansible/vagrant_inventory

@@ -1,6 +1,3 @@
-[vagrant]
-ci.dlang.io
-
 [container_hosts]
 ci.dlang.io
 
@@ -13,6 +10,10 @@ jenkins.ci.lxd
 runner-01.ci.lxd
 runner-02.ci.lxd
 
+[ci_containers:vars]
+jenkins_url=http://jenkins.ci.lxd:8080/
+jenkins_use_github_auth=False
+
 [ci_containers:vars]
 # ssh proxy into container (running ssh itself)
 ansible_ssh_common_args=-o ProxyCommand="ssh -o IdentityFile=.vagrant/machines/ci.dlang.io/virtualbox/private_key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p ubuntu@172.16.1.2" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
@@ -22,3 +23,11 @@ ansible_ssh_user=root
 [jenkins_slaves]
 runner-01.ci.lxd
 runner-02.ci.lxd
+
+[vagrant_hosts:children]
+container_hosts
+
+# all vagrant group members
+[vagrant:children]
+container_hosts
+containers

ansible/vars/passwords.yml

@@ -1,8 +1,15 @@
 $ANSIBLE_VAULT;1.1;AES256
-35313838313037626136353436643364363066663232323430356462366637393230653734373163
-3737376465373333346638633431623861346239616231610a326465363763333238366237663632
-62366137336236633236383337343166366239323634333365326534343463373533313235303633
-3133383636333338640a303863363139383635326432306664656332653364613966386662666633
-36393766333437356532663138326139373332663136633937373334646237616466363563363162
-66626465323237623965653834653431383064653361653039656635663834636163336631633730
-666638323532643636346366663530373964
+36613061396366396131376134353639663165396332326362306361323731386664306535343734
+3261306536643535386137323336323531633766346366390a613161313939383739663934333663
+39623235346339643536303230373235666435303430326463353432353661383633323338363038
+3564653934303830390a303530663163323432383865666665373836393731303236303630613363
+34346533363837393161333931366562373831306164393364643137666334363836656135373835
+37616133373365626637633762353731393330666464633736393631626664653565633163613537
+63333535633435643166366333356263623662643336363463353036333666623735396139396139
+33666631363235383631326663346561643133303563316133343933666138626138663338633932
+33643162306265643234313438666565383333313164356335393561373262306338336361363935
+33616530313931303434353664383763636431666161303064353831643535633163363032326238
+32383732663261366466626466646632346635373331343766383531393431303764393363373362
+32343835383866303763653964303832313962376135346236336332346635393163343535366132
+39373366616461653163393533663764373730313832343234336430353236313764316337363434
+3639663235616164323237643130333230343238333738646639