From 49abbb5315e980b6dd54b11818304d1d101c5757 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Sun, 23 Aug 2015 21:09:24 +1000 Subject: [PATCH 01/15] added libcurl4-openssl-dev to git compile for https:// URLs --- setup/raspberry-pi/ansible-playbook.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/setup/raspberry-pi/ansible-playbook.yaml b/setup/raspberry-pi/ansible-playbook.yaml index 20962696a..85d6df8f3 100644 --- a/setup/raspberry-pi/ansible-playbook.yaml +++ b/setup/raspberry-pi/ansible-playbook.yaml @@ -80,14 +80,19 @@ tags: jenkins - name: git | Install required packages for git compile - apt: name=gettext update_cache=yes state=latest + apt: name={{ item }} update_cache=yes state=latest + with_items: + - gettext + - libcurl4-openssl-dev tags: git - name: git | Download, compile and install git 2.5 - shell: curl https://www.kernel.org/pub/software/scm/git/git-2.5.0.tar.xz -O && \ + shell: rm -rf git-2.5.0* && \ + curl https://www.kernel.org/pub/software/scm/git/git-2.5.0.tar.xz -O && \ tar -xvf git-2.5.0.tar.xz && \ cd git-2.5.0/ && \ ./configure --prefix=/usr --with-gitconfig=/etc/gitconfig && \ make && \ - sudo make install + sudo make install && \ + rm -rf git-2.5.0* tags: git From 30700f38cce7d99ce8855ed49a35132e03e92f4c Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Sun, 23 Aug 2015 21:24:40 +1000 Subject: [PATCH 02/15] config for new nodejs.org server --- setup/www/ansible-playbook.yaml | 46 ++++++--- setup/www/ansible-vars.yaml | 4 +- setup/www/host_vars/.gitignore | 2 +- setup/www/host_vars/iojs-www.tmpl | 4 - setup/www/host_vars/nodejs.org.tmpl | 4 + setup/www/resources/.gitignore | 4 +- setup/www/resources/github-webhook.json | 17 +++- setup/www/resources/iojs.org | 39 ++++++++ setup/www/resources/nodejs.org | 125 ++++++++++++++++++++++++ setup/www/resources/old_nodejs.org.conf | 124 +++++++++++++++++++++++ 10 files changed, 344 insertions(+), 25 deletions(-) delete mode 100644 setup/www/host_vars/iojs-www.tmpl create mode 100644 setup/www/host_vars/nodejs.org.tmpl create mode 100644 setup/www/resources/nodejs.org create mode 100644 setup/www/resources/old_nodejs.org.conf diff --git a/setup/www/ansible-playbook.yaml b/setup/www/ansible-playbook.yaml index c4a899912..7034a0654 100644 --- a/setup/www/ansible-playbook.yaml +++ b/setup/www/ansible-playbook.yaml @@ -1,5 +1,5 @@ --- -- hosts: iojs-www +- hosts: nodejs.org remote_user: root @@ -16,7 +16,7 @@ tags: general - name: Node.js | Add the NodeSource Node.js repo - command: "bash -c 'curl -sL https://deb.nodesource.com/setup | bash -'" + command: "bash -c 'curl -sL https://deb.nodesource.com/setup_iojs_3.x | bash -'" tags: node - name: General | Install required packages @@ -56,8 +56,12 @@ replace: dest=/etc/github-webhook.json regexp="\{\{github_secret\}\}" replace="{{ github_secret }}" tags: webhook - - name: GitHub Webhook | Copy update command to config - replace: dest=/etc/github-webhook.json regexp="\{\{update_command\}\}" replace="{{ update_command }}" + - name: GitHub Webhook | Copy update nodejs command to config + replace: dest=/etc/github-webhook.json regexp="\{\{update_nodejs_command\}\}" replace="{{ update_nodejs_command }}" + tags: webhook + + - name: GitHub Webhook | Copy update iojs command to config + replace: dest=/etc/github-webhook.json regexp="\{\{update_iojs_command\}\}" replace="{{ update_iojs_command }}" tags: webhook - name: GitHub Webhook | Copy Upstart config @@ -68,22 +72,38 @@ service: name=github-webhook state=started tags: webhook - - name: Setup | Initial clone + - name: Setup | Initial nodejs clone + remote_user: "{{ server_user }}" + command: "bash -c '{{ clone_nodejs_command }}'" + tags: setup + + - name: Setup | Initial iojs clone remote_user: "{{ server_user }}" - command: "bash -c '{{ clone_command }}'" + command: "bash -c '{{ clone_iojs_command }}'" tags: setup - - name: Setup | Initial update + - name: Setup | Initial nodejs update remote_user: "{{ server_user }}" - command: "bash -c '{{ update_command }}'" + command: "bash -c '{{ update_nodejs_command }}'" tags: setup - - name: nginx | Copy site config - copy: src=./resources/iojs.org dest=/etc/nginx/sites-available/iojs.org mode=0644 + - name: Setup | Initial iojs update + remote_user: "{{ server_user }}" + command: "bash -c '{{ update_iojs_command }}'" + tags: setup + + - name: nginx | Copy site configs + copy: src=./resources/{{ item }} dest=/etc/nginx/sites-available/{{ item }} mode=0644 + with_items: + - nodejs.org + - iojs.org tags: nginx - - name: nginx | Create config symlink - file: src=/etc/nginx/sites-available/iojs.org dest=/etc/nginx/sites-enabled/00-iojs.org state=link + - name: nginx | Create config symlinks + file: src=/etc/nginx/sites-available/{{ item }} dest=/etc/nginx/sites-enabled/00-{{ item }} state=link + with_items: + - nodejs.org + - iojs.org tags: nginx - name: nginx | Generate DH params @@ -93,7 +113,9 @@ - name: nginx | Copy site certificates copy: src=./resources/{{ item }} dest=/etc/nginx/ssl/{{ item }} mode=0644 with_items: + - nodejs_chained.crt - iojs_chained.crt + - nodejs.key - iojs.key tags: nginx diff --git a/setup/www/ansible-vars.yaml b/setup/www/ansible-vars.yaml index c24683246..580bd9c33 100644 --- a/setup/www/ansible-vars.yaml +++ b/setup/www/ansible-vars.yaml @@ -1,8 +1,10 @@ --- -server_user: iojs +server_user: nodejs ssh_users: - rvagg - indutny + - jbergstroem + - orangemocha packages: - nodejs - nginx diff --git a/setup/www/host_vars/.gitignore b/setup/www/host_vars/.gitignore index 304d0b150..35bd93942 100644 --- a/setup/www/host_vars/.gitignore +++ b/setup/www/host_vars/.gitignore @@ -1 +1 @@ -iojs-www +nodejs.org \ No newline at end of file diff --git a/setup/www/host_vars/iojs-www.tmpl b/setup/www/host_vars/iojs-www.tmpl deleted file mode 100644 index 4fbea869a..000000000 --- a/setup/www/host_vars/iojs-www.tmpl +++ /dev/null @@ -1,4 +0,0 @@ ---- -github_secret: "INSERT SECRET FROM WEBHOOK HERE" -clone_command: "cd /home/iojs/ && rm -rf website.github && git clone https://github.com/iojs/website.git website.github" -update_command: "cd /home/iojs/website.github/ && git checkout master && git reset --hard && git clean -fdx && git pull origin master && rsync -avz --delete --exclude .git /home/iojs/website.github/public/ /home/iojs/www/" diff --git a/setup/www/host_vars/nodejs.org.tmpl b/setup/www/host_vars/nodejs.org.tmpl new file mode 100644 index 000000000..833a7ebd6 --- /dev/null +++ b/setup/www/host_vars/nodejs.org.tmpl @@ -0,0 +1,4 @@ +--- +github_secret: "INSERT SECRET FROM WEBHOOK HERE" +clone_command: "cd /home/nodejs/ && rm -rf new.nodejs.org.github && git clone https://github.com/nodejs/new.nodejs.org.git new.nodejs.org.github" +update_command: "cd /home/nodejs/new.nodejs.org.github/ && git checkout master && git reset --hard && git clean -fdx && git pull origin master && rsync -avz --delete --exclude .git /home/nodejs/new.nodejs.org.github/public/ /home/nodejs/www/" diff --git a/setup/www/resources/.gitignore b/setup/www/resources/.gitignore index 130181699..be870b428 100644 --- a/setup/www/resources/.gitignore +++ b/setup/www/resources/.gitignore @@ -1,2 +1,2 @@ -iojs_chained.crt -iojs.key +*.crt +*.key diff --git a/setup/www/resources/github-webhook.json b/setup/www/resources/github-webhook.json index c59401015..819535753 100644 --- a/setup/www/resources/github-webhook.json +++ b/setup/www/resources/github-webhook.json @@ -3,9 +3,16 @@ "path": "/webhook", "secret": "{{github_secret}}", "log": "/var/log/github-webhook.log", - "rules": [{ - "event": "push", - "match": "ref == \"refs/heads/master\" && repository.full_name == \"iojs/website\"", - "exec": "{{update_command}}" - }] + "rules": [ + { + "event": "push", + "match": "ref == \"refs/heads/master\" && repository.full_name == \"nodejs/new.nodejs.org\"", + "exec": "{{update_nodejs_command}}" + }, + { + "event": "push", + "match": "ref == \"refs/heads/master\" && repository.full_name == \"nodejs/iojs.org\"", + "exec": "{{update_iojs_command}}" + } + ] } diff --git a/setup/www/resources/iojs.org b/setup/www/resources/iojs.org index 49ed463a8..f7f4a3887 100644 --- a/setup/www/resources/iojs.org +++ b/setup/www/resources/iojs.org @@ -1,10 +1,12 @@ server { listen 80; + listen [::]:80; return 301 https://iojs.org$request_uri; } server { listen 443 ssl spdy; + listen [::]:443 ipv6only=on ssl spdy; server_name iojs.org; @@ -26,6 +28,9 @@ server { spdy_keepalive_timeout 300; spdy_headers_comp 9; + keepalive_timeout 60; + server_tokens off; + resolver 8.8.4.4 8.8.8.8 valid=300s; resolver_timeout 10s; @@ -33,6 +38,10 @@ server { add_header X-Frame-Options DENY; add_header X-Content-Type-Options nosniff; + log_format main '$remote_addr - $remote_user [$time_local] $request ' + '"$status" $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /var/log/nginx/iojs.org-access.log; error_log /var/log/nginx/iojs.org-error.log; @@ -46,9 +55,39 @@ server { rewrite ^(.*)$ https://iojs.org$1; } + location ~ \.json$ { + add_header access-control-allow-origin *; + } + + location ~ /(dist|docs)/v[0-9]+\.[0-9]+\.[0-9]+/.*\.(tar\.gz|msi|exe|pdb|lib|html|json) { + expires max; + } + location / { root /home/iojs/www; index index.html; default_type text/plain; } + + location /download { + alias /home/dist/public; + autoindex on; + default_type text/plain; + } + + location /dist { + alias /home/dist/public/release/; + autoindex on; + default_type text/plain; + } + + location /api { + alias /home/dist/public/release/latest/doc/api; + autoindex on; + default_type text/plain; + } + + location /download-stats.json { + alias /home/iojs/download-stats.json; + } } diff --git a/setup/www/resources/nodejs.org b/setup/www/resources/nodejs.org new file mode 100644 index 000000000..0179518e6 --- /dev/null +++ b/setup/www/resources/nodejs.org @@ -0,0 +1,125 @@ +server { + listen 80; + listen [::]:80; + return 301 https://nodejs.org$request_uri; + + # TODO: old nodejs.org allowed /dist/ and *.json through on 80, should we + # also do the same here? The suggestion was that it would break npm/travis-ci + # although that's likely to be old versions + # If we let it through then we need to copy some of the 443 config here +} + +server { + listen 80; + listen [::]:80; + server_name doc.nodejs.org docs.nodejs.org; + rewrite ^/(v[0-9]+\.[0-9]+\.[0-9]+)(/?.*)$ https://nodejs.org/docs/$1$2 permanent; + rewrite /(.*)$ https://nodejs.org/docs/latest/$1 permanent; +} + +server { + listen 80; + listen [::]:80; + server_name api.nodejs.org; + rewrite ^/(v[0-9]+\.[0-9]+\.[0-9]+)(/?.*)$ https://nodejs.org/docs/$1/api$2 permanent; + rewrite /(.*)$ https://nodejs.org/docs/latest/api/$1 permanent; +} + +server { + listen 80; + listen [::]:80; + server_name dist.nodejs.org; + rewrite /(.*)$ http://nodejs.org/dist/$1 permanent; +} + +server { + listen 443 ssl spdy; + listen [::]:443 ipv6only=on ssl spdy; + + server_name nodejs.org; + + ssl_certificate ssl/nodejs_chained.crt; + ssl_certificate_key ssl/nodejs.key; + ssl_trusted_certificate ssl/nodejs_chained.crt; + ssl_dhparam ssl/dhparam.pem; + + ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS; + ssl_prefer_server_ciphers on; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 24h; + + ssl_stapling on; + ssl_stapling_verify on; + + spdy_keepalive_timeout 300; + spdy_headers_comp 9; + + keepalive_timeout 60; + server_tokens off; + + resolver 8.8.4.4 8.8.8.8 valid=300s; + resolver_timeout 10s; + + add_header Strict-Transport-Security max-age=63072000; + add_header X-Frame-Options DENY; + add_header X-Content-Type-Options nosniff; + + log_format main '$remote_addr - $remote_user [$time_local] $request ' + '"$status" $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/nodejs.org-access.log; + error_log /var/log/nginx/nodejs.org-error.log; + + gzip on; + gzip_static on; + gzip_disable "MSIE [1-6]\."; + default_type text/html; + gzip_types text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript; + + if ($host ~* ^www\.){ + rewrite ^(.*)$ https://nodejs.org$1; + } + + location ~ \.json$ { + add_header access-control-allow-origin *; + } + + location ~ /(dist|docs)/v[0-9]+\.[0-9]+\.[0-9]+/.*\.(tar\.gz|msi|exe|pdb|lib|html|json) { + expires max; + } + + location /documentation/ { + rewrite ^/documentation/api(.*)$ /api$1 permanent; + } + + location / { + root /home/nodejs/www; + index index.html; + default_type text/plain; + } + + location /download { + alias /home/dist/public; + autoindex on; + default_type text/plain; + } + + location /dist { + alias /home/dist/public/release/; + autoindex on; + default_type text/plain; + } + + location /api { + alias /home/dist/public/release/latest/doc/api; + autoindex on; + default_type text/plain; + } + + location /download-stats.json { + alias /home/iojs/download-stats.json; + } +} diff --git a/setup/www/resources/old_nodejs.org.conf b/setup/www/resources/old_nodejs.org.conf new file mode 100644 index 000000000..8bc007cfe --- /dev/null +++ b/setup/www/resources/old_nodejs.org.conf @@ -0,0 +1,124 @@ +user www www; +worker_processes 1; +error_log /var/log/nginx/error.log; +pid /var/spool/nginx/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /opt/local/etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] $request ' + '"$status" $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /var/log/nginx/access.log main; + + keepalive_timeout 60; + server_tokens off; + + server { + listen 80; + server_name nodejs.org 8.12.44.238 www.nodejs.org; + + access_log /var/log/nginx/nodejs.access_log main; + error_log /var/log/nginx/nodejs.error_log info; + + autoindex on; + + location ~ \.json$ { + add_header access-control-allow-origin *; + } + + location ~ /(dist|docs)/v[0-9]+\.[0-9]+\.[0-9]+/.*\.(tar\.gz|msi|exe|pdb|lib|html|json) { + expires max; + } + + location ~ /blog(.*) { + rewrite ^/blog(.*) http://blog.nodejs.org/$1 permanent; + } + + location /documentation/ { + rewrite ^/documentation/api(.*)$ /api$1 permanent; + } + + # don't redirect dist to https as it breaks nvm/travis-ci + # probably the same is true for json + location ~ ^/(?!(dist/|dist$|\.json$)) { + rewrite ^ https://nodejs.org$request_uri permanent; + } + + root /home/node/web/nodejs.org; + } + + server { + listen 443 ssl; + server_name nodejs.org 8.12.44.238 www.nodejs.org; + + ssl_certificate /opt/local/etc/ssl/NODEJS-SSL/nodejs.org.combined.crt; + ssl_certificate_key /opt/local/etc/ssl/NODEJS-SSL/nodejs.org.key; + + access_log /var/log/nginx/nodejs.access_log main; + error_log /var/log/nginx/nodejs.error_log info; + + autoindex on; + + location ~ \.json$ { + add_header access-control-allow-origin *; + } + + location ~ /(dist|docs)/v[0-9]+\.[0-9]+\.[0-9]+/.*\.(tar\.gz|msi|exe|pdb|lib|html|json) { + expires max; + } + + location ~ /blog(.*) { + rewrite ^/blog(.*) http://blog.nodejs.org/$1 permanent; + } + + location /documentation/ { + rewrite ^/documentation/api(.*)$ /api$1 permanent; + } + + root /home/node/web/nodejs.org; + } + + server { + listen 80; + server_name blog.nodejs.org; + + access_log /var/log/nginx/nodejs.access_log main; + error_log /var/log/nginx/nodejs.error_log info; + + autoindex on; + root /home/node/web/nodejs.org/blog; + + if (-f $request_filename/index.html) { + rewrite (.*) $1/index.html break; + } + if (-f $request_filename/index.xml) { + rewrite (.*) $1/index.xml break; + } + } + + server { + listen 80; + server_name doc.nodejs.org docs.nodejs.org; + rewrite ^/(v[0-9]+\.[0-9]+\.[0-9]+)(/?.*)$ https://nodejs.org/docs/$1$2 permanent; + rewrite /(.*)$ https://nodejs.org/docs/latest/$1 permanent; + } + + server { + listen 80; + server_name api.nodejs.org; + rewrite ^/(v[0-9]+\.[0-9]+\.[0-9]+)(/?.*)$ https://nodejs.org/docs/$1/api$2 permanent; + rewrite /(.*)$ https://nodejs.org/docs/latest/api/$1 permanent; + } + + server { + listen 80; + server_name dist.nodejs.org; + rewrite /(.*)$ http://nodejs.org/dist/$1 permanent; + } +} From 7ddfa1c35fd560f0d86836851c2e18ea7774a2aa Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 28 Aug 2015 22:39:49 +1000 Subject: [PATCH 03/15] working config for both sites --- setup/ansible-inventory | 4 ++ setup/www/ansible-playbook.yaml | 84 +++++++++++++------------ setup/www/ansible-vars.yaml | 3 +- setup/www/host_vars/.gitignore | 2 +- setup/www/host_vars/node-www.tmpl | 2 + setup/www/host_vars/nodejs.org.tmpl | 4 -- setup/www/resources/.gitignore | 1 + setup/www/resources/build-site.sh | 59 +++++++++++++++++ setup/www/resources/check-build-site.sh | 14 +++++ setup/www/resources/github-webhook.conf | 2 +- setup/www/resources/github-webhook.json | 6 +- setup/www/resources/iojs.org | 6 +- setup/www/resources/nodejs.org | 11 ++-- setup/www/resources/old_mime.types | 74 ++++++++++++++++++++++ 14 files changed, 210 insertions(+), 62 deletions(-) create mode 100644 setup/www/host_vars/node-www.tmpl delete mode 100644 setup/www/host_vars/nodejs.org.tmpl create mode 100755 setup/www/resources/build-site.sh create mode 100755 setup/www/resources/check-build-site.sh create mode 100644 setup/www/resources/old_mime.types diff --git a/setup/ansible-inventory b/setup/ansible-inventory index e2997445b..0e14e4950 100644 --- a/setup/ansible-inventory +++ b/setup/ansible-inventory @@ -69,6 +69,9 @@ iojs-ns-pi2-5 [iojs-www] iojs-www +[node-www] +node-www + [iojs-jenkins] iojs-jenkins @@ -79,6 +82,7 @@ iojs-build-ubuntu1204 iojs-build-debian8 iojs-build-containers iojs-www +node-www iojs-jenkins iojs-armv7-wheezy diff --git a/setup/www/ansible-playbook.yaml b/setup/www/ansible-playbook.yaml index 7034a0654..f2d58b1bc 100644 --- a/setup/www/ansible-playbook.yaml +++ b/setup/www/ansible-playbook.yaml @@ -1,5 +1,5 @@ --- -- hosts: nodejs.org +- hosts: node-www remote_user: root @@ -7,6 +7,10 @@ - include_vars: ansible-vars.yaml tags: vars + - name: Node.js | Add the NodeSource Node.js repo + command: "bash -c 'curl -sL https://deb.nodesource.com/setup_iojs_3.x | bash -'" + tags: general + - name: General | APT Update apt: update_cache=yes tags: general @@ -15,17 +19,13 @@ apt: upgrade=full tags: general - - name: Node.js | Add the NodeSource Node.js repo - command: "bash -c 'curl -sL https://deb.nodesource.com/setup_iojs_3.x | bash -'" - tags: node - - name: General | Install required packages apt: name={{ item }} update_cache=yes state=latest with_items: packages tags: general - - name: User | Add {{ server_user }} user - user: name="{{ server_user }}" shell=/bin/bash + - name: User | Add nodejs user + user: name="nodejs" shell=/bin/bash tags: user - name: User | Download pubkey(s) @@ -39,11 +39,19 @@ with_items: ssh_users tags: user - - name: General | Create authorized_keys for {{ server_user }} - authorized_key: user="{{ server_user }}" key="{{ lookup('file', '/tmp/' + item + '.keys') }}" + - name: General | Create authorized_keys for nodejs + authorized_key: user="nodejs" key="{{ lookup('file', '/tmp/' + item + '.keys') }}" with_items: ssh_users tags: user + - name: Docker | Add the Docker.io repo + command: "bash -c 'curl -sL http://get.docker.io/ | bash -'" + tags: docker + + - name: Docker | Add {{ system_user }} to docker group + command: usermod -aG docker nodejs + tags: docker + - name: GitHub Webhook | Install github-webhook command: "npm install github-webhook -g" tags: webhook @@ -56,40 +64,36 @@ replace: dest=/etc/github-webhook.json regexp="\{\{github_secret\}\}" replace="{{ github_secret }}" tags: webhook - - name: GitHub Webhook | Copy update nodejs command to config - replace: dest=/etc/github-webhook.json regexp="\{\{update_nodejs_command\}\}" replace="{{ update_nodejs_command }}" - tags: webhook - - - name: GitHub Webhook | Copy update iojs command to config - replace: dest=/etc/github-webhook.json regexp="\{\{update_iojs_command\}\}" replace="{{ update_iojs_command }}" - tags: webhook - - name: GitHub Webhook | Copy Upstart config copy: src=./resources/github-webhook.conf dest=/etc/init/github-webhook.conf mode=0644 tags: webhook + - name: GitHub Webhook | Copy build-site scripts + copy: src=./resources/{{ item }} dest=/home/nodejs/{{ item }} mode=0700 + remote_user: "nodejs" + with_items: + - build-site.sh + - check-build-site.sh + tags: webhook + - name: GitHub Webhook | Start service service: name=github-webhook state=started tags: webhook - - name: Setup | Initial nodejs clone - remote_user: "{{ server_user }}" - command: "bash -c '{{ clone_nodejs_command }}'" + - name: Setup | Make /home/iojs + file: path=/home/iojs state=directory mode=0755 owner=nodejs tags: setup - - name: Setup | Initial iojs clone - remote_user: "{{ server_user }}" - command: "bash -c '{{ clone_iojs_command }}'" + - name: Setup | Make /home/nodejs/.npm + file: path=/home/nodejs/.npm state=directory mode=0755 owner=nodejs tags: setup - - name: Setup | Initial nodejs update - remote_user: "{{ server_user }}" - command: "bash -c '{{ update_nodejs_command }}'" - tags: setup - - - name: Setup | Initial iojs update - remote_user: "{{ server_user }}" - command: "bash -c '{{ update_iojs_command }}'" + - name: Setup | Initial nodejs and iojs clone and update + remote_user: "nodejs" + command: "{{ item }}" + with_items: + - "/home/nodejs/build-site.sh nodejs" + - "/home/nodejs/build-site.sh iojs" tags: setup - name: nginx | Copy site configs @@ -99,15 +103,16 @@ - iojs.org tags: nginx - - name: nginx | Create config symlinks - file: src=/etc/nginx/sites-available/{{ item }} dest=/etc/nginx/sites-enabled/00-{{ item }} state=link - with_items: - - nodejs.org - - iojs.org + - name: nginx | Create nodejs config symlink + file: src=/etc/nginx/sites-available/nodejs.org dest=/etc/nginx/sites-enabled/00-nodejs.org state=link + tags: nginx + + - name: nginx | Create iojs config symlink + file: src=/etc/nginx/sites-available/iojs.org dest=/etc/nginx/sites-enabled/00-iojs.org state=link tags: nginx - - name: nginx | Generate DH params - command: "bash -c 'mkdir -p /etc/nginx/ssl/ && openssl dhparam -out /etc/nginx/ssl/dhparam.pem 4096'" + - name: Setup | Make /etc/nginx/ssl/ + file: path=/etc/nginx/ssl/ state=directory mode=0755 owner=root tags: nginx - name: nginx | Copy site certificates @@ -117,6 +122,7 @@ - iojs_chained.crt - nodejs.key - iojs.key + - dhparam.pem tags: nginx - name: nginx | Delete default config @@ -137,4 +143,4 @@ - name: nginx | Restart service service: name=nginx state=restarted - tags: webhook + tags: nginx diff --git a/setup/www/ansible-vars.yaml b/setup/www/ansible-vars.yaml index 580bd9c33..1fa07f5fb 100644 --- a/setup/www/ansible-vars.yaml +++ b/setup/www/ansible-vars.yaml @@ -2,10 +2,9 @@ server_user: nodejs ssh_users: - rvagg - - indutny - jbergstroem - orangemocha packages: - - nodejs + - iojs - nginx - git diff --git a/setup/www/host_vars/.gitignore b/setup/www/host_vars/.gitignore index 35bd93942..549628fb0 100644 --- a/setup/www/host_vars/.gitignore +++ b/setup/www/host_vars/.gitignore @@ -1 +1 @@ -nodejs.org \ No newline at end of file +node-www \ No newline at end of file diff --git a/setup/www/host_vars/node-www.tmpl b/setup/www/host_vars/node-www.tmpl new file mode 100644 index 000000000..597817397 --- /dev/null +++ b/setup/www/host_vars/node-www.tmpl @@ -0,0 +1,2 @@ +--- +github_secret: "INSERT SECRET FROM WEBHOOK HERE" diff --git a/setup/www/host_vars/nodejs.org.tmpl b/setup/www/host_vars/nodejs.org.tmpl deleted file mode 100644 index 833a7ebd6..000000000 --- a/setup/www/host_vars/nodejs.org.tmpl +++ /dev/null @@ -1,4 +0,0 @@ ---- -github_secret: "INSERT SECRET FROM WEBHOOK HERE" -clone_command: "cd /home/nodejs/ && rm -rf new.nodejs.org.github && git clone https://github.com/nodejs/new.nodejs.org.git new.nodejs.org.github" -update_command: "cd /home/nodejs/new.nodejs.org.github/ && git checkout master && git reset --hard && git clean -fdx && git pull origin master && rsync -avz --delete --exclude .git /home/nodejs/new.nodejs.org.github/public/ /home/nodejs/www/" diff --git a/setup/www/resources/.gitignore b/setup/www/resources/.gitignore index be870b428..99faad5d7 100644 --- a/setup/www/resources/.gitignore +++ b/setup/www/resources/.gitignore @@ -1,2 +1,3 @@ *.crt *.key +*.pem \ No newline at end of file diff --git a/setup/www/resources/build-site.sh b/setup/www/resources/build-site.sh new file mode 100755 index 000000000..2be0b8c03 --- /dev/null +++ b/setup/www/resources/build-site.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +set -e + +site=$1 + +if [ "X$site" != "Xiojs" ] && [ "X$site" != "Xnodejs" ]; then + echo "Usage: build-site.sh < iojs | nodejs >" + exit 1 +fi + +pidof -s -o '%PPID' -x $(basename $0) > /dev/null 2>&1 && \ + echo "$(basename $0) already running" && \ + exit 1 + +clonedir=/home/${site}/website.github + +if [ ! -d "${clonedir}" ]; then + repo="${site}.org" + #TODO: remove this when repo is renamed + if [ "$site" == "nodejs" ]; then + repo="new.${site}.org" + fi + git clone https://github.com/nodejs/${repo}.git $clonedir +fi + +if [ "$site" == "nodejs" ]; then + build_cmd="npm run build" + rsync_from="build/" +else + build_cmd="node_modules/.bin/gulp build" + rsync_from="public/" +fi + +cd $clonedir +git reset --hard +git clean -fdx +git fetch origin +git checkout origin/master + +docker pull iojs:latest +docker run \ + --rm \ + -v ${clonedir}:/website/ \ + -v /home/nodejs/.npm:/npm/ \ + iojs:latest \ + bash -c " \ + addgroup nodejs --gid 1000 && \ + adduser nodejs --uid 1000 --gid 1000 --gecos nodejs --disabled-password && \ + su nodejs -c ' \ + npm config set loglevel http && \ + npm config set cache /npm/ && \ + cd /website/ && \ + npm install --cache-min 1440 --production && \ + $build_cmd \ + ' \ + " + +rsync -avz --delete --exclude .git ${clonedir}/${rsync_from} /home/${site}/www/ diff --git a/setup/www/resources/check-build-site.sh b/setup/www/resources/check-build-site.sh new file mode 100755 index 000000000..62cf33bee --- /dev/null +++ b/setup/www/resources/check-build-site.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +site=$1 + +if [ "X$site" != "Xiojs" ] && [ "X$site" != "Xnodejs" ]; then + echo "Usage: check-build-site.sh < iojs | nodejs >" + exit 1 +fi + +indexjson=/home/${site}/dist/public/release/index.json +indexhtml=/home/${site}//www/en/index.html +buildsite=/home/nodejs/build-site.sh + +[ $indexjson -nt $indexhtml ] && $buildsite $site \ No newline at end of file diff --git a/setup/www/resources/github-webhook.conf b/setup/www/resources/github-webhook.conf index 58edffa33..d09aaafe4 100644 --- a/setup/www/resources/github-webhook.conf +++ b/setup/www/resources/github-webhook.conf @@ -4,6 +4,6 @@ stop on runlevel [!2345] respawn respawn limit 10 5 -setuid iojs +setuid nodejs exec github-webhook --config /etc/github-webhook.json diff --git a/setup/www/resources/github-webhook.json b/setup/www/resources/github-webhook.json index 819535753..9b0325705 100644 --- a/setup/www/resources/github-webhook.json +++ b/setup/www/resources/github-webhook.json @@ -2,17 +2,17 @@ "port": 9999, "path": "/webhook", "secret": "{{github_secret}}", - "log": "/var/log/github-webhook.log", + "log": "/home/nodejs/github-webhook.log", "rules": [ { "event": "push", "match": "ref == \"refs/heads/master\" && repository.full_name == \"nodejs/new.nodejs.org\"", - "exec": "{{update_nodejs_command}}" + "exec": "/home/nodejs/build-site.sh nodejs" }, { "event": "push", "match": "ref == \"refs/heads/master\" && repository.full_name == \"nodejs/iojs.org\"", - "exec": "{{update_iojs_command}}" + "exec": "/home/nodejs/build-site.sh iojs" } ] } diff --git a/setup/www/resources/iojs.org b/setup/www/resources/iojs.org index f7f4a3887..4fb5eb28f 100644 --- a/setup/www/resources/iojs.org +++ b/setup/www/resources/iojs.org @@ -6,7 +6,7 @@ server { server { listen 443 ssl spdy; - listen [::]:443 ipv6only=on ssl spdy; + #listen [::]:443 ipv6only=on ssl spdy; server_name iojs.org; @@ -38,10 +38,6 @@ server { add_header X-Frame-Options DENY; add_header X-Content-Type-Options nosniff; - log_format main '$remote_addr - $remote_user [$time_local] $request ' - '"$status" $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/iojs.org-access.log; error_log /var/log/nginx/iojs.org-error.log; diff --git a/setup/www/resources/nodejs.org b/setup/www/resources/nodejs.org index 0179518e6..e965d9143 100644 --- a/setup/www/resources/nodejs.org +++ b/setup/www/resources/nodejs.org @@ -1,5 +1,5 @@ server { - listen 80; + listen 80 default_server; listen [::]:80; return 301 https://nodejs.org$request_uri; @@ -33,8 +33,8 @@ server { } server { - listen 443 ssl spdy; - listen [::]:443 ipv6only=on ssl spdy; + listen 443 default_server ssl spdy; + listen [::]:443 default_server ipv6only=on ssl spdy; server_name nodejs.org; @@ -66,10 +66,6 @@ server { add_header X-Frame-Options DENY; add_header X-Content-Type-Options nosniff; - log_format main '$remote_addr - $remote_user [$time_local] $request ' - '"$status" $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/nodejs.org-access.log; error_log /var/log/nginx/nodejs.org-error.log; @@ -99,6 +95,7 @@ server { root /home/nodejs/www; index index.html; default_type text/plain; + rewrite ^/$ /en/ redirect; } location /download { diff --git a/setup/www/resources/old_mime.types b/setup/www/resources/old_mime.types new file mode 100644 index 000000000..325b58575 --- /dev/null +++ b/setup/www/resources/old_mime.types @@ -0,0 +1,74 @@ + +types { + text/html html htm shtml; + text/css css; + text/xml xml; + image/gif gif; + image/jpeg jpeg jpg; + application/x-javascript js; + application/json json; + application/atom+xml atom; + application/rss+xml rss; + + text/mathml mml; + text/plain txt md markdown asc; + text/vnd.sun.j2me.app-descriptor jad; + text/vnd.wap.wml wml; + text/x-component htc; + + image/png png; + image/tiff tif tiff; + image/vnd.wap.wbmp wbmp; + image/x-icon ico; + image/x-jng jng; + image/x-ms-bmp bmp; + image/svg+xml svg; + + application/java-archive jar war ear; + application/mac-binhex40 hqx; + application/msword doc; + application/pdf pdf; + application/postscript ps eps ai; + application/rtf rtf; + application/vnd.ms-excel xls; + application/vnd.ms-powerpoint ppt; + application/vnd.wap.wmlc wmlc; + application/vnd.wap.xhtml+xml xhtml; + application/vnd.google-earth.kml+xml kml; + application/vnd.google-earth.kmz kmz; + application/x-cocoa cco; + application/x-java-archive-diff jardiff; + application/x-java-jnlp-file jnlp; + application/x-makeself run; + application/x-perl pl pm; + application/x-pilot prc pdb; + application/x-rar-compressed rar; + application/x-redhat-package-manager rpm; + application/x-sea sea; + application/x-shockwave-flash swf; + application/x-stuffit sit; + application/x-tcl tcl tk; + application/x-x509-ca-cert der pem crt; + application/x-xpinstall xpi; + application/zip zip; + + application/octet-stream bin exe dll; + application/octet-stream deb; + application/octet-stream dmg; + application/octet-stream eot; + application/octet-stream iso img; + application/octet-stream msi msp msm; + + audio/midi mid midi kar; + audio/mpeg mp3; + audio/x-realaudio ra; + + video/3gpp 3gpp 3gp; + video/mpeg mpeg mpg; + video/quicktime mov; + video/x-flv flv; + video/x-mng mng; + video/x-ms-asf asx asf; + video/x-ms-wmv wmv; + video/x-msvideo avi; +} From 48a04bbc285295d433e5da2386012dacc754a906 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Sun, 30 Aug 2015 17:33:17 +1000 Subject: [PATCH 04/15] modularise ansible, added cloudfuse & more --- setup/www/ansible-playbook.yaml | 139 ++---------------- setup/www/ansible-vars.yaml | 8 +- setup/www/host_vars/node-www.tmpl | 6 + setup/www/resources/config/.cloudfuse | 3 + .../{ => config}/github-webhook.conf | 0 .../{ => config}/github-webhook.json | 0 setup/www/resources/{ => config}/iojs.org | 0 setup/www/resources/{ => config}/nodejs.org | 0 .../www/resources/{ => config}/old_mime.types | 0 .../{ => config}/old_nodejs.org.conf | 0 .../www/resources/{ => scripts}/build-site.sh | 2 + setup/www/resources/scripts/cdn-purge.sh | 45 ++++++ .../{ => scripts}/check-build-site.sh | 0 .../www/resources/scripts/queue-cdn-purge.sh | 13 ++ setup/www/resources/secrets/makesecrets.sh | 44 ++++++ setup/www/tasks/base.yaml | 19 +++ setup/www/tasks/cloudfuse.yaml | 41 ++++++ setup/www/tasks/docker.yaml | 7 + setup/www/tasks/nginx.yaml | 63 ++++++++ setup/www/tasks/site-setup.yaml | 71 +++++++++ setup/www/tasks/user.yaml | 72 +++++++++ setup/www/tasks/webhook.yaml | 28 ++++ 22 files changed, 430 insertions(+), 131 deletions(-) create mode 100644 setup/www/resources/config/.cloudfuse rename setup/www/resources/{ => config}/github-webhook.conf (100%) rename setup/www/resources/{ => config}/github-webhook.json (100%) rename setup/www/resources/{ => config}/iojs.org (100%) rename setup/www/resources/{ => config}/nodejs.org (100%) rename setup/www/resources/{ => config}/old_mime.types (100%) rename setup/www/resources/{ => config}/old_nodejs.org.conf (100%) rename setup/www/resources/{ => scripts}/build-site.sh (97%) create mode 100644 setup/www/resources/scripts/cdn-purge.sh rename setup/www/resources/{ => scripts}/check-build-site.sh (100%) create mode 100644 setup/www/resources/scripts/queue-cdn-purge.sh create mode 100755 setup/www/resources/secrets/makesecrets.sh create mode 100644 setup/www/tasks/base.yaml create mode 100644 setup/www/tasks/cloudfuse.yaml create mode 100644 setup/www/tasks/docker.yaml create mode 100644 setup/www/tasks/nginx.yaml create mode 100644 setup/www/tasks/site-setup.yaml create mode 100644 setup/www/tasks/user.yaml create mode 100644 setup/www/tasks/webhook.yaml diff --git a/setup/www/ansible-playbook.yaml b/setup/www/ansible-playbook.yaml index f2d58b1bc..84eee627b 100644 --- a/setup/www/ansible-playbook.yaml +++ b/setup/www/ansible-playbook.yaml @@ -1,146 +1,25 @@ --- - hosts: node-www - remote_user: root tasks: - include_vars: ansible-vars.yaml tags: vars - - name: Node.js | Add the NodeSource Node.js repo - command: "bash -c 'curl -sL https://deb.nodesource.com/setup_iojs_3.x | bash -'" - tags: general - - - name: General | APT Update - apt: update_cache=yes - tags: general - - - name: General | APT Upgrade - apt: upgrade=full - tags: general - - - name: General | Install required packages - apt: name={{ item }} update_cache=yes state=latest - with_items: packages - tags: general - - - name: User | Add nodejs user - user: name="nodejs" shell=/bin/bash + - include: tasks/user.yaml tags: user - - name: User | Download pubkey(s) - get_url: url=https://github.com/{{ item }}.keys dest=/tmp/{{ item }}.keys - delegate_to: 127.0.0.1 - with_items: ssh_users - tags: user - - - name: General | Create authorized_keys for root - authorized_key: user="root" key="{{ lookup('file', '/tmp/' + item + '.keys') }}" - with_items: ssh_users - tags: user - - - name: General | Create authorized_keys for nodejs - authorized_key: user="nodejs" key="{{ lookup('file', '/tmp/' + item + '.keys') }}" - with_items: ssh_users - tags: user - - - name: Docker | Add the Docker.io repo - command: "bash -c 'curl -sL http://get.docker.io/ | bash -'" - tags: docker - - - name: Docker | Add {{ system_user }} to docker group - command: usermod -aG docker nodejs + - include: tasks/docker.yaml tags: docker - - name: GitHub Webhook | Install github-webhook - command: "npm install github-webhook -g" - tags: webhook - - - name: GitHub Webhook | Copy config - copy: src=./resources/github-webhook.json dest=/etc/github-webhook.json mode=0644 - tags: webhook - - - name: GitHub Webhook | Copy secret to config - replace: dest=/etc/github-webhook.json regexp="\{\{github_secret\}\}" replace="{{ github_secret }}" - tags: webhook - - - name: GitHub Webhook | Copy Upstart config - copy: src=./resources/github-webhook.conf dest=/etc/init/github-webhook.conf mode=0644 - tags: webhook - - - name: GitHub Webhook | Copy build-site scripts - copy: src=./resources/{{ item }} dest=/home/nodejs/{{ item }} mode=0700 - remote_user: "nodejs" - with_items: - - build-site.sh - - check-build-site.sh - tags: webhook - - - name: GitHub Webhook | Start service - service: name=github-webhook state=started - tags: webhook - - - name: Setup | Make /home/iojs - file: path=/home/iojs state=directory mode=0755 owner=nodejs - tags: setup - - - name: Setup | Make /home/nodejs/.npm - file: path=/home/nodejs/.npm state=directory mode=0755 owner=nodejs - tags: setup - - - name: Setup | Initial nodejs and iojs clone and update - remote_user: "nodejs" - command: "{{ item }}" - with_items: - - "/home/nodejs/build-site.sh nodejs" - - "/home/nodejs/build-site.sh iojs" + - include: tasks/site-setup.yaml tags: setup - - name: nginx | Copy site configs - copy: src=./resources/{{ item }} dest=/etc/nginx/sites-available/{{ item }} mode=0644 - with_items: - - nodejs.org - - iojs.org - tags: nginx - - - name: nginx | Create nodejs config symlink - file: src=/etc/nginx/sites-available/nodejs.org dest=/etc/nginx/sites-enabled/00-nodejs.org state=link - tags: nginx - - - name: nginx | Create iojs config symlink - file: src=/etc/nginx/sites-available/iojs.org dest=/etc/nginx/sites-enabled/00-iojs.org state=link - tags: nginx - - - name: Setup | Make /etc/nginx/ssl/ - file: path=/etc/nginx/ssl/ state=directory mode=0755 owner=root - tags: nginx - - - name: nginx | Copy site certificates - copy: src=./resources/{{ item }} dest=/etc/nginx/ssl/{{ item }} mode=0644 - with_items: - - nodejs_chained.crt - - iojs_chained.crt - - nodejs.key - - iojs.key - - dhparam.pem - tags: nginx + - include: tasks/cloudfuse.yaml + tags: cloudfuse - - name: nginx | Delete default config - file: path=/etc/nginx/sites-enabled/default state=absent - tags: nginx - - - name: nginx | Add .pkg mime-type - lineinfile: dest=/etc/nginx/mime.types line='application/octet-stream pkg;' insertafter='^types.*' - tags: nginx - - - name: nginx | Add .xz mime-type - lineinfile: dest=/etc/nginx/mime.types line='application/x-xz xz;' insertafter='^types.*' - tags: nginx - - - name: nginx | Use official .gz mime-type - lineinfile: dest=/etc/nginx/mime.types line='application/gzip gz;' insertafter='^types.*' - tags: nginx + - include: tasks/webhook.yaml + tags: webhook - - name: nginx | Restart service - service: name=nginx state=restarted - tags: nginx + - include: tasks/nginx.yaml + tags: nginx \ No newline at end of file diff --git a/setup/www/ansible-vars.yaml b/setup/www/ansible-vars.yaml index 1fa07f5fb..8ffb8fda4 100644 --- a/setup/www/ansible-vars.yaml +++ b/setup/www/ansible-vars.yaml @@ -1,9 +1,15 @@ --- server_user: nodejs -ssh_users: +root_users: - rvagg - jbergstroem - orangemocha +dist_users: + - cjihrig + - sam-github + - fishrock123 + - jasnell + - rvagg packages: - iojs - nginx diff --git a/setup/www/host_vars/node-www.tmpl b/setup/www/host_vars/node-www.tmpl index 597817397..b905f4d5e 100644 --- a/setup/www/host_vars/node-www.tmpl +++ b/setup/www/host_vars/node-www.tmpl @@ -1,2 +1,8 @@ --- github_secret: "INSERT SECRET FROM WEBHOOK HERE" +cdn_api_key: "INSERT CLOUDFLARE API KEY" +cdn_api_email: "INSERT CLOUDFLARE API EMAIL" +cdn_api_iojs_id: "INSERT CLOUDFLARE API iojs.org ID" +cdn_api_nodejs_id: "INSERT CLOUDFLARE API nodejs.org ID" +cloudfuse_user: "INSERT RACKSPACE CLOUD USERNAME" +cloudfuse_key: "INSERT RACKSPACE CLOUD API KEY" diff --git a/setup/www/resources/config/.cloudfuse b/setup/www/resources/config/.cloudfuse new file mode 100644 index 000000000..7b521149a --- /dev/null +++ b/setup/www/resources/config/.cloudfuse @@ -0,0 +1,3 @@ +username={{ cloudfuse_user }} +api_key={{ cloudfuse_key }} +region=IAD \ No newline at end of file diff --git a/setup/www/resources/github-webhook.conf b/setup/www/resources/config/github-webhook.conf similarity index 100% rename from setup/www/resources/github-webhook.conf rename to setup/www/resources/config/github-webhook.conf diff --git a/setup/www/resources/github-webhook.json b/setup/www/resources/config/github-webhook.json similarity index 100% rename from setup/www/resources/github-webhook.json rename to setup/www/resources/config/github-webhook.json diff --git a/setup/www/resources/iojs.org b/setup/www/resources/config/iojs.org similarity index 100% rename from setup/www/resources/iojs.org rename to setup/www/resources/config/iojs.org diff --git a/setup/www/resources/nodejs.org b/setup/www/resources/config/nodejs.org similarity index 100% rename from setup/www/resources/nodejs.org rename to setup/www/resources/config/nodejs.org diff --git a/setup/www/resources/old_mime.types b/setup/www/resources/config/old_mime.types similarity index 100% rename from setup/www/resources/old_mime.types rename to setup/www/resources/config/old_mime.types diff --git a/setup/www/resources/old_nodejs.org.conf b/setup/www/resources/config/old_nodejs.org.conf similarity index 100% rename from setup/www/resources/old_nodejs.org.conf rename to setup/www/resources/config/old_nodejs.org.conf diff --git a/setup/www/resources/build-site.sh b/setup/www/resources/scripts/build-site.sh similarity index 97% rename from setup/www/resources/build-site.sh rename to setup/www/resources/scripts/build-site.sh index 2be0b8c03..50a41ee86 100755 --- a/setup/www/resources/build-site.sh +++ b/setup/www/resources/scripts/build-site.sh @@ -57,3 +57,5 @@ docker run \ " rsync -avz --delete --exclude .git ${clonedir}/${rsync_from} /home/${site}/www/ + +/home/nodejs/queue-cdn-purge.sh $site diff --git a/setup/www/resources/scripts/cdn-purge.sh b/setup/www/resources/scripts/cdn-purge.sh new file mode 100644 index 000000000..804bf8512 --- /dev/null +++ b/setup/www/resources/scripts/cdn-purge.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e + +api_key="{{cdn_api_key}}" +api_email="{{cdn_api_email}}" +iojs_id="{{cdn_api_iojs_id}}" +nodejs_id="{{cdn_api_nodejs_id}}" + +site=$1 + +if [ "X$site" == "Xiojs" ]; then + zone_id="${iojs_id}" +elif [ "X$site" == "Xnodejs" ]; then + zone_id="${nodejs_id}" +elif [ "X$site" == "X" ]; then + # no arg? try both then + "$0" nodejs + "$0" iojs + exit 0 +else + echo "Usage: cdn-purge.sh < iojs | nodejs >" + exit 1 +fi + +if ! [ -f /tmp/cdnpurge.$site ]; then + exit 0 +fi + +rm -f /tmp/cdnpurge.$site + +# list zones: +# +#curl -X GET \ +# "https://api.cloudflare.com/client/v4/zones/" \ +# -H "X-Auth-Email: ${api_email}" \ +# -H "X-Auth-Key: ${api_key}" + +# purge full cache +curl -X DELETE \ + "https://api.cloudflare.com/client/v4/zones/${zone_id}/purge_cache" \ + -H "X-Auth-Email: ${api_email}" \ + -H "X-Auth-Key: ${api_key}" \ + -H "Content-Type: application/json" \ + --data '{"purge_everything":true}' diff --git a/setup/www/resources/check-build-site.sh b/setup/www/resources/scripts/check-build-site.sh similarity index 100% rename from setup/www/resources/check-build-site.sh rename to setup/www/resources/scripts/check-build-site.sh diff --git a/setup/www/resources/scripts/queue-cdn-purge.sh b/setup/www/resources/scripts/queue-cdn-purge.sh new file mode 100644 index 000000000..05f7094d9 --- /dev/null +++ b/setup/www/resources/scripts/queue-cdn-purge.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +site=$1 + +if [ "X$site" != "Xiojs" ] && [ "X$site" != "Xnodejs" ]; then + echo "Usage: queue-cdn-purge.sh < iojs | nodejs >" + exit 1 +fi + +umask 000 +touch /tmp/cdnpurge.$site diff --git a/setup/www/resources/secrets/makesecrets.sh b/setup/www/resources/secrets/makesecrets.sh new file mode 100755 index 000000000..1ff4bece8 --- /dev/null +++ b/setup/www/resources/secrets/makesecrets.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -e +secretsdir=$1 + +if ! [ -d $secretsdir ] || ! [ -d ${secretsdir}/.git ]; then + echo "Usage: makesecrets.sh " + exit 1 +fi + +function bork { + echo "Something borked, perhaps you got your passphrase wrong? ..." + exit 1 +} + +#if ! [ -f dhparam.pem ]; then +# echo "Generating 4096 dhparam, this may take some time ..." +# openssl dhparam -out dhparam.pem 4096 +#fi + +read -p "Enter GPG passphrase: " -s gpgpass + +echo "" + +gpgcmd="gpg --batch -q -q --passphrase '${gpgpass}'" + +echo "Extracting dhparam.pem..." +bash -c "$gpgcmd" < ${secretsdir}/build/release/dhparam.pem 1> ./dhparam.pem 2> /dev/null || bork + +echo "Extracting nodejs.key..." +bash -c "$gpgcmd" < ${secretsdir}/build/release/star_nodejs.org.key 1> ./nodejs.key 2> /dev/null || bork + +echo "Extracting iojs.key..." +bash -c "$gpgcmd" < ${secretsdir}/build/release/star_iojs.org.key 1> ./iojs.key 2> /dev/null || bork + +echo "Extracting nodejs_chained.crt..." +bash -c "$gpgcmd" < ${secretsdir}/build/release/star_nodejs.org.crt 1> ./nodejs_chained.crt 2> /dev/null || bork +bash -c "$gpgcmd" < ${secretsdir}/build/release/star_nodejs.org-COMODORSADomainValidationSecureServerCA.crt 1>> ./nodejs_chained.crt 2> /dev/null || bork +bash -c "$gpgcmd" < ${secretsdir}/build/release/star_nodejs.org-AddTrustExternalCARoot.crt 1>> ./nodejs_chained.crt 2> /dev/null || bork + +echo "Extracting iojs_chained.crt..." +bash -c "$gpgcmd" < ${secretsdir}/build/release/star_iojs.org.crt 1> ./iojs_chained.crt 2> /dev/null || bork +bash -c "$gpgcmd" < ${secretsdir}/build/release/star_iojs.org-COMODORSADomainValidationSecureServerCA.crt 1>> ./iojs_chained.crt 2> /dev/null || bork +bash -c "$gpgcmd" < ${secretsdir}/build/release/star_iojs.org-AddTrustExternalCARoot.crt 1>> ./iojs_chained.crt 2> /dev/null || bork diff --git a/setup/www/tasks/base.yaml b/setup/www/tasks/base.yaml new file mode 100644 index 000000000..9a12e6b25 --- /dev/null +++ b/setup/www/tasks/base.yaml @@ -0,0 +1,19 @@ +- name: Base | Add the NodeSource Node.js repo + command: "bash -c 'curl -sL https://deb.nodesource.com/setup_iojs_3.x | bash -'" + tags: base + +- name: Base | APT Update + apt: update_cache=yes + tags: base + +- name: Base | APT Upgrade + apt: upgrade=full + tags: base + +- name: Base | Install required packages + apt: + name: '{{ item }}' + update_cache: yes + state: latest + with_items: packages + tags: base diff --git a/setup/www/tasks/cloudfuse.yaml b/setup/www/tasks/cloudfuse.yaml new file mode 100644 index 000000000..d2d5786ad --- /dev/null +++ b/setup/www/tasks/cloudfuse.yaml @@ -0,0 +1,41 @@ +- name: CloudFuse | Install required packages for CloudFuse compile + apt: + name: "{{ item }}" + update_cache: yes + state: latest + with_items: + - build-essential + - libcurl4-openssl-dev + - libxml2 + - libxml2-dev + - libfuse-dev + - fuse + tags: cloudfuse + +- name: CloudFuse | Download source + shell: cd /usr/src/ && \ + curl -sL http://github.com/redbo/cloudfuse/tarball/master | tar -zx && \ + cd redbo-cloudfuse-*/ && \ + ./configure && \ + make && \ + make install + tags: cloudfuse + +- name: CloudFuse | Copy config file + copy: + src: ./resources/config/.cloudfuse + dest: /root/.cloudfuse + mode: 0600 + owner: root + group: root + tags: cloudfuse + +- name: CloudFuse | Add /etc/fstab entry + lineinfile: + dest: /etc/fstab + line: 'cloudfuse /home/nodejs/cloudfuse fuse auto,_netdev,defaults,uid=1000,gid=1000,umask=002,allow_other 0 0' + tags: cloudfuse + +- name: CloudFuse | Mount all + shell: mount -a + tags: cloudfuse diff --git a/setup/www/tasks/docker.yaml b/setup/www/tasks/docker.yaml new file mode 100644 index 000000000..953bb4f1c --- /dev/null +++ b/setup/www/tasks/docker.yaml @@ -0,0 +1,7 @@ +- name: Docker | Add the Docker.io repo + command: "bash -c 'curl -sL http://get.docker.io/ | bash -'" + tags: docker + +- name: Docker | Add {{ system_user }} to docker group + command: usermod -aG docker nodejs + tags: docker diff --git a/setup/www/tasks/nginx.yaml b/setup/www/tasks/nginx.yaml new file mode 100644 index 000000000..d279e36b2 --- /dev/null +++ b/setup/www/tasks/nginx.yaml @@ -0,0 +1,63 @@ +- name: nginx | Copy site configs + copy: + src: ./resources/config/{{ item }} + dest: /etc/nginx/sites-available/{{ item }} + mode: 0644 + with_items: + - nodejs.org + - iojs.org + tags: nginx + +- name: nginx | Create nginx config symlinks + file: + src: '/etc/nginx/sites-available/{{ item.src }}' + dest: '/etc/nginx/sites-enabled/{{ item.dest }}' + state: link + with_items: + - { src: nodejs.org, dest: 00-nodejs.org } + - { src: iojs.org, dest: 01-iojs.org } + tags: nginx + +- name: nginx | Make /etc/nginx/ssl/ + file: + path: /etc/nginx/ssl/ + state: directory + mode: 0755 + owner: root + tags: nginx + +- name: nginx | Copy site certificates + copy: + src: ./resources/secrets/{{ item }} + dest: /etc/nginx/ssl/{{ item }} + mode: 0644 + with_items: + - nodejs_chained.crt + - iojs_chained.crt + - nodejs.key + - iojs.key + - dhparam.pem + tags: nginx + +- name: nginx | Delete default config + file: + path: /etc/nginx/sites-enabled/default + state: absent + tags: nginx + +- name: nginx | Add .pkg mime-type + lineinfile: + dest: /etc/nginx/mime.types + line: '{{ item }}' + insertafter: '^types.*' + with_items: + - 'application/octet-stream pkg;' + - 'application/x-xz xz;' + - 'application/gzip gz;' + tags: nginx + +- name: nginx | Restart service + service: + name: nginx + state: restarted + tags: nginx diff --git a/setup/www/tasks/site-setup.yaml b/setup/www/tasks/site-setup.yaml new file mode 100644 index 000000000..4427cd9ce --- /dev/null +++ b/setup/www/tasks/site-setup.yaml @@ -0,0 +1,71 @@ +- name: Site Setup | Copy build-site scripts + copy: + src: ./resources/scripts/{{ item }} + dest: /home/nodejs/{{ item }} + mode: 0700 + owner: nodejs + group: nodejs + with_items: + - build-site.sh + - check-build-site.sh + tags: setup + +- name: Site Setup | Copy cdn-purge.sh script + copy: + src: ./resources/scripts/cdn-purge.sh + dest: /home/nodejs/cdn-purge.sh + mode: 0700 + owner: root + group: root + tags: setup + +- name: Site Setup | Replace cdn-purge.sh secrets + replace: + dest: /home/nodejs/cdn-purge.sh + regexp: "{{ item.regexp }}" + replace: "{{ item.replace }}" + with_items: + - { regexp: '\{\{cdn_api_key\}\}', replace: '{{ cdn_api_key }}' } + - { regexp: '\{\{cdn_api_email\}\}', replace: '{{ cdn_api_email }}' } + - { regexp: '\{\{cdn_api_iojs_id\}\}', replace: '{{ cdn_api_iojs_id }}' } + - { regexp: '\{\{cdn_api_nodejs_id\}\}', replace: '{{ cdn_api_nodejs_id }}' } + tags: setup + +- name: Site Setup | Add cdn-purge.sh to crontab + lineinfile: + dest: /etc/crontab + line: '* * * * * root /home/nodejs/cdn-purge.sh' + tags: setup + +- name: Site Setup | Copy queue-cdn-purge.sh script + copy: + src: ./resources/scripts/queue-cdn-purge.sh + dest: /home/nodejs/queue-cdn-purge.sh + mode: 0755 + owner: root + group: root + tags: setup + +- name: Site Setup | Make /home/iojs + file: + path: /home/iojs + state: directory + mode: 0755 + owner: nodejs + tags: setup + +- name: Site Setup | Make /home/nodejs/.npm + file: + path: /home/nodejs/.npm + state: directory + mode: 0755 + owner: nodejs + tags: setup + +- name: Site Setup | Initial nodejs and iojs clone and update + remote_user: "nodejs" + command: "{{ item }}" + with_items: + - "/home/nodejs/build-site.sh nodejs" + - "/home/nodejs/build-site.sh iojs" + tags: setup diff --git a/setup/www/tasks/user.yaml b/setup/www/tasks/user.yaml new file mode 100644 index 000000000..8ef09c649 --- /dev/null +++ b/setup/www/tasks/user.yaml @@ -0,0 +1,72 @@ +- name: User | Add nodejs, dist, staging users + user: + name: "{{ item }}" + shell: /bin/bash + with_items: + - nodejs + - dist + - staging + tags: user + +- name: User | Download pubkeys (root_users) + get_url: + url: https://github.com/{{ item }}.keys + dest: /tmp/{{ item }}.keys + delegate_to: 127.0.0.1 + with_items: root_users + tags: user + +- name: User | Download pubkeys (dist_users) + get_url: + url: https://github.com/{{ item }}.keys + dest: /tmp/{{ item }}.keys + delegate_to: 127.0.0.1 + with_items: dist_users + tags: user + +- name: User | Create authorized_keys for root + authorized_key: + user: "root" + key: "{{ lookup('file', '/tmp/' + item + '.keys') }}" + with_items: root_users + tags: user + +- name: User | Create authorized_keys for nodejs + authorized_key: + user: "nodejs" + key: "{{ lookup('file', '/tmp/' + item + '.keys') }}" + with_items: root_users + tags: user + +- name: User | Create authorized_keys for dist (root_users) + authorized_key: + user: "dist" + key: "{{ lookup('file', '/tmp/' + item + '.keys') }}" + with_items: root_users + tags: user + +- name: User | Create authorized_keys for dist (dist_users) + authorized_key: + user: "dist" + key: "{{ lookup('file', '/tmp/' + item + '.keys') }}" + with_items: dist_users + tags: user + +- name: User | Set up dist/{nodejs,iojs}/public directories + file: + path: /home/dist/{{ item }} + state: directory + mode: 0755 + owner: dist + with_items: + - nodejs/public/next-nightly + - nodejs/public/nightly + - nodejs/public/rc + - nodejs/public/release + - nodejs/public/test + - iojs/public/next-nightly + - iojs/public/nightly + - iojs/public/rc + - iojs/public/release + - iojs/public/test + tags: user diff --git a/setup/www/tasks/webhook.yaml b/setup/www/tasks/webhook.yaml new file mode 100644 index 000000000..f06c68e04 --- /dev/null +++ b/setup/www/tasks/webhook.yaml @@ -0,0 +1,28 @@ +- name: GitHub Webhook | Install github-webhook + command: "npm install github-webhook -g" + tags: webhook + +- name: GitHub Webhook | Copy config + copy: + src: ./resources/config/github-webhook.json + dest: /etc/github-webhook.json + mode: 0644 + tags: webhook + +- name: GitHub Webhook | Copy secret to config + replace: + dest: /etc/github-webhook.json + regexp: '\{\{github_secret\}\}' + replace: '{{ github_secret }}' + tags: webhook + +- name: GitHub Webhook | Copy Upstart config + copy: + src: ./resources/config/github-webhook.conf + dest: /etc/init/github-webhook.conf + mode: 0644 + tags: webhook + +- name: GitHub Webhook | Start service + service: name=github-webhook state=started + tags: webhook \ No newline at end of file From 810873682d9d53932c51cf35fea748e5d5e5ff0a Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Sun, 30 Aug 2015 18:51:04 +1000 Subject: [PATCH 05/15] move dist/web tools to www --- {tools/dist => setup/www/tools}/dist-indexer/LICENSE | 0 {tools/dist => setup/www/tools}/dist-indexer/README.md | 0 {tools/dist => setup/www/tools}/dist-indexer/decode-ref.js | 0 {tools/dist => setup/www/tools}/dist-indexer/dist-indexer.js | 0 {tools/dist => setup/www/tools}/dist-indexer/ls-types.js | 0 {tools/dist => setup/www/tools}/dist-indexer/package.json | 0 .../dist => setup/www/tools}/dist-indexer/transform-filename.js | 0 {tools => setup/www/tools}/nightly-builder/.gitignore | 0 {tools => setup/www/tools}/nightly-builder/build-required.js | 0 {tools => setup/www/tools}/nightly-builder/latest-build.js | 0 {tools => setup/www/tools}/nightly-builder/latest-commit.js | 0 {tools => setup/www/tools}/nightly-builder/nightly-builder.js | 0 {tools => setup/www/tools}/nightly-builder/package.json | 0 {tools => setup/www/tools}/nightly-builder/test.js | 0 {tools => setup/www/tools}/nightly-builder/trigger-build.js | 0 {tools/dist => setup/www/tools}/promote/README.md | 0 {tools/dist => setup/www/tools}/promote/_promote.sh | 0 {tools/dist => setup/www/tools}/promote/_resha.sh | 0 {tools/dist => setup/www/tools}/promote/promote_nightly.sh | 0 {tools/dist => setup/www/tools}/promote/promote_release.sh | 0 {tools/dist => setup/www/tools}/promote/resha_release.sh | 0 {tools/dist => setup/www/tools}/promote/settings | 0 {tools/dist => setup/www/tools}/update-download-stats.sh | 0 23 files changed, 0 insertions(+), 0 deletions(-) rename {tools/dist => setup/www/tools}/dist-indexer/LICENSE (100%) rename {tools/dist => setup/www/tools}/dist-indexer/README.md (100%) rename {tools/dist => setup/www/tools}/dist-indexer/decode-ref.js (100%) rename {tools/dist => setup/www/tools}/dist-indexer/dist-indexer.js (100%) rename {tools/dist => setup/www/tools}/dist-indexer/ls-types.js (100%) rename {tools/dist => setup/www/tools}/dist-indexer/package.json (100%) rename {tools/dist => setup/www/tools}/dist-indexer/transform-filename.js (100%) rename {tools => setup/www/tools}/nightly-builder/.gitignore (100%) rename {tools => setup/www/tools}/nightly-builder/build-required.js (100%) rename {tools => setup/www/tools}/nightly-builder/latest-build.js (100%) rename {tools => setup/www/tools}/nightly-builder/latest-commit.js (100%) rename {tools => setup/www/tools}/nightly-builder/nightly-builder.js (100%) rename {tools => setup/www/tools}/nightly-builder/package.json (100%) rename {tools => setup/www/tools}/nightly-builder/test.js (100%) rename {tools => setup/www/tools}/nightly-builder/trigger-build.js (100%) rename {tools/dist => setup/www/tools}/promote/README.md (100%) rename {tools/dist => setup/www/tools}/promote/_promote.sh (100%) rename {tools/dist => setup/www/tools}/promote/_resha.sh (100%) rename {tools/dist => setup/www/tools}/promote/promote_nightly.sh (100%) rename {tools/dist => setup/www/tools}/promote/promote_release.sh (100%) rename {tools/dist => setup/www/tools}/promote/resha_release.sh (100%) rename {tools/dist => setup/www/tools}/promote/settings (100%) rename {tools/dist => setup/www/tools}/update-download-stats.sh (100%) diff --git a/tools/dist/dist-indexer/LICENSE b/setup/www/tools/dist-indexer/LICENSE similarity index 100% rename from tools/dist/dist-indexer/LICENSE rename to setup/www/tools/dist-indexer/LICENSE diff --git a/tools/dist/dist-indexer/README.md b/setup/www/tools/dist-indexer/README.md similarity index 100% rename from tools/dist/dist-indexer/README.md rename to setup/www/tools/dist-indexer/README.md diff --git a/tools/dist/dist-indexer/decode-ref.js b/setup/www/tools/dist-indexer/decode-ref.js similarity index 100% rename from tools/dist/dist-indexer/decode-ref.js rename to setup/www/tools/dist-indexer/decode-ref.js diff --git a/tools/dist/dist-indexer/dist-indexer.js b/setup/www/tools/dist-indexer/dist-indexer.js similarity index 100% rename from tools/dist/dist-indexer/dist-indexer.js rename to setup/www/tools/dist-indexer/dist-indexer.js diff --git a/tools/dist/dist-indexer/ls-types.js b/setup/www/tools/dist-indexer/ls-types.js similarity index 100% rename from tools/dist/dist-indexer/ls-types.js rename to setup/www/tools/dist-indexer/ls-types.js diff --git a/tools/dist/dist-indexer/package.json b/setup/www/tools/dist-indexer/package.json similarity index 100% rename from tools/dist/dist-indexer/package.json rename to setup/www/tools/dist-indexer/package.json diff --git a/tools/dist/dist-indexer/transform-filename.js b/setup/www/tools/dist-indexer/transform-filename.js similarity index 100% rename from tools/dist/dist-indexer/transform-filename.js rename to setup/www/tools/dist-indexer/transform-filename.js diff --git a/tools/nightly-builder/.gitignore b/setup/www/tools/nightly-builder/.gitignore similarity index 100% rename from tools/nightly-builder/.gitignore rename to setup/www/tools/nightly-builder/.gitignore diff --git a/tools/nightly-builder/build-required.js b/setup/www/tools/nightly-builder/build-required.js similarity index 100% rename from tools/nightly-builder/build-required.js rename to setup/www/tools/nightly-builder/build-required.js diff --git a/tools/nightly-builder/latest-build.js b/setup/www/tools/nightly-builder/latest-build.js similarity index 100% rename from tools/nightly-builder/latest-build.js rename to setup/www/tools/nightly-builder/latest-build.js diff --git a/tools/nightly-builder/latest-commit.js b/setup/www/tools/nightly-builder/latest-commit.js similarity index 100% rename from tools/nightly-builder/latest-commit.js rename to setup/www/tools/nightly-builder/latest-commit.js diff --git a/tools/nightly-builder/nightly-builder.js b/setup/www/tools/nightly-builder/nightly-builder.js similarity index 100% rename from tools/nightly-builder/nightly-builder.js rename to setup/www/tools/nightly-builder/nightly-builder.js diff --git a/tools/nightly-builder/package.json b/setup/www/tools/nightly-builder/package.json similarity index 100% rename from tools/nightly-builder/package.json rename to setup/www/tools/nightly-builder/package.json diff --git a/tools/nightly-builder/test.js b/setup/www/tools/nightly-builder/test.js similarity index 100% rename from tools/nightly-builder/test.js rename to setup/www/tools/nightly-builder/test.js diff --git a/tools/nightly-builder/trigger-build.js b/setup/www/tools/nightly-builder/trigger-build.js similarity index 100% rename from tools/nightly-builder/trigger-build.js rename to setup/www/tools/nightly-builder/trigger-build.js diff --git a/tools/dist/promote/README.md b/setup/www/tools/promote/README.md similarity index 100% rename from tools/dist/promote/README.md rename to setup/www/tools/promote/README.md diff --git a/tools/dist/promote/_promote.sh b/setup/www/tools/promote/_promote.sh similarity index 100% rename from tools/dist/promote/_promote.sh rename to setup/www/tools/promote/_promote.sh diff --git a/tools/dist/promote/_resha.sh b/setup/www/tools/promote/_resha.sh similarity index 100% rename from tools/dist/promote/_resha.sh rename to setup/www/tools/promote/_resha.sh diff --git a/tools/dist/promote/promote_nightly.sh b/setup/www/tools/promote/promote_nightly.sh similarity index 100% rename from tools/dist/promote/promote_nightly.sh rename to setup/www/tools/promote/promote_nightly.sh diff --git a/tools/dist/promote/promote_release.sh b/setup/www/tools/promote/promote_release.sh similarity index 100% rename from tools/dist/promote/promote_release.sh rename to setup/www/tools/promote/promote_release.sh diff --git a/tools/dist/promote/resha_release.sh b/setup/www/tools/promote/resha_release.sh similarity index 100% rename from tools/dist/promote/resha_release.sh rename to setup/www/tools/promote/resha_release.sh diff --git a/tools/dist/promote/settings b/setup/www/tools/promote/settings similarity index 100% rename from tools/dist/promote/settings rename to setup/www/tools/promote/settings diff --git a/tools/dist/update-download-stats.sh b/setup/www/tools/update-download-stats.sh similarity index 100% rename from tools/dist/update-download-stats.sh rename to setup/www/tools/update-download-stats.sh From 6d40006c3ed138dcd4689e7714c572791a27efae Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Sun, 30 Aug 2015 23:08:24 +1000 Subject: [PATCH 06/15] moar updates, added dist-indexer --- setup/www/ansible-playbook.yaml | 9 ++- setup/www/resources/config/iojs.org | 6 +- setup/www/resources/config/nodejs.org | 58 +++++++++----- .../resources/scripts/restore-cloudfuse.sh | 12 +++ .../www/resources/scripts/rsync-cloudfuse.sh | 14 ++++ setup/www/resources/secrets/makesecrets.sh | 10 ++- setup/www/tasks/cloudfuse.yaml | 9 +++ setup/www/tasks/dist-indexer.yaml | 29 +++++++ setup/www/tasks/nginx.yaml | 2 +- setup/www/tasks/nightly-builder.yaml | 20 +++++ setup/www/tools/dist-indexer/decode-ref.js | 2 + setup/www/tools/dist-indexer/dist-indexer.js | 79 ++++++++++++++----- setup/www/tools/dist-indexer/package.json | 3 +- .../tools/dist-indexer/transform-filename.js | 17 +++- 14 files changed, 222 insertions(+), 48 deletions(-) create mode 100644 setup/www/resources/scripts/restore-cloudfuse.sh create mode 100644 setup/www/resources/scripts/rsync-cloudfuse.sh create mode 100644 setup/www/tasks/dist-indexer.yaml create mode 100644 setup/www/tasks/nightly-builder.yaml diff --git a/setup/www/ansible-playbook.yaml b/setup/www/ansible-playbook.yaml index 84eee627b..69da653f7 100644 --- a/setup/www/ansible-playbook.yaml +++ b/setup/www/ansible-playbook.yaml @@ -22,4 +22,11 @@ tags: webhook - include: tasks/nginx.yaml - tags: nginx \ No newline at end of file + tags: nginx + + - include: tasks/dist-indexer.yaml + tags: dist-indexer + + - include: tasks/nightly-builder.yaml + tags: nightly-builder + diff --git a/setup/www/resources/config/iojs.org b/setup/www/resources/config/iojs.org index 4fb5eb28f..c5b6f62b3 100644 --- a/setup/www/resources/config/iojs.org +++ b/setup/www/resources/config/iojs.org @@ -66,19 +66,19 @@ server { } location /download { - alias /home/dist/public; + alias /home/dist/iojs/public; autoindex on; default_type text/plain; } location /dist { - alias /home/dist/public/release/; + alias /home/dist/iojs/public/release/; autoindex on; default_type text/plain; } location /api { - alias /home/dist/public/release/latest/doc/api; + alias /home/dist/iojs/public/release/latest/doc/api; autoindex on; default_type text/plain; } diff --git a/setup/www/resources/config/nodejs.org b/setup/www/resources/config/nodejs.org index e965d9143..454ecd373 100644 --- a/setup/www/resources/config/nodejs.org +++ b/setup/www/resources/config/nodejs.org @@ -79,16 +79,8 @@ server { rewrite ^(.*)$ https://nodejs.org$1; } - location ~ \.json$ { - add_header access-control-allow-origin *; - } - - location ~ /(dist|docs)/v[0-9]+\.[0-9]+\.[0-9]+/.*\.(tar\.gz|msi|exe|pdb|lib|html|json) { - expires max; - } - location /documentation/ { - rewrite ^/documentation/api(.*)$ /api$1 permanent; + rewrite ^/documentation/api(.*)$ /api$1 permanent; } location / { @@ -96,27 +88,55 @@ server { index index.html; default_type text/plain; rewrite ^/$ /en/ redirect; + + location ~ \.json$ { + add_header access-control-allow-origin *; + } + + location ~ /(dist|docs)/v[0-9]+\.[0-9]+\.[0-9]+/.*\.(tar\.gz|msi|exe|pdb|lib|html|json) { + expires max; + } } location /download { - alias /home/dist/public; + alias /home/dist/nodejs/public; autoindex on; default_type text/plain; + + location ~ \.json$ { + add_header access-control-allow-origin *; + } + + location ~ /(dist|docs)/v[0-9]+\.[0-9]+\.[0-9]+/.*\.(tar\.gz|msi|exe|pdb|lib|html|json) { + expires max; + } } location /dist { - alias /home/dist/public/release/; - autoindex on; - default_type text/plain; + alias /home/dist/nodejs/public/release/; + autoindex on; + default_type text/plain; + + location ~ \.json$ { + add_header access-control-allow-origin *; + } + + location ~ /(dist|docs)/v[0-9]+\.[0-9]+\.[0-9]+/.*\.(tar\.gz|msi|exe|pdb|lib|html|json) { + expires max; + } } location /api { - alias /home/dist/public/release/latest/doc/api; - autoindex on; - default_type text/plain; - } + alias /home/dist/nodejs/public/release/latest/doc/api; + autoindex on; + default_type text/plain; + + location ~ \.json$ { + add_header access-control-allow-origin *; + } - location /download-stats.json { - alias /home/iojs/download-stats.json; + location ~ /(dist|docs)/v[0-9]+\.[0-9]+\.[0-9]+/.*\.(tar\.gz|msi|exe|pdb|lib|html|json) { + expires max; + } } } diff --git a/setup/www/resources/scripts/restore-cloudfuse.sh b/setup/www/resources/scripts/restore-cloudfuse.sh new file mode 100644 index 000000000..c13265872 --- /dev/null +++ b/setup/www/resources/scripts/restore-cloudfuse.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +for d in nodejs iojs; do + srcdir=/home/nodejs/cloudfuse/${d}.org/public/release/ + dstdir=/home/dist/${d}/public/release/ + timesfile=/home/nodejs/cloudfuse/${d}.org/public.times + + mkdir -p $dstdir + rsync -avz $srcdir $dstdir --ignore-times --size-only + cat $timesfile | awk -F'|' '{ print "touch -d\""$2"\" \""$1"\"" }' + find $srddir -fprintf $timesfile '%p|%TY-%Tm-%Td %TT\n' +done diff --git a/setup/www/resources/scripts/rsync-cloudfuse.sh b/setup/www/resources/scripts/rsync-cloudfuse.sh new file mode 100644 index 000000000..b65a8a98c --- /dev/null +++ b/setup/www/resources/scripts/rsync-cloudfuse.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +for d in nodejs iojs; do + srcdir=/home/dist/${d}/public/release/ + dstdir=/home/nodejs/cloudfuse/${d}.org/public/release/ + timesfile=/home/nodejs/cloudfuse/${d}.org/public.times + + mkdir -p $dstdir + rsync \ + -avz --delete --ignore-times --size-only \ + --exclude=rc/ --exclude=nightly/ --exclude=next-nightly/ \ + $srcdir $dstdir + find $srddir -fprintf %timesfile '%p|%TY-%Tm-%Td %TT\n' +done diff --git a/setup/www/resources/secrets/makesecrets.sh b/setup/www/resources/secrets/makesecrets.sh index 1ff4bece8..fec41a14b 100755 --- a/setup/www/resources/secrets/makesecrets.sh +++ b/setup/www/resources/secrets/makesecrets.sh @@ -35,10 +35,16 @@ bash -c "$gpgcmd" < ${secretsdir}/build/release/star_iojs.org.key 1> ./iojs.key echo "Extracting nodejs_chained.crt..." bash -c "$gpgcmd" < ${secretsdir}/build/release/star_nodejs.org.crt 1> ./nodejs_chained.crt 2> /dev/null || bork +echo "" >> ./nodejs_chained.crt bash -c "$gpgcmd" < ${secretsdir}/build/release/star_nodejs.org-COMODORSADomainValidationSecureServerCA.crt 1>> ./nodejs_chained.crt 2> /dev/null || bork -bash -c "$gpgcmd" < ${secretsdir}/build/release/star_nodejs.org-AddTrustExternalCARoot.crt 1>> ./nodejs_chained.crt 2> /dev/null || bork +echo "" >> ./nodejs_chained.crt +bash -c "$gpgcmd" < ${secretsdir}/build/release/star_nodejs.org-COMODORSAAddTrustCA.crt 1>> ./nodejs_chained.crt 2> /dev/null || bork +echo "" >> ./nodejs_chained.crt echo "Extracting iojs_chained.crt..." bash -c "$gpgcmd" < ${secretsdir}/build/release/star_iojs.org.crt 1> ./iojs_chained.crt 2> /dev/null || bork +echo "" >> ./iojs_chained.crt bash -c "$gpgcmd" < ${secretsdir}/build/release/star_iojs.org-COMODORSADomainValidationSecureServerCA.crt 1>> ./iojs_chained.crt 2> /dev/null || bork -bash -c "$gpgcmd" < ${secretsdir}/build/release/star_iojs.org-AddTrustExternalCARoot.crt 1>> ./iojs_chained.crt 2> /dev/null || bork +echo "" >> ./iojs_chained.crt +bash -c "$gpgcmd" < ${secretsdir}/build/release/star_iojs.org-COMODORSAAddTrustCA.crt 1>> ./iojs_chained.crt 2> /dev/null || bork +echo "" >> ./iojs_chained.crt diff --git a/setup/www/tasks/cloudfuse.yaml b/setup/www/tasks/cloudfuse.yaml index d2d5786ad..7a3617c49 100644 --- a/setup/www/tasks/cloudfuse.yaml +++ b/setup/www/tasks/cloudfuse.yaml @@ -39,3 +39,12 @@ - name: CloudFuse | Mount all shell: mount -a tags: cloudfuse + +- name: CloudFuse | Copy backup script + copy: + src: ./resources/scripts/rsync-cloudfuse.sh + dest: /home/nodejs/rsync-cloudfuse.sh + mode: 0700 + owner: nodejs + group: nodejs + tags: cloudfuse diff --git a/setup/www/tasks/dist-indexer.yaml b/setup/www/tasks/dist-indexer.yaml new file mode 100644 index 000000000..771f92e63 --- /dev/null +++ b/setup/www/tasks/dist-indexer.yaml @@ -0,0 +1,29 @@ +- name: dist-indexer | Make /home/dist/tools/dist-indexer + file: + path: /home/dist/tools/dist-indexer + state: directory + mode: 0755 + owner: dist + group: dist + tags: setup + +- name: dist-indexer | Copy source files + copy: + src: tools/dist-indexer/ + dest: /home/dist/tools/dist-indexer/ + owner: dist + group: dist + tags: dist-indexer + +- name: dist-indexer | Set executables + file: + path: /home/dist/tools/dist-indexer/{{ item }} + mode: 0755 + with_items: + - dist-indexer.js + - ls-types.js + +- name: dist-indexer | npm install + shell: cd /home/dist/tools/dist-indexer/ && npm install + remote_user: dist + tags: dist-indexer diff --git a/setup/www/tasks/nginx.yaml b/setup/www/tasks/nginx.yaml index d279e36b2..ca6ed766e 100644 --- a/setup/www/tasks/nginx.yaml +++ b/setup/www/tasks/nginx.yaml @@ -45,7 +45,7 @@ state: absent tags: nginx -- name: nginx | Add .pkg mime-type +- name: nginx | Add extra mime-type lineinfile: dest: /etc/nginx/mime.types line: '{{ item }}' diff --git a/setup/www/tasks/nightly-builder.yaml b/setup/www/tasks/nightly-builder.yaml new file mode 100644 index 000000000..f5967996d --- /dev/null +++ b/setup/www/tasks/nightly-builder.yaml @@ -0,0 +1,20 @@ +- name: Site Setup | Make /home/dist/tools/nightly-builder + file: + path: /home/dist/tools/nightly-builder + state: directory + mode: 0755 + owner: dist + group: dist + tags: setup + +- name: nightly-builder | Copy source files + copy: + src: tools/nightly-builder/ + dest: /home/dist/tools/nightly-builder/ + owner: dist + group: dist + tags: nightly-builder + +- name: nightly-builder | npm install + command: cd /home/dist/tools/nightly-builder/ && npm install + tags: nightly-builder diff --git a/setup/www/tools/dist-indexer/decode-ref.js b/setup/www/tools/dist-indexer/decode-ref.js index 7e857d865..588bc0aac 100644 --- a/setup/www/tools/dist-indexer/decode-ref.js +++ b/setup/www/tools/dist-indexer/decode-ref.js @@ -19,6 +19,8 @@ if (module === require.main) { , { dir: 'v2.3.1-next-nightly201506308f6f4280c6' , ref: '8f6f4280c6' } , { dir: 'v3.0.0-rc.1' , ref: 'v3.0.0-rc.1' } , { dir: 'v33.22.1-rc.111' , ref: 'v33.22.1-rc.111' } + , { dir: 'v0.6.1' , ref: 'v0.6.1' } + , { dir: 'v0.5.1' , ref: 'v0.5.1' } ] tests.forEach(function (test) { diff --git a/setup/www/tools/dist-indexer/dist-indexer.js b/setup/www/tools/dist-indexer/dist-indexer.js index 0d066b661..5f9fcd6c6 100755 --- a/setup/www/tools/dist-indexer/dist-indexer.js +++ b/setup/www/tools/dist-indexer/dist-indexer.js @@ -9,6 +9,7 @@ const fs = require('fs') , after = require('after') , hyperquest = require('hyperquest') , bl = require('bl') + , semver = require('semver') , transformFilename = require('./transform-filename') , decodeRef = require('./decode-ref') @@ -22,10 +23,16 @@ const fs = require('fs') `${githubContentUrl}/deps/v8/src/version.cc` , `${githubContentUrl}/deps/v8/include/v8-version.h` ] - , uvVersionUrl = `${githubContentUrl}/deps/uv/include/uv-version.h` + , uvVersionUrl = [ + `${githubContentUrl}/deps/uv/include/uv-version.h` + , `${githubContentUrl}/deps/uv/src/version.c` + ] , sslVersionUrl = `${githubContentUrl}/deps/openssl/openssl/Makefile` , zlibVersionUrl = `${githubContentUrl}/deps/zlib/zlib.h` - , modVersionUrl = `${githubContentUrl}/src/node_version.h` + , modVersionUrl = [ + `${githubContentUrl}/src/node_version.h` + , `${githubContentUrl}/src/node.h` + ] , githubOptions = { headers: { 'accept': 'text/plain,application/vnd.github.v3.raw' } } @@ -145,19 +152,36 @@ function fetchUvVersion (gitref, callback) { if (version) return setImmediate(callback.bind(null, null, version)) - fetch(uvVersionUrl, gitref, function (err, rawData) { + fetch(uvVersionUrl[0], gitref, function (err, rawData) { if (err) return callback(err) version = rawData.split('\n').map(function (line) { - return line.match(/^#define UV_VERSION_(?:MAJOR|MINOR|PATCH)\s+(\d+)$/) - }) - .filter(Boolean) - .map(function (m) { return m[1] }) - .join('.') + return line.match(/^#define UV_VERSION_(?:MAJOR|MINOR|PATCH)\s+(\d+)$/) + }) + .filter(Boolean) + .map(function (m) { return m[1] }) + .join('.') - cachePut(gitref, 'uv', version) - callback(null, version) + if (version) { + cachePut(gitref, 'uv', version) + return callback(null, version) + } + + fetch(uvVersionUrl[1], gitref, function (err, rawData) { + if (err) + return callback(err) + + version = rawData.split('\n').map(function (line) { + return line.match(/^#define UV_VERSION_(?:MAJOR|MINOR|PATCH)\s+(\d+)$/) + }) + .filter(Boolean) + .map(function (m) { return m[1] }) + .join('.') + + cachePut(gitref, 'uv', version) + callback(null, version) + }) }) } @@ -203,15 +227,27 @@ function fetchModVersion (gitref, callback) { if (version) return setImmediate(callback.bind(null, null, version)) - fetch(modVersionUrl, gitref, function (err, rawData) { + fetch(modVersionUrl[0], gitref, function (err, rawData) { if (err) return callback(err) var m = rawData.match(/^#define NODE_MODULE_VERSION\s+([^\s]+)\s+.+$/m) version = m && m[1] - cachePut(gitref, 'mod', version) - callback(null, version) + if (version) { + cachePut(gitref, 'mod', version) + return callback(null, version) + } + + fetch(modVersionUrl[1], gitref, function (err, rawData) { + if (err) + return callback(err) + + m = rawData.match(/^#define NODE_MODULE_VERSION\s+\(?([^\s\)]+)\)?\s+.+$/m) + version = m && m[1] + cachePut(gitref, 'mod', version) + callback(null, version) + }) }) } @@ -238,6 +274,7 @@ function dirDate (dir, callback) { function dirFiles (dir, callback) { + //TODO: look in SHASUMS.txt as well for older versions fs.readFile(path.join(argv.dist, dir, 'SHASUMS256.txt'), 'utf8', afterReadFile) function afterReadFile (err, contents) { @@ -245,12 +282,12 @@ function dirFiles (dir, callback) { return callback(err) var files = contents.split('\n').map(function (line) { - var seg = line.split(/\s+/) - return seg.length >= 2 && seg[1] - }) - .map(transformFilename) - .filter(Boolean) - .sort() + var seg = line.split(/\s+/) + return seg.length >= 2 && seg[1] + }) + .map(transformFilename) + .filter(Boolean) + .sort() callback(null, files) } @@ -375,6 +412,10 @@ function afterMap (err, dirs) { if (err) throw err + dirs.sort(function (d1, d2) { + return semver.compare(d2.version, d1.version) + }) + saveVersionCache() dirs = dirs.filter(Boolean) diff --git a/setup/www/tools/dist-indexer/package.json b/setup/www/tools/dist-indexer/package.json index 09d2fb487..f2b8a3202 100644 --- a/setup/www/tools/dist-indexer/package.json +++ b/setup/www/tools/dist-indexer/package.json @@ -10,6 +10,7 @@ "bl": "~0.9.3", "hyperquest": "~1.0.1", "map-async": "~0.1.1", - "minimist": "~1.1.0" + "minimist": "~1.1.0", + "semver": "~5.0.1" } } diff --git a/setup/www/tools/dist-indexer/transform-filename.js b/setup/www/tools/dist-indexer/transform-filename.js index 67709e696..9ad042e05 100644 --- a/setup/www/tools/dist-indexer/transform-filename.js +++ b/setup/www/tools/dist-indexer/transform-filename.js @@ -2,24 +2,28 @@ const assert = require('assert') , types = { 'tar.gz' : 'src' , 'darwin-x64' : 'osx-x64-tar' + , 'darwin-x86' : 'osx-x86-tar' , 'pkg' : 'osx-x64-pkg' , 'linux-arm64' : 'linux-arm64' , 'linux-armv7l' : 'linux-armv7l' , 'linux-armv6l' : 'linux-armv6l' , 'linux-x64' : 'linux-x64' , 'linux-x86' : 'linux-x86' + , 'sunos-x64' : 'sunos-x64' + , 'sunos-x86' : 'sunos-x86' , 'x64.msi' : 'win-x64-msi' , 'x86.msi' : 'win-x86-msi' , 'win-x64/iojs.exe' : 'win-x64-exe' , 'win-x86/iojs.exe' : 'win-x86-exe' + , 'node.exe' : 'win-x86-exe' + , 'x64/node.exe' : 'win-x64-exe' , 'headers' : 'headers' } function transformFilename (file) { - file = file && file.replace(/^iojs-v\d\.\d\.\d-?((rc\.\d+|(next-)?nightly\d{8}[^-\.]+)-?)?\.?/, '') + file = file && file.replace(/^(?:\.\/)?(?:iojs|node)-v\d+\.\d+\.\d+-?((rc\.\d+|(next-)?nightly\d{8}[^-\.]+)-?)?\.?/, '') .replace(/\.tar\.gz$/, '') - return types[file] } @@ -101,6 +105,15 @@ if (module === require.main) { , { file: 'SHASUMS256.txt' } , { file: 'win-x64/iojs.exe', type: 'win-x64-exe' } , { file: 'win-x86/iojs.exe', type: 'win-x86-exe' } + , { file: 'node-v0.11.9-darwin-x64.tar.gz', type: 'osx-x64-tar' } + , { file: 'node-v0.11.9-darwin-x86.tar.gz', type: 'osx-x86-tar' } + , { file: 'node-v0.11.9-linux-x64.tar.gz', type: 'linux-x64' } + , { file: 'node-v0.11.9-linux-x86.tar.gz', type: 'linux-x86' } + , { file: 'node-v0.11.9-sunos-x64.tar.gz', type: 'sunos-x64' } + , { file: 'node-v0.11.9-sunos-x86.tar.gz', type: 'sunos-x86' } + , { file: 'node-v0.11.9.tar.gz', type: 'src' } + , { file: 'node.exe', type: 'win-x86-exe' } + , { file: 'x64/node.exe', type: 'win-x64-exe' } ] tests.forEach(function (test) { From 1ff5675b3c3cdabd6975870febb17c345f5ad61d Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Mon, 31 Aug 2015 10:54:03 +1000 Subject: [PATCH 07/15] improve cert creation --- setup/www/resources/secrets/makesecrets.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup/www/resources/secrets/makesecrets.sh b/setup/www/resources/secrets/makesecrets.sh index fec41a14b..ecf6a4791 100755 --- a/setup/www/resources/secrets/makesecrets.sh +++ b/setup/www/resources/secrets/makesecrets.sh @@ -40,6 +40,8 @@ bash -c "$gpgcmd" < ${secretsdir}/build/release/star_nodejs.org-COMODORSADomainV echo "" >> ./nodejs_chained.crt bash -c "$gpgcmd" < ${secretsdir}/build/release/star_nodejs.org-COMODORSAAddTrustCA.crt 1>> ./nodejs_chained.crt 2> /dev/null || bork echo "" >> ./nodejs_chained.crt +bash -c "$gpgcmd" < ${secretsdir}/build/release/star_nodejs.org-AddTrustExternalCARoot.crt 1>> ./nodejs_chained.crt 2> /dev/null || bork +echo "" >> ./nodejs_chained.crt echo "Extracting iojs_chained.crt..." bash -c "$gpgcmd" < ${secretsdir}/build/release/star_iojs.org.crt 1> ./iojs_chained.crt 2> /dev/null || bork @@ -48,3 +50,5 @@ bash -c "$gpgcmd" < ${secretsdir}/build/release/star_iojs.org-COMODORSADomainVal echo "" >> ./iojs_chained.crt bash -c "$gpgcmd" < ${secretsdir}/build/release/star_iojs.org-COMODORSAAddTrustCA.crt 1>> ./iojs_chained.crt 2> /dev/null || bork echo "" >> ./iojs_chained.crt +bash -c "$gpgcmd" < ${secretsdir}/build/release/star_iojs.org-AddTrustExternalCARoot.crt 1>> ./iojs_chained.crt 2> /dev/null || bork +echo "" >> ./nodejs_chained.crt From ce2f9f70184d31db04fe5a97d43d8e0312d54da8 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Mon, 31 Aug 2015 14:41:13 +1000 Subject: [PATCH 08/15] added more tools & latest-linker for making symlinks --- setup/www/ansible-playbook.yaml | 6 ++ setup/www/resources/config/iojs.org | 2 +- setup/www/resources/config/nodejs.org | 2 +- setup/www/resources/secrets/makesecrets.sh | 6 ++ setup/www/tasks/dist-indexer.yaml | 2 +- setup/www/tasks/latest-linker.yaml | 25 ++++++ setup/www/tasks/nightly-builder.yaml | 4 +- setup/www/tasks/promote.yaml | 17 ++++ setup/www/tasks/user.yaml | 25 ++++++ setup/www/tools/dist/dist-promotable | 3 + setup/www/tools/dist/dist-promote | 8 ++ setup/www/tools/dist/dist-sign | 8 ++ setup/www/tools/latest-linker/.gitignore | 1 + .../www/tools/latest-linker/latest-linker.js | 85 +++++++++++++++++++ setup/www/tools/latest-linker/package.json | 15 ++++ 15 files changed, 204 insertions(+), 5 deletions(-) create mode 100644 setup/www/tasks/latest-linker.yaml create mode 100644 setup/www/tasks/promote.yaml create mode 100644 setup/www/tools/dist/dist-promotable create mode 100644 setup/www/tools/dist/dist-promote create mode 100644 setup/www/tools/dist/dist-sign create mode 100644 setup/www/tools/latest-linker/.gitignore create mode 100755 setup/www/tools/latest-linker/latest-linker.js create mode 100644 setup/www/tools/latest-linker/package.json diff --git a/setup/www/ansible-playbook.yaml b/setup/www/ansible-playbook.yaml index 69da653f7..cfe06eeca 100644 --- a/setup/www/ansible-playbook.yaml +++ b/setup/www/ansible-playbook.yaml @@ -30,3 +30,9 @@ - include: tasks/nightly-builder.yaml tags: nightly-builder + - include: tasks/promote.yaml + tags: promote + + - include: tasks/latest-linker.yaml + tags: latest-linker + diff --git a/setup/www/resources/config/iojs.org b/setup/www/resources/config/iojs.org index c5b6f62b3..99a9910bf 100644 --- a/setup/www/resources/config/iojs.org +++ b/setup/www/resources/config/iojs.org @@ -78,7 +78,7 @@ server { } location /api { - alias /home/dist/iojs/public/release/latest/doc/api; + alias /home/dist/iojs/public/release/latest/docs/api; autoindex on; default_type text/plain; } diff --git a/setup/www/resources/config/nodejs.org b/setup/www/resources/config/nodejs.org index 454ecd373..8570e183c 100644 --- a/setup/www/resources/config/nodejs.org +++ b/setup/www/resources/config/nodejs.org @@ -127,7 +127,7 @@ server { } location /api { - alias /home/dist/nodejs/public/release/latest/doc/api; + alias /home/dist/nodejs/public/release/latest/docs/api; autoindex on; default_type text/plain; diff --git a/setup/www/resources/secrets/makesecrets.sh b/setup/www/resources/secrets/makesecrets.sh index ecf6a4791..dc90dfe50 100755 --- a/setup/www/resources/secrets/makesecrets.sh +++ b/setup/www/resources/secrets/makesecrets.sh @@ -52,3 +52,9 @@ bash -c "$gpgcmd" < ${secretsdir}/build/release/star_iojs.org-COMODORSAAddTrustC echo "" >> ./iojs_chained.crt bash -c "$gpgcmd" < ${secretsdir}/build/release/star_iojs.org-AddTrustExternalCARoot.crt 1>> ./iojs_chained.crt 2> /dev/null || bork echo "" >> ./nodejs_chained.crt + +echo "Extracting staging_id_rsa_public.key..." +bash -c "$gpgcmd" < ${secretsdir}/build/release/staging_id_rsa_public.key 1>> ./staging_id_rsa_public.key 2> /dev/null || bork +echo "" >> ./staging_id_rsa_public.key + + diff --git a/setup/www/tasks/dist-indexer.yaml b/setup/www/tasks/dist-indexer.yaml index 771f92e63..b3d33b271 100644 --- a/setup/www/tasks/dist-indexer.yaml +++ b/setup/www/tasks/dist-indexer.yaml @@ -5,7 +5,7 @@ mode: 0755 owner: dist group: dist - tags: setup + tags: dist-indexer - name: dist-indexer | Copy source files copy: diff --git a/setup/www/tasks/latest-linker.yaml b/setup/www/tasks/latest-linker.yaml new file mode 100644 index 000000000..2b558e6e0 --- /dev/null +++ b/setup/www/tasks/latest-linker.yaml @@ -0,0 +1,25 @@ +- name: Site Setup | Make /home/dist/tools/latest-linker + file: + path: /home/dist/tools/latest-linker + state: directory + mode: 0755 + owner: dist + group: dist + tags: latest-linker + +- name: latest-linker | Copy source files + copy: + src: tools/latest-linker/ + dest: /home/dist/tools/latest-linker/ + owner: dist + group: dist + tags: latest-linker + +- name: latest-linker | npm install + shell: cd /home/dist/tools/latest-linker/ && npm install + tags: latest-linker + +- name: latest-linker | Set executables + file: + path: /home/dist/tools/latest-linker/latest-linker.js + mode: 0755 diff --git a/setup/www/tasks/nightly-builder.yaml b/setup/www/tasks/nightly-builder.yaml index f5967996d..c04159a73 100644 --- a/setup/www/tasks/nightly-builder.yaml +++ b/setup/www/tasks/nightly-builder.yaml @@ -5,7 +5,7 @@ mode: 0755 owner: dist group: dist - tags: setup + tags: nightly-builder - name: nightly-builder | Copy source files copy: @@ -16,5 +16,5 @@ tags: nightly-builder - name: nightly-builder | npm install - command: cd /home/dist/tools/nightly-builder/ && npm install + shell: cd /home/dist/tools/nightly-builder/ && npm install tags: nightly-builder diff --git a/setup/www/tasks/promote.yaml b/setup/www/tasks/promote.yaml new file mode 100644 index 000000000..868156553 --- /dev/null +++ b/setup/www/tasks/promote.yaml @@ -0,0 +1,17 @@ +- name: promote | Make /home/staging/tools/promote + file: + path: /home/staging/tools/promote + state: directory + mode: 0755 + owner: staging + group: staging + tags: promote + +- name: promote | Copy source files + copy: + src: tools/promote/ + dest: /home/staging/tools/promote/ + owner: root + group: root + mode: 0750 + tags: promote diff --git a/setup/www/tasks/user.yaml b/setup/www/tasks/user.yaml index 8ef09c649..e7604fec8 100644 --- a/setup/www/tasks/user.yaml +++ b/setup/www/tasks/user.yaml @@ -70,3 +70,28 @@ - iojs/public/release - iojs/public/test tags: user + +- name: User | Insert SSH public key to staging + authorized_key: + user: "staging" + key: "{{ item }}" + with_file: + - resources/secrets/staging_id_rsa_public.key + tags: user + +- name: User | Set up staging/{nodejs,iojs} directories + file: + path: /home/staging/{{ item }} + state: directory + mode: 0775 + owner: staging + with_items: + - nodejs/custom + - nodejs/next-nightly + - nodejs/nightly + - nodejs/release + - iojs/custom + - iojs/next-nightly + - iojs/nightly + - iojs/release + tags: user diff --git a/setup/www/tools/dist/dist-promotable b/setup/www/tools/dist/dist-promotable new file mode 100644 index 000000000..124a5fdc1 --- /dev/null +++ b/setup/www/tools/dist/dist-promotable @@ -0,0 +1,3 @@ +#!/bin/sh + +(cd /home/staging/staging/release/ && /home/dist/dist-indexer/ls-types.js -d *) diff --git a/setup/www/tools/dist/dist-promote b/setup/www/tools/dist/dist-promote new file mode 100644 index 000000000..d7954ad2d --- /dev/null +++ b/setup/www/tools/dist/dist-promote @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +if [ "X$1" == "X" ]; then + echo "Must supply a version string to promote" + exit 1 +fi + +/home/staging/promote/promote_release.sh $1 diff --git a/setup/www/tools/dist/dist-sign b/setup/www/tools/dist/dist-sign new file mode 100644 index 000000000..27d0e9306 --- /dev/null +++ b/setup/www/tools/dist/dist-sign @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +if [ "X$1" == "X" ]; then + echo "Must supply a version string to sign" + exit 1 +fi + +/home/staging/promote/resha_release.sh $1 diff --git a/setup/www/tools/latest-linker/.gitignore b/setup/www/tools/latest-linker/.gitignore new file mode 100644 index 000000000..b512c09d4 --- /dev/null +++ b/setup/www/tools/latest-linker/.gitignore @@ -0,0 +1 @@ +node_modules \ No newline at end of file diff --git a/setup/www/tools/latest-linker/latest-linker.js b/setup/www/tools/latest-linker/latest-linker.js new file mode 100755 index 000000000..497239567 --- /dev/null +++ b/setup/www/tools/latest-linker/latest-linker.js @@ -0,0 +1,85 @@ +#!/usr/bin/env node + +'use strict' + +const fs = require('fs') + , path = require('path') + , semver = require('semver') + , map = require('map-async') + + +if (process.argv.length < 3) + throw new Error('Please provide a downloads directory location') + +const dir = process.argv[2] + +if (!fs.statSync(dir).isDirectory()) + throw new Error('Please provide a downloads directory location') + +map( + fs.readdirSync(dir).map(function (d) { return path.join(dir, d) }) + , function (d, callback) { + fs.stat(d, function (err, stat) { callback(null, { d: d, stat: stat }) }) + } + , afterMap +) + +function afterMap (err, dirs) { + if (err) + throw err + + dirs = dirs.filter(function (d) { return d.stat && d.stat.isDirectory() }) + .map(function (d) { return path.basename(d.d) }) + .map(function (d) { try { return semver(d) } catch (e) {} }) + .filter(Boolean) + .filter(function (d) { return semver.satisfies(d, '~0.10 || ~0.12 || >= 1.0') }) + .map(function (d) { return d.raw }) + + dirs.sort(function (d1, d2) { return semver.compare(d1, d2) }) + + link('0.10.x', dirs) + max = link('0.12.x', dirs) + for (var i = 1;; i++) + if (!link(`${i}.x`, dirs)) break + + var max = link(null, dirs) + , tbreg = new RegExp(`(\\w+)-${max}.tar.gz`) + + var tarball = fs.readdirSync(path.join(dir, 'latest')) + .filter(function (f) { + return tbreg.test(f) + }) + + if (tarball.length != 1) + throw new Error('Could not find latest.tar.gz') + + tarball = tarball[0] + var name = tarball.match(tbreg)[1] + var dst = path.join(dir, `${name}-latest.tar.gz`) + try { fs.unlinkSync(dst) } catch (e) {} + fs.symlinkSync(path.join(dir, 'latest', tarball), dst) +} + + +function link (line, dirs) { + var range = line ? `${line[0] == '0' ? '~' : '^'}${line}` : '*' + , max = semver.maxSatisfying(dirs, range) + + if (!max) return false + + function symlink (name) { + var dst = path.join(dir, name) + try { fs.unlinkSync(dst) } catch (e) {} + fs.symlinkSync(path.join(dir, max), dst) + } + + + if (line) { + symlink(`latest-v${line}`) + } else { + symlink('latest') + } + + return max +} + diff --git a/setup/www/tools/latest-linker/package.json b/setup/www/tools/latest-linker/package.json new file mode 100644 index 000000000..1d91915c5 --- /dev/null +++ b/setup/www/tools/latest-linker/package.json @@ -0,0 +1,15 @@ +{ + "name": "latest-linker", + "version": "1.0.0", + "description": "", + "main": "latest-linker.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "Rod (http://r.va.gg/)", + "license": "MIT", + "dependencies": { + "map-async": "~0.1.1", + "semver": "~5.0.1" + } +} From 3d90a74ab6ecbef064b57e805bc5a5be4d4438aa Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Mon, 31 Aug 2015 22:52:05 +1000 Subject: [PATCH 09/15] major reorg of on-server resources & directories --- setup/www/ansible-vars.yaml | 1 + setup/www/resources/config/iojs.org | 56 ++++++++++--------- setup/www/resources/config/logrotate-nginx | 23 ++++++++ setup/www/resources/config/nodejs.org | 44 ++++++--------- setup/www/resources/scripts/build-site.sh | 4 +- .../www/resources/scripts/check-build-site.sh | 4 +- .../resources/scripts/restore-cloudfuse.sh | 2 +- .../www/resources/scripts/rsync-cloudfuse.sh | 2 +- setup/www/tasks/cloudfuse.yaml | 2 +- setup/www/tasks/docker.yaml | 2 +- setup/www/tasks/nginx.yaml | 9 +++ setup/www/tasks/promote.yaml | 15 ++++- setup/www/tasks/site-setup.yaml | 9 ++- setup/www/tasks/user.yaml | 26 +++++---- setup/www/tools/dist/dist-promotable | 11 +++- setup/www/tools/dist/dist-promote | 15 ++++- setup/www/tools/dist/dist-sign | 15 ++++- setup/www/tools/promote/_promote.sh | 8 ++- setup/www/tools/promote/_resha.sh | 12 ++-- setup/www/tools/promote/promote_nightly.sh | 22 +++++++- setup/www/tools/promote/promote_release.sh | 24 ++++++-- setup/www/tools/promote/resha_release.sh | 20 ++++++- setup/www/tools/promote/settings | 4 +- 23 files changed, 230 insertions(+), 100 deletions(-) create mode 100644 setup/www/resources/config/logrotate-nginx diff --git a/setup/www/ansible-vars.yaml b/setup/www/ansible-vars.yaml index 8ffb8fda4..bbc26ed00 100644 --- a/setup/www/ansible-vars.yaml +++ b/setup/www/ansible-vars.yaml @@ -6,6 +6,7 @@ root_users: - orangemocha dist_users: - cjihrig + - chrisdickinson - sam-github - fishrock123 - jasnell diff --git a/setup/www/resources/config/iojs.org b/setup/www/resources/config/iojs.org index 99a9910bf..14d266d7b 100644 --- a/setup/www/resources/config/iojs.org +++ b/setup/www/resources/config/iojs.org @@ -1,14 +1,15 @@ server { listen 80; - listen [::]:80; + listen iojs.org:80; + server_name iojs.org www.iojs.org; + return 301 https://iojs.org$request_uri; } server { listen 443 ssl spdy; - #listen [::]:443 ipv6only=on ssl spdy; - - server_name iojs.org; + listen iojs.org:443 ssl spdy; + server_name iojs.org www.iojs.org; ssl_certificate ssl/iojs_chained.crt; ssl_certificate_key ssl/iojs.key; @@ -44,46 +45,51 @@ server { gzip on; gzip_static on; gzip_disable "MSIE [1-6]\."; - default_type text/html; gzip_types text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript; if ($host ~* ^www\.){ rewrite ^(.*)$ https://iojs.org$1; } - location ~ \.json$ { - add_header access-control-allow-origin *; - } - - location ~ /(dist|docs)/v[0-9]+\.[0-9]+\.[0-9]+/.*\.(tar\.gz|msi|exe|pdb|lib|html|json) { - expires max; - } + root /home/www/iojs; + default_type text/plain; + index index.html; location / { - root /home/iojs/www; - index index.html; - default_type text/plain; + rewrite ^/$ /en/ redirect; + + location ~ \.json$ { + add_header access-control-allow-origin *; + } } location /download { - alias /home/dist/iojs/public; + alias /home/dist/iojs; autoindex on; default_type text/plain; + + location ~ \.json$ { + add_header access-control-allow-origin *; + } } location /dist { - alias /home/dist/iojs/public/release/; - autoindex on; - default_type text/plain; + alias /home/dist/iojs/release/; + autoindex on; + default_type text/plain; + + location ~ \.json$ { + add_header access-control-allow-origin *; + } } location /api { - alias /home/dist/iojs/public/release/latest/docs/api; - autoindex on; - default_type text/plain; - } + alias /home/dist/iojs/release/latest/docs/api; + autoindex on; + default_type text/plain; - location /download-stats.json { - alias /home/iojs/download-stats.json; + location ~ \.json$ { + add_header access-control-allow-origin *; + } } } diff --git a/setup/www/resources/config/logrotate-nginx b/setup/www/resources/config/logrotate-nginx new file mode 100644 index 000000000..eb45caec3 --- /dev/null +++ b/setup/www/resources/config/logrotate-nginx @@ -0,0 +1,23 @@ +/var/log/nginx/*.log { + daily + missingok + compresscmd /usr/bin/xz + uncompresscmd /usr/bin/unxz + compressext .xz + compress + delaycompress + notifempty + create 0640 www-data adm + dateext + dateformat "%Y%m%d" + dateyesterday + sharedscripts + prerotate + if [ -d /etc/logrotate.d/httpd-prerotate ]; then \ + run-parts /etc/logrotate.d/httpd-prerotate; \ + fi \ + endscript + postrotate + [ -s /run/nginx.pid ] && kill -USR1 `cat /run/nginx.pid` + endscript +} \ No newline at end of file diff --git a/setup/www/resources/config/nodejs.org b/setup/www/resources/config/nodejs.org index 8570e183c..cfb52c5b0 100644 --- a/setup/www/resources/config/nodejs.org +++ b/setup/www/resources/config/nodejs.org @@ -1,18 +1,20 @@ +log_format main '$remote_addr - $remote_user [$time_local] $request ' + '"$status" $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + server { listen 80 default_server; listen [::]:80; - return 301 https://nodejs.org$request_uri; + server_name www.nodejs.org nodejs.org; - # TODO: old nodejs.org allowed /dist/ and *.json through on 80, should we - # also do the same here? The suggestion was that it would break npm/travis-ci - # although that's likely to be old versions - # If we let it through then we need to copy some of the 443 config here + return 301 https://nodejs.org$request_uri; } server { listen 80; listen [::]:80; server_name doc.nodejs.org docs.nodejs.org; + rewrite ^/(v[0-9]+\.[0-9]+\.[0-9]+)(/?.*)$ https://nodejs.org/docs/$1$2 permanent; rewrite /(.*)$ https://nodejs.org/docs/latest/$1 permanent; } @@ -21,6 +23,7 @@ server { listen 80; listen [::]:80; server_name api.nodejs.org; + rewrite ^/(v[0-9]+\.[0-9]+\.[0-9]+)(/?.*)$ https://nodejs.org/docs/$1/api$2 permanent; rewrite /(.*)$ https://nodejs.org/docs/latest/api/$1 permanent; } @@ -29,6 +32,7 @@ server { listen 80; listen [::]:80; server_name dist.nodejs.org; + rewrite /(.*)$ http://nodejs.org/dist/$1 permanent; } @@ -72,7 +76,6 @@ server { gzip on; gzip_static on; gzip_disable "MSIE [1-6]\."; - default_type text/html; gzip_types text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript; if ($host ~* ^www\.){ @@ -83,60 +86,45 @@ server { rewrite ^/documentation/api(.*)$ /api$1 permanent; } + root /home/www/nodejs; + default_type text/plain; + index index.html; + location / { - root /home/nodejs/www; - index index.html; - default_type text/plain; rewrite ^/$ /en/ redirect; location ~ \.json$ { add_header access-control-allow-origin *; } - - location ~ /(dist|docs)/v[0-9]+\.[0-9]+\.[0-9]+/.*\.(tar\.gz|msi|exe|pdb|lib|html|json) { - expires max; - } } location /download { - alias /home/dist/nodejs/public; + alias /home/dist/nodejs; autoindex on; default_type text/plain; location ~ \.json$ { add_header access-control-allow-origin *; } - - location ~ /(dist|docs)/v[0-9]+\.[0-9]+\.[0-9]+/.*\.(tar\.gz|msi|exe|pdb|lib|html|json) { - expires max; - } } location /dist { - alias /home/dist/nodejs/public/release/; + alias /home/dist/nodejs/release/; autoindex on; default_type text/plain; location ~ \.json$ { add_header access-control-allow-origin *; } - - location ~ /(dist|docs)/v[0-9]+\.[0-9]+\.[0-9]+/.*\.(tar\.gz|msi|exe|pdb|lib|html|json) { - expires max; - } } location /api { - alias /home/dist/nodejs/public/release/latest/docs/api; + alias /home/dist/nodejs/release/latest/docs/api; autoindex on; default_type text/plain; location ~ \.json$ { add_header access-control-allow-origin *; } - - location ~ /(dist|docs)/v[0-9]+\.[0-9]+\.[0-9]+/.*\.(tar\.gz|msi|exe|pdb|lib|html|json) { - expires max; - } } } diff --git a/setup/www/resources/scripts/build-site.sh b/setup/www/resources/scripts/build-site.sh index 50a41ee86..b68f2b1a0 100755 --- a/setup/www/resources/scripts/build-site.sh +++ b/setup/www/resources/scripts/build-site.sh @@ -13,7 +13,7 @@ pidof -s -o '%PPID' -x $(basename $0) > /dev/null 2>&1 && \ echo "$(basename $0) already running" && \ exit 1 -clonedir=/home/${site}/website.github +clonedir=/home/www/github/${site} if [ ! -d "${clonedir}" ]; then repo="${site}.org" @@ -56,6 +56,6 @@ docker run \ ' \ " -rsync -avz --delete --exclude .git ${clonedir}/${rsync_from} /home/${site}/www/ +rsync -avz --delete --exclude .git ${clonedir}/${rsync_from} /home/www/${site}/ /home/nodejs/queue-cdn-purge.sh $site diff --git a/setup/www/resources/scripts/check-build-site.sh b/setup/www/resources/scripts/check-build-site.sh index 62cf33bee..3224c30bc 100755 --- a/setup/www/resources/scripts/check-build-site.sh +++ b/setup/www/resources/scripts/check-build-site.sh @@ -7,8 +7,8 @@ if [ "X$site" != "Xiojs" ] && [ "X$site" != "Xnodejs" ]; then exit 1 fi -indexjson=/home/${site}/dist/public/release/index.json -indexhtml=/home/${site}//www/en/index.html +indexjson=/home/dist/${site}/release/index.json +indexhtml=/home/www/${site}/en/index.html buildsite=/home/nodejs/build-site.sh [ $indexjson -nt $indexhtml ] && $buildsite $site \ No newline at end of file diff --git a/setup/www/resources/scripts/restore-cloudfuse.sh b/setup/www/resources/scripts/restore-cloudfuse.sh index c13265872..e35ff3c53 100644 --- a/setup/www/resources/scripts/restore-cloudfuse.sh +++ b/setup/www/resources/scripts/restore-cloudfuse.sh @@ -2,7 +2,7 @@ for d in nodejs iojs; do srcdir=/home/nodejs/cloudfuse/${d}.org/public/release/ - dstdir=/home/dist/${d}/public/release/ + dstdir=/home/dist/${d}/release/ timesfile=/home/nodejs/cloudfuse/${d}.org/public.times mkdir -p $dstdir diff --git a/setup/www/resources/scripts/rsync-cloudfuse.sh b/setup/www/resources/scripts/rsync-cloudfuse.sh index b65a8a98c..8e35123f3 100644 --- a/setup/www/resources/scripts/rsync-cloudfuse.sh +++ b/setup/www/resources/scripts/rsync-cloudfuse.sh @@ -1,7 +1,7 @@ #!/bin/bash for d in nodejs iojs; do - srcdir=/home/dist/${d}/public/release/ + srcdir=/home/dist/${d}/release/ dstdir=/home/nodejs/cloudfuse/${d}.org/public/release/ timesfile=/home/nodejs/cloudfuse/${d}.org/public.times diff --git a/setup/www/tasks/cloudfuse.yaml b/setup/www/tasks/cloudfuse.yaml index 7a3617c49..6d8f5664d 100644 --- a/setup/www/tasks/cloudfuse.yaml +++ b/setup/www/tasks/cloudfuse.yaml @@ -12,7 +12,7 @@ - fuse tags: cloudfuse -- name: CloudFuse | Download source +- name: CloudFuse | Download and compile shell: cd /usr/src/ && \ curl -sL http://github.com/redbo/cloudfuse/tarball/master | tar -zx && \ cd redbo-cloudfuse-*/ && \ diff --git a/setup/www/tasks/docker.yaml b/setup/www/tasks/docker.yaml index 953bb4f1c..3f51ed680 100644 --- a/setup/www/tasks/docker.yaml +++ b/setup/www/tasks/docker.yaml @@ -2,6 +2,6 @@ command: "bash -c 'curl -sL http://get.docker.io/ | bash -'" tags: docker -- name: Docker | Add {{ system_user }} to docker group +- name: Docker | Add nodejs to docker group command: usermod -aG docker nodejs tags: docker diff --git a/setup/www/tasks/nginx.yaml b/setup/www/tasks/nginx.yaml index ca6ed766e..67edec23f 100644 --- a/setup/www/tasks/nginx.yaml +++ b/setup/www/tasks/nginx.yaml @@ -56,6 +56,15 @@ - 'application/gzip gz;' tags: nginx +- name: nginx | Copy nginx logrotate config + copy: + src: ./resources/config/logrotate-nginx + dest: /etc/logrotate.d/nginx + mode: 0644 + owner: root + group: root + tags: nginx + - name: nginx | Restart service service: name: nginx diff --git a/setup/www/tasks/promote.yaml b/setup/www/tasks/promote.yaml index 868156553..fc3fbfebb 100644 --- a/setup/www/tasks/promote.yaml +++ b/setup/www/tasks/promote.yaml @@ -12,6 +12,19 @@ src: tools/promote/ dest: /home/staging/tools/promote/ owner: root - group: root + group: dist mode: 0750 tags: promote + +- name: promote | Copy dist promote files + copy: + src: tools/dist/{{ item }} + dest: /usr/local/bin/{{ item }} + owner: dist + group: dist + mode: 0750 + with_items: + - dist-promotable + - dist-promote + - dist-sign + tags: promote diff --git a/setup/www/tasks/site-setup.yaml b/setup/www/tasks/site-setup.yaml index 4427cd9ce..84973db50 100644 --- a/setup/www/tasks/site-setup.yaml +++ b/setup/www/tasks/site-setup.yaml @@ -46,12 +46,17 @@ group: root tags: setup -- name: Site Setup | Make /home/iojs +- name: Site Setup | Make /home/www file: - path: /home/iojs + path: "{{ item }}" state: directory mode: 0755 owner: nodejs + with_items: + - /home/www + - /home/www/iojs + - /home/www/nodejs + - /home/www/github tags: setup - name: Site Setup | Make /home/nodejs/.npm diff --git a/setup/www/tasks/user.yaml b/setup/www/tasks/user.yaml index e7604fec8..55cd1715f 100644 --- a/setup/www/tasks/user.yaml +++ b/setup/www/tasks/user.yaml @@ -8,6 +8,10 @@ - staging tags: user +- name: User | Add dist to staging group + command: usermod -aG staging dist + tags: user + - name: User | Download pubkeys (root_users) get_url: url: https://github.com/{{ item }}.keys @@ -52,23 +56,23 @@ with_items: dist_users tags: user -- name: User | Set up dist/{nodejs,iojs}/public directories +- name: User | Set up dist/{nodejs,iojs}/ directories file: path: /home/dist/{{ item }} state: directory mode: 0755 owner: dist with_items: - - nodejs/public/next-nightly - - nodejs/public/nightly - - nodejs/public/rc - - nodejs/public/release - - nodejs/public/test - - iojs/public/next-nightly - - iojs/public/nightly - - iojs/public/rc - - iojs/public/release - - iojs/public/test + - nodejs/next-nightly + - nodejs/nightly + - nodejs/rc + - nodejs/release + - nodejs/test + - iojs/next-nightly + - iojs/nightly + - iojs/rc + - iojs/release + - iojs/test tags: user - name: User | Insert SSH public key to staging diff --git a/setup/www/tools/dist/dist-promotable b/setup/www/tools/dist/dist-promotable index 124a5fdc1..a3b0ee31b 100644 --- a/setup/www/tools/dist/dist-promotable +++ b/setup/www/tools/dist/dist-promotable @@ -1,3 +1,12 @@ #!/bin/sh -(cd /home/staging/staging/release/ && /home/dist/dist-indexer/ls-types.js -d *) +set -e + +site=$1 + +if [ "X$site" != "Xiojs" ] && [ "X$site" != "Xnodejs" ]; then + echo "Usage: dist-promotable < iojs | nodejs >" + exit 1 +fi + +(cd /home/staging/${site}/release/ && /home/dist/tools/dist-indexer/ls-types.js -d *) diff --git a/setup/www/tools/dist/dist-promote b/setup/www/tools/dist/dist-promote index d7954ad2d..8debe8b33 100644 --- a/setup/www/tools/dist/dist-promote +++ b/setup/www/tools/dist/dist-promote @@ -1,8 +1,17 @@ #!/usr/bin/env bash -if [ "X$1" == "X" ]; then - echo "Must supply a version string to promote" +set -e + +site=$1 + +if [ "X$site" != "Xiojs" ] && [ "X$site" != "Xnodejs" ]; then + echo "Usage: dist-promote < iojs | nodejs > " + exit 1 +fi + +if [ "X$2" == "X" ]; then + echo "Usage: dist-promote < iojs | nodejs > " exit 1 fi -/home/staging/promote/promote_release.sh $1 +/home/staging/tools/promote/promote_release.sh $1 $2 diff --git a/setup/www/tools/dist/dist-sign b/setup/www/tools/dist/dist-sign index 27d0e9306..c52debbae 100644 --- a/setup/www/tools/dist/dist-sign +++ b/setup/www/tools/dist/dist-sign @@ -1,8 +1,17 @@ #!/usr/bin/env bash -if [ "X$1" == "X" ]; then - echo "Must supply a version string to sign" +set -e + +site=$1 + +if [ "X$site" != "Xiojs" ] && [ "X$site" != "Xnodejs" ]; then + echo "Usage: dist-sign < iojs | nodejs > " + exit 1 +fi + +if [ "X$2" == "X" ]; then + echo "Usage: dist-sign < iojs | nodejs > " exit 1 fi -/home/staging/promote/resha_release.sh $1 +/home/staging/tools/promote/resha_release.sh $1 $2 diff --git a/setup/www/tools/promote/_promote.sh b/setup/www/tools/promote/_promote.sh index 49c47c908..577f3676b 100755 --- a/setup/www/tools/promote/_promote.sh +++ b/setup/www/tools/promote/_promote.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +__dirname="$(CDPATH= cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + if [ -z ${srcdir+x} ]; then echo "\$srcdir is not set" exit 1 @@ -10,7 +12,8 @@ if [ -z ${dstdir+x} ]; then exit 1 fi -version=$1 +site=$1 +version=$2 for subdir in $(cd $srcdir && ls); do @@ -38,9 +41,10 @@ for subdir in $(cd $srcdir && ls); do done if [ "X${version}" == "X" ] && [ "$resha" == "yes" ]; then - /home/staging/promote/_resha.sh $dstdir $subdir + ${__dirname}/_resha.sh $site $dstdir $subdir fi + /home/nodejs/queue-cdn-purge.sh $site fi done diff --git a/setup/www/tools/promote/_resha.sh b/setup/www/tools/promote/_resha.sh index 70eafdb2c..b006b0d85 100755 --- a/setup/www/tools/promote/_resha.sh +++ b/setup/www/tools/promote/_resha.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash -dstdir=$1 -version=$2 +set -e + +site=$1 +dstdir=$2 +version=$3 if [ "X${dstdir}" == "X" ]; then echo "dstdir argument not provided" @@ -13,8 +16,9 @@ if [ "X${version}" == "X" ]; then exit 1 fi -(cd "${dstdir}/${version}" && shasum -a256 $(ls iojs* win-*/* 2> /dev/null) > SHASUMS256.txt) || exit 1 +(cd "${dstdir}/${version}" && shasum -a256 $(ls node* iojs* win-*/* 2> /dev/null) > SHASUMS256.txt) || exit 1 echo "${dstdir}/${version}/SHASUMS256.txt" -/home/dist/dist-indexer/dist-indexer.js --dist $dstdir --indexjson ${dstdir}/index.json --indextab ${dstdir}/index.tab +/home/dist/tools/dist-indexer/dist-indexer.js --dist $dstdir --indexjson ${dstdir}/index.json --indextab ${dstdir}/index.tab find "${dstdir}/${version}" -type f -exec chmod 644 '{}' \; find "${dstdir}/${version}" -type d -exec chmod 755 '{}' \; + diff --git a/setup/www/tools/promote/promote_nightly.sh b/setup/www/tools/promote/promote_nightly.sh index ffa6c3c89..46993b029 100755 --- a/setup/www/tools/promote/promote_nightly.sh +++ b/setup/www/tools/promote/promote_nightly.sh @@ -1,13 +1,29 @@ #!/usr/bin/env bash -. /home/staging/promote/settings +set -e + +site=$1 + +__dirname="$(CDPATH= cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [ "X$site" != "Xiojs" ] && [ "X$site" != "Xnodejs" ]; then + echo "Usage: promote_nightly.sh < iojs | nodejs > " + exit 1 +fi + +if [ "X$2" == "X" ]; then + echo "Usage: promote_nightly.sh < iojs | nodejs > " + exit 1 +fi + +. ${__dirname}/settings srcdir=$nightly_srcdir dstdir=$nightly_dstdir -. /home/staging/promote/_promote.sh +. ${__dirname}/_promote.sh $site $2 srcdir=$next_nightly_srcdir dstdir=$next_nightly_dstdir -. /home/staging/promote/_promote.sh +. ${__dirname}/_promote.sh $site $2 diff --git a/setup/www/tools/promote/promote_release.sh b/setup/www/tools/promote/promote_release.sh index f6d9b6d74..9371406ed 100755 --- a/setup/www/tools/promote/promote_release.sh +++ b/setup/www/tools/promote/promote_release.sh @@ -1,12 +1,26 @@ #!/usr/bin/env bash -. /home/staging/promote/settings +set -e + +site=$1 + +__dirname="$(CDPATH= cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [ "X$site" != "Xiojs" ] && [ "X$site" != "Xnodejs" ]; then + echo "Usage: promote_release.sh < iojs | nodejs > " + exit 1 +fi + +if [ "X$2" == "X" ]; then + echo "Usage: promote_release.sh < iojs | nodejs > " + exit 1 +fi + +. ${__dirname}/settings srcdir=$release_srcdir dstdir=$release_dstdir -. /home/staging/promote/_promote.sh $1 +. ${__dirname}/_promote.sh $site $2 -latest=$(ls $dstdir | grep ^v | tail -1) -rm -f ${dstdir}/latest -ln -s $latest ${dstdir}/latest +/home/dist/tools/latest-linker/latest-linker.js /home/dist/${site}/release/ diff --git a/setup/www/tools/promote/resha_release.sh b/setup/www/tools/promote/resha_release.sh index 9034c6350..cdd46b495 100755 --- a/setup/www/tools/promote/resha_release.sh +++ b/setup/www/tools/promote/resha_release.sh @@ -1,6 +1,20 @@ #!/usr/bin/env bash -. /home/staging/promote/settings +site=$1 + +__dirname="$(CDPATH= cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [ "X$site" != "Xiojs" ] && [ "X$site" != "Xnodejs" ]; then + echo "Usage: resha_release.sh < iojs | nodejs > " + exit 1 +fi + +if [ "X$2" == "X" ]; then + echo "Usage: resha_release.sh < iojs | nodejs > " + exit 1 +fi + +. ${__dirname}/settings dstdir=$release_dstdir @@ -9,4 +23,6 @@ if [ "X${1}" == "X" ]; then exit 1 fi -/home/staging/promote/_resha.sh $dstdir $1 +${__dirname}/_resha.sh $site $dstdir $2 + +/home/nodejs/queue-cdn-purge.sh $site \ No newline at end of file diff --git a/setup/www/tools/promote/settings b/setup/www/tools/promote/settings index 6a79dcc05..20c827703 100755 --- a/setup/www/tools/promote/settings +++ b/setup/www/tools/promote/settings @@ -1,5 +1,5 @@ -staging_rootdir=/home/staging/staging/ -dist_rootdir=/home/dist/public/ +staging_rootdir=/home/staging/${site}/ +dist_rootdir=/home/dist/${site}/ nightly_srcdir=${staging_rootdir}nightly next_nightly_srcdir=${staging_rootdir}next-nightly release_srcdir=${staging_rootdir}release From d0ff6ad1e91b9aa7b6a586109dc557dd33439295 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Tue, 1 Sep 2015 13:04:04 +1000 Subject: [PATCH 10/15] fix nightly promote script --- setup/www/tools/promote/promote_nightly.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/www/tools/promote/promote_nightly.sh b/setup/www/tools/promote/promote_nightly.sh index 46993b029..7684b5632 100755 --- a/setup/www/tools/promote/promote_nightly.sh +++ b/setup/www/tools/promote/promote_nightly.sh @@ -21,9 +21,9 @@ fi srcdir=$nightly_srcdir dstdir=$nightly_dstdir -. ${__dirname}/_promote.sh $site $2 +. ${__dirname}/_promote.sh $site srcdir=$next_nightly_srcdir dstdir=$next_nightly_dstdir -. ${__dirname}/_promote.sh $site $2 +. ${__dirname}/_promote.sh $site From 54961dd8b0df8cc57d0e5b889f22b0af0c3a745b Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Tue, 1 Sep 2015 13:04:11 +1000 Subject: [PATCH 11/15] fixed redirects and whatnot --- setup/www/resources/config/nodejs.org | 137 +++++++++++++++++++++++--- 1 file changed, 126 insertions(+), 11 deletions(-) diff --git a/setup/www/resources/config/nodejs.org b/setup/www/resources/config/nodejs.org index cfb52c5b0..c2866573f 100644 --- a/setup/www/resources/config/nodejs.org +++ b/setup/www/resources/config/nodejs.org @@ -7,7 +7,103 @@ server { listen [::]:80; server_name www.nodejs.org nodejs.org; - return 301 https://nodejs.org$request_uri; + access_log /var/log/nginx/nodejs.org-access.log; + error_log /var/log/nginx/nodejs.org-error.log; + + keepalive_timeout 60; + server_tokens off; + + gzip on; + gzip_static on; + gzip_disable "MSIE [1-6]\."; + gzip_types text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript; + + # let the blog.nodejs.org redirector handle this + location ~ /blog(.*) { + rewrite ^/blog(.*) http://blog.nodejs.org/$1 permanent; + } + + location ~ ^/(?!(dist/|dist$|\.json$)) { + rewrite ^ https://nodejs.org$request_uri permanent; + } + + root /home/www/nodejs; + default_type text/plain; + index index.html; + + if ($host ~* ^www\.){ + rewrite ^(.*)$ http://$server_name$1; + } + + location / { + location ~ \.json$ { + add_header access-control-allow-origin *; + } + } + + location /dist { + alias /home/dist/nodejs/release/; + autoindex on; + default_type text/plain; + + location ~ \.json$ { + add_header access-control-allow-origin *; + } + } +} + +server { + listen 80; + listen [::]:80; + server_name blog.nodejs.org; + + rewrite ^/\d+/\d+/\d+/(?:node-v(?:ersion-)?|version-)(\d+)[-\.](\d+)[-\.](\d+).*$ https://nodejs.org/en/blog/release/v$1.$2.$3/ permanent; + + rewrite ^/2015/05/16/node-leaders-are-building-an-open-foundation/$ https://nodejs.org/en/blog/community/node-leaders-building-open-neutral-foundation/ permanent; + rewrite ^/2015/05/16/the-nodejs-foundation-benefits-all/$ https://nodejs.org/en/blog/community/foundation-benefits-all/ permanent; + rewrite ^/2014/01/17/nodejs-road-ahead/$ https://nodejs.org/en/blog/nodejs-road-ahead/ permanent; + rewrite ^/2013/12/03/bnoordhuis-departure/$ https://nodejs.org/en/blog/uncategorized/bnoordhuis-departure/ permanent; + rewrite ^/2013/11/26/npm-post-mortem/$ https://nodejs.org/en/blog/npm/2013-outage-postmortem/ permanent; + rewrite ^/2013/10/22/cve-2013-4450-http-server-pipeline-flood-dos/$ https://nodejs.org/en/blog/vulnerability/http-server-pipeline-flood-dos/ permanent; + rewrite ^/2015/05/08/transitions/$ https://nodejs.org/en/blog/community/transitions/ permanent; + rewrite ^/2015/05/08/next-chapter/$ https://nodejs.org/en/blog/community/next-chapter/ permanent; + rewrite ^/2013/02/08/peer-dependencies/$ https://nodejs.org/en/blog/npm/peer-dependencies/ permanent; + rewrite ^/2012/12/21/streams2/$ https://nodejs.org/en/blog/feature/streams2/ permanent; + rewrite ^/2012/09/30/bert-belder-libuv-lxjs-2012/$ https://nodejs.org/en/blog/video/bert-belder-libuv-lxjs-2012/ permanent; + rewrite ^/2012/05/08/bryan-cantrill-instrumenting-the-real-time-web/$ https://nodejs.org/en/blog/video/bryan-cantrill-instrumenting-the-real-time-web/ permanent; + rewrite ^/2012/05/07/http-server-security-vulnerability-please-upgrade-to-0-6-17/$ https://nodejs.org/en/blog/vulnerability/http-server-security-vulnerability-please-upgrade-to-0-6-17/ permanent; + rewrite ^/2012/05/02/multi-server-continuous-deployment-with-fleet/$ https://nodejs.org/en/blog/module/multi-server-continuous-deployment-with-fleet/ permanent; + rewrite ^/2012/04/25/profiling-node-js/$ https://nodejs.org/en/blog/uncategorized/profiling-node-js/ permanent; + rewrite ^/2012/03/28/service-logging-in-json-with-bunyan/$ https://nodejs.org/en/blog/module/service-logging-in-json-with-bunyan/ permanent; + rewrite ^/2012/02/27/managing-node-js-dependencies-with-shrinkwrap/$ https://nodejs.org/en/blog/npm/managing-node-js-dependencies-with-shrinkwrap/ permanent; + rewrite ^/2011/12/15/growing-up/$ https://nodejs.org/en/blog/uncategorized/growing-up/ permanent; + rewrite ^/2011/10/26/version-0-6/$ https://nodejs.org/en/blog/uncategorized/version-0-6/ permanent; + rewrite ^/2011/10/05/an-easy-way-to-build-scalable-network-programs/$ https://nodejs.org/en/blog/uncategorized/an-easy-way-to-build-scalable-network-programs/ permanent; + rewrite ^/2011/09/23/libuv-status-report/$ https://nodejs.org/en/blog/uncategorized/libuv-status-report/ permanent; + rewrite ^/2011/09/08/ldapjs-a-reprise-of-ldap/$ https://nodejs.org/en/blog/uncategorized/ldapjs-a-reprise-of-ldap/ permanent; + rewrite ^/2011/08/29/some-new-node-projects/$ https://nodejs.org/en/blog/uncategorized/some-new-node-projects/ permanent; + rewrite ^/2011/08/12/the-videos-from-node-meetup/$ https://nodejs.org/en/blog/uncategorized/the-videos-from-node-meetup/ permanent; + rewrite ^/2011/08/03/node-meetup-this-thursday/$ https://nodejs.org/en/blog/uncategorized/node-meetup-this-thursday/ permanent; + rewrite ^/2011/07/11/evolving-the-node-js-brand/$ https://nodejs.org/en/blog/uncategorized/evolving-the-node-js-brand/ permanent; + rewrite ^/2011/06/24/porting-node-to-windows-with-microsoft.+s-help/$ https://nodejs.org/en/blog/uncategorized/porting-node-to-windows-with-microsofts-help/ permanent; + rewrite ^/2011/05/01/npm-1-0-released/$ https://nodejs.org/en/blog/npm/npm-1-0-released/ permanent; + rewrite ^/2011/04/29/trademark/$ https://nodejs.org/en/blog/uncategorized/trademark/ permanent; + rewrite ^/2011/04/28/node-office-hours-cut-short/$ https://nodejs.org/en/blog/uncategorized/node-office-hours-cut-short/ permanent; + rewrite ^/2011/04/07/npm-1-0-link/$ https://nodejs.org/en/blog/npm/npm-1-0-link/ permanent; + rewrite ^/2011/04/05/development-environment/$ https://nodejs.org/en/blog/uncategorized/development-environment/ permanent; + rewrite ^/2014/12/05/listening-to-the-community/$ https://nodejs.org/en/blog/advisory-board/listening-to-the-community/ permanent; + rewrite ^/2014/12/03/advisory-board-update/$ https://nodejs.org/en/blog/advisory-board/advisory-board-update/ permanent; + rewrite ^/2011/03/25/jobs-nodejs-org/$ https://nodejs.org/en/blog/uncategorized/jobs-nodejs-org/ permanent; + rewrite ^/2011/03/24/npm-1-0-global-vs-local-installation/$ https://nodejs.org/en/blog/npm/npm-1-0-global-vs-local-installation/ permanent; + rewrite ^/2011/03/24/office-hours/$ https://nodejs.org/en/blog/uncategorized/office-hours/ permanent; + rewrite ^/2011/03/18/npm-1-0-the-new-ls/$ https://nodejs.org/en/blog/npm/npm-1-0-the-new-ls/ permanent; + rewrite ^/2011/03/18/welcome-to-the-node-blog/$ https://nodejs.org/en/blog/video/welcome-to-the-node-blog/ permanent; + rewrite ^/2014/07/31/v8-memory-corruption-stack-overflow/$ https://nodejs.org/en/blog/vulnerability/v8-memory-corruption-stack-overflow/ permanent; + rewrite ^/2014/07/29/building-nodejs-together/$ https://nodejs.org/en/blog/community/building-nodejs-together/ permanent; + rewrite ^/2014/06/16/openssl-and-breaking-utf-8-change/$ https://nodejs.org/en/blog/vulnerability/openssl-and-utf8/ permanent; + rewrite ^/2014/06/11/notes-from-the-road/$ https://nodejs.org/en/blog/uncategorized/notes-from-the-road/ permanent; + + rewrite ^/(.*)$ https://nodejs.org/en/blog/ permanent; } server { @@ -15,8 +111,7 @@ server { listen [::]:80; server_name doc.nodejs.org docs.nodejs.org; - rewrite ^/(v[0-9]+\.[0-9]+\.[0-9]+)(/?.*)$ https://nodejs.org/docs/$1$2 permanent; - rewrite /(.*)$ https://nodejs.org/docs/latest/$1 permanent; + rewrite ^/(.*)$ https://nodejs.org/en/docs/ permanent; } server { @@ -24,8 +119,7 @@ server { listen [::]:80; server_name api.nodejs.org; - rewrite ^/(v[0-9]+\.[0-9]+\.[0-9]+)(/?.*)$ https://nodejs.org/docs/$1/api$2 permanent; - rewrite /(.*)$ https://nodejs.org/docs/latest/api/$1 permanent; + rewrite ^/(.*)$ https://nodejs.org/api/ permanent; } server { @@ -33,14 +127,14 @@ server { listen [::]:80; server_name dist.nodejs.org; - rewrite /(.*)$ http://nodejs.org/dist/$1 permanent; + rewrite ^/(.*)$ https://nodejs.org/dist/ permanent; } server { listen 443 default_server ssl spdy; listen [::]:443 default_server ipv6only=on ssl spdy; - server_name nodejs.org; + server_name nodejs.org www.nodejs.org; ssl_certificate ssl/nodejs_chained.crt; ssl_certificate_key ssl/nodejs.key; @@ -66,9 +160,9 @@ server { resolver 8.8.4.4 8.8.8.8 valid=300s; resolver_timeout 10s; - add_header Strict-Transport-Security max-age=63072000; - add_header X-Frame-Options DENY; - add_header X-Content-Type-Options nosniff; + #add_header Strict-Transport-Security max-age=63072000; + #add_header X-Frame-Options DENY; + #add_header X-Content-Type-Options nosniff; access_log /var/log/nginx/nodejs.org-access.log; error_log /var/log/nginx/nodejs.org-error.log; @@ -79,7 +173,7 @@ server { gzip_types text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript; if ($host ~* ^www\.){ - rewrite ^(.*)$ https://nodejs.org$1; + rewrite ^(.*)$ https://$server_name$1; } location /documentation/ { @@ -127,4 +221,25 @@ server { add_header access-control-allow-origin *; } } + + rewrite ^/about/advisory-board/(.*)$ https://$server_name/en/about/organization/ permanent; + rewrite ^/advisory-board/(.*)$ https://$server_name/en/about/organization/ permanent; + rewrite ^/about/organization/tsc-meetings/$ https://$server_name/en/foundation/tsc/minutes/ permanent; + rewrite ^/about/organization/tsc-meetings/(.*?)/minutes.html$ https://$server_name/en/foundation/tsc/minutes/$1/ permanent; + rewrite ^/about/security/$ https://$server_name/en/security/ permanent; + rewrite ^/contribute/$ https://$server_name/en/get-involved/ permanent; + rewrite ^/contribute/accepting_contributions.html$ https://github.com/nodejs/dev-policy permanent; + rewrite ^/contribute/becoming_collaborator.html$ https://$server_name/en/get-involved/ permanent; + rewrite ^/contribute/code_contributions/$ https://$server_name/en/get-involved/ permanent; + rewrite ^/contribute/code_contributions/workflow.html$ https://$server_name/en/get-involved/ permanent; + rewrite ^/documentation/(.*)$ https://$server_name/en/docs/ permanent; + rewrite ^/foundation/blog.html$ https://$server_name/en/blog/ permanent; + rewrite ^/foundation/members.html$ https://$server_name/en/foundation/members/ permanent; + + rewrite ^/images/foundation-visual-guidelines.pdf$ https://$server_name/static/documents/foundation-visual-guidelines.pdf permanent; + rewrite ^/images/logos/js-black(.*)$ https://$server_name/static/images/logos/js-black$1 permanent; + rewrite ^/images/logos/nodejs-(.*)$ https://$server_name/static/images/logos/nodejs-$1 permanent; + rewrite ^/images/node-foundation-by-laws.pdf$ https://$server_name/static/documents/node-foundation-by-laws.pdf permanent; + rewrite ^/images/.*trademark-policy.pdf$ https://$server_name/static/documents/trademark-policy.pdf permanent; + rewrite ^/video/(.*)$ https://$server_name/static/video/$1 permanent; } From a8a3d7964d25cb6a4306f6c0a2a8571559260a1f Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Tue, 1 Sep 2015 21:32:57 +1000 Subject: [PATCH 12/15] more redirects and fixes for nodejs.org switchover --- setup/www/resources/config/iojs.org | 12 +- setup/www/resources/config/nodejs.org | 113 ++++++++++++++++-- setup/www/tasks/user.yaml | 2 + .../www/tools/latest-linker/latest-linker.js | 66 ++++++++-- setup/www/tools/promote/promote_release.sh | 2 +- 5 files changed, 169 insertions(+), 26 deletions(-) diff --git a/setup/www/resources/config/iojs.org b/setup/www/resources/config/iojs.org index 14d266d7b..a412eb1f2 100644 --- a/setup/www/resources/config/iojs.org +++ b/setup/www/resources/config/iojs.org @@ -83,8 +83,18 @@ server { } } + location /docs { + alias /home/dist/iojs/docs/; + autoindex on; + default_type text/html; + + location ~ \.json$ { + add_header access-control-allow-origin *; + } + } + location /api { - alias /home/dist/iojs/release/latest/docs/api; + alias /home/dist/iojs/docs/latest/api; autoindex on; default_type text/plain; diff --git a/setup/www/resources/config/nodejs.org b/setup/www/resources/config/nodejs.org index c2866573f..7e079c8d4 100644 --- a/setup/www/resources/config/nodejs.org +++ b/setup/www/resources/config/nodejs.org @@ -103,6 +103,10 @@ server { rewrite ^/2014/06/16/openssl-and-breaking-utf-8-change/$ https://nodejs.org/en/blog/vulnerability/openssl-and-utf8/ permanent; rewrite ^/2014/06/11/notes-from-the-road/$ https://nodejs.org/en/blog/uncategorized/notes-from-the-road/ permanent; + rewrite ^/((atom|feed|rss)(/|\.xml)|(feed))$ https://nodejs.org/en/feed/blog.xml permanent; + rewrite ^/(feed/)?release/?$ https://nodejs.org/en/feed/releases.xml permanent; + rewrite ^/(feed/)?vulnerability/?$ https://nodejs.org/en/feed/vulnerability.xml permanent; + rewrite ^/(.*)$ https://nodejs.org/en/blog/ permanent; } @@ -134,7 +138,7 @@ server { listen 443 default_server ssl spdy; listen [::]:443 default_server ipv6only=on ssl spdy; - server_name nodejs.org www.nodejs.org; + server_name nodejs.org www.nodejs.org blog.nodejs.org; ssl_certificate ssl/nodejs_chained.crt; ssl_certificate_key ssl/nodejs.key; @@ -172,10 +176,16 @@ server { gzip_disable "MSIE [1-6]\."; gzip_types text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript; - if ($host ~* ^www\.){ + if ($host ~* ^www\.) { rewrite ^(.*)$ https://$server_name$1; } + if ($host ~* ^blog\.) { + # keep the blog rewrites in the blog.nodejs.org:80 config + # and shunt blog.nodejs.org:443 requests through there + rewrite ^(.*)$ http://blog.nodejs.org$1; + } + location /documentation/ { rewrite ^/documentation/api(.*)$ /api$1 permanent; } @@ -212,8 +222,18 @@ server { } } + location /docs { + alias /home/dist/nodejs/docs/; + autoindex on; + default_type text/html; + + location ~ \.json$ { + add_header access-control-allow-origin *; + } + } + location /api { - alias /home/dist/nodejs/release/latest/docs/api; + alias /home/dist/nodejs/docs/latest/api; autoindex on; default_type text/plain; @@ -222,24 +242,93 @@ server { } } - rewrite ^/about/advisory-board/(.*)$ https://$server_name/en/about/organization/ permanent; - rewrite ^/advisory-board/(.*)$ https://$server_name/en/about/organization/ permanent; - rewrite ^/about/organization/tsc-meetings/$ https://$server_name/en/foundation/tsc/minutes/ permanent; + location /robots.txt { + alias /home/www/nodejs/static/robots.txt; + default_type text/plain; + } + + rewrite ^/about/advisory-board(.*)$ https://$server_name/en/about/organization/ permanent; + rewrite ^/advisory-board(.*)$ https://$server_name/en/about/organization/ permanent; + rewrite ^/about/organization/tsc-meetings/?$ https://$server_name/en/foundation/tsc/minutes/ permanent; rewrite ^/about/organization/tsc-meetings/(.*?)/minutes.html$ https://$server_name/en/foundation/tsc/minutes/$1/ permanent; - rewrite ^/about/security/$ https://$server_name/en/security/ permanent; - rewrite ^/contribute/$ https://$server_name/en/get-involved/ permanent; + rewrite ^/about/security/?$ https://$server_name/en/security/ permanent; + rewrite ^/contribute/?$ https://$server_name/en/get-involved/ permanent; rewrite ^/contribute/accepting_contributions.html$ https://github.com/nodejs/dev-policy permanent; rewrite ^/contribute/becoming_collaborator.html$ https://$server_name/en/get-involved/ permanent; - rewrite ^/contribute/code_contributions/$ https://$server_name/en/get-involved/ permanent; + rewrite ^/contribute/code_contributions/?$ https://$server_name/en/get-involved/ permanent; rewrite ^/contribute/code_contributions/workflow.html$ https://$server_name/en/get-involved/ permanent; - rewrite ^/documentation/(.*)$ https://$server_name/en/docs/ permanent; + rewrite ^/documentation(.*)$ https://$server_name/en/docs/ permanent; rewrite ^/foundation/blog.html$ https://$server_name/en/blog/ permanent; rewrite ^/foundation/members.html$ https://$server_name/en/foundation/members/ permanent; - rewrite ^/images/foundation-visual-guidelines.pdf$ https://$server_name/static/documents/foundation-visual-guidelines.pdf permanent; rewrite ^/images/logos/js-black(.*)$ https://$server_name/static/images/logos/js-black$1 permanent; rewrite ^/images/logos/nodejs-(.*)$ https://$server_name/static/images/logos/nodejs-$1 permanent; rewrite ^/images/node-foundation-by-laws.pdf$ https://$server_name/static/documents/node-foundation-by-laws.pdf permanent; rewrite ^/images/.*trademark-policy.pdf$ https://$server_name/static/documents/trademark-policy.pdf permanent; - rewrite ^/video/(.*)$ https://$server_name/static/video/$1 permanent; + rewrite ^/video(.*)$ https://$server_name/static/video$1 permanent; + rewrite ^/changelog.html$ https://github.com/nodejs/node/blob/master/CHANGELOG.md permanent; + rewrite ^/api.html$ https://$server_name/api/ permanent; + rewrite ^/index.html$ https://$server_name/ permanent; + + rewrite ^/(20\d\d/\d\d/\d\d/.*)$ http://blog.nodejs.org/$1 permanent; + + rewrite ^/about/?$ https://$server_name/en/about/ permanent; + rewrite ^/about/advisory-board/?$ https://$server_name/en/about/organization/ permanent; + rewrite ^/about/advisory-board/members/?$ https://$server_name/en/about/organization/ permanent; + rewrite ^/about/organization/?$ https://$server_name/en/about/organization/ permanent; + rewrite ^/about/organization/tsc-meetings/(\d\d\d\d-\d\d-\d\d)/?$ https://$server_name/en/foundation/tsc/minutes/$1/ permanent; + rewrite ^/about/organization/tsc-meetings/(\d\d\d\d-\d\d-\d\d)/minutes.html$ https://$server_name/en/foundation/tsc/minutes/$1/ permanent; + rewrite ^/about/releases/?$ https://$server_name/en/about/releases/ permanent; + rewrite ^/about/resources/?$ https://$server_name/en/about/resources/ permanent; + rewrite ^/about/security/?$ https://$server_name/en/security/ permanent; + rewrite ^/about/trademark/?$ https://$server_name/en/about/trademark/ permanent; + rewrite ^/blog/?$ https://$server_name/en/blog/ permanent; + rewrite ^/community/?$ https://$server_name/en/get-involved/ permanent; + rewrite ^/foundation/?$ https://$server_name/en/foundation/ permanent; + + rewrite ^/dist/staging/(.*)$ https://$server_name/dist/$1 permanent; + + # RSS Feeds + rewrite ^/((atom|feed|rss)(/|\.xml)|(feed))$ https://$server_name/en/feed/blog.xml permanent; + rewrite ^/feed/release/?$ https://$server_name/en/feed/releases.xml permanent; + rewrite ^/feed/vulnerability/?$ https://$server_name/en/feed/vulnerability.xml permanent; + + # Asset rewrites + rewrite ^/(?!static/)?favicon\.ico$ https://$server_name/static/favicon.ico permanent; + rewrite ^/apple-touch-icon.*\.png$ https://$server_name/static/apple-touch-icon.png permanent; + rewrite ^/trademark-policy.pdf$ https://$server_name/static/documents/trademark-policy.pdf permanent; + + rewrite ^/logos/ https://$server_name/static/images/logos/ permanent; + rewrite ^/logos/monitor.png https://$server_name/static/images/logos/monitor.png permanent; + rewrite ^/logos/nodejs(.*)$ https://$server_name/static/images/logos/nodejs$1 permanent; + + # Foundation member logos + rewrite ^/images/logos/apigee_logo_md.png$ https://$server_name/static/images/foundation/apigee_logo_md.png permanent; + rewrite ^/images/logos/codefreshLogo.png$ https://$server_name/static/images/foundation/codefreshLogo.png permanent; + rewrite ^/images/logos/codefreshLogo2.png$ https://$server_name/static/images/foundation/codefreshLogo2.png permanent; + rewrite ^/images/logos/digitaloceanLogo.png$ https://$server_name/static/images/foundation/digitaloceanLogo.png permanent; + rewrite ^/images/logos/famousLogo.jpg$ https://$server_name/static/images/foundation/famousLogo.jpg permanent; + rewrite ^/images/logos/famousLogo.png$ https://$server_name/static/images/foundation/famousLogo.png permanent; + rewrite ^/images/logos/fidelityLogo.png$ https://$server_name/static/images/foundation/fidelityLogo.png permanent; + rewrite ^/images/logos/godaddyLogo.png$ https://$server_name/static/images/foundation/godaddyLogo.png permanent; + rewrite ^/images/logos/grouponLogo.png$ https://$server_name/static/images/foundation/grouponLogo.png permanent; + rewrite ^/images/logos/ibmLogo.png$ https://$server_name/static/images/foundation/ibmLogo.png permanent; + rewrite ^/images/logos/intelLogo.png$ https://$server_name/static/images/foundation/intelLogo.png permanent; + rewrite ^/images/logos/joyentLogo.svg$ https://$server_name/static/images/foundation/joyentLogo.svg permanent; + rewrite ^/images/logos/microsoftLogo.png$ https://$server_name/static/images/foundation/microsoftLogo.png permanent; + rewrite ^/images/logos/microsoftLogo2.png$ https://$server_name/static/images/foundation/microsoftLogo2.png permanent; + rewrite ^/images/logos/modulusLogo.png$ https://$server_name/static/images/foundation/modulusLogo.png permanent; + rewrite ^/images/logos/nearformLogo.png$ https://$server_name/static/images/foundation/nearformLogo.png permanent; + rewrite ^/images/logos/nearformLogo2.png$ https://$server_name/static/images/foundation/nearformLogo2.png permanent; + rewrite ^/images/logos/nodesourceLogo.png$ https://$server_name/static/images/foundation/nodesourceLogo.png permanent; + rewrite ^/images/logos/npmLogo.png$ https://$server_name/static/images/foundation/npmLogo.png permanent; + rewrite ^/images/logos/paypalLogo.png$ https://$server_name/static/images/foundation/paypalLogo.png permanent; + rewrite ^/images/logos/sapLogo.png$ https://$server_name/static/images/foundation/sapLogo.png permanent; + rewrite ^/images/logos/saucelabsLogo.png$ https://$server_name/static/images/foundation/saucelabsLogo.png permanent; + rewrite ^/images/logos/saucelabsLogo2.png$ https://$server_name/static/images/foundation/saucelabsLogo2.png permanent; + rewrite ^/images/logos/strongloopLogo.png$ https://$server_name/static/images/foundation/strongloopLogo.png permanent; + rewrite ^/images/logos/yldLogo.png$ https://$server_name/static/images/foundation/yldLogo.png permanent; + rewrite ^/images/logos/yldLogo2.png$ https://$server_name/static/images/foundation/yldLogo2.png permanent; + + rewrite ^/images/(.*) https://$server_name/static/images/$1 permanent; } diff --git a/setup/www/tasks/user.yaml b/setup/www/tasks/user.yaml index 55cd1715f..d480feec6 100644 --- a/setup/www/tasks/user.yaml +++ b/setup/www/tasks/user.yaml @@ -68,11 +68,13 @@ - nodejs/rc - nodejs/release - nodejs/test + - nodejs/docs - iojs/next-nightly - iojs/nightly - iojs/rc - iojs/release - iojs/test + - iojs/docs tags: user - name: User | Insert SSH public key to staging diff --git a/setup/www/tools/latest-linker/latest-linker.js b/setup/www/tools/latest-linker/latest-linker.js index 497239567..f22b2c87e 100755 --- a/setup/www/tools/latest-linker/latest-linker.js +++ b/setup/www/tools/latest-linker/latest-linker.js @@ -9,12 +9,16 @@ const fs = require('fs') if (process.argv.length < 3) - throw new Error('Please provide a downloads directory location') + throw new Error('Usage: latest-linker.js [docs directory]') -const dir = process.argv[2] +const dir = path.resolve(process.argv[2]) + , docsDir = process.argv[3] && path.resolve(process.argv[3]) if (!fs.statSync(dir).isDirectory()) - throw new Error('Please provide a downloads directory location') + throw new Error('Usage: latest-linker.js [docs directory]') + +if (docsDir && !fs.statSync(docsDir).isDirectory()) + throw new Error('Usage: latest-linker.js [docs directory]') map( fs.readdirSync(dir).map(function (d) { return path.join(dir, d) }) @@ -24,16 +28,19 @@ map( , afterMap ) -function afterMap (err, dirs) { +function afterMap (err, allDirs) { if (err) throw err - dirs = dirs.filter(function (d) { return d.stat && d.stat.isDirectory() }) - .map(function (d) { return path.basename(d.d) }) - .map(function (d) { try { return semver(d) } catch (e) {} }) - .filter(Boolean) - .filter(function (d) { return semver.satisfies(d, '~0.10 || ~0.12 || >= 1.0') }) - .map(function (d) { return d.raw }) + allDirs = allDirs.filter(function (d) { return d.stat && d.stat.isDirectory() }) + .map(function (d) { return path.basename(d.d) }) + .map(function (d) { try { return semver(d) } catch (e) {} }) + .filter(Boolean) + + makeDocsLinks(allDirs.map(function (v) { return v.raw })) + + var dirs = allDirs.filter(function (d) { return semver.satisfies(d, '~0.10 || ~0.12 || >= 1.0') }) + .map(function (d) { return d.raw }) dirs.sort(function (d1, d2) { return semver.compare(d1, d2) }) @@ -61,6 +68,31 @@ function afterMap (err, dirs) { } +function makeDocsLinks (versions) { + if (!docsDir) + return + + versions.forEach(function (version) { + var src = path.join(dir, version, 'docs') + , dst = path.join(docsDir, version) + + fs.stat(src, function (err, stat) { + if (err) + throw err + + if (!stat.isDirectory()) + return + + fs.unlink(dst, function () { + fs.symlink(src, dst, function (err) { + if (err) + throw err + }) + }) + }) + }) +} + function link (line, dirs) { var range = line ? `${line[0] == '0' ? '~' : '^'}${line}` : '*' , max = semver.maxSatisfying(dirs, range) @@ -69,10 +101,20 @@ function link (line, dirs) { function symlink (name) { var dst = path.join(dir, name) + , src = path.join(dir, max) + try { fs.unlinkSync(dst) } catch (e) {} - fs.symlinkSync(path.join(dir, max), dst) - } + fs.symlinkSync(src, dst) + if (!docsDir) + return + + var dsrc = path.join(dir, max, 'docs') + , ddst = path.join(docsDir, name) + + try { fs.unlinkSync(ddst) } catch (e) {} + fs.symlinkSync(dsrc, ddst) + } if (line) { symlink(`latest-v${line}`) diff --git a/setup/www/tools/promote/promote_release.sh b/setup/www/tools/promote/promote_release.sh index 9371406ed..d8d55d0ef 100755 --- a/setup/www/tools/promote/promote_release.sh +++ b/setup/www/tools/promote/promote_release.sh @@ -23,4 +23,4 @@ dstdir=$release_dstdir . ${__dirname}/_promote.sh $site $2 -/home/dist/tools/latest-linker/latest-linker.js /home/dist/${site}/release/ +/home/dist/tools/latest-linker/latest-linker.js /home/dist/${site}/release/ /home/dist/${site}/docs/ From 9f4784b8205c322a5afffc24a49dae560e2f5d91 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Wed, 2 Sep 2015 11:39:57 +1000 Subject: [PATCH 13/15] fix logrotate file name format --- setup/www/resources/config/logrotate-nginx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/www/resources/config/logrotate-nginx b/setup/www/resources/config/logrotate-nginx index eb45caec3..03582b84a 100644 --- a/setup/www/resources/config/logrotate-nginx +++ b/setup/www/resources/config/logrotate-nginx @@ -9,8 +9,9 @@ notifempty create 0640 www-data adm dateext - dateformat "%Y%m%d" + dateformat .%Y%m%d.%s dateyesterday + maxsize 500M sharedscripts prerotate if [ -d /etc/logrotate.d/httpd-prerotate ]; then \ From 5cd6609a4fa85ef846296b74a735319eae337a01 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Wed, 2 Sep 2015 11:40:05 +1000 Subject: [PATCH 14/15] introduce legacy /docs/ resources redirects --- setup/www/resources/config/nodejs.org | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/setup/www/resources/config/nodejs.org b/setup/www/resources/config/nodejs.org index 7e079c8d4..cc66ccb8d 100644 --- a/setup/www/resources/config/nodejs.org +++ b/setup/www/resources/config/nodejs.org @@ -330,5 +330,15 @@ server { rewrite ^/images/logos/yldLogo.png$ https://$server_name/static/images/foundation/yldLogo.png permanent; rewrite ^/images/logos/yldLogo2.png$ https://$server_name/static/images/foundation/yldLogo2.png permanent; + # legacy v0.12.x docs/ html + rewrite ^/pipe.css$ https://$server_name/static/legacy/pipe.css permanent; + rewrite ^/sh_vim-dark.css$ https://$server_name/static/legacy/sh_vim-dark.css permanent; + rewrite ^/lfcollab.css$ https://$server_name/static/legacy/lfcollab.css permanent; + rewrite ^/images/stripe.png$ https://$server_name/static/legacy/stripe.png permanent; + rewrite ^/images/walmart-thumb.jpg$ https://$server_name/static/legacy/walmart-thumb.jpg permanent; + rewrite ^/sh_main.js$ https://$server_name/static/legacy/sh_main.js permanent; + rewrite ^/sh_javascript.min.js$ https://$server_name/static/legacy/sh_javascript.min.js permanent; + + rewrite ^/images/(.*) https://$server_name/static/images/$1 permanent; } From 802ca03588ed6a5844830fb6dfbf876b717dbffc Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Wed, 2 Sep 2015 15:02:29 +1000 Subject: [PATCH 15/15] tweak log_format to show x-forwarded-for --- setup/www/resources/config/iojs.org | 2 +- setup/www/resources/config/nodejs.org | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/setup/www/resources/config/iojs.org b/setup/www/resources/config/iojs.org index a412eb1f2..e324196f4 100644 --- a/setup/www/resources/config/iojs.org +++ b/setup/www/resources/config/iojs.org @@ -39,7 +39,7 @@ server { add_header X-Frame-Options DENY; add_header X-Content-Type-Options nosniff; - access_log /var/log/nginx/iojs.org-access.log; + access_log /var/log/nginx/iojs.org-access.log nodejs; error_log /var/log/nginx/iojs.org-error.log; gzip on; diff --git a/setup/www/resources/config/nodejs.org b/setup/www/resources/config/nodejs.org index cc66ccb8d..05b9650bb 100644 --- a/setup/www/resources/config/nodejs.org +++ b/setup/www/resources/config/nodejs.org @@ -1,13 +1,22 @@ -log_format main '$remote_addr - $remote_user [$time_local] $request ' - '"$status" $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; +# default combined nginx log_format: +#log_format combined '$remote_addr - $remote_user [$time_local] ' +# '"$request" $status $body_bytes_sent ' +# '"$http_referer" "$http_user_agent"'; + +log_format joyent '$remote_addr - $remote_user [$time_local] ' + '$request "$status" $body_bytes_sent ' + '"$http_referer" "$http_user_agent" "$http_x_forwarded_for"'; + +log_format nodejs '$remote_addr - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent" "$http_x_forwarded_for"'; server { listen 80 default_server; listen [::]:80; server_name www.nodejs.org nodejs.org; - access_log /var/log/nginx/nodejs.org-access.log; + access_log /var/log/nginx/nodejs.org-access.log nodejs; error_log /var/log/nginx/nodejs.org-error.log; keepalive_timeout 60; @@ -168,7 +177,7 @@ server { #add_header X-Frame-Options DENY; #add_header X-Content-Type-Options nosniff; - access_log /var/log/nginx/nodejs.org-access.log; + access_log /var/log/nginx/nodejs.org-access.log nodejs; error_log /var/log/nginx/nodejs.org-error.log; gzip on;