feat: 🎸 No config needed for development (#42)

https://trello.com/c/Re9paxRB/316-remove-need-for-env-file

Author: JohanObrink

.env

@@ -10,6 +10,7 @@ COPY ./package.json .
 COPY ./package-lock.json .
 RUN npm install --production
 
+COPY ./scripts ./scripts
 COPY ./migrations ./migrations
 COPY ./lib ./lib
 

Dockerfile

@@ -7,34 +7,47 @@
 
 Backend for managing consents and data flow
 
+## What data is stored and where?
+
+- Personal data is stored encrypted in the `PDS` (eg. Dropbox) specified by the user
+- Metadata (consents, permissions, id:s) and client data are stored in `Postgres` (this will probably change)
+- Temporary consent requests are stored in `Redis`
+- Requests and errors are logged in `elasticsearch/APM` for debugging purposes in test/dev
+
 ## Configuration
-Create a file named `.env` in the project directory, example for a developers machine:
+
+The application runs with default cofiguration if `NODE_ENV` is `development`
+(default) or `test` (default when running tests). When running in `production`
+these environment variables *must* be set or the application will not start.
+
+These variables may optionally be overridden in an `.env` file on your dev
+machine.
+
 ```
-HOST=http://localhost:3000
-# development, test or production
+# Run mode: development|test|production
 NODE_ENV=development
-# optional, defaults to 3000
+
+# Host port, defaults to 3000
 PORT=3000
-# optional, apm will not be used if APM_SERVER is not set
-APM_SERVER=http://localhost:8200
-# optional, defaults to ''
-APM_TOKEN=abc
+# Full host address, defaults to http://[your ip]:[port]
+HOST=https://myservice.work
 
-# For Postgres migrations
-DATABASE_URL=postgres://postgresuser:postgrespassword@localhost:5432/mydata
+# Postgres
+PGHOST='postgres-service'
+PGPORT='5432'
+PGUSER='some-username'
+PGPASSWORD='some-very-secret-password'
+PGDATABASE='egendata'
 
+# Application performance management. turned off if val is ''
+APM_SERVER=http://apm-service:8200
+APM_TOKEN=mysecrettoken
 
-# Client keys (generate your own or use these FOR TESTING ONLY)
-PUBLIC_KEY="-----BEGIN RSA PUBLIC KEY-----\nMIGJAoGBAOBHQz1EEVPboCx5o1jL2Dlhf5hCFuyQVMCnRoI7qH/zE8A7OR3uw74u\nqvHguOzyK5RO/slRvHz6aX7sgwpiOkXHh4VDWRwRb0gvnFIopwe3Y7fn1zLkpsET\nGqPgWvYmSYIT5dwwlrkYY6ek0oH3amYm186SNUFVDbzSf+Pyy7ILAgMBAAE=\n-----END RSA PUBLIC KEY-----\n"
+# Operator key (generate your own!!!)
 PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\nMIICXQIBAAKBgQDgR0M9RBFT26AseaNYy9g5YX+YQhbskFTAp0aCO6h/8xPAOzkd\n7sO+Lqrx4Ljs8iuUTv7JUbx8+ml+7IMKYjpFx4eFQ1kcEW9IL5xSKKcHt2O359cy\n5KbBExqj4Fr2JkmCE+XcMJa5GGOnpNKB92pmJtfOkjVBVQ280n/j8suyCwIDAQAB\nAoGBAMwGqBl86ZJy0nSDN2EZF5ujoXJ+dOJBrogP5CmnYfL7y3Ttq1kakwFY7PPb\nLf+HkrN5ZXj5HVJIb14ihFcW4tBR2EtABhuv2J6ZNx0KnDxUj+mJlb7GNgr5eayI\nUibIu8/eQh2+CGMilI/KR8zlRiHpD8BgttfBaRktGIrzklQJAkEA9C8JgnAGUbPp\n3rc3dEZR6pEcOGI5Fjo3uvhbOYO5oa4tJszNF1Fh1oUmn17J6yoMnh0qPG4snL2B\nOgSB8OCOnwJBAOshovf7obbVZFzQ7ikYImT/pqz7f7eV1+Uv1MRfGsXAc0EAXDrh\nAPiJ5icWkeRDCFxaTAy/8lrDGgDcL2CSoRUCQQCem4L4x91C6rMJaEbL7vU8gL8s\n3JgqGOykNLfElwxXubQ4VKUO9Vywo9JfiIlth+WkOlt53zJ5KRqsXcstdB8PAkAo\nw6IfYA6/Reuqc8Z2dWqxG+lnoAqaZ24Qm+RFTz+y/RR+NnPG+W9Tp4SxTiZo7n4q\nlLUOmNCJj72YXJQSKBmpAkAyDc4PrJ3nFt45BOEnRuXE60Lv3VzLPdWggOLcKTbW\nr6NAWQS0VNdXEmJVmdoKFhJAeUvLrXPtBGqPS7HO6A8A\n-----END RSA PRIVATE KEY-----\n"
 ```
-- `PORT` is the port for this service
-- `APM_SERVER` and `APM_TOKEN` is so that this service can reach [APM](https://www.npmjs.com/package/elastic-apm-node) for logging requests and errors
 
-Good luck.
+## Logging
 
-## What data is stored and where?
-- Personal data is stored encrypted in the `PDS` (eg. Dropbox) specified by the user
-- Metadata (consents, permissions, id:s) and client data are stored in `Postgres` (this will probably change)
-- Temporary consent requests are stored in `Redis`
-- Requests and errors are logged in `elasticsearch/APM` for debugging purposes in test/dev
+The Operator is instrumented to use [Elastic APM](https://www.elastic.co/products/apm)
+for performance and error logging.

README.md

@@ -1,6 +1,6 @@
-const pds = require(`../../lib/adapters/pds`)
-const dropbox = require(`../../lib/adapters/pds/dropbox`)
-jest.mock(`../../lib/adapters/pds/dropbox`)
+const pds = require('../../lib/adapters/pds')
+const dropbox = require('../../lib/adapters/pds/dropbox')
+jest.mock('../../lib/adapters/pds/dropbox')
 
 describe('adapters/pds', () => {
   describe('#get', () => {

__test__/adapters/pds.test.js

@@ -54,11 +54,13 @@ describe('data', () => {
         sub: '26eb214f-287b-4def-943c-55a6eefa2d91',
         aud: 'https://smoothoperator.com',
         iss: 'https://mycv.work',
-        paths: [ {
-          domain: 'https://mycv.work',
-          area: 'favorite_cats',
-          data: { txt: 'Some huge JWE' }
-        } ],
+        paths: [
+          {
+            domain: 'https://mycv.work',
+            area: 'favorite_cats',
+            data: { txt: 'Some huge JWE' }
+          }
+        ],
         iat: 1562150432,
         exp: 1562154032
       }
@@ -129,10 +131,12 @@ describe('data', () => {
         sub: 'd82054d3-4115-49a0-ac5c-3325273d53b2',
         aud: 'https://smoothoperator.com',
         iss: 'https://mycv.work',
-        paths: [ {
-          domain: 'https://mycv.work',
-          area: 'favorite_cats'
-        } ],
+        paths: [
+          {
+            domain: 'https://mycv.work',
+            area: 'favorite_cats'
+          }
+        ],
         iat: 1562323351,
         exp: 1562326951
       }

__test__/data.test.js

@@ -56,10 +56,12 @@ describe('tokens', () => {
         sub: 'd82054d3-4115-49a0-ac5c-3325273d53b2',
         aud: 'https://smoothoperator.com',
         iss: 'https://mycv.work',
-        paths: [ {
-          domain,
-          area
-        } ],
+        paths: [
+          {
+            domain,
+            area
+          }
+        ],
         iat: 1562323351,
         exp: 1562326951
       }

__test__/services/tokens.test.js

@@ -1,17 +1,19 @@
+const config = require('../config')
+const { info } = require('../logger')
 let apm
 
-if (process.env.APM_SERVER) {
+if (config.get('APM_SERVER')) {
   apm = require('elastic-apm-node').start({
-    serviceName: process.env.APP_NAME || 'egendata-operator', // Allowed characters: a-z, A-Z, 0-9, -, _, and space
-    secretToken: process.env.APM_TOKEN || '', // Use if APM Server requires a token
-    serverUrl: process.env.APM_SERVER, // Set APM Server URL
-    captureBody: (process.env.NODE_ENV === 'production') // Don't save request body in production
+    serviceName: config.get('APP_NAME'), // Allowed characters: a-z, A-Z, 0-9, -, _, and space
+    secretToken: config.get('APM_TOKEN'), // Use if APM Server requires a token
+    serverUrl: config.get('APM_SERVER'), // Set APM Server URL
+    captureBody: (config.get('NODE_ENV') === 'production') // Don't save request body in production
       ? 'off'
       : 'errors'
   })
-  console.log('APM instrumentation done')
+  info('APM instrumentation done')
 } else {
-  console.log('No APM instrumentation configured')
+  info('No APM instrumentation configured')
 }
 
 function setTransactionName (name) {

lib/adapters/apm.js

@@ -1,17 +1,18 @@
 const { Client } = require('pg')
-const config = {
-  host: process.env.PGHOST || 'localhost',
-  port: process.env.PGPORT || '5432',
-  user: process.env.PGUSER || 'postgresuser',
-  password: process.env.PGPASSWORD || 'postgrespassword',
-  database: process.env.PGDATABASE || 'mydata'
+const config = require('../config')
+const pgconfig = {
+  host: config.get('PGHOST'),
+  port: config.get('PGPORT'),
+  user: config.get('PGUSER'),
+  password: config.get('PGPASSWORD'),
+  database: config.get('PGDATABASE')
 }
 
 const wait = (ms) => new Promise(resolve => setTimeout(() => resolve(), ms))
 
 async function connect (attemptNo = 0) {
   try {
-    const client = new Client(config)
+    const client = new Client(pgconfig)
     await client.connect()
     return client
   } catch (err) {
@@ -49,7 +50,7 @@ async function transaction (queries) {
     await conn.query('BEGIN')
     const results = []
     for (const args of queries) {
-      let result = await conn.query(...args)
+      const result = await conn.query(...args)
       results.push(result)
     }
     await conn.query('COMMIT')
@@ -67,4 +68,4 @@ async function transaction (queries) {
   }
 }
 
-module.exports = { query, multiple, transaction }
+module.exports = { connect, query, multiple, transaction }

lib/adapters/postgres.js

@@ -0,0 +1,104 @@
+require('dotenv').config()
+const { JWK } = require('@panva/jose')
+const config = require('nconf')
+  .argv()
+  .env()
+  .defaults({
+    NODE_ENV: 'development',
+    APP_NAME: 'egendata-operator'
+  })
+
+function exitWith (message) {
+  process.stderr.write(`${message}\n`)
+  process.exit(1)
+}
+
+function privateKey (host, rsaKey) {
+  const rxPEM = /^-----BEGIN RSA PRIVATE KEY-----\n([a-zA-Z0-9+/=]*\n)*-----END RSA PRIVATE KEY-----\n?$/
+  if (typeof rsaKey !== 'string' || !rxPEM.test(rsaKey)) {
+    exitWith('Unknown key format for PRIVATE_KEY')
+  }
+
+  const keyConfig = {
+    kid: `${host}/jwks/operator_key`,
+    use: 'sig'
+  }
+  return JWK.asKey(rsaKey, keyConfig).toJWK(true)
+}
+
+function publicKey ({ e, kid, kty, n, use }) {
+  return { e, kid, kty, n, use }
+}
+
+function checkMissingKeys (config, keys) {
+  const missing = keys.filter(key => {
+    return !config.get(key)
+  })
+  if (missing.length > 0) {
+    exitWith(`Your config is missing the following keys: ${missing.join(', ')}`)
+  }
+}
+
+let ip, PORT, HOST, PGHOST, PGPORT, PGUSER, PGPASSWORD, PGDATABASE, PRIVATE_KEY, PUBLIC_KEY
+
+switch (config.get('NODE_ENV')) {
+  case 'development':
+    ip = require('ip').address()
+    PORT = config.get('PORT') || '3000'
+    HOST = `http://${ip}:${PORT}`
+    PGHOST = config.get('PGHOST') || ip
+    PGPORT = config.get('PGPORT') || '5432'
+    PGUSER = config.get('PGUSER') || 'postgresuser'
+    PGPASSWORD = config.get('PGPASSWORD') || 'postgrespassword'
+    PGDATABASE = config.get('PGDATABASE') || 'mydata'
+    PRIVATE_KEY = privateKey(HOST, config.get('PRIVATE_KEY') || '-----BEGIN RSA PRIVATE KEY-----\nMIICXQIBAAKBgQDgR0M9RBFT26AseaNYy9g5YX+YQhbskFTAp0aCO6h/8xPAOzkd\n7sO+Lqrx4Ljs8iuUTv7JUbx8+ml+7IMKYjpFx4eFQ1kcEW9IL5xSKKcHt2O359cy\n5KbBExqj4Fr2JkmCE+XcMJa5GGOnpNKB92pmJtfOkjVBVQ280n/j8suyCwIDAQAB\nAoGBAMwGqBl86ZJy0nSDN2EZF5ujoXJ+dOJBrogP5CmnYfL7y3Ttq1kakwFY7PPb\nLf+HkrN5ZXj5HVJIb14ihFcW4tBR2EtABhuv2J6ZNx0KnDxUj+mJlb7GNgr5eayI\nUibIu8/eQh2+CGMilI/KR8zlRiHpD8BgttfBaRktGIrzklQJAkEA9C8JgnAGUbPp\n3rc3dEZR6pEcOGI5Fjo3uvhbOYO5oa4tJszNF1Fh1oUmn17J6yoMnh0qPG4snL2B\nOgSB8OCOnwJBAOshovf7obbVZFzQ7ikYImT/pqz7f7eV1+Uv1MRfGsXAc0EAXDrh\nAPiJ5icWkeRDCFxaTAy/8lrDGgDcL2CSoRUCQQCem4L4x91C6rMJaEbL7vU8gL8s\n3JgqGOykNLfElwxXubQ4VKUO9Vywo9JfiIlth+WkOlt53zJ5KRqsXcstdB8PAkAo\nw6IfYA6/Reuqc8Z2dWqxG+lnoAqaZ24Qm+RFTz+y/RR+NnPG+W9Tp4SxTiZo7n4q\nlLUOmNCJj72YXJQSKBmpAkAyDc4PrJ3nFt45BOEnRuXE60Lv3VzLPdWggOLcKTbW\nr6NAWQS0VNdXEmJVmdoKFhJAeUvLrXPtBGqPS7HO6A8A\n-----END RSA PRIVATE KEY-----\n')
+    PUBLIC_KEY = publicKey(PRIVATE_KEY)
+
+    module.exports = config.defaults({
+      PORT,
+      HOST,
+      PGHOST,
+      PGPORT,
+      PGUSER,
+      PGPASSWORD,
+      PGDATABASE,
+      PRIVATE_KEY,
+      PUBLIC_KEY,
+      APM_SERVER: `http://${ip}:8200`,
+      APM_TOKEN: 'abc'
+    })
+    break
+  case 'test':
+    PORT = '3000'
+    HOST = `http://localhost:${PORT}`
+    PGHOST = 'localhost'
+    PGPORT = '5432'
+    PGUSER = 'postgresuser'
+    PGPASSWORD = 'postgrespassword'
+    PGDATABASE = 'mydata'
+    PRIVATE_KEY = privateKey(HOST, '-----BEGIN RSA PRIVATE KEY-----\nMIICXQIBAAKBgQDgR0M9RBFT26AseaNYy9g5YX+YQhbskFTAp0aCO6h/8xPAOzkd\n7sO+Lqrx4Ljs8iuUTv7JUbx8+ml+7IMKYjpFx4eFQ1kcEW9IL5xSKKcHt2O359cy\n5KbBExqj4Fr2JkmCE+XcMJa5GGOnpNKB92pmJtfOkjVBVQ280n/j8suyCwIDAQAB\nAoGBAMwGqBl86ZJy0nSDN2EZF5ujoXJ+dOJBrogP5CmnYfL7y3Ttq1kakwFY7PPb\nLf+HkrN5ZXj5HVJIb14ihFcW4tBR2EtABhuv2J6ZNx0KnDxUj+mJlb7GNgr5eayI\nUibIu8/eQh2+CGMilI/KR8zlRiHpD8BgttfBaRktGIrzklQJAkEA9C8JgnAGUbPp\n3rc3dEZR6pEcOGI5Fjo3uvhbOYO5oa4tJszNF1Fh1oUmn17J6yoMnh0qPG4snL2B\nOgSB8OCOnwJBAOshovf7obbVZFzQ7ikYImT/pqz7f7eV1+Uv1MRfGsXAc0EAXDrh\nAPiJ5icWkeRDCFxaTAy/8lrDGgDcL2CSoRUCQQCem4L4x91C6rMJaEbL7vU8gL8s\n3JgqGOykNLfElwxXubQ4VKUO9Vywo9JfiIlth+WkOlt53zJ5KRqsXcstdB8PAkAo\nw6IfYA6/Reuqc8Z2dWqxG+lnoAqaZ24Qm+RFTz+y/RR+NnPG+W9Tp4SxTiZo7n4q\nlLUOmNCJj72YXJQSKBmpAkAyDc4PrJ3nFt45BOEnRuXE60Lv3VzLPdWggOLcKTbW\nr6NAWQS0VNdXEmJVmdoKFhJAeUvLrXPtBGqPS7HO6A8A\n-----END RSA PRIVATE KEY-----\n')
+    PUBLIC_KEY = publicKey(PRIVATE_KEY)
+
+    module.exports = config.defaults({
+      PORT,
+      HOST,
+      PGHOST,
+      PGPORT,
+      PGUSER,
+      PGPASSWORD,
+      PGDATABASE,
+      PRIVATE_KEY,
+      PUBLIC_KEY,
+      APM_SERVER: '',
+      APM_TOKEN: ''
+    })
+    break
+  default:
+    checkMissingKeys(config, ['PORT', 'HOST', 'DATABASE_URL', 'PRIVATE_KEY'])
+
+    config.set('PRIVATE_KEY', privateKey(config.get('HOST'), '-----BEGIN RSA PRIVATE KEY-----\nMIICXQIBAAKBgQDgR0M9RBFT26AseaNYy9g5YX+YQhbskFTAp0aCO6h/8xPAOzkd\n7sO+Lqrx4Ljs8iuUTv7JUbx8+ml+7IMKYjpFx4eFQ1kcEW9IL5xSKKcHt2O359cy\n5KbBExqj4Fr2JkmCE+XcMJa5GGOnpNKB92pmJtfOkjVBVQ280n/j8suyCwIDAQAB\nAoGBAMwGqBl86ZJy0nSDN2EZF5ujoXJ+dOJBrogP5CmnYfL7y3Ttq1kakwFY7PPb\nLf+HkrN5ZXj5HVJIb14ihFcW4tBR2EtABhuv2J6ZNx0KnDxUj+mJlb7GNgr5eayI\nUibIu8/eQh2+CGMilI/KR8zlRiHpD8BgttfBaRktGIrzklQJAkEA9C8JgnAGUbPp\n3rc3dEZR6pEcOGI5Fjo3uvhbOYO5oa4tJszNF1Fh1oUmn17J6yoMnh0qPG4snL2B\nOgSB8OCOnwJBAOshovf7obbVZFzQ7ikYImT/pqz7f7eV1+Uv1MRfGsXAc0EAXDrh\nAPiJ5icWkeRDCFxaTAy/8lrDGgDcL2CSoRUCQQCem4L4x91C6rMJaEbL7vU8gL8s\n3JgqGOykNLfElwxXubQ4VKUO9Vywo9JfiIlth+WkOlt53zJ5KRqsXcstdB8PAkAo\nw6IfYA6/Reuqc8Z2dWqxG+lnoAqaZ24Qm+RFTz+y/RR+NnPG+W9Tp4SxTiZo7n4q\nlLUOmNCJj72YXJQSKBmpAkAyDc4PrJ3nFt45BOEnRuXE60Lv3VzLPdWggOLcKTbW\nr6NAWQS0VNdXEmJVmdoKFhJAeUvLrXPtBGqPS7HO6A8A\n-----END RSA PRIVATE KEY-----\n'))
+    config.set('PUBLIC_KEY', publicKey(config.get('PRIVATE_KEY')))
+
+    module.exports = config
+    break
+}

lib/config.js

@@ -14,8 +14,8 @@ async function read ({ payload }, res, next) {
   }))
   const result = await multiple(statements)
   const reads = []
-  for (let { rows } of result) {
-    for (let row of rows) {
+  for (const { rows } of result) {
+    for (const row of rows) {
       const { pdsProvider, pdsCredentials, domain, area } = camelCase(row)
       const { readFile } = get({ pdsProvider, pdsCredentials })
 
@@ -53,7 +53,7 @@ async function read ({ payload }, res, next) {
 
 function toDataMap (paths) {
   const map = {}
-  for (let { domain, area, data } of paths) {
+  for (const { domain, area, data } of paths) {
     if (!map[domain]) {
       map[domain] = {}
     }
@@ -73,8 +73,8 @@ async function write ({ payload }, res, next) {
   const dataMap = toDataMap(payload.paths)
   const results = await multiple(statements)
   const writes = []
-  for (let { rows } of results) {
-    for (let row of rows) {
+  for (const { rows } of results) {
+    for (const row of rows) {
       const { pdsProvider, pdsCredentials, domain, area } = camelCase(row)
       const { outputFile } = get({ pdsProvider, pdsCredentials })