From daeda5976c389594eb4a9f72ca0d967b61ad3185 Mon Sep 17 00:00:00 2001 From: Dmitry Yakovlev Date: Wed, 29 Jun 2022 15:23:04 +1200 Subject: [PATCH 01/10] Remove body-parser as deprecated --- package.json | 1 - src/server-app.js | 5 ++--- yarn.lock | 57 ----------------------------------------------- 3 files changed, 2 insertions(+), 61 deletions(-) diff --git a/package.json b/package.json index b0be04c2..edfb2ba7 100644 --- a/package.json +++ b/package.json @@ -76,7 +76,6 @@ "babel-polyfill": "^6.26.0", "babel-preset-es2015": "^6.24.1", "babel-preset-stage-0": "^6.24.1", - "body-parser": "^1.19.2", "claudia": "^5.14.1", "mocha": "^7.0.0", "nodemon": "^2.0.14", diff --git a/src/server-app.js b/src/server-app.js index dec5276c..52405525 100644 --- a/src/server-app.js +++ b/src/server-app.js @@ -1,6 +1,5 @@ import express from 'express'; import cors from 'cors'; -import bodyParser from 'body-parser'; import router from './routes/index.js'; import requestValidator from './middleware/requestValidator.js'; import userManager from './components/userManager'; @@ -18,8 +17,8 @@ let corsConfig = { // Parse incoming requests data app.use(cors(corsConfig)); -app.use(bodyParser.json()); -app.use(bodyParser.urlencoded({ extended: false })); +app.use(express.urlencoded({ extended: false })); +app.use(express.json()); app.get('/user/public', UserController.getPublic); app.use( basicAuth({ diff --git a/yarn.lock b/yarn.lock index c9e14fdb..cde5b207 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2588,22 +2588,6 @@ body-parser@1.19.0: raw-body "2.4.0" type-is "~1.6.17" -body-parser@^1.19.2: - version "1.19.2" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.19.2.tgz#4714ccd9c157d44797b8b5607d72c0b89952f26e" - integrity sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw== - dependencies: - bytes "3.1.2" - content-type "~1.0.4" - debug "2.6.9" - depd "~1.1.2" - http-errors "1.8.1" - iconv-lite "0.4.24" - on-finished "~2.3.0" - qs "6.9.7" - raw-body "2.4.3" - type-is "~1.6.18" - boxen@^5.0.0: version "5.1.2" resolved "https://registry.yarnpkg.com/boxen/-/boxen-5.1.2.tgz#788cb686fc83c1f486dfa8a40c68fc2b831d2b50" @@ -2714,11 +2698,6 @@ bytes@3.1.0: resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6" integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg== -bytes@3.1.2: - version "3.1.2" - resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" - integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg== - cacheable-request@^6.0.0: version "6.1.0" resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-6.1.0.tgz#20ffb8bd162ba4be11e9567d823db651052ca912" @@ -3885,17 +3864,6 @@ http-errors@1.7.2: statuses ">= 1.5.0 < 2" toidentifier "1.0.0" -http-errors@1.8.1: - version "1.8.1" - resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.8.1.tgz#7c3f28577cbc8a207388455dbd62295ed07bd68c" - integrity sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g== - dependencies: - depd "~1.1.2" - inherits "2.0.4" - setprototypeof "1.2.0" - statuses ">= 1.5.0 < 2" - toidentifier "1.0.1" - http-errors@~1.7.2: version "1.7.3" resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.3.tgz#6c619e4f9c60308c38519498c14fbb10aacebb06" @@ -5081,11 +5049,6 @@ qs@6.7.0: resolved "https://registry.yarnpkg.com/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc" integrity sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ== -qs@6.9.7: - version "6.9.7" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.9.7.tgz#4610846871485e1e048f44ae3b94033f0e675afe" - integrity sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw== - qs@~6.5.2: version "6.5.2" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" @@ -5111,16 +5074,6 @@ raw-body@2.4.0: iconv-lite "0.4.24" unpipe "1.0.0" -raw-body@2.4.3: - version "2.4.3" - resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.4.3.tgz#8f80305d11c2a0a545c2d9d89d7a0286fcead43c" - integrity sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g== - dependencies: - bytes "3.1.2" - http-errors "1.8.1" - iconv-lite "0.4.24" - unpipe "1.0.0" - rc@^1.2.8: version "1.2.8" resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" @@ -5464,11 +5417,6 @@ setprototypeof@1.1.1: resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683" integrity sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw== -setprototypeof@1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" - integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw== - shallow-clone@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-3.0.1.tgz#8f2981ad92531f55035b01fb230769a40e02efa3" @@ -5744,11 +5692,6 @@ toidentifier@1.0.0: resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553" integrity sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw== -toidentifier@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" - integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== - touch@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/touch/-/touch-3.1.0.tgz#fe365f5f75ec9ed4e56825e0bb76d24ab74af83b" From 63924a31b32b73c3d8a4cb83f3de3b8f239608fd Mon Sep 17 00:00:00 2001 From: Dmitry Yakovlev Date: Tue, 26 Jul 2022 15:16:10 +1200 Subject: [PATCH 02/10] Add Dockerfile --- .dockerignore | 10 ++++++++++ Dockerfile | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..8556bad4 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +node_modules + +.git +.github + +dist + +docker-compose.yml +Dockerfile +README.md \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..faa82520 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM node:14.17.1-slim as node +FROM ubuntu:focal-20220531 as base +COPY --from=node /usr/local/include/ /usr/local/include/ +COPY --from=node /usr/local/lib/ /usr/local/lib/ +COPY --from=node /usr/local/bin/ /usr/local/bin/ +COPY --from=node /opt/ /opt + +RUN apt-get update \ + && apt-get -qq install -y --no-install-recommends \ + git ca-certificates tini \ + && rm -rf /var/lib/apt/lists/* + +ENV NODE_ENV=production +EXPOSE 5000 + +RUN groupadd --gid 1000 node \ + && useradd --uid 1000 --gid node --shell /bin/bash --create-home node +RUN mkdir /storage-node && chown -R node:node /storage-node +WORKDIR /storage-node +USER node +COPY --chown=node:node package.json yarn.lock ./ +RUN echo $PATH +RUN yarn --version +RUN yarn install --prod --frozen-lockfile + +FROM base as source +COPY --chown=node:node . . + +### prod stage +FROM source as prod +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["node", "./src/server.js"] \ No newline at end of file From c08b0da47678159010072332ecd86b9ade4e34d5 Mon Sep 17 00:00:00 2001 From: Dmitry Yakovlev Date: Tue, 26 Jul 2022 17:07:33 +1200 Subject: [PATCH 03/10] Replace require with import for the js work without babel --- package.json | 1 + src/components/dbManager.js | 27 ++++++++++++--------------- src/components/userManager.js | 16 ++++++++-------- src/components/utils.js | 2 +- src/controllers/user.js | 6 +++--- src/middleware/requestValidator.js | 14 +++++++------- src/routes/index.js | 2 +- src/server-app.js | 14 ++++++++------ src/server.js | 2 +- test/permissions.js | 14 ++++++++------ 10 files changed, 50 insertions(+), 48 deletions(-) diff --git a/package.json b/package.json index edfb2ba7..c325a6df 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "version": "1.2.0", "description": "Verida Storage Node middleware that bridges decentralised identities so they can control access to databases within a CouchDB storage engine", "main": "dist/server.js", + "type": "module", "scripts": { "clean": "rm -rf dist", "test": "mocha -mocha --require @babel/polyfill --require @babel/register './test/**/*.js' --timeout 30000", diff --git a/src/components/dbManager.js b/src/components/dbManager.js index 821d378e..6d8e8e54 100644 --- a/src/components/dbManager.js +++ b/src/components/dbManager.js @@ -1,6 +1,6 @@ -const CouchDb = require('nano'); -import Utils from './utils'; +import CouchDb from 'nano'; import _ from 'lodash'; +import Utils from './utils.js'; class DbManager { constructor() { @@ -10,15 +10,14 @@ class DbManager { async createDatabase(username, databaseName, applicationName, options) { let couch = this._getCouch(); - let response; // Create database try { - response = await couch.db.create(databaseName); + await couch.db.create(databaseName); } catch (err) { // The database may already exist, or may have been deleted so a file // already exists. // In that case, ignore the error and continue - if (err.error != 'file_exists') { + if (err.error !== 'file_exists') { throw err; } } @@ -52,7 +51,6 @@ class DbManager { async deleteDatabase(databaseName) { let couch = this._getCouch(); - let response; // Create database try { return await couch.db.destroy(databaseName); @@ -145,12 +143,12 @@ class DbManager { await this._insertOrUpdate(db, writeDoc, '_design/only_permit_write_users'); } catch (err) { // CouchDB throws a document update conflict without any obvious reason - if (err.reason != 'Document update conflict.') { + if (err.reason !== 'Document update conflict.') { throw err; } } - if (permissions.write == 'public') { + if (permissions.write === 'public') { // If the public has write permissions, disable public from deleting records try { const deleteFunction = @@ -164,7 +162,7 @@ class DbManager { await this._insertOrUpdate(db, deleteDoc, '_design/disable_public_delete'); } catch (err) { // CouchDB throws a document update conflict without any obvious reason - if (err.reason != 'Document update conflict.') { + if (err.reason !== 'Document update conflict.') { throw err; } } @@ -185,7 +183,7 @@ class DbManager { try { doc = await db.get(id); } catch (err) { - if (err.reason != 'missing') { + if (err.reason !== 'missing') { throw err; } } @@ -209,7 +207,7 @@ class DbManager { this._couch = new CouchDb({ url: dsn, requestDefaults: { - rejectUnauthorized: process.env.DB_REJECT_UNAUTHORIZED_SSL.toLowerCase() != 'false', + rejectUnauthorized: process.env.DB_REJECT_UNAUTHORIZED_SSL.toLowerCase() !== 'false', }, }); } @@ -223,10 +221,9 @@ class DbManager { * @param {*} password */ buildDsn(username, password) { - let env = process.env; - return ( - env.DB_PROTOCOL + '://' + username + ':' + password + '@' + env.DB_HOST + ':' + env.DB_PORT - ); + const { DB_PROTOCOL, DB_HOST, DB_PORT } = process.env; + + return `${DB_PROTOCOL}://${username}:${password}@${DB_HOST}:${DB_PORT}`; } } diff --git a/src/components/userManager.js b/src/components/userManager.js index dab23c9a..b703aa2b 100644 --- a/src/components/userManager.js +++ b/src/components/userManager.js @@ -1,4 +1,4 @@ -const CouchDb = require('nano'); +import CouchDb from 'nano'; import crypto from 'crypto'; class UserManager { @@ -9,7 +9,8 @@ class UserManager { /** * Get a user by DID * - * @param {} did + * @param {string} username - did + * @param {string} signature */ async getByUsername(username, signature) { let couch = this._getCouch(); @@ -74,7 +75,7 @@ class UserManager { try { await usersDb.insert(userData); } catch (err) { - if (err.error == 'conflict') { + if (err.error === 'conflict') { // this is ok - we can continue after this. console.info('Public user not created -- already existed. This is ok and can continue'); } else { @@ -92,7 +93,7 @@ class UserManager { url: dsn, //log: console.log, requestDefaults: { - rejectUnauthorized: process.env.DB_REJECT_UNAUTHORIZED_SSL.toLowerCase() != 'false', + rejectUnauthorized: process.env.DB_REJECT_UNAUTHORIZED_SSL.toLowerCase() !== 'false', }, }); } @@ -101,10 +102,9 @@ class UserManager { } buildDsn(username, password) { - let env = process.env; - return ( - env.DB_PROTOCOL + '://' + username + ':' + password + '@' + env.DB_HOST + ':' + env.DB_PORT - ); + const { DB_PROTOCOL, DB_HOST, DB_PORT } = process.env; + + return `${DB_PROTOCOL}://${username}:${password}@${DB_HOST}:${DB_PORT}`; } } diff --git a/src/components/utils.js b/src/components/utils.js index be6974b5..71e20460 100644 --- a/src/components/utils.js +++ b/src/components/utils.js @@ -1,4 +1,4 @@ -const crypto = require('crypto'); +import crypto from 'crypto'; class Utils { generateUsernameFromRequest(req) { diff --git a/src/controllers/user.js b/src/controllers/user.js index f5da8250..a485096c 100644 --- a/src/controllers/user.js +++ b/src/controllers/user.js @@ -1,6 +1,6 @@ -import UserManager from '../components/userManager'; -import DbManager from '../components/dbManager'; -import Utils from '../components/utils'; +import UserManager from '../components/userManager.js'; +import DbManager from '../components/dbManager.js'; +import Utils from '../components/utils.js'; class UserController { // @todo: Enforce HTTPS diff --git a/src/middleware/requestValidator.js b/src/middleware/requestValidator.js index bba30a18..e80df522 100644 --- a/src/middleware/requestValidator.js +++ b/src/middleware/requestValidator.js @@ -1,6 +1,5 @@ -const basicAuth = require('express-basic-auth'); import { DIDClient } from '@verida/did-client'; -const mcache = require('memory-cache'); +import mcache from 'memory-cache'; let didClient; @@ -10,9 +9,10 @@ class RequestValidator { * * @todo: cache the signature verifications * - * @param {*} did - * @param {*} password - * @param {*} req + * @param {string} did - username + * @param {string} signature - password + * @param {Request} req + * @param {function} cb */ authorize(did, signature, req, cb) { did = did.replace(/_/g, ':').toLowerCase(); @@ -58,13 +58,13 @@ class RequestValidator { } }; - const promise = new Promise((resolve, rejects) => { + const promise = new Promise((resolve, _rejects) => { authCheck(); resolve(); }); } - getUnauthorizedResponse(req) { + getUnauthorizedResponse(_req) { return { status: 'fail', code: 90, diff --git a/src/routes/index.js b/src/routes/index.js index 887a92e5..6d694f01 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -1,5 +1,5 @@ import express from 'express'; -import UserController from '../controllers/user'; +import UserController from '../controllers/user.js'; const router = express.Router(); diff --git a/src/server-app.js b/src/server-app.js index 52405525..23dd1dae 100644 --- a/src/server-app.js +++ b/src/server-app.js @@ -1,12 +1,14 @@ import express from 'express'; import cors from 'cors'; +import dotenv from 'dotenv'; +import basicAuth from 'express-basic-auth'; + import router from './routes/index.js'; import requestValidator from './middleware/requestValidator.js'; -import userManager from './components/userManager'; -import UserController from './controllers/user'; -require('dotenv').config(); +import userManager from './components/userManager.js'; +import UserController from './controllers/user.js'; -const basicAuth = require('express-basic-auth'); +dotenv.config(); // Set up the express app const app = express(); @@ -35,6 +37,6 @@ app.use(function (req, res, next) { }); app.use(router); -userManager.ensurePublicUser() +userManager.ensurePublicUser(); -module.exports=app \ No newline at end of file +export default app; \ No newline at end of file diff --git a/src/server.js b/src/server.js index 1eaf3091..17e645fe 100644 --- a/src/server.js +++ b/src/server.js @@ -1,4 +1,4 @@ -const app = require('./server-app'); +import app from './server-app.js'; const PORT = process.env.PORT ? process.env.PORT : 5000; app.listen(PORT, () => { diff --git a/test/permissions.js b/test/permissions.js index beca43b9..4ea537cb 100644 --- a/test/permissions.js +++ b/test/permissions.js @@ -1,14 +1,16 @@ -var assert = require('assert'); -require('dotenv').config(); +import assert from 'assert'; +import dotenv from 'dotenv'; -import DbManager from '../src/components/dbManager'; -import UserManager from '../src/components/userManager'; -import Utils from '../src/components/utils'; +import DbManager from '../src/components/dbManager.js'; +import UserManager from '../src/components/userManager.js'; +import Utils from '../src/components/utils.js'; -const CouchDb = require('nano'); +import CouchDb from 'nano'; import PouchDb from 'pouchdb'; import { resolve } from 'path'; +dotenv.config(); + describe('Permissions', function () { var ownerUser, userUser, user2User, user3User, user4User; var ownerDb, userDb, user2Db, user3Db, user4Db, publicDb; From 6e3f54c6ee463916b8f7ff6c8eb971075b535225 Mon Sep 17 00:00:00 2001 From: Dmitry Yakovlev Date: Tue, 26 Jul 2022 17:08:11 +1200 Subject: [PATCH 04/10] Add docker-compose --- .dockerignore | 2 +- .gitignore | 3 ++- Dockerfile | 2 +- docker-compose.yml | 21 +++++++++++++++++++++ sample.env | 14 +++++++------- src/server-app.js | 2 +- 6 files changed, 33 insertions(+), 11 deletions(-) create mode 100644 docker-compose.yml diff --git a/.dockerignore b/.dockerignore index 8556bad4..ae7451b3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,4 +7,4 @@ dist docker-compose.yml Dockerfile -README.md \ No newline at end of file +README.md diff --git a/.gitignore b/.gitignore index cb1ff751..4cdd09e8 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ dist .DS_Store .vscode curl_tests +dbdata .env -.env* \ No newline at end of file +.env* diff --git a/Dockerfile b/Dockerfile index faa82520..4e74af9d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,4 +29,4 @@ COPY --chown=node:node . . ### prod stage FROM source as prod ENTRYPOINT ["/usr/bin/tini", "--"] -CMD ["node", "./src/server.js"] \ No newline at end of file +CMD ["node", "./src/server.js"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..b9ac8fd6 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,21 @@ +version: '3' +services: + couchdb: + image: couchdb + restart: always + ports: + - "5984:5984" + environment: + - COUCHDB_USER=${DB_USER} + - COUCHDB_PASSWORD=${DB_PASS} + volumes: + - ./dbdata:/opt/couchdb/data + + storage-node: + build: . + ports: + - "5000:5000" + environment: + PORT: 5000 + DB_HOST: couchdb + restart: on-failure diff --git a/sample.env b/sample.env index a5712138..fd3696be 100644 --- a/sample.env +++ b/sample.env @@ -1,13 +1,13 @@ HASH_KEY=this_is_not_the_prod_hash_key -DID_SERVER_URL="https://dids.testnet.verida.io:5001" +DID_SERVER_URL=https://dids.testnet.verida.io:5001 DID_CACHE_DURATION=3600 -DB_PROTOCOL="http" -DB_USER="admin" -DB_PASS="admin" -DB_HOST="localhost" +DB_PROTOCOL=http +DB_USER=admin +DB_PASS=admin +DB_HOST=localhost DB_PORT=5984 DB_REJECT_UNAUTHORIZED_SSL=false // alpha numeric only -DB_PUBLIC_USER="784c2n780c9cn0789" -DB_PUBLIC_PASS="784c2n780c9cn0789" +DB_PUBLIC_USER=784c2n780c9cn0789 +DB_PUBLIC_PASS=784c2n780c9cn0789 diff --git a/src/server-app.js b/src/server-app.js index 23dd1dae..bbd10947 100644 --- a/src/server-app.js +++ b/src/server-app.js @@ -39,4 +39,4 @@ app.use(router); userManager.ensurePublicUser(); -export default app; \ No newline at end of file +export default app; From 8fa65a9cba1c5d1653f8831852cde98ea40038f3 Mon Sep 17 00:00:00 2001 From: Dmitry Yakovlev Date: Tue, 26 Jul 2022 17:26:31 +1200 Subject: [PATCH 05/10] Update node to latest 14 --- .nvmrc | 2 +- Dockerfile | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.nvmrc b/.nvmrc index 2831228e..a3eb5a03 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -14.17.1 +14.20.0 diff --git a/Dockerfile b/Dockerfile index 4e74af9d..b94f7211 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,10 @@ -FROM node:14.17.1-slim as node +FROM node:14.20.0-slim as node FROM ubuntu:focal-20220531 as base COPY --from=node /usr/local/include/ /usr/local/include/ COPY --from=node /usr/local/lib/ /usr/local/lib/ COPY --from=node /usr/local/bin/ /usr/local/bin/ -COPY --from=node /opt/ /opt +# this ensures we fix simlinks for npx and yarn +RUN corepack disable && corepack enable RUN apt-get update \ && apt-get -qq install -y --no-install-recommends \ @@ -19,8 +20,6 @@ RUN mkdir /storage-node && chown -R node:node /storage-node WORKDIR /storage-node USER node COPY --chown=node:node package.json yarn.lock ./ -RUN echo $PATH -RUN yarn --version RUN yarn install --prod --frozen-lockfile FROM base as source From 560bad1f58998a06929d0cde38bce4711c433d1e Mon Sep 17 00:00:00 2001 From: Dmitry Yakovlev Date: Tue, 2 Aug 2022 14:00:06 +1200 Subject: [PATCH 06/10] Add gh workflow to build and publish docker image --- .dockerignore | 2 +- .github/workflows/build-docker.yml | 93 ++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build-docker.yml diff --git a/.dockerignore b/.dockerignore index ae7451b3..bf34f419 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,7 +4,7 @@ node_modules .github dist - +.env* docker-compose.yml Dockerfile README.md diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml new file mode 100644 index 00000000..b38ee6a5 --- /dev/null +++ b/.github/workflows/build-docker.yml @@ -0,0 +1,93 @@ +name: build-docker-image + +on: + push: + branches: + - 'main' + tags: + - 'v*' + pull_request: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + # qemu and buildx are for multi-platform images and caching + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + # set up local caching + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: | + verida/storage-node + ghcr.io/verida/storage-node + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + + # get the correct AW role to push image to ECR + - name: Configure AWS credentials from Testnet account + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-duration-seconds: 1500 + role-to-assume: arn:aws:iam::737954963756:role/github-testnet-deploy-role + aws-region: us-east-1 + + # login to push to DockerHub +# - name: Login to DockerHub +# uses: docker/login-action@v2 +# with: +# username: ${{ secrets.DOCKERHUB_USERNAME }} +# password: ${{ secrets.DOCKERHUB_TOKEN }} + + # login to push to GHCR + - name: Login to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v3 + with: + platforms: linux/amd64 # ,linux/arm64,linux/arm/v7 + context: . + push: ${{ github.event_name != 'pull_request' }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + tags: ${{ steps.meta.output.tags }} + labels: ${{ steps.meta.output.labels }} + + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache From 2da54c93af0f3d1700179f8df539b1f6e2026249 Mon Sep 17 00:00:00 2001 From: Dmitry Yakovlev Date: Thu, 4 Aug 2022 19:13:33 +1200 Subject: [PATCH 07/10] Rearrange dockerfile to improve image size --- .dockerignore | 3 ++- Dockerfile | 22 +++++++++++++--------- docker-compose.yml | 4 +++- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.dockerignore b/.dockerignore index bf34f419..7ed7b71f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,4 +7,5 @@ dist .env* docker-compose.yml Dockerfile -README.md +*.md +claudia.* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index b94f7211..8cfaab1e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,13 +3,6 @@ FROM ubuntu:focal-20220531 as base COPY --from=node /usr/local/include/ /usr/local/include/ COPY --from=node /usr/local/lib/ /usr/local/lib/ COPY --from=node /usr/local/bin/ /usr/local/bin/ -# this ensures we fix simlinks for npx and yarn -RUN corepack disable && corepack enable - -RUN apt-get update \ - && apt-get -qq install -y --no-install-recommends \ - git ca-certificates tini \ - && rm -rf /var/lib/apt/lists/* ENV NODE_ENV=production EXPOSE 5000 @@ -18,14 +11,25 @@ RUN groupadd --gid 1000 node \ && useradd --uid 1000 --gid node --shell /bin/bash --create-home node RUN mkdir /storage-node && chown -R node:node /storage-node WORKDIR /storage-node + +FROM base as build +# this ensures we fix simlinks for npx and yarn +RUN corepack disable && corepack enable + +RUN apt-get update \ + && apt-get -qq install -y --no-install-recommends \ + git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + USER node COPY --chown=node:node package.json yarn.lock ./ RUN yarn install --prod --frozen-lockfile FROM base as source -COPY --chown=node:node . . +COPY --from=build --chown=node:node /storage-node/node_modules ./node_modules +COPY . . ### prod stage +# Note: use --init option when running the container to have better signal forwarding FROM source as prod -ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["node", "./src/server.js"] diff --git a/docker-compose.yml b/docker-compose.yml index b9ac8fd6..80aa9863 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3' services: couchdb: image: couchdb @@ -19,3 +18,6 @@ services: PORT: 5000 DB_HOST: couchdb restart: on-failure + init: true + env_file: + - .env From cd703ea8765a7eae514a966c42b006915ead583b Mon Sep 17 00:00:00 2001 From: Dmitry Yakovlev Date: Fri, 26 Aug 2022 10:13:11 +1200 Subject: [PATCH 08/10] Uncomment dockerhub --- .dockerignore | 2 +- .github/workflows/build-docker.yml | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.dockerignore b/.dockerignore index 7ed7b71f..57e4035f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,4 +8,4 @@ dist docker-compose.yml Dockerfile *.md -claudia.* \ No newline at end of file +claudia.* diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index b38ee6a5..b4c8b27c 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -58,11 +58,12 @@ jobs: aws-region: us-east-1 # login to push to DockerHub -# - name: Login to DockerHub -# uses: docker/login-action@v2 -# with: -# username: ${{ secrets.DOCKERHUB_USERNAME }} -# password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to DockerHub + if: github.event_name != 'pull_request' + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} # login to push to GHCR - name: Login to GHCR From b6289e122e6f6f701e77ced3b266ea9792e2a126 Mon Sep 17 00:00:00 2001 From: Dmitry Yakovlev Date: Fri, 2 Sep 2022 14:39:27 +1200 Subject: [PATCH 09/10] Add docker info and single node config to readme --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index c7a5c833..92e8740c 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,9 @@ $env:DB_PUBLIC_PASS="784c2n780c9cn0789" - A valid user must be enforced for security reasons ``` +[couchdb] +single_node=true + [httpd] WWW-Authenticate = Basic realm="administrator" enable_cors = true @@ -87,3 +90,14 @@ See the [Claudia Docs for information](https://claudiajs.com/news/2016/11/24/cla Verida staff can see the [internal Verida repo]( https://github.com/verida/infrastructure/blob/develop/storage_node.md) for docs on this. +## Docker + +You can spin up storage node API on your machine with Docker: +```shell +docker run --init --env-file=.env verida/storage-node:latest +``` + +Using the example [docker-compose.yml](./docker-compose.yml) you can run storage node together with CouchDB. +```shell +docker compose up +``` \ No newline at end of file From 0992ea6791d32ce67a9f89c6c9d9471a1cb48c65 Mon Sep 17 00:00:00 2001 From: Chris Date: Fri, 4 Nov 2022 11:58:23 +1030 Subject: [PATCH 10/10] Update to latest verida dependencies. Remove module so tests run. --- package.json | 3 +-- yarn.lock | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 213f2d07..aa6111b0 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,6 @@ "version": "1.2.0", "description": "Verida Storage Node middleware that bridges decentralised identities so they can control access to databases within a CouchDB storage engine", "main": "dist/server.js", - "type": "module", "scripts": { "clean": "rm -rf dist", "tests": "mocha -mocha --require @babel/polyfill --require @babel/register './test/**/*.js' --timeout 30000", @@ -52,7 +51,7 @@ "homepage": "https://github.com/verida/storage-node/README.md", "dependencies": { "@babel/runtime": "^7.16.7", - "@verida/did-client": "^2.0.0-rc2", + "@verida/did-client": "^2.0.0-rc3", "@verida/encryption-utils": "^2.0.0-rc1", "aws-serverless-express": "^3.4.0", "cors": "^2.8.5", diff --git a/yarn.lock b/yarn.lock index 656d8930..48171b9b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1677,6 +1677,19 @@ deepcopy "^2.1.0" ethers "^5.5.1" +"@verida/did-client@^2.0.0-rc3": + version "2.0.0-rc3" + resolved "https://registry.yarnpkg.com/@verida/did-client/-/did-client-2.0.0-rc3.tgz#b3b5165fe76d5479deed9cab059ac6a28a01b08e" + integrity sha512-3IULtenrXBk8StAXSH2fN54lIKsbKwIEjocZDamSvkrd0oXtc2guwnonJIBB63tjbfd1JmteX/+ciLV5sjCrMg== + dependencies: + "@verida/did-document" "^2.0.0-rc3" + "@verida/vda-did" "^2.0.0-rc3" + "@verida/vda-did-resolver" "^2.0.0-rc3" + "@verida/web3" "^2.0.0-rc3" + axios "^0.23.0" + deepcopy "^2.1.0" + ethers "^5.5.1" + "@verida/did-document@^2.0.0-rc1": version "2.0.0-rc1" resolved "https://registry.yarnpkg.com/@verida/did-document/-/did-document-2.0.0-rc1.tgz#38ec512e4a1d27433de10fc9c8e158bdf43960e8" @@ -1702,6 +1715,19 @@ ethers "^5.7.2" lodash "^4.17.21" +"@verida/did-document@^2.0.0-rc3": + version "2.0.0-rc3" + resolved "https://registry.yarnpkg.com/@verida/did-document/-/did-document-2.0.0-rc3.tgz#897b99ecca2a074dfe09163af8007584fb6a7566" + integrity sha512-zKWewLM+q1qe00/cyEuv5hRUMRxnMSAoMYyb3Byhndf0nGtkrwRl4qXNYYfEaTDkYxTC4haGsogjfX/dYQRjYQ== + dependencies: + "@ethersproject/bignumber" "^5.7.0" + "@verida/encryption-utils" "^2.0.0-rc1" + "@verida/keyring" "^2.0.0-rc1" + "@verida/vda-did-resolver" "^2.0.0-rc3" + did-resolver "^3.1.2" + ethers "^5.7.2" + lodash "^4.17.21" + "@verida/encryption-utils@^2.0.0-rc1": version "2.0.0-rc1" resolved "https://registry.yarnpkg.com/@verida/encryption-utils/-/encryption-utils-2.0.0-rc1.tgz#ad1eb6c5001d59930f87b2d7c0272cfe1b8c3196" @@ -1781,6 +1807,24 @@ did-resolver "^3.1.5" ethers "^5.7.0" +"@verida/vda-did-resolver@^2.0.0-rc3": + version "2.0.0-rc3" + resolved "https://registry.yarnpkg.com/@verida/vda-did-resolver/-/vda-did-resolver-2.0.0-rc3.tgz#6d19a49c8e35cf0a55d97290e8438b89968cb34e" + integrity sha512-Ofh9OytgazFy3Oq3Y+wOZHY2z1h4s3i8cDpPGFhkvYs9oLZFgjF6EVfjOgU2WiAth8qEceVNg2ehbi7oP02H+Q== + dependencies: + "@ethersproject/abi" "^5.5.0" + "@ethersproject/abstract-signer" "^5.5.0" + "@ethersproject/address" "^5.5.0" + "@ethersproject/basex" "^5.5.0" + "@ethersproject/bignumber" "^5.5.0" + "@ethersproject/contracts" "^5.5.0" + "@ethersproject/providers" "^5.5.0" + "@ethersproject/transactions" "^5.5.0" + "@verida/encryption-utils" "^2.0.0-rc1" + "@verida/web3" "^2.0.0-rc3" + did-resolver "^3.1.5" + ethers "^5.7.0" + "@verida/vda-did@^2.0.0-rc2": version "2.0.0-rc2" resolved "https://registry.yarnpkg.com/@verida/vda-did/-/vda-did-2.0.0-rc2.tgz#8318fcbad16a3cf133643f600e15d86a4d68bfa7" @@ -1798,6 +1842,23 @@ "@verida/vda-did-resolver" "^2.0.0-rc2" did-jwt "^5.12.3" +"@verida/vda-did@^2.0.0-rc3": + version "2.0.0-rc3" + resolved "https://registry.yarnpkg.com/@verida/vda-did/-/vda-did-2.0.0-rc3.tgz#69bd5cf13096794c8a0ecf1b03782b58d23ae45b" + integrity sha512-28uKuVGqHWC+uEt7kprH1AmZxs8sIAKk/jcsRYBjR6fLjsEP/yL7vS6gbVyraaaYcK76ybuo3tmPhShNZr0Meg== + dependencies: + "@ethersproject/abstract-signer" "^5.5.0" + "@ethersproject/base64" "^5.5.0" + "@ethersproject/basex" "^5.5.0" + "@ethersproject/bytes" "^5.5.0" + "@ethersproject/providers" "^5.5.0" + "@ethersproject/signing-key" "^5.5.0" + "@ethersproject/strings" "^5.5.0" + "@ethersproject/transactions" "^5.5.0" + "@ethersproject/wallet" "^5.5.0" + "@verida/vda-did-resolver" "^2.0.0-rc3" + did-jwt "^5.12.3" + "@verida/web3@^2.0.0-rc1": version "2.0.0-rc1" resolved "https://registry.yarnpkg.com/@verida/web3/-/web3-2.0.0-rc1.tgz#0f63b455120306d9b920a232bc87367355af6f15" @@ -1814,6 +1875,14 @@ axios "^0.27.2" ethers "^5.7.0" +"@verida/web3@^2.0.0-rc3": + version "2.0.0-rc3" + resolved "https://registry.yarnpkg.com/@verida/web3/-/web3-2.0.0-rc3.tgz#04f2d6e2b581fd6fbf63a0582c906bc78bbfeed5" + integrity sha512-l2SRpYegEqgRMSoe0Z9F2D4+jn3kO9zC4INpWdxnoL3IV09dsK0AHOkL8L/zj9BJWczVGBpix6R7CAgVj1RKJQ== + dependencies: + axios "^0.27.2" + ethers "^5.7.0" + abbrev@1: version "1.1.1" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"