From 90a88b4593c7346e246731703b2199aed0743242 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 8 Nov 2022 08:47:59 +1030 Subject: [PATCH 01/13] Add scaffolding for new did storage support. Refactor routes. --- src/routes/{index.js => private.js} | 0 src/routes/public.js | 16 ++++++ src/server-app.js | 22 +++---- src/services/didStorage/controller.js | 63 ++++++++++++++++++++ src/services/didStorage/routes.js | 13 +++++ test/config.js | 2 +- test/did-storage.js | 82 +++++++++++++++++++++++++++ 7 files changed, 183 insertions(+), 15 deletions(-) rename src/routes/{index.js => private.js} (100%) create mode 100644 src/routes/public.js create mode 100644 src/services/didStorage/controller.js create mode 100644 src/services/didStorage/routes.js create mode 100644 test/did-storage.js diff --git a/src/routes/index.js b/src/routes/private.js similarity index 100% rename from src/routes/index.js rename to src/routes/private.js diff --git a/src/routes/public.js b/src/routes/public.js new file mode 100644 index 00000000..2095f811 --- /dev/null +++ b/src/routes/public.js @@ -0,0 +1,16 @@ +import express from 'express'; +import UserController from '../controllers/user.js'; +import AuthController from '../controllers/auth.js'; + +const router = express.Router(); + +// Specify public endpoints +router.get('/auth/public', UserController.getPublic); +router.post('/auth/generateAuthJwt', AuthController.generateAuthJwt); +router.post('/auth/authenticate', AuthController.authenticate); +router.post('/auth/connect', AuthController.connect); +router.post('/auth/regenerateRefreshToken', AuthController.regenerateRefreshToken); +router.post('/auth/invalidateDeviceId', AuthController.invalidateDeviceId); +router.post('/auth/isTokenValid', AuthController.isTokenValid); + +export default router; \ No newline at end of file diff --git a/src/server-app.js b/src/server-app.js index e53616b1..d567afce 100644 --- a/src/server-app.js +++ b/src/server-app.js @@ -3,11 +3,12 @@ import cors from 'cors'; import dotenv from 'dotenv'; import basicAuth from 'express-basic-auth'; -import router from './routes/index.js'; +import privateRoutes from './routes/private.js'; +import publicRoutes from './routes/public.js'; +import didStorageRoutes from './services/didStorage/routes' + import requestValidator from './middleware/requestValidator.js'; import userManager from './components/userManager.js'; -import UserController from './controllers/user.js'; -import AuthController from './controllers/auth.js'; import AuthManager from './components/authManager.js'; dotenv.config(); @@ -19,22 +20,15 @@ let corsConfig = { //origin: process.env.CORS_HOST }; + // Parse incoming requests data app.use(cors(corsConfig)); app.use(express.urlencoded({ extended: false })); app.use(express.json()); - -// Specify public endpoints -app.get('/auth/public', UserController.getPublic); -app.post('/auth/generateAuthJwt', AuthController.generateAuthJwt); -app.post('/auth/authenticate', AuthController.authenticate); -app.post('/auth/connect', AuthController.connect); -app.post('/auth/regenerateRefreshToken', AuthController.regenerateRefreshToken); -app.post('/auth/invalidateDeviceId', AuthController.invalidateDeviceId); -app.post('/auth/isTokenValid', AuthController.isTokenValid); - +app.use(didStorageRoutes); +app.use(publicRoutes); app.use(requestValidator); -app.use(router); +app.use(privateRoutes); AuthManager.initDb(); userManager.ensureDefaultDatabases(); diff --git a/src/services/didStorage/controller.js b/src/services/didStorage/controller.js new file mode 100644 index 00000000..66cc8e46 --- /dev/null +++ b/src/services/didStorage/controller.js @@ -0,0 +1,63 @@ +import { create } from "lodash"; + +class DidStorage { + + async create(req, res) { + const did = req.params.did + + return res.status(200).send({ + status: "success-create", + data: { + "did": did + } + }); + } + + async update(req, res) { + const did = req.params.did + + return res.status(200).send({ + status: "success-update", + data: { + "did": did + } + }); + } + + async delete(req, res) { + const did = req.params.did + + return res.status(200).send({ + status: "success-delete", + data: { + "did": did + } + }); + } + + async get(req, res) { + const did = req.params.did + + return res.status(200).send({ + status: "success-get", + data: { + "did": did + } + }); + } + + async migrate(req, res) { + const did = req.params.did + + return res.status(200).send({ + status: "success-migrate", + data: { + "did": did + } + }); + } + +} + +const didStorage = new DidStorage(); +export default didStorage; \ No newline at end of file diff --git a/src/services/didStorage/routes.js b/src/services/didStorage/routes.js new file mode 100644 index 00000000..6414da7f --- /dev/null +++ b/src/services/didStorage/routes.js @@ -0,0 +1,13 @@ +import express from 'express'; +import DidStorageController from './controller.js'; + +const router = express.Router(); + +// Specify public endpoints +router.post('/did/:did/migrate', DidStorageController.migrate); +router.post('/did/:did', DidStorageController.create); +router.put('/did/:did', DidStorageController.update); +router.delete('/did/:did', DidStorageController.delete); +router.get('/did/:did', DidStorageController.get); + +export default router; \ No newline at end of file diff --git a/test/config.js b/test/config.js index de934218..f3744a5b 100644 --- a/test/config.js +++ b/test/config.js @@ -11,7 +11,7 @@ export default { }, }, ENVIRONMENT: EnvironmentType.TESTNET, - SERVER_URL: `https://sn-acacia1.tn.verida.tech`, + SERVER_URL: `http://localhost:5000`, VDA_PRIVATE_KEY: '0x19d3b996ec98a9a536efdffbae41e5eaaf117765a587483c69195c9460165c34', CONTEXT_NAME: 'Verida Storage Node Test: Test Application 1', DATABASE_SERVER: 'https://sn-acacia1.tn.verida.tech/', // http://localhost:5000/ for local testing when running local @verida/storage-node diff --git a/test/did-storage.js b/test/did-storage.js new file mode 100644 index 00000000..a636909e --- /dev/null +++ b/test/did-storage.js @@ -0,0 +1,82 @@ +import assert from 'assert'; +import dotenv from 'dotenv'; +import Axios from 'axios' +import {ethers} from 'ethers' + +import AuthManager from "../src/components/authManager"; +import TestUtils from "./utils" + +import CONFIG from './config' +const { CONTEXT_NAME, SERVER_URL, TEST_DEVICE_ID } = CONFIG + +const DID_URL = `${SERVER_URL}/did` + + +let authJwt, accountInfo, authRequestId +let refreshToken, accessToken, newRefreshToken + +const wallet = ethers.Wallet.createRandom() +const DID = `did:vda:${wallet.address}` + +describe("DID Storage Tests", function() { + /*this.beforeAll(async () => { + //await AuthManager.initDb() -- This is required if the server is running locally and has never been run before, run just once + //await TestUtils.ensureVeridaAccount(CONFIG.VDA_PRIVATE_KEY) -- This is required if the private key has never been initilaized with an application context, run just once + accountInfo = await TestUtils.connectAccount(CONFIG.VDA_PRIVATE_KEY) + })*/ + + describe("Create", () => { + it("Success", async () => { + console.log(`${DID_URL}/${DID}`) + const createResult = await Axios.post(`${DID_URL}/${DID}`, { + hello: 'world' + }); + + console.log(createResult.data) + }) + }) + + describe("Update", () => { + it("Success", async () => { + console.log(`${DID_URL}/${DID}`) + const createResult = await Axios.put(`${DID_URL}/${DID}`, { + hello: 'world' + }); + + console.log(createResult.data) + }) + }) + + describe("Delete", () => { + it("Success", async () => { + console.log(`${DID_URL}/${DID}`) + const createResult = await Axios.delete(`${DID_URL}/${DID}`, { + hello: 'world' + }); + + console.log(createResult.data) + }) + }) + + describe("Get", () => { + it("Success", async () => { + console.log(`${DID_URL}/${DID}`) + const createResult = await Axios.get(`${DID_URL}/${DID}`, { + hello: 'world' + }); + + console.log(createResult.data) + }) + }) + + describe("Migrate", () => { + it("Success", async () => { + console.log(`${DID_URL}/${DID}/migrate`) + const createResult = await Axios.post(`${DID_URL}/${DID}/migrate`, { + hello: 'world' + }); + + console.log(createResult.data) + }) + }) +}) \ No newline at end of file From 21433cf8f2506a96b6154521b7d1258a3484be67 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 8 Nov 2022 16:50:17 +1030 Subject: [PATCH 02/13] Initial progress on create and get endpoints --- package.json | 1 + sample.env | 5 +- src/components/dbManager.js | 2 + src/components/userManager.js | 14 ++++- src/services/didStorage/controller.js | 91 +++++++++++++++++++++++---- src/services/didStorage/utils.js | 68 ++++++++++++++++++++ test/did-storage.js | 90 ++++++++++++++++++-------- 7 files changed, 229 insertions(+), 42 deletions(-) create mode 100644 src/services/didStorage/utils.js diff --git a/package.json b/package.json index aa6111b0..0b276297 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "dependencies": { "@babel/runtime": "^7.16.7", "@verida/did-client": "^2.0.0-rc3", + "@verida/did-document": "^2.0.0-rc3", "@verida/encryption-utils": "^2.0.0-rc1", "aws-serverless-express": "^3.4.0", "cors": "^2.8.5", diff --git a/sample.env b/sample.env index 1137169f..3df467a8 100644 --- a/sample.env +++ b/sample.env @@ -14,13 +14,14 @@ ACCESS_TOKEN_EXPIRY=300 REFRESH_JWT_SIGN_PK=insert-random-refresh-symmetric-key # 30 Days REFRESH_TOKEN_EXPIRY=2592000 -DB_REFRESH_TOKENS="verida_refresh_tokens" -DB_DB_INFO="verida_db_info" +DB_REFRESH_TOKENS=verida_refresh_tokens +DB_DB_INFO=verida_db_info # How often garbage collection runs (1=100%, 0.5 = 50%) GC_PERCENT=0.1 // alpha numeric only DB_PUBLIC_USER=784c2n780c9cn0789 DB_PUBLIC_PASS=784c2n780c9cn0789 +DB_DIDS=verida_dids PORT=5151 diff --git a/src/components/dbManager.js b/src/components/dbManager.js index f8003771..bf8ab705 100644 --- a/src/components/dbManager.js +++ b/src/components/dbManager.js @@ -2,6 +2,8 @@ import Utils from './utils.js'; import _ from 'lodash'; import Db from "./db.js" import EncryptionUtils from "@verida/encryption-utils" +import dotenv from 'dotenv'; +dotenv.config(); class DbManager { diff --git a/src/components/userManager.js b/src/components/userManager.js index 1b9326cb..53e16b98 100644 --- a/src/components/userManager.js +++ b/src/components/userManager.js @@ -1,7 +1,7 @@ import crypto from 'crypto'; -//import jwt from 'jsonwebtoken'; import Db from './db.js' -//import AuthManager from './authManager.js'; +import dotenv from 'dotenv'; +dotenv.config(); class UserManager { @@ -100,6 +100,16 @@ class UserManager { throw err; } } + + try { + await couch.db.create(process.env.DB_DIDS) + } catch (err) { + if (err.message == "The database could not be created, the file already exists.") { + console.log("DID database not created -- already existed"); + } else { + throw err; + } + } } } diff --git a/src/services/didStorage/controller.js b/src/services/didStorage/controller.js index 66cc8e46..00d1f95d 100644 --- a/src/services/didStorage/controller.js +++ b/src/services/didStorage/controller.js @@ -1,16 +1,54 @@ -import { create } from "lodash"; +import { DIDDocument } from "@verida/did-document" +import Utils from './utils' class DidStorage { async create(req, res) { - const did = req.params.did + // Verify request parameters + if (!req.params.did) { + return Utils.error(res, `No DID specified`) + } - return res.status(200).send({ - status: "success-create", - data: { - "did": did + if (!req.body.document) { + return Utils.error(res, `No document specified`) + } + + const did = req.params.did.toLowerCase() + const didDocument = new DIDDocument(req.body.document) + const jsonDoc = didDocument.export() + + try { + Utils.verifyDocument(did, didDocument, { + versionId: 0 + }) + } catch (err) { + console.error(err.message) + return Utils.error(res, `Invalid DID Document: ${err.message}`) + } + + // @ todo Ensure there is currently no entry for the given DID in the DID Registry + // OR + // there is currently an entry and it references this storage node endpoint + + // Save the DID document + const didDb = Utils.getDidDocumentDb() + + // Create CouchDB database user matching username and password + const documentData = { + _id: jsonDoc.id, + ...jsonDoc + }; + + try { + await didDb.insert(documentData); + return Utils.success(res, {}); + } catch (err) { + if (err.error == 'conflict') { + return Utils.error(res, `DID Document already exists. Use PUT request to update.`) } - }); + + return Utils.error(res, `Unknown error: ${err.message}`) + } } async update(req, res) { @@ -36,14 +74,41 @@ class DidStorage { } async get(req, res) { - const did = req.params.did + const did = req.params.did.toLowerCase() + const allVersions = req.query.allVersions && req.query.allVersions === 'true' - return res.status(200).send({ - status: "success-get", - data: { - "did": did + try { + const document = await Utils.getDidDocument(did, allVersions) + + let result + if (allVersions) { + // strip `_id`, `_rev` and `_revisions` from documents + console.log(document) + const versions = document.map(item => { + delete item['_id'] + delete item['_rev'] + delete item['_revisions'] + + return item + }) + + result = { + versions + } + } else { + delete document['_id'] + result = document } - }); + + return Utils.success(res, result) + } catch (err) { + if (err.reason == 'missing') { + return Utils.error(res, `DID Document not found.`, 404) + } + + console.error(err) + return Utils.error(res, `Unknown error: ${err.message}`) + } } async migrate(req, res) { diff --git a/src/services/didStorage/utils.js b/src/services/didStorage/utils.js new file mode 100644 index 00000000..ce7b182f --- /dev/null +++ b/src/services/didStorage/utils.js @@ -0,0 +1,68 @@ +import Db from "../../components/db.js" + +class Utils { + + // @todo + verifyDocument(did, document, expectedValues = {}) { + console.log('verifying doc') + + const doc = document.export() + for (let key in expectedValues) { + if (doc[key] != expectedValues[key]) { + throw new Error(`Invalid value for ${key}`) + } + } + + //throw new Error(`versionId not set`) + /* + versionId + created + updated + deactivated + proof — A string representing the full DID Document as a JSON encoded string that has been hashed using keccak256 and signed with ed25519, the default Ethereum based signature scheme. + */ + + return true + } + + async error(res, message, httpStatus=400) { + return res.status(httpStatus).send({ + status: "fail", + message + }) + } + + async success(res, data) { + return res.status(200).send({ + status: "success", + data + }) + } + + getDidDocumentDb() { + const couch = Db.getCouch() + return couch.db.use(process.env.DB_DIDS); + } + + async getDidDocument(did, all=false) { + const db = this.getDidDocumentDb() + + /*const res = await db.list({ + include_docs: true + }) + console.log(res.rows[3])*/ + + const options = {} + if (all) { + options.meta = true + } + + console.log(did, options) + + return db.get(did.toLowerCase(), options) + } + +} + +const utils = new Utils(); +export default utils; \ No newline at end of file diff --git a/test/did-storage.js b/test/did-storage.js index a636909e..64bc28aa 100644 --- a/test/did-storage.js +++ b/test/did-storage.js @@ -1,22 +1,22 @@ -import assert from 'assert'; -import dotenv from 'dotenv'; import Axios from 'axios' -import {ethers} from 'ethers' - -import AuthManager from "../src/components/authManager"; -import TestUtils from "./utils" +import assert from 'assert'; +import { ethers } from 'ethers' +import { DIDDocument } from '@verida/did-document' import CONFIG from './config' -const { CONTEXT_NAME, SERVER_URL, TEST_DEVICE_ID } = CONFIG +const { SERVER_URL } = CONFIG const DID_URL = `${SERVER_URL}/did` +const wallet = ethers.Wallet.createRandom() -let authJwt, accountInfo, authRequestId -let refreshToken, accessToken, newRefreshToken +//const DID_ADDRESS = wallet.address +//const DID = `did:vda:testnet:${DID_ADDRESS}` +//const DID_PK = wallet.signingKey.publicKey -const wallet = ethers.Wallet.createRandom() -const DID = `did:vda:${wallet.address}` +const DID_ADDRESS = '0x56f2c429fC8fdd4911F472a3c451341EAEC989a2' +const DID = 'did:vda:testnet:0x56f2c429fC8fdd4911F472a3c451341EAEC989a2' +const DID_PK = '0x04d1c85058d70c637f8ec46df26cbe855829a51f3335731352e2d1587e478b66e350e49bd4650685039c4b4e0adab5bb2a680d5a13dfb176a311544ec503999f4f' describe("DID Storage Tests", function() { /*this.beforeAll(async () => { @@ -27,30 +27,70 @@ describe("DID Storage Tests", function() { describe("Create", () => { it("Success", async () => { - console.log(`${DID_URL}/${DID}`) + const doc = new DIDDocument(DID, DID_PK) + const createResult = await Axios.post(`${DID_URL}/${DID}`, { - hello: 'world' + document: doc.export() }); - console.log(createResult.data) + assert.equal(createResult.data.status, 'success', 'Success response') + }) + + it("Fail - Duplicate DID Document", async () => { + const doc = new DIDDocument(DID, DID_PK) + + try { + await Axios.post(`${DID_URL}/${DID}`, { + document: doc.export() + }); + + assert.fail('DID Document was created a second time') + } catch (err) { + assert.equal(err.response.data.status, 'fail', 'DID Document create failed') + assert.ok(err.response.data.message.match('DID Document already exists'), 'Rejected because DID Document already exists') + } }) }) - describe("Update", () => { - it("Success", async () => { - console.log(`${DID_URL}/${DID}`) - const createResult = await Axios.put(`${DID_URL}/${DID}`, { - hello: 'world' - }); + describe.only("Get", () => { + it("Success - Latest", async () => { + const getResult = await Axios.get(`${DID_URL}/${DID}`); - console.log(createResult.data) + assert.ok(getResult.data.status, 'success', 'Success response') + + // @tgodo: re-build document and compare it matches + //console.log(getResult.data) + }) + + it("Fail - Invalid DID", async () => { + try { + const getResult = await Axios.get(`${DID_URL}/abc123`); + + assert.fail(`DID Document was found, when it shouldn't have`) + } catch (err) { + assert.equal(err.response.data.status, 'fail', 'Get DID Document failed') + assert.ok(err.response.data.message.match('DID Document not found'), `Rejected because DID Document doesn't exists`) + } + }) + + // versionId + // versionTime + // allVersions + + it("Success - All versions", async () => { + const getResult = await Axios.get(`${DID_URL}/${DID}?allVersions=true`); + + assert.ok(getResult.data.status, 'success', 'Success response') + + // @tgodo: re-build document and compare it matches + console.log(getResult.data) }) }) - describe("Delete", () => { + describe("Update", () => { it("Success", async () => { console.log(`${DID_URL}/${DID}`) - const createResult = await Axios.delete(`${DID_URL}/${DID}`, { + const createResult = await Axios.put(`${DID_URL}/${DID}`, { hello: 'world' }); @@ -58,10 +98,10 @@ describe("DID Storage Tests", function() { }) }) - describe("Get", () => { + describe("Delete", () => { it("Success", async () => { console.log(`${DID_URL}/${DID}`) - const createResult = await Axios.get(`${DID_URL}/${DID}`, { + const createResult = await Axios.delete(`${DID_URL}/${DID}`, { hello: 'world' }); From e7ecd6180b09857f0c607fbf87b8e25a3ef4f49a Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 9 Nov 2022 12:19:25 +1030 Subject: [PATCH 03/13] Updates working. Get all versions working. --- src/components/userManager.js | 7 ++ src/services/didStorage/controller.js | 93 ++++++++++++++++----------- src/services/didStorage/utils.js | 54 +++++++++++++--- test/did-storage.js | 87 +++++++++++++++++++------ 4 files changed, 175 insertions(+), 66 deletions(-) diff --git a/src/components/userManager.js b/src/components/userManager.js index 53e16b98..7404b082 100644 --- a/src/components/userManager.js +++ b/src/components/userManager.js @@ -103,6 +103,13 @@ class UserManager { try { await couch.db.create(process.env.DB_DIDS) + const dbDids = couch.db.use(process.env.DB_DIDS) + await dbDids.createIndex({ + index: { + fields: ['id'] + }, + name: 'did' + }) } catch (err) { if (err.message == "The database could not be created, the file already exists.") { console.log("DID database not created -- already existed"); diff --git a/src/services/didStorage/controller.js b/src/services/didStorage/controller.js index 00d1f95d..c9acb38a 100644 --- a/src/services/didStorage/controller.js +++ b/src/services/didStorage/controller.js @@ -35,7 +35,7 @@ class DidStorage { // Create CouchDB database user matching username and password const documentData = { - _id: jsonDoc.id, + _id: `${jsonDoc.id}-0`, ...jsonDoc }; @@ -52,14 +52,57 @@ class DidStorage { } async update(req, res) { - const did = req.params.did + // Verify request parameters + if (!req.params.did) { + return Utils.error(res, `No DID specified`) + } - return res.status(200).send({ - status: "success-update", - data: { - "did": did + if (!req.body.document) { + return Utils.error(res, `No document specified`) + } + + const did = req.params.did.toLowerCase() + const didDocument = new DIDDocument(req.body.document) + const jsonDoc = didDocument.export() + + let existingDoc + try { + existingDoc = await Utils.getDidDocument(did) + if (!existingDoc) { + return Utils.error(res, `DID Document not found`) } - }); + + const nextVersionId = existingDoc.versionId + 1 + Utils.verifyDocument(did, didDocument, { + versionId: nextVersionId + }) + } catch (err) { + return Utils.error(res, `Invalid DID Document: ${err.message}`) + } + + // @ todo Ensure there is currently no entry for the given DID in the DID Registry + // OR + // there is currently an entry and it references this storage node endpoint + + // Save the DID document + const didDb = Utils.getDidDocumentDb() + + // Create CouchDB database user matching username and password + const documentData = { + _id: `${jsonDoc.id}-${jsonDoc.versionId}`, + ...jsonDoc + }; + + try { + await didDb.insert(documentData); + return Utils.success(res, {}); + } catch (err) { + /*if (err.error == 'conflict') { + return Utils.error(res, `DID Document already exists. Use PUT request to update.`) + }*/ + + return Utils.error(res, `Unknown error: ${err.message}`) + } } async delete(req, res) { @@ -77,38 +120,12 @@ class DidStorage { const did = req.params.did.toLowerCase() const allVersions = req.query.allVersions && req.query.allVersions === 'true' - try { - const document = await Utils.getDidDocument(did, allVersions) - - let result - if (allVersions) { - // strip `_id`, `_rev` and `_revisions` from documents - console.log(document) - const versions = document.map(item => { - delete item['_id'] - delete item['_rev'] - delete item['_revisions'] - - return item - }) - - result = { - versions - } - } else { - delete document['_id'] - result = document - } - - return Utils.success(res, result) - } catch (err) { - if (err.reason == 'missing') { - return Utils.error(res, `DID Document not found.`, 404) - } - - console.error(err) - return Utils.error(res, `Unknown error: ${err.message}`) + const result = await Utils.getDidDocument(did, allVersions) + if (!result) { + return Utils.error(res, `DID Document not found.`, 404) } + + return Utils.success(res, result) } async migrate(req, res) { diff --git a/src/services/didStorage/utils.js b/src/services/didStorage/utils.js index ce7b182f..c869fee6 100644 --- a/src/services/didStorage/utils.js +++ b/src/services/didStorage/utils.js @@ -44,22 +44,56 @@ class Utils { return couch.db.use(process.env.DB_DIDS); } - async getDidDocument(did, all=false) { + async getDidDocument(did, allVersions=false) { const db = this.getDidDocumentDb() - /*const res = await db.list({ - include_docs: true - }) - console.log(res.rows[3])*/ + const query = { + selector: { + id: did + }, + fields: ['id', 'versionId'], + sort: [ + {'versionId': 'desc'} + ], + limit: 1 + } + + const result = await db.find(query) + + if (result.docs.length === 0) { + return + } + + const latestDoc = result.docs[0] + let resultDocs = [latestDoc] - const options = {} - if (all) { - options.meta = true + if (allVersions) { + const latestVersion = latestDoc.versionId + const keys = [] + for (let i = 0; i<=latestVersion; i++) { + keys.push(`${did}-${i}`) + } + + // Fetch all the versions + const allDocs = await db.fetch({ keys }) + resultDocs = allDocs.rows } - console.log(did, options) + const docs = resultDocs.map(item => { + if (item.doc) { + item = item.doc + } + + delete item['_id'] + delete item['_rev'] + return item + }) + + if (!allVersions) { + return docs[0] + } - return db.get(did.toLowerCase(), options) + return docs } } diff --git a/test/did-storage.js b/test/did-storage.js index 64bc28aa..18adc0e8 100644 --- a/test/did-storage.js +++ b/test/did-storage.js @@ -4,19 +4,23 @@ import { ethers } from 'ethers' import { DIDDocument } from '@verida/did-document' import CONFIG from './config' +import { id } from 'ethers/utils'; const { SERVER_URL } = CONFIG const DID_URL = `${SERVER_URL}/did` const wallet = ethers.Wallet.createRandom() -//const DID_ADDRESS = wallet.address -//const DID = `did:vda:testnet:${DID_ADDRESS}` -//const DID_PK = wallet.signingKey.publicKey +const DID_ADDRESS = wallet.address +const DID = `did:vda:testnet:${DID_ADDRESS}` +const DID_PK = wallet.signingKey.publicKey -const DID_ADDRESS = '0x56f2c429fC8fdd4911F472a3c451341EAEC989a2' -const DID = 'did:vda:testnet:0x56f2c429fC8fdd4911F472a3c451341EAEC989a2' -const DID_PK = '0x04d1c85058d70c637f8ec46df26cbe855829a51f3335731352e2d1587e478b66e350e49bd4650685039c4b4e0adab5bb2a680d5a13dfb176a311544ec503999f4f' + +/* +const DID_ADDRESS = '0xDd07ddBC34cE9794495B2d464073975ec2376930' +const DID = 'did:vda:testnet:0xDd07ddBC34cE9794495B2d464073975ec2376930' +const DID_PK = '0x04025e031149315eae2fea4514a4a47ca6b86c84fb89a17ab9cd3d808033ca8475f92cd204a768ea1c41cb3431527673957e4dfbcfc741e5d1f2c97a647da14874' +*/ describe("DID Storage Tests", function() { /*this.beforeAll(async () => { @@ -25,6 +29,12 @@ describe("DID Storage Tests", function() { accountInfo = await TestUtils.connectAccount(CONFIG.VDA_PRIVATE_KEY) })*/ + this.beforeAll(async () => { + console.log('Executing with:') + console.log(`DID: ${DID}`) + console.log(`DID_PK: ${DID_PK}`) + }) + describe("Create", () => { it("Success", async () => { const doc = new DIDDocument(DID, DID_PK) @@ -52,7 +62,53 @@ describe("DID Storage Tests", function() { }) }) - describe.only("Get", () => { + describe("Update", () => { + it("Fail - Not next versionId", async () => { + const doc = new DIDDocument(DID, DID_PK) + + try { + await Axios.put(`${DID_URL}/${DID}`, { + document: doc.export() + }); + + assert.fail('DID Document was updated with invalid versionId') + } catch (err) { + assert.equal(err.response.data.status, 'fail', 'DID Document create failed') + console.log(err.response.data) + assert.ok(err.response.data.message.match('Invalid DID Document: Invalid value for versionId'), 'Rejected because incorrect version') + } + }) + + it("Fail - Invalid DID", async () => { + try { + const doc = new DIDDocument(DID, DID_PK) + await Axios.put(`${DID_URL}/abc123`, { + document: doc.export() + }); + + assert.fail(`DID Document was found, when it shouldn't have`) + } catch (err) { + assert.equal(err.response.data.status, 'fail', 'Get DID Document failed') + console.log(err.response.data) + assert.ok(err.response.data.message.match('DID Document not found'), `Rejected because DID Document doesn't exists`) + } + }) + + it("Success", async () => { + const basicDoc = new DIDDocument(DID, DID_PK) + const document = basicDoc.export() + document.versionId = document.versionId + 1 + + console.log(`${DID_URL}/${DID}`) + const createResult = await Axios.put(`${DID_URL}/${DID}`, { + document + }); + + assert.equal(createResult.data.status, 'success', 'Success response') + }) + }) + + describe("Get", () => { it("Success - Latest", async () => { const getResult = await Axios.get(`${DID_URL}/${DID}`); @@ -82,19 +138,14 @@ describe("DID Storage Tests", function() { assert.ok(getResult.data.status, 'success', 'Success response') - // @tgodo: re-build document and compare it matches - console.log(getResult.data) - }) - }) + console.log(getResult.data.data) - describe("Update", () => { - it("Success", async () => { - console.log(`${DID_URL}/${DID}`) - const createResult = await Axios.put(`${DID_URL}/${DID}`, { - hello: 'world' - }); + assert.equal(getResult.data.data.length, 2, 'Two versions returned') + assert.equal(getResult.data.data[0].versionId, 0, 'First doc is version 0') + assert.equal(getResult.data.data[1].versionId, 1, 'Second doc is version 1') - console.log(createResult.data) + // @tgodo: re-build document and compare it matches + }) }) From b53bb9d007813dc6559766cc678acec3605586b1 Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 9 Nov 2022 14:29:22 +1030 Subject: [PATCH 04/13] Deletion now working. All tests passing. --- src/services/didStorage/controller.js | 39 ++++++++--- src/services/didStorage/utils.js | 11 ++-- test/did-storage.js | 94 +++++++++++++++++++-------- 3 files changed, 103 insertions(+), 41 deletions(-) diff --git a/src/services/didStorage/controller.js b/src/services/didStorage/controller.js index c9acb38a..961c1125 100644 --- a/src/services/didStorage/controller.js +++ b/src/services/didStorage/controller.js @@ -84,7 +84,6 @@ class DidStorage { // OR // there is currently an entry and it references this storage node endpoint - // Save the DID document const didDb = Utils.getDidDocumentDb() // Create CouchDB database user matching username and password @@ -106,13 +105,35 @@ class DidStorage { } async delete(req, res) { - const did = req.params.did + if (!req.params.did) { + return Utils.error(res, `No DID specified`) + } - return res.status(200).send({ - status: "success-delete", - data: { - "did": did - } + const did = req.params.did.toLowerCase() + const didDocuments = await Utils.getDidDocument(did, true, false) + + if (!didDocuments || didDocuments.length === 0) { + return Utils.error(res, `DID Document not found`) + } + + const didDb = Utils.getDidDocumentDb() + const docs = [] + + didDocuments.forEach(doc => { + docs.push({ + _id: doc._id, + _rev: doc._rev, + _deleted: true + }) + }) + + const deleteResponse = await didDb.bulk({ + docs + }) + + return Utils.success(res, { + did, + revisions: docs.length }); } @@ -129,6 +150,8 @@ class DidStorage { } async migrate(req, res) { + return Utils.error(res, `Not implemented (yet)`, 404) + /* const did = req.params.did return res.status(200).send({ @@ -136,7 +159,7 @@ class DidStorage { data: { "did": did } - }); + });*/ } } diff --git a/src/services/didStorage/utils.js b/src/services/didStorage/utils.js index c869fee6..923a7b3f 100644 --- a/src/services/didStorage/utils.js +++ b/src/services/didStorage/utils.js @@ -4,8 +4,6 @@ class Utils { // @todo verifyDocument(did, document, expectedValues = {}) { - console.log('verifying doc') - const doc = document.export() for (let key in expectedValues) { if (doc[key] != expectedValues[key]) { @@ -44,7 +42,7 @@ class Utils { return couch.db.use(process.env.DB_DIDS); } - async getDidDocument(did, allVersions=false) { + async getDidDocument(did, allVersions=false, stripCouchMetadata=true) { const db = this.getDidDocumentDb() const query = { @@ -83,9 +81,12 @@ class Utils { if (item.doc) { item = item.doc } + + if (stripCouchMetadata) { + delete item['_id'] + delete item['_rev'] + } - delete item['_id'] - delete item['_rev'] return item }) diff --git a/test/did-storage.js b/test/did-storage.js index 18adc0e8..9289fa82 100644 --- a/test/did-storage.js +++ b/test/did-storage.js @@ -11,16 +11,21 @@ const DID_URL = `${SERVER_URL}/did` const wallet = ethers.Wallet.createRandom() -const DID_ADDRESS = wallet.address -const DID = `did:vda:testnet:${DID_ADDRESS}` -const DID_PK = wallet.signingKey.publicKey - - -/* -const DID_ADDRESS = '0xDd07ddBC34cE9794495B2d464073975ec2376930' -const DID = 'did:vda:testnet:0xDd07ddBC34cE9794495B2d464073975ec2376930' -const DID_PK = '0x04025e031149315eae2fea4514a4a47ca6b86c84fb89a17ab9cd3d808033ca8475f92cd204a768ea1c41cb3431527673957e4dfbcfc741e5d1f2c97a647da14874' -*/ +//const WALLET_TYPE = 'manual' +const WALLET_TYPE = 'create' + +let DID_ADDRESS, DID, DID_PK + +if (WALLET_TYPE == 'create') { + DID_ADDRESS = wallet.address + DID = `did:vda:testnet:${DID_ADDRESS}` + DID_PK = wallet.signingKey.publicKey +} +else { + DID_ADDRESS = '0x3529bEae0adE19C53c9Dbcd08B8b20510E455e45' + DID = 'did:vda:testnet:0x3529bEae0adE19C53c9Dbcd08B8b20510E455e45' + DID_PK = '0x04648d3bdcdce7c0a47a25a8a19d19748f1e6767b6f5f3f0895ca04192bef84e90657b85a803b0573d9d4b1d8c6fb16ac97e3ccc2d8826dec524da5274a5ea7ef4' +} describe("DID Storage Tests", function() { /*this.beforeAll(async () => { @@ -74,7 +79,6 @@ describe("DID Storage Tests", function() { assert.fail('DID Document was updated with invalid versionId') } catch (err) { assert.equal(err.response.data.status, 'fail', 'DID Document create failed') - console.log(err.response.data) assert.ok(err.response.data.message.match('Invalid DID Document: Invalid value for versionId'), 'Rejected because incorrect version') } }) @@ -89,7 +93,6 @@ describe("DID Storage Tests", function() { assert.fail(`DID Document was found, when it shouldn't have`) } catch (err) { assert.equal(err.response.data.status, 'fail', 'Get DID Document failed') - console.log(err.response.data) assert.ok(err.response.data.message.match('DID Document not found'), `Rejected because DID Document doesn't exists`) } }) @@ -99,7 +102,6 @@ describe("DID Storage Tests", function() { const document = basicDoc.export() document.versionId = document.versionId + 1 - console.log(`${DID_URL}/${DID}`) const createResult = await Axios.put(`${DID_URL}/${DID}`, { document }); @@ -128,39 +130,75 @@ describe("DID Storage Tests", function() { assert.ok(err.response.data.message.match('DID Document not found'), `Rejected because DID Document doesn't exists`) } }) - - // versionId - // versionTime - // allVersions it("Success - All versions", async () => { const getResult = await Axios.get(`${DID_URL}/${DID}?allVersions=true`); assert.ok(getResult.data.status, 'success', 'Success response') - - console.log(getResult.data.data) - assert.equal(getResult.data.data.length, 2, 'Two versions returned') assert.equal(getResult.data.data[0].versionId, 0, 'First doc is version 0') assert.equal(getResult.data.data[1].versionId, 1, 'Second doc is version 1') - - // @tgodo: re-build document and compare it matches - }) + + // Get by versionId + // Get by versionTime }) describe("Delete", () => { + it("Fail - Invalid DID", async () => { + try { + await Axios.delete(`${DID_URL}/abc123`); + + assert.fail(`DID Document was found, when it shouldn't have`) + } catch (err) { + assert.equal(err.response.data.status, 'fail', 'Get DID Document failed') + assert.ok(err.response.data.message.match('DID Document not found'), `Rejected because DID Document doesn't exists`) + } + }) + it("Success", async () => { - console.log(`${DID_URL}/${DID}`) - const createResult = await Axios.delete(`${DID_URL}/${DID}`, { + const deleteResult = await Axios.delete(`${DID_URL}/${DID}`, { hello: 'world' }); - console.log(createResult.data) + assert.ok(deleteResult.data.status, 'success', 'Success response') + assert.equal(deleteResult.data.data.revisions, 2, 'Two versions deleted') + }) + + it("Fail - Deleted", async () => { + try { + await Axios.delete(`${DID_URL}/${DID}`); + + assert.fail(`DID Document was found, when it shouldn't have`) + } catch (err) { + assert.equal(err.response.data.status, 'fail', 'Get DID Document failed') + assert.ok(err.response.data.message.match('DID Document not found'), `Rejected because DID Document doesn't exists`) + } + }) + }) + + describe("Create again - After deletion", () => { + it("Success", async () => { + const doc = new DIDDocument(DID, DID_PK) + + const createResult = await Axios.post(`${DID_URL}/${DID}`, { + document: doc.export() + }); + + assert.equal(createResult.data.status, 'success', 'Success response') + }) + + it("Success - Document exists", async () => { + const getResult = await Axios.get(`${DID_URL}/${DID}`); + + assert.ok(getResult.data.status, 'success', 'Success response') + + // @tgodo: re-build document and compare it matches + //console.log(getResult.data) }) }) - describe("Migrate", () => { + /*describe("Migrate", () => { it("Success", async () => { console.log(`${DID_URL}/${DID}/migrate`) const createResult = await Axios.post(`${DID_URL}/${DID}/migrate`, { @@ -169,5 +207,5 @@ describe("DID Storage Tests", function() { console.log(createResult.data) }) - }) + })*/ }) \ No newline at end of file From 48b857b76af2edb6eadcb89cd64be98c1e6ae42e Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 9 Nov 2022 16:58:40 +1030 Subject: [PATCH 05/13] Support field validation for create, including proof verification --- src/services/didStorage/controller.js | 5 +++ src/services/didStorage/utils.js | 45 +++++++++++++++++++++------ test/did-storage.js | 25 ++++++++++----- 3 files changed, 58 insertions(+), 17 deletions(-) diff --git a/src/services/didStorage/controller.js b/src/services/didStorage/controller.js index 961c1125..3a9e6a9d 100644 --- a/src/services/didStorage/controller.js +++ b/src/services/didStorage/controller.js @@ -74,8 +74,13 @@ class DidStorage { const nextVersionId = existingDoc.versionId + 1 Utils.verifyDocument(did, didDocument, { + created: existingDoc.created, versionId: nextVersionId }) + + if (existingDoc.updated > jsonDoc.updated) { + return Utils.error(res, `updated must be after the current document`) + } } catch (err) { return Utils.error(res, `Invalid DID Document: ${err.message}`) } diff --git a/src/services/didStorage/utils.js b/src/services/didStorage/utils.js index 923a7b3f..706e4bca 100644 --- a/src/services/didStorage/utils.js +++ b/src/services/didStorage/utils.js @@ -5,20 +5,47 @@ class Utils { // @todo verifyDocument(did, document, expectedValues = {}) { const doc = document.export() + if (doc.id != did) { + throw new Error(`DID must match ID in the DID Document`) + } + for (let key in expectedValues) { if (doc[key] != expectedValues[key]) { throw new Error(`Invalid value for ${key}`) } } - //throw new Error(`versionId not set`) - /* - versionId - created - updated - deactivated - proof — A string representing the full DID Document as a JSON encoded string that has been hashed using keccak256 and signed with ed25519, the default Ethereum based signature scheme. - */ + const requiredFields = ['versionId', 'created', 'updated', 'proof'] + requiredFields.forEach(field => { + if (!doc.hasOwnProperty(field)) { + throw new Error(`Missing required field (${field})`) + } + }) + + if (typeof(doc.versionId) !== 'number') { + console.log(doc.versionId, typeof(doc.versionId)) + throw new Error(`versionId must be a number`) + } + + // ie: 2020-12-20T19:17:47Z + // @see https://www.w3.org/TR/did-core/#did-document-metadata + if (!Date.parse(doc.created) || document.buildTimestamp(new Date(Date.parse(doc.created))) != doc.created) { + throw new Error(`created must be a valid timestamp`) + } + + // ie: 2020-12-20T19:17:47Z + // @see https://www.w3.org/TR/did-core/#did-document-metadata + if (!Date.parse(doc.updated) || document.buildTimestamp(new Date(Date.parse(doc.updated))) != doc.updated) { + throw new Error(`created must be a valid timestamp`) + } + + if (doc.deactivated && typeof(doc.deactivated) !== 'boolean') { + throw new Error(`deactivated must be a valid boolean value`) + } + + if (!document.verifyProof()) { + throw new Error(`Invalid proof`) + } return true } @@ -86,7 +113,7 @@ class Utils { delete item['_id'] delete item['_rev'] } - + return item }) diff --git a/test/did-storage.js b/test/did-storage.js index 9289fa82..df7f0cab 100644 --- a/test/did-storage.js +++ b/test/did-storage.js @@ -14,17 +14,19 @@ const wallet = ethers.Wallet.createRandom() //const WALLET_TYPE = 'manual' const WALLET_TYPE = 'create' -let DID_ADDRESS, DID, DID_PK +let DID_ADDRESS, DID, DID_PK, DID_PRIVATE_KEY if (WALLET_TYPE == 'create') { DID_ADDRESS = wallet.address DID = `did:vda:testnet:${DID_ADDRESS}` DID_PK = wallet.signingKey.publicKey + DID_PRIVATE_KEY = wallet.privateKey } else { DID_ADDRESS = '0x3529bEae0adE19C53c9Dbcd08B8b20510E455e45' DID = 'did:vda:testnet:0x3529bEae0adE19C53c9Dbcd08B8b20510E455e45' DID_PK = '0x04648d3bdcdce7c0a47a25a8a19d19748f1e6767b6f5f3f0895ca04192bef84e90657b85a803b0573d9d4b1d8c6fb16ac97e3ccc2d8826dec524da5274a5ea7ef4' + DID_PRIVATE_KEY = '0xadc3930bb646015be35da24140d3fafa2c0c8fbfaefb85d25122ddc7384670f9' } describe("DID Storage Tests", function() { @@ -37,18 +39,25 @@ describe("DID Storage Tests", function() { this.beforeAll(async () => { console.log('Executing with:') console.log(`DID: ${DID}`) - console.log(`DID_PK: ${DID_PK}`) + console.log(`DID_PUB_KEY: ${DID_PK}`) + console.log(`DID_PRIVATE_KEY: ${DID_PRIVATE_KEY}`) }) describe("Create", () => { - it("Success", async () => { - const doc = new DIDDocument(DID, DID_PK) + it.only("Success", async () => { + try { + const doc = new DIDDocument(DID, DID_PK) + doc.signProof(wallet.privateKey) - const createResult = await Axios.post(`${DID_URL}/${DID}`, { - document: doc.export() - }); + const createResult = await Axios.post(`${DID_URL}/${DID}`, { + document: doc.export() + }); - assert.equal(createResult.data.status, 'success', 'Success response') + assert.equal(createResult.data.status, 'success', 'Success response') + } catch (err) { + console.error(err.response.data) + assert.fail(err.response.data.message) + } }) it("Fail - Duplicate DID Document", async () => { From c98914c7099d79660c0471f42dbca4ef23b9c601 Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 9 Nov 2022 17:30:54 +1030 Subject: [PATCH 06/13] Tests destroy database before starting. Fix create and update tests to work with new verification. --- src/components/userManager.js | 17 ------- src/server-app.js | 2 + src/services/didStorage/controller.js | 1 + src/services/didStorage/utils.js | 35 ++++++++++++-- test/did-storage.js | 68 ++++++++++++++++++--------- 5 files changed, 79 insertions(+), 44 deletions(-) diff --git a/src/components/userManager.js b/src/components/userManager.js index 7404b082..bc4abcca 100644 --- a/src/components/userManager.js +++ b/src/components/userManager.js @@ -100,23 +100,6 @@ class UserManager { throw err; } } - - try { - await couch.db.create(process.env.DB_DIDS) - const dbDids = couch.db.use(process.env.DB_DIDS) - await dbDids.createIndex({ - index: { - fields: ['id'] - }, - name: 'did' - }) - } catch (err) { - if (err.message == "The database could not be created, the file already exists.") { - console.log("DID database not created -- already existed"); - } else { - throw err; - } - } } } diff --git a/src/server-app.js b/src/server-app.js index d567afce..ea5229aa 100644 --- a/src/server-app.js +++ b/src/server-app.js @@ -10,6 +10,7 @@ import didStorageRoutes from './services/didStorage/routes' import requestValidator from './middleware/requestValidator.js'; import userManager from './components/userManager.js'; import AuthManager from './components/authManager.js'; +import didUtils from './services/didStorage/utils' dotenv.config(); @@ -32,5 +33,6 @@ app.use(privateRoutes); AuthManager.initDb(); userManager.ensureDefaultDatabases(); +didUtils.createDb() export default app; diff --git a/src/services/didStorage/controller.js b/src/services/didStorage/controller.js index 3a9e6a9d..03506c84 100644 --- a/src/services/didStorage/controller.js +++ b/src/services/didStorage/controller.js @@ -68,6 +68,7 @@ class DidStorage { let existingDoc try { existingDoc = await Utils.getDidDocument(did) + if (!existingDoc) { return Utils.error(res, `DID Document not found`) } diff --git a/src/services/didStorage/utils.js b/src/services/didStorage/utils.js index 706e4bca..0c4a9c10 100644 --- a/src/services/didStorage/utils.js +++ b/src/services/didStorage/utils.js @@ -1,4 +1,6 @@ import Db from "../../components/db.js" +import dotenv from 'dotenv'; +dotenv.config(); class Utils { @@ -11,7 +13,7 @@ class Utils { for (let key in expectedValues) { if (doc[key] != expectedValues[key]) { - throw new Error(`Invalid value for ${key}`) + throw new Error(`Missing value for ${key} (Expected ${expectedValues[key]})`) } } @@ -23,13 +25,14 @@ class Utils { }) if (typeof(doc.versionId) !== 'number') { - console.log(doc.versionId, typeof(doc.versionId)) throw new Error(`versionId must be a number`) } // ie: 2020-12-20T19:17:47Z // @see https://www.w3.org/TR/did-core/#did-document-metadata if (!Date.parse(doc.created) || document.buildTimestamp(new Date(Date.parse(doc.created))) != doc.created) { + console.log(new Date(Date.parse(doc.created))) + console.log(doc.created) throw new Error(`created must be a valid timestamp`) } @@ -64,9 +67,32 @@ class Utils { }) } + getDb() { + return Db.getCouch() + } + + async createDb() { + try { + const couch = this.getDb() + await couch.db.create(process.env.DB_DIDS) + const dbDids = couch.db.use(process.env.DB_DIDS) + await dbDids.createIndex({ + index: { + fields: ['id'] + }, + name: 'did' + }) + } catch (err) { + if (err.message == "The database could not be created, the file already exists.") { + console.log("DID database not created -- already existed"); + } else { + throw err; + } + } + } + getDidDocumentDb() { - const couch = Db.getCouch() - return couch.db.use(process.env.DB_DIDS); + return this.getDb().use(process.env.DB_DIDS); } async getDidDocument(did, allVersions=false, stripCouchMetadata=true) { @@ -76,7 +102,6 @@ class Utils { selector: { id: did }, - fields: ['id', 'versionId'], sort: [ {'versionId': 'desc'} ], diff --git a/test/did-storage.js b/test/did-storage.js index df7f0cab..7ea4b26c 100644 --- a/test/did-storage.js +++ b/test/did-storage.js @@ -3,8 +3,11 @@ import assert from 'assert'; import { ethers } from 'ethers' import { DIDDocument } from '@verida/did-document' +import dotenv from 'dotenv'; +dotenv.config(); + +import Utils from '../src/services/didStorage/utils' import CONFIG from './config' -import { id } from 'ethers/utils'; const { SERVER_URL } = CONFIG const DID_URL = `${SERVER_URL}/did` @@ -22,29 +25,25 @@ if (WALLET_TYPE == 'create') { DID_PK = wallet.signingKey.publicKey DID_PRIVATE_KEY = wallet.privateKey } -else { - DID_ADDRESS = '0x3529bEae0adE19C53c9Dbcd08B8b20510E455e45' - DID = 'did:vda:testnet:0x3529bEae0adE19C53c9Dbcd08B8b20510E455e45' - DID_PK = '0x04648d3bdcdce7c0a47a25a8a19d19748f1e6767b6f5f3f0895ca04192bef84e90657b85a803b0573d9d4b1d8c6fb16ac97e3ccc2d8826dec524da5274a5ea7ef4' - DID_PRIVATE_KEY = '0xadc3930bb646015be35da24140d3fafa2c0c8fbfaefb85d25122ddc7384670f9' -} -describe("DID Storage Tests", function() { - /*this.beforeAll(async () => { - //await AuthManager.initDb() -- This is required if the server is running locally and has never been run before, run just once - //await TestUtils.ensureVeridaAccount(CONFIG.VDA_PRIVATE_KEY) -- This is required if the private key has never been initilaized with an application context, run just once - accountInfo = await TestUtils.connectAccount(CONFIG.VDA_PRIVATE_KEY) - })*/ +let masterDidDoc +describe("DID Storage Tests", function() { this.beforeAll(async () => { console.log('Executing with:') console.log(`DID: ${DID}`) console.log(`DID_PUB_KEY: ${DID_PK}`) console.log(`DID_PRIVATE_KEY: ${DID_PRIVATE_KEY}`) + + // clear DID database + const couch = Utils.getDb() + await couch.db.destroy(process.env.DB_DIDS) + await Utils.createDb() + console.log('destroyed!') }) - describe("Create", () => { - it.only("Success", async () => { + describe.only("Create", () => { + it("Success", async () => { try { const doc = new DIDDocument(DID, DID_PK) doc.signProof(wallet.privateKey) @@ -53,6 +52,8 @@ describe("DID Storage Tests", function() { document: doc.export() }); + masterDidDoc = doc.export() + assert.equal(createResult.data.status, 'success', 'Success response') } catch (err) { console.error(err.response.data) @@ -62,6 +63,7 @@ describe("DID Storage Tests", function() { it("Fail - Duplicate DID Document", async () => { const doc = new DIDDocument(DID, DID_PK) + doc.signProof(wallet.privateKey) try { await Axios.post(`${DID_URL}/${DID}`, { @@ -76,9 +78,14 @@ describe("DID Storage Tests", function() { }) }) - describe("Update", () => { + describe.only("Update", () => { it("Fail - Not next versionId", async () => { const doc = new DIDDocument(DID, DID_PK) + doc.setAttributes({ + created: masterDidDoc.created + }) + + doc.signProof(wallet.privateKey) try { await Axios.put(`${DID_URL}/${DID}`, { @@ -88,13 +95,18 @@ describe("DID Storage Tests", function() { assert.fail('DID Document was updated with invalid versionId') } catch (err) { assert.equal(err.response.data.status, 'fail', 'DID Document create failed') - assert.ok(err.response.data.message.match('Invalid DID Document: Invalid value for versionId'), 'Rejected because incorrect version') + assert.equal(err.response.data.message, 'Invalid DID Document: Missing value for versionId (Expected 1)', 'Rejected because incorrect version') } }) it("Fail - Invalid DID", async () => { try { const doc = new DIDDocument(DID, DID_PK) + doc.setAttributes({ + created: masterDidDoc.created + }) + doc.signProof(wallet.privateKey) + await Axios.put(`${DID_URL}/abc123`, { document: doc.export() }); @@ -108,14 +120,25 @@ describe("DID Storage Tests", function() { it("Success", async () => { const basicDoc = new DIDDocument(DID, DID_PK) + const document = basicDoc.export() - document.versionId = document.versionId + 1 + basicDoc.setAttributes({ + created: masterDidDoc.created, + versionId: document.versionId + 1 + }) - const createResult = await Axios.put(`${DID_URL}/${DID}`, { - document - }); + basicDoc.signProof(wallet.privateKey) + + try { + const createResult = await Axios.put(`${DID_URL}/${DID}`, { + document: basicDoc.export() + }); - assert.equal(createResult.data.status, 'success', 'Success response') + assert.equal(createResult.data.status, 'success', 'Success response') + } catch (err) { + console.error(err.response.data) + assert.fail('Error updating') + } }) }) @@ -189,6 +212,7 @@ describe("DID Storage Tests", function() { describe("Create again - After deletion", () => { it("Success", async () => { const doc = new DIDDocument(DID, DID_PK) + doc.signProof(wallet.privateKey) const createResult = await Axios.post(`${DID_URL}/${DID}`, { document: doc.export() From a8db1cb84f8a9afac3b791c65299566868f48677 Mon Sep 17 00:00:00 2001 From: Chris Date: Thu, 10 Nov 2022 10:46:46 +1030 Subject: [PATCH 07/13] Fix missing versionId field on index --- src/services/didStorage/utils.js | 2 +- test/did-storage.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/didStorage/utils.js b/src/services/didStorage/utils.js index 0c4a9c10..5ec179b3 100644 --- a/src/services/didStorage/utils.js +++ b/src/services/didStorage/utils.js @@ -78,7 +78,7 @@ class Utils { const dbDids = couch.db.use(process.env.DB_DIDS) await dbDids.createIndex({ index: { - fields: ['id'] + fields: ['id', 'versionId'] }, name: 'did' }) diff --git a/test/did-storage.js b/test/did-storage.js index 7ea4b26c..47f04c57 100644 --- a/test/did-storage.js +++ b/test/did-storage.js @@ -42,7 +42,7 @@ describe("DID Storage Tests", function() { console.log('destroyed!') }) - describe.only("Create", () => { + describe("Create", () => { it("Success", async () => { try { const doc = new DIDDocument(DID, DID_PK) @@ -78,7 +78,7 @@ describe("DID Storage Tests", function() { }) }) - describe.only("Update", () => { + describe("Update", () => { it("Fail - Not next versionId", async () => { const doc = new DIDDocument(DID, DID_PK) doc.setAttributes({ From a67c9563d13b13740ec6eb1a42791d012dafe9c3 Mon Sep 17 00:00:00 2001 From: Chris Date: Sat, 12 Nov 2022 08:35:40 +1030 Subject: [PATCH 08/13] Upgrade to latest ethers version --- package.json | 2 +- test/did-storage.js | 16 ++++++--------- yarn.lock | 48 --------------------------------------------- 3 files changed, 7 insertions(+), 59 deletions(-) diff --git a/package.json b/package.json index 0b276297..424d411f 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "cors": "^2.8.5", "did-resolver": "^3.1.0", "dotenv": "^8.2.0", - "ethers": "^4.0.42", + "ethers": "^5.7.2", "express": "^4.17.1", "express-basic-auth": "git+https://github.com/Mozzler/express-basic-auth.git", "jsonwebtoken": "^8.5.1", diff --git a/test/did-storage.js b/test/did-storage.js index 47f04c57..7f55cca8 100644 --- a/test/did-storage.js +++ b/test/did-storage.js @@ -14,17 +14,12 @@ const DID_URL = `${SERVER_URL}/did` const wallet = ethers.Wallet.createRandom() -//const WALLET_TYPE = 'manual' -const WALLET_TYPE = 'create' - let DID_ADDRESS, DID, DID_PK, DID_PRIVATE_KEY -if (WALLET_TYPE == 'create') { - DID_ADDRESS = wallet.address - DID = `did:vda:testnet:${DID_ADDRESS}` - DID_PK = wallet.signingKey.publicKey - DID_PRIVATE_KEY = wallet.privateKey -} +DID_ADDRESS = wallet.address +DID = `did:vda:testnet:${DID_ADDRESS}` +DID_PK = wallet.publicKey +DID_PRIVATE_KEY = wallet.privateKey let masterDidDoc @@ -146,7 +141,8 @@ describe("DID Storage Tests", function() { it("Success - Latest", async () => { const getResult = await Axios.get(`${DID_URL}/${DID}`); - assert.ok(getResult.data.status, 'success', 'Success response') + assert.ok(getResult.data.status, 'Success response') + assert.equal(getResult.data.data.id, DID.toLowerCase(), 'DID mathces') // @tgodo: re-build document and compare it matches //console.log(getResult.data) diff --git a/yarn.lock b/yarn.lock index 48171b9b..4da09321 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3687,21 +3687,6 @@ etag@~1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg== -ethers@^4.0.42: - version "4.0.49" - resolved "https://registry.yarnpkg.com/ethers/-/ethers-4.0.49.tgz#0eb0e9161a0c8b4761be547396bbe2fb121a8894" - integrity sha512-kPltTvWiyu+OktYy1IStSO16i2e7cS9D9OxZ81q2UUaiNPVrm/RTcbxamCXF9VUSKzJIdJV68EAIhTEVBalRWg== - dependencies: - aes-js "3.0.0" - bn.js "^4.11.9" - elliptic "6.5.4" - hash.js "1.1.3" - js-sha3 "0.5.7" - scrypt-js "2.0.4" - setimmediate "1.0.4" - uuid "2.0.1" - xmlhttprequest "1.8.0" - ethers@^5.5.1: version "5.7.0" resolved "https://registry.yarnpkg.com/ethers/-/ethers-5.7.0.tgz#0055da174b9e076b242b8282638bc94e04b39835" @@ -4207,14 +4192,6 @@ hash-base@^3.0.0: readable-stream "^3.6.0" safe-buffer "^5.2.0" -hash.js@1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/hash.js/-/hash.js-1.1.3.tgz#340dedbe6290187151c1ea1d777a3448935df846" - integrity sha512-/UETyP0W22QILqS+6HowevwhEFJ3MBJnwTf75Qob9Wz9t0DPuisL8kW8YZMK62dHAKE1c1p+gY1TtOLY+USEHA== - dependencies: - inherits "^2.0.3" - minimalistic-assert "^1.0.0" - hash.js@1.1.7, hash.js@^1.0.0, hash.js@^1.0.3: version "1.1.7" resolved "https://registry.yarnpkg.com/hash.js/-/hash.js-1.1.7.tgz#0babca538e8d4ee4a0f8988d68866537a003cf42" @@ -4540,11 +4517,6 @@ jmespath@0.16.0: resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.16.0.tgz#b15b0a85dfd4d930d43e69ed605943c802785076" integrity sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw== -js-sha3@0.5.7: - version "0.5.7" - resolved "https://registry.yarnpkg.com/js-sha3/-/js-sha3-0.5.7.tgz#0d4ffd8002d5333aabaf4a23eed2f6374c9f28e7" - integrity sha512-GII20kjaPX0zJ8wzkTbNDYMY7msuZcTWk8S5UOh6806Jq/wz1J8/bnr8uGU0DAUmYDjj2Mr4X1cW8v/GLYnR+g== - js-sha3@0.8.0, js-sha3@^0.8.0: version "0.8.0" resolved "https://registry.yarnpkg.com/js-sha3/-/js-sha3-0.8.0.tgz#b9b7a5da73afad7dedd0f8c463954cbde6818840" @@ -5849,11 +5821,6 @@ schema-utils@^2.6.5: ajv "^6.12.4" ajv-keywords "^3.5.2" -scrypt-js@2.0.4: - version "2.0.4" - resolved "https://registry.yarnpkg.com/scrypt-js/-/scrypt-js-2.0.4.tgz#32f8c5149f0797672e551c07e230f834b6af5f16" - integrity sha512-4KsaGcPnuhtCZQCxFxN3GVYIhKFPTdLd8PLC552XwbMndtD0cjRFAhDuuydXQ0h08ZfPgzqe6EKHozpuH74iDw== - scrypt-js@3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/scrypt-js/-/scrypt-js-3.0.1.tgz#d314a57c2aef69d1ad98a138a21fe9eafa9ee312" @@ -5913,11 +5880,6 @@ set-blocking@^2.0.0: resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" integrity sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw== -setimmediate@1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.4.tgz#20e81de622d4a02588ce0c8da8973cbcf1d3138f" - integrity sha512-/TjEmXQVEzdod/FFskf3o7oOAsGhHf2j1dZqRFbDzq4F3mvvxflIIi4Hd3bLQE9y/CpwqfSQam5JakI/mi3Pog== - setprototypeof@1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" @@ -6377,11 +6339,6 @@ utils-merge@1.0.1: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== -uuid@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-2.0.1.tgz#c2a30dedb3e535d72ccf82e343941a50ba8533ac" - integrity sha512-nWg9+Oa3qD2CQzHIP4qKUqwNfzKn8P0LtFhotaCTFchsV7ZfDhAybeip/HZVeMIpZi9JgY1E3nUlwaCmZT1sEg== - uuid@8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.0.0.tgz#bc6ccf91b5ff0ac07bbcdbf1c7c4e150db4dbb6c" @@ -6524,11 +6481,6 @@ xmlbuilder@~9.0.1: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== -xmlhttprequest@1.8.0: - version "1.8.0" - resolved "https://registry.yarnpkg.com/xmlhttprequest/-/xmlhttprequest-1.8.0.tgz#67fe075c5c24fef39f9d65f5f7b7fe75171968fc" - integrity sha512-58Im/U0mlVBLM38NdZjHyhuMtCqa61469k2YP/AaPbvCoV9aQGUpbJBj1QRm2ytRiVQBD/fsw7L2bJGDVQswBA== - xtend@^4.0.2, xtend@~4.0.0, xtend@~4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" From d45fb8f4e0834aef10284d8e43c678c63cb1fd87 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 15 Nov 2022 17:21:10 +1030 Subject: [PATCH 09/13] Update error messages. Wrap versions response in a versions parameter. --- src/services/didStorage/utils.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/services/didStorage/utils.js b/src/services/didStorage/utils.js index 5ec179b3..2db1ecfa 100644 --- a/src/services/didStorage/utils.js +++ b/src/services/didStorage/utils.js @@ -13,7 +13,7 @@ class Utils { for (let key in expectedValues) { if (doc[key] != expectedValues[key]) { - throw new Error(`Missing value for ${key} (Expected ${expectedValues[key]})`) + throw new Error(`Incorrect value for ${key} (Expected ${expectedValues[key]})`) } } @@ -146,7 +146,9 @@ class Utils { return docs[0] } - return docs + return { + versions: docs + } } } From 8fe2cf33a20a594894789586f0f237e184f7d150 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 15 Nov 2022 17:24:53 +1030 Subject: [PATCH 10/13] Fix handling of all versions being wrapped in a versions parameter. --- src/services/didStorage/controller.js | 11 +++++++++-- test/did-storage.js | 8 ++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/services/didStorage/controller.js b/src/services/didStorage/controller.js index 03506c84..17afe4b5 100644 --- a/src/services/didStorage/controller.js +++ b/src/services/didStorage/controller.js @@ -61,6 +61,8 @@ class DidStorage { return Utils.error(res, `No document specified`) } + // @todo: Verify signature + const did = req.params.did.toLowerCase() const didDocument = new DIDDocument(req.body.document) const jsonDoc = didDocument.export() @@ -115,13 +117,16 @@ class DidStorage { return Utils.error(res, `No DID specified`) } + // @todo: Verify signature + const did = req.params.did.toLowerCase() - const didDocuments = await Utils.getDidDocument(did, true, false) + const versionResponse = await Utils.getDidDocument(did, true, false) - if (!didDocuments || didDocuments.length === 0) { + if (!versionResponse || !versionResponse.versions || versionResponse.versions.length === 0) { return Utils.error(res, `DID Document not found`) } + const didDocuments = versionResponse.versions const didDb = Utils.getDidDocumentDb() const docs = [] @@ -160,6 +165,8 @@ class DidStorage { /* const did = req.params.did + // @todo: Verify signature + return res.status(200).send({ status: "success-migrate", data: { diff --git a/test/did-storage.js b/test/did-storage.js index 7f55cca8..5e9acdea 100644 --- a/test/did-storage.js +++ b/test/did-storage.js @@ -90,7 +90,7 @@ describe("DID Storage Tests", function() { assert.fail('DID Document was updated with invalid versionId') } catch (err) { assert.equal(err.response.data.status, 'fail', 'DID Document create failed') - assert.equal(err.response.data.message, 'Invalid DID Document: Missing value for versionId (Expected 1)', 'Rejected because incorrect version') + assert.equal(err.response.data.message, 'Invalid DID Document: Incorrect value for versionId (Expected 1)', 'Rejected because incorrect version') } }) @@ -163,9 +163,9 @@ describe("DID Storage Tests", function() { const getResult = await Axios.get(`${DID_URL}/${DID}?allVersions=true`); assert.ok(getResult.data.status, 'success', 'Success response') - assert.equal(getResult.data.data.length, 2, 'Two versions returned') - assert.equal(getResult.data.data[0].versionId, 0, 'First doc is version 0') - assert.equal(getResult.data.data[1].versionId, 1, 'Second doc is version 1') + assert.equal(getResult.data.data.versions.length, 2, 'Two versions returned') + assert.equal(getResult.data.data.versions[0].versionId, 0, 'First doc is version 0') + assert.equal(getResult.data.data.versions[1].versionId, 1, 'Second doc is version 1') }) // Get by versionId From 810b648f1688e684951bf6dbaf9a5188b66cc097 Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 16 Nov 2022 10:03:24 +1030 Subject: [PATCH 11/13] Ensure signature verification for update, delete --- src/services/didStorage/controller.js | 27 +++++++++++++++++++++++---- test/did-storage.js | 27 ++++++++++++++++++++++----- test/utils.js | 9 +++++++++ 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/src/services/didStorage/controller.js b/src/services/didStorage/controller.js index 17afe4b5..6547cbc6 100644 --- a/src/services/didStorage/controller.js +++ b/src/services/didStorage/controller.js @@ -1,4 +1,5 @@ import { DIDDocument } from "@verida/did-document" +import { now } from "lodash" import Utils from './utils' class DidStorage { @@ -26,7 +27,7 @@ class DidStorage { return Utils.error(res, `Invalid DID Document: ${err.message}`) } - // @ todo Ensure there is currently no entry for the given DID in the DID Registry + // @todo Ensure there is currently no entry for the given DID in the DID Registry // OR // there is currently an entry and it references this storage node endpoint @@ -61,8 +62,6 @@ class DidStorage { return Utils.error(res, `No document specified`) } - // @todo: Verify signature - const did = req.params.did.toLowerCase() const didDocument = new DIDDocument(req.body.document) const jsonDoc = didDocument.export() @@ -117,15 +116,35 @@ class DidStorage { return Utils.error(res, `No DID specified`) } - // @todo: Verify signature + if (!req.headers.signature) { + return Utils.error(res, `No signature specified`) + } const did = req.params.did.toLowerCase() + const signature = req.headers.signature const versionResponse = await Utils.getDidDocument(did, true, false) + if (!versionResponse || !versionResponse.versions || versionResponse.versions.length === 0) { return Utils.error(res, `DID Document not found`) } + // Verify signature with the most recent DID document + const didDocument = new DIDDocument(versionResponse.versions[0]) + let validSig = false + const nowInMinutes = Math.round((new Date()).getTime() / 1000 / 60) + for (let i=-1; i<=1; i++) { + const proofString = `Delete DID Document ${did} at ${nowInMinutes+i}` + if (didDocument.verifySig(proofString, signature)) { + validSig = true + break + } + } + + if (!validSig) { + return Utils.error(res, `Invalid signature`) + } + const didDocuments = versionResponse.versions const didDb = Utils.getDidDocumentDb() const docs = [] diff --git a/test/did-storage.js b/test/did-storage.js index 5e9acdea..1bd22423 100644 --- a/test/did-storage.js +++ b/test/did-storage.js @@ -7,6 +7,7 @@ import dotenv from 'dotenv'; dotenv.config(); import Utils from '../src/services/didStorage/utils' +import TestUtils from './utils' import CONFIG from './config' const { SERVER_URL } = CONFIG @@ -34,7 +35,6 @@ describe("DID Storage Tests", function() { const couch = Utils.getDb() await couch.db.destroy(process.env.DB_DIDS) await Utils.createDb() - console.log('destroyed!') }) describe("Create", () => { @@ -173,9 +173,20 @@ describe("DID Storage Tests", function() { }) describe("Delete", () => { + let signature + this.beforeAll(() => { + const nowInMinutes = Math.round((new Date()).getTime() / 1000 / 60) + const proofString = `Delete DID Document ${DID.toLowerCase()} at ${nowInMinutes}` + signature = TestUtils.signString(proofString, wallet.privateKey) + }) + it("Fail - Invalid DID", async () => { try { - await Axios.delete(`${DID_URL}/abc123`); + await Axios.delete(`${DID_URL}/abc123`, { + headers: { + signature + } + }); assert.fail(`DID Document was found, when it shouldn't have`) } catch (err) { @@ -186,7 +197,9 @@ describe("DID Storage Tests", function() { it("Success", async () => { const deleteResult = await Axios.delete(`${DID_URL}/${DID}`, { - hello: 'world' + headers: { + signature + } }); assert.ok(deleteResult.data.status, 'success', 'Success response') @@ -195,9 +208,13 @@ describe("DID Storage Tests", function() { it("Fail - Deleted", async () => { try { - await Axios.delete(`${DID_URL}/${DID}`); + await Axios.delete(`${DID_URL}/${DID}`, { + headers: { + signature + } + }); - assert.fail(`DID Document was found, when it shouldn't have`) + assert.fail(`DID Document was deleted, when it shouldn't have`) } catch (err) { assert.equal(err.response.data.status, 'fail', 'Get DID Document failed') assert.ok(err.response.data.message.match('DID Document not found'), `Rejected because DID Document doesn't exists`) diff --git a/test/utils.js b/test/utils.js index 4616cc55..350b3a72 100644 --- a/test/utils.js +++ b/test/utils.js @@ -2,6 +2,7 @@ import Axios from 'axios' import PouchDb from 'pouchdb' import { AutoAccount } from "@verida/account-node" import { Network } from "@verida/client-ts" +import EncryptionUtils from '@verida/encryption-utils' import CONFIG from './config.js' @@ -66,6 +67,14 @@ class Utils { return response } + signString(str, privateKey) { + if (privateKey == 'string') { + privateKey = new Uint8Array(Buffer.from(privateKey.substr(2),'hex')) + } + + return EncryptionUtils.signData(str, privateKey) + } + } const utils = new Utils() From b7d4b165bec4279d75ed285d0d4b5d2942d69406 Mon Sep 17 00:00:00 2001 From: tahpot Date: Wed, 16 Nov 2022 21:02:19 +1030 Subject: [PATCH 12/13] Implement storage limits and status information for users and the system (#45) --- sample.env | 4 ++++ src/components/db.js | 8 +++++++ src/components/userManager.js | 40 ++++++++++++++++++++++++++++++++--- src/controllers/auth.js | 29 +++++++++++++++++++------ src/controllers/system.js | 28 ++++++++++++++++++++++++ src/controllers/user.js | 31 +++++++++++++++------------ src/routes/public.js | 2 ++ test/server.js | 10 +++++++++ 8 files changed, 129 insertions(+), 23 deletions(-) create mode 100644 src/controllers/system.js diff --git a/sample.env b/sample.env index c98e1ffb..78115778 100644 --- a/sample.env +++ b/sample.env @@ -20,6 +20,10 @@ DB_DB_INFO=verida_db_info GC_PERCENT=0.1 # Verida Private Key as hex string (used to sign responses). Including leading 0x. VDA_PRIVATE_KEY= +# Default maximum number of Megabytes for a storage context +DEFAULT_USER_CONTEXT_LIMIT_MB=10 +# Maximum number of users supported by this node +MAX_USERS=10000 // alpha numeric only DB_PUBLIC_USER=784c2n780c9cn0789 diff --git a/src/components/db.js b/src/components/db.js index 566ffdf6..60e821a0 100644 --- a/src/components/db.js +++ b/src/components/db.js @@ -29,6 +29,14 @@ class Db { return env.DB_PROTOCOL + "://" + env.DB_HOST + ":" + env.DB_PORT; } + // Total number of users in the system + async totalUsers() { + const couch = db.getCouch() + const usersDb = couch.db.use('_users') + const info = await usersDb.info() + return info.doc_count + } + } const db = new Db() diff --git a/src/components/userManager.js b/src/components/userManager.js index bc4abcca..9b56bebb 100644 --- a/src/components/userManager.js +++ b/src/components/userManager.js @@ -1,5 +1,8 @@ import crypto from 'crypto'; import Db from './db.js' +import Utils from './utils.js' +import DbManager from './dbManager.js'; + import dotenv from 'dotenv'; dotenv.config(); @@ -27,16 +30,26 @@ class UserManager { } async create(username, signature) { + const maxUsers = parseInt(process.env.MAX_USERS) + const currentUsers = await Db.totalUsers() + + if (currentUsers >= maxUsers) { + throw new Error('Maximum user limit reached') + } + const couch = Db.getCouch() const password = crypto.createHash('sha256').update(signature).digest("hex") + const storageLimit = process.env.DEFAULT_USER_CONTEXT_LIMIT_MB*1048576 + // Create CouchDB database user matching username and password let userData = { _id: `org.couchdb.user:${username}`, name: username, password: password, type: "user", - roles: [] + roles: [], + storageLimit }; let usersDb = couch.db.use('_users'); @@ -53,8 +66,6 @@ class UserManager { } } - - /** * Ensure we have a public user in the database for accessing public data */ @@ -102,6 +113,29 @@ class UserManager { } } + async getUsage(did, contextName) { + const username = Utils.generateUsername(did, contextName); + const user = await this.getByUsername(username); + const databases = await DbManager.getUserDatabases(did, contextName) + + const result = { + databases: 0, + bytes: 0, + storageLimit: user.storageLimit + } + + for (let d in databases) { + const database = databases[d] + const dbInfo = await DbManager.getUserDatabase(did, contextName, database.databaseName) + result.databases++ + result.bytes += dbInfo.info.sizes.file + } + + const usage = result.bytes / parseInt(result.storageLimit) + result.usagePercent = Number(usage.toFixed(4)) + return result + } + } let userManager = new UserManager(); diff --git a/src/controllers/auth.js b/src/controllers/auth.js index 50477f36..7c93503b 100644 --- a/src/controllers/auth.js +++ b/src/controllers/auth.js @@ -51,13 +51,20 @@ class AuthController { // Create the user if they don't exist if (!user) { - const response = await UserManager.create(username, signature); - if (!response || !response.id) { + try { + const response = await UserManager.create(username, signature); + if (!response || !response.id) { + return res.status(500).send({ + status: "fail", + data: { + "auth": "User does not exist and unable to create" + } + }) + } + } catch (err) { return res.status(500).send({ - status: "fail", - data: { - "auth": "User does not exist and unable to create" - } + status: 'fail', + message: err.message }) } } @@ -84,6 +91,15 @@ class AuthController { async connect(req, res) { const refreshToken = req.body.refreshToken; const contextName = req.body.contextName; + const did = req.body.did + + const userUsage = await UserManager.getUsage(did, contextName) + if (userUsage.usagePercent >= 100) { + return res.status(400).send({ + status: "fail", + message: 'Storage limit reached' + }); + } const accessToken = await AuthManager.generateAccessToken(refreshToken, contextName); @@ -92,7 +108,6 @@ class AuthController { status: "success", accessToken, host: Db.buildHost() // required to know the CouchDB host - // username: removed, don't think it is needed }); } else { diff --git a/src/controllers/system.js b/src/controllers/system.js new file mode 100644 index 00000000..bb33542c --- /dev/null +++ b/src/controllers/system.js @@ -0,0 +1,28 @@ +import db from '../components/db' +import Utils from '../components/utils' +import packageJson from '../../package.json' + +import dotenv from 'dotenv'; +dotenv.config(); + +class SystemController { + + async status(req, res) { + const currentUsers = await db.totalUsers() + + const results = { + maxUsers: parseInt(process.env.MAX_USERS), + currentUsers, + version: packageJson.version + } + + return Utils.signedResponse({ + status: "success", + results + }, res); + } + +} + +const systemController = new SystemController(); +export default systemController; \ No newline at end of file diff --git a/src/controllers/user.js b/src/controllers/user.js index f3c334a4..71ffd82a 100644 --- a/src/controllers/user.js +++ b/src/controllers/user.js @@ -1,4 +1,5 @@ import DbManager from '../components/dbManager.js'; +import UserManager from '../components/userManager.js'; import Utils from '../components/utils.js'; import Db from '../components/db.js' @@ -37,6 +38,14 @@ class UserController { }); } + const userUsage = await UserManager.getUsage(did, contextName) + if (userUsage.usagePercent >= 100) { + return res.status(400).send({ + status: "fail", + message: 'Storage limit reached' + }); + } + const databaseHash = Utils.generateDatabaseName(did, contextName, databaseName) let success; @@ -148,6 +157,14 @@ class UserController { const databaseHash = Utils.generateDatabaseName(did, contextName, databaseName) try { + const userUsage = await UserManager.getUsage(did, contextName) + if (userUsage.usagePercent >= 100) { + return res.status(400).send({ + status: "fail", + message: 'Storage limit reached' + }); + } + let success = await DbManager.updateDatabase(username, databaseHash, contextName, options); if (success) { await DbManager.saveUserDatabase(did, contextName, databaseName, databaseHash, options.permissions) @@ -245,19 +262,7 @@ class UserController { } try { - const databases = await DbManager.getUserDatabases(did, contextName) - - const result = { - databases: 0, - bytes: 0 - } - - for (let d in databases) { - const database = databases[d] - const dbInfo = await DbManager.getUserDatabase(did, contextName, database.databaseName) - result.databases++ - result.bytes += dbInfo.info.sizes.file - } + const result = await UserManager.getUsage(did, contextName) return Utils.signedResponse({ status: "success", diff --git a/src/routes/public.js b/src/routes/public.js index 2095f811..18d372d9 100644 --- a/src/routes/public.js +++ b/src/routes/public.js @@ -1,11 +1,13 @@ import express from 'express'; import UserController from '../controllers/user.js'; import AuthController from '../controllers/auth.js'; +import SystemController from '../controllers/system.js'; const router = express.Router(); // Specify public endpoints router.get('/auth/public', UserController.getPublic); +router.get('/status', SystemController.status); router.post('/auth/generateAuthJwt', AuthController.generateAuthJwt); router.post('/auth/authenticate', AuthController.authenticate); router.post('/auth/connect', AuthController.connect); diff --git a/test/server.js b/test/server.js index e5404b77..4aadd7e1 100644 --- a/test/server.js +++ b/test/server.js @@ -257,6 +257,8 @@ describe("Server tests", function() { const result = response.data.result assert.equal(result.databases, 2, 'Expected number of databases') assert.ok(result.bytes > 0, 'More than 0 bytes used') + assert.ok(result.usagePercent > 0, 'More than 0 percentage usage') + assert.equal(result.storageLimit, process.env.DEFAULT_USER_CONTEXT_LIMIT_MB*1048576, 'Storage limit is 100Mb') }) // @todo: updates @@ -312,7 +314,15 @@ describe("Server tests", function() { assert.ok(response.data.results.indexOf('DeleteAll_3') >= 0, 'Deleted correct databases (DeleteAll_3)') assert.ok(TestUtils.verifySignature(response), 'Have a valid signature in response') }) + }) + describe("Server info", () => { + it("Status", async () => { + const response = await Axios.get(`${SERVER_URL}/status`); + assert.equal(response.data.results.maxUsers, process.env.MAX_USERS, 'Correct maximum number of users') + assert.ok(response.data.results.currentUsers > 2, 'At least two users') + assert.ok(response.data.results.version && response.data.results.version.length, 'Version specified') + }) }) }) \ No newline at end of file From 83eec755c23dc0642b03e2b0ad8c0720657e8530 Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 23 Nov 2022 09:49:54 +1030 Subject: [PATCH 13/13] Remove redundant hash key --- sample.env | 1 - 1 file changed, 1 deletion(-) diff --git a/sample.env b/sample.env index 78115778..0462b00a 100644 --- a/sample.env +++ b/sample.env @@ -1,4 +1,3 @@ -HASH_KEY=this_is_not_the_prod_hash_key DID_RPC_URL= DID_NETWORK=testnet DID_CACHE_DURATION=3600