From 738a8a7801f5b35945a3e57375bca442b022653b Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Fri, 9 Dec 2022 09:47:19 +1030
Subject: [PATCH 01/28] Rename applicationName to contextName. Add replicator
 permissions.

---
 src/components/dbManager.js | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/components/dbManager.js b/src/components/dbManager.js
index b7fab40d..90fab256 100644
--- a/src/components/dbManager.js
+++ b/src/components/dbManager.js
@@ -112,7 +112,7 @@ class DbManager {
         }
     }
 
-    async createDatabase(username, databaseName, applicationName, options) {
+    async createDatabase(username, databaseName, contextName, options) {
         let couch = Db.getCouch();
 
         // Create database
@@ -130,7 +130,7 @@ class DbManager {
         let db = couch.db.use(databaseName);
 
         try {
-            await this.configurePermissions(db, username, applicationName, options.permissions);
+            await this.configurePermissions(db, username, contextName, options.permissions);
         } catch (err) {
             //console.log("configure error");
             //console.log(err);
@@ -139,7 +139,7 @@ class DbManager {
         return true;
     }
 
-    async updateDatabase(username, databaseName, applicationName, options) {
+    async updateDatabase(username, databaseName, contextName, options) {
         const couch = Db.getCouch();
         const db = couch.db.use(databaseName);
 
@@ -152,7 +152,7 @@ class DbManager {
         }
 
         try {
-            await this.configurePermissions(db, username, applicationName, options.permissions);
+            await this.configurePermissions(db, username, contextName, options.permissions);
         } catch (err) {
             //console.log("update database error");
             //console.log(err);
@@ -183,7 +183,7 @@ class DbManager {
         }
     }
 
-    async configurePermissions(db, username, applicationName, permissions) {
+    async configurePermissions(db, username, contextName, permissions) {
         permissions = permissions ? permissions : {};
 
         let owner = username;
@@ -197,8 +197,8 @@ class DbManager {
 
         switch (permissions.write) {
             case "users":
-                writeUsers = _.union(writeUsers, Utils.didsToUsernames(permissions.writeList, applicationName));
-                deleteUsers = _.union(deleteUsers, Utils.didsToUsernames(permissions.deleteList, applicationName));
+                writeUsers = _.union(writeUsers, Utils.didsToUsernames(permissions.writeList, contextName));
+                deleteUsers = _.union(deleteUsers, Utils.didsToUsernames(permissions.deleteList, contextName));
                 break;
             case "public":
                 writeUsers = writeUsers.concat([process.env.DB_PUBLIC_USER]);
@@ -207,7 +207,7 @@ class DbManager {
 
         switch (permissions.read) {
             case "users":
-                readUsers = _.union(readUsers, Utils.didsToUsernames(permissions.readList, applicationName));
+                readUsers = _.union(readUsers, Utils.didsToUsernames(permissions.readList, contextName));
                 break;
             case "public":
                 readUsers = readUsers.concat([process.env.DB_PUBLIC_USER]);
@@ -215,6 +215,8 @@ class DbManager {
         }
 
         const dbMembers = _.union(readUsers, writeUsers);
+        const contextHash = Utils.generateHash(contextName)
+        const replicaterRole = `${contextHash}-replicater`
 
         let securityDoc = {
             admins: {
@@ -224,7 +226,7 @@ class DbManager {
             members: {
                 // this grants read access to all members
                 names: dbMembers,
-                roles: []
+                roles: [replicaterRole]
             }
         };
 
@@ -240,7 +242,7 @@ class DbManager {
         let deleteUsersJson = JSON.stringify(deleteUsers);
 
         try {
-            const writeFunction = "\n    function(newDoc, oldDoc, userCtx, secObj) {\n        if (" + writeUsersJson + ".indexOf(userCtx.name) == -1) throw({ unauthorized: 'User is not permitted to write to database' });\n}";
+            const writeFunction = `\n    function(newDoc, oldDoc, userCtx, secObj) {\n        if (${writeUsersJson}.indexOf(userCtx.name) == -1 && userCtx.roles.indexOf('${replicaterRole}') == -1) throw({ unauthorized: 'User is not permitted to write to database' });\n}`;
             const writeDoc = {
                 "validate_doc_update": writeFunction
             };
@@ -256,7 +258,7 @@ class DbManager {
         if (permissions.write === "public") {
             // If the public has write permissions, disable public from deleting records
             try {
-                const deleteFunction = "\n    function(newDoc, oldDoc, userCtx, secObj) {\n        if ("+deleteUsersJson+".indexOf(userCtx.name) == -1 && newDoc._deleted) throw({ unauthorized: 'User is not permitted to delete from database' });\n}";
+                const deleteFunction = `\n    function(newDoc, oldDoc, userCtx, secObj) {\n        if (${deleteUsersJson}.indexOf(userCtx.name) == -1 && userCtx.roles.indexOf('${replicaterRole}') == -1 && newDoc._deleted) throw({ unauthorized: 'User is not permitted to delete from database' });\n}`;
                 const deleteDoc = {
                     "validate_doc_update": deleteFunction
                 };

From a43a44cbb3e590ad0afa4518c935e7e0609f0c81 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Fri, 9 Dec 2022 09:47:36 +1030
Subject: [PATCH 02/28] Add generateHash utility method

---
 src/components/utils.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/components/utils.js b/src/components/utils.js
index bda772af..95bd2f40 100644
--- a/src/components/utils.js
+++ b/src/components/utils.js
@@ -2,6 +2,10 @@ import EncryptionUtils from "@verida/encryption-utils"
 
 class Utils {
 
+    generateHash(value) {
+        return EncryptionUtils.hash(value).substring(2);
+    }
+
     generateUsername(did, contextName) {
         did = did.toLowerCase()
         const text = [

From 237b6c8ae80687073375ebae8288a0051355f727 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Fri, 9 Dec 2022 11:11:49 +1030
Subject: [PATCH 03/28] Refactor so context databases are in their own database
 that will be replicated, instead of all stored in a single database. Other
 minor fixes and test improvements.

---
 sample.env                    |   1 -
 src/components/authManager.js |   4 +-
 src/components/dbManager.js   | 121 +++++++++++++++++++++-------------
 src/components/userManager.js |  17 -----
 src/components/utils.js       |   9 +++
 src/controllers/auth.js       |   4 ++
 src/controllers/user.js       |  12 ++--
 src/routes/private.js         |   1 +
 src/routes/public.js          |   1 +
 test/server.js                |  20 +++++-
 10 files changed, 118 insertions(+), 72 deletions(-)

diff --git a/sample.env b/sample.env
index 410c43f5..048a569a 100644
--- a/sample.env
+++ b/sample.env
@@ -22,7 +22,6 @@ REFRESH_JWT_SIGN_PK=insert-random-refresh-symmetric-key
 # 30 Days
 REFRESH_TOKEN_EXPIRY=2592000
 DB_REFRESH_TOKENS=verida_refresh_tokens
-DB_DB_INFO=verida_db_info
 # How often garbage collection runs (1=100%, 0.5 = 50%)
 GC_PERCENT=0.1
 # Verida Private Key as hex string (used to sign responses). Including leading 0x.
diff --git a/src/components/authManager.js b/src/components/authManager.js
index e69868e7..78079655 100644
--- a/src/components/authManager.js
+++ b/src/components/authManager.js
@@ -109,7 +109,7 @@ class AuthManager {
             const result = didDocument.verifySig(consentMessage, signature)
 
             if (!result) {
-                console.warning('Invalid signature when verifying signed consent message')
+                console.info('Invalid signature when verifying signed consent message')
                 // Invalid signature
                 return false
             }
@@ -118,7 +118,7 @@ class AuthManager {
         } catch (err) {
             // @todo: Log error
             // Likely unable to resolve DID or invalid signature
-            console.warning(`Unable to resolve DID or invalid signature: ${err.message}`)
+            console.info(`Unable to resolve DID or invalid signature: ${err.message}`)
             return false
         }
     }
diff --git a/src/components/dbManager.js b/src/components/dbManager.js
index 90fab256..22cac04a 100644
--- a/src/components/dbManager.js
+++ b/src/components/dbManager.js
@@ -11,13 +11,47 @@ class DbManager {
         this.error = null;
     }
 
-    async saveUserDatabase(did, contextName, databaseName, databaseHash, permissions) {
+    async saveUserDatabase(did, owner, contextName, databaseName, databaseHash, permissions) {
         const couch = Db.getCouch()
-        const userDatabaseName = process.env.DB_DB_INFO
-        const db = couch.db.use(userDatabaseName)
+        const didContextHash = Utils.generateDidContextHash(did, contextName)
+        const didContextDbName = `c${didContextHash}`
 
-        const text = `${did.toLowerCase()}/${contextName}/${databaseName}`
-        const id = EncryptionUtils.hash(text).substring(2)
+        // Create database for storing all the databases for this user context
+        let db
+        try {
+            await couch.db.create(didContextDbName);
+            db = couch.db.use(didContextDbName);
+
+            const replicaterRole = `r${didContextHash}-replicater`
+
+            let securityDoc = {
+                admins: {
+                    names: [owner],
+                    roles: []
+                },
+                members: {
+                    names: [owner],
+                    roles: [replicaterRole]
+                }
+            };
+
+            // Insert security document to ensure owner is the admin and any other read / write users can access the database
+            try {
+                await this._insertOrUpdate(db, securityDoc, '_security');
+            } catch (err) {
+                return false;
+            }
+        } catch (err) {
+            // The didContext database may already exist, or may have been deleted so a file
+            // already exists.
+            // In that case, ignore the error and continue
+            if (err.error != "file_exists") {
+                throw err;
+            }
+        }
+        
+        db = couch.db.use(didContextDbName);
+        const id = Utils.generateDatabaseName(did, contextName, databaseName)
 
         try {
             const result = await this._insertOrUpdate(db, {
@@ -26,7 +60,7 @@ class DbManager {
                 contextName,
                 databaseName,
                 databaseHash,
-                permissions
+                permissions: permissions ? permissions : {}
             }, id)
         } catch (err) {
             throw err
@@ -35,41 +69,36 @@ class DbManager {
 
     async getUserDatabases(did, contextName) {
         const couch = Db.getCouch()
-        const userDatabaseName = process.env.DB_DB_INFO
-        const db = couch.db.use(userDatabaseName)
+        const didContextHash = Utils.generateDidContextHash(did, contextName)
+        const didContextDbName = `c${didContextHash}`
 
         try {
-            const query = {
-                selector: {
-                    did,
-                    contextName
-                },
-                limit: 1000
-            }
+            const db = couch.db.use(didContextDbName)
+            const result = await db.list({include_docs: true, limit: 1000})
+            const finalResult = result.rows.map((item) => {
+                delete item.doc['_id']
+                delete item.doc['_rev']
 
-            const results = await db.find(query)
-            const finalResult = results.docs.map((item) => {
-                delete item['_id']
-                delete item['_rev']
-
-                return item
+                return item.doc
             })
 
             return finalResult
         } catch (err) {
-            if (err.reason != "missing") {
+            if (err.reason != 'missing' && err.error != 'not_found') {
                 throw err;
             }
+
+            return []
         }
     }
 
     async getUserDatabase(did, contextName, databaseName) {
         const couch = Db.getCouch()
-        const userDatabaseName = process.env.DB_DB_INFO
-        const db = couch.db.use(userDatabaseName)
+        const didContextHash = Utils.generateDidContextHash(did, contextName)
+        const didContextDbName = `c${didContextHash}`
+        const db = couch.db.use(didContextDbName)
 
-        const text = `${did.toLowerCase()}/${contextName}/${databaseName}`
-        const id = EncryptionUtils.hash(text).substring(2)
+        const id = Utils.generateDatabaseName(did, contextName, databaseName)
 
         try {
             const doc = await db.get(id)
@@ -86,7 +115,7 @@ class DbManager {
 
             return result
         } catch (err) {
-            if (err.reason == "missing") {
+            if (err.reason == 'missing' || err.reason == 'deleted') {
                 return false
             }
 
@@ -94,13 +123,13 @@ class DbManager {
         }
     }
 
-    async deleteUserDatabase(did, contextName, databaseName, databaseHash) {
+    async deleteUserDatabase(did, contextName, databaseName) {
         const couch = Db.getCouch()
-        const userDatabaseName = process.env.DB_DB_INFO
-        const db = couch.db.use(userDatabaseName)
+        const didContextHash = Utils.generateDidContextHash(did, contextName)
+        const didContextDbName = `c${didContextHash}`
+        const db = couch.db.use(didContextDbName)
 
-        const text = `${did.toLowerCase()}/${contextName}/${databaseName}`
-        const id = EncryptionUtils.hash(text).substring(2)
+        const id = Utils.generateDatabaseName(did, contextName, databaseName)
 
         try {
             await this._insertOrUpdate(db, {
@@ -112,12 +141,12 @@ class DbManager {
         }
     }
 
-    async createDatabase(username, databaseName, contextName, options) {
+    async createDatabase(did, username, databaseHash, contextName, options) {
         let couch = Db.getCouch();
 
         // Create database
         try {
-            await couch.db.create(databaseName);
+            await couch.db.create(databaseHash);
         } catch (err) {
             // The database may already exist, or may have been deleted so a file
             // already exists.
@@ -127,10 +156,10 @@ class DbManager {
             }
         }
 
-        let db = couch.db.use(databaseName);
+        let db = couch.db.use(databaseHash);
 
         try {
-            await this.configurePermissions(db, username, contextName, options.permissions);
+            await this.configurePermissions(did, db, username, contextName, options.permissions);
         } catch (err) {
             //console.log("configure error");
             //console.log(err);
@@ -139,12 +168,12 @@ class DbManager {
         return true;
     }
 
-    async updateDatabase(username, databaseName, contextName, options) {
+    async updateDatabase(did, username, databaseHash, contextName, options) {
         const couch = Db.getCouch();
-        const db = couch.db.use(databaseName);
+        const db = couch.db.use(databaseHash);
 
         // Do a sanity check to confirm the username is an admin of the database
-        const perms = await couch.request({db: databaseName, method: 'get', path: '/_security'})
+        const perms = await couch.request({db: databaseHash, method: 'get', path: '/_security'})
         const usernameIsAdmin = perms.admins.names.includes(username)
 
         if (!usernameIsAdmin) {
@@ -152,7 +181,7 @@ class DbManager {
         }
 
         try {
-            await this.configurePermissions(db, username, contextName, options.permissions);
+            await this.configurePermissions(did, db, username, contextName, options.permissions);
         } catch (err) {
             //console.log("update database error");
             //console.log(err);
@@ -161,11 +190,11 @@ class DbManager {
         return true;
     }
 
-    async deleteDatabase(databaseName, username) {
+    async deleteDatabase(databaseHash, username) {
         const couch = Db.getCouch();
 
         // Do a sanity check to confirm the username is an admin of the database
-        const perms = await couch.request({db: databaseName, method: 'get', path: '/_security'})
+        const perms = await couch.request({db: databaseHash, method: 'get', path: '/_security'})
         const usernameIsAdmin = perms.admins.names.includes(username)
 
         if (!usernameIsAdmin) {
@@ -174,7 +203,7 @@ class DbManager {
 
         // Create database
         try {
-            return await couch.db.destroy(databaseName);
+            return await couch.db.destroy(databaseHash);
         } catch (err) {
             // The database may already exist, or may have been deleted so a file
             // already exists.
@@ -183,7 +212,7 @@ class DbManager {
         }
     }
 
-    async configurePermissions(db, username, contextName, permissions) {
+    async configurePermissions(did, db, username, contextName, permissions) {
         permissions = permissions ? permissions : {};
 
         let owner = username;
@@ -215,8 +244,8 @@ class DbManager {
         }
 
         const dbMembers = _.union(readUsers, writeUsers);
-        const contextHash = Utils.generateHash(contextName)
-        const replicaterRole = `${contextHash}-replicater`
+        const didContextHash = Utils.generateDidContextHash(did, contextName)
+        const replicaterRole = `r${didContextHash}-replicater`
 
         let securityDoc = {
             admins: {
diff --git a/src/components/userManager.js b/src/components/userManager.js
index 1e5ce152..ea57186d 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -94,23 +94,6 @@ class UserManager {
                 throw err;
             }
         }
-
-        try {
-            await couch.db.create(process.env.DB_DB_INFO)
-            const dbInfo = couch.db.use(process.env.DB_DB_INFO)
-            await dbInfo.createIndex({
-                index: {
-                    fields: ['did', 'contextName']
-                },
-                name: 'didContext'
-            })
-        } catch (err) {
-            if (err.message == "The database could not be created, the file already exists.") {
-                console.log("Info database not created -- already existed");
-            } else {
-                throw err;
-            }
-        }
     }
 
     async getUsage(did, contextName) {
diff --git a/src/components/utils.js b/src/components/utils.js
index 95bd2f40..bc3e7a3f 100644
--- a/src/components/utils.js
+++ b/src/components/utils.js
@@ -6,6 +6,15 @@ class Utils {
         return EncryptionUtils.hash(value).substring(2);
     }
 
+    generateDidContextHash(did, contextName) {
+        let text = [
+            did.toLowerCase(),
+            contextName
+        ].join("/");
+
+        return this.generateHash(text)
+    }
+
     generateUsername(did, contextName) {
         did = did.toLowerCase()
         const text = [
diff --git a/src/controllers/auth.js b/src/controllers/auth.js
index 7c93503b..b02589ec 100644
--- a/src/controllers/auth.js
+++ b/src/controllers/auth.js
@@ -200,6 +200,10 @@ class AuthController {
         }
     }
 
+    async replicationCreds(req, res) {
+
+    }
+
 }
 
 const authController = new AuthController();
diff --git a/src/controllers/user.js b/src/controllers/user.js
index dd7e7597..559037f4 100644
--- a/src/controllers/user.js
+++ b/src/controllers/user.js
@@ -16,7 +16,6 @@ class UserController {
                 }
             }, res);
         } catch (err) {
-            console.error(err);
             return res.status(500).send({
                 status: "fail",
                 message: err.message
@@ -58,9 +57,9 @@ class UserController {
 
         let success;
         try {
-            success = await DbManager.createDatabase(username, databaseHash, contextName, options);
+            success = await DbManager.createDatabase(did, username, databaseHash, contextName, options);
             if (success) {
-                await DbManager.saveUserDatabase(did, contextName, databaseName, databaseHash, options.permissions)
+                await DbManager.saveUserDatabase(did, username, contextName, databaseName, databaseHash, options.permissions)
 
                 return Utils.signedResponse({
                     status: "success"
@@ -173,9 +172,9 @@ class UserController {
                 });
             }
 
-            let success = await DbManager.updateDatabase(username, databaseHash, contextName, options);
+            let success = await DbManager.updateDatabase(did, username, databaseHash, contextName, options);
             if (success) {
-                await DbManager.saveUserDatabase(did, contextName, databaseName, databaseHash, options.permissions)
+                await DbManager.saveUserDatabase(did, username, contextName, databaseName, databaseHash, options.permissions)
 
                 return Utils.signedResponse({
                     status: "success"
@@ -284,6 +283,9 @@ class UserController {
         }
     }
 
+    async checkReplication(req, res) {
+    }
+
 }
 
 const userController = new UserController();
diff --git a/src/routes/private.js b/src/routes/private.js
index fdc3090e..cb196fda 100644
--- a/src/routes/private.js
+++ b/src/routes/private.js
@@ -10,5 +10,6 @@ router.post('/user/deleteDatabases', UserController.deleteDatabases);
 router.post('/user/databases', UserController.databases);
 router.post('/user/databaseInfo', UserController.databaseInfo);
 router.post('/user/usage', UserController.usage);
+router.post('/user/checkReplication', UserController.checkReplication);
 
 export default router;
\ No newline at end of file
diff --git a/src/routes/public.js b/src/routes/public.js
index c78e12f4..cdcf4c18 100644
--- a/src/routes/public.js
+++ b/src/routes/public.js
@@ -10,6 +10,7 @@ const router = express.Router();
 router.get('/auth/public', UserController.getPublic);
 router.get('/status', SystemController.status);
 
+router.get('/auth/replicationCreds', AuthController.replicationCreds);
 router.post('/auth/generateAuthJwt', AuthController.generateAuthJwt);
 router.post('/auth/authenticate', AuthController.authenticate);
 router.post('/auth/connect', AuthController.connect);
diff --git a/test/server.js b/test/server.js
index 16d61e12..30127d89 100644
--- a/test/server.js
+++ b/test/server.js
@@ -279,7 +279,24 @@ describe("Server tests", function() {
 
             assert.equal(response.data.status, "success", "Successful delete response")
 
-            const response2 = await Axios.post(`${SERVER_URL}/user/deleteDatabase`, {
+            // confirm the database doesn't exist
+            try {
+                const response2 = await Axios.post(`${SERVER_URL}/user/databaseInfo`, {
+                    databaseName,
+                    did: accountInfo.did,
+                    contextName: CONTEXT_NAME
+                }, {
+                    headers: {
+                        Authorization: `Bearer ${accessToken}`
+                    }
+                });
+
+                assert.fail('Expected a 404 because the database shouldnt be found')
+            } catch (err) {
+                assert.equal(err.response.data.message, 'Database not found', 'Database not found')
+            }
+
+            const response3 = await Axios.post(`${SERVER_URL}/user/deleteDatabase`, {
                 databaseName: databaseName2,
                 did: accountInfo.did,
                 contextName: CONTEXT_NAME
@@ -288,6 +305,7 @@ describe("Server tests", function() {
                     Authorization: `Bearer ${accessToken}`
                 }
             });
+            assert.equal(response3.data.status, 'success', 'Successful delete response')
 
             assert.ok(TestUtils.verifySignature(response), 'Have a valid signature in response')
         })

From a6834a67c7b788a49e5fdc54ba154956afc51b70 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Fri, 9 Dec 2022 12:55:55 +1030
Subject: [PATCH 04/28] Implement untested auth/replicationCreds

---
 package.json                  |  2 +-
 src/components/authManager.js | 64 +++++++++++++++++++++++++++-----
 src/components/utils.js       | 18 +++++++++
 src/controllers/auth.js       | 69 +++++++++++++++++++++++++++++++++++
 yarn.lock                     |  9 +++++
 5 files changed, 151 insertions(+), 11 deletions(-)

diff --git a/package.json b/package.json
index 1d2c4fcf..39ad5959 100644
--- a/package.json
+++ b/package.json
@@ -44,6 +44,7 @@
     "@verida/did-document": "^2.0.0-rc4",
     "@verida/encryption-utils": "^2.0.0-rc1",
     "aws-serverless-express": "^3.4.0",
+    "axios": "^1.2.1",
     "cors": "^2.8.5",
     "did-resolver": "^3.1.0",
     "dotenv": "^8.2.0",
@@ -65,7 +66,6 @@
     "@babel/preset-env": "^7.20.2",
     "@verida/account-node": "^2.0.0-rc4",
     "@verida/client-ts": "^2.0.0-rc4",
-    "axios": "^0.27.2",
     "claudia": "^5.14.1",
     "ethers": "^5.7.2",
     "mocha": "^7.0.0",
diff --git a/src/components/authManager.js b/src/components/authManager.js
index 78079655..0875b4f8 100644
--- a/src/components/authManager.js
+++ b/src/components/authManager.js
@@ -7,6 +7,7 @@ import { DIDClient } from '@verida/did-client'
 import EncryptionUtils from '@verida/encryption-utils';
 import Utils from './utils.js';
 import Db from './db.js';
+import dbManager from './dbManager.js';
 
 dotenv.config();
 
@@ -80,6 +81,27 @@ class AuthManager {
     }
 
     async verifySignedConsentMessage(did, contextName, signature, consentMessage) {
+        // Verify the signature signed the correct string
+        try {
+            const didDocument = await this.getDidDocument(did)
+            const result = didDocument.verifySig(consentMessage, signature)
+
+            if (!result) {
+                console.info('Invalid signature when verifying signed consent message')
+                // Invalid signature
+                return false
+            }
+
+            return true
+        } catch (err) {
+            // @todo: Log error
+            // Likely unable to resolve DID or invalid signature
+            console.info(`Unable to resolve DID or invalid signature: ${err.message}`)
+            return false
+        }
+    }
+
+    async getDidDocument(did) {
         // Verify the signature signed the correct string
         const cacheKey = did
 
@@ -106,19 +128,11 @@ class AuthManager {
                 }
             }
 
-            const result = didDocument.verifySig(consentMessage, signature)
-
-            if (!result) {
-                console.info('Invalid signature when verifying signed consent message')
-                // Invalid signature
-                return false
-            }
-
-            return true
+            return didDocument
         } catch (err) {
             // @todo: Log error
             // Likely unable to resolve DID or invalid signature
-            console.info(`Unable to resolve DID or invalid signature: ${err.message}`)
+            console.info(`Unable to resolve DID`)
             return false
         }
     }
@@ -397,6 +411,36 @@ class AuthManager {
         await tokenDb.createIndex(expiryIndex);
     }
 
+    async ensureReplicationCredentials(endpointUri, password) {
+        const username = Utils.generateReplicaterUsername(endpointUri)
+
+        const couch = Db.getCouch('internal');
+        const usersDb = await couch.db.users('_users')
+        let user
+        try {
+            const id = `org.couchdb.user:${username}`
+            user = await usersDb.get(id)
+        } catch (err) {
+            console.log(err)
+            throw err
+        }
+
+        if (!user && !password) {
+            throw new Error(`Unable to create user: User doesn't exist and no password specified`)
+        }
+
+        if (user && password) {
+            // Update the password
+            user.password = password
+            try {
+                await dbManager._insertOrUpdate(usersDb, user, user.id)
+            } catch (err) {
+                console.log(err)
+                throw new Error(`Unable to update password: ${err.message}`)
+            }
+        }
+    }
+
     // @todo: garbage collection
     async gc() {
         const GC_PERCENT = process.env.GC_PERCENT
diff --git a/src/components/utils.js b/src/components/utils.js
index bc3e7a3f..bac37cf2 100644
--- a/src/components/utils.js
+++ b/src/components/utils.js
@@ -6,6 +6,10 @@ class Utils {
         return EncryptionUtils.hash(value).substring(2);
     }
 
+    generateReplicaterUsername(endpointUri) {
+        return `r${this.generateHash(endpointUri)}`
+    }
+
     generateDidContextHash(did, contextName) {
         let text = [
             did.toLowerCase(),
@@ -58,6 +62,20 @@ class Utils {
         });
     }
 
+    async error(res, message, httpStatus=400) {
+        return res.status(httpStatus).send({
+            status: "fail",
+            message
+        })
+    }
+
+    async success(res, data) {
+        return res.status(200).send({
+            status: "success",
+            data
+        })
+    }
+
 }
 
 let utils = new Utils();
diff --git a/src/controllers/auth.js b/src/controllers/auth.js
index b02589ec..d18b1908 100644
--- a/src/controllers/auth.js
+++ b/src/controllers/auth.js
@@ -2,6 +2,8 @@ import UserManager from '../components/userManager.js';
 import Utils from '../components/utils.js';
 import AuthManager from '../components/authManager.js';
 import Db from '../components/db.js';
+import Axios from 'axios'
+import EncryptionUtils from '@verida/encryption-utils';
 
 class AuthController {
 
@@ -200,8 +202,75 @@ class AuthController {
         }
     }
 
+    // If password is not an empty string, it will update the password to match
+    // If no user exists, must specify a password
     async replicationCreds(req, res) {
+        const {
+            endpointUri,
+            //did,
+            //contextName,
+            timestampMinutes,
+            password,
+            signature
+        } = req.body
+        
+        // Verify params
+        if (!endpointUri) {
+            return Utils.error(res, 'Endpoint not specified')
+        }
+
+        if (!timestampMinutes) {
+            return Utils.error(res, 'Timestamp not specified')
+        }
+
+        if (!signature) {
+            return Utils.error(res, 'Signature not specified')
+        }
+
+        // Lookup DID document and confirm endpointUri is a valid endpoint
+        const didDocument = await AuthManager.getDidDocument(did)
+        const endpoints = didDocument.locateServiceEndpoint(contextName, 'database')
+        console.log(endpoints)
+        if (endpoints.indexOf(endpointUri) === -1) {
+            return Utils.error(res, 'Invalid endpoint 1')
+        }
+        
+        // Pull endpoint public key from /status and verify the signature
+        let endpointPublicKey
+        try {
+            const result = await Axios.get(`${endpointUri}/status`)
+            console.log(result.data)
+            endpointPublicKey = result.data.publicKey
+            const params = {
+                endpointUri,
+                timestampMinutes,
+                password
+            }
+
+            if (!EncryptionUtils.verifySig(params, signature, endpointPublicKey)) {
+                return Utils.error(res, 'Invalid signature')
+            }
+        } catch (err) {
+            console.log(err)
+            return Utils.error(res, 'Invalid endpoint 2')
+        }
+
+        let decryptedPassword
+        if (password !== '') {
+            try {
+                const sharedKey = EncryptionUtils.sharedKey(endpointPublicKey, process.env.VDA_PRIVATE_KEY)
+                decryptedPassword = EncryptionUtils.asymDecrypt(password, sharedKey)
+            } catch (err) {
+                console.log(err)
+                return Utils.error(res, 'Invalid password encryption')
+            }
+        }
 
+        try {
+            await AuthManager.ensureReplicationCredentials(endpointUri, decryptedPassword)
+        } catch (err) {
+            return Utils.error(res, err.message)
+        }
     }
 
 }
diff --git a/yarn.lock b/yarn.lock
index a29153b2..c705ebc5 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2116,6 +2116,15 @@ axios@^0.27.2:
     follow-redirects "^1.14.9"
     form-data "^4.0.0"
 
+axios@^1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-1.2.1.tgz#44cf04a3c9f0c2252ebd85975361c026cb9f864a"
+  integrity sha512-I88cFiGu9ryt/tfVEi4kX2SITsvDddTajXTOFmt2uK1ZVA8LytjtdeyefdQWEf5PU8w+4SSJDoYnggflB5tW4A==
+  dependencies:
+    follow-redirects "^1.15.0"
+    form-data "^4.0.0"
+    proxy-from-env "^1.1.0"
+
 babel-plugin-polyfill-corejs2@^0.3.2:
   version "0.3.2"
   resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.3.2.tgz#e4c31d4c89b56f3cf85b92558954c66b54bd972d"

From 81e27b9ad1299561cbf1da81c592ef2c81296868 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Sun, 11 Dec 2022 08:35:48 +1030
Subject: [PATCH 05/28] Implementation complete, untested.

---
 sample.env                    |  1 +
 src/components/authManager.js | 62 ++++++++++++++++++++++++++++++
 src/components/userManager.js | 71 +++++++++++++++++++++++++++++++++++
 src/components/utils.js       | 13 +++++++
 src/controllers/user.js       | 17 +++++++++
 5 files changed, 164 insertions(+)

diff --git a/sample.env b/sample.env
index 048a569a..2dc6e094 100644
--- a/sample.env
+++ b/sample.env
@@ -35,5 +35,6 @@ MAX_USERS=10000
 DB_PUBLIC_USER=784c2n780c9cn0789
 DB_PUBLIC_PASS=784c2n780c9cn0789
 DB_DIDS=verida_dids
+DB_REPLICATER_CREDS=verida_replicater_creds
 
 PORT=5151
diff --git a/src/components/authManager.js b/src/components/authManager.js
index 0875b4f8..d09eb7ba 100644
--- a/src/components/authManager.js
+++ b/src/components/authManager.js
@@ -8,6 +8,7 @@ import EncryptionUtils from '@verida/encryption-utils';
 import Utils from './utils.js';
 import Db from './db.js';
 import dbManager from './dbManager.js';
+import { ethers } from 'ethers'
 
 dotenv.config();
 
@@ -412,7 +413,9 @@ class AuthManager {
     }
 
     async ensureReplicationCredentials(endpointUri, password) {
+        console.log(`ensureReplicationCredentials(${endpointUri}, ${password})`)
         const username = Utils.generateReplicaterUsername(endpointUri)
+        console.log(`- username: ${username}`)
 
         const couch = Db.getCouch('internal');
         const usersDb = await couch.db.users('_users')
@@ -432,6 +435,7 @@ class AuthManager {
         if (user && password) {
             // Update the password
             user.password = password
+            console.log(`- user: ${user}`)
             try {
                 await dbManager._insertOrUpdate(usersDb, user, user.id)
             } catch (err) {
@@ -441,6 +445,64 @@ class AuthManager {
         }
     }
 
+    async fetchReplicaterCredentials(endpointUri, did, contextName) {
+        console.log(`fetchReplicaterCredentials(${endpointUri}, ${did}, ${contextName})`)
+        // Check process.env.DB_REPLICATER_CREDS for existing credentials
+        const couch = Db.getCouch('internal');
+        const replicaterCredsDb = await couch.db.users(process.env.DB_REPLICATER_CREDS)
+        const replicaterHash = Utils.generateReplicatorHash(endpointUri, did, contextName)
+        console.log(`- replicaterHash: ${replicaterHash}`)
+
+        let creds = replicaterCredsDb.get(replicaterHash)
+        
+        if (!creds) {
+            const timestampMinutes = Math.floor(Date.now() / 1000 / 60)
+
+            // Generate a HD wallet to create a new private key to be used
+            // to generate a deterministic password for the endpoint
+            const wallet = new ethers.Wallet(process.env.VDA_PRIVATE_KEY)
+            const hdWallet = ethers.utils.HDNode.fromMnemonic(wallet.mnemonic)
+            const secondaryWallet = hdWallet.derivePath("1")
+            const password = EncryptionUtils.symEncrypt(replicaterHash, Buffer.from(secondaryWallet.privateKey, 'hex'))
+
+            const requestBody = {
+                endpointUri,
+                timestampMinutes,
+                password
+            }
+
+            const signature = ''
+            requestBody.signature = signature
+
+            console.log(`- requestBody: ${requestBody}`)
+
+            // Fetch credentials from the endpointUri
+            const result = Axios.get(`${endpointUri}'/auth/replicationCreds`, requestBody)
+            console.log(`- result: ${result.data}`)
+
+            // Save them
+            creds = {
+                _id: replicaterHash,
+                username: Utils.generateReplicaterUsername(endpointUri),
+                password
+            }
+
+            console.log(`- creds: ${creds}`)
+
+            try {
+                await dbManager._insertOrUpdate(replicaterCredsDb, creds, creds._id)
+            } catch (err) {
+                console.log(err)
+                throw new Error(`Unable to save replicater password : ${err.message} (${endpointUri})`)
+            }
+        }
+
+        return {
+            username: creds.username,
+            password: creds.password
+        }
+    }
+
     // @todo: garbage collection
     async gc() {
         const GC_PERCENT = process.env.GC_PERCENT
diff --git a/src/components/userManager.js b/src/components/userManager.js
index ea57186d..c61aa69a 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -2,6 +2,7 @@ import crypto from 'crypto';
 import Db from './db.js'
 import Utils from './utils.js'
 import DbManager from './dbManager.js';
+import AuthManager from './authManager';
 
 import dotenv from 'dotenv';
 dotenv.config();
@@ -119,6 +120,76 @@ class UserManager {
         return result
     }
 
+    async checkReplication(did, contextName, databaseName) {
+        console.log(`checkReplication(${did}, ${contextName}, ${databaseName})`)
+        // Lookup DID document and get list of endpoints for this context
+        const didDocument = await AuthManager.getDidDocument(did)
+        const endpoints = didDocument.locateServiceEndpoint(contextName, 'database')
+
+        console.log(`- endpoints: ${endpoints}`)
+
+        let databases = []
+        if (databaseName) {
+            // Only check a single database
+            databases.push(databaseName)
+        } else {
+            // Fetch all databases for this context
+            let userDatabases = await DbManager.getUserDatabases(did, contextName)
+            databases = userDatabases.map(item => item.databaseName)
+        }
+
+        console.log('- databases', databases)
+
+        // Ensure there is a replication entry for each
+        const couch = Db.getCouch('internal')
+        const replicationDb = couch.db.use('_replicator')
+
+        for (let d in databases) {
+            const dbName = databases[d]
+
+            for (let e in endpoints) {
+                const replicatorId = Utils.generateReplicatorHash(endpointUri, did, contextName)
+                const record = await replicationDb.get(replicatorId)
+
+                if (!record) {
+                    console.log(`- no record: ${endpointUri}`)
+                    // No record, so create it
+                    // Check if we have credentials
+                    // No credentials? Ask for them from the endpoint
+                    const { endpointUsername, endpointPassword } = AuthManager.fetchReplicaterCredentials(endpointUri, did, contextName)
+
+                    const dbHash = Utils.generateDatabaseName(did, contextName, dbName)
+                    const replicationRecord = {
+                        _id: `${replicatorId}-${dbhash}`,
+                        source: `host: ${Db.buildHost()}/${dbHash}`,
+                        target: {
+                            url: `${endpointUri}/${dbHash}`,
+                            auth: {
+                                basic: {
+                                    username: endpointUsername,
+                                    password: endpointPassword
+                                }
+                            }
+                        },
+                        create_target: true,
+                        continous: true
+                    }
+
+                    console.log('- replicationRecord')
+                    console.log(replicationRecord)
+
+                    try {
+                        await dbManager._insertOrUpdate(replicationDb, replicationRecord, replicationRecord.id)
+                    } catch (err) {
+                        console.log(err)
+                        throw new Error(`Unable to update password: ${err.message}`)
+                    }
+                }
+            }
+        }
+        
+    }
+
 }
 
 let userManager = new UserManager();
diff --git a/src/components/utils.js b/src/components/utils.js
index bac37cf2..c815a344 100644
--- a/src/components/utils.js
+++ b/src/components/utils.js
@@ -45,6 +45,19 @@ class Utils {
         return "v" + hash
     }
 
+    generateReplicatorHash(endpointUri, did, contextName) {
+        let text = [
+            endpointUri,
+            did.toLowerCase(),
+            contextName
+        ].join("/");
+        
+        const hash = EncryptionUtils.hash(text).substring(2);
+
+        // Database name must start with a letter
+        return "e" + hash
+    }
+
     didsToUsernames(dids, contextName) {
         return dids ? dids.map(did => this.generateUsername(did.toLowerCase(), contextName)) : []
     }
diff --git a/src/controllers/user.js b/src/controllers/user.js
index 559037f4..7eb487c2 100644
--- a/src/controllers/user.js
+++ b/src/controllers/user.js
@@ -284,6 +284,23 @@ class UserController {
     }
 
     async checkReplication(req, res) {
+        const did = req.tokenData.did
+        const contextName = req.tokenData.contextName
+        const databaseName = req.body.databaseName
+
+        try {
+            const result = await UserManager.checkReplication(did, contextName, databaseName)
+
+            return Utils.signedResponse({
+                status: "success",
+                result
+            }, res);
+        } catch (err) {
+            return res.status(500).send({
+                status: "fail",
+                message: err.message
+            });
+        }
     }
 
 }

From f4f4b594f8abe2d2954c43643b6c7fc945a705fe Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Mon, 12 Dec 2022 07:32:34 +1030
Subject: [PATCH 06/28] Add comments for next steps

---
 src/components/userManager.js |  2 ++
 test/replication.js           | 19 +++++++++++++++++++
 2 files changed, 21 insertions(+)
 create mode 100644 test/replication.js

diff --git a/src/components/userManager.js b/src/components/userManager.js
index c61aa69a..a892b1c4 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -187,6 +187,8 @@ class UserManager {
                 }
             }
         }
+
+        // @todo: Remove any replication entries for deleted databases
         
     }
 
diff --git a/test/replication.js b/test/replication.js
new file mode 100644
index 00000000..c26253e8
--- /dev/null
+++ b/test/replication.js
@@ -0,0 +1,19 @@
+
+
+/**
+ * Config:
+ * 1. admin credentials for endpoint1 and endpoint2
+ * 
+ * Steps:
+ * 
+ * 1. Create a new VDA private key
+ * 2. Create new DID document (using DIDClient) for the private key with two testing endpoints (local)
+ * 3. Create three test databases (db1, db2, db3) via `createDatabase()`
+ * 4. Call `checkReplication(db1)` on endpoint1, then endpoint2
+ * 5. Verify the data is being replicated for db1, but not db2 (create 3 records on endpoint1, verify its on endpoint2, create 3 records on d2, verify not on endpoint2)
+ * 6. Call `checkReplication()` on endpoint1, then endpoint2
+ * 7. Verify the data is being replicated for db2 and db3
+ * 8. Delete db1
+ * 9. Call `checkReplication()` on endpoint1, endpoint2
+ * 10. Verify replication entry is removed from both, Verify database is deleted from both
+ */
\ No newline at end of file

From d5ba2635d7f429f9de27741163a6d71fd6b5af09 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Mon, 12 Dec 2022 10:29:18 +1030
Subject: [PATCH 07/28] First pass at replication test

---
 test/replication.js | 215 ++++++++++++++++++++++++++++++++++++++++----
 test/utils.js       |  30 ++++++-
 2 files changed, 226 insertions(+), 19 deletions(-)

diff --git a/test/replication.js b/test/replication.js
index c26253e8..8e6cd8e4 100644
--- a/test/replication.js
+++ b/test/replication.js
@@ -1,19 +1,200 @@
+import Axios from 'axios'
+import assert from 'assert';
+import { ethers } from 'ethers'
+import { DIDDocument } from '@verida/did-document'
+import { DIDClient } from '@verida/did-client';
+import { AutoAccount } from "@verida/account-node"
+import { Keyring } from '@verida/keyring';
 
+import dotenv from 'dotenv';
+dotenv.config();
 
-/**
- * Config:
- * 1. admin credentials for endpoint1 and endpoint2
- * 
- * Steps:
- * 
- * 1. Create a new VDA private key
- * 2. Create new DID document (using DIDClient) for the private key with two testing endpoints (local)
- * 3. Create three test databases (db1, db2, db3) via `createDatabase()`
- * 4. Call `checkReplication(db1)` on endpoint1, then endpoint2
- * 5. Verify the data is being replicated for db1, but not db2 (create 3 records on endpoint1, verify its on endpoint2, create 3 records on d2, verify not on endpoint2)
- * 6. Call `checkReplication()` on endpoint1, then endpoint2
- * 7. Verify the data is being replicated for db2 and db3
- * 8. Delete db1
- * 9. Call `checkReplication()` on endpoint1, endpoint2
- * 10. Verify replication entry is removed from both, Verify database is deleted from both
- */
\ No newline at end of file
+import Utils from '../src/services/didStorage/utils'
+import TestUtils from './utils'
+import CONFIG from './config'
+
+const CONTEXT_NAME = 'Verida Test: Storage Node Replication'
+// @todo: use three endpoints
+const ENDPOINT_DSN = {
+    'http://192.168.1.117:5000': 'http://admin:admin@192.168.1.117:5984',
+    'http://192.168.1.118:5000': 'http://admin:admin@192.168.1.117:5984'
+}
+const ENDPOINTS = Object.keys(ENDPOINT_AUTH)
+const TEST_DATABASES = ['db1', 'db2', 'db3']
+const TEST_DEVICE_ID = 'Device 1'
+
+const didClient = new DIDClient(CONFIG.DID_CLIENT_CONFIG)
+
+describe("Replication tests", function() {
+
+    this.beforeAll(async () => {
+        // Create a new VDA private key
+        const wallet = ethers.Wallet.createRandom()
+        const DID_ADDRESS = wallet.address
+        const DID = `did:vda:testnet:${DID_ADDRESS}`
+        const DID_PUBLIC_KEY = wallet.publicKey
+        const DID_PRIVATE_KEY = wallet.privateKey
+        const keyring = new Keyring(wallet.mnemonic.phrase)
+        await didClient.authenticate(DID_PRIVATE_KEY, 'web3', CONFIG.DID_CLIENT_CONFIG.web3Config, ENDPOINTS)
+
+        console.log(DID_ADDRESS, DID, DID_PRIVATE_KEY, DID_PUBLIC_KEY, wallet.mnemonic.phrase)
+
+        // Create a new VDA account using our test endpoints
+        const account = new AutoAccount({
+            defaultDatabaseServer: {
+                type: 'VeridaDatabase',
+                endpointUri: ENDPOINTS
+            },
+            defaultMessageServer: {
+                type: 'VeridaMessage',
+                endpointUri: ENDPOINTS
+            },
+        }, {
+            privateKey: wallet.privateKey,
+            didClientConfig: CONFIG.DID_CLIENT_CONFIG,
+            environment: CONFIG.ENVIRONMENT
+        })
+
+        // Create new DID document (using DIDClient) for the private key with two testing endpoints (local)
+        const doc = new DIDDocument(DID, DID_PUBLIC_KEY)
+        await doc.addContext(CONTEXT_NAME, keyring, DID_PRIVATE_KEY, {
+            database: {
+                type: 'VeridaDatabase',
+                endpointUri: ENDPOINTS
+            },
+            messaging: {
+                type: 'VeridaMessage',
+                endpointUri: ENDPOINTS
+            },
+        })
+        const endpointResponses = await didClient.save(doc)
+        console.log(endpointResponses)
+        console.log(doc.export())
+
+        // Fetch an auth token for each server
+        const AUTH_TOKENS = {}
+        const CONNECTIONS = {}
+        for (let i in ENDPOINTS) {
+            const endpoint = ENDPOINTS[i]
+            const authJwtResult = await Axios.post(`${SERVER_URL}/auth/generateAuthJwt`, {
+                did: DID,
+                contextName: CONTEXT_NAME
+            });
+    
+            authRequestId = authJwtResult.data.authJwt.authRequestId
+            authJwt = authJwtResult.data.authJwt.authJwt
+            const consentMessage = `Authenticate this application context: "${CONTEXT_NAME}"?\n\n${DID.toLowerCase()}\n${authRequestId}`
+            const signature = await accountInfo.account.sign(consentMessage)
+
+            const authenticateResponse = await Axios.post(`${endpoint}/auth/authenticate`, {
+                authJwt,
+                did: DID,
+                contextName: CONTEXT_NAME,
+                signature,
+                deviceId: TEST_DEVICE_ID
+            })
+            AUTH_TOKENS[endpoint] = authenticateResponse.data.accessToken
+        }
+
+        console.log(AUTH_TOKENS)
+    })
+
+    describe("Create test databases", async () => {
+        // Create the test databases on the first endpoint
+        let endpoint = ENDPOINTS[0]
+        for (let i in TEST_DATABASES) {
+            const dbName = TEST_DATABASES[i]
+            const response = await Utils.createDatabase(dbName, DID, CONTEXT_NAME, AUTH_TOKENS[endpoint], endpoint)
+            console.log(`createDatabase (${dbName}) on ${endpoint} response:`)
+            console.log(response)
+        }
+
+        // Call `checkReplication(db1)` on all the endpoints (first database only)
+        it.only('can initialise replication for one database via checkReplication()', async () => {
+            for (let i in ENDPOINTS) {
+                const endpoint = ENDPOINTS[i]
+                const result = await Utils.checkReplication(endpoint, AUTH_TOKENS[endpoint], TEST_DATABASES[0])
+                console.log(`checkReplication on ${endpoint} for ${TEST_DATABASES[0]}`)
+                console.log(result)
+
+                const conn = Utils.buildPouchDsn(ENDPOINT_DSN[endpoint], '_replicator')
+                const replicationEntry = await conn.get(`${endpoint}/${TEST_DATABASE[0]}`)
+                console.log(`${endpoint} _replication entry for ${TEST_DATABASE[0]}`)
+                console.log(replicationEntry)
+                assert.ok(replicationEntry)
+            }
+        })
+
+        // Verify data saved to db1 is being replicated for all endpoints
+        it('verify data is replicated for first database only', async () => {
+            // Create three records
+            const endpoint0db1Connection = Utils.buildPouchDsn(ENDPOINT_DSN[endpoint], TEST_DATABASES[0])
+            await endpoint0db1Connection.put({db1endpoint1: 'world1'})
+            await endpoint0db1Connection.put({db1endpoint1: 'world2'})
+            await endpoint0db1Connection.put({db1endpoint1: 'world3'})
+
+            // Check the three records are correctly replicated on all the other databases
+            for (let i in ENDPOINTS) {
+                if (i === 0) {
+                    // skip first database
+                    continue
+                }
+
+                const conn = Utils.buildPouchDsn(ENDPOINT_DSN[endpoint], TEST_DATABASES[0])
+                const docs = await conn.allDocs({include_docs: true})
+                console.log(`Endpoint ${endpoint} has docs:`)
+                console.log(docs)
+                assert.equals(docs.rows.length, 3, 'Three rows returned')
+            }
+        })
+
+        // Verify data saved to db2 is NOT replicated for all endpoints
+        it('verify data is not replicated for second database', async () => {
+            // Create three records on second database
+            const endpoint1db2Connection = Utils.buildPouchDsn(ENDPOINT_DSN[endpoint], TEST_DATABASES[1])
+            await endpoint1db2Connection.put({db2endpoint2: 'world1'})
+            await endpoint1db2Connection.put({db2endpoint2: 'world2'})
+            await endpoint1db2Connection.put({db2endpoint2: 'world3'})
+
+            // Check the three records are correctly replicated on all the other databases
+            for (let i in ENDPOINTS) {
+                if (i === 1) {
+                    // skip second database
+                    continue
+                }
+
+                const conn = Utils.buildPouchDsn(ENDPOINT_DSN[endpoint], TEST_DATABASES[1])
+                const docs = await conn.allDocs({include_docs: true})
+                console.log(`Endpoint ${endpoint} has docs:`)
+                console.log(docs)
+                assert.equals(docs.rows.length, 0, 'No rows returned')
+            }
+        })
+
+        it('can initialise replication for all database via checkReplication()', async () => {
+            for (let i in ENDPOINTS) {
+                const endpoint = ENDPOINTS[i]
+                const result = await Utils.checkReplication(endpoint, AUTH_TOKENS[endpoint])
+                console.log(`checkReplication on ${endpoint} for all databases`)
+                console.log(result)
+            }
+
+            // @todo: check the replication database as per above
+        })
+
+        it('verify data is being replicated for all databases', async () => {
+
+        })
+
+        it('can delete a database', () => {})
+
+        it('can remove a database replication entry when via checkReplication()', () => {})
+
+        it('verify database is deleted from all endpoints', () => {})
+    })
+
+    this.afterAll(async () => {
+        // Delete all replication entries
+        // Delete all databases
+    })
+})
\ No newline at end of file
diff --git a/test/utils.js b/test/utils.js
index 5f45f116..4b83bf02 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -61,8 +61,34 @@ class Utils {
         });
     }
 
-    async createDatabase(databaseName, did, contextName, accessToken) {
-        const response = await Axios.post(`${CONFIG.SERVER_URL}/user/createDatabase`, {
+    buildPouchDsn(dsn, dbName) {
+        return new PouchDb(`${dsn}/${dbName}`, {
+            requestDefaults: {
+                rejectUnauthorized: false
+            }
+        });
+    }
+
+    async createDatabase(databaseName, did, contextName, accessToken, serverUrl) {
+        if (!serverUrl) {
+            serverUrl = CONFIG.SERVER_URL
+        }
+
+        const response = await Axios.post(`${serverUrl}/user/createDatabase`, {
+            databaseName,
+            did,
+            contextName
+        }, {
+            headers: {
+                Authorization: `Bearer ${accessToken}`
+            }
+        });
+        
+        return response
+    }
+
+    async checkReplication(endpointUri, accessToken, databaseName) {
+        const response = await Axios.post(`${endpointUri}/user/checkReplication`, {
             databaseName,
             did,
             contextName

From 2fa42091d0bdeb4aaaea71513f69d8eee7028c7c Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Tue, 13 Dec 2022 14:04:08 +1030
Subject: [PATCH 08/28] Fix missing endpoint variable

---
 src/components/userManager.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/components/userManager.js b/src/components/userManager.js
index a892b1c4..cde9ecb6 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -126,7 +126,8 @@ class UserManager {
         const didDocument = await AuthManager.getDidDocument(did)
         const endpoints = didDocument.locateServiceEndpoint(contextName, 'database')
 
-        console.log(`- endpoints: ${endpoints}`)
+        console.log(`- endpoints:`)
+        console.log(endpoints)
 
         let databases = []
         if (databaseName) {
@@ -148,6 +149,7 @@ class UserManager {
             const dbName = databases[d]
 
             for (let e in endpoints) {
+                const endpointUri = endpoints[e]
                 const replicatorId = Utils.generateReplicatorHash(endpointUri, did, contextName)
                 const record = await replicationDb.get(replicatorId)
 

From e34169e941adb045355a3e4fc00e5ae26d417416 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Wed, 14 Dec 2022 19:23:36 +1030
Subject: [PATCH 09/28] Bug fixes and enhanced logging

---
 sample.env                    |   7 ++-
 src/components/authManager.js | 107 ++++++++++++++++++++++----------
 src/components/db.js          |   2 +-
 src/components/dbManager.js   |   5 +-
 src/components/userManager.js | 113 +++++++++++++++++++++++-----------
 src/components/utils.js       |   4 ++
 src/controllers/auth.js       |  74 ++++++++++++++--------
 src/routes/public.js          |   2 +-
 8 files changed, 213 insertions(+), 101 deletions(-)

diff --git a/sample.env b/sample.env
index 2dc6e094..8f9a99a2 100644
--- a/sample.env
+++ b/sample.env
@@ -6,14 +6,17 @@ DB_PASS="admin"
 
 # Internal hostname (Used internally by the storage node for accessing CouchDb).
 DB_PROTOCOL_INTERNAL="http"
-DB_HOST_INTERNAL="192.168.68.117"
+DB_HOST_INTERNAL="localhost"
 DB_PORT_INTERNAL=5984
 
 # External hostname (returned to network users to access CouchDb)
 DB_PROTOCOL_EXTERNAL="http"
-DB_HOST_EXTERNAL="192.168.68.117"
+DB_HOST_EXTERNAL="localhost"
 DB_PORT_EXTERNAL=5984
 
+# The public URI of this storage node server (Will match what is stored in DID Documents)
+ENDPOINT_URI="http://localhost:5000"
+
 DB_REJECT_UNAUTHORIZED_SSL=true
 ACCESS_JWT_SIGN_PK=insert-random-access-symmetric-key
 # 5 Minutes
diff --git a/src/components/authManager.js b/src/components/authManager.js
index d09eb7ba..57c2d961 100644
--- a/src/components/authManager.js
+++ b/src/components/authManager.js
@@ -8,7 +8,7 @@ import EncryptionUtils from '@verida/encryption-utils';
 import Utils from './utils.js';
 import Db from './db.js';
 import dbManager from './dbManager.js';
-import { ethers } from 'ethers'
+import Axios from 'axios'
 
 dotenv.config();
 
@@ -396,6 +396,17 @@ class AuthManager {
             }
         }
 
+        try {
+            await couch.db.create(process.env.DB_REPLICATER_CREDS)
+        } catch (err) {
+            if (err.message.match(/already exists/)) {
+                // Database already exists
+            } else {
+                console.error(err)
+                throw err
+            }
+        }
+
         const tokenDb = couch.db.use(process.env.DB_REFRESH_TOKENS);
 
         const deviceIndex = {
@@ -415,82 +426,112 @@ class AuthManager {
     async ensureReplicationCredentials(endpointUri, password) {
         console.log(`ensureReplicationCredentials(${endpointUri}, ${password})`)
         const username = Utils.generateReplicaterUsername(endpointUri)
+        const id = `org.couchdb.user:${username}`
         console.log(`- username: ${username}`)
 
         const couch = Db.getCouch('internal');
-        const usersDb = await couch.db.users('_users')
+        const usersDb = await couch.db.use('_users')
         let user
         try {
-            const id = `org.couchdb.user:${username}`
             user = await usersDb.get(id)
-        } catch (err) {
-            console.log(err)
-            throw err
-        }
 
-        if (!user && !password) {
-            throw new Error(`Unable to create user: User doesn't exist and no password specified`)
-        }
+            // User exists, check if we need to update the password
+            if (!password) {
+                console.log(`User exists, NOT updating password`)
+                // No password, so no need to update and just confirm the user exists
+                return "exists"
+            }
 
-        if (user && password) {
-            // Update the password
+            // User exists and we need to update the password
+            console.log(`User exists, updating password`)
             user.password = password
-            console.log(`- user: ${user}`)
             try {
-                await dbManager._insertOrUpdate(usersDb, user, user.id)
+                await dbManager._insertOrUpdate(usersDb, user, user._id)
+                return "updated"
             } catch (err) {
                 console.log(err)
                 throw new Error(`Unable to update password: ${err.message}`)
             }
+        } catch (err) {
+            if (err.error !== 'not_found') {
+                throw err
+            }
+
+            // Need to create the user
+            try {
+                console.log('replication user didnt exist, so creating')
+                console.log(id)
+                await dbManager._insertOrUpdate(usersDb, {
+                    _id: id,
+                    name: username,
+                    password,
+                    type: "user",
+                    roles: []
+                }, id)
+
+                return "created"
+            } catch (err) {
+                console.log(err)
+                throw new Error(`Unable to create replication user: ${err.message}`)
+            }
         }
     }
 
     async fetchReplicaterCredentials(endpointUri, did, contextName) {
-        console.log(`fetchReplicaterCredentials(${endpointUri}, ${did}, ${contextName})`)
         // Check process.env.DB_REPLICATER_CREDS for existing credentials
         const couch = Db.getCouch('internal');
-        const replicaterCredsDb = await couch.db.users(process.env.DB_REPLICATER_CREDS)
+        const replicaterCredsDb = await couch.db.use(process.env.DB_REPLICATER_CREDS)
         const replicaterHash = Utils.generateReplicatorHash(endpointUri, did, contextName)
-        console.log(`- replicaterHash: ${replicaterHash}`)
-
-        let creds = replicaterCredsDb.get(replicaterHash)
         
+        console.log(`${Utils.serverUri()}: Fetching credentials for ${endpointUri}`)
+
+        let creds
+        try {
+            creds = await replicaterCredsDb.get(replicaterHash)
+            console.log(`${Utils.serverUri()}: Located credentials for ${endpointUri}`)
+        } catch (err) {
+            // If credentials aren't found, that's okay we will create them below
+            if (err.error != 'not_found') {
+                console.log('rethrowing')
+                throw err
+            }
+        }
+
         if (!creds) {
+            console.log(`${Utils.serverUri()}: No credentials found for ${endpointUri}... creating.`)
             const timestampMinutes = Math.floor(Date.now() / 1000 / 60)
 
-            // Generate a HD wallet to create a new private key to be used
-            // to generate a deterministic password for the endpoint
-            const wallet = new ethers.Wallet(process.env.VDA_PRIVATE_KEY)
-            const hdWallet = ethers.utils.HDNode.fromMnemonic(wallet.mnemonic)
-            const secondaryWallet = hdWallet.derivePath("1")
-            const password = EncryptionUtils.symEncrypt(replicaterHash, Buffer.from(secondaryWallet.privateKey, 'hex'))
+            // Generate a random password
+            const secretKeyBytes = EncryptionUtils.randomKey(32)
+            const password = Buffer.from(secretKeyBytes).toString('hex')
 
             const requestBody = {
+                did,
+                contextName,
                 endpointUri,
                 timestampMinutes,
                 password
             }
 
-            const signature = ''
+            const privateKeyBytes = new Uint8Array(Buffer.from(process.env.VDA_PRIVATE_KEY.substring(2), 'hex'))
+            const signature = EncryptionUtils.signData(requestBody, privateKeyBytes)
             requestBody.signature = signature
 
-            console.log(`- requestBody: ${requestBody}`)
-
             // Fetch credentials from the endpointUri
-            const result = Axios.get(`${endpointUri}'/auth/replicationCreds`, requestBody)
-            console.log(`- result: ${result.data}`)
+            console.log(`${Utils.serverUri()}: Requesting the creation of credentials for ${endpointUri}`)
+            const result = await Axios.post(`${endpointUri}/auth/replicationCreds`, requestBody)
+            console.log(`${Utils.serverUri()}: Credentials returned for ${endpointUri}`)
+            console.log(result.data)
 
-            // Save them
             creds = {
                 _id: replicaterHash,
                 username: Utils.generateReplicaterUsername(endpointUri),
                 password
             }
 
-            console.log(`- creds: ${creds}`)
-
             try {
                 await dbManager._insertOrUpdate(replicaterCredsDb, creds, creds._id)
+                console.log(`${Utils.serverUri()}: Credentials saved for ${endpointUri}`)
             } catch (err) {
                 console.log(err)
                 throw new Error(`Unable to save replicater password : ${err.message} (${endpointUri})`)
diff --git a/src/components/db.js b/src/components/db.js
index 4c19dfda..0cb814c0 100644
--- a/src/components/db.js
+++ b/src/components/db.js
@@ -28,7 +28,7 @@ class Db {
         return PROTOCOL + "://" + username + ":" + password + "@" + HOST + ":" + PORT;
     }
 
-    // Build external hostname that users will connect to
+    // Build external hostname that users will connect to the storage node
     buildHost() {
         let env = process.env;
         return env.DB_PROTOCOL_EXTERNAL + "://" + env.DB_HOST_EXTERNAL + ":" + env.DB_PORT_EXTERNAL;
diff --git a/src/components/dbManager.js b/src/components/dbManager.js
index 22cac04a..eba2a716 100644
--- a/src/components/dbManager.js
+++ b/src/components/dbManager.js
@@ -1,7 +1,6 @@
 import Utils from './utils.js';
 import _ from 'lodash';
 import Db from "./db.js"
-import EncryptionUtils from "@verida/encryption-utils"
 import dotenv from 'dotenv';
 dotenv.config();
 
@@ -324,9 +323,9 @@ class DbManager {
         if (doc._rev) {
             newDoc._rev = doc._rev;
             newDoc._id = id;
-            return db.insert(newDoc);
+            return await db.insert(newDoc);
         } else {
-            return db.insert(newDoc, id);
+            return await db.insert(newDoc, id);
         }
     }
 
diff --git a/src/components/userManager.js b/src/components/userManager.js
index cde9ecb6..726b50df 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -120,30 +120,63 @@ class UserManager {
         return result
     }
 
+    /**
+     * Confirm replication is correctly configured for a given DID and application context.
+     * 
+     * If a storage node is being added or removed to the application context, it must be the
+     * last node to have checkReplication called. This ensures the node has a list of all the
+     * active databases and can ensure it is replicating correctly to the other nodes.
+     * 
+     * The client SDK should call checkReplication() when opening a context to ensure the replication is working as expected.
+     * 
+     * @param {*} did 
+     * @param {*} contextName 
+     * @param {*} databaseName (optional) If not specified, checks all databases
+     */
     async checkReplication(did, contextName, databaseName) {
-        console.log(`checkReplication(${did}, ${contextName}, ${databaseName})`)
+        console.log(`${Utils.serverUri()}: checkReplication(${did}, ${contextName}, ${databaseName})`)
         // Lookup DID document and get list of endpoints for this context
         const didDocument = await AuthManager.getDidDocument(did)
-        const endpoints = didDocument.locateServiceEndpoint(contextName, 'database')
+        const didService = didDocument.locateServiceEndpoint(contextName, 'database')
+        let endpoints = didService.serviceEndpoint
+
+        // Confirm this endpoint is in the list of endpoints
+        const endpointIndex = allEndpoints.indexOf(Utils.serverUri())
+        if (endpointIndex === -1) {
+            throw new Error('Server not a valid endpoint for this DID and context')
+        }
+
+        // Remove this endpoint from the list of endpoints to check
+        endpoints.splice(endpointIndex, 1)
 
         console.log(`- endpoints:`)
         console.log(endpoints)
 
         let databases = []
         if (databaseName) {
+            console.log(`${Utils.serverUri()}: Only checking ${databaseName})`)
             // Only check a single database
             databases.push(databaseName)
         } else {
             // Fetch all databases for this context
             let userDatabases = await DbManager.getUserDatabases(did, contextName)
             databases = userDatabases.map(item => item.databaseName)
+            console.log(`${Utils.serverUri()}: Cecking ${databases.length}) databases`)
         }
 
-        console.log('- databases', databases)
+        //console.log('- databases', databases)
 
         // Ensure there is a replication entry for each
         const couch = Db.getCouch('internal')
-        const replicationDb = couch.db.use('_replicator')
+        let replicationDb
+        try {
+            replicationDb = couch.db.use('_replicator')
+            console.log('got db')
+        } catch (err) {
+            console.log('!')
+            console.log(err)
+            throw err
+        }
 
         for (let d in databases) {
             const dbName = databases[d]
@@ -151,47 +184,55 @@ class UserManager {
             for (let e in endpoints) {
                 const endpointUri = endpoints[e]
                 const replicatorId = Utils.generateReplicatorHash(endpointUri, did, contextName)
-                const record = await replicationDb.get(replicatorId)
-
-                if (!record) {
-                    console.log(`- no record: ${endpointUri}`)
-                    // No record, so create it
-                    // Check if we have credentials
-                    // No credentials? Ask for them from the endpoint
-                    const { endpointUsername, endpointPassword } = AuthManager.fetchReplicaterCredentials(endpointUri, did, contextName)
-
-                    const dbHash = Utils.generateDatabaseName(did, contextName, dbName)
-                    const replicationRecord = {
-                        _id: `${replicatorId}-${dbhash}`,
-                        source: `host: ${Db.buildHost()}/${dbHash}`,
-                        target: {
-                            url: `${endpointUri}/${dbHash}`,
-                            auth: {
-                                basic: {
-                                    username: endpointUsername,
-                                    password: endpointPassword
+                const dbHash = Utils.generateDatabaseName(did, contextName, dbName)
+                let record
+                try {
+                    record = await replicationDb.get(`${replicatorId}-${dbHash}`)
+                    console.log(`${Utils.serverUri()}: Located replication record for ${endpointUri} (${replicatorId})`)
+                } catch (err) {
+                    if (err.message == 'missing' || err.reason == 'deleted') {
+                        console.log(`${Utils.serverUri()}: Located replication record for ${endpointUri}... creating.`)
+                        // No record, so create it
+                        // Check if we have credentials
+                        // No credentials? Ask for them from the endpoint
+                        const { endpointUsername, endpointPassword } = await AuthManager.fetchReplicaterCredentials(endpointUri, did, contextName)
+                        console.log(`${Utils.serverUri()}: Located replication credentials for ${endpointUri} (${endpointUsername}, ${endpointPassword})`)
+
+                        const replicationRecord = {
+                            _id: `${replicatorId}-${dbHash}`,
+                            source: `${Db.buildHost()}/${dbHash}`,
+                            target: {
+                                url: `${endpointUri}/${dbHash}`,
+                                auth: {
+                                    basic: {
+                                        username: endpointUsername,
+                                        password: endpointPassword
+                                    }
                                 }
-                            }
-                        },
-                        create_target: true,
-                        continous: true
+                            },
+                            create_target: true,
+                            continous: true
+                        }
+
+                        try {
+                            await DbManager._insertOrUpdate(replicationDb, replicationRecord, replicationRecord._id)
+                            console.log(`${Utils.serverUri()}: Saved replication entry for ${endpointUri} (${replicatorId})`)
+                        } catch (err) {
+                            console.log(`${Utils.serverUri()}: Error saving replication entry for ${endpointUri} (${replicatorId})`)
+                            console.log(err)
+                            throw new Error(`Unable to create replication entry: ${err.message}`)
+                        }
                     }
-
-                    console.log('- replicationRecord')
-                    console.log(replicationRecord)
-
-                    try {
-                        await dbManager._insertOrUpdate(replicationDb, replicationRecord, replicationRecord.id)
-                    } catch (err) {
+                    else {
+                        console.log(`${Utils.serverUri()}: Unknown error fetching replication entry for ${endpointUri} (${replicatorId})`)
                         console.log(err)
-                        throw new Error(`Unable to update password: ${err.message}`)
+                        throw err
                     }
                 }
             }
         }
 
         // @todo: Remove any replication entries for deleted databases
-        
     }
 
 }
diff --git a/src/components/utils.js b/src/components/utils.js
index c815a344..6f1dd2ff 100644
--- a/src/components/utils.js
+++ b/src/components/utils.js
@@ -89,6 +89,10 @@ class Utils {
         })
     }
 
+    serverUri() {
+        return process.env.ENDPOINT_URI
+    }
+
 }
 
 let utils = new Utils();
diff --git a/src/controllers/auth.js b/src/controllers/auth.js
index d18b1908..aaf5469c 100644
--- a/src/controllers/auth.js
+++ b/src/controllers/auth.js
@@ -202,13 +202,27 @@ class AuthController {
         }
     }
 
-    // If password is not an empty string, it will update the password to match
-    // If no user exists, must specify a password
+    /**
+     * Ensure replication credentials exist on the server
+     * 
+     * If the password is an empty string, will just determine if the user exists or not
+     * If the password is not an empty string, it will update the password to match
+     * If no user exists, must specify a password
+     * 
+     * Return status is either:
+     * 1. `created` (user created)
+     * 2. `updated` (password updated)
+     * 3. `exists` (user existed, but password unchanged)
+     * 
+     * @param {*} req 
+     * @param {*} res 
+     * @returns 
+     */
     async replicationCreds(req, res) {
         const {
             endpointUri,
-            //did,
-            //contextName,
+            did,
+            contextName,
             timestampMinutes,
             password,
             signature
@@ -223,51 +237,61 @@ class AuthController {
             return Utils.error(res, 'Timestamp not specified')
         }
 
+        // @todo: verify timestampMinutes is within range
+
+        if (!did) {
+            return Utils.error(res, 'DID not specified')
+        }
+
+        if (!contextName) {
+            return Utils.error(res, 'Context not specified')
+        }
+
         if (!signature) {
             return Utils.error(res, 'Signature not specified')
         }
 
         // Lookup DID document and confirm endpointUri is a valid endpoint
         const didDocument = await AuthManager.getDidDocument(did)
-        const endpoints = didDocument.locateServiceEndpoint(contextName, 'database')
-        console.log(endpoints)
+        const endpointService = didDocument.locateServiceEndpoint(contextName, 'database')
+        const endpoints = endpointService.serviceEndpoint
         if (endpoints.indexOf(endpointUri) === -1) {
-            return Utils.error(res, 'Invalid endpoint 1')
+            return Utils.error(res, `Invalid endpoint (${endpointUri}): DID not linked (${did})`)
+        }
+
+        // Confirm this endpoint is linked to the DID and context
+        const thisEndpointUri = Utils.serverUri()
+        if (endpoints.indexOf(thisEndpointUri)) {
+            return Utils.error(res, `Invalid DID and context: Not associated with this endpoint`)
         }
         
         // Pull endpoint public key from /status and verify the signature
         let endpointPublicKey
         try {
-            const result = await Axios.get(`${endpointUri}/status`)
-            console.log(result.data)
-            endpointPublicKey = result.data.publicKey
+            const response = await Axios.get(`${endpointUri}/status`)
+            console.log(response.data)
+
+            endpointPublicKey = response.data.results.publicKey
             const params = {
+                did,
+                contextName,
                 endpointUri,
                 timestampMinutes,
                 password
             }
 
             if (!EncryptionUtils.verifySig(params, signature, endpointPublicKey)) {
-                return Utils.error(res, 'Invalid signature')
+                return Utils.error(res, 'Invalid signature', 401)
             }
         } catch (err) {
-            console.log(err)
-            return Utils.error(res, 'Invalid endpoint 2')
-        }
-
-        let decryptedPassword
-        if (password !== '') {
-            try {
-                const sharedKey = EncryptionUtils.sharedKey(endpointPublicKey, process.env.VDA_PRIVATE_KEY)
-                decryptedPassword = EncryptionUtils.asymDecrypt(password, sharedKey)
-            } catch (err) {
-                console.log(err)
-                return Utils.error(res, 'Invalid password encryption')
-            }
+            return Utils.error(res, `Unknown error: ${err.message}`)
         }
 
         try {
-            await AuthManager.ensureReplicationCredentials(endpointUri, decryptedPassword)
+            const result = await AuthManager.ensureReplicationCredentials(endpointUri, password)
+            return Utils.signedResponse({
+                result
+            }, res)
         } catch (err) {
             return Utils.error(res, err.message)
         }
diff --git a/src/routes/public.js b/src/routes/public.js
index cdcf4c18..c32bd7c3 100644
--- a/src/routes/public.js
+++ b/src/routes/public.js
@@ -10,7 +10,7 @@ const router = express.Router();
 router.get('/auth/public', UserController.getPublic);
 router.get('/status', SystemController.status);
 
-router.get('/auth/replicationCreds', AuthController.replicationCreds);
+router.post('/auth/replicationCreds', AuthController.replicationCreds);
 router.post('/auth/generateAuthJwt', AuthController.generateAuthJwt);
 router.post('/auth/authenticate', AuthController.authenticate);
 router.post('/auth/connect', AuthController.connect);

From 07ed3f765c6978569bfe0cb0cb50fa032271be4a Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Wed, 14 Dec 2022 19:50:02 +1030
Subject: [PATCH 10/28] Bug fixes from testing

---
 src/components/userManager.js | 2 +-
 src/controllers/auth.js       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/components/userManager.js b/src/components/userManager.js
index 726b50df..3179ed9b 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -141,7 +141,7 @@ class UserManager {
         let endpoints = didService.serviceEndpoint
 
         // Confirm this endpoint is in the list of endpoints
-        const endpointIndex = allEndpoints.indexOf(Utils.serverUri())
+        const endpointIndex = endpoints.indexOf(Utils.serverUri())
         if (endpointIndex === -1) {
             throw new Error('Server not a valid endpoint for this DID and context')
         }
diff --git a/src/controllers/auth.js b/src/controllers/auth.js
index aaf5469c..7b75fa8a 100644
--- a/src/controllers/auth.js
+++ b/src/controllers/auth.js
@@ -220,7 +220,7 @@ class AuthController {
      */
     async replicationCreds(req, res) {
         const {
-            endpointUri,
+            endpointUri,        // endpoint making the request
             did,
             contextName,
             timestampMinutes,
@@ -261,7 +261,7 @@ class AuthController {
 
         // Confirm this endpoint is linked to the DID and context
         const thisEndpointUri = Utils.serverUri()
-        if (endpoints.indexOf(thisEndpointUri)) {
+        if (endpoints.indexOf(thisEndpointUri) === -1) {
             return Utils.error(res, `Invalid DID and context: Not associated with this endpoint`)
         }
         

From e8c7187f061c966e57f263af99ea50652e6770aa Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Wed, 14 Dec 2022 19:51:57 +1030
Subject: [PATCH 11/28] Bug fixes

---
 src/components/authManager.js | 26 ++++++++++++++++++++++----
 src/components/userManager.js |  2 +-
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/src/components/authManager.js b/src/components/authManager.js
index 57c2d961..b5f06aa3 100644
--- a/src/components/authManager.js
+++ b/src/components/authManager.js
@@ -407,6 +407,17 @@ class AuthManager {
             }
         }
 
+        try {
+            await couch.db.create('_replicator')
+        } catch (err) {
+            if (err.message.match(/already exists/)) {
+                // Database already exists
+            } else {
+                console.error(err)
+                throw err
+            }
+        }
+
         const tokenDb = couch.db.use(process.env.DB_REFRESH_TOKENS);
 
         const deviceIndex = {
@@ -508,7 +519,7 @@ class AuthManager {
             const requestBody = {
                 did,
                 contextName,
-                endpointUri,
+                endpointUri: Utils.serverUri(),
                 timestampMinutes,
                 password
             }
@@ -519,9 +530,16 @@ class AuthManager {
 
             // Fetch credentials from the endpointUri
             console.log(`${Utils.serverUri()}: Requesting the creation of credentials for ${endpointUri}`)
-            const result = await Axios.post(`${endpointUri}/auth/replicationCreds`, requestBody)
-            console.log(`${Utils.serverUri()}: Credentials returned for ${endpointUri}`)
-            console.log(result.data)
+            try {
+                const result = await Axios.post(`${endpointUri}/auth/replicationCreds`, requestBody)
+                console.log(`${Utils.serverUri()}: Credentials returned for ${endpointUri}`)
+            } catch (err) {
+                if (err.response) {
+                    throw Error(`Unable to obtain credentials from ${endpointUri} (${err.response.data.message})`)
+                }
+
+                throw err
+            }
 
             creds = {
                 _id: replicaterHash,
diff --git a/src/components/userManager.js b/src/components/userManager.js
index 3179ed9b..78c78bdc 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -191,7 +191,7 @@ class UserManager {
                     console.log(`${Utils.serverUri()}: Located replication record for ${endpointUri} (${replicatorId})`)
                 } catch (err) {
                     if (err.message == 'missing' || err.reason == 'deleted') {
-                        console.log(`${Utils.serverUri()}: Located replication record for ${endpointUri}... creating.`)
+                        console.log(`${Utils.serverUri()}: Replication record for ${endpointUri} is missing... creating.`)
                         // No record, so create it
                         // Check if we have credentials
                         // No credentials? Ask for them from the endpoint

From 9913781eba011ade52fa32ac93227fbd56817bed Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Wed, 14 Dec 2022 20:45:16 +1030
Subject: [PATCH 12/28] Support generating correct CouchURI for server

---
 src/components/authManager.js | 23 +++++++++++++++++++----
 src/components/userManager.js | 10 +++++-----
 src/controllers/system.js     |  3 ++-
 3 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/src/components/authManager.js b/src/components/authManager.js
index b5f06aa3..a77b3a75 100644
--- a/src/components/authManager.js
+++ b/src/components/authManager.js
@@ -531,8 +531,21 @@ class AuthManager {
             // Fetch credentials from the endpointUri
             console.log(`${Utils.serverUri()}: Requesting the creation of credentials for ${endpointUri}`)
             try {
-                const result = await Axios.post(`${endpointUri}/auth/replicationCreds`, requestBody)
-                console.log(`${Utils.serverUri()}: Credentials returned for ${endpointUri}`)
+                await Axios.post(`${endpointUri}/auth/replicationCreds`, requestBody)
+                console.log(`${Utils.serverUri()}: Credentials generated for ${endpointUri}`)
+            } catch (err) {
+                if (err.response) {
+                    throw Error(`Unable to obtain credentials from ${endpointUri} (${err.response.data.message})`)
+                }
+
+                throw err
+            }
+
+            let couchUri
+            try {
+                statusResponse = await Axios.get(`${endpointUri}/status`)
+                console.log(`${Utils.serverUri()}: Status fetched ${endpointUri}`)
+                couchUri = statusResponse.data.results.couchUri
             } catch (err) {
                 if (err.response) {
                     throw Error(`Unable to obtain credentials from ${endpointUri} (${err.response.data.message})`)
@@ -544,7 +557,8 @@ class AuthManager {
             creds = {
                 _id: replicaterHash,
                 username: Utils.generateReplicaterUsername(endpointUri),
-                password
+                password,
+                couchUri
             }
 
             try {
@@ -558,7 +572,8 @@ class AuthManager {
 
         return {
             username: creds.username,
-            password: creds.password
+            password: creds.password,
+            couchUri: creds.couchUri
         }
     }
 
diff --git a/src/components/userManager.js b/src/components/userManager.js
index 78c78bdc..65876948 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -195,18 +195,18 @@ class UserManager {
                         // No record, so create it
                         // Check if we have credentials
                         // No credentials? Ask for them from the endpoint
-                        const { endpointUsername, endpointPassword } = await AuthManager.fetchReplicaterCredentials(endpointUri, did, contextName)
-                        console.log(`${Utils.serverUri()}: Located replication credentials for ${endpointUri} (${endpointUsername}, ${endpointPassword})`)
+                        const { username, password, couchUri } = await AuthManager.fetchReplicaterCredentials(endpointUri, did, contextName)
+                        console.log(`${Utils.serverUri()}: Located replication credentials for ${endpointUri} (${username}, ${password}, ${couchUri})`)
 
                         const replicationRecord = {
                             _id: `${replicatorId}-${dbHash}`,
                             source: `${Db.buildHost()}/${dbHash}`,
                             target: {
-                                url: `${endpointUri}/${dbHash}`,
+                                url: `${couchUri}/${dbHash}`,
                                 auth: {
                                     basic: {
-                                        username: endpointUsername,
-                                        password: endpointPassword
+                                        username,
+                                        password
                                     }
                                 }
                             },
diff --git a/src/controllers/system.js b/src/controllers/system.js
index a994df2b..313bb44b 100644
--- a/src/controllers/system.js
+++ b/src/controllers/system.js
@@ -17,7 +17,8 @@ class SystemController {
             maxUsers: parseInt(process.env.MAX_USERS),
             currentUsers,
             version: packageJson.version,
-            publicKey: wallet.publicKey
+            publicKey: wallet.publicKey,
+            couchUri: db.buildHost()
         }
 
         return Utils.signedResponse({

From 6532f5c5663df7390b3b0d75adae62e2d6a4f94b Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Wed, 14 Dec 2022 21:02:33 +1030
Subject: [PATCH 13/28] Bug fixes

---
 src/components/authManager.js |  7 ++++---
 src/components/userManager.js | 12 ++----------
 2 files changed, 6 insertions(+), 13 deletions(-)

diff --git a/src/components/authManager.js b/src/components/authManager.js
index a77b3a75..32a4fc1f 100644
--- a/src/components/authManager.js
+++ b/src/components/authManager.js
@@ -543,9 +543,9 @@ class AuthManager {
 
             let couchUri
             try {
-                statusResponse = await Axios.get(`${endpointUri}/status`)
-                console.log(`${Utils.serverUri()}: Status fetched ${endpointUri}`)
+                const statusResponse = await Axios.get(`${endpointUri}/status`)
                 couchUri = statusResponse.data.results.couchUri
+                console.log(`${Utils.serverUri()}: Status fetched ${endpointUri} with CouchURI: ${couchUri}`)
             } catch (err) {
                 if (err.response) {
                     throw Error(`Unable to obtain credentials from ${endpointUri} (${err.response.data.message})`)
@@ -556,7 +556,8 @@ class AuthManager {
 
             creds = {
                 _id: replicaterHash,
-                username: Utils.generateReplicaterUsername(endpointUri),
+                // Use this server username
+                username: Utils.generateReplicaterUsername(Utils.serverUri()),
                 password,
                 couchUri
             }
diff --git a/src/components/userManager.js b/src/components/userManager.js
index 65876948..299b757d 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -138,7 +138,7 @@ class UserManager {
         // Lookup DID document and get list of endpoints for this context
         const didDocument = await AuthManager.getDidDocument(did)
         const didService = didDocument.locateServiceEndpoint(contextName, 'database')
-        let endpoints = didService.serviceEndpoint
+        let endpoints = [...didService.serviceEndpoint] // create a copy as this is cached and we will modify later
 
         // Confirm this endpoint is in the list of endpoints
         const endpointIndex = endpoints.indexOf(Utils.serverUri())
@@ -168,15 +168,7 @@ class UserManager {
 
         // Ensure there is a replication entry for each
         const couch = Db.getCouch('internal')
-        let replicationDb
-        try {
-            replicationDb = couch.db.use('_replicator')
-            console.log('got db')
-        } catch (err) {
-            console.log('!')
-            console.log(err)
-            throw err
-        }
+        const replicationDb = couch.db.use('_replicator')\
 
         for (let d in databases) {
             const dbName = databases[d]

From 35d49478a2fa13ae0a80d08206d7dc4a6ecebe44 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Wed, 14 Dec 2022 21:03:08 +1030
Subject: [PATCH 14/28] Fix syntax error

---
 src/components/userManager.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/components/userManager.js b/src/components/userManager.js
index 299b757d..fe8e6884 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -168,7 +168,7 @@ class UserManager {
 
         // Ensure there is a replication entry for each
         const couch = Db.getCouch('internal')
-        const replicationDb = couch.db.use('_replicator')\
+        const replicationDb = couch.db.use('_replicator')
 
         for (let d in databases) {
             const dbName = databases[d]

From f883ab28a7db9fa6b1c6c1d554be91478ad2b0ad Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Wed, 14 Dec 2022 21:09:28 +1030
Subject: [PATCH 15/28] First tests passing

---
 test/replication.js | 258 ++++++++++++++++++++++++++++----------------
 test/utils.js       |   4 +-
 2 files changed, 167 insertions(+), 95 deletions(-)

diff --git a/test/replication.js b/test/replication.js
index 8e6cd8e4..f6a3cbb1 100644
--- a/test/replication.js
+++ b/test/replication.js
@@ -5,130 +5,190 @@ import { DIDDocument } from '@verida/did-document'
 import { DIDClient } from '@verida/did-client';
 import { AutoAccount } from "@verida/account-node"
 import { Keyring } from '@verida/keyring';
+import ComponentUtils from '../src/components/utils'
+import CouchDb from 'nano'
 
 import dotenv from 'dotenv';
 dotenv.config();
 
-import Utils from '../src/services/didStorage/utils'
-import TestUtils from './utils'
+import Utils from './utils'
 import CONFIG from './config'
 
 const CONTEXT_NAME = 'Verida Test: Storage Node Replication'
 // @todo: use three endpoints
 const ENDPOINT_DSN = {
-    'http://192.168.1.117:5000': 'http://admin:admin@192.168.1.117:5984',
-    'http://192.168.1.118:5000': 'http://admin:admin@192.168.1.117:5984'
+    'http://192.168.68.117:5000': 'http://admin:admin@192.168.68.117:5984',
+    'http://192.168.68.118:5000': 'http://admin:admin@192.168.68.118:5984',
 }
-const ENDPOINTS = Object.keys(ENDPOINT_AUTH)
+const ENDPOINTS = Object.keys(ENDPOINT_DSN)
+const ENDPOINTS_DID = ENDPOINTS.map(item => `${item}/did/`)
+const ENDPOINTS_COUCH = {}
+ENDPOINTS.forEach(key => {
+    ENDPOINTS_COUCH[key] = key.replace('5000', '5984')
+})
 const TEST_DATABASES = ['db1', 'db2', 'db3']
 const TEST_DEVICE_ID = 'Device 1'
 
 const didClient = new DIDClient(CONFIG.DID_CLIENT_CONFIG)
 
 describe("Replication tests", function() {
+    let DID, DID_ADDRESS, DID_PUBLIC_KEY, DID_PRIVATE_KEY, keyring, wallet, account, AUTH_TOKENS
 
-    this.beforeAll(async () => {
-        // Create a new VDA private key
-        const wallet = ethers.Wallet.createRandom()
-        const DID_ADDRESS = wallet.address
-        const DID = `did:vda:testnet:${DID_ADDRESS}`
-        const DID_PUBLIC_KEY = wallet.publicKey
-        const DID_PRIVATE_KEY = wallet.privateKey
-        const keyring = new Keyring(wallet.mnemonic.phrase)
-        await didClient.authenticate(DID_PRIVATE_KEY, 'web3', CONFIG.DID_CLIENT_CONFIG.web3Config, ENDPOINTS)
-
-        console.log(DID_ADDRESS, DID, DID_PRIVATE_KEY, DID_PUBLIC_KEY, wallet.mnemonic.phrase)
-
-        // Create a new VDA account using our test endpoints
-        const account = new AutoAccount({
-            defaultDatabaseServer: {
-                type: 'VeridaDatabase',
-                endpointUri: ENDPOINTS
-            },
-            defaultMessageServer: {
-                type: 'VeridaMessage',
-                endpointUri: ENDPOINTS
-            },
-        }, {
-            privateKey: wallet.privateKey,
-            didClientConfig: CONFIG.DID_CLIENT_CONFIG,
-            environment: CONFIG.ENVIRONMENT
-        })
+    describe("Create test databases", async () => {
+        this.timeout(200 * 1000)
 
-        // Create new DID document (using DIDClient) for the private key with two testing endpoints (local)
-        const doc = new DIDDocument(DID, DID_PUBLIC_KEY)
-        await doc.addContext(CONTEXT_NAME, keyring, DID_PRIVATE_KEY, {
-            database: {
-                type: 'VeridaDatabase',
-                endpointUri: ENDPOINTS
-            },
-            messaging: {
-                type: 'VeridaMessage',
-                endpointUri: ENDPOINTS
-            },
-        })
-        const endpointResponses = await didClient.save(doc)
-        console.log(endpointResponses)
-        console.log(doc.export())
-
-        // Fetch an auth token for each server
-        const AUTH_TOKENS = {}
-        const CONNECTIONS = {}
-        for (let i in ENDPOINTS) {
-            const endpoint = ENDPOINTS[i]
-            const authJwtResult = await Axios.post(`${SERVER_URL}/auth/generateAuthJwt`, {
-                did: DID,
-                contextName: CONTEXT_NAME
-            });
+        this.beforeAll(async () => {
+            // Create a new VDA private key
+            //wallet = ethers.Wallet.createRandom()
+            wallet = ethers.Wallet.fromMnemonic('pave online install gift glimpse purpose truth loan arm wing west option')
+            DID_ADDRESS = wallet.address
+            DID = `did:vda:testnet:${DID_ADDRESS}`
+            DID_PUBLIC_KEY = wallet.publicKey
+            DID_PRIVATE_KEY = wallet.privateKey
+            keyring = new Keyring(wallet.mnemonic.phrase)
+            await didClient.authenticate(DID_PRIVATE_KEY, 'web3', CONFIG.DID_CLIENT_CONFIG.web3Config, ENDPOINTS_DID)
     
-            authRequestId = authJwtResult.data.authJwt.authRequestId
-            authJwt = authJwtResult.data.authJwt.authJwt
-            const consentMessage = `Authenticate this application context: "${CONTEXT_NAME}"?\n\n${DID.toLowerCase()}\n${authRequestId}`
-            const signature = await accountInfo.account.sign(consentMessage)
-
-            const authenticateResponse = await Axios.post(`${endpoint}/auth/authenticate`, {
-                authJwt,
-                did: DID,
-                contextName: CONTEXT_NAME,
-                signature,
-                deviceId: TEST_DEVICE_ID
+            console.log(DID_ADDRESS, DID, DID_PRIVATE_KEY, DID_PUBLIC_KEY, wallet.mnemonic.phrase)
+    
+            // Create a new VDA account using our test endpoints
+            account = new AutoAccount({
+                defaultDatabaseServer: {
+                    type: 'VeridaDatabase',
+                    endpointUri: ENDPOINTS
+                },
+                defaultMessageServer: {
+                    type: 'VeridaMessage',
+                    endpointUri: ENDPOINTS
+                },
+            }, {
+                privateKey: wallet.privateKey,
+                didClientConfig: CONFIG.DID_CLIENT_CONFIG,
+                environment: CONFIG.ENVIRONMENT
+            })
+    
+            // Create new DID document (using DIDClient) for the private key with two testing endpoints (local)
+            let doc = await didClient.get(DID)
+            if (!doc) {
+                doc = new DIDDocument(DID, DID_PUBLIC_KEY)
+            }
+            await doc.addContext(CONTEXT_NAME, keyring, DID_PRIVATE_KEY, {
+                database: {
+                    type: 'VeridaDatabase',
+                    endpointUri: ENDPOINTS
+                },
+                messaging: {
+                    type: 'VeridaMessage',
+                    endpointUri: ENDPOINTS
+                },
             })
-            AUTH_TOKENS[endpoint] = authenticateResponse.data.accessToken
-        }
 
-        console.log(AUTH_TOKENS)
-    })
+            try {
+                const endpointResponses = await didClient.save(doc)
+            } catch (err) {
+                console.log(err)
+                console.log(didClient.getLastEndpointErrors())
+            }
+    
+            // Fetch an auth token for each server
+            AUTH_TOKENS = {}
+            for (let i in ENDPOINTS) {
+                const endpoint = ENDPOINTS[i]
+                console.log(`Authenticating with ${endpoint}`)
+                const authJwtResult = await Axios.post(`${endpoint}/auth/generateAuthJwt`, {
+                    did: DID,
+                    contextName: CONTEXT_NAME
+                });
+        
+                const authRequestId = authJwtResult.data.authJwt.authRequestId
+                const authJwt = authJwtResult.data.authJwt.authJwt
+                const consentMessage = `Authenticate this application context: "${CONTEXT_NAME}"?\n\n${DID.toLowerCase()}\n${authRequestId}`
+                const signature = await account.sign(consentMessage)
+    
+                const authenticateResponse = await Axios.post(`${endpoint}/auth/authenticate`, {
+                    authJwt,
+                    did: DID,
+                    contextName: CONTEXT_NAME,
+                    signature,
+                    deviceId: TEST_DEVICE_ID
+                })
+                AUTH_TOKENS[endpoint] = authenticateResponse.data.accessToken
+            }
+    
+            console.log(`auth tokens for the endpoints:`)
+            console.log(AUTH_TOKENS)
+        })
 
-    describe("Create test databases", async () => {
         // Create the test databases on the first endpoint
-        let endpoint = ENDPOINTS[0]
-        for (let i in TEST_DATABASES) {
-            const dbName = TEST_DATABASES[i]
-            const response = await Utils.createDatabase(dbName, DID, CONTEXT_NAME, AUTH_TOKENS[endpoint], endpoint)
-            console.log(`createDatabase (${dbName}) on ${endpoint} response:`)
-            console.log(response)
-        }
+        it.only('can create the test databases on the first endpoint', async () => {
+            let endpoint = ENDPOINTS[0]
+            for (let i in TEST_DATABASES) {
+                const dbName = TEST_DATABASES[i]
+                console.log(`createDatabase (${dbName}) on ${endpoint}`)
+                const response = await Utils.createDatabase(dbName, DID, CONTEXT_NAME, AUTH_TOKENS[endpoint], endpoint)
+                assert.equal(response.data.status, 'success', 'database created')
+            }
+        })
 
         // Call `checkReplication(db1)` on all the endpoints (first database only)
         it.only('can initialise replication for one database via checkReplication()', async () => {
-            for (let i in ENDPOINTS) {
-                const endpoint = ENDPOINTS[i]
-                const result = await Utils.checkReplication(endpoint, AUTH_TOKENS[endpoint], TEST_DATABASES[0])
-                console.log(`checkReplication on ${endpoint} for ${TEST_DATABASES[0]}`)
-                console.log(result)
+            // @todo: fix code so endpoint doesn't create replication entries to itself
+            try {
+                for (let i in ENDPOINTS) {
+                    const endpoint = ENDPOINTS[i]
+                    const result = await Utils.checkReplication(endpoint, AUTH_TOKENS[endpoint], TEST_DATABASES[0])
+                    console.log(`${endpoint}: checkReplication on for ${TEST_DATABASES[0]}`)
+                    console.log(result.data)
+
+                    assert.equal(result.data.status, 'success', 'Check replication completed successfully')
+
+                    const conn = Utils.buildPouchDsn(ENDPOINT_DSN[endpoint], '_replicator')
+                    console.log(`${endpoint}: Connecting to ${endpoint}/${TEST_DATABASES[0]}`)
+
+                    let replicationEntry
+                    // Check replications are occurring to all the other endpoints (but not this endpoint)
+                    for (let e in ENDPOINTS) {
+                        const endpointCheckUri = ENDPOINTS[e]
+                        if (endpointCheckUri == endpoint) {
+                            continue
+                        }
 
-                const conn = Utils.buildPouchDsn(ENDPOINT_DSN[endpoint], '_replicator')
-                const replicationEntry = await conn.get(`${endpoint}/${TEST_DATABASE[0]}`)
-                console.log(`${endpoint} _replication entry for ${TEST_DATABASE[0]}`)
-                console.log(replicationEntry)
-                assert.ok(replicationEntry)
+                        const replicatorId = ComponentUtils.generateReplicatorHash(endpointCheckUri, DID, CONTEXT_NAME)
+                        const replicatorUsername = ComponentUtils.generateReplicaterUsername(endpoint)
+                        const dbHash = ComponentUtils.generateDatabaseName(DID, CONTEXT_NAME, TEST_DATABASES[0])
+                        console.log(`${endpoint}: (${endpointCheckUri}) Locating _replication entry for ${TEST_DATABASES[0]} (${replicatorId}-${dbHash})`)
+
+                        let replicationEntry
+                        try {
+                            replicationEntry = await conn.get(`${replicatorId}-${dbHash}`)
+                        } catch (err) {
+                            console.log('pouchdb connection error')
+                            console.log(err.message)
+                            assert.fail('Replication record not created')
+                        }
+
+                        assert.ok(replicationEntry)
+                        assert.ok(replicationEntry.source, `Have a source for ${endpointCheckUri}`)
+                        assert.ok(replicationEntry.target, `Have a target for ${endpointCheckUri}`)
+                        assert.equal(replicationEntry.source, `${ENDPOINTS_COUCH[endpoint]}/${dbHash}`, `Source URI is correct for ${endpointCheckUri}`)
+                        assert.equal(replicationEntry.target.url, `${ENDPOINTS_COUCH[endpointCheckUri]}/${dbHash}`, `Destination URI is correct for ${endpointCheckUri}`)
+
+                        assert.ok(replicationEntry.target.auth, `Have target.auth for ${endpointCheckUri}`)
+                        assert.ok(replicationEntry.target.auth.basic, `Have target.auth.basic for ${endpointCheckUri}`)
+                        assert.ok(replicationEntry.target.auth.basic.username, `Have target.auth.basic.username for ${endpointCheckUri}`)
+                        assert.ok(replicationEntry.target.auth.basic.password, `Have target.auth.basic.password for ${endpointCheckUri}`)
+                        assert.equal(replicationEntry.target.auth.basic.username, replicatorUsername, `Target username is correct for ${endpointCheckUri}`)
+                    }
+                }
+            } catch (err) {
+                console.log(err)
+                assert.fail('error')
             }
         })
 
         // Verify data saved to db1 is being replicated for all endpoints
         it('verify data is replicated for first database only', async () => {
             // Create three records
-            const endpoint0db1Connection = Utils.buildPouchDsn(ENDPOINT_DSN[endpoint], TEST_DATABASES[0])
+            const endpoint0db1Connection = Utils.buildPouchDsn(ENDPOINTS_COUCH[endpoint], TEST_DATABASES[0])
             await endpoint0db1Connection.put({db1endpoint1: 'world1'})
             await endpoint0db1Connection.put({db1endpoint1: 'world2'})
             await endpoint0db1Connection.put({db1endpoint1: 'world3'})
@@ -194,7 +254,21 @@ describe("Replication tests", function() {
     })
 
     this.afterAll(async () => {
-        // Delete all replication entries
+        // Clear replication related databases
+        for (let endpoint in ENDPOINT_DSN) {
+            const conn = new CouchDb({
+                url: ENDPOINT_DSN[endpoint],
+                requestDefaults: {
+                    rejectUnauthorized: process.env.DB_REJECT_UNAUTHORIZED_SSL.toLowerCase() !== "false"
+                }
+            })
+            await conn.db.destroy('_replicator')
+            await conn.db.create('_replicator')
+            await conn.db.destroy('verida_replicater_creds')
+            await conn.db.create('verida_replicater_creds')
+        }
+
+        // Delete created replication users
         // Delete all databases
     })
 })
\ No newline at end of file
diff --git a/test/utils.js b/test/utils.js
index 4b83bf02..8538239d 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -89,9 +89,7 @@ class Utils {
 
     async checkReplication(endpointUri, accessToken, databaseName) {
         const response = await Axios.post(`${endpointUri}/user/checkReplication`, {
-            databaseName,
-            did,
-            contextName
+            databaseName
         }, {
             headers: {
                 Authorization: `Bearer ${accessToken}`

From 33022a33a2bde6ed908f4a88010789163a31be1e Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Wed, 14 Dec 2022 21:14:11 +1030
Subject: [PATCH 16/28] Fix clearing databases. Add warnings.

---
 test/replication.js | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/test/replication.js b/test/replication.js
index f6a3cbb1..ee8aaefe 100644
--- a/test/replication.js
+++ b/test/replication.js
@@ -31,6 +31,15 @@ const TEST_DEVICE_ID = 'Device 1'
 
 const didClient = new DIDClient(CONFIG.DID_CLIENT_CONFIG)
 
+/**
+ * WARNING: ONLY RUN THIS TEST ON LOCALHOST
+ * 
+ * It deletes `_replicator` and `verida_replicator_creds` databases on all CouchDB
+ * endpoints upon completion of the tests.
+ * 
+ * This is necessary to reset the couch instnaces to a known state (empty)
+ */
+
 describe("Replication tests", function() {
     let DID, DID_ADDRESS, DID_PUBLIC_KEY, DID_PRIVATE_KEY, keyring, wallet, account, AUTH_TOKENS
 
@@ -253,8 +262,9 @@ describe("Replication tests", function() {
         it('verify database is deleted from all endpoints', () => {})
     })
 
+    // WARNING: This should never run on production!
     this.afterAll(async () => {
-        // Clear replication related databases
+        // Clear replication related databases to reset them for the next run
         for (let endpoint in ENDPOINT_DSN) {
             const conn = new CouchDb({
                 url: ENDPOINT_DSN[endpoint],
@@ -262,9 +272,14 @@ describe("Replication tests", function() {
                     rejectUnauthorized: process.env.DB_REJECT_UNAUTHORIZED_SSL.toLowerCase() !== "false"
                 }
             })
-            await conn.db.destroy('_replicator')
+
+            try {
+                await conn.db.destroy('_replicator')
+            } catch (err) {}
+            try {
+                await conn.db.destroy('verida_replicater_creds')
+            } catch (err) {}
             await conn.db.create('_replicator')
-            await conn.db.destroy('verida_replicater_creds')
             await conn.db.create('verida_replicater_creds')
         }
 

From 5077cadcecf1a91dfc8ed0cfc9e6bdf3891b4c85 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Wed, 14 Dec 2022 22:38:02 +1030
Subject: [PATCH 17/28] Ensure replicator user has did context role

---
 src/components/authManager.js | 42 ++++++++++++++++++++++-------------
 src/components/userManager.js |  2 +-
 src/controllers/auth.js       |  8 ++++++-
 3 files changed, 35 insertions(+), 17 deletions(-)

diff --git a/src/components/authManager.js b/src/components/authManager.js
index 32a4fc1f..c41e176a 100644
--- a/src/components/authManager.js
+++ b/src/components/authManager.js
@@ -434,7 +434,7 @@ class AuthManager {
         await tokenDb.createIndex(expiryIndex);
     }
 
-    async ensureReplicationCredentials(endpointUri, password) {
+    async ensureReplicationCredentials(endpointUri, password, replicaterRole) {
         console.log(`ensureReplicationCredentials(${endpointUri}, ${password})`)
         const username = Utils.generateReplicaterUsername(endpointUri)
         const id = `org.couchdb.user:${username}`
@@ -446,22 +446,34 @@ class AuthManager {
         try {
             user = await usersDb.get(id)
 
+            let userRequiresUpdate = false
+            if (!user.roles.indexOf(replicaterRole)) {
+                console.log(`User exists, but needs the replicatorRole added (${replicaterRole})`)
+                user.roles.push(replicaterRole)
+                userRequiresUpdate = true
+            }
+
             // User exists, check if we need to update the password
-            if (!password) {
-                console.log(`User exists, NOT updating password`)
-                // No password, so no need to update and just confirm the user exists
-                return "exists"
+            if (password) {
+                user.password = password
+                userRequiresUpdate = true
+                console.log(`User exists and password needs updating`)
             }
 
-            // User exists and we need to update the password
-            console.log(`User exists, updating password`)
-            user.password = password
-            try {
-                await dbManager._insertOrUpdate(usersDb, user, user._id)
-                return "updated"
-            } catch (err) {
-                console.log(err)
-                throw new Error(`Unable to update password: ${err.message}`)
+            if (userRequiresUpdate) {
+                // User exists and we need to update the password or roles
+                console.log(`User exists, updating password and / or roles`)
+                
+                try {
+                    await dbManager._insertOrUpdate(usersDb, user, user._id)
+                    return "updated"
+                } catch (err) {
+                    console.log(err)
+                    throw new Error(`Unable to update password: ${err.message}`)
+                }
+            } else {
+                // Nothing needed to change, so respond indicating the user exists
+                return "exists"
             }
         } catch (err) {
             if (err.error !== 'not_found') {
@@ -477,7 +489,7 @@ class AuthManager {
                     name: username,
                     password,
                     type: "user",
-                    roles: []
+                    roles: [replicaterRole]
                 }, id)
 
                 return "created"
diff --git a/src/components/userManager.js b/src/components/userManager.js
index fe8e6884..c0d4046b 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -161,7 +161,7 @@ class UserManager {
             // Fetch all databases for this context
             let userDatabases = await DbManager.getUserDatabases(did, contextName)
             databases = userDatabases.map(item => item.databaseName)
-            console.log(`${Utils.serverUri()}: Cecking ${databases.length}) databases`)
+            console.log(`${Utils.serverUri()}: Checking ${databases.length}) databases`)
         }
 
         //console.log('- databases', databases)
diff --git a/src/controllers/auth.js b/src/controllers/auth.js
index 7b75fa8a..a09d7674 100644
--- a/src/controllers/auth.js
+++ b/src/controllers/auth.js
@@ -214,6 +214,9 @@ class AuthController {
      * 2. `updated` (password updated)
      * 3. `exists` (user existed, but password unchanged)
      * 
+     * It's essential to ensure the user has the replication role that grants them
+     * access to all databases associated with a context
+     * 
      * @param {*} req 
      * @param {*} res 
      * @returns 
@@ -287,8 +290,11 @@ class AuthController {
             return Utils.error(res, `Unknown error: ${err.message}`)
         }
 
+        const didContextHash = Utils.generateDidContextHash(did, contextName)
+        const replicaterRole = `r${didContextHash}-replicater`
+
         try {
-            const result = await AuthManager.ensureReplicationCredentials(endpointUri, password)
+            const result = await AuthManager.ensureReplicationCredentials(endpointUri, password, replicaterRole)
             return Utils.signedResponse({
                 result
             }, res)

From 34d184909acc409cf8024e4e134cf280159f77fa Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Thu, 15 Dec 2022 08:21:16 +1030
Subject: [PATCH 18/28] Bug fixes

---
 src/components/authManager.js |  4 ++--
 src/components/userManager.js | 17 ++++++++++++++---
 src/controllers/user.js       | 18 +++++++++++++-----
 3 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/src/components/authManager.js b/src/components/authManager.js
index c41e176a..a7891fd0 100644
--- a/src/components/authManager.js
+++ b/src/components/authManager.js
@@ -438,7 +438,7 @@ class AuthManager {
         console.log(`ensureReplicationCredentials(${endpointUri}, ${password})`)
         const username = Utils.generateReplicaterUsername(endpointUri)
         const id = `org.couchdb.user:${username}`
-        console.log(`- username: ${username}`)
+        console.log(`- username: ${username} for ${endpointUri}`)
 
         const couch = Db.getCouch('internal');
         const usersDb = await couch.db.use('_users')
@@ -447,7 +447,7 @@ class AuthManager {
             user = await usersDb.get(id)
 
             let userRequiresUpdate = false
-            if (!user.roles.indexOf(replicaterRole)) {
+            if (user.roles.indexOf(replicaterRole) == -1) {
                 console.log(`User exists, but needs the replicatorRole added (${replicaterRole})`)
                 user.roles.push(replicaterRole)
                 userRequiresUpdate = true
diff --git a/src/components/userManager.js b/src/components/userManager.js
index c0d4046b..2d1505f8 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -110,9 +110,20 @@ class UserManager {
 
         for (let d in databases) {
             const database = databases[d]
-            const dbInfo = await DbManager.getUserDatabase(did, contextName, database.databaseName)
-            result.databases++
-            result.bytes += dbInfo.info.sizes.file
+            try {
+                const dbInfo = await DbManager.getUserDatabase(did, contextName, database.databaseName)
+                result.databases++
+                result.bytes += dbInfo.info.sizes.file
+            } catch (err) {
+                if (err.error == 'not_found') {
+                    console.log('not found', database.databaseName)
+                    // Database doesn't exist, so remove from the list of databases
+                    await DbManager.deleteUserDatabase(did, contextName, database.databaseName)
+                    continue
+                }
+                
+                throw err
+            }
         }
 
         const usage = result.bytes / parseInt(result.storageLimit)
diff --git a/src/controllers/user.js b/src/controllers/user.js
index 7eb487c2..bc23eb88 100644
--- a/src/controllers/user.js
+++ b/src/controllers/user.js
@@ -45,15 +45,23 @@ class UserController {
             });
         }
 
-        const userUsage = await UserManager.getUsage(did, contextName)
-        if (userUsage.usagePercent >= 100) {
-            return res.status(400).send({
+        try {
+            const userUsage = await UserManager.getUsage(did, contextName)
+            if (userUsage.usagePercent >= 100) {
+                return res.status(400).send({
+                    status: "fail",
+                    message: 'Storage limit reached'
+                });
+            }
+        } catch (err) {
+            return res.status(500).send({
                 status: "fail",
-                message: 'Storage limit reached'
-            });
+                message: err.message
+            })
         }
 
         const databaseHash = Utils.generateDatabaseName(did, contextName, databaseName)
+        console.log(`creating ${databaseHash}`)
 
         let success;
         try {

From ea1772a718ba8e413b5adf73f73a01098becfa34 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Thu, 15 Dec 2022 09:36:18 +1030
Subject: [PATCH 19/28] Try basic auth header

---
 src/components/userManager.js | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/components/userManager.js b/src/components/userManager.js
index 2d1505f8..59f8d7d0 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -201,17 +201,24 @@ class UserManager {
                         const { username, password, couchUri } = await AuthManager.fetchReplicaterCredentials(endpointUri, did, contextName)
                         console.log(`${Utils.serverUri()}: Located replication credentials for ${endpointUri} (${username}, ${password}, ${couchUri})`)
 
+                        const authBuffer = Buffer.from(`${REPLICATOR_CREDS[endpoint1].username}:${REPLICATOR_CREDS[endpoint1].password}`);
+                        const authBase64 = authBuffer.toString('base64')
+                        console.log(authBase64)
+
                         const replicationRecord = {
                             _id: `${replicatorId}-${dbHash}`,
                             source: `${Db.buildHost()}/${dbHash}`,
                             target: {
                                 url: `${couchUri}/${dbHash}`,
-                                auth: {
+                                headers: {
+                                    Authorization: `Basic ${authBase64}`
+                                }
+                                /*auth: {
                                     basic: {
                                         username,
                                         password
                                     }
-                                }
+                                }*/
                             },
                             create_target: true,
                             continous: true

From a06c32ccbfb6caffac26fe5397cd38659cd3ae87 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Thu, 15 Dec 2022 11:08:42 +1030
Subject: [PATCH 20/28] Update replication document creation

---
 src/components/userManager.js | 38 +++++++++++++++++++++++------------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/src/components/userManager.js b/src/components/userManager.js
index 59f8d7d0..d5da7b91 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -201,27 +201,39 @@ class UserManager {
                         const { username, password, couchUri } = await AuthManager.fetchReplicaterCredentials(endpointUri, did, contextName)
                         console.log(`${Utils.serverUri()}: Located replication credentials for ${endpointUri} (${username}, ${password}, ${couchUri})`)
 
-                        const authBuffer = Buffer.from(`${REPLICATOR_CREDS[endpoint1].username}:${REPLICATOR_CREDS[endpoint1].password}`);
-                        const authBase64 = authBuffer.toString('base64')
-                        console.log(authBase64)
+                        const remoteAuthBuffer = Buffer.from(`${username}:${password}`);
+                        const remoteAuthBase64 = remoteAuthBuffer.toString('base64')
+                        console.log(username, password, remoteAuthBase64)
+
+                        const localAuthBuffer = Buffer.from(`${process.env.DB_USER}:${process.env.DB_PASS}`);
+                        const localAuthBase64 = localAuthBuffer.toString('base64')
+                        console.log(username, password, localAuthBase64)
 
                         const replicationRecord = {
                             _id: `${replicatorId}-${dbHash}`,
-                            source: `${Db.buildHost()}/${dbHash}`,
+                            user_ctx: {
+                                name: process.env.DB_USER,
+                                roles: [
+                                    '_admin',
+                                    '_reader',
+                                    '_writer'
+                                ]
+                            },
+                            source: {
+                                url: `http://localhost:${process.env.DB_PORT_INTERNAL}/${dbHash}`,
+                                headers: {
+                                    Authorization: `Basic ${localAuthBase64}`
+                                }
+                            },
                             target: {
                                 url: `${couchUri}/${dbHash}`,
                                 headers: {
-                                    Authorization: `Basic ${authBase64}`
+                                    Authorization: `Basic ${remoteAuthBase64}`
                                 }
-                                /*auth: {
-                                    basic: {
-                                        username,
-                                        password
-                                    }
-                                }*/
                             },
-                            create_target: true,
-                            continous: true
+                            create_target: false,
+                            continous: true,
+                            owner: 'admin'
                         }
 
                         try {

From bd4088a5306702e38a36cd5d322e8e854783acd9 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Thu, 15 Dec 2022 12:17:50 +1030
Subject: [PATCH 21/28] Create new replicater user for replication instead of
 using admin

---
 sample.env                    |  2 ++
 src/components/authManager.js | 20 ++++++++++++++++++++
 src/components/userManager.js | 17 ++++++++---------
 3 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/sample.env b/sample.env
index 8f9a99a2..b93ce2e8 100644
--- a/sample.env
+++ b/sample.env
@@ -3,6 +3,8 @@ DID_NETWORK=testnet
 DID_CACHE_DURATION=3600
 DB_USER="admin"
 DB_PASS="admin"
+DB_REPLICATION_USER=""
+DB_REPLICATION_PASS=""
 
 # Internal hostname (Used internally by the storage node for accessing CouchDb).
 DB_PROTOCOL_INTERNAL="http"
diff --git a/src/components/authManager.js b/src/components/authManager.js
index a7891fd0..8863ceb3 100644
--- a/src/components/authManager.js
+++ b/src/components/authManager.js
@@ -407,6 +407,26 @@ class AuthManager {
             }
         }
 
+        // Create replication user with access to all databases
+        try {
+            const userDb = couch.db.user('_users')
+            const username = process.env.DB_REPLICATION_USER
+            const password = process.env.DB_REPLICATION_PASS
+            const id = `org.couchdb.user:${username}`
+            const userRow = {
+                _id: id,
+                name: username,
+                password,
+                type: "user",
+                roles: ['replicater-local']
+            }
+
+            await userDb.put(userRow, userRow._id)
+
+        } catch (err) {
+            console.log(err)
+        }
+
         try {
             await couch.db.create('_replicator')
         } catch (err) {
diff --git a/src/components/userManager.js b/src/components/userManager.js
index d5da7b91..9f94a2bc 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -180,6 +180,11 @@ class UserManager {
         // Ensure there is a replication entry for each
         const couch = Db.getCouch('internal')
         const replicationDb = couch.db.use('_replicator')
+        const didContextHash = Utils.generateDidContextHash(did, contextName)
+        const replicaterRole = `r${didContextHash}-replicater`
+        const localAuthBuffer = Buffer.from(`${process.env.DB_REPLICATION_USER}:${process.env.DB_REPLICATION_PASS}`);
+        const localAuthBase64 = localAuthBuffer.toString('base64')
+        console.log(username, password, localAuthBase64)
 
         for (let d in databases) {
             const dbName = databases[d]
@@ -205,18 +210,12 @@ class UserManager {
                         const remoteAuthBase64 = remoteAuthBuffer.toString('base64')
                         console.log(username, password, remoteAuthBase64)
 
-                        const localAuthBuffer = Buffer.from(`${process.env.DB_USER}:${process.env.DB_PASS}`);
-                        const localAuthBase64 = localAuthBuffer.toString('base64')
-                        console.log(username, password, localAuthBase64)
-
                         const replicationRecord = {
                             _id: `${replicatorId}-${dbHash}`,
                             user_ctx: {
-                                name: process.env.DB_USER,
+                                name: process.env.DB_REPLICATION_USER,
                                 roles: [
-                                    '_admin',
-                                    '_reader',
-                                    '_writer'
+                                    replicaterRole
                                 ]
                             },
                             source: {
@@ -232,7 +231,7 @@ class UserManager {
                                 }
                             },
                             create_target: false,
-                            continous: true,
+                            continuous: true,
                             owner: 'admin'
                         }
 

From 5b39538f343cdc56cdebf21893dd3f6137c100a1 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Thu, 15 Dec 2022 12:50:26 +1030
Subject: [PATCH 22/28] Bug fix replication

---
 src/components/authManager.js | 8 +++++---
 src/components/dbManager.js   | 2 +-
 src/components/userManager.js | 7 ++-----
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/components/authManager.js b/src/components/authManager.js
index 8863ceb3..7e07c381 100644
--- a/src/components/authManager.js
+++ b/src/components/authManager.js
@@ -409,7 +409,7 @@ class AuthManager {
 
         // Create replication user with access to all databases
         try {
-            const userDb = couch.db.user('_users')
+            const userDb = couch.db.use('_users')
             const username = process.env.DB_REPLICATION_USER
             const password = process.env.DB_REPLICATION_PASS
             const id = `org.couchdb.user:${username}`
@@ -421,10 +421,12 @@ class AuthManager {
                 roles: ['replicater-local']
             }
 
-            await userDb.put(userRow, userRow._id)
+            await userDb.insert(userRow, userRow._id)
 
         } catch (err) {
-            console.log(err)
+            if (err.error != 'conflict') {
+                throw err
+            }
         }
 
         try {
diff --git a/src/components/dbManager.js b/src/components/dbManager.js
index eba2a716..d300282c 100644
--- a/src/components/dbManager.js
+++ b/src/components/dbManager.js
@@ -254,7 +254,7 @@ class DbManager {
             members: {
                 // this grants read access to all members
                 names: dbMembers,
-                roles: [replicaterRole]
+                roles: [replicaterRole, 'replicater-local']
             }
         };
 
diff --git a/src/components/userManager.js b/src/components/userManager.js
index 9f94a2bc..87b931ab 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -184,7 +184,7 @@ class UserManager {
         const replicaterRole = `r${didContextHash}-replicater`
         const localAuthBuffer = Buffer.from(`${process.env.DB_REPLICATION_USER}:${process.env.DB_REPLICATION_PASS}`);
         const localAuthBase64 = localAuthBuffer.toString('base64')
-        console.log(username, password, localAuthBase64)
+        console.log(process.env.DB_REPLICATION_USER, process.env.DB_REPLICATION_PASS, localAuthBase64)
 
         for (let d in databases) {
             const dbName = databases[d]
@@ -213,10 +213,7 @@ class UserManager {
                         const replicationRecord = {
                             _id: `${replicatorId}-${dbHash}`,
                             user_ctx: {
-                                name: process.env.DB_REPLICATION_USER,
-                                roles: [
-                                    replicaterRole
-                                ]
+                                name: process.env.DB_REPLICATION_USER
                             },
                             source: {
                                 url: `http://localhost:${process.env.DB_PORT_INTERNAL}/${dbHash}`,

From f4f0f66ecafe93efa294ca65f0c215567aaa147f Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Thu, 15 Dec 2022 13:15:59 +1030
Subject: [PATCH 23/28] Update working tests

---
 test/replication.js | 165 +++++++++++++++++++++++++++++++++-----------
 1 file changed, 126 insertions(+), 39 deletions(-)

diff --git a/test/replication.js b/test/replication.js
index ee8aaefe..125331b3 100644
--- a/test/replication.js
+++ b/test/replication.js
@@ -31,6 +31,10 @@ const TEST_DEVICE_ID = 'Device 1'
 
 const didClient = new DIDClient(CONFIG.DID_CLIENT_CONFIG)
 
+function buildDsn(hostname, username, password) {
+    return hostname.replace('://', `://${username}:${password}@`)
+}
+
 /**
  * WARNING: ONLY RUN THIS TEST ON LOCALHOST
  * 
@@ -38,10 +42,14 @@ const didClient = new DIDClient(CONFIG.DID_CLIENT_CONFIG)
  * endpoints upon completion of the tests.
  * 
  * This is necessary to reset the couch instnaces to a known state (empty)
+ * 
+ * Note: CouchDB replicator interval must be set to 2 seconds (in couch config)
+ * to ensure replication is activated during these tests
  */
 
 describe("Replication tests", function() {
-    let DID, DID_ADDRESS, DID_PUBLIC_KEY, DID_PRIVATE_KEY, keyring, wallet, account, AUTH_TOKENS
+    let DID, DID_ADDRESS, DID_PUBLIC_KEY, DID_PRIVATE_KEY, keyring, wallet, account, AUTH_TOKENS, TEST_DATABASE_HASH
+    let REPLICATOR_CREDS = {}
 
     describe("Create test databases", async () => {
         this.timeout(200 * 1000)
@@ -56,6 +64,9 @@ describe("Replication tests", function() {
             DID_PRIVATE_KEY = wallet.privateKey
             keyring = new Keyring(wallet.mnemonic.phrase)
             await didClient.authenticate(DID_PRIVATE_KEY, 'web3', CONFIG.DID_CLIENT_CONFIG.web3Config, ENDPOINTS_DID)
+
+            TEST_DATABASE_HASH = TEST_DATABASES.map(item => ComponentUtils.generateDatabaseName(DID, CONTEXT_NAME, item))
+            console.log(TEST_DATABASE_HASH)
     
             console.log(DID_ADDRESS, DID, DID_PRIVATE_KEY, DID_PUBLIC_KEY, wallet.mnemonic.phrase)
     
@@ -128,13 +139,16 @@ describe("Replication tests", function() {
         })
 
         // Create the test databases on the first endpoint
-        it.only('can create the test databases on the first endpoint', async () => {
-            let endpoint = ENDPOINTS[0]
-            for (let i in TEST_DATABASES) {
-                const dbName = TEST_DATABASES[i]
-                console.log(`createDatabase (${dbName}) on ${endpoint}`)
-                const response = await Utils.createDatabase(dbName, DID, CONTEXT_NAME, AUTH_TOKENS[endpoint], endpoint)
-                assert.equal(response.data.status, 'success', 'database created')
+        it.only('can create the test databases on the endpoints', async () => {
+            for (let i in ENDPOINTS) {
+                let endpoint = ENDPOINTS[i]
+                for (let i in TEST_DATABASES) {
+                    const dbName = TEST_DATABASES[i]
+                    console.log(`createDatabase (${dbName}) on ${endpoint}`)
+                    const response = await Utils.createDatabase(dbName, DID, CONTEXT_NAME, AUTH_TOKENS[endpoint], endpoint)
+                    console.log('created')
+                    assert.equal(response.data.status, 'success', 'database created')
+                }
             }
         })
 
@@ -144,17 +158,26 @@ describe("Replication tests", function() {
             try {
                 for (let i in ENDPOINTS) {
                     const endpoint = ENDPOINTS[i]
+                    console.log(`${endpoint}: Calling checkReplication() on for ${TEST_DATABASES[0]}`)
                     const result = await Utils.checkReplication(endpoint, AUTH_TOKENS[endpoint], TEST_DATABASES[0])
-                    console.log(`${endpoint}: checkReplication on for ${TEST_DATABASES[0]}`)
-                    console.log(result.data)
-
                     assert.equal(result.data.status, 'success', 'Check replication completed successfully')
+                }
 
-                    const conn = Utils.buildPouchDsn(ENDPOINT_DSN[endpoint], '_replicator')
-                    console.log(`${endpoint}: Connecting to ${endpoint}/${TEST_DATABASES[0]}`)
+                // Sleep 5ms to have replication time to initialise
+                console.log('Sleeping so replication has time to do its thing...')
+                await Utils.sleep(5000)
 
-                    let replicationEntry
-                    // Check replications are occurring to all the other endpoints (but not this endpoint)
+                for (let i in ENDPOINTS) {
+                    const endpoint = ENDPOINTS[i]
+                    const couch = new CouchDb({
+                        url: ENDPOINT_DSN[endpoint],
+                        requestDefaults: {
+                            rejectUnauthorized: process.env.DB_REJECT_UNAUTHORIZED_SSL.toLowerCase() !== "false"
+                        }
+                    })
+                    const conn = couch.db.use('_replicator')
+
+                    // Check replications entries have been created for all the other endpoints (but not this endpoint)
                     for (let e in ENDPOINTS) {
                         const endpointCheckUri = ENDPOINTS[e]
                         if (endpointCheckUri == endpoint) {
@@ -175,17 +198,21 @@ describe("Replication tests", function() {
                             assert.fail('Replication record not created')
                         }
 
+                        console.log(replicationEntry)
+                        // Check info is accurate
                         assert.ok(replicationEntry)
                         assert.ok(replicationEntry.source, `Have a source for ${endpointCheckUri}`)
                         assert.ok(replicationEntry.target, `Have a target for ${endpointCheckUri}`)
-                        assert.equal(replicationEntry.source, `${ENDPOINTS_COUCH[endpoint]}/${dbHash}`, `Source URI is correct for ${endpointCheckUri}`)
+                        assert.equal(replicationEntry.source.url, `http://localhost:5984/${dbHash}`, `Source URI is correct for ${endpointCheckUri}`)
                         assert.equal(replicationEntry.target.url, `${ENDPOINTS_COUCH[endpointCheckUri]}/${dbHash}`, `Destination URI is correct for ${endpointCheckUri}`)
 
-                        assert.ok(replicationEntry.target.auth, `Have target.auth for ${endpointCheckUri}`)
-                        assert.ok(replicationEntry.target.auth.basic, `Have target.auth.basic for ${endpointCheckUri}`)
-                        assert.ok(replicationEntry.target.auth.basic.username, `Have target.auth.basic.username for ${endpointCheckUri}`)
-                        assert.ok(replicationEntry.target.auth.basic.password, `Have target.auth.basic.password for ${endpointCheckUri}`)
-                        assert.equal(replicationEntry.target.auth.basic.username, replicatorUsername, `Target username is correct for ${endpointCheckUri}`)
+                        REPLICATOR_CREDS[endpoint] = replicationEntry.target.headers
+
+                        const replicationResponse = await Axios.get(`${ENDPOINT_DSN[endpoint]}/_scheduler/docs/_replicator/${replicatorId}-${dbHash}`)
+                        assert.ok(replicationResponse, 'Have a replication job')
+
+                        const status = replicationResponse.data
+                        assert.ok(['pending', 'running'].indexOf(status.state) !== -1, 'Replication is active')
                     }
                 }
             } catch (err) {
@@ -194,33 +221,63 @@ describe("Replication tests", function() {
             }
         })
 
+        it.only('verify replication user can write to first database', async () => {
+            const endpoint0 = ENDPOINTS[0]
+            const endpoint1 = ENDPOINTS[1]
+
+            const couch = new CouchDb({
+                url: ENDPOINT_DSN[endpoint0],
+                requestDefaults: {
+                    headers: REPLICATOR_CREDS[endpoint1],
+                    rejectUnauthorized: process.env.DB_REJECT_UNAUTHORIZED_SSL.toLowerCase() !== "false"
+                }
+            })
+
+            console.log(`${endpoint0}: Creating three test records on ${TEST_DATABASES[0]} (${TEST_DATABASE_HASH[0]}) using credentials from ${endpoint1}`)
+            const endpoint0db1Connection = couch.db.use(TEST_DATABASE_HASH[0])
+            const result1 = await endpoint0db1Connection.insert({_id: '1', sourceEndpoint: endpoint0})
+            assert.ok(result1.ok, 'Record 1 saved')
+            const result2 = await endpoint0db1Connection.insert({_id: '2', sourceEndpoint: endpoint0})
+            assert.ok(result2.ok, 'Record 2 saved')
+            const result3 = await endpoint0db1Connection.insert({_id: '3', sourceEndpoint: endpoint0})
+            assert.ok(result3.ok, 'Record 3 saved')
+        })
+
         // Verify data saved to db1 is being replicated for all endpoints
-        it('verify data is replicated for first database only', async () => {
-            // Create three records
-            const endpoint0db1Connection = Utils.buildPouchDsn(ENDPOINTS_COUCH[endpoint], TEST_DATABASES[0])
-            await endpoint0db1Connection.put({db1endpoint1: 'world1'})
-            await endpoint0db1Connection.put({db1endpoint1: 'world2'})
-            await endpoint0db1Connection.put({db1endpoint1: 'world3'})
+        it.only('verify data is replicated on all endpoints for first database', async () => {
+            // Sleep 5ms to have replication time to do its thing
+            console.log('Sleeping so replication has time to do its thing...')
+            await Utils.sleep(5000)
 
             // Check the three records are correctly replicated on all the other databases
             for (let i in ENDPOINTS) {
-                if (i === 0) {
-                    // skip first database
+                if (i == 0) {
+                    // skip first endpoint
                     continue
                 }
 
-                const conn = Utils.buildPouchDsn(ENDPOINT_DSN[endpoint], TEST_DATABASES[0])
-                const docs = await conn.allDocs({include_docs: true})
-                console.log(`Endpoint ${endpoint} has docs:`)
+                const externalEndpoint = ENDPOINTS[i]
+                const couch = new CouchDb({
+                    url: ENDPOINT_DSN[externalEndpoint],
+                    requestDefaults: {
+                        headers: REPLICATOR_CREDS[externalEndpoint],
+                        rejectUnauthorized: process.env.DB_REJECT_UNAUTHORIZED_SSL.toLowerCase() !== "false"
+                    }
+                })
+                const conn = couch.db.use(TEST_DATABASE_HASH[0])
+
+                console.log(`${externalEndpoint}: Verifying endpoint has docs`)
+                const docs = await conn.list({include_docs: true})
+                console.log(`Endpoint ${externalEndpoint} has docs:`)
                 console.log(docs)
-                assert.equals(docs.rows.length, 3, 'Three rows returned')
+                assert.equal(docs.rows.length, 3, `Three rows returned from ${externalEndpoint}/${TEST_DATABASES[0]} (${TEST_DATABASE_HASH[0]})`)
             }
         })
 
         // Verify data saved to db2 is NOT replicated for all endpoints
         it('verify data is not replicated for second database', async () => {
             // Create three records on second database
-            const endpoint1db2Connection = Utils.buildPouchDsn(ENDPOINT_DSN[endpoint], TEST_DATABASES[1])
+            const endpoint1db2Connection = Utils.buildPouchDsn(ENDPOINT_DSN[endpoint], TEST_DATABASE_HASH[1])
             await endpoint1db2Connection.put({db2endpoint2: 'world1'})
             await endpoint1db2Connection.put({db2endpoint2: 'world2'})
             await endpoint1db2Connection.put({db2endpoint2: 'world3'})
@@ -264,7 +321,8 @@ describe("Replication tests", function() {
 
     // WARNING: This should never run on production!
     this.afterAll(async () => {
-        // Clear replication related databases to reset them for the next run
+        console.log('Destroying _replicator, verida_replicater_creds and test databases on ALL endpoints')
+
         for (let endpoint in ENDPOINT_DSN) {
             const conn = new CouchDb({
                 url: ENDPOINT_DSN[endpoint],
@@ -273,6 +331,7 @@ describe("Replication tests", function() {
                 }
             })
 
+            // Clear replication related databases to reset them for the next run
             try {
                 await conn.db.destroy('_replicator')
             } catch (err) {}
@@ -281,9 +340,37 @@ describe("Replication tests", function() {
             } catch (err) {}
             await conn.db.create('_replicator')
             await conn.db.create('verida_replicater_creds')
-        }
 
-        // Delete created replication users
-        // Delete all databases
+            // Delete test databases
+            for (let d in TEST_DATABASE_HASH) {
+                const databaseName = TEST_DATABASE_HASH[d]
+                try {
+                    console.log(`Destroying ${databaseName}`)
+                    await conn.db.destroy(databaseName)
+                } catch (err) {}
+            }
+
+            // Delete created replication users
+            for (let i in ENDPOINTS) {
+                const endpointExternal = ENDPOINTS[i]
+                if (endpointExternal == endpoint) {
+                    continue
+                }
+
+                try {
+                    const username = ComponentUtils.generateReplicaterUsername(endpointExternal)
+                    const users = conn.db.use('_users')
+                    console.log(`Deleting replication user ${username} for ${endpointExternal} from ${endpoint}`)
+                    const doc = await users.get(`org.couchdb.user:${username}`)
+                    await users.destroy(`org.couchdb.user:${username}`, doc._rev)
+                } catch (err) {
+                    if (err.error != 'not_found') {
+                        console.log(`Unable to delete user`)
+                        console.log(err)   
+                    }
+                }
+            }
+        }
     })
-})
\ No newline at end of file
+})
+

From 0e2000b64b1bb7ad79074d83d95a413548039d3d Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Thu, 15 Dec 2022 13:23:45 +1030
Subject: [PATCH 24/28] All tests written so far, pass!

---
 test/replication.js | 92 ++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 87 insertions(+), 5 deletions(-)

diff --git a/test/replication.js b/test/replication.js
index 125331b3..bf89d907 100644
--- a/test/replication.js
+++ b/test/replication.js
@@ -257,10 +257,16 @@ describe("Replication tests", function() {
                 }
 
                 const externalEndpoint = ENDPOINTS[i]
+
+                console.log(ENDPOINTS_COUCH[externalEndpoint])
+                console.log(REPLICATOR_CREDS[ENDPOINTS[0]])
+
+                // Connect to the external endpoint, using the credentials from the
+                // first endpoint to confirm it has access (plus admin user doesnt have access)
                 const couch = new CouchDb({
-                    url: ENDPOINT_DSN[externalEndpoint],
+                    url: ENDPOINTS_COUCH[externalEndpoint],
                     requestDefaults: {
-                        headers: REPLICATOR_CREDS[externalEndpoint],
+                        headers: REPLICATOR_CREDS[ENDPOINTS[0]],
                         rejectUnauthorized: process.env.DB_REJECT_UNAUTHORIZED_SSL.toLowerCase() !== "false"
                     }
                 })
@@ -268,9 +274,8 @@ describe("Replication tests", function() {
 
                 console.log(`${externalEndpoint}: Verifying endpoint has docs`)
                 const docs = await conn.list({include_docs: true})
-                console.log(`Endpoint ${externalEndpoint} has docs:`)
-                console.log(docs)
-                assert.equal(docs.rows.length, 3, `Three rows returned from ${externalEndpoint}/${TEST_DATABASES[0]} (${TEST_DATABASE_HASH[0]})`)
+                // Note: There is a design document, which is why the number is actually 4
+                assert.equal(docs.rows.length, 4, `Three rows returned from ${externalEndpoint}/${TEST_DATABASES[0]} (${TEST_DATABASE_HASH[0]})`)
             }
         })
 
@@ -374,3 +379,80 @@ describe("Replication tests", function() {
     })
 })
 
+
+/**
+ * 
+ * 
+ *
+{
+  "_id": "e9bf718dc5221dfac6fad45b1e5ef604b68ea80eaf1b92ca96c4ffa6753ab2eef-vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181",
+  "_rev": "2-389b0a07c2e17bb81a7a3b57a8bac939",
+  "source": {
+    "url": "http://192.168.68.117:5984/vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181",
+    "headers": {
+      "Authorization": "Basic cjNlNzBkZmI1YTFiYzNiMzk2NGVjMjQ3MTJjNzc1NGQwYTNjMDlmOWM1MzZjOTU3MjlmMjE5ODI4ZmU5ZjA2MTc6ZTY4ODU4ZWNkZDBlZmFkMjhlMGVjM2FhY2Q4M2IwNDBjYzg0ZDUxYWZkNGQ0OWNkYjgxNDVkYjQyZTA5NDNjZA=="
+    }
+  },
+  "target": {
+    "url": "http://192.168.68.118:5984/vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181",
+    "headers": {
+      "Authorization": "Basic cmYwODA0MjRmOTlmYjA3MjNmYzQzM2QxZWFjZDc1YTY0YWVkNzY2MDBlOWM5NWZlNjA5ZDQ2ZGNjZDkyZmM4M2U6NWE1MWUwN2I2Yjk0YWI2YWE2YjA4YzMzYmJlY2E0YzQwMTMxYzYxY2YyOGZhM2FhNTg2ZTdiNmIxZjE5ODRkMw=="
+    }
+  },
+  "create_target": true,
+  "continous": true,
+  "owner": "admin"
+}
+
+
+curl http://192.168.68.117:5984/vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181 \
+ -H "Authorization: Basic cjNlNzBkZmI1YTFiYzNiMzk2NGVjMjQ3MTJjNzc1NGQwYTNjMDlmOWM1MzZjOTU3MjlmMjE5ODI4ZmU5ZjA2MTc6ZTY4ODU4ZWNkZDBlZmFkMjhlMGVjM2FhY2Q4M2IwNDBjYzg0ZDUxYWZkNGQ0OWNkYjgxNDVkYjQyZTA5NDNjZA=="
+
+ curl http://192.168.68.118:5984/vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181 \
+ -H "Authorization: Basic cmYwODA0MjRmOTlmYjA3MjNmYzQzM2QxZWFjZDc1YTY0YWVkNzY2MDBlOWM5NWZlNjA5ZDQ2ZGNjZDkyZmM4M2U6NWE1MWUwN2I2Yjk0YWI2YWE2YjA4YzMzYmJlY2E0YzQwMTMxYzYxY2YyOGZhM2FhNTg2ZTdiNmIxZjE5ODRkMw=="
+
+curl http://localhost:5984/vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181 \
+ -H "Authorization: Basic cjNlNzBkZmI1YTFiYzNiMzk2NGVjMjQ3MTJjNzc1NGQwYTNjMDlmOWM1MzZjOTU3MjlmMjE5ODI4ZmU5ZjA2MTc6NTBiNjI3ZGRkZTIxYTI2NDU0OTQ3NjUzMTI5ZmJkNDRhZDU1M2I5YmFhODZhODRkMTRhYWY0ZWQ1YjdkMjAzOQ=="
+
+curl http://localhost:5984/vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181 \
+ -H "Authorization: Basic OThudzc4bjc5Y3c0Y3dxODkwOmNuNzA4OTRjNzQ4OTBjNDg5MDB3MXg5NDA4OQ=="
+
+{
+  "_id": "e9bf718dc5221dfac6fad45b1e5ef604b68ea80eaf1b92ca96c4ffa6753ab2eef-vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181",
+  "_rev": "3-a32a18ab1e5627a223718875a157caab",
+  "user_ctx": {
+    "name": "admin",
+    "roles": [
+      "_admin",
+      "_reader",
+      "_writer"
+    ]
+  },
+  "source": {
+    "url": "http://localhost:5984/vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181",
+    "headers": {
+      "Authorization": "Basic cjNlNzBkZmI1YTFiYzNiMzk2NGVjMjQ3MTJjNzc1NGQwYTNjMDlmOWM1MzZjOTU3MjlmMjE5ODI4ZmU5ZjA2MTc6ZTY4ODU4ZWNkZDBlZmFkMjhlMGVjM2FhY2Q4M2IwNDBjYzg0ZDUxYWZkNGQ0OWNkYjgxNDVkYjQyZTA5NDNjZA=="
+    }
+  },
+  "target": {
+    "url": "http://192.168.68.118:5984/vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181",
+    "headers": {
+      "Authorization": "Basic cmYwODA0MjRmOTlmYjA3MjNmYzQzM2QxZWFjZDc1YTY0YWVkNzY2MDBlOWM5NWZlNjA5ZDQ2ZGNjZDkyZmM4M2U6NWE1MWUwN2I2Yjk0YWI2YWE2YjA4YzMzYmJlY2E0YzQwMTMxYzYxY2YyOGZhM2FhNTg2ZTdiNmIxZjE5ODRkMw=="
+    }
+  },
+  "create_target": false,
+  "continuous": true,
+  "owner": "admin"
+}
+
+
+
+vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181
+
+ "username": "rf080424f99fb0723fc433d1eacd75a64aed76600e9c95fe609d46dccd92fc83e",
+  "password": "3f43e35bfdbf094461e415a956658825cc9e7d47e8bd4a7e6e746afee3064961",
+
+rf080424f99fb0723fc433d1eacd75a64aed76600e9c95fe609d46dccd92fc83e:3f43e35bfdbf094461e415a956658825cc9e7d47e8bd4a7e6e746afee3064961
+
+ * 
+ */
\ No newline at end of file

From eae007d5bcc1766067626f17ac966638a4af9896 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Thu, 15 Dec 2022 13:24:13 +1030
Subject: [PATCH 25/28] Remove commented test code

---
 test/replication.js | 80 +--------------------------------------------
 1 file changed, 1 insertion(+), 79 deletions(-)

diff --git a/test/replication.js b/test/replication.js
index bf89d907..b48cf01d 100644
--- a/test/replication.js
+++ b/test/replication.js
@@ -377,82 +377,4 @@ describe("Replication tests", function() {
             }
         }
     })
-})
-
-
-/**
- * 
- * 
- *
-{
-  "_id": "e9bf718dc5221dfac6fad45b1e5ef604b68ea80eaf1b92ca96c4ffa6753ab2eef-vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181",
-  "_rev": "2-389b0a07c2e17bb81a7a3b57a8bac939",
-  "source": {
-    "url": "http://192.168.68.117:5984/vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181",
-    "headers": {
-      "Authorization": "Basic cjNlNzBkZmI1YTFiYzNiMzk2NGVjMjQ3MTJjNzc1NGQwYTNjMDlmOWM1MzZjOTU3MjlmMjE5ODI4ZmU5ZjA2MTc6ZTY4ODU4ZWNkZDBlZmFkMjhlMGVjM2FhY2Q4M2IwNDBjYzg0ZDUxYWZkNGQ0OWNkYjgxNDVkYjQyZTA5NDNjZA=="
-    }
-  },
-  "target": {
-    "url": "http://192.168.68.118:5984/vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181",
-    "headers": {
-      "Authorization": "Basic cmYwODA0MjRmOTlmYjA3MjNmYzQzM2QxZWFjZDc1YTY0YWVkNzY2MDBlOWM5NWZlNjA5ZDQ2ZGNjZDkyZmM4M2U6NWE1MWUwN2I2Yjk0YWI2YWE2YjA4YzMzYmJlY2E0YzQwMTMxYzYxY2YyOGZhM2FhNTg2ZTdiNmIxZjE5ODRkMw=="
-    }
-  },
-  "create_target": true,
-  "continous": true,
-  "owner": "admin"
-}
-
-
-curl http://192.168.68.117:5984/vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181 \
- -H "Authorization: Basic cjNlNzBkZmI1YTFiYzNiMzk2NGVjMjQ3MTJjNzc1NGQwYTNjMDlmOWM1MzZjOTU3MjlmMjE5ODI4ZmU5ZjA2MTc6ZTY4ODU4ZWNkZDBlZmFkMjhlMGVjM2FhY2Q4M2IwNDBjYzg0ZDUxYWZkNGQ0OWNkYjgxNDVkYjQyZTA5NDNjZA=="
-
- curl http://192.168.68.118:5984/vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181 \
- -H "Authorization: Basic cmYwODA0MjRmOTlmYjA3MjNmYzQzM2QxZWFjZDc1YTY0YWVkNzY2MDBlOWM5NWZlNjA5ZDQ2ZGNjZDkyZmM4M2U6NWE1MWUwN2I2Yjk0YWI2YWE2YjA4YzMzYmJlY2E0YzQwMTMxYzYxY2YyOGZhM2FhNTg2ZTdiNmIxZjE5ODRkMw=="
-
-curl http://localhost:5984/vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181 \
- -H "Authorization: Basic cjNlNzBkZmI1YTFiYzNiMzk2NGVjMjQ3MTJjNzc1NGQwYTNjMDlmOWM1MzZjOTU3MjlmMjE5ODI4ZmU5ZjA2MTc6NTBiNjI3ZGRkZTIxYTI2NDU0OTQ3NjUzMTI5ZmJkNDRhZDU1M2I5YmFhODZhODRkMTRhYWY0ZWQ1YjdkMjAzOQ=="
-
-curl http://localhost:5984/vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181 \
- -H "Authorization: Basic OThudzc4bjc5Y3c0Y3dxODkwOmNuNzA4OTRjNzQ4OTBjNDg5MDB3MXg5NDA4OQ=="
-
-{
-  "_id": "e9bf718dc5221dfac6fad45b1e5ef604b68ea80eaf1b92ca96c4ffa6753ab2eef-vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181",
-  "_rev": "3-a32a18ab1e5627a223718875a157caab",
-  "user_ctx": {
-    "name": "admin",
-    "roles": [
-      "_admin",
-      "_reader",
-      "_writer"
-    ]
-  },
-  "source": {
-    "url": "http://localhost:5984/vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181",
-    "headers": {
-      "Authorization": "Basic cjNlNzBkZmI1YTFiYzNiMzk2NGVjMjQ3MTJjNzc1NGQwYTNjMDlmOWM1MzZjOTU3MjlmMjE5ODI4ZmU5ZjA2MTc6ZTY4ODU4ZWNkZDBlZmFkMjhlMGVjM2FhY2Q4M2IwNDBjYzg0ZDUxYWZkNGQ0OWNkYjgxNDVkYjQyZTA5NDNjZA=="
-    }
-  },
-  "target": {
-    "url": "http://192.168.68.118:5984/vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181",
-    "headers": {
-      "Authorization": "Basic cmYwODA0MjRmOTlmYjA3MjNmYzQzM2QxZWFjZDc1YTY0YWVkNzY2MDBlOWM5NWZlNjA5ZDQ2ZGNjZDkyZmM4M2U6NWE1MWUwN2I2Yjk0YWI2YWE2YjA4YzMzYmJlY2E0YzQwMTMxYzYxY2YyOGZhM2FhNTg2ZTdiNmIxZjE5ODRkMw=="
-    }
-  },
-  "create_target": false,
-  "continuous": true,
-  "owner": "admin"
-}
-
-
-
-vf19e24d684fb546dd9c9015336ce4005c12522f2cdedc066b2baf26b04351181
-
- "username": "rf080424f99fb0723fc433d1eacd75a64aed76600e9c95fe609d46dccd92fc83e",
-  "password": "3f43e35bfdbf094461e415a956658825cc9e7d47e8bd4a7e6e746afee3064961",
-
-rf080424f99fb0723fc433d1eacd75a64aed76600e9c95fe609d46dccd92fc83e:3f43e35bfdbf094461e415a956658825cc9e7d47e8bd4a7e6e746afee3064961
-
- * 
- */
\ No newline at end of file
+})
\ No newline at end of file

From 2ef69ff9e684fe2a360040c6054115c45c069568 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Thu, 15 Dec 2022 17:09:28 +1030
Subject: [PATCH 26/28] Code cleanup

---
 src/components/userManager.js | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/components/userManager.js b/src/components/userManager.js
index 87b931ab..b60acef6 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -165,7 +165,7 @@ class UserManager {
 
         let databases = []
         if (databaseName) {
-            console.log(`${Utils.serverUri()}: Only checking ${databaseName})`)
+            console.log(`${Utils.serverUri()}: Only checking ${databaseName}`)
             // Only check a single database
             databases.push(databaseName)
         } else {
@@ -175,13 +175,10 @@ class UserManager {
             console.log(`${Utils.serverUri()}: Checking ${databases.length}) databases`)
         }
 
-        //console.log('- databases', databases)
-
         // Ensure there is a replication entry for each
         const couch = Db.getCouch('internal')
         const replicationDb = couch.db.use('_replicator')
-        const didContextHash = Utils.generateDidContextHash(did, contextName)
-        const replicaterRole = `r${didContextHash}-replicater`
+
         const localAuthBuffer = Buffer.from(`${process.env.DB_REPLICATION_USER}:${process.env.DB_REPLICATION_PASS}`);
         const localAuthBase64 = localAuthBuffer.toString('base64')
         console.log(process.env.DB_REPLICATION_USER, process.env.DB_REPLICATION_PASS, localAuthBase64)
@@ -196,7 +193,7 @@ class UserManager {
                 let record
                 try {
                     record = await replicationDb.get(`${replicatorId}-${dbHash}`)
-                    console.log(`${Utils.serverUri()}: Located replication record for ${endpointUri} (${replicatorId})`)
+                    console.log(`${Utils.serverUri()}: Located replication record for ${dbHash} on ${endpointUri} (${replicatorId})`)
                 } catch (err) {
                     if (err.message == 'missing' || err.reason == 'deleted') {
                         console.log(`${Utils.serverUri()}: Replication record for ${endpointUri} is missing... creating.`)

From f5190905feafe851fe96e40e0c391f36ea0cb8d3 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Thu, 15 Dec 2022 17:26:01 +1030
Subject: [PATCH 27/28] Code cleanup. Replication test cleanup. Verify
 timestamp when fetching replication creds.

---
 src/components/authManager.js |  36 ++---
 src/components/dbManager.js   |   2 -
 src/components/userManager.js |  27 ++--
 src/controllers/auth.js       |   8 +-
 src/controllers/user.js       |   2 -
 test/replication.js           | 262 ++++++++++++++++++++++++----------
 6 files changed, 213 insertions(+), 124 deletions(-)

diff --git a/src/components/authManager.js b/src/components/authManager.js
index 7e07c381..6ed30ee9 100644
--- a/src/components/authManager.js
+++ b/src/components/authManager.js
@@ -95,7 +95,6 @@ class AuthManager {
 
             return true
         } catch (err) {
-            // @todo: Log error
             // Likely unable to resolve DID or invalid signature
             console.info(`Unable to resolve DID or invalid signature: ${err.message}`)
             return false
@@ -120,9 +119,6 @@ class AuthManager {
                 }
 
                 didDocument = await didClient.get(did)
-                
-                // @todo: check if the doc was auto-generated or actually
-                // stored on chain? if not on chain, don't cache
                 if (didDocument) {
                     const { DID_CACHE_DURATION }  = process.env
                     mcache.put(cacheKey, didDocument, DID_CACHE_DURATION * 1000)
@@ -131,7 +127,6 @@ class AuthManager {
 
             return didDocument
         } catch (err) {
-            // @todo: Log error
             // Likely unable to resolve DID or invalid signature
             console.info(`Unable to resolve DID`)
             return false
@@ -457,10 +452,8 @@ class AuthManager {
     }
 
     async ensureReplicationCredentials(endpointUri, password, replicaterRole) {
-        console.log(`ensureReplicationCredentials(${endpointUri}, ${password})`)
         const username = Utils.generateReplicaterUsername(endpointUri)
         const id = `org.couchdb.user:${username}`
-        console.log(`- username: ${username} for ${endpointUri}`)
 
         const couch = Db.getCouch('internal');
         const usersDb = await couch.db.use('_users')
@@ -470,7 +463,7 @@ class AuthManager {
 
             let userRequiresUpdate = false
             if (user.roles.indexOf(replicaterRole) == -1) {
-                console.log(`User exists, but needs the replicatorRole added (${replicaterRole})`)
+                //console.log(`User exists, but needs the replicatorRole added (${replicaterRole})`)
                 user.roles.push(replicaterRole)
                 userRequiresUpdate = true
             }
@@ -479,18 +472,17 @@ class AuthManager {
             if (password) {
                 user.password = password
                 userRequiresUpdate = true
-                console.log(`User exists and password needs updating`)
+                //console.log(`User exists and password needs updating`)
             }
 
             if (userRequiresUpdate) {
                 // User exists and we need to update the password or roles
-                console.log(`User exists, updating password and / or roles`)
+                //console.log(`User exists, updating password and / or roles`)
                 
                 try {
                     await dbManager._insertOrUpdate(usersDb, user, user._id)
                     return "updated"
                 } catch (err) {
-                    console.log(err)
                     throw new Error(`Unable to update password: ${err.message}`)
                 }
             } else {
@@ -504,8 +496,7 @@ class AuthManager {
 
             // Need to create the user
             try {
-                console.log('replication user didnt exist, so creating')
-                console.log(id)
+                //console.log('Replication user didnt exist, so creating')
                 await dbManager._insertOrUpdate(usersDb, {
                     _id: id,
                     name: username,
@@ -516,7 +507,6 @@ class AuthManager {
 
                 return "created"
             } catch (err) {
-                console.log(err)
                 throw new Error(`Unable to create replication user: ${err.message}`)
             }
         }
@@ -528,22 +518,21 @@ class AuthManager {
         const replicaterCredsDb = await couch.db.use(process.env.DB_REPLICATER_CREDS)
         const replicaterHash = Utils.generateReplicatorHash(endpointUri, did, contextName)
         
-        console.log(`${Utils.serverUri()}: Fetching credentials for ${endpointUri}`)
+        //console.log(`${Utils.serverUri()}: Fetching credentials for ${endpointUri}`)
 
         let creds
         try {
             creds = await replicaterCredsDb.get(replicaterHash)
-            console.log(`${Utils.serverUri()}: Located credentials for ${endpointUri}`)
+            //console.log(`${Utils.serverUri()}: Located credentials for ${endpointUri}`)
         } catch (err) {
             // If credentials aren't found, that's okay we will create them below
             if (err.error != 'not_found') {
-                console.log('rethrowing')
                 throw err
             }
         }
 
         if (!creds) {
-            console.log(`${Utils.serverUri()}: No credentials found for ${endpointUri}... creating.`)
+            //console.log(`${Utils.serverUri()}: No credentials found for ${endpointUri}... creating.`)
             const timestampMinutes = Math.floor(Date.now() / 1000 / 60)
 
             // Generate a random password
@@ -563,10 +552,10 @@ class AuthManager {
             requestBody.signature = signature
 
             // Fetch credentials from the endpointUri
-            console.log(`${Utils.serverUri()}: Requesting the creation of credentials for ${endpointUri}`)
+            //console.log(`${Utils.serverUri()}: Requesting the creation of credentials for ${endpointUri}`)
             try {
                 await Axios.post(`${endpointUri}/auth/replicationCreds`, requestBody)
-                console.log(`${Utils.serverUri()}: Credentials generated for ${endpointUri}`)
+                //console.log(`${Utils.serverUri()}: Credentials generated for ${endpointUri}`)
             } catch (err) {
                 if (err.response) {
                     throw Error(`Unable to obtain credentials from ${endpointUri} (${err.response.data.message})`)
@@ -579,7 +568,7 @@ class AuthManager {
             try {
                 const statusResponse = await Axios.get(`${endpointUri}/status`)
                 couchUri = statusResponse.data.results.couchUri
-                console.log(`${Utils.serverUri()}: Status fetched ${endpointUri} with CouchURI: ${couchUri}`)
+                //console.log(`${Utils.serverUri()}: Status fetched ${endpointUri} with CouchURI: ${couchUri}`)
             } catch (err) {
                 if (err.response) {
                     throw Error(`Unable to obtain credentials from ${endpointUri} (${err.response.data.message})`)
@@ -598,9 +587,8 @@ class AuthManager {
 
             try {
                 await dbManager._insertOrUpdate(replicaterCredsDb, creds, creds._id)
-                console.log(`${Utils.serverUri()}: Credentials saved for ${endpointUri}`)
+                //console.log(`${Utils.serverUri()}: Credentials saved for ${endpointUri}`)
             } catch (err) {
-                console.log(err)
                 throw new Error(`Unable to save replicater password : ${err.message} (${endpointUri})`)
             }
         }
@@ -612,7 +600,7 @@ class AuthManager {
         }
     }
 
-    // @todo: garbage collection
+    // Garbage collection of refresh tokens
     async gc() {
         const GC_PERCENT = process.env.GC_PERCENT
         const random = Math.random()
diff --git a/src/components/dbManager.js b/src/components/dbManager.js
index d300282c..d1ecd814 100644
--- a/src/components/dbManager.js
+++ b/src/components/dbManager.js
@@ -221,8 +221,6 @@ class DbManager {
         let readUsers = [owner];
         let deleteUsers = [owner];
 
-        // @todo Support modifying user lists after db has been created
-
         switch (permissions.write) {
             case "users":
                 writeUsers = _.union(writeUsers, Utils.didsToUsernames(permissions.writeList, contextName));
diff --git a/src/components/userManager.js b/src/components/userManager.js
index b60acef6..eb5b61f7 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -116,7 +116,6 @@ class UserManager {
                 result.bytes += dbInfo.info.sizes.file
             } catch (err) {
                 if (err.error == 'not_found') {
-                    console.log('not found', database.databaseName)
                     // Database doesn't exist, so remove from the list of databases
                     await DbManager.deleteUserDatabase(did, contextName, database.databaseName)
                     continue
@@ -145,7 +144,7 @@ class UserManager {
      * @param {*} databaseName (optional) If not specified, checks all databases
      */
     async checkReplication(did, contextName, databaseName) {
-        console.log(`${Utils.serverUri()}: checkReplication(${did}, ${contextName}, ${databaseName})`)
+        //console.log(`${Utils.serverUri()}: checkReplication(${did}, ${contextName}, ${databaseName})`)
         // Lookup DID document and get list of endpoints for this context
         const didDocument = await AuthManager.getDidDocument(did)
         const didService = didDocument.locateServiceEndpoint(contextName, 'database')
@@ -160,19 +159,16 @@ class UserManager {
         // Remove this endpoint from the list of endpoints to check
         endpoints.splice(endpointIndex, 1)
 
-        console.log(`- endpoints:`)
-        console.log(endpoints)
-
         let databases = []
         if (databaseName) {
-            console.log(`${Utils.serverUri()}: Only checking ${databaseName}`)
+            //console.log(`${Utils.serverUri()}: Only checking ${databaseName}`)
             // Only check a single database
             databases.push(databaseName)
         } else {
             // Fetch all databases for this context
             let userDatabases = await DbManager.getUserDatabases(did, contextName)
             databases = userDatabases.map(item => item.databaseName)
-            console.log(`${Utils.serverUri()}: Checking ${databases.length}) databases`)
+            //console.log(`${Utils.serverUri()}: Checking ${databases.length}) databases`)
         }
 
         // Ensure there is a replication entry for each
@@ -181,7 +177,6 @@ class UserManager {
 
         const localAuthBuffer = Buffer.from(`${process.env.DB_REPLICATION_USER}:${process.env.DB_REPLICATION_PASS}`);
         const localAuthBase64 = localAuthBuffer.toString('base64')
-        console.log(process.env.DB_REPLICATION_USER, process.env.DB_REPLICATION_PASS, localAuthBase64)
 
         for (let d in databases) {
             const dbName = databases[d]
@@ -193,19 +188,18 @@ class UserManager {
                 let record
                 try {
                     record = await replicationDb.get(`${replicatorId}-${dbHash}`)
-                    console.log(`${Utils.serverUri()}: Located replication record for ${dbHash} on ${endpointUri} (${replicatorId})`)
+                    //console.log(`${Utils.serverUri()}: Located replication record for ${dbHash} on ${endpointUri} (${replicatorId})`)
                 } catch (err) {
                     if (err.message == 'missing' || err.reason == 'deleted') {
-                        console.log(`${Utils.serverUri()}: Replication record for ${endpointUri} is missing... creating.`)
+                        //console.log(`${Utils.serverUri()}: Replication record for ${endpointUri} is missing... creating.`)
                         // No record, so create it
                         // Check if we have credentials
                         // No credentials? Ask for them from the endpoint
                         const { username, password, couchUri } = await AuthManager.fetchReplicaterCredentials(endpointUri, did, contextName)
-                        console.log(`${Utils.serverUri()}: Located replication credentials for ${endpointUri} (${username}, ${password}, ${couchUri})`)
+                        //console.log(`${Utils.serverUri()}: Located replication credentials for ${endpointUri} (${username}, ${password}, ${couchUri})`)
 
                         const remoteAuthBuffer = Buffer.from(`${username}:${password}`);
                         const remoteAuthBase64 = remoteAuthBuffer.toString('base64')
-                        console.log(username, password, remoteAuthBase64)
 
                         const replicationRecord = {
                             _id: `${replicatorId}-${dbHash}`,
@@ -231,22 +225,21 @@ class UserManager {
 
                         try {
                             await DbManager._insertOrUpdate(replicationDb, replicationRecord, replicationRecord._id)
-                            console.log(`${Utils.serverUri()}: Saved replication entry for ${endpointUri} (${replicatorId})`)
+                            //console.log(`${Utils.serverUri()}: Saved replication entry for ${endpointUri} (${replicatorId})`)
                         } catch (err) {
-                            console.log(`${Utils.serverUri()}: Error saving replication entry for ${endpointUri} (${replicatorId})`)
-                            console.log(err)
+                            //console.log(`${Utils.serverUri()}: Error saving replication entry for ${endpointUri} (${replicatorId}): ${err.message}`)
                             throw new Error(`Unable to create replication entry: ${err.message}`)
                         }
                     }
                     else {
-                        console.log(`${Utils.serverUri()}: Unknown error fetching replication entry for ${endpointUri} (${replicatorId})`)
-                        console.log(err)
+                        //console.log(`${Utils.serverUri()}: Unknown error fetching replication entry for ${endpointUri} (${replicatorId}): ${err.message}`)
                         throw err
                     }
                 }
             }
         }
 
+        // @todo: Find any replication errors and handle them nicely
         // @todo: Remove any replication entries for deleted databases
     }
 
diff --git a/src/controllers/auth.js b/src/controllers/auth.js
index a09d7674..4dbb49ff 100644
--- a/src/controllers/auth.js
+++ b/src/controllers/auth.js
@@ -240,7 +240,12 @@ class AuthController {
             return Utils.error(res, 'Timestamp not specified')
         }
 
-        // @todo: verify timestampMinutes is within range
+        // Verify timestampMinutes is within two minutes of now
+        const currentTimestampMinutes = Math.floor(Date.now() / 1000 / 60)
+        const diff = currentTimestampMinutes - timestampMinutes
+        if (diff > 2 || diff < -2) {
+            return Utils.error(res, `Timestamp is out of range ${diff}`)
+        }
 
         if (!did) {
             return Utils.error(res, 'DID not specified')
@@ -272,7 +277,6 @@ class AuthController {
         let endpointPublicKey
         try {
             const response = await Axios.get(`${endpointUri}/status`)
-            console.log(response.data)
 
             endpointPublicKey = response.data.results.publicKey
             const params = {
diff --git a/src/controllers/user.js b/src/controllers/user.js
index bc23eb88..abe4eca4 100644
--- a/src/controllers/user.js
+++ b/src/controllers/user.js
@@ -61,7 +61,6 @@ class UserController {
         }
 
         const databaseHash = Utils.generateDatabaseName(did, contextName, databaseName)
-        console.log(`creating ${databaseHash}`)
 
         let success;
         try {
@@ -161,7 +160,6 @@ class UserController {
     }
 
     // Update permissions on a user's database
-    // @todo: database name should be in plain text, then hashed
     async updateDatabase(req, res) {
         const username = req.tokenData.username
         const did = req.tokenData.did
diff --git a/test/replication.js b/test/replication.js
index b48cf01d..2a9b5f00 100644
--- a/test/replication.js
+++ b/test/replication.js
@@ -14,8 +14,20 @@ dotenv.config();
 import Utils from './utils'
 import CONFIG from './config'
 
+// Enable verbose logging of what the tests are doing
+const LOGGING_ENABLED = false
+
+// Use a pre-built mnemonic where the first private key is a Verida DID private key
+const MNEMONIC = 'pave online install gift glimpse purpose truth loan arm wing west option'
+
+// Context name to use for the tests
 const CONTEXT_NAME = 'Verida Test: Storage Node Replication'
-// @todo: use three endpoints
+
+// Endpoints to use for testing
+// WARNING!!!
+// Only ever use local network endpoints.
+// These tests will delete the `_replicator` database and `verida_replicator_creds` on
+// ALL endpoints when they are complete.
 const ENDPOINT_DSN = {
     'http://192.168.68.117:5000': 'http://admin:admin@192.168.68.117:5984',
     'http://192.168.68.118:5000': 'http://admin:admin@192.168.68.118:5984',
@@ -31,8 +43,20 @@ const TEST_DEVICE_ID = 'Device 1'
 
 const didClient = new DIDClient(CONFIG.DID_CLIENT_CONFIG)
 
-function buildDsn(hostname, username, password) {
-    return hostname.replace('://', `://${username}:${password}@`)
+function buildEndpointConnection(externalEndpoint, endpointCreds) {
+    return new CouchDb({
+        url: externalEndpoint,
+        requestDefaults: {
+            headers: endpointCreds,
+            rejectUnauthorized: process.env.DB_REJECT_UNAUTHORIZED_SSL.toLowerCase() !== "false"
+        }
+    })
+}
+
+function log(output) {
+    if (LOGGING_ENABLED) {
+        console.log(output)
+    }
 }
 
 /**
@@ -56,8 +80,12 @@ describe("Replication tests", function() {
 
         this.beforeAll(async () => {
             // Create a new VDA private key
-            //wallet = ethers.Wallet.createRandom()
-            wallet = ethers.Wallet.fromMnemonic('pave online install gift glimpse purpose truth loan arm wing west option')
+            if (MNEMONIC) {
+                wallet = ethers.Wallet.fromMnemonic(MNEMONIC)
+            } else {
+                wallet = ethers.Wallet.createRandom()
+            }
+            
             DID_ADDRESS = wallet.address
             DID = `did:vda:testnet:${DID_ADDRESS}`
             DID_PUBLIC_KEY = wallet.publicKey
@@ -66,9 +94,8 @@ describe("Replication tests", function() {
             await didClient.authenticate(DID_PRIVATE_KEY, 'web3', CONFIG.DID_CLIENT_CONFIG.web3Config, ENDPOINTS_DID)
 
             TEST_DATABASE_HASH = TEST_DATABASES.map(item => ComponentUtils.generateDatabaseName(DID, CONTEXT_NAME, item))
-            console.log(TEST_DATABASE_HASH)
     
-            console.log(DID_ADDRESS, DID, DID_PRIVATE_KEY, DID_PUBLIC_KEY, wallet.mnemonic.phrase)
+            log(DID_ADDRESS, DID, DID_PRIVATE_KEY, DID_PUBLIC_KEY, wallet.mnemonic.phrase)
     
             // Create a new VDA account using our test endpoints
             account = new AutoAccount({
@@ -105,15 +132,15 @@ describe("Replication tests", function() {
             try {
                 const endpointResponses = await didClient.save(doc)
             } catch (err) {
-                console.log(err)
-                console.log(didClient.getLastEndpointErrors())
+                log(err)
+                log(didClient.getLastEndpointErrors())
             }
     
             // Fetch an auth token for each server
             AUTH_TOKENS = {}
             for (let i in ENDPOINTS) {
                 const endpoint = ENDPOINTS[i]
-                console.log(`Authenticating with ${endpoint}`)
+                log(`Authenticating with ${endpoint}`)
                 const authJwtResult = await Axios.post(`${endpoint}/auth/generateAuthJwt`, {
                     did: DID,
                     contextName: CONTEXT_NAME
@@ -133,9 +160,6 @@ describe("Replication tests", function() {
                 })
                 AUTH_TOKENS[endpoint] = authenticateResponse.data.accessToken
             }
-    
-            console.log(`auth tokens for the endpoints:`)
-            console.log(AUTH_TOKENS)
         })
 
         // Create the test databases on the first endpoint
@@ -144,9 +168,7 @@ describe("Replication tests", function() {
                 let endpoint = ENDPOINTS[i]
                 for (let i in TEST_DATABASES) {
                     const dbName = TEST_DATABASES[i]
-                    console.log(`createDatabase (${dbName}) on ${endpoint}`)
                     const response = await Utils.createDatabase(dbName, DID, CONTEXT_NAME, AUTH_TOKENS[endpoint], endpoint)
-                    console.log('created')
                     assert.equal(response.data.status, 'success', 'database created')
                 }
             }
@@ -158,13 +180,13 @@ describe("Replication tests", function() {
             try {
                 for (let i in ENDPOINTS) {
                     const endpoint = ENDPOINTS[i]
-                    console.log(`${endpoint}: Calling checkReplication() on for ${TEST_DATABASES[0]}`)
+                    log(`${endpoint}: Calling checkReplication() on for ${TEST_DATABASES[0]}`)
                     const result = await Utils.checkReplication(endpoint, AUTH_TOKENS[endpoint], TEST_DATABASES[0])
                     assert.equal(result.data.status, 'success', 'Check replication completed successfully')
                 }
 
                 // Sleep 5ms to have replication time to initialise
-                console.log('Sleeping so replication has time to do its thing...')
+                log('Sleeping so replication has time to do its thing...')
                 await Utils.sleep(5000)
 
                 for (let i in ENDPOINTS) {
@@ -185,20 +207,18 @@ describe("Replication tests", function() {
                         }
 
                         const replicatorId = ComponentUtils.generateReplicatorHash(endpointCheckUri, DID, CONTEXT_NAME)
-                        const replicatorUsername = ComponentUtils.generateReplicaterUsername(endpoint)
                         const dbHash = ComponentUtils.generateDatabaseName(DID, CONTEXT_NAME, TEST_DATABASES[0])
-                        console.log(`${endpoint}: (${endpointCheckUri}) Locating _replication entry for ${TEST_DATABASES[0]} (${replicatorId}-${dbHash})`)
+                        log(`${endpoint}: (${endpointCheckUri}) Locating _replication entry for ${TEST_DATABASES[0]} (${replicatorId}-${dbHash})`)
 
                         let replicationEntry
                         try {
                             replicationEntry = await conn.get(`${replicatorId}-${dbHash}`)
                         } catch (err) {
-                            console.log('pouchdb connection error')
-                            console.log(err.message)
+                            log('pouchdb connection error')
+                            log(err.message)
                             assert.fail('Replication record not created')
                         }
 
-                        console.log(replicationEntry)
                         // Check info is accurate
                         assert.ok(replicationEntry)
                         assert.ok(replicationEntry.source, `Have a source for ${endpointCheckUri}`)
@@ -216,7 +236,7 @@ describe("Replication tests", function() {
                     }
                 }
             } catch (err) {
-                console.log(err)
+                log(err)
                 assert.fail('error')
             }
         })
@@ -225,15 +245,9 @@ describe("Replication tests", function() {
             const endpoint0 = ENDPOINTS[0]
             const endpoint1 = ENDPOINTS[1]
 
-            const couch = new CouchDb({
-                url: ENDPOINT_DSN[endpoint0],
-                requestDefaults: {
-                    headers: REPLICATOR_CREDS[endpoint1],
-                    rejectUnauthorized: process.env.DB_REJECT_UNAUTHORIZED_SSL.toLowerCase() !== "false"
-                }
-            })
+            const couch = buildEndpointConnection(ENDPOINT_DSN[endpoint0], REPLICATOR_CREDS[endpoint1])
 
-            console.log(`${endpoint0}: Creating three test records on ${TEST_DATABASES[0]} (${TEST_DATABASE_HASH[0]}) using credentials from ${endpoint1}`)
+            log(`${endpoint0}: Creating three test records on ${TEST_DATABASES[0]} (${TEST_DATABASE_HASH[0]}) using credentials from ${endpoint1}`)
             const endpoint0db1Connection = couch.db.use(TEST_DATABASE_HASH[0])
             const result1 = await endpoint0db1Connection.insert({_id: '1', sourceEndpoint: endpoint0})
             assert.ok(result1.ok, 'Record 1 saved')
@@ -246,7 +260,7 @@ describe("Replication tests", function() {
         // Verify data saved to db1 is being replicated for all endpoints
         it.only('verify data is replicated on all endpoints for first database', async () => {
             // Sleep 5ms to have replication time to do its thing
-            console.log('Sleeping so replication has time to do its thing...')
+            log('Sleeping so replication has time to do its thing...')
             await Utils.sleep(5000)
 
             // Check the three records are correctly replicated on all the other databases
@@ -258,75 +272,169 @@ describe("Replication tests", function() {
 
                 const externalEndpoint = ENDPOINTS[i]
 
-                console.log(ENDPOINTS_COUCH[externalEndpoint])
-                console.log(REPLICATOR_CREDS[ENDPOINTS[0]])
-
                 // Connect to the external endpoint, using the credentials from the
                 // first endpoint to confirm it has access (plus admin user doesnt have access)
-                const couch = new CouchDb({
-                    url: ENDPOINTS_COUCH[externalEndpoint],
-                    requestDefaults: {
-                        headers: REPLICATOR_CREDS[ENDPOINTS[0]],
-                        rejectUnauthorized: process.env.DB_REJECT_UNAUTHORIZED_SSL.toLowerCase() !== "false"
-                    }
-                })
+                const couch = buildEndpointConnection(ENDPOINTS_COUCH[externalEndpoint], REPLICATOR_CREDS[ENDPOINTS[0]])
                 const conn = couch.db.use(TEST_DATABASE_HASH[0])
 
-                console.log(`${externalEndpoint}: Verifying endpoint has docs`)
+                log(`${externalEndpoint}: Verifying endpoint has docs`)
                 const docs = await conn.list({include_docs: true})
+
                 // Note: There is a design document, which is why the number is actually 4
                 assert.equal(docs.rows.length, 4, `Three rows returned from ${externalEndpoint}/${TEST_DATABASES[0]} (${TEST_DATABASE_HASH[0]})`)
             }
         })
 
-        // Verify data saved to db2 is NOT replicated for all endpoints
-        it('verify data is not replicated for second database', async () => {
-            // Create three records on second database
-            const endpoint1db2Connection = Utils.buildPouchDsn(ENDPOINT_DSN[endpoint], TEST_DATABASE_HASH[1])
-            await endpoint1db2Connection.put({db2endpoint2: 'world1'})
-            await endpoint1db2Connection.put({db2endpoint2: 'world2'})
-            await endpoint1db2Connection.put({db2endpoint2: 'world3'})
-
-            // Check the three records are correctly replicated on all the other databases
+        it.only('can initialise replication for all database via checkReplication()', async () => {
             for (let i in ENDPOINTS) {
-                if (i === 1) {
-                    // skip second database
+                const endpoint = ENDPOINTS[i]
+                log(`${endpoint}: Calling checkReplication() on all databases for ${endpoint}`)
+                const result = await Utils.checkReplication(endpoint, AUTH_TOKENS[endpoint])
+                assert.equal(result.data.status, 'success', 'Check replication completed successfully')
+            }
+        })
+
+        it.only('verify data is being replicated for all databases and endpoints', async () => {
+            // Sleep 5ms to have replication time to initialise
+            log('Sleeping so replication has time to do its thing...')
+            await Utils.sleep(5000)
+
+            let recordCount = 0
+            // Create data on every database, on every endpoint, and verify on every other endpoint
+            for (let i in TEST_DATABASES) {
+                // skip first database as we've already used it
+                if (i == 0) {
                     continue
                 }
 
-                const conn = Utils.buildPouchDsn(ENDPOINT_DSN[endpoint], TEST_DATABASES[1])
-                const docs = await conn.allDocs({include_docs: true})
-                console.log(`Endpoint ${endpoint} has docs:`)
-                console.log(docs)
-                assert.equals(docs.rows.length, 0, 'No rows returned')
+                const dbName = TEST_DATABASES[i]
+                const dbHash = TEST_DATABASE_HASH[i]
+                const createdDatabaseIds = []
+                
+                log(`${dbName} (${dbHash}): Creating a record on every endpoint`)
+                for (let e in ENDPOINTS) {
+                    const endpoint = ENDPOINTS[e]
+
+                    // Use the credentials of a different server as the local server doesn't have permissions
+                    // to write (even as admin)
+                    let creds
+                    if (e == 0) {
+                        creds = REPLICATOR_CREDS[ENDPOINTS[1]]
+                    } else {
+                        creds = REPLICATOR_CREDS[ENDPOINTS[0]]
+                    }
+
+                    // create a record on this endpoint
+                    const couch = buildEndpointConnection(ENDPOINTS_COUCH[endpoint], creds)
+                    const conn = couch.db.use(dbHash)
+                    const id = String(recordCount++)
+                    createdDatabaseIds.push(id)
+                    await conn.insert({_id: id, dbName, dbHash, endpoint})
+                }
+
+                log(`${dbName} (${dbHash}): Done (${createdDatabaseIds.length}). Sleeping for replication to do its thing...`)
+                await Utils.sleep(5000)
+
+                for (let e in ENDPOINTS) {
+                    const endpoint = ENDPOINTS[e]
+
+                    // Use the credentials of a different server as the local server doesn't have permissions
+                    // to write (even as admin)
+                    let creds
+                    if (e == 0) {
+                        creds = REPLICATOR_CREDS[ENDPOINTS[1]]
+                    } else {
+                        creds = REPLICATOR_CREDS[ENDPOINTS[0]]
+                    }
+
+                    // create a record on this endpoint
+                    const couch = buildEndpointConnection(ENDPOINTS_COUCH[endpoint], creds)
+                    const conn = couch.db.use(dbHash)
+                    
+                    // confirm all the records exist
+                    for (let j in createdDatabaseIds) {
+                        const createdId = createdDatabaseIds[j]
+                        const result = await conn.get(createdId)
+                        assert.equal(result._id, createdId, 'Record deleted')
+                    }
+                }
             }
         })
 
-        it('can initialise replication for all database via checkReplication()', async () => {
-            for (let i in ENDPOINTS) {
-                const endpoint = ENDPOINTS[i]
-                const result = await Utils.checkReplication(endpoint, AUTH_TOKENS[endpoint])
-                console.log(`checkReplication on ${endpoint} for all databases`)
-                console.log(result)
+        it.only('can delete a database', async () => {
+            // delete a database from all endpoints
+            for (let e in ENDPOINTS) {
+                const endpoint = ENDPOINTS[e]
+                const response = await Utils.deleteDatabase(TEST_DATABASES[0], DID, CONTEXT_NAME, AUTH_TOKENS[endpoint], endpoint)
+                assert.ok(response.data.status, 'success', `Database ${TEST_DATABASES[0]} deleted from ${endpoint}`)
             }
-
-            // @todo: check the replication database as per above
         })
 
-        it('verify data is being replicated for all databases', async () => {
+        it.only('verify database is completely deleted from all endpoints', async () => {
+            const dbHash = TEST_DATABASE_HASH[0]
 
-        })
+            for (let e in ENDPOINTS) {
+                const endpoint = ENDPOINTS[e]
+
+                // Use the credentials of a different server as the local server doesn't have permissions
+                // to write (even as admin)
+                let creds
+                if (e == 0) {
+                    creds = REPLICATOR_CREDS[ENDPOINTS[1]]
+                } else {
+                    creds = REPLICATOR_CREDS[ENDPOINTS[0]]
+                }
+
+                const couch = buildEndpointConnection(ENDPOINTS_COUCH[endpoint], creds)
+
+                // verify database is deleted from each endpoint
+                log(`${endpoint}: Verifying database is deleted (${TEST_DATABASES[0]}) from ${endpoint}`)
+                const dbConn = couch.db.use(dbHash)
+                try {
+                    await dbConn.get('0')
+                    assert.fail(`${dbHash} wasnt deleted from ${endpoint}`)
+                } catch (err) {
+                    assert.equal(err.reason, 'Database does not exist.')
+                }
+
+                // verify all replication entries for the database is removed from this endpoint
+                const couchAdmin = new CouchDb({
+                    url: ENDPOINT_DSN[endpoint],
+                    requestDefaults: {
+                        rejectUnauthorized: process.env.DB_REJECT_UNAUTHORIZED_SSL.toLowerCase() !== "false"
+                    }
+                })
 
-        it('can delete a database', () => {})
+                const replicationConn = couchAdmin.db.use('_replicator')
 
-        it('can remove a database replication entry when via checkReplication()', () => {})
+                log(`${endpoint}: Verifying all replication entries are deleted (${TEST_DATABASES[0]}) from ${endpoint}`)
+                for (let i in ENDPOINTS) {
+                    const endpointCheckUri = ENDPOINTS[i]
+                    if (endpointCheckUri == endpoint) {
+                        continue
+                    }
+
+                    const replicatorId = ComponentUtils.generateReplicatorHash(endpointCheckUri, DID, CONTEXT_NAME)
+                    const dbHash = ComponentUtils.generateDatabaseName(DID, CONTEXT_NAME, TEST_DATABASES[0])
+                    log(`${endpoint}: Verifying replication entry for ${endpointCheckUri} is deleted from endpoint ${endpoint} (${replicatorId}-${dbHash})`)
+
+                    try {
+                        await replicationConn.get(`${replicatorId}-${dbHash}`)
+                    } catch (err) {
+                        assert.equal(err.error, 'not_found', 'Replication entry not found')
+                    }
+                }
+                
+            }
+        })
 
-        it('verify database is deleted from all endpoints', () => {})
+        // can handle a storage node that goes down at any part in the process
     })
 
     // WARNING: This should never run on production!
     this.afterAll(async () => {
-        console.log('Destroying _replicator, verida_replicater_creds and test databases on ALL endpoints')
+        //return
+        log('Destroying _replicator, verida_replicater_creds and test databases on ALL endpoints')
 
         for (let endpoint in ENDPOINT_DSN) {
             const conn = new CouchDb({
@@ -350,7 +458,7 @@ describe("Replication tests", function() {
             for (let d in TEST_DATABASE_HASH) {
                 const databaseName = TEST_DATABASE_HASH[d]
                 try {
-                    console.log(`Destroying ${databaseName}`)
+                    log(`Destroying ${databaseName}`)
                     await conn.db.destroy(databaseName)
                 } catch (err) {}
             }
@@ -365,13 +473,13 @@ describe("Replication tests", function() {
                 try {
                     const username = ComponentUtils.generateReplicaterUsername(endpointExternal)
                     const users = conn.db.use('_users')
-                    console.log(`Deleting replication user ${username} for ${endpointExternal} from ${endpoint}`)
+                    log(`Deleting replication user ${username} for ${endpointExternal} from ${endpoint}`)
                     const doc = await users.get(`org.couchdb.user:${username}`)
                     await users.destroy(`org.couchdb.user:${username}`, doc._rev)
                 } catch (err) {
                     if (err.error != 'not_found') {
-                        console.log(`Unable to delete user`)
-                        console.log(err)   
+                        log(`Unable to delete user`)
+                        log(err)   
                     }
                 }
             }

From 5f814c6b88bb67d3aa6825dc8cc5dcd441cf0688 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Fri, 16 Dec 2022 09:40:43 +1030
Subject: [PATCH 28/28] Add comments in .env file

---
 sample.env | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/sample.env b/sample.env
index b93ce2e8..0acf3b05 100644
--- a/sample.env
+++ b/sample.env
@@ -1,8 +1,13 @@
 DID_RPC_URL=
 DID_NETWORK=testnet
 DID_CACHE_DURATION=3600
+# Admin username and password (for system operations)
+# MUST be set to something random
 DB_USER="admin"
 DB_PASS="admin"
+# Replication username and password (for replicating data to other nodes)
+# MUST be set to something random
+# MUST not change once the node is operational
 DB_REPLICATION_USER=""
 DB_REPLICATION_PASS=""