From 350d4759f7fe2faf75960b17b68f11a9c08bc4ee Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Tue, 20 Dec 2022 19:12:16 +1030
Subject: [PATCH 1/6] Ensure user databases are replicated

---
 src/components/userManager.js | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/components/userManager.js b/src/components/userManager.js
index eb5b61f7..dd0a1bcb 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -168,6 +168,13 @@ class UserManager {
             // Fetch all databases for this context
             let userDatabases = await DbManager.getUserDatabases(did, contextName)
             databases = userDatabases.map(item => item.databaseName)
+
+            // Ensure the user database list database is included in the list of databases
+            const didContextHash = Utils.generateDidContextHash(did, contextName)
+            const didContextDbName = `c${didContextHash}`
+
+            // prefix with `hashed::` to indicate this database name is already hash
+            databases.push(`hashed::${didContextDbName}`)
             //console.log(`${Utils.serverUri()}: Checking ${databases.length}) databases`)
         }
 
@@ -178,13 +185,19 @@ class UserManager {
         const localAuthBuffer = Buffer.from(`${process.env.DB_REPLICATION_USER}:${process.env.DB_REPLICATION_PASS}`);
         const localAuthBase64 = localAuthBuffer.toString('base64')
 
+        // Ensure all databases have replication entries
         for (let d in databases) {
             const dbName = databases[d]
+            let dbHash
+            if (dbName.substring(0,8) == 'hashed::') {
+                dbHash = Utils.generateDatabaseName(did, contextName, dbName)
+            } else {
+                dbHash = dbname.substring(8)
+            }
 
             for (let e in endpoints) {
                 const endpointUri = endpoints[e]
                 const replicatorId = Utils.generateReplicatorHash(endpointUri, did, contextName)
-                const dbHash = Utils.generateDatabaseName(did, contextName, dbName)
                 let record
                 try {
                     record = await replicationDb.get(`${replicatorId}-${dbHash}`)

From e18ff8ad9ddb2a15ce412452150d98314b6235b4 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Wed, 21 Dec 2022 08:05:25 +1030
Subject: [PATCH 2/6] Add checkReplication tests confirming databases are
 re-created

---
 src/components/userManager.js | 64 ++++++++++++++++++++++++++++-------
 test/replication.js           | 57 +++++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+), 12 deletions(-)

diff --git a/src/components/userManager.js b/src/components/userManager.js
index dd0a1bcb..f6f4b78a 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -159,22 +159,32 @@ class UserManager {
         // Remove this endpoint from the list of endpoints to check
         endpoints.splice(endpointIndex, 1)
 
-        let databases = []
+        let databases = {}
         if (databaseName) {
             //console.log(`${Utils.serverUri()}: Only checking ${databaseName}`)
             // Only check a single database
-            databases.push(databaseName)
+            if (!userDatabases[databaseName]) {
+                console.log('User database not found!')
+                return
+            }
+            
+            databases[databaseName] = userDatabases[databaseName]
         } else {
             // Fetch all databases for this context
             let userDatabases = await DbManager.getUserDatabases(did, contextName)
-            databases = userDatabases.map(item => item.databaseName)
+            for (let i in userDatabases) {
+                const item = userDatabases[i]
+                databases[item.databaseName] = item
+            }
 
             // Ensure the user database list database is included in the list of databases
             const didContextHash = Utils.generateDidContextHash(did, contextName)
             const didContextDbName = `c${didContextHash}`
 
-            // prefix with `hashed::` to indicate this database name is already hash
-            databases.push(`hashed::${didContextDbName}`)
+            databases[didContextDbName] = {
+                databaseName: didContextDbName,
+                databasehash: didContextDbName
+            }
             //console.log(`${Utils.serverUri()}: Checking ${databases.length}) databases`)
         }
 
@@ -187,13 +197,7 @@ class UserManager {
 
         // Ensure all databases have replication entries
         for (let d in databases) {
-            const dbName = databases[d]
-            let dbHash
-            if (dbName.substring(0,8) == 'hashed::') {
-                dbHash = Utils.generateDatabaseName(did, contextName, dbName)
-            } else {
-                dbHash = dbname.substring(8)
-            }
+            const dbHash = databases[d].databaseHash
 
             for (let e in endpoints) {
                 const endpointUri = endpoints[e]
@@ -254,6 +258,42 @@ class UserManager {
 
         // @todo: Find any replication errors and handle them nicely
         // @todo: Remove any replication entries for deleted databases
+
+        // Check user databases are configured correctly
+        await this.checkDatabases(userDatabases)
+    }
+
+    /**
+     * Check all the databases in the user database list exist
+     * 
+     * @todo: How to check they have the correct permissions?
+     */
+    async checkDatabases(userDatabases) {
+        const couch = Db.getCouch('internal')
+
+        for (let d in userDatabases) {
+            const database = userDatabases[d]
+            
+            // Try to create database
+            try {
+                console.log(`Checking ${database.databaseHash} (${database.databaseName}) exists`)
+                await couch.db.create(database.databaseHash);
+
+                // Database didn't exist, so create it properly
+                const options = {}
+                if (database.permissions) {
+                    options.permissions = database.permissions
+                }
+
+                const username = Utils.generateUsername(did, contextName)
+                console.log(`Database didn't exist, so creating`)
+                await DbManager.createDatabase(database.did, username, database.databaseHash, database.contextName, options)
+
+            } catch (err) {
+                // The database may already exist, or may have been deleted so a file already exists.
+                // In that case, ignore the error and continue
+            }
+        }
     }
 
 }
diff --git a/test/replication.js b/test/replication.js
index 6c88dd75..43f3bfa8 100644
--- a/test/replication.js
+++ b/test/replication.js
@@ -349,6 +349,63 @@ describe("Replication tests", function() {
             }
         })
 
+        it('verify non-replicated database is fixed with checkReplication()', async () => {
+            // manually delete the database replication entry from endpoint 1
+            // call checkReplication() on endpoint 1
+            // verify the replication entry exists and is valid
+        })
+
+        it('verify missing database is correctly created with checkReplication(databaseName)', async () => {
+            // manually delete the database from endpoint 1
+            console.log(`Destroying ${TEST_DATABASE_HASH[0]}`)
+            const endpoint1 = ENDPOINTS[0]
+            const creds = REPLICATOR_CREDS[endpoint1]
+            const couch = buildEndpointConnection(ENDPOINTS_DSN[endpoint1], creds)
+            await couch.db.destroy(TEST_DATABASE_HASH[0])
+            console.log(`Destroyed`)
+
+            // call checkReplication() on endpoint 1
+            console.log(`Calling checkReplication(${TEST_DATABASE_HASH[0]})`)
+            const result = await Utils.checkReplication(endpoint1, AUTH_TOKENS[endpoint1], TEST_DATABASES[0])
+            console.log(result)
+
+            // verify the database has been re-created
+            const conn = couch.db.use(TEST_DATABASE_HASH[0])
+            try {
+                const results = await conn.list()
+                console.log(results)
+                assert.ok(true, 'Database exists')
+            } catch (err) {
+                assert.fail(`Database doesn't exist`)
+            }  
+        })
+
+        // Do it again, but without specifying the database
+        it('verify missing database is correctly created with checkReplication()', async () => {
+            // manually delete the database from endpoint 1
+            console.log(`Destroying ${TEST_DATABASE_HASH[0]}`)
+            const endpoint1 = ENDPOINTS[0]
+            const creds = REPLICATOR_CREDS[endpoint1]
+            const couch = buildEndpointConnection(ENDPOINTS_DSN[endpoint1], creds)
+            await couch.db.destroy(TEST_DATABASE_HASH[0])
+            console.log(`Destroyed`)
+
+            // call checkReplication() on endpoint 1
+            console.log(`Calling checkReplication(${TEST_DATABASE_HASH[0]})`)
+            const result = await Utils.checkReplication(endpoint1, AUTH_TOKENS[endpoint1])
+            console.log(result)
+
+            // verify the database has been re-created
+            const conn = couch.db.use(TEST_DATABASE_HASH[0])
+            try {
+                const results = await conn.list()
+                console.log(results)
+                assert.ok(true, 'Database exists')
+            } catch (err) {
+                assert.fail(`Database doesn't exist`)
+            }
+        })
+
         it('can delete a database', async () => {
             // delete a database from all endpoints
             for (let e in ENDPOINTS) {

From b93babf893856c0f6735432079979802540efadf Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Wed, 21 Dec 2022 08:25:04 +1030
Subject: [PATCH 3/6] Fix user database issues. Add debug logging.

---
 src/components/userManager.js | 22 +++++++++++++++++-----
 src/controllers/user.js       |  1 +
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/src/components/userManager.js b/src/components/userManager.js
index f6f4b78a..3826019b 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -144,12 +144,15 @@ class UserManager {
      * @param {*} databaseName (optional) If not specified, checks all databases
      */
     async checkReplication(did, contextName, databaseName) {
-        //console.log(`${Utils.serverUri()}: checkReplication(${did}, ${contextName}, ${databaseName})`)
+        console.log(`${Utils.serverUri()}: checkReplication(${did}, ${contextName}, ${databaseName})`)
         // Lookup DID document and get list of endpoints for this context
         const didDocument = await AuthManager.getDidDocument(did)
         const didService = didDocument.locateServiceEndpoint(contextName, 'database')
         let endpoints = [...didService.serviceEndpoint] // create a copy as this is cached and we will modify later
 
+        console.log(endpoints)
+        console.log('serverUrl', Utils.serverUri())
+
         // Confirm this endpoint is in the list of endpoints
         const endpointIndex = endpoints.indexOf(Utils.serverUri())
         if (endpointIndex === -1) {
@@ -159,19 +162,28 @@ class UserManager {
         // Remove this endpoint from the list of endpoints to check
         endpoints.splice(endpointIndex, 1)
 
+        const userDatabases = await DbManager.getUserDatabases(did, contextName)
+
         let databases = {}
         if (databaseName) {
             //console.log(`${Utils.serverUri()}: Only checking ${databaseName}`)
+            for (let i in userDatabases) {
+                const item = userDatabases[i]
+                if (item.databaseName == databaseName) {
+                    databases[item.databaseName] = item
+                }
+            }
+            console.log(databases)
+            
             // Only check a single database
-            if (!userDatabases[databaseName]) {
+            if (!Object.keys(databases).length === 0) {
+                console.log(userDatabases)
+                console.log(databaseName)
                 console.log('User database not found!')
                 return
             }
-            
-            databases[databaseName] = userDatabases[databaseName]
         } else {
             // Fetch all databases for this context
-            let userDatabases = await DbManager.getUserDatabases(did, contextName)
             for (let i in userDatabases) {
                 const item = userDatabases[i]
                 databases[item.databaseName] = item
diff --git a/src/controllers/user.js b/src/controllers/user.js
index abe4eca4..37bed87b 100644
--- a/src/controllers/user.js
+++ b/src/controllers/user.js
@@ -290,6 +290,7 @@ class UserController {
     }
 
     async checkReplication(req, res) {
+        console.log(`checkReplication()`)
         const did = req.tokenData.did
         const contextName = req.tokenData.contextName
         const databaseName = req.body.databaseName

From 1ea54b7110b07e68e6655e64bce1c5eaa4243e99 Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Wed, 21 Dec 2022 09:09:59 +1030
Subject: [PATCH 4/6] Bug fix database creation via checkReplication()

---
 src/components/userManager.js | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/components/userManager.js b/src/components/userManager.js
index 3826019b..4f2f1f48 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -174,7 +174,7 @@ class UserManager {
                 }
             }
             console.log(databases)
-            
+
             // Only check a single database
             if (!Object.keys(databases).length === 0) {
                 console.log(userDatabases)
@@ -297,8 +297,7 @@ class UserManager {
                     options.permissions = database.permissions
                 }
 
-                const username = Utils.generateUsername(did, contextName)
-                console.log(`Database didn't exist, so creating`)
+                const username = Utils.generateUsername(database.did, database.contextName)
                 await DbManager.createDatabase(database.did, username, database.databaseHash, database.contextName, options)
 
             } catch (err) {

From dabd226e54febbee4283c85e33918ba87514efbb Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Wed, 21 Dec 2022 15:28:46 +1030
Subject: [PATCH 5/6] Basic replication unit tests passing

---
 src/components/userManager.js |   1 -
 test/replication.js           | 106 +++++++++++++++++++++++-----------
 2 files changed, 71 insertions(+), 36 deletions(-)

diff --git a/src/components/userManager.js b/src/components/userManager.js
index 4f2f1f48..12e23948 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -299,7 +299,6 @@ class UserManager {
 
                 const username = Utils.generateUsername(database.did, database.contextName)
                 await DbManager.createDatabase(database.did, username, database.databaseHash, database.contextName, options)
-
             } catch (err) {
                 // The database may already exist, or may have been deleted so a file already exists.
                 // In that case, ignore the error and continue
diff --git a/test/replication.js b/test/replication.js
index 43f3bfa8..39baca9d 100644
--- a/test/replication.js
+++ b/test/replication.js
@@ -18,6 +18,7 @@ import CONFIG from './config'
 const LOGGING_ENABLED = false
 
 // Use a pre-built mnemonic where the first private key is a Verida DID private key
+// mnemonic with a Verida DID that points to 2x local endpoints
 const MNEMONIC = 'pave online install gift glimpse purpose truth loan arm wing west option'
 
 // Context name to use for the tests
@@ -217,7 +218,7 @@ describe("Replication tests", function() {
                         } catch (err) {
                             log('pouchdb connection error')
                             log(err.message)
-                            assert.fail('Replication record not created')
+                            assert.fail(`Replication record not created (${replicatorId}-${dbHash})`)
                         }
 
                         // Check info is accurate
@@ -300,7 +301,7 @@ describe("Replication tests", function() {
         it('verify data is being replicated for all databases and endpoints', async () => {
             // Sleep 1s to have replication time to initialise
             log('Sleeping so replication has time to do its thing...')
-            await Utils.sleep(1000)
+            await Utils.sleep(5000)
 
             let recordCount = 0
             // Create data on every database, on every endpoint, and verify on every other endpoint
@@ -328,84 +329,119 @@ describe("Replication tests", function() {
                 }
 
                 log(`${dbName} (${dbHash}): Done (${createdDatabaseIds.length}). Sleeping for replication to do its thing...`)
-                await Utils.sleep(1000)
-
-                for (let e in ENDPOINTS) {
-                    const endpoint = ENDPOINTS[e]
-
-                    const creds = REPLICATOR_CREDS[endpoint]
+                await Utils.sleep(5000)
 
-                    // create a record on this endpoint
-                    const couch = buildEndpointConnection(ENDPOINTS_COUCH[endpoint], creds)
-                    const conn = couch.db.use(dbHash)
-                    
-                    // confirm all the records exist
-                    for (let j in createdDatabaseIds) {
-                        const createdId = createdDatabaseIds[j]
-                        const result = await conn.get(createdId)
-                        assert.equal(result._id, createdId, 'Record deleted')
+                try {
+                    for (let e in ENDPOINTS) {
+                        const endpoint = ENDPOINTS[e]
+
+                        const creds = REPLICATOR_CREDS[endpoint]
+
+                        // create a record on this endpoint
+                        const couch = buildEndpointConnection(ENDPOINTS_COUCH[endpoint], creds)
+                        const conn = couch.db.use(dbHash)
+                        
+                        // confirm all the records exist
+                        for (let j in createdDatabaseIds) {
+                            const createdId = createdDatabaseIds[j]
+                            const result = await conn.get(createdId)
+                            assert.equal(result._id, createdId, 'Record deleted')
+                        }
                     }
+                } catch (err) {
+                    console.log(err)
+                    throw err
                 }
             }
         })
 
         it('verify non-replicated database is fixed with checkReplication()', async () => {
-            // manually delete the database replication entry from endpoint 1
+            // manually delete the database replication entry from endpoint 1 to endpoint 2
+            const endpoint1 = ENDPOINTS[0]
+            const endpoint2 = ENDPOINTS[1]
+            const couch = buildEndpointConnection(ENDPOINT_DSN[endpoint1], {})
+            const replicatorId = ComponentUtils.generateReplicatorHash(endpoint2, DID, CONTEXT_NAME)
+            const dbHash = ComponentUtils.generateDatabaseName(DID, CONTEXT_NAME, TEST_DATABASES[0])
+            const conn = couch.db.use(`_replicator`)
+
+            log(`${endpoint1}: (${endpoint2}) Locating _replication entry for ${TEST_DATABASES[0]} (${replicatorId}-${dbHash})`)
+            let replicationEntry
+            try {
+                replicationEntry = await conn.get(`${replicatorId}-${dbHash}`)
+                const destroyResult = await conn.destroy(replicationEntry._id, replicationEntry._rev)
+            } catch (err) {
+                log(err)
+                assert.fail(`Replication record not found (${replicatorId}-${dbHash})`)
+            }
+
             // call checkReplication() on endpoint 1
+            const result = await Utils.checkReplication(endpoint1, AUTH_TOKENS[endpoint1], TEST_DATABASES[0])
+            assert.equal(result.data.status, 'success', 'checkReplication() success')
+
             // verify the replication entry exists and is valid
+            try {
+                const newReplicationEntry = await conn.get(`${replicatorId}-${dbHash}`)
+                assert.equal(newReplicationEntry._id, replicationEntry._id, 'Replication entry found with correct _id')
+                assert.ok(newReplicationEntry._rev != replicationEntry._rev, 'Replication entry found with different revision')
+            } catch (err) {
+                log(err.message)
+                assert.fail(`Replication record not found (${replicatorId}-${dbHash})`)
+            }
         })
 
         it('verify missing database is correctly created with checkReplication(databaseName)', async () => {
             // manually delete the database from endpoint 1
-            console.log(`Destroying ${TEST_DATABASE_HASH[0]}`)
             const endpoint1 = ENDPOINTS[0]
-            const creds = REPLICATOR_CREDS[endpoint1]
-            const couch = buildEndpointConnection(ENDPOINTS_DSN[endpoint1], creds)
+            const couch = buildEndpointConnection(ENDPOINT_DSN[endpoint1], {})
             await couch.db.destroy(TEST_DATABASE_HASH[0])
-            console.log(`Destroyed`)
 
             // call checkReplication() on endpoint 1
-            console.log(`Calling checkReplication(${TEST_DATABASE_HASH[0]})`)
             const result = await Utils.checkReplication(endpoint1, AUTH_TOKENS[endpoint1], TEST_DATABASES[0])
-            console.log(result)
+            assert.equal(result.data.status, 'success', 'checkReplication() success')
 
             // verify the database has been re-created
             const conn = couch.db.use(TEST_DATABASE_HASH[0])
             try {
                 const results = await conn.list()
-                console.log(results)
-                assert.ok(true, 'Database exists')
+                assert.ok(results, 'Database exists')
             } catch (err) {
+                console.log(err)
                 assert.fail(`Database doesn't exist`)
-            }  
+            }
         })
 
         // Do it again, but without specifying the database
         it('verify missing database is correctly created with checkReplication()', async () => {
             // manually delete the database from endpoint 1
-            console.log(`Destroying ${TEST_DATABASE_HASH[0]}`)
             const endpoint1 = ENDPOINTS[0]
-            const creds = REPLICATOR_CREDS[endpoint1]
-            const couch = buildEndpointConnection(ENDPOINTS_DSN[endpoint1], creds)
+            const couch = buildEndpointConnection(ENDPOINT_DSN[endpoint1], {})
             await couch.db.destroy(TEST_DATABASE_HASH[0])
-            console.log(`Destroyed`)
 
             // call checkReplication() on endpoint 1
-            console.log(`Calling checkReplication(${TEST_DATABASE_HASH[0]})`)
             const result = await Utils.checkReplication(endpoint1, AUTH_TOKENS[endpoint1])
-            console.log(result)
+            assert.equal(result.data.status, 'success', 'checkReplication() success')
 
             // verify the database has been re-created
             const conn = couch.db.use(TEST_DATABASE_HASH[0])
             try {
                 const results = await conn.list()
-                console.log(results)
-                assert.ok(true, 'Database exists')
+                assert.ok(results, 'Database exists')
             } catch (err) {
+                console.log(err)
                 assert.fail(`Database doesn't exist`)
             }
         })
 
+        // @todo make sure database permissions are correct when the database is re-created
+
+        // @todo detects the storage node is no longer included in the DID document and deletes everything
+
+        // @todo inject a fake database into storage node 1, call checkReplication() on storage node 2, make sure it's not created
+
+        it('verify deleted database is correctly removed with checkReplication()', async () => {
+            // @todo
+        })
+
         it('can delete a database', async () => {
             // delete a database from all endpoints
             for (let e in ENDPOINTS) {

From d5fa6ce59b75bf29ca082c0ae843c63ff214714f Mon Sep 17 00:00:00 2001
From: Chris <chris@verida.io>
Date: Wed, 21 Dec 2022 15:31:44 +1030
Subject: [PATCH 6/6] Remove excess debug output

---
 src/components/userManager.js | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/src/components/userManager.js b/src/components/userManager.js
index 12e23948..e9fb57bb 100644
--- a/src/components/userManager.js
+++ b/src/components/userManager.js
@@ -144,15 +144,12 @@ class UserManager {
      * @param {*} databaseName (optional) If not specified, checks all databases
      */
     async checkReplication(did, contextName, databaseName) {
-        console.log(`${Utils.serverUri()}: checkReplication(${did}, ${contextName}, ${databaseName})`)
+        //console.log(`${Utils.serverUri()}: checkReplication(${did}, ${contextName}, ${databaseName})`)
         // Lookup DID document and get list of endpoints for this context
         const didDocument = await AuthManager.getDidDocument(did)
         const didService = didDocument.locateServiceEndpoint(contextName, 'database')
         let endpoints = [...didService.serviceEndpoint] // create a copy as this is cached and we will modify later
 
-        console.log(endpoints)
-        console.log('serverUrl', Utils.serverUri())
-
         // Confirm this endpoint is in the list of endpoints
         const endpointIndex = endpoints.indexOf(Utils.serverUri())
         if (endpointIndex === -1) {
@@ -173,13 +170,9 @@ class UserManager {
                     databases[item.databaseName] = item
                 }
             }
-            console.log(databases)
 
             // Only check a single database
             if (!Object.keys(databases).length === 0) {
-                console.log(userDatabases)
-                console.log(databaseName)
-                console.log('User database not found!')
                 return
             }
         } else {
@@ -288,7 +281,7 @@ class UserManager {
             
             // Try to create database
             try {
-                console.log(`Checking ${database.databaseHash} (${database.databaseName}) exists`)
+                //console.log(`Checking ${database.databaseHash} (${database.databaseName}) exists`)
                 await couch.db.create(database.databaseHash);
 
                 // Database didn't exist, so create it properly