Feature/52 user db list replicated

src/components/userManager.js

@@ -159,15 +159,37 @@ class UserManager {
         // Remove this endpoint from the list of endpoints to check
         endpoints.splice(endpointIndex, 1)
 
-        let databases = []
+        const userDatabases = await DbManager.getUserDatabases(did, contextName)
+
+        let databases = {}
         if (databaseName) {
             //console.log(`${Utils.serverUri()}: Only checking ${databaseName}`)
+            for (let i in userDatabases) {
+                const item = userDatabases[i]
+                if (item.databaseName == databaseName) {
+                    databases[item.databaseName] = item
+                }
+            }
+
             // Only check a single database
-            databases.push(databaseName)
+            if (!Object.keys(databases).length === 0) {
+                return
+            }
         } else {
             // Fetch all databases for this context
-            let userDatabases = await DbManager.getUserDatabases(did, contextName)
-            databases = userDatabases.map(item => item.databaseName)
+            for (let i in userDatabases) {
+                const item = userDatabases[i]
+                databases[item.databaseName] = item
+            }
+
+            // Ensure the user database list database is included in the list of databases
+            const didContextHash = Utils.generateDidContextHash(did, contextName)
+            const didContextDbName = `c${didContextHash}`
+
+            databases[didContextDbName] = {
+                databaseName: didContextDbName,
+                databasehash: didContextDbName
+            }
             //console.log(`${Utils.serverUri()}: Checking ${databases.length}) databases`)
         }
 
@@ -178,13 +200,13 @@ class UserManager {
         const localAuthBuffer = Buffer.from(`${process.env.DB_REPLICATION_USER}:${process.env.DB_REPLICATION_PASS}`);
         const localAuthBase64 = localAuthBuffer.toString('base64')
 
+        // Ensure all databases have replication entries
         for (let d in databases) {
-            const dbName = databases[d]
+            const dbHash = databases[d].databaseHash
 
             for (let e in endpoints) {
                 const endpointUri = endpoints[e]
                 const replicatorId = Utils.generateReplicatorHash(endpointUri, did, contextName)
-                const dbHash = Utils.generateDatabaseName(did, contextName, dbName)
                 let record
                 try {
                     record = await replicationDb.get(`${replicatorId}-${dbHash}`)
@@ -241,6 +263,40 @@ class UserManager {
 
         // @todo: Find any replication errors and handle them nicely
         // @todo: Remove any replication entries for deleted databases
+
+        // Check user databases are configured correctly
+        await this.checkDatabases(userDatabases)
+    }
+
+    /**
+     * Check all the databases in the user database list exist
+     * 
+     * @todo: How to check they have the correct permissions?
+     */
+    async checkDatabases(userDatabases) {
+        const couch = Db.getCouch('internal')
+
+        for (let d in userDatabases) {
+            const database = userDatabases[d]
+
+            // Try to create database
+            try {
+                //console.log(`Checking ${database.databaseHash} (${database.databaseName}) exists`)
+                await couch.db.create(database.databaseHash);
+
+                // Database didn't exist, so create it properly
+                const options = {}
+                if (database.permissions) {
+                    options.permissions = database.permissions
+                }
+
+                const username = Utils.generateUsername(database.did, database.contextName)
+                await DbManager.createDatabase(database.did, username, database.databaseHash, database.contextName, options)
+            } catch (err) {
+                // The database may already exist, or may have been deleted so a file already exists.
+                // In that case, ignore the error and continue
+            }
+        }
     }
 
 }

src/controllers/user.js

@@ -290,6 +290,7 @@ class UserController {
     }
 
     async checkReplication(req, res) {
+        console.log(`checkReplication()`)
         const did = req.tokenData.did
         const contextName = req.tokenData.contextName
         const databaseName = req.body.databaseName

test/replication.js

@@ -18,6 +18,7 @@ import CONFIG from './config'
 const LOGGING_ENABLED = false
 
 // Use a pre-built mnemonic where the first private key is a Verida DID private key
+// mnemonic with a Verida DID that points to 2x local endpoints
 const MNEMONIC = 'pave online install gift glimpse purpose truth loan arm wing west option'
 
 // Context name to use for the tests
@@ -217,7 +218,7 @@ describe("Replication tests", function() {
                         } catch (err) {
                             log('pouchdb connection error')
                             log(err.message)
-                            assert.fail('Replication record not created')
+                            assert.fail(`Replication record not created (${replicatorId}-${dbHash})`)
                         }
 
                         // Check info is accurate
@@ -300,7 +301,7 @@ describe("Replication tests", function() {
         it('verify data is being replicated for all databases and endpoints', async () => {
             // Sleep 1s to have replication time to initialise
             log('Sleeping so replication has time to do its thing...')
-            await Utils.sleep(1000)
+            await Utils.sleep(5000)
 
             let recordCount = 0
             // Create data on every database, on every endpoint, and verify on every other endpoint
@@ -328,27 +329,119 @@ describe("Replication tests", function() {
                 }
 
                 log(`${dbName} (${dbHash}): Done (${createdDatabaseIds.length}). Sleeping for replication to do its thing...`)
-                await Utils.sleep(1000)
-
-                for (let e in ENDPOINTS) {
-                    const endpoint = ENDPOINTS[e]
-
-                    const creds = REPLICATOR_CREDS[endpoint]
+                await Utils.sleep(5000)
 
-                    // create a record on this endpoint
-                    const couch = buildEndpointConnection(ENDPOINTS_COUCH[endpoint], creds)
-                    const conn = couch.db.use(dbHash)
-
-                    // confirm all the records exist
-                    for (let j in createdDatabaseIds) {
-                        const createdId = createdDatabaseIds[j]
-                        const result = await conn.get(createdId)
-                        assert.equal(result._id, createdId, 'Record deleted')
+                try {
+                    for (let e in ENDPOINTS) {
+                        const endpoint = ENDPOINTS[e]
+
+                        const creds = REPLICATOR_CREDS[endpoint]
+
+                        // create a record on this endpoint
+                        const couch = buildEndpointConnection(ENDPOINTS_COUCH[endpoint], creds)
+                        const conn = couch.db.use(dbHash)
+
+                        // confirm all the records exist
+                        for (let j in createdDatabaseIds) {
+                            const createdId = createdDatabaseIds[j]
+                            const result = await conn.get(createdId)
+                            assert.equal(result._id, createdId, 'Record deleted')
+                        }
                     }
+                } catch (err) {
+                    console.log(err)
+                    throw err
                 }
             }
         })
 
+        it('verify non-replicated database is fixed with checkReplication()', async () => {
+            // manually delete the database replication entry from endpoint 1 to endpoint 2
+            const endpoint1 = ENDPOINTS[0]
+            const endpoint2 = ENDPOINTS[1]
+            const couch = buildEndpointConnection(ENDPOINT_DSN[endpoint1], {})
+            const replicatorId = ComponentUtils.generateReplicatorHash(endpoint2, DID, CONTEXT_NAME)
+            const dbHash = ComponentUtils.generateDatabaseName(DID, CONTEXT_NAME, TEST_DATABASES[0])
+            const conn = couch.db.use(`_replicator`)
+
+            log(`${endpoint1}: (${endpoint2}) Locating _replication entry for ${TEST_DATABASES[0]} (${replicatorId}-${dbHash})`)
+            let replicationEntry
+            try {
+                replicationEntry = await conn.get(`${replicatorId}-${dbHash}`)
+                const destroyResult = await conn.destroy(replicationEntry._id, replicationEntry._rev)
+            } catch (err) {
+                log(err)
+                assert.fail(`Replication record not found (${replicatorId}-${dbHash})`)
+            }
+
+            // call checkReplication() on endpoint 1
+            const result = await Utils.checkReplication(endpoint1, AUTH_TOKENS[endpoint1], TEST_DATABASES[0])
+            assert.equal(result.data.status, 'success', 'checkReplication() success')
+
+            // verify the replication entry exists and is valid
+            try {
+                const newReplicationEntry = await conn.get(`${replicatorId}-${dbHash}`)
+                assert.equal(newReplicationEntry._id, replicationEntry._id, 'Replication entry found with correct _id')
+                assert.ok(newReplicationEntry._rev != replicationEntry._rev, 'Replication entry found with different revision')
+            } catch (err) {
+                log(err.message)
+                assert.fail(`Replication record not found (${replicatorId}-${dbHash})`)
+            }
+        })
+
+        it('verify missing database is correctly created with checkReplication(databaseName)', async () => {
+            // manually delete the database from endpoint 1
+            const endpoint1 = ENDPOINTS[0]
+            const couch = buildEndpointConnection(ENDPOINT_DSN[endpoint1], {})
+            await couch.db.destroy(TEST_DATABASE_HASH[0])
+
+            // call checkReplication() on endpoint 1
+            const result = await Utils.checkReplication(endpoint1, AUTH_TOKENS[endpoint1], TEST_DATABASES[0])
+            assert.equal(result.data.status, 'success', 'checkReplication() success')
+
+            // verify the database has been re-created
+            const conn = couch.db.use(TEST_DATABASE_HASH[0])
+            try {
+                const results = await conn.list()
+                assert.ok(results, 'Database exists')
+            } catch (err) {
+                console.log(err)
+                assert.fail(`Database doesn't exist`)
+            }
+        })
+
+        // Do it again, but without specifying the database
+        it('verify missing database is correctly created with checkReplication()', async () => {
+            // manually delete the database from endpoint 1
+            const endpoint1 = ENDPOINTS[0]
+            const couch = buildEndpointConnection(ENDPOINT_DSN[endpoint1], {})
+            await couch.db.destroy(TEST_DATABASE_HASH[0])
+
+            // call checkReplication() on endpoint 1
+            const result = await Utils.checkReplication(endpoint1, AUTH_TOKENS[endpoint1])
+            assert.equal(result.data.status, 'success', 'checkReplication() success')
+
+            // verify the database has been re-created
+            const conn = couch.db.use(TEST_DATABASE_HASH[0])
+            try {
+                const results = await conn.list()
+                assert.ok(results, 'Database exists')
+            } catch (err) {
+                console.log(err)
+                assert.fail(`Database doesn't exist`)
+            }
+        })
+
+        // @todo make sure database permissions are correct when the database is re-created
+
+        // @todo detects the storage node is no longer included in the DID document and deletes everything
+
+        // @todo inject a fake database into storage node 1, call checkReplication() on storage node 2, make sure it's not created
+
+        it('verify deleted database is correctly removed with checkReplication()', async () => {
+            // @todo
+        })
+
         it('can delete a database', async () => {
             // delete a database from all endpoints
             for (let e in ENDPOINTS) {