From 690200fa165e418854c7799b077b1b61f5b82428 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Tue, 13 Feb 2024 15:36:43 +1100 Subject: [PATCH] feat: adding `secrets env` rpc method --- src/client/callers/index.ts | 3 + src/client/callers/vaultsSecretsEnv.ts | 12 ++ src/client/handlers/VaultsSecretsEnv.ts | 89 ++++++++ src/client/handlers/index.ts | 3 + src/git/utils.ts | 2 +- src/vaults/VaultOps.ts | 4 +- src/vaults/utils.ts | 37 +++- tests/client/handlers/vaults.test.ts | 266 ++++++++++++++++++++++++ tests/vaults/utils.test.ts | 8 +- 9 files changed, 413 insertions(+), 11 deletions(-) create mode 100644 src/client/callers/vaultsSecretsEnv.ts create mode 100644 src/client/handlers/VaultsSecretsEnv.ts diff --git a/src/client/callers/index.ts b/src/client/callers/index.ts index c2b6f9d0f..54ae4805d 100644 --- a/src/client/callers/index.ts +++ b/src/client/callers/index.ts @@ -60,6 +60,7 @@ import vaultsRename from './vaultsRename'; import vaultsScan from './vaultsScan'; import vaultsSecretsDelete from './vaultsSecretsDelete'; import vaultsSecretsEdit from './vaultsSecretsEdit'; +import vaultsSecretsEnv from './vaultsSecretsEnv'; import vaultsSecretsGet from './vaultsSecretsGet'; import vaultsSecretsList from './vaultsSecretsList'; import vaultsSecretsMkdir from './vaultsSecretsMkdir'; @@ -135,6 +136,7 @@ const clientManifest = { vaultsScan, vaultsSecretsDelete, vaultsSecretsEdit, + vaultsSecretsEnv, vaultsSecretsGet, vaultsSecretsList, vaultsSecretsMkdir, @@ -209,6 +211,7 @@ export { vaultsScan, vaultsSecretsDelete, vaultsSecretsEdit, + vaultsSecretsEnv, vaultsSecretsGet, vaultsSecretsList, vaultsSecretsMkdir, diff --git a/src/client/callers/vaultsSecretsEnv.ts b/src/client/callers/vaultsSecretsEnv.ts new file mode 100644 index 000000000..ba6e07d1b --- /dev/null +++ b/src/client/callers/vaultsSecretsEnv.ts @@ -0,0 +1,12 @@ +import type { HandlerTypes } from '@matrixai/rpc'; +import type VaultsSecretsEnv from '../handlers/VaultsSecretsEnv'; +import { DuplexCaller } from '@matrixai/rpc'; + +type CallerTypes = HandlerTypes; + +const vaultsSecretsEnv = new DuplexCaller< + CallerTypes['input'], + CallerTypes['output'] +>(); + +export default vaultsSecretsEnv; diff --git a/src/client/handlers/VaultsSecretsEnv.ts b/src/client/handlers/VaultsSecretsEnv.ts new file mode 100644 index 000000000..6b0a230ac --- /dev/null +++ b/src/client/handlers/VaultsSecretsEnv.ts @@ -0,0 +1,89 @@ +import type { DB } from '@matrixai/db'; +import type { + ClientRPCRequestParams, + ClientRPCResponseResult, + SecretIdentifierMessage, + SecretContentMessage, +} from '../types'; +import type VaultManager from '../../vaults/VaultManager'; +import { DuplexHandler } from '@matrixai/rpc'; +import * as vaultsUtils from '../../vaults/utils'; +import * as vaultsErrors from '../../vaults/errors'; + +class VaultsSecretsList extends DuplexHandler< + { + vaultManager: VaultManager; + db: DB; + }, + ClientRPCRequestParams, + ClientRPCResponseResult +> { + public handle = async function* ( + input: AsyncIterableIterator< + ClientRPCRequestParams + >, + _cancel, + _meta, + ctx, + ): AsyncGenerator> { + if (ctx.signal.aborted) throw ctx.signal.reason; + const { vaultManager, db }: { vaultManager: VaultManager; db: DB } = + this.container; + + return yield* db.withTransactionG(async function* (tran): AsyncGenerator< + ClientRPCResponseResult + > { + if (ctx.signal.aborted) throw ctx.signal.reason; + for await (const secretIdentifierMessage of input) { + const { nameOrId, secretName } = secretIdentifierMessage; + const vaultIdFromName = await vaultManager.getVaultId(nameOrId, tran); + const vaultId = vaultIdFromName ?? vaultsUtils.decodeVaultId(nameOrId); + if (vaultId == null) { + throw new vaultsErrors.ErrorVaultsVaultUndefined(); + } + const secrets = await vaultManager.withVaults( + [vaultId], + async (vault) => { + const results: Array<{ + filePath: string; + value: string; + }> = []; + return await vault.readF(async (fs) => { + try { + for await (const filePath of vaultsUtils.walkFs( + fs, + secretName, + )) { + const fileContents = await fs.readFile(filePath); + results.push({ + filePath, + value: fileContents.toString(), + }); + } + } catch (e) { + if (e.code === 'ENOENT') { + throw new vaultsErrors.ErrorSecretsSecretUndefined( + `Secret with name: ${secretName} does not exist`, + { cause: e }, + ); + } + throw e; + } + return results; + }); + }, + tran, + ); + for (const { filePath, value } of secrets) { + yield { + nameOrId: nameOrId, + secretName: filePath, + secretContent: value, + }; + } + } + }); + }; +} + +export default VaultsSecretsList; diff --git a/src/client/handlers/index.ts b/src/client/handlers/index.ts index 4449f294c..be76bf227 100644 --- a/src/client/handlers/index.ts +++ b/src/client/handlers/index.ts @@ -75,6 +75,7 @@ import VaultsRename from './VaultsRename'; import VaultsScan from './VaultsScan'; import VaultsSecretsDelete from './VaultsSecretsDelete'; import VaultsSecretsEdit from './VaultsSecretsEdit'; +import VaultsSecretsEnv from './VaultsSecretsEnv'; import VaultsSecretsGet from './VaultsSecretsGet'; import VaultsSecretsList from './VaultsSecretsList'; import VaultsSecretsMkdir from './VaultsSecretsMkdir'; @@ -175,6 +176,7 @@ const serverManifest = (container: { vaultsScan: new VaultsScan(container), vaultsSecretsDelete: new VaultsSecretsDelete(container), vaultsSecretsEdit: new VaultsSecretsEdit(container), + vaultsSecretsEnv: new VaultsSecretsEnv(container), vaultsSecretsGet: new VaultsSecretsGet(container), vaultsSecretsList: new VaultsSecretsList(container), vaultsSecretsMkdir: new VaultsSecretsMkdir(container), @@ -249,6 +251,7 @@ export { VaultsScan, VaultsSecretsDelete, VaultsSecretsEdit, + VaultsSecretsEnv, VaultsSecretsGet, VaultsSecretsList, VaultsSecretsMkdir, diff --git a/src/git/utils.ts b/src/git/utils.ts index a6218a373..65889b9d2 100644 --- a/src/git/utils.ts +++ b/src/git/utils.ts @@ -192,7 +192,7 @@ async function listRefs( const packedMap = packedRefs(fs, gitdir); let files: string[] = []; try { - for await (const file of vaultsUtils.readdirRecursively( + for await (const file of vaultsUtils.readDirRecursively( fs, path.join(gitdir, filepath), )) { diff --git a/src/vaults/VaultOps.ts b/src/vaults/VaultOps.ts index 3de1edd1c..ac6a7a75c 100644 --- a/src/vaults/VaultOps.ts +++ b/src/vaults/VaultOps.ts @@ -204,7 +204,7 @@ async function addSecretDirectory( const absoluteDirPath = path.resolve(secretDirectory); await vault.writeF(async (efs) => { - for await (const secretPath of vaultsUtils.readdirRecursively( + for await (const secretPath of vaultsUtils.readDirRecursively( fs, absoluteDirPath, )) { @@ -253,7 +253,7 @@ async function addSecretDirectory( async function listSecrets(vault: Vault): Promise { return await vault.readF(async (efs) => { const secrets: string[] = []; - for await (const secret of vaultsUtils.readdirRecursively(efs)) { + for await (const secret of vaultsUtils.readDirRecursively(efs)) { secrets.push(secret); } return secrets; diff --git a/src/vaults/utils.ts b/src/vaults/utils.ts index 191d52385..e9f216e62 100644 --- a/src/vaults/utils.ts +++ b/src/vaults/utils.ts @@ -1,7 +1,14 @@ import type { EncryptedFS } from 'encryptedfs'; -import type { VaultRef, VaultAction, CommitId } from './types'; +import type { + VaultRef, + VaultAction, + CommitId, + FileSystemReadable, +} from './types'; import type { NodeId } from '../ids/types'; +import type { Path } from 'encryptedfs/dist/types'; import path from 'path'; +import { pathJoin } from 'encryptedfs/dist/utils'; import { tagLast, refs, vaultActions } from './types'; import * as nodesUtils from '../nodes/utils'; import * as validationErrors from '../validation/errors'; @@ -36,19 +43,40 @@ function commitAuthor(nodeId: NodeId): { name: string; email: string } { }; } -async function* readdirRecursively(fs, dir = '.') { +async function* readDirRecursively(fs, dir = '.') { const dirents = await fs.promises.readdir(dir); for (const dirent of dirents) { const res = path.join(dir, dirent.toString()); const stat = await fs.promises.stat(res); if (stat.isDirectory()) { - yield* readdirRecursively(fs, res); + yield* readDirRecursively(fs, res); } else if (stat.isFile()) { yield res; } } } +async function* walkFs( + efs: FileSystemReadable, + path: string = '.', +): AsyncGenerator { + const shortList: Array = [path]; + let path_: Path | undefined = undefined; + while ((path_ = shortList.shift()) != null) { + const pathStat = await efs.stat(path_); + if (pathStat.isDirectory()) { + // Push contents to shortlist + const newPaths = await efs.readdir(path_); + shortList.push( + ...newPaths.map((v) => pathJoin(path_!.toString(), v.toString())), + ); + } else { + // Is a file so we yield the path + yield path_; + } + } +} + function isVaultAction(action: any): action is VaultAction { if (typeof action !== 'string') return false; return (vaultActions as Readonly>).includes(action); @@ -84,7 +112,8 @@ export { commitAuthor, isVaultAction, parseVaultAction, - readdirRecursively, + readDirRecursively, + walkFs, deleteObject, }; diff --git a/tests/client/handlers/vaults.test.ts b/tests/client/handlers/vaults.test.ts index 562e50dfe..000a5434b 100644 --- a/tests/client/handlers/vaults.test.ts +++ b/tests/client/handlers/vaults.test.ts @@ -4,6 +4,7 @@ import type { VaultId } from '@/ids'; import type NodeManager from '@/nodes/NodeManager'; import type { LogEntryMessage, + SecretContentMessage, VaultListMessage, VaultPermissionMessage, } from '@/client/types'; @@ -31,6 +32,7 @@ import { VaultsRename, VaultsSecretsDelete, VaultsSecretsEdit, + VaultsSecretsEnv, VaultsSecretsGet, VaultsSecretsList, VaultsSecretsMkdir, @@ -51,6 +53,7 @@ import { vaultsRename, vaultsSecretsDelete, vaultsSecretsEdit, + vaultsSecretsEnv, vaultsSecretsGet, vaultsSecretsList, vaultsSecretsMkdir, @@ -942,6 +945,269 @@ describe('vaultsSecretsEdit', () => { }); }); }); +describe('vaultsSecretEnv', () => { + const logger = new Logger('vaultsSecretEnv test', LogLevel.WARN, [ + new StreamHandler( + formatting.format`${formatting.level}:${formatting.keys}:${formatting.msg}`, + ), + ]); + const password = 'helloWorld'; + const localhost = '127.0.0.1'; + let dataDir: string; + let db: DB; + let keyRing: KeyRing; + let tlsConfig: TLSConfig; + let clientService: ClientService; + let webSocketClient: WebSocketClient; + let rpcClient: RPCClient<{ + vaultsSecretsEnv: typeof vaultsSecretsEnv; + }>; + let vaultManager: VaultManager; + beforeEach(async () => { + dataDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'polykey-test-'), + ); + const keysPath = path.join(dataDir, 'keys'); + keyRing = await KeyRing.createKeyRing({ + password, + keysPath, + passwordOpsLimit: keysUtils.passwordOpsLimits.min, + passwordMemLimit: keysUtils.passwordMemLimits.min, + strictMemoryLock: false, + logger, + }); + tlsConfig = await testsUtils.createTLSConfig(keyRing.keyPair); + const dbPath = path.join(dataDir, 'db'); + db = await DB.createDB({ + dbPath, + logger, + }); + const vaultsPath = path.join(dataDir, 'vaults'); + vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + db, + acl: {} as ACL, + keyRing, + nodeManager: {} as NodeManager, + gestaltGraph: {} as GestaltGraph, + notificationsManager: {} as NotificationsManager, + logger, + }); + clientService = new ClientService({ + tlsConfig, + logger: logger.getChild(ClientService.name), + }); + await clientService.start({ + manifest: { + vaultsSecretsEnv: new VaultsSecretsEnv({ + db, + vaultManager, + }), + }, + host: localhost, + }); + webSocketClient = await WebSocketClient.createWebSocketClient({ + config: { + verifyPeer: false, + }, + host: localhost, + logger: logger.getChild(WebSocketClient.name), + port: clientService.port, + }); + rpcClient = new RPCClient({ + manifest: { + vaultsSecretsEnv, + }, + streamFactory: () => webSocketClient.connection.newStream(), + toError: networkUtils.toError, + logger: logger.getChild(RPCClient.name), + }); + }); + afterEach(async () => { + await clientService?.stop({ force: true }); + await webSocketClient.destroy({ force: true }); + await vaultManager.stop(); + await db.stop(); + await keyRing.stop(); + await fs.promises.rm(dataDir, { + force: true, + recursive: true, + }); + }); + + test('should get secrets', async () => { + // Demonstrating we can pull out multiple secrets across separate vaults + const vaultName1 = 'vault1'; + const vaultName2 = 'vault2'; + const secretName1 = 'secret1'; + const secretName2 = 'secret2'; + const secretName3 = 'secret3'; + const secretName4 = 'secret4'; + const vaultId1 = await vaultManager.createVault(vaultName1); + await vaultManager.withVaults([vaultId1], async (vault) => { + await vault.writeF(async (efs) => { + await efs.writeFile(secretName1, secretName1); + await efs.writeFile(secretName2, secretName2); + }); + }); + const vaultId2 = await vaultManager.createVault(vaultName2); + await vaultManager.withVaults([vaultId2], async (vault) => { + await vault.writeF(async (efs) => { + await efs.writeFile(secretName3, secretName3); + await efs.writeFile(secretName4, secretName4); + }); + }); + + const secrets = [ + [vaultName1, secretName1], + [vaultName1, secretName2], + [vaultName2, secretName3], + [vaultName2, secretName4], + ]; + + const duplexStream = await rpcClient.methods.vaultsSecretsEnv(); + const writeP = (async () => { + const writer = duplexStream.writable.getWriter(); + for (const [name, secret] of secrets) { + await writer.write({ + nameOrId: name, + secretName: secret, + }); + } + await writer.close(); + })(); + const results: Array = []; + for await (const value of duplexStream.readable) { + results.push(value); + } + await writeP; + + expect(results[0]).toMatchObject({ + nameOrId: vaultName1, + secretName: secretName1, + secretContent: secretName1, + }); + expect(results[1]).toMatchObject({ + nameOrId: vaultName1, + secretName: secretName2, + secretContent: secretName2, + }); + expect(results[2]).toMatchObject({ + nameOrId: vaultName2, + secretName: secretName3, + secretContent: secretName3, + }); + expect(results[3]).toMatchObject({ + nameOrId: vaultName2, + secretName: secretName4, + secretContent: secretName4, + }); + }); + test('should get secrets by directory', async () => { + // Demonstrating we can pull out multiple secrets across separate vaults + const vaultName1 = 'vault1'; + const dirName1 = 'dir1'; + const dirName2 = 'dir2'; + const dirName3 = 'dir3'; + const secretName1 = 'secret1'; + const secretName2 = 'secret2'; + const secretName3 = 'secret3'; + const secretName4 = 'secret4'; + const vaultId1 = await vaultManager.createVault(vaultName1); + + await vaultManager.withVaults([vaultId1], async (vault) => { + await vault.writeF(async (efs) => { + await efs.mkdir(dirName1); + await efs.writeFile(`${dirName1}/${secretName1}`, secretName1); + await efs.writeFile(`${dirName1}/${secretName2}`, secretName2); + await efs.mkdir(dirName2); + await efs.writeFile(`${dirName2}/${secretName3}`, secretName3); + await efs.mkdir(`${dirName2}/${dirName3}`); + await efs.writeFile( + `${dirName2}/${dirName3}/${secretName4}`, + secretName4, + ); + }); + }); + + const secrets = [ + [vaultName1, dirName1], + [vaultName1, dirName2], + ]; + + const duplexStream = await rpcClient.methods.vaultsSecretsEnv(); + const writeP = (async () => { + const writer = duplexStream.writable.getWriter(); + for (const [name, secret] of secrets) { + await writer.write({ + nameOrId: name, + secretName: secret, + }); + } + await writer.close(); + })(); + const results: Map = new Map(); + for await (const value of duplexStream.readable) { + results.set(value.secretName, value); + } + await writeP; + expect(results.size).toBe(4); + expect(results.has(`${dirName1}/${secretName1}`)).toBeTrue(); + expect(results.get(`${dirName1}/${secretName1}`)).toMatchObject({ + nameOrId: vaultName1, + secretName: `${dirName1}/${secretName1}`, + secretContent: secretName1, + }); + expect(results.has(`${dirName1}/${secretName2}`)).toBeTrue(); + expect(results.get(`${dirName1}/${secretName2}`)).toMatchObject({ + nameOrId: vaultName1, + secretName: `${dirName1}/${secretName2}`, + secretContent: secretName2, + }); + expect(results.has(`${dirName2}/${dirName3}/${secretName4}`)).toBeTrue(); + expect(results.get(`${dirName2}/${dirName3}/${secretName4}`)).toMatchObject( + { + nameOrId: vaultName1, + secretName: `${dirName2}/${dirName3}/${secretName4}`, + secretContent: secretName4, + }, + ); + expect(results.has(`${dirName2}/${secretName3}`)).toBeTrue(); + expect(results.get(`${dirName2}/${secretName3}`)).toMatchObject({ + nameOrId: vaultName1, + secretName: `${dirName2}/${secretName3}`, + secretContent: secretName3, + }); + }); + test('errors should be descriptive', async () => { + // Demonstrating we can pull out multiple secrets across separate vaults + const vaultName1 = 'vault1'; + await vaultManager.createVault(vaultName1); + + const secrets = [[vaultName1, 'noSecret']]; + + const duplexStream = await rpcClient.methods.vaultsSecretsEnv(); + const writeP = (async () => { + const writer = duplexStream.writable.getWriter(); + for (const [name, secret] of secrets) { + await writer.write({ + nameOrId: name, + secretName: secret, + }); + } + await writer.close(); + })(); + await testsUtils.expectRemoteError( + (async () => { + for await (const _ of duplexStream.readable) { + // Do nothing until it throws + } + })(), + vaultsErrors.ErrorSecretsSecretUndefined, + ); + await writeP; + }); +}); describe('vaultsSecretsMkdir', () => { const logger = new Logger('vaultsSecretsMkdir test', LogLevel.WARN, [ new StreamHandler( diff --git a/tests/vaults/utils.test.ts b/tests/vaults/utils.test.ts index ef510a179..9ec256385 100644 --- a/tests/vaults/utils.test.ts +++ b/tests/vaults/utils.test.ts @@ -40,14 +40,14 @@ describe('Vaults utils', () => { const filePath1 = path.join('dir', 'file'); await efs.promises.writeFile(filePath1, 'content'); let files: string[] = []; - for await (const file of vaultsUtils.readdirRecursively(efs, './')) { + for await (const file of vaultsUtils.readDirRecursively(efs, './')) { files.push(file); } expect(files).toStrictEqual([filePath1]); files = []; const filePath2 = path.join('dir', 'dir2', 'dir3', 'file'); await efs.promises.writeFile(filePath2, 'content'); - for await (const file of vaultsUtils.readdirRecursively(efs)) { + for await (const file of vaultsUtils.readDirRecursively(efs)) { files.push(file); } expect(files.sort()).toStrictEqual([filePath1, filePath2].sort()); @@ -60,14 +60,14 @@ describe('Vaults utils', () => { const filePath1 = path.join(dataDir, 'dir', 'file'); await fs.promises.writeFile(filePath1, 'content'); let files: string[] = []; - for await (const file of vaultsUtils.readdirRecursively(fs, dataDir)) { + for await (const file of vaultsUtils.readDirRecursively(fs, dataDir)) { files.push(file); } expect(files).toStrictEqual([filePath1]); files = []; const filePath2 = path.join(dataDir, 'dir', 'dir2', 'dir3', 'file'); await fs.promises.writeFile(filePath2, 'content'); - for await (const file of vaultsUtils.readdirRecursively(fs, dataDir)) { + for await (const file of vaultsUtils.readDirRecursively(fs, dataDir)) { files.push(file); } expect(files.sort()).toStrictEqual([filePath1, filePath2].sort());