From be9f19a0c697045f4d3a2c4056896bc0668acde8 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Thu, 15 Feb 2024 13:08:03 +1100 Subject: [PATCH 1/9] deps: updated `polykey` to `^1.2.1-alpha.49` [cvi skip] --- package-lock.json | 48 +++++++++++++++++++++++------------------------ package.json | 2 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/package-lock.json b/package-lock.json index bc25d09a..f5b4a97d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,7 +40,7 @@ "nexpect": "^0.6.0", "node-gyp-build": "^4.4.0", "nodemon": "^3.0.1", - "polykey": "^1.2.1-alpha.47", + "polykey": "^1.2.1-alpha.49", "prettier": "^3.0.0", "shelljs": "^0.8.5", "shx": "^0.3.4", @@ -1476,9 +1476,9 @@ ] }, "node_modules/@matrixai/quic": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/@matrixai/quic/-/quic-1.1.4.tgz", - "integrity": "sha512-WhwoFzw2fuxDCxzY/WTZnMHg2snSGpGvifh4K6iRPfnvUfKME0ikFty7EMkRt8bQiAes4yxBbcI72/wBSeTYMA==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@matrixai/quic/-/quic-1.2.1.tgz", + "integrity": "sha512-Dz/IzRrYIV407aG3GAM7+4eoqntBZ/QOP4sBugd/5ZgiS8vwIdXQzzfGfetcM9SRp3I67v5teXDpavb4JAcWYg==", "dev": true, "dependencies": { "@matrixai/async-cancellable": "^1.1.1", @@ -1493,16 +1493,16 @@ "ip-num": "^1.5.0" }, "optionalDependencies": { - "@matrixai/quic-darwin-arm64": "1.1.4", - "@matrixai/quic-darwin-x64": "1.1.4", - "@matrixai/quic-linux-x64": "1.1.4", - "@matrixai/quic-win32-x64": "1.1.4" + "@matrixai/quic-darwin-arm64": "1.2.1", + "@matrixai/quic-darwin-x64": "1.2.1", + "@matrixai/quic-linux-x64": "1.2.1", + "@matrixai/quic-win32-x64": "1.2.1" } }, "node_modules/@matrixai/quic-darwin-arm64": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/@matrixai/quic-darwin-arm64/-/quic-darwin-arm64-1.1.4.tgz", - "integrity": "sha512-/Dho39T4sjbdOeC01F4rDesssQ9nvfdH57Ap95870LoY1KwXWh+c7cAtdv9FVma2R+9cqdKNkfsMFcS68Y15KA==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@matrixai/quic-darwin-arm64/-/quic-darwin-arm64-1.2.1.tgz", + "integrity": "sha512-TutHRdQoPYPvwg8Fs6t+JmedKWn/7mJG63c5ZFyw9D58a90OzYr47re2qknKNjH/dgBIyybhBYTAltkOcXxzOg==", "cpu": [ "arm64" ], @@ -1512,9 +1512,9 @@ ] }, "node_modules/@matrixai/quic-darwin-x64": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/@matrixai/quic-darwin-x64/-/quic-darwin-x64-1.1.4.tgz", - "integrity": "sha512-soxYopXZjoHxSMvpoNEKQQS1uQcXK2aIuaJHDgZINz4UbeLW0NsoEqMgUJKXeWZNR+aaVxhL5fE5+5Ypwej80Q==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@matrixai/quic-darwin-x64/-/quic-darwin-x64-1.2.1.tgz", + "integrity": "sha512-HMIhCHjAvvAgSap2gQyKYgWyhikDtmU9KbqtReMT3tP8KG3jNPAw2YuF7vucVR9k6FaMufISLYyBvWQSadTxIg==", "cpu": [ "x64" ], @@ -1524,9 +1524,9 @@ ] }, "node_modules/@matrixai/quic-linux-x64": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/@matrixai/quic-linux-x64/-/quic-linux-x64-1.1.4.tgz", - "integrity": "sha512-LRKkVX++UZEj00XhfD0cPe4uFQJeHOBpQl+v6BIL7LSB9KxsmQRgbDh5mvxvWAwnh5EI51aSgn4auWEdHMDjTw==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@matrixai/quic-linux-x64/-/quic-linux-x64-1.2.1.tgz", + "integrity": "sha512-WgvzATQhuVMV1cCiV0rRS9Tn3BL6LZ37OWBeQY20OJB4yaF9FR3xgUdATY3AVgF1FAdwvCHUJCF5BgzQmq8NMw==", "cpu": [ "x64" ], @@ -1536,9 +1536,9 @@ ] }, "node_modules/@matrixai/quic-win32-x64": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/@matrixai/quic-win32-x64/-/quic-win32-x64-1.1.4.tgz", - "integrity": "sha512-JNIQn1GRCRDm23v7hkqNVRm0oV928AHVfNPXJOB6FqBtbmbe/DaqUnppFi3qqUxm5Agguhd1rgNfKkL4HRhq8A==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@matrixai/quic-win32-x64/-/quic-win32-x64-1.2.1.tgz", + "integrity": "sha512-2F/llbBRzd2ohLLBZSW2ZIQTq8EHZ6jsrkv+/x0gggvRKaoPKmE2UMqcAYrfX/h5D48Gb3/JynD6AtFYGSBKFw==", "cpu": [ "x64" ], @@ -7488,9 +7488,9 @@ } }, "node_modules/polykey": { - "version": "1.2.1-alpha.47", - "resolved": "https://registry.npmjs.org/polykey/-/polykey-1.2.1-alpha.47.tgz", - "integrity": "sha512-nFQkDY8oE3ZcQgRlKrDIYFhYXzxJbFh3UAAxIovPG9HwO33jXbHckmsqLdir9p9wKgjvA57/RR7WTD5BaNh7Tg==", + "version": "1.2.1-alpha.49", + "resolved": "https://registry.npmjs.org/polykey/-/polykey-1.2.1-alpha.49.tgz", + "integrity": "sha512-81kWTeJ6M8x6I/cbuHMEBFLo7K6cbT7NNNVPfNsnp8JJaeGQmwTAODCS4WGP7wqAC0NVTFRaqYMUpNVvbJ9RNQ==", "dev": true, "dependencies": { "@matrixai/async-cancellable": "^1.1.1", @@ -7503,7 +7503,7 @@ "@matrixai/id": "^3.3.6", "@matrixai/logger": "^3.1.2", "@matrixai/mdns": "^1.2.5", - "@matrixai/quic": "^1.1.4", + "@matrixai/quic": "^1.2.1", "@matrixai/resources": "^1.1.5", "@matrixai/rpc": "^0.5.0", "@matrixai/timer": "^1.1.3", diff --git a/package.json b/package.json index bc9189e9..0a22d369 100644 --- a/package.json +++ b/package.json @@ -134,7 +134,7 @@ "nexpect": "^0.6.0", "node-gyp-build": "^4.4.0", "nodemon": "^3.0.1", - "polykey": "^1.2.1-alpha.47", + "polykey": "^1.2.1-alpha.49", "prettier": "^3.0.0", "shelljs": "^0.8.5", "shx": "^0.3.4", From b0d5f1af7c1ce51d220b88c77df3138a26762ba1 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Thu, 22 Feb 2024 17:40:33 +1100 Subject: [PATCH 2/9] deps: added `@matrixai/exec` as a dependency [ci skip] --- package-lock.json | 51 +++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 2 files changed, 52 insertions(+) diff --git a/package-lock.json b/package-lock.json index f5b4a97d..f4df2144 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ }, "devDependencies": { "@matrixai/errors": "^1.2.0", + "@matrixai/exec": "^0.1.1", "@matrixai/logger": "^3.1.0", "@swc/core": "1.3.82", "@swc/jest": "^0.2.29", @@ -1421,6 +1422,56 @@ "integrity": "sha512-bZrNCwzYeFalGQpn8qa/jgD10mUAwLRbv6xGMI7gGz1f+vE65d3GPoJ6JoFOJSg9iCmRSayQJ+IipH3LMATvDA==", "devOptional": true }, + "node_modules/@matrixai/exec": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@matrixai/exec/-/exec-0.1.1.tgz", + "integrity": "sha512-jKq8cVRFMPOuKyNfTjLrg+LoIZPLkDrRs0uQyx5fv8v1Uc1eiBOnQsvOh33VyVCsFMiK+aSukLoE/7xh8Yq5GQ==", + "dev": true, + "optionalDependencies": { + "@matrixai/exec-darwin-arm64": "0.1.1", + "@matrixai/exec-darwin-x64": "0.1.1", + "@matrixai/exec-linux-x64": "0.1.1" + } + }, + "node_modules/@matrixai/exec-darwin-arm64": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@matrixai/exec-darwin-arm64/-/exec-darwin-arm64-0.1.1.tgz", + "integrity": "sha512-JabHQa+/uA8Viqab7br40kOWVVIhotoyQnVQppzUt8ed/7npWczQQKtdspAafG3AjUSCxcg6aptaUX3NZ8/A4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@matrixai/exec-darwin-x64": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@matrixai/exec-darwin-x64/-/exec-darwin-x64-0.1.1.tgz", + "integrity": "sha512-i7ShSWrM8bbB9GUtMbhTlyRjzAB/hw1eDYHEFFNFJvMJ220p902TbJTajb9+OxHD/6vk3arslo1uA9okr5ZHvA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@matrixai/exec-linux-x64": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@matrixai/exec-linux-x64/-/exec-linux-x64-0.1.1.tgz", + "integrity": "sha512-AlGoFHGDSDsFjUXjTnVXRGMLYfb/Y1ApblSQ4xWhZo/xwqi0of9+85LxDicboEPl5k6teRbYV9BY4DRmZYbFfA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, "node_modules/@matrixai/id": { "version": "3.3.6", "resolved": "https://registry.npmjs.org/@matrixai/id/-/id-3.3.6.tgz", diff --git a/package.json b/package.json index 0a22d369..ce0fe4c9 100644 --- a/package.json +++ b/package.json @@ -109,6 +109,7 @@ "devDependencies": { "@matrixai/errors": "^1.2.0", "@matrixai/logger": "^3.1.0", + "@matrixai/exec": "^0.1.1", "@swc/core": "1.3.82", "@swc/jest": "^0.2.29", "@types/jest": "^29.5.2", From 66fcbdb929bfeec0ecca52c7e6399719bd82b079 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Mon, 12 Feb 2024 17:58:05 +1100 Subject: [PATCH 3/9] feat: secrets env command --- npmDepsHash | 2 +- src/secrets/CommandEnv.ts | 349 +++++++++++++++---------------- src/secrets/CommandSecrets.ts | 4 +- src/utils/options.ts | 20 ++ src/utils/parsers.ts | 5 +- tests/secrets/env.test.ts | 382 ++++++++++++++++++++++++++++++++++ 6 files changed, 578 insertions(+), 184 deletions(-) create mode 100644 tests/secrets/env.test.ts diff --git a/npmDepsHash b/npmDepsHash index 4e62ea74..d87741b0 100644 --- a/npmDepsHash +++ b/npmDepsHash @@ -1 +1 @@ -sha256-buIzGNjR5CMM7BNlw5srJtVTSH/6Ru+UsMeXeNOjgZ4= +sha256-LZ3QekhwOZj3IkbnZVdC/lbUvwhbJXROzd9oTxrMs7s= diff --git a/src/secrets/CommandEnv.ts b/src/secrets/CommandEnv.ts index 48014650..c19cb40c 100644 --- a/src/secrets/CommandEnv.ts +++ b/src/secrets/CommandEnv.ts @@ -1,179 +1,170 @@ -// Import { spawn } from 'child_process'; -// import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; -// import * as vaultsPB from 'polykey/dist/proto/js/polykey/v1/vaults/vaults_pb'; -// import * as secretsPB from 'polykey/dist/proto/js/polykey/v1/secrets/secrets_pb'; -// import PolykeyClient from 'polykey/dist/PolykeyClient'; -// import * as utils from 'polykey/dist/utils'; -// import * as binUtils from '../utils'; -// import * as CLIErrors from '../errors'; -// import * as grpcErrors from 'polykey/dist/grpc/errors'; - -// import CommandPolykey from '../CommandPolykey'; -// import * as binOptions from '../utils/options'; - -// class CommandEnv extends CommandPolykey { -// constructor(...args: ConstructorParameters) { -// super(...args); -// this.name('env'); -// this.description('Secrets Env'); -// this.option( -// '--command ', -// 'In the environment of the derivation, run the shell command cmd in an interactive shell (Use --run to use a non-interactive shell instead)', -// ); -// this.option( -// '--run ', -// 'In the environment of the derivation, run the shell command cmd in a non-interactive shell, meaning (among other things) that if you hit Ctrl-C while the command is running, the shell exits (Use --command to use an interactive shell instead)', -// ); -// this.arguments( -// "Secrets to inject into env, of the format ':[=]', you can also control what the environment variable will be called using '[]' (defaults to upper, snake case of the original secret name)", -// ); -// this.addOption(binOptions.nodeId); -// this.addOption(binOptions.clientHost); -// this.addOption(binOptions.clientPort); -// this.action(async (options, command) => { - -// }); -// } -// } - -// export default CommandEnv; - -// OLD COMMAND -// const env = binUtils.createCommand('env', { -// description: 'Runs a modified environment with injected secrets', -// nodePath: true, -// verbose: true, -// format: true, -// }); -// env.option( -// '--command ', -// 'In the environment of the derivation, run the shell command cmd in an interactive shell (Use --run to use a non-interactive shell instead)', -// ); -// env.option( -// '--run ', -// 'In the environment of the derivation, run the shell command cmd in a non-interactive shell, meaning (among other things) that if you hit Ctrl-C while the command is running, the shell exits (Use --command to use an interactive shell instead)', -// ); -// env.arguments( -// "Secrets to inject into env, of the format ':[=]', you can also control what the environment variable will be called using '[]' (defaults to upper, snake case of the original secret name)", -// ); -// env.action(async (options, command) => { -// const clientConfig = {}; -// clientConfig['logger'] = new Logger('CLI Logger', LogLevel.WARN, [ -// new StreamHandler(), -// ]); -// if (options.verbose) { -// clientConfig['logger'].setLevel(LogLevel.DEBUG); -// } -// clientConfig['nodePath'] = options.nodePath -// ? options.nodePath -// : utils.getDefaultNodePath(); - -// const client = await PolykeyClient.createPolykeyClient(clientConfig); -// const vaultMessage = new vaultsPB.Vault(); -// const secretMessage = new secretsPB.Secret(); -// secretMessage.setVault(vaultMessage); -// const secretPathList: string[] = Array.from(command.args.values()); - -// try { -// if (secretPathList.length < 1) { -// throw new CLIErrors.ErrorSecretsUndefined(); -// } - -// const parsedPathList: { -// vaultName: string; -// secretName: string; -// variableName: string; -// }[] = []; - -// for (const path of secretPathList) { -// if (!binUtils.pathRegex.test(path)) { -// throw new CLIErrors.ErrorSecretPathFormat(); -// } - -// const [, vaultName, secretName, variableName] = path.match( -// binUtils.pathRegex, -// )!; -// parsedPathList.push({ -// vaultName, -// secretName, -// variableName: -// variableName ?? secretName.toUpperCase().replace('-', '_'), -// }); -// } - -// const secretEnv = { ...process.env }; - -// await client.start({}); -// const grpcClient = client.grpcClient; - -// for (const obj of parsedPathList) { -// vaultMessage.setNameOrId(obj.vaultName); -// secretMessage.setSecretName(obj.secretName); -// const res = await binUtils.unaryCallCARL( -// client, -// attemptUnaryCall(client, grpcClient.vaultsSecretsGet), -// )(secretMessage); - -// const secret = res.getSecretName(); -// secretEnv[obj.variableName] = secret; -// } - -// const shellPath = process.env.SHELL ?? 'sh'; -// const args: string[] = []; - -// if (options.command && options.run) { -// throw new CLIErrors.ErrorInvalidArguments( -// 'Only one of --command or --run can be specified', -// ); -// } else if (options.command) { -// args.push('-i'); -// args.push('-c'); -// args.push(`"${options.command}"`); -// } else if (options.run) { -// args.push('-c'); -// args.push(`"${options.run}"`); -// } - -// const shell = spawn(shellPath, args, { -// stdio: 'inherit', -// env: secretEnv, -// shell: true, -// }); - -// shell.on('close', (code) => { -// if (code !== 0) { -// process.stdout.write( -// binUtils.outputFormatter({ -// type: options.format === 'json' ? 'json' : 'list', -// data: [`Terminated with ${code}`], -// }), -// ); -// } -// }); -// } catch (err) { -// if (err instanceof grpcErrors.ErrorGRPCClientTimeout) { -// process.stderr.write(`${err.message}\n`); -// } -// if (err instanceof grpcErrors.ErrorGRPCServerNotStarted) { -// process.stderr.write(`${err.message}\n`); -// } else { -// process.stderr.write( -// binUtils.outputFormatter({ -// type: 'error', -// description: err.description, -// message: err.message, -// }), -// ); -// throw err; -// } -// } finally { -// await client.stop(); -// options.nodePath = undefined; -// options.verbose = undefined; -// options.format = undefined; -// options.command = undefined; -// options.run = undefined; -// } -// }); - -// export default env; +import type PolykeyClient from 'polykey/dist/PolykeyClient'; +import path from 'path'; +import * as utils from 'polykey/dist/utils'; +import { exec } from '@matrixai/exec'; +import * as binProcessors from '../utils/processors'; +import * as binUtils from '../utils'; +import CommandPolykey from '../CommandPolykey'; +import * as binOptions from '../utils/options'; + +class CommandEnv extends CommandPolykey { + constructor(...args: ConstructorParameters) { + super(...args); + this.name('env'); + this.description( + 'Run a command with the given secrets and env variables. If no command is specified then the variables are printed to stdout.', + ); + this.addOption(binOptions.nodeId); + this.addOption(binOptions.clientHost); + this.addOption(binOptions.clientPort); + this.addOption(binOptions.envVariables); + this.addOption(binOptions.envFormat); + this.argument('[cmd] [argv...]', 'command and arguments'); + this.action(async (args: Array, options) => { + const [cmd, ...argv] = args; + const { + env: envVariables, + outputFormat, + }: { + env: Array<[string, string, string?]>; + outputFormat: 'dotenv' | 'json' | 'prepend'; + } = options; + + // There are a few stages here + // 1. parse the desired secrets + // 2. obtain the desired secrets + // 3. switching behaviour here based on parameters + // a. exec the command with the provided env variables from the secrets + // b. output the env variables in the desired format + + const { default: PolykeyClient } = await import( + 'polykey/dist/PolykeyClient' + ); + const clientOptions = await binProcessors.processClientOptions( + options.nodePath, + options.nodeId, + options.clientHost, + options.clientPort, + this.fs, + this.logger.getChild(binProcessors.processClientOptions.name), + ); + const meta = await binProcessors.processAuthentication( + options.passwordFile, + this.fs, + ); + + let pkClient: PolykeyClient; + this.exitHandlers.handlers.push(async () => { + if (pkClient != null) await pkClient.stop(); + }); + try { + pkClient = await PolykeyClient.createPolykeyClient({ + nodeId: clientOptions.nodeId, + host: clientOptions.clientHost, + port: clientOptions.clientPort, + options: { + nodePath: options.nodePath, + }, + logger: this.logger.getChild(PolykeyClient.name), + }); + + // Getting envs + const envp = await binUtils.retryAuthentication(async (auth) => { + const responseStream = + await pkClient.rpcClient.methods.vaultsSecretsEnv(); + // Writing desired secrets + const secretRenameMap = new Map(); + const writeP = (async () => { + const writer = responseStream.writable.getWriter(); + let first = true; + for (const envVariable of envVariables) { + const [nameOrId, secretName, secretNameNew] = envVariable; + secretRenameMap.set(secretName, secretNameNew); + await writer.write({ + nameOrId, + secretName, + metadata: first ? auth : undefined, + }); + first = false; + } + await writer.close(); + })(); + + const envp: Record = {}; + for await (const value of responseStream.readable) { + const { secretName, secretContent } = value; + let newName = secretRenameMap.get(secretName); + newName = newName ?? path.basename(secretName); + envp[newName] = secretContent; + } + await writeP; + return envp; + }, meta); + // End connection early to avoid errors on server + await pkClient.stop(); + + // Here we want to switch between the different usages + if (cmd != null) { + // If a cmd is provided then we default to exec it + exec.execvp(cmd, argv, envp); + } else { + // Otherwise we switch between output formats + switch (outputFormat) { + case 'dotenv': + { + // Formatting as a .env file + let data = ''; + for (const [key, value] of Object.entries(envp)) { + data += `${key}="${value}"\n`; + } + process.stdout.write( + binUtils.outputFormatter({ + type: 'raw', + data, + }), + ); + } + break; + case 'prepend': + { + // Formatting as a command input + let first = true; + let data = ''; + for (const [key, value] of Object.entries(envp)) { + data += `${first ? '' : ' '}${key}="${value}"`; + first = false; + } + process.stdout.write( + binUtils.outputFormatter({ + type: 'raw', + data, + }), + ); + } + break; + case 'json': + { + const data = {}; + for (const [key, value] of Object.entries(envp)) { + data[key] = value; + } + process.stdout.write( + binUtils.outputFormatter({ + type: 'json', + data: data, + }), + ); + } + break; + default: + utils.never(); + } + } + } finally { + if (pkClient! != null) await pkClient.stop(); + } + }); + } +} + +export default CommandEnv; diff --git a/src/secrets/CommandSecrets.ts b/src/secrets/CommandSecrets.ts index 0cf1c766..fe24384a 100644 --- a/src/secrets/CommandSecrets.ts +++ b/src/secrets/CommandSecrets.ts @@ -2,7 +2,7 @@ import CommandCreate from './CommandCreate'; import CommandDelete from './CommandDelete'; import CommandDir from './CommandDir'; import CommandEdit from './CommandEdit'; -// Import CommandEnv from './CommandEnv'; +import CommandEnv from './CommandEnv'; import CommandGet from './CommandGet'; import CommandList from './CommandList'; import CommandMkdir from './CommandMkdir'; @@ -20,7 +20,7 @@ class CommandSecrets extends CommandPolykey { this.addCommand(new CommandDelete(...args)); this.addCommand(new CommandDir(...args)); this.addCommand(new CommandEdit(...args)); - // This.addCommand(new CommandEnv(...args)); + this.addCommand(new CommandEnv(...args)); this.addCommand(new CommandGet(...args)); this.addCommand(new CommandList(...args)); this.addCommand(new CommandMkdir(...args)); diff --git a/src/utils/options.ts b/src/utils/options.ts index 8c494218..e24cbf3a 100644 --- a/src/utils/options.ts +++ b/src/utils/options.ts @@ -197,6 +197,24 @@ const commitId = new commander.Option( 'Id for a specific commit to read from', ); +const envVariables = new commander.Option('-e --env ', 'specify envs') + .makeOptionMandatory(true) + .argParser( + (value: string, previous: Array<[string, string, string?]> | undefined) => { + const acc = previous ?? []; + acc.push(binParsers.parseSecretPath(value)); + return acc; + }, + ); + +// '-f, --format ', 'Output Format' +const envFormat = new commander.Option( + '-of --output-format ', + 'How the env variables are formatted when outputted. Only used if no commands are executed', +) + .choices(['dotenv', 'json', 'prepend']) + .default('dotenv'); + export { nodePath, format, @@ -225,4 +243,6 @@ export { passwordMemLimit, depth, commitId, + envVariables, + envFormat, }; diff --git a/src/utils/parsers.ts b/src/utils/parsers.ts index 8a164082..866fd1a7 100644 --- a/src/utils/parsers.ts +++ b/src/utils/parsers.ts @@ -71,8 +71,9 @@ function parseSecretPath(secretPath: string): [string, string, string?] { `${secretPath} is not of the format :`, ); } - const [, vaultName, directoryPath] = secretPath.match(secretPathRegex)!; - return [vaultName, directoryPath, undefined]; + const [, vaultName, directoryPath, value] = + secretPath.match(secretPathRegex)!; + return [vaultName, directoryPath, value]; } const parseInteger: (data: string) => number = validateParserToArgParser( diff --git a/tests/secrets/env.test.ts b/tests/secrets/env.test.ts new file mode 100644 index 00000000..deaf0712 --- /dev/null +++ b/tests/secrets/env.test.ts @@ -0,0 +1,382 @@ +import type { VaultName } from 'polykey/dist/vaults/types'; +import path from 'path'; +import fs from 'fs'; +import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; +import PolykeyAgent from 'polykey/dist/PolykeyAgent'; +import { vaultOps } from 'polykey/dist/vaults'; +import * as keysUtils from 'polykey/dist/keys/utils'; +import * as testUtils from '../utils'; + +describe('commandEnv', () => { + const logger = new Logger('CLI Test', LogLevel.WARN, [new StreamHandler()]); + const password = 'password'; + const vaultName = 'vault' as VaultName; + let dataDir: string; + let polykeyAgent: PolykeyAgent; + let passwordFile: string; + let command: Array; + + beforeEach(async () => { + dataDir = await fs.promises.mkdtemp( + path.join(globalThis.tmpDir, 'polykey-test-'), + ); + passwordFile = path.join(dataDir, 'passwordFile'); + await fs.promises.writeFile(passwordFile, 'password'); + polykeyAgent = await PolykeyAgent.createPolykeyAgent({ + password, + options: { + nodePath: dataDir, + agentServiceHost: '127.0.0.1', + clientServiceHost: '127.0.0.1', + keys: { + passwordOpsLimit: keysUtils.passwordOpsLimits.min, + passwordMemLimit: keysUtils.passwordMemLimits.min, + strictMemoryLock: false, + }, + }, + logger: logger, + }); + // Authorize session + await testUtils.pkStdio( + ['agent', 'unlock', '-np', dataDir, '--password-file', passwordFile], + { + env: {}, + cwd: dataDir, + }, + ); + }); + afterEach(async () => { + await polykeyAgent.stop(); + await fs.promises.rm(dataDir, { + force: true, + recursive: true, + }); + }); + + test('can select 1 env variable', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, 'SECRET', 'this is the secret1'); + }); + + command = [ + 'secrets', + 'env', + '-np', + dataDir, + '-e', + `${vaultName}:SECRET`, + '--', + 'node', + '-e', + 'console.log(JSON.stringify(process.env))', + ]; + + const result = await testUtils.pkExec([...command]); + expect(result.exitCode).toBe(0); + const jsonOut = JSON.parse(result.stdout); + expect(jsonOut['SECRET']).toBe('this is the secret1'); + }); + test('can select multiple env variables', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, 'SECRET1', 'this is the secret1'); + await vaultOps.addSecret(vault, 'SECRET2', 'this is the secret2'); + }); + + command = [ + 'secrets', + 'env', + '-np', + dataDir, + '-e', + `${vaultName}:SECRET1`, + `${vaultName}:SECRET2`, + '--', + 'node', + '-e', + 'console.log(JSON.stringify(process.env))', + ]; + + const result = await testUtils.pkExec([...command]); + expect(result.exitCode).toBe(0); + const jsonOut = JSON.parse(result.stdout); + expect(jsonOut['SECRET1']).toBe('this is the secret1'); + expect(jsonOut['SECRET2']).toBe('this is the secret2'); + }); + test('can select a directory of env variables', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, 'SECRET1', 'this is the secret1'); + await vaultOps.mkdir(vault, 'dir1'); + await vaultOps.addSecret(vault, 'dir1/SECRET2', 'this is the secret2'); + await vaultOps.addSecret(vault, 'dir1/SECRET3', 'this is the secret3'); + }); + + command = [ + 'secrets', + 'env', + '-np', + dataDir, + '-e', + `${vaultName}:dir1`, + '--', + 'node', + '-e', + 'console.log(JSON.stringify(process.env))', + ]; + + const result = await testUtils.pkExec([...command]); + expect(result.exitCode).toBe(0); + const jsonOut = JSON.parse(result.stdout); + expect(jsonOut['SECRET1']).toBeUndefined(); + expect(jsonOut['SECRET2']).toBe('this is the secret2'); + expect(jsonOut['SECRET3']).toBe('this is the secret3'); + }); + test('can select and rename an env variable', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, 'SECRET', 'this is the secret'); + }); + + command = [ + 'secrets', + 'env', + '-np', + dataDir, + '-e', + `${vaultName}:SECRET=SECRET_NEW`, + '--', + 'node', + '-e', + 'console.log(JSON.stringify(process.env))', + ]; + + const result = await testUtils.pkExec([...command]); + expect(result.exitCode).toBe(0); + const jsonOut = JSON.parse(result.stdout); + expect(jsonOut['SECRET']).toBeUndefined(); + expect(jsonOut['SECRET_NEW']).toBe('this is the secret'); + }); + test('can not rename a directory of env variables', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.mkdir(vault, 'dir'); + await vaultOps.addSecret(vault, 'dir1/SECRET1', 'this is the secret1'); + await vaultOps.addSecret(vault, 'dir1/SECRET2', 'this is the secret2'); + }); + + command = [ + 'secrets', + 'env', + '-np', + dataDir, + '-e', + `${vaultName}:dir1=SECRET_NEW`, + '--', + 'node', + '-e', + 'console.log(JSON.stringify(process.env))', + ]; + + const result = await testUtils.pkExec([...command]); + expect(result.exitCode).toBe(0); + const jsonOut = JSON.parse(result.stdout); + expect(jsonOut['SECRET_NEW']).toBeUndefined(); + expect(jsonOut['SECRET1']).toBe('this is the secret1'); + expect(jsonOut['SECRET2']).toBe('this is the secret2'); + }); + test('can mix and match env variables', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, 'SECRET1', 'this is the secret1'); + await vaultOps.addSecret(vault, 'SECRET2', 'this is the secret2'); + await vaultOps.mkdir(vault, 'dir1'); + await vaultOps.addSecret(vault, 'dir1/SECRET3', 'this is the secret3'); + await vaultOps.addSecret(vault, 'dir1/SECRET4', 'this is the secret4'); + }); + + command = [ + 'secrets', + 'env', + '-np', + dataDir, + '-e', + `${vaultName}:SECRET1`, + `${vaultName}:SECRET2`, + `${vaultName}:dir1`, + '--', + 'node', + '-e', + 'console.log(JSON.stringify(process.env))', + ]; + + const result = await testUtils.pkExec([...command]); + expect(result.exitCode).toBe(0); + const jsonOut = JSON.parse(result.stdout); + expect(jsonOut['SECRET1']).toBe('this is the secret1'); + expect(jsonOut['SECRET2']).toBe('this is the secret2'); + expect(jsonOut['SECRET3']).toBe('this is the secret3'); + expect(jsonOut['SECRET4']).toBe('this is the secret4'); + }); + test('existing env are passed through', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, 'SECRET1', 'this is the secret1'); + }); + + command = [ + 'secrets', + 'env', + '-np', + dataDir, + '-e', + `${vaultName}:SECRET1`, + '--', + 'node', + '-e', + 'console.log(JSON.stringify(process.env))', + ]; + + const result = await testUtils.pkExec([...command], { + env: { + EXISTING: 'existing var', + }, + }); + expect(result.exitCode).toBe(0); + const jsonOut = JSON.parse(result.stdout); + expect(jsonOut['SECRET1']).toBe('this is the secret1'); + expect(jsonOut['EXISTING']).toBe('existing var'); + }); + test('handles duplicate secret names', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, 'SECRET1', 'this is the secret1'); + await vaultOps.addSecret(vault, 'SECRET2', 'this is the secret2'); + await vaultOps.addSecret(vault, 'SECRET3', 'this is the secret3'); + await vaultOps.mkdir(vault, 'dir1'); + await vaultOps.addSecret(vault, 'dir1/SECRET4', 'this is the secret4'); + }); + + command = [ + 'secrets', + 'env', + '-np', + dataDir, + '-e', + `${vaultName}:SECRET1`, + `${vaultName}:SECRET2=SECRET1`, + `${vaultName}:SECRET3=SECRET4`, + `${vaultName}:dir1`, + '--', + 'node', + '-e', + 'console.log(JSON.stringify(process.env))', + ]; + + const result = await testUtils.pkExec([...command]); + expect(result.exitCode).toBe(0); + const jsonOut = JSON.parse(result.stdout); + // Latter set envs override former ones, so secrets should be 2 and 4 + expect(jsonOut['SECRET1']).toBe('this is the secret2'); + expect(jsonOut['SECRET2']).toBeUndefined(); + expect(jsonOut['SECRET3']).toBeUndefined(); + expect(jsonOut['SECRET4']).toBe('this is the secret4'); + }); + test('should output .env format', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, 'SECRET1', 'this is the secret1'); + await vaultOps.addSecret(vault, 'SECRET2', 'this is the secret2'); + await vaultOps.mkdir(vault, 'dir1'); + await vaultOps.addSecret(vault, 'dir1/SECRET3', 'this is the secret3'); + await vaultOps.addSecret(vault, 'dir1/SECRET4', 'this is the secret4'); + }); + + command = [ + 'secrets', + 'env', + '-np', + dataDir, + '-e', + `${vaultName}:.`, + '--output-format', + 'dotenv', + ]; + + const result = await testUtils.pkExec([...command]); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('SECRET1="this is the secret1"'); + expect(result.stdout).toContain('SECRET2="this is the secret2"'); + expect(result.stdout).toContain('SECRET3="this is the secret3"'); + expect(result.stdout).toContain('SECRET4="this is the secret4"'); + }); + test('should output json format', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, 'SECRET1', 'this is the secret1'); + await vaultOps.addSecret(vault, 'SECRET2', 'this is the secret2'); + await vaultOps.mkdir(vault, 'dir1'); + await vaultOps.addSecret(vault, 'dir1/SECRET3', 'this is the secret3'); + await vaultOps.addSecret(vault, 'dir1/SECRET4', 'this is the secret4'); + }); + + command = [ + 'secrets', + 'env', + '-np', + dataDir, + '-e', + `${vaultName}:.`, + '--output-format', + 'json', + ]; + + const result = await testUtils.pkExec([...command]); + expect(result.exitCode).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ + SECRET1: 'this is the secret1', + SECRET2: 'this is the secret2', + SECRET3: 'this is the secret3', + SECRET4: 'this is the secret4', + }); + }); + test('should output prepend format', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, 'SECRET1', 'this is the secret1'); + await vaultOps.addSecret(vault, 'SECRET2', 'this is the secret2'); + await vaultOps.mkdir(vault, 'dir1'); + await vaultOps.addSecret(vault, 'dir1/SECRET3', 'this is the secret3'); + await vaultOps.addSecret(vault, 'dir1/SECRET4', 'this is the secret4'); + }); + + command = [ + 'secrets', + 'env', + '-np', + dataDir, + '-e', + `${vaultName}:.`, + '--output-format', + 'prepend', + ]; + + const result = await testUtils.pkExec([...command]); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe( + 'SECRET1="this is the secret1" SECRET2="this is the secret2" SECRET3="this is the secret3" SECRET4="this is the secret4"', + ); + }); +}); From fee88090efd86ec589ddb6f2e5f3716c9249a9ef Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Mon, 26 Feb 2024 14:50:36 +1100 Subject: [PATCH 4/9] feat: added duplicate and invalid env name handling flags [ci skip] --- src/errors.ts | 13 +++ src/secrets/CommandEnv.ts | 59 +++++++++- src/utils/options.ts | 20 +++- src/utils/parsers.ts | 18 +++ src/utils/utils.ts | 3 + tests/secrets/env.test.ts | 226 +++++++++++++++++++++++++++++++++++++- 6 files changed, 330 insertions(+), 9 deletions(-) diff --git a/src/errors.ts b/src/errors.ts index 9d26dbf2..1d6de160 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -141,6 +141,17 @@ class ErrorPolykeyCLINodePingFailed extends ErrorPolykeyCLI { exitCode = 1; } +class ErrorPolykeyCLIInvalidEnvName extends ErrorPolykeyCLI { + static description = + 'Secret retrieved has an invalid environment variable name'; + exitCode = sysexits.USAGE; +} + +class ErrorPolykeyCLIDuplicateEnvName extends ErrorPolykeyCLI { + static description = 'Environment variable name already retrieved'; + exitCode = sysexits.USAGE; +} + export { ErrorPolykeyCLI, ErrorPolykeyCLIUncaughtException, @@ -159,4 +170,6 @@ export { ErrorPolykeyCLIAgentProcess, ErrorPolykeyCLINodeFindFailed, ErrorPolykeyCLINodePingFailed, + ErrorPolykeyCLIInvalidEnvName, + ErrorPolykeyCLIDuplicateEnvName, }; diff --git a/src/secrets/CommandEnv.ts b/src/secrets/CommandEnv.ts index c19cb40c..a8627d67 100644 --- a/src/secrets/CommandEnv.ts +++ b/src/secrets/CommandEnv.ts @@ -4,6 +4,7 @@ import * as utils from 'polykey/dist/utils'; import { exec } from '@matrixai/exec'; import * as binProcessors from '../utils/processors'; import * as binUtils from '../utils'; +import * as binErrors from '../errors'; import CommandPolykey from '../CommandPolykey'; import * as binOptions from '../utils/options'; @@ -19,15 +20,21 @@ class CommandEnv extends CommandPolykey { this.addOption(binOptions.clientPort); this.addOption(binOptions.envVariables); this.addOption(binOptions.envFormat); + this.addOption(binOptions.envInvalid); + this.addOption(binOptions.envDuplicate); this.argument('[cmd] [argv...]', 'command and arguments'); this.action(async (args: Array, options) => { const [cmd, ...argv] = args; const { env: envVariables, - outputFormat, + envInvalid, + envDuplicate, + envFormat, }: { env: Array<[string, string, string?]>; - outputFormat: 'dotenv' | 'json' | 'prepend'; + envInvalid: 'error' | 'warn' | 'ignore'; + envDuplicate: 'keep' | 'overwrite' | 'warn' | 'error'; + envFormat: 'dotenv' | 'json' | 'prepend'; } = options; // There are a few stages here @@ -94,7 +101,51 @@ class CommandEnv extends CommandPolykey { for await (const value of responseStream.readable) { const { secretName, secretContent } = value; let newName = secretRenameMap.get(secretName); - newName = newName ?? path.basename(secretName); + if (newName == null) { + const secretEnvName = path.basename(secretName); + // Validating name + if (!binUtils.validEnvRegex.test(secretEnvName)) { + switch (envInvalid) { + case 'error': + throw new binErrors.ErrorPolykeyCLIInvalidEnvName( + `The following env variable name (${secretEnvName}) is invalid`, + ); + case 'warn': + this.logger.warn( + `The following env variable name (${secretEnvName}) is invalid and was dropped`, + ); + // Fallthrough + case 'ignore': + continue; + default: + utils.never(); + } + } + newName = secretEnvName; + } + // Handling duplicate names + if (envp[newName] != null) { + switch (envDuplicate) { + // Continue without modifying + case 'error': + throw new binErrors.ErrorPolykeyCLIDuplicateEnvName( + `The env variable (${newName}) is duplicate`, + ); + // Fallthrough + case 'keep': + continue; + // Log a warning and overwrite + case 'warn': + this.logger.warn( + `The env variable (${newName}) is duplicate, overwriting`, + ); + // Fallthrough + case 'overwrite': + break; + default: + utils.never(); + } + } envp[newName] = secretContent; } await writeP; @@ -109,7 +160,7 @@ class CommandEnv extends CommandPolykey { exec.execvp(cmd, argv, envp); } else { // Otherwise we switch between output formats - switch (outputFormat) { + switch (envFormat) { case 'dotenv': { // Formatting as a .env file diff --git a/src/utils/options.ts b/src/utils/options.ts index e24cbf3a..072ce2e1 100644 --- a/src/utils/options.ts +++ b/src/utils/options.ts @@ -202,19 +202,33 @@ const envVariables = new commander.Option('-e --env ', 'specify envs') .argParser( (value: string, previous: Array<[string, string, string?]> | undefined) => { const acc = previous ?? []; - acc.push(binParsers.parseSecretPath(value)); + acc.push(binParsers.parseEnvPath(value)); return acc; }, ); // '-f, --format ', 'Output Format' const envFormat = new commander.Option( - '-of --output-format ', + '-ef --env-format ', 'How the env variables are formatted when outputted. Only used if no commands are executed', ) .choices(['dotenv', 'json', 'prepend']) .default('dotenv'); +const envInvalid = new commander.Option( + '-ei --env-invalid ', + 'How invalid env variable names are handled when retrieving secrets. `error` will throw, `warn` will log a warning and drop and `ignore` will silently drop.', +) + .choices(['error', 'warn', 'ignore']) + .default('error'); + +const envDuplicate = new commander.Option( + '-ed --env-duplicate ', + 'How duplicate env variable names are handled. `keep` will keep the exising secret, `overwrite` will overwrite existing with the new secret, `warn` will log a warning and overwrite and `error` will throw.', +) + .choices(['keep', 'overwrite', 'warn', 'error']) + .default('overwrite'); + export { nodePath, format, @@ -245,4 +259,6 @@ export { commitId, envVariables, envFormat, + envInvalid, + envDuplicate, }; diff --git a/src/utils/parsers.ts b/src/utils/parsers.ts index 866fd1a7..307d0c45 100644 --- a/src/utils/parsers.ts +++ b/src/utils/parsers.ts @@ -76,6 +76,23 @@ function parseSecretPath(secretPath: string): [string, string, string?] { return [vaultName, directoryPath, value]; } +function parseEnvPath(secretPath: string): [string, string, string?] { + // E.g. If 'vault1:a/b/c', ['vault1', 'a/b/c'] is returned + // If 'vault1:a/b/c=VARIABLE', ['vault1, 'a/b/c', 'VARIABLE'] is returned + // VARIABLE must be a valid ENV variable name + const secretPathRegex = + /^([\w-]+)(?::)([\w\-\\\/\.\$]+)(?:=)?([a-zA-Z_]+[a-zA-Z0-9_]*)?$/; + // /^([\w-]+)(?::)([\w\-\\\/\.\$]+)(?:=)?([a-zA-Z_][\w]+)?$/; + if (!secretPathRegex.test(secretPath)) { + throw new commander.InvalidArgumentError( + `${secretPath} is not of the format :`, + ); + } + const [, vaultName, directoryPath, value] = + secretPath.match(secretPathRegex)!; + return [vaultName, directoryPath, value]; +} + const parseInteger: (data: string) => number = validateParserToArgParser( validationUtils.parseInteger, ); @@ -132,6 +149,7 @@ export { validateParserToArgListParser, parseCoreCount, parseSecretPath, + parseEnvPath, parseInteger, parseNumber, parseNodeId, diff --git a/src/utils/utils.ts b/src/utils/utils.ts index bf925b90..8ff76bd5 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -509,6 +509,8 @@ function remoteErrorCause(e: any): [any, number] { return [errorCause, depth]; } +const validEnvRegex = /[a-zA-Z_]+[a-zA-Z0-9_]*/; + export { verboseToLogLevel, standardErrorReplacer, @@ -527,6 +529,7 @@ export { decodeEscapedWrapped, decodeEscaped, decodeEscapedRegex, + validEnvRegex, }; export type { OutputObject }; diff --git a/tests/secrets/env.test.ts b/tests/secrets/env.test.ts index deaf0712..55dd842c 100644 --- a/tests/secrets/env.test.ts +++ b/tests/secrets/env.test.ts @@ -5,6 +5,7 @@ import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; import PolykeyAgent from 'polykey/dist/PolykeyAgent'; import { vaultOps } from 'polykey/dist/vaults'; import * as keysUtils from 'polykey/dist/keys/utils'; +import { sysexits } from 'polykey/dist/utils'; import * as testUtils from '../utils'; describe('commandEnv', () => { @@ -309,7 +310,7 @@ describe('commandEnv', () => { dataDir, '-e', `${vaultName}:.`, - '--output-format', + '--env-format', 'dotenv', ]; @@ -338,7 +339,7 @@ describe('commandEnv', () => { dataDir, '-e', `${vaultName}:.`, - '--output-format', + '--env-format', 'json', ]; @@ -369,7 +370,7 @@ describe('commandEnv', () => { dataDir, '-e', `${vaultName}:.`, - '--output-format', + '--env-format', 'prepend', ]; @@ -379,4 +380,223 @@ describe('commandEnv', () => { 'SECRET1="this is the secret1" SECRET2="this is the secret2" SECRET3="this is the secret3" SECRET4="this is the secret4"', ); }); + test('testing valid and invalid rename inputs', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, 'SECRET', 'this is the secret'); + }); + + const valid = [ + 'one', + 'ONE', + 'one_two', + 'ONE_two', + 'one_TWO', + 'ONE_TWO', + 'ONE123', + 'ONE_123', + ]; + + const invalid = ['123', '123abc', '123_123', '123_abc', '123 abc', ' ']; + + command = [ + 'secrets', + 'env', + '-np', + dataDir, + '-e', + `${vaultName}:SECRET=one_123_ABC`, + '--', + 'node', + '-e', + 'console.log(JSON.stringify(process.env))', + ]; + + // Checking valid + const result = await testUtils.pkExec([ + 'secrets', + 'env', + '-np', + dataDir, + '-e', + ...valid.map((v) => `${vaultName}:SECRET=${v}`), + ]); + expect(result.exitCode).toBe(0); + + // Checking invalid + for (const nameNew of invalid) { + const result = await testUtils.pkExec([ + 'secrets', + 'env', + '-np', + dataDir, + '-e', + `${vaultName}:SECRET=${nameNew}`, + ]); + expect(result.exitCode).toBe(sysexits.USAGE); + } + }); + test('invalid handled with error', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, '123', 'this is an invalid secret'); + }); + + // Checking valid + const result = await testUtils.pkExec([ + 'secrets', + 'env', + '-np', + dataDir, + '-ei', + 'error', + '-e', + `${vaultName}:.`, + ]); + expect(result.exitCode).toBe(64); + }); + test('invalid handled with warn', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, '123', 'this is an invalid secret'); + }); + + // Checking valid + const result = await testUtils.pkExec([ + 'secrets', + 'env', + '-np', + dataDir, + '-ei', + 'warn', + '-e', + `${vaultName}:.`, + ]); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe(''); + expect(result.stderr).toInclude( + 'The following env variable name (123) is invalid and was dropped', + ); + }); + test('invalid handled with ignore', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, '123', 'this is an invalid secret'); + }); + + // Checking valid + const result = await testUtils.pkExec([ + 'secrets', + 'env', + '-np', + dataDir, + '-ei', + 'ignore', + '-e', + `${vaultName}:.`, + ]); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe(''); + expect(result.stderr).not.toInclude( + 'The following env variable name (123) is invalid and was dropped', + ); + }); + test('duplicate handled with error', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, 'secret', 'this is a secret'); + await vaultOps.mkdir(vault, 'dir'); + await vaultOps.addSecret(vault, 'dir/secret', 'this is a secret'); + }); + + // Checking valid + const result = await testUtils.pkExec([ + 'secrets', + 'env', + '-np', + dataDir, + '-ed', + 'error', + '-e', + `${vaultName}:.`, + ]); + expect(result.exitCode).toBe(64); + expect(result.stderr).toInclude('ErrorPolykeyCLIDuplicateEnvName'); + }); + test('duplicate handled with warn', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, 'secret', 'this is a secret'); + await vaultOps.mkdir(vault, 'dir'); + await vaultOps.addSecret(vault, 'dir/secret', 'this is a secret'); + }); + + // Checking valid + const result = await testUtils.pkExec([ + 'secrets', + 'env', + '-np', + dataDir, + '-ed', + 'warn', + '-e', + `${vaultName}:.`, + ]); + expect(result.exitCode).toBe(0); + expect(result.stderr).toInclude( + 'The env variable (secret) is duplicate, overwriting', + ); + }); + test('duplicate handled with keep', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, 'secret', 'this is a secret1'); + await vaultOps.mkdir(vault, 'dir'); + await vaultOps.addSecret(vault, 'dir/secret', 'this is a secret2'); + }); + + // Checking valid + const result = await testUtils.pkExec([ + 'secrets', + 'env', + '-np', + dataDir, + '-ed', + 'keep', + '-e', + `${vaultName}:.`, + ]); + expect(result.exitCode).toBe(0); + expect(result.stdout).toInclude('this is a secret1'); + }); + test('duplicate handled with overwrite', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, 'secret', 'this is a secret1'); + await vaultOps.mkdir(vault, 'dir'); + await vaultOps.addSecret(vault, 'dir/secret', 'this is a secret2'); + }); + + // Checking valid + const result = await testUtils.pkExec([ + 'secrets', + 'env', + '-np', + dataDir, + '-ed', + 'overwrite', + '-e', + `${vaultName}:.`, + ]); + expect(result.exitCode).toBe(0); + expect(result.stdout).toInclude('this is a secret2'); + }); }); From a58a8e3b6fabfe7c5cc675b4a8c8a534572a0056 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Tue, 27 Feb 2024 12:30:44 +1100 Subject: [PATCH 5/9] feat: fallback `spawnSync` for platforms not supported by `js-exec` [ci skip] --- src/secrets/CommandEnv.ts | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/secrets/CommandEnv.ts b/src/secrets/CommandEnv.ts index a8627d67..a82181b6 100644 --- a/src/secrets/CommandEnv.ts +++ b/src/secrets/CommandEnv.ts @@ -1,7 +1,7 @@ import type PolykeyClient from 'polykey/dist/PolykeyClient'; import path from 'path'; +import os from 'os'; import * as utils from 'polykey/dist/utils'; -import { exec } from '@matrixai/exec'; import * as binProcessors from '../utils/processors'; import * as binUtils from '../utils'; import * as binErrors from '../errors'; @@ -156,8 +156,31 @@ class CommandEnv extends CommandPolykey { // Here we want to switch between the different usages if (cmd != null) { - // If a cmd is provided then we default to exec it - exec.execvp(cmd, argv, envp); + // If a cmd is| provided then we default to exec it + const platform = os.platform(); + switch (platform) { + case 'linux': + // Fallthrough + case 'darwin': + { + const { exec } = await import('@matrixai/exec'); + exec.execvp(cmd, argv, envp); + } + break; + default: { + const { spawnSync } = await import('child_process'); + const result = spawnSync(cmd, argv, { + env: { + ...process.env, + ...envp, + }, + shell: false, + windowsHide: true, + stdio: 'inherit', + }); + process.exit(result.status ?? 255); + } + } } else { // Otherwise we switch between output formats switch (envFormat) { From 86419dc712e3f115d1f2ddf495a998aeb93d8147 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Tue, 27 Feb 2024 14:17:00 +1100 Subject: [PATCH 6/9] fix: general polish [ci skip] --- src/secrets/CommandEnv.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/secrets/CommandEnv.ts b/src/secrets/CommandEnv.ts index a82181b6..3f5a4567 100644 --- a/src/secrets/CommandEnv.ts +++ b/src/secrets/CommandEnv.ts @@ -13,7 +13,7 @@ class CommandEnv extends CommandPolykey { super(...args); this.name('env'); this.description( - 'Run a command with the given secrets and env variables. If no command is specified then the variables are printed to stdout.', + `Run a command with the given secrets and env variables. If no command is specified then the variables are printed to stdout in the format specified by env-format.\n\nWhen selecting secrets with --env secrets with invalid names can be selected. By default when these are encountered then the command will throw an error. This behaviour can be modified with '--env-invalid'. the invalid name can be silently dropped with 'ignore' or logged out with 'warn'\n\nDuplicate secret names can be specified, by default with 'overwrite' the env variable will be overwritten with the latest found secret of that name. It can be specified to 'keep' the first found secret of that name, 'error' to throw if there is a duplicate and 'warn' to log a warning while overwriting.`, ); this.addOption(binOptions.nodeId); this.addOption(binOptions.clientHost); From 98adacd0ddb1321220ae1616f60d1e139ac1e54a Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Tue, 27 Feb 2024 14:43:22 +1100 Subject: [PATCH 7/9] deps: updated `@matrixai/exec` to `^0.1.2` [ci skip] --- npmDepsHash | 2 +- package-lock.json | 32 ++++++++++++++++---------------- package.json | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/npmDepsHash b/npmDepsHash index d87741b0..9675f740 100644 --- a/npmDepsHash +++ b/npmDepsHash @@ -1 +1 @@ -sha256-LZ3QekhwOZj3IkbnZVdC/lbUvwhbJXROzd9oTxrMs7s= +sha256-Tkycu9+7Kwjvuaplw38b/t1N4ZRdsOack5OC0ZO0cxo= diff --git a/package-lock.json b/package-lock.json index f4df2144..9a09d8cf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ }, "devDependencies": { "@matrixai/errors": "^1.2.0", - "@matrixai/exec": "^0.1.1", + "@matrixai/exec": "^0.1.2", "@matrixai/logger": "^3.1.0", "@swc/core": "1.3.82", "@swc/jest": "^0.2.29", @@ -1423,20 +1423,20 @@ "devOptional": true }, "node_modules/@matrixai/exec": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@matrixai/exec/-/exec-0.1.1.tgz", - "integrity": "sha512-jKq8cVRFMPOuKyNfTjLrg+LoIZPLkDrRs0uQyx5fv8v1Uc1eiBOnQsvOh33VyVCsFMiK+aSukLoE/7xh8Yq5GQ==", + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@matrixai/exec/-/exec-0.1.2.tgz", + "integrity": "sha512-5hvfIDrNqMjTRdpPNxZ5wdJe9uzJSRzRAy8nycc7s1kFhmvYteIiiE0oRRGIRzaKWkQQicGypr+W3jJd/1r3kQ==", "dev": true, "optionalDependencies": { - "@matrixai/exec-darwin-arm64": "0.1.1", - "@matrixai/exec-darwin-x64": "0.1.1", - "@matrixai/exec-linux-x64": "0.1.1" + "@matrixai/exec-darwin-arm64": "0.1.2", + "@matrixai/exec-darwin-x64": "0.1.2", + "@matrixai/exec-linux-x64": "0.1.2" } }, "node_modules/@matrixai/exec-darwin-arm64": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@matrixai/exec-darwin-arm64/-/exec-darwin-arm64-0.1.1.tgz", - "integrity": "sha512-JabHQa+/uA8Viqab7br40kOWVVIhotoyQnVQppzUt8ed/7npWczQQKtdspAafG3AjUSCxcg6aptaUX3NZ8/A4g==", + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@matrixai/exec-darwin-arm64/-/exec-darwin-arm64-0.1.2.tgz", + "integrity": "sha512-sl2uD8Dbv3pAOVyUJsnDkNflh+xd4h8k/svQX4DycepcTIGKClBRc7Y09krgwcVPqjQH/daWVhDsKFDyNx0JGw==", "cpu": [ "arm64" ], @@ -1447,9 +1447,9 @@ ] }, "node_modules/@matrixai/exec-darwin-x64": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@matrixai/exec-darwin-x64/-/exec-darwin-x64-0.1.1.tgz", - "integrity": "sha512-i7ShSWrM8bbB9GUtMbhTlyRjzAB/hw1eDYHEFFNFJvMJ220p902TbJTajb9+OxHD/6vk3arslo1uA9okr5ZHvA==", + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@matrixai/exec-darwin-x64/-/exec-darwin-x64-0.1.2.tgz", + "integrity": "sha512-qGV9ePdnjMloVSgAhCSWR3aBnyG9Af+o6Ma+LRgpv4VGlykxz+z+3zGxrJbEB1zJXF4ID7TD94TlBiRPC9zKmg==", "cpu": [ "x64" ], @@ -1460,9 +1460,9 @@ ] }, "node_modules/@matrixai/exec-linux-x64": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@matrixai/exec-linux-x64/-/exec-linux-x64-0.1.1.tgz", - "integrity": "sha512-AlGoFHGDSDsFjUXjTnVXRGMLYfb/Y1ApblSQ4xWhZo/xwqi0of9+85LxDicboEPl5k6teRbYV9BY4DRmZYbFfA==", + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@matrixai/exec-linux-x64/-/exec-linux-x64-0.1.2.tgz", + "integrity": "sha512-EG0/EZOVvaDLqHLNoBp7mrA1ugbXs2N9EbGM6WyjP3xp8T7ILVHB1gGyiNUEXwN6hQ8wvEnqaLQ8+NRltXAoyg==", "cpu": [ "x64" ], diff --git a/package.json b/package.json index ce0fe4c9..0299be5d 100644 --- a/package.json +++ b/package.json @@ -109,7 +109,7 @@ "devDependencies": { "@matrixai/errors": "^1.2.0", "@matrixai/logger": "^3.1.0", - "@matrixai/exec": "^0.1.1", + "@matrixai/exec": "^0.1.2", "@swc/core": "1.3.82", "@swc/jest": "^0.2.29", "@types/jest": "^29.5.2", From 2725fac938ef4b9605ae0a9cf7fdecae8da50e90 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Tue, 27 Feb 2024 15:39:17 +1100 Subject: [PATCH 8/9] fix: merging `--env-format` into the `--format` option [ci skip] --- src/secrets/CommandEnv.ts | 18 ++++++++++++++---- src/utils/options.ts | 9 --------- tests/secrets/env.test.ts | 37 +++++++++++++++++++++++++++++++++---- 3 files changed, 47 insertions(+), 17 deletions(-) diff --git a/src/secrets/CommandEnv.ts b/src/secrets/CommandEnv.ts index 3f5a4567..bd52464a 100644 --- a/src/secrets/CommandEnv.ts +++ b/src/secrets/CommandEnv.ts @@ -11,6 +11,15 @@ import * as binOptions from '../utils/options'; class CommandEnv extends CommandPolykey { constructor(...args: ConstructorParameters) { super(...args); + + // Modify the --format choices to include `dotenv` and `prepend` + // @ts-ignore: missing type for `options` + const formatOption = this.options.filter((v) => v.short === '-f')[0]; + if (formatOption == null) { + utils.never('The -f --format option should exist'); + } + formatOption.argChoices.push('dotenv', 'prepend'); + this.name('env'); this.description( `Run a command with the given secrets and env variables. If no command is specified then the variables are printed to stdout in the format specified by env-format.\n\nWhen selecting secrets with --env secrets with invalid names can be selected. By default when these are encountered then the command will throw an error. This behaviour can be modified with '--env-invalid'. the invalid name can be silently dropped with 'ignore' or logged out with 'warn'\n\nDuplicate secret names can be specified, by default with 'overwrite' the env variable will be overwritten with the latest found secret of that name. It can be specified to 'keep' the first found secret of that name, 'error' to throw if there is a duplicate and 'warn' to log a warning while overwriting.`, @@ -19,7 +28,6 @@ class CommandEnv extends CommandPolykey { this.addOption(binOptions.clientHost); this.addOption(binOptions.clientPort); this.addOption(binOptions.envVariables); - this.addOption(binOptions.envFormat); this.addOption(binOptions.envInvalid); this.addOption(binOptions.envDuplicate); this.argument('[cmd] [argv...]', 'command and arguments'); @@ -29,12 +37,12 @@ class CommandEnv extends CommandPolykey { env: envVariables, envInvalid, envDuplicate, - envFormat, + format, }: { env: Array<[string, string, string?]>; envInvalid: 'error' | 'warn' | 'ignore'; envDuplicate: 'keep' | 'overwrite' | 'warn' | 'error'; - envFormat: 'dotenv' | 'json' | 'prepend'; + format: 'human' | 'dotenv' | 'json' | 'prepend'; } = options; // There are a few stages here @@ -183,7 +191,9 @@ class CommandEnv extends CommandPolykey { } } else { // Otherwise we switch between output formats - switch (envFormat) { + switch (format) { + case 'human': + // Fallthrough case 'dotenv': { // Formatting as a .env file diff --git a/src/utils/options.ts b/src/utils/options.ts index 072ce2e1..746ad351 100644 --- a/src/utils/options.ts +++ b/src/utils/options.ts @@ -207,14 +207,6 @@ const envVariables = new commander.Option('-e --env ', 'specify envs') }, ); -// '-f, --format ', 'Output Format' -const envFormat = new commander.Option( - '-ef --env-format ', - 'How the env variables are formatted when outputted. Only used if no commands are executed', -) - .choices(['dotenv', 'json', 'prepend']) - .default('dotenv'); - const envInvalid = new commander.Option( '-ei --env-invalid ', 'How invalid env variable names are handled when retrieving secrets. `error` will throw, `warn` will log a warning and drop and `ignore` will silently drop.', @@ -258,7 +250,6 @@ export { depth, commitId, envVariables, - envFormat, envInvalid, envDuplicate, }; diff --git a/tests/secrets/env.test.ts b/tests/secrets/env.test.ts index 55dd842c..5ff5db46 100644 --- a/tests/secrets/env.test.ts +++ b/tests/secrets/env.test.ts @@ -292,7 +292,7 @@ describe('commandEnv', () => { expect(jsonOut['SECRET3']).toBeUndefined(); expect(jsonOut['SECRET4']).toBe('this is the secret4'); }); - test('should output .env format', async () => { + test('should output human format', async () => { const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { @@ -310,7 +310,36 @@ describe('commandEnv', () => { dataDir, '-e', `${vaultName}:.`, - '--env-format', + '--format', + 'human', + ]; + + const result = await testUtils.pkExec([...command]); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('SECRET1="this is the secret1"'); + expect(result.stdout).toContain('SECRET2="this is the secret2"'); + expect(result.stdout).toContain('SECRET3="this is the secret3"'); + expect(result.stdout).toContain('SECRET4="this is the secret4"'); + }); + test('should output dotenv format', async () => { + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, 'SECRET1', 'this is the secret1'); + await vaultOps.addSecret(vault, 'SECRET2', 'this is the secret2'); + await vaultOps.mkdir(vault, 'dir1'); + await vaultOps.addSecret(vault, 'dir1/SECRET3', 'this is the secret3'); + await vaultOps.addSecret(vault, 'dir1/SECRET4', 'this is the secret4'); + }); + + command = [ + 'secrets', + 'env', + '-np', + dataDir, + '-e', + `${vaultName}:.`, + '--format', 'dotenv', ]; @@ -339,7 +368,7 @@ describe('commandEnv', () => { dataDir, '-e', `${vaultName}:.`, - '--env-format', + '--format', 'json', ]; @@ -370,7 +399,7 @@ describe('commandEnv', () => { dataDir, '-e', `${vaultName}:.`, - '--env-format', + '--format', 'prepend', ]; From 92be342e1563499dcac7cec28e8ab5f149df7e1e Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Tue, 27 Feb 2024 15:53:08 +1100 Subject: [PATCH 9/9] build: updating `npmDepsHash` [ci skip] --- npmDepsHash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/npmDepsHash b/npmDepsHash index 9675f740..525867f3 100644 --- a/npmDepsHash +++ b/npmDepsHash @@ -1 +1 @@ -sha256-Tkycu9+7Kwjvuaplw38b/t1N4ZRdsOack5OC0ZO0cxo= +sha256-hS9LURKsHIb8bg/LEbGGkrHP4oAxDGeQKvopl8PhuL4=