From a01b36faa9ac314aae1958c1793873be0291ff50 Mon Sep 17 00:00:00 2001 From: Junjie Gao Date: Mon, 12 May 2025 07:08:05 +0000 Subject: [PATCH 1/3] fix: remove the --verbose flag Signed-off-by: Junjie Gao --- cmd/notation/internal/flag/options.go | 6 +----- specs/cmd/blob.md | 3 --- specs/cmd/certificate.md | 2 -- specs/cmd/inspect.md | 1 - specs/cmd/key.md | 3 --- specs/cmd/list.md | 1 - specs/cmd/login.md | 1 - specs/cmd/logout.md | 1 - specs/cmd/plugin.md | 2 -- specs/cmd/sign.md | 1 - specs/cmd/verify.md | 1 - specs/error-handling-guideline.md | 4 ++-- 12 files changed, 3 insertions(+), 23 deletions(-) diff --git a/cmd/notation/internal/flag/options.go b/cmd/notation/internal/flag/options.go index 44cec099e..50b693eb9 100644 --- a/cmd/notation/internal/flag/options.go +++ b/cmd/notation/internal/flag/options.go @@ -59,22 +59,18 @@ func (opts *SignerFlagOpts) ApplyFlagsToCommand(command *cobra.Command) { // LoggingFlagOpts cmd opts for logging. type LoggingFlagOpts struct { - Debug bool - Verbose bool + Debug bool } // ApplyFlags applies flags to a command flag set. func (opts *LoggingFlagOpts) ApplyFlags(fs *pflag.FlagSet) { fs.BoolVarP(&opts.Debug, "debug", "d", false, "debug mode") - fs.BoolVarP(&opts.Verbose, "verbose", "v", false, "verbose mode") } // InitializeLogger sets up the logger based on common options. func (opts *LoggingFlagOpts) InitializeLogger(ctx context.Context) context.Context { if opts.Debug { ctx = trace.WithLoggerLevel(ctx, logrus.DebugLevel) - } else if opts.Verbose { - ctx = trace.WithLoggerLevel(ctx, logrus.InfoLevel) } else { return ctx } diff --git a/specs/cmd/blob.md b/specs/cmd/blob.md index 792cb0843..782bcecee 100644 --- a/specs/cmd/blob.md +++ b/specs/cmd/blob.md @@ -92,7 +92,6 @@ Flags: --timestamp-root-cert string filepath of timestamp authority root certificate --timestamp-url string RFC 3161 Timestamping Authority (TSA) server URL -m, --user-metadata stringArray {key}={value} pairs that are added to the signature payload - -v, --verbose verbose mode ``` ### notation blob inspect @@ -106,7 +105,6 @@ Usage: Flags: -o, --output string output format, options: 'json', 'tree' (default "tree") -d, --debug debug mode - -v, --verbose verbose mode -h, --help help for inspect ``` @@ -185,7 +183,6 @@ Flags: --policy-name string policy name to verify against. If not provided, the global policy is used if exists -s --signature string filepath of the signature to be verified -m, --user-metadata stringArray user defined {key}={value} pairs that must be present in the signature for successful verification if provided - -v, --verbose verbose mode ``` ## Usage diff --git a/specs/cmd/certificate.md b/specs/cmd/certificate.md index 65eb129cc..b6e258242 100644 --- a/specs/cmd/certificate.md +++ b/specs/cmd/certificate.md @@ -90,7 +90,6 @@ Flags: -h, --help help for list -s, --store string specify named store -t, --type string specify trust store type, options: ca, signingAuthority, tsa - -v, --verbose verbose mode ``` ### notation certificate show @@ -106,7 +105,6 @@ Flags: -h, --help help for show -s, --store string specify named store -t, --type string specify trust store type, options: ca, signingAuthority, tsa - -v, --verbose verbose mode ``` ### notation certificate delete diff --git a/specs/cmd/inspect.md b/specs/cmd/inspect.md index f4d1465b2..a1fd33779 100644 --- a/specs/cmd/inspect.md +++ b/specs/cmd/inspect.md @@ -45,7 +45,6 @@ Flags: -o, --output string output format, options: 'json', 'tree' (default "tree") -p, --password string password for registry operations (default to $NOTATION_PASSWORD if not specified) -u, --username string username for registry operations (default to $NOTATION_USERNAME if not specified) - -v, --verbose verbose mode ``` ## Usage diff --git a/specs/cmd/key.md b/specs/cmd/key.md index dbd9fd72a..293222de7 100644 --- a/specs/cmd/key.md +++ b/specs/cmd/key.md @@ -39,7 +39,6 @@ Flags: --id string key id (required if --plugin is set) --plugin string signing plugin name --plugin-config stringArray {key}={value} pairs that are passed as it is to a plugin, refer plugin's documentation to set appropriate values - -v, --verbose verbose mode ``` ### notation key delete @@ -53,7 +52,6 @@ Usage: Flags: -d, --debug debug mode -h, --help help for delete - -v, --verbose verbose mode ``` ### notation key list @@ -86,7 +84,6 @@ Flags: -d, --debug debug mode --default mark as default -h, --help help for update - -v, --verbose verbose mode ``` ## Usage diff --git a/specs/cmd/list.md b/specs/cmd/list.md index 2540633d0..c64621d4c 100644 --- a/specs/cmd/list.md +++ b/specs/cmd/list.md @@ -34,7 +34,6 @@ Flags: --oci-layout [Experimental] list signatures stored in OCI image layout -p, --password string password for registry operations (default to $NOTATION_PASSWORD if not specified) -u, --username string username for registry operations (default to $NOTATION_USERNAME if not specified) - -v, --verbose verbose mode ``` ## Usage diff --git a/specs/cmd/login.md b/specs/cmd/login.md index 90f37e5c2..f441f35d4 100644 --- a/specs/cmd/login.md +++ b/specs/cmd/login.md @@ -19,7 +19,6 @@ Flags: -p, --password string password for registry operations (default to $NOTATION_PASSWORD if not specified) --password-stdin take the password from stdin -u, --username string username for registry operations (default to $NOTATION_USERNAME if not specified) - -v, --verbose verbose mode ``` ## Usage diff --git a/specs/cmd/logout.md b/specs/cmd/logout.md index 70e88a77a..70b0deac5 100644 --- a/specs/cmd/logout.md +++ b/specs/cmd/logout.md @@ -15,7 +15,6 @@ Usage: Flags: -d, --debug debug mode -h, --help help for logout - -v, --verbose verbose mode ``` ## Usage diff --git a/specs/cmd/plugin.md b/specs/cmd/plugin.md index c0460ab50..2b87fef39 100644 --- a/specs/cmd/plugin.md +++ b/specs/cmd/plugin.md @@ -53,7 +53,6 @@ Flags: -h, --help help for install --sha256sum string must match SHA256 of the plugin source, required when "--url" flag is set --url install plugin from an HTTPS URL. The plugin download timeout is 10m0s - -v, --verbose verbose mode Aliases: install, add @@ -70,7 +69,6 @@ Usage: Flags: -d, --debug debug mode -h, --help help for remove - -v, --verbose verbose mode -y, --yes do not prompt for confirmation Aliases: uninstall, remove, rm diff --git a/specs/cmd/sign.md b/specs/cmd/sign.md index 379fe86d4..e1beb2e4a 100644 --- a/specs/cmd/sign.md +++ b/specs/cmd/sign.md @@ -46,7 +46,6 @@ Flags: --timestamp-url string RFC 3161 Timestamping Authority (TSA) server URL -u, --username string username for registry operations (default to $NOTATION_USERNAME if not specified) -m, --user-metadata stringArray {key}={value} pairs that are added to the signature payload - -v, --verbose verbose mode ``` ### Set config property for OCI image manifest diff --git a/specs/cmd/verify.md b/specs/cmd/verify.md index 6d91904de..dba173537 100644 --- a/specs/cmd/verify.md +++ b/specs/cmd/verify.md @@ -48,7 +48,6 @@ Flags: --scope string [Experimental] set trust policy scope for artifact verification, required and can only be used when flag "--oci-layout" is set -u, --username string username for registry operations (default to $NOTATION_USERNAME if not specified) -m, --user-metadata stringArray user defined {key}={value} pairs that must be present in the signature for successful verification if provided - -v, --verbose verbose mode ``` ## Usage diff --git a/specs/error-handling-guideline.md b/specs/error-handling-guideline.md index ef06ddcc5..95bedefd5 100644 --- a/specs/error-handling-guideline.md +++ b/specs/error-handling-guideline.md @@ -14,7 +14,7 @@ First and foremost, make the error messages descriptive and informative. Error m Second, when necessary, it is highly suggested for Notation CLI contributors to provide recommendations for users how to resolve the problems based on the error messages they encountered. Showing descriptive words and straightforward prompt with executable commands as a potential solution is a good practice for error messages. -Third, for unhandled errors you didn't expect the user to run into. For that, have a way to view full traceback information as well as full debug or verbose logs output, and instructions on how to submit a bug. +Third, for unhandled errors you didn't expect the user to run into. For that, have a way to view full traceback information as well as full debug logs output, and instructions on how to submit a bug. Fourth, signal-to-noise ratio is crucial. The more irrelevant output you produce, the longer it's going to take the user to figure out what they did wrong. If your program produces multiple errors of the same type, consider grouping them under a single explanatory header instead of printing many similar-looking lines. @@ -31,7 +31,7 @@ Last, error logs can also be useful for post-mortem debugging and can also be wr - Print human readable error message. If the error message is mainly from the server and varies by different servers, tell users that the error response is from server. This implies that users may need to contact server side for troubleshooting. - Provide specific and actionable prompt message with argument suggestion or show the example usage for reference. (e.g, Instead of showing flag or argument options is missing, please provide available argument options and guide users to `--help` to view more examples). - If the actionable prompt message is too long to show in the CLI output, consider guide users to Notation user manual or troubleshooting guide with the versioned permanent link. -- If the error message is not enough for troubleshooting, guide users to use `--verbose` to print much more detailed logs. +- If the error message is not enough for troubleshooting, guide users to use `--debug` to print much more detailed logs. - If server returns an error without any [message or detail](https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc.3/spec.md#error-codes), consider providing customized error logs to make it clearer. The original server logs can be displayed in debug mode. - As a security tool, `notation` SHOULD prompt users to stop upon verification errors. From 3413616430ea5c782b1d7157ad1a92443f2c3af6 Mon Sep 17 00:00:00 2001 From: Junjie Gao Date: Mon, 12 May 2025 07:12:18 +0000 Subject: [PATCH 2/3] fix: update E2E Signed-off-by: Junjie Gao --- test/e2e/suite/command/inspect.go | 4 +- test/e2e/suite/command/list.go | 4 +- test/e2e/suite/command/sign.go | 6 +-- test/e2e/suite/command/verify.go | 14 +++--- test/e2e/suite/plugin/install.go | 12 ++--- test/e2e/suite/trustpolicy/trust_store.go | 18 ++++---- .../e2e/suite/trustpolicy/trusted_identity.go | 26 +++++------ .../suite/trustpolicy/verification_level.go | 46 +++++++++---------- 8 files changed, 65 insertions(+), 65 deletions(-) diff --git a/test/e2e/suite/command/inspect.go b/test/e2e/suite/command/inspect.go index a811855f6..a6bfe611c 100644 --- a/test/e2e/suite/command/inspect.go +++ b/test/e2e/suite/command/inspect.go @@ -116,7 +116,7 @@ var _ = Describe("notation inspect", func() { notation.Exec("sign", "--force-referrers-tag", artifact.ReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("inspect", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("inspect", artifact.ReferenceWithDigest(), "-d"). MatchKeyWords(inspectSuccessfully...) }) }) @@ -126,7 +126,7 @@ var _ = Describe("notation inspect", func() { notation.Exec("sign", "--force-referrers-tag=false", artifact.ReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("inspect", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("inspect", artifact.ReferenceWithDigest(), "-d"). MatchKeyWords(inspectSuccessfully...) }) }) diff --git a/test/e2e/suite/command/list.go b/test/e2e/suite/command/list.go index 18c36f22a..c81550c83 100644 --- a/test/e2e/suite/command/list.go +++ b/test/e2e/suite/command/list.go @@ -91,7 +91,7 @@ var _ = Describe("notation list", func() { notation.Exec("sign", "--force-referrers-tag", artifact.ReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("list", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("list", artifact.ReferenceWithDigest(), "-d"). MatchKeyWords( "└── application/vnd.cncf.notary.signature", "└── sha256:", @@ -104,7 +104,7 @@ var _ = Describe("notation list", func() { notation.Exec("sign", "--force-referrers-tag=false", artifact.ReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("list", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("list", artifact.ReferenceWithDigest(), "-d"). MatchKeyWords( "└── application/vnd.cncf.notary.signature", "└── sha256:", diff --git a/test/e2e/suite/command/sign.go b/test/e2e/suite/command/sign.go index c357cf2e6..9bf0c9be3 100644 --- a/test/e2e/suite/command/sign.go +++ b/test/e2e/suite/command/sign.go @@ -86,7 +86,7 @@ var _ = Describe("notation sign", func() { MatchKeyWords(SignSuccessfully) notation.WithDescription("verify by tag schema"). - Exec("verify", artifact.ReferenceWithDigest(), "-v"). + Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchKeyWords(VerifySuccessfully) }) }) @@ -98,7 +98,7 @@ var _ = Describe("notation sign", func() { MatchKeyWords(SignSuccessfully) notation.WithDescription("verify by referrers api"). - Exec("verify", artifact.ReferenceWithDigest(), "-v"). + Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchKeyWords(VerifySuccessfully) }) }) @@ -132,7 +132,7 @@ var _ = Describe("notation sign", func() { // sleep to wait for expiry time.Sleep(2100 * time.Millisecond) - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("expiry validation failed."). MatchErrKeyWords("signature verification failed for all the signatures") }) diff --git a/test/e2e/suite/command/verify.go b/test/e2e/suite/command/verify.go index 2dd862bbc..5ac662089 100644 --- a/test/e2e/suite/command/verify.go +++ b/test/e2e/suite/command/verify.go @@ -29,7 +29,7 @@ var _ = Describe("notation verify", func() { notation.Exec("sign", artifact.ReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchKeyWords(VerifySuccessfully) }) }) @@ -39,7 +39,7 @@ var _ = Describe("notation verify", func() { notation.Exec("sign", artifact.ReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("verify", artifact.ReferenceWithTag(), "-v"). + notation.Exec("verify", artifact.ReferenceWithTag(), "-d"). MatchKeyWords(VerifySuccessfully) }) }) @@ -69,7 +69,7 @@ var _ = Describe("notation verify", func() { notation.Exec("sign", "--force-referrers-tag", artifact.ReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchKeyWords(VerifySuccessfully) }) }) @@ -79,7 +79,7 @@ var _ = Describe("notation verify", func() { notation.Exec("sign", "--force-referrers-tag=false", artifact.ReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchKeyWords(VerifySuccessfully) }) }) @@ -163,7 +163,7 @@ var _ = Describe("notation verify", func() { MatchKeyWords(SignSuccessfully) vhost.UpdateEnv(map[string]string{"NOTATION_CONFIG": "/not/exist"}) - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("trust policy is not present") }) }) @@ -174,7 +174,7 @@ var _ = Describe("notation verify", func() { MatchKeyWords(SignSuccessfully) vhost.UpdateEnv(map[string]string{"NOTATION_CONFIG": vhost.AbsolutePath(NotationDirName)}) - notation.Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchKeyWords(VerifySuccessfully) }) }) @@ -185,7 +185,7 @@ var _ = Describe("notation verify", func() { MatchKeyWords(SignSuccessfully) vhost.UpdateEnv(map[string]string{"NOTATION_CACHE": vhost.AbsolutePath(NotationDirName)}) - notation.Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchKeyWords(VerifySuccessfully) }) }) diff --git a/test/e2e/suite/plugin/install.go b/test/e2e/suite/plugin/install.go index 6ac7728d5..ce3f4e8d2 100644 --- a/test/e2e/suite/plugin/install.go +++ b/test/e2e/suite/plugin/install.go @@ -67,14 +67,14 @@ var _ = Describe("notation plugin install", func() { It("with zip bomb single file exceeds 256 MiB size limit in zip format", func() { Host(nil, func(notation *utils.ExecOpts, _ *Artifact, vhost *utils.VirtualHost) { - notation.ExpectFailure().Exec("plugin", "install", "--file", filepath.Join(NotationE2EMaliciousPluginArchivePath, "large_file_zip.zip"), "-v"). + notation.ExpectFailure().Exec("plugin", "install", "--file", filepath.Join(NotationE2EMaliciousPluginArchivePath, "large_file_zip.zip"), "-d"). MatchErrContent("Error: plugin installation failed: total file size reached the 256 MiB size limit\n") }) }) It("with zip bomb single file exceeds 256 MiB size limit in tar.gz format", func() { Host(nil, func(notation *utils.ExecOpts, _ *Artifact, vhost *utils.VirtualHost) { - notation.ExpectFailure().Exec("plugin", "install", "--file", filepath.Join(NotationE2EMaliciousPluginArchivePath, "large_file_tarGz.tar.gz"), "-v"). + notation.ExpectFailure().Exec("plugin", "install", "--file", filepath.Join(NotationE2EMaliciousPluginArchivePath, "large_file_tarGz.tar.gz"), "-d"). MatchErrContent("Error: plugin installation failed: total file size reached the 256 MiB size limit\n") }) }) @@ -97,21 +97,21 @@ var _ = Describe("notation plugin install", func() { Fail(fmt.Sprintf("failed to write file %s: %v", targetPath, err)) } - notation.ExpectFailure().Exec("plugin", "install", "--file", targetPath, "-v"). + notation.ExpectFailure().Exec("plugin", "install", "--file", targetPath, "-d"). MatchErrContent("Error: plugin installation failed: total file size reached the 256 MiB size limit\n") }) }) It("with zip slip", func() { Host(nil, func(notation *utils.ExecOpts, _ *Artifact, vhost *utils.VirtualHost) { - notation.ExpectFailure().Exec("plugin", "install", "--file", filepath.Join(NotationE2EMaliciousPluginArchivePath, "zip_slip.zip"), "-v"). + notation.ExpectFailure().Exec("plugin", "install", "--file", filepath.Join(NotationE2EMaliciousPluginArchivePath, "zip_slip.zip"), "-d"). MatchErrContent("Error: plugin installation failed: file name in zip cannot contain '..', but found \"../../../../../../../../tmp/evil.txt\"\n") }) }) It("with valid plugin file path", func() { Host(nil, func(notation *utils.ExecOpts, _ *Artifact, vhost *utils.VirtualHost) { - notation.Exec("plugin", "install", "--file", NotationE2EPluginTarGzPath, "-v"). + notation.Exec("plugin", "install", "--file", NotationE2EPluginTarGzPath, "-d"). MatchContent("Successfully installed plugin e2e-plugin, version 1.0.0\n") }) }) @@ -135,7 +135,7 @@ var _ = Describe("notation plugin install", func() { It("with plugin already installed but force install", func() { Host(nil, func(notation *utils.ExecOpts, _ *Artifact, vhost *utils.VirtualHost) { - notation.Exec("plugin", "install", "--file", NotationE2EPluginTarGzPath, "-v"). + notation.Exec("plugin", "install", "--file", NotationE2EPluginTarGzPath, "-d"). MatchContent("Successfully installed plugin e2e-plugin, version 1.0.0\n") notation.Exec("plugin", "install", "--file", NotationE2EPluginTarGzPath, "--force"). diff --git a/test/e2e/suite/trustpolicy/trust_store.go b/test/e2e/suite/trustpolicy/trust_store.go index 9f1a90ecd..6c322fc39 100644 --- a/test/e2e/suite/trustpolicy/trust_store.go +++ b/test/e2e/suite/trustpolicy/trust_store.go @@ -29,7 +29,7 @@ var _ = Describe("notation trust policy trust store test", func() { artifact := GenerateArtifact("e2e-valid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords(`trust policy statement "e2e" is either missing trust stores or trusted identities, both must be specified`) }) }) @@ -40,7 +40,7 @@ var _ = Describe("notation trust policy trust store test", func() { artifact := GenerateArtifact("e2e-valid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("the trust store \"invalid_store\" of type \"ca\" does not exist") }) }) @@ -51,7 +51,7 @@ var _ = Describe("notation trust policy trust store test", func() { artifact := GenerateArtifact("e2e-valid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords(`trust policy statement "e2e" uses an unsupported trust store name "" in trust store value "ca:". Named store name needs to follow [a-zA-Z0-9_.-]+ format`) }) }) @@ -62,7 +62,7 @@ var _ = Describe("notation trust policy trust store test", func() { artifact := GenerateArtifact("e2e-valid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords(`trust policy statement "e2e" has malformed trust store value "*". The required format is :`) }) }) @@ -71,7 +71,7 @@ var _ = Describe("notation trust policy trust store test", func() { Host(nil, func(notation *utils.ExecOpts, artifact1 *Artifact, vhost *utils.VirtualHost) { // artifact1 signed with new_e2e.crt OldNotation(AuthOption("", ""), AddKeyOption(filepath.Join(NotationE2ELocalKeysDir, "e2e.key"), filepath.Join(NotationE2ELocalKeysDir, "new_e2e.crt"))). - Exec("sign", artifact1.ReferenceWithDigest(), "-v"). + Exec("sign", artifact1.ReferenceWithDigest(), "-d"). MatchKeyWords(SignSuccessfully) // artifact2 signed with e2e.crt @@ -85,11 +85,11 @@ var _ = Describe("notation trust policy trust store test", func() { ) notation.WithDescription("verify artifact1 with trust store ca/e2e-new"). - Exec("verify", artifact1.ReferenceWithDigest(), "-v"). + Exec("verify", artifact1.ReferenceWithDigest(), "-d"). MatchKeyWords(VerifySuccessfully) notation.WithDescription("verify artifact2 with trust store ca/e2e"). - Exec("verify", artifact2.ReferenceWithDigest(), "-v"). + Exec("verify", artifact2.ReferenceWithDigest(), "-d"). MatchKeyWords(VerifySuccessfully) }) }) @@ -98,7 +98,7 @@ var _ = Describe("notation trust policy trust store test", func() { Skip("overlapped trust stores were not checked") Host(nil, func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { // artifact signed with new_e2e.crt - notation.Exec("sign", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("sign", artifact.ReferenceWithDigest(), "-d"). MatchKeyWords(SignSuccessfully) // setup overlapped trust store @@ -106,7 +106,7 @@ var _ = Describe("notation trust policy trust store test", func() { AddTrustPolicyOption("overlapped_trust_store_trustpolicy.json", false), AddTrustStoreOption("e2e", filepath.Join(NotationE2ELocalKeysDir, "e2e.crt"))) - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords(VerifyFailed) }) }) diff --git a/test/e2e/suite/trustpolicy/trusted_identity.go b/test/e2e/suite/trustpolicy/trusted_identity.go index e822eb1ce..03ad30e77 100644 --- a/test/e2e/suite/trustpolicy/trusted_identity.go +++ b/test/e2e/suite/trustpolicy/trusted_identity.go @@ -28,7 +28,7 @@ var _ = Describe("notation trust policy trusted identity test", func() { vhost.SetOption(AddTrustPolicyOption("unset_trusted_identity_trustpolicy.json", false)) artifact := GenerateArtifact("e2e-valid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords(`trust policy statement "e2e" is either missing trust stores or trusted identities, both must be specified`) }) }) @@ -38,7 +38,7 @@ var _ = Describe("notation trust policy trusted identity test", func() { vhost.SetOption(AddTrustPolicyOption("valid_trusted_identity_trustpolicy.json", false)) artifact := GenerateArtifact("e2e-valid-signature", "") - notation.Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchKeyWords(VerifySuccessfully) }) }) @@ -48,7 +48,7 @@ var _ = Describe("notation trust policy trusted identity test", func() { vhost.SetOption(AddTrustPolicyOption("invalid_trusted_identity_trustpolicy.json", false)) artifact := GenerateArtifact("e2e-valid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("Failure reason: signing certificate from the digital signature does not match the X.509 trusted identities", VerifyFailed) }) @@ -59,7 +59,7 @@ var _ = Describe("notation trust policy trusted identity test", func() { vhost.SetOption(AddTrustPolicyOption("malformed_trusted_identity_trustpolicy.json", false)) artifact := GenerateArtifact("e2e-valid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords(`trust policy statement "e2e" has trusted identity "x509.subject CN=e2e,O=Notary,L=Seattle,ST=WA,C=US" missing separator`) }) }) @@ -69,7 +69,7 @@ var _ = Describe("notation trust policy trusted identity test", func() { vhost.SetOption(AddTrustPolicyOption("empty_trusted_identity_trustpolicy.json", false)) artifact := GenerateArtifact("e2e-valid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords(`trust policy statement "e2e" has an empty trusted identity`) }) }) @@ -78,7 +78,7 @@ var _ = Describe("notation trust policy trusted identity test", func() { Host(nil, func(notation *utils.ExecOpts, artifact1 *Artifact, vhost *utils.VirtualHost) { // artifact1 signed with new_e2e.crt OldNotation(AuthOption("", ""), AddKeyOption(filepath.Join(NotationE2ELocalKeysDir, "e2e.key"), filepath.Join(NotationE2ELocalKeysDir, "new_e2e.crt"))). - Exec("sign", artifact1.ReferenceWithDigest(), "-v"). + Exec("sign", artifact1.ReferenceWithDigest(), "-d"). MatchKeyWords(SignSuccessfully) // artifact2 signed with e2e.crt @@ -91,10 +91,10 @@ var _ = Describe("notation trust policy trusted identity test", func() { AddTrustStoreOption("e2e", filepath.Join(NotationE2ELocalKeysDir, "e2e.crt")), ) - notation.Exec("verify", artifact1.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact1.ReferenceWithDigest(), "-d"). MatchKeyWords(VerifySuccessfully) - notation.Exec("verify", artifact2.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact2.ReferenceWithDigest(), "-d"). MatchKeyWords(VerifySuccessfully) }) }) @@ -104,7 +104,7 @@ var _ = Describe("notation trust policy trusted identity test", func() { vhost.SetOption(AddTrustPolicyOption("overlapped_trusted_identity_trustpolicy.json", false)) artifact := GenerateArtifact("e2e-valid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords(`trust policy statement "e2e" has overlapping x509 trustedIdentities`) }) }) @@ -114,7 +114,7 @@ var _ = Describe("notation trust policy trusted identity test", func() { vhost.SetOption(AddTrustPolicyOption("wildcard_plus_other_trusted_identity_trustpolicy.json", false)) artifact := GenerateArtifact("e2e-valid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords(`trust policy statement "e2e" uses a wildcard trusted identity '*', a wildcard identity cannot be used in conjunction with other values`) }) }) @@ -124,7 +124,7 @@ var _ = Describe("notation trust policy trusted identity test", func() { vhost.SetOption(AddTrustPolicyOption("missing_organization_trusted_identity_trustpolicy.json", false)) artifact := GenerateArtifact("e2e-valid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords(`distinguished name (DN) " CN=e2e,L=Seattle,ST=WA,C=US" has no mandatory RDN attribute for "O", it must contain 'C', 'ST' or 'S', and 'O' RDN attributes at a minimum`) }) }) @@ -134,7 +134,7 @@ var _ = Describe("notation trust policy trusted identity test", func() { vhost.SetOption(AddTrustPolicyOption("missing_state_trusted_identity_trustpolicy.json", false)) artifact := GenerateArtifact("e2e-valid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords(`distinguished name (DN) " CN=e2e,O=Notary,L=Seattle,C=US" has no mandatory RDN attribute for "ST", it must contain 'C', 'ST' or 'S', and 'O' RDN attributes at a minimum`) }) }) @@ -144,7 +144,7 @@ var _ = Describe("notation trust policy trusted identity test", func() { vhost.SetOption(AddTrustPolicyOption("missing_country_trusted_identity_trustpolicy.json", false)) artifact := GenerateArtifact("e2e-valid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords(`distinguished name (DN) " CN=e2e,O=Notary,L=Seattle,ST=WA" has no mandatory RDN attribute for "C", it must contain 'C', 'ST' or 'S', and 'O' RDN attributes at a minimum`) }) }) diff --git a/test/e2e/suite/trustpolicy/verification_level.go b/test/e2e/suite/trustpolicy/verification_level.go index d18e6fb0d..18d8024f6 100644 --- a/test/e2e/suite/trustpolicy/verification_level.go +++ b/test/e2e/suite/trustpolicy/verification_level.go @@ -27,7 +27,7 @@ var _ = Describe("notation trust policy verification level test", func() { Host(BaseOptions(), func(notation *utils.ExecOpts, _ *Artifact, vhost *utils.VirtualHost) { artifact := GenerateArtifact("e2e-expired-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("expiry validation failed.", VerifyFailed) }) @@ -42,7 +42,7 @@ var _ = Describe("notation trust policy verification level test", func() { AddTrustStoreOption("e2e", filepath.Join(NotationE2EConfigPath, "localkeys", "expired_e2e.crt")), ) - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("authenticTimestamp validation failed", VerifyFailed) }) @@ -59,7 +59,7 @@ var _ = Describe("notation trust policy verification level test", func() { // trust store. artifact := GenerateArtifact("e2e-valid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("authenticity validation failed", VerifyFailed) }) @@ -69,7 +69,7 @@ var _ = Describe("notation trust policy verification level test", func() { Host(BaseOptions(), func(notation *utils.ExecOpts, _ *Artifact, vhost *utils.VirtualHost) { artifact := GenerateArtifact("e2e-invalid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("integrity validation failed", VerifyFailed) }) @@ -81,7 +81,7 @@ var _ = Describe("notation trust policy verification level test", func() { artifact := GenerateArtifact("e2e-expired-signature", "") - notation.Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("expiry was set to \"log\" and failed with error: digital signature has expired"). MatchKeyWords(VerifySuccessfully) }) @@ -96,7 +96,7 @@ var _ = Describe("notation trust policy verification level test", func() { AddTrustStoreOption("e2e", filepath.Join(NotationE2EConfigPath, "localkeys", "expired_e2e.crt")), ) - notation.Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("Warning: authenticTimestamp was set to \"log\"", "after certificate \"O=Internet Widgits Pty Ltd,ST=Some-State,C=AU\" validity period, it was expired at \"Tue, 27 Jun 2023 06:10:00 +0000\""). MatchKeyWords(VerifySuccessfully) @@ -114,7 +114,7 @@ var _ = Describe("notation trust policy verification level test", func() { // trust store. artifact := GenerateArtifact("e2e-valid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("authenticity validation failed", VerifyFailed) }) @@ -126,7 +126,7 @@ var _ = Describe("notation trust policy verification level test", func() { artifact := GenerateArtifact("e2e-invalid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("integrity validation failed", VerifyFailed) }) @@ -138,7 +138,7 @@ var _ = Describe("notation trust policy verification level test", func() { artifact := GenerateArtifact("e2e-expired-signature", "") - notation.Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("digital signature has expired", "expiry was set to \"log\""). MatchKeyWords(VerifySuccessfully) @@ -154,7 +154,7 @@ var _ = Describe("notation trust policy verification level test", func() { AddTrustStoreOption("e2e", filepath.Join(NotationE2EConfigPath, "localkeys", "expired_e2e.crt")), ) - notation.Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("Warning: authenticTimestamp was set to \"log\"", "after certificate \"O=Internet Widgits Pty Ltd,ST=Some-State,C=AU\" validity period, it was expired at \"Tue, 27 Jun 2023 06:10:00 +0000\""). MatchKeyWords(VerifySuccessfully) @@ -172,7 +172,7 @@ var _ = Describe("notation trust policy verification level test", func() { // trust store. artifact := GenerateArtifact("e2e-valid-signature", "") - notation.Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("Warning: authenticity was set to \"log\"", "the signature's certificate chain does not contain any trusted certificate"). MatchKeyWords(VerifySuccessfully) @@ -185,7 +185,7 @@ var _ = Describe("notation trust policy verification level test", func() { artifact := GenerateArtifact("e2e-invalid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("integrity validation failed", VerifyFailed) }) @@ -197,7 +197,7 @@ var _ = Describe("notation trust policy verification level test", func() { artifact := GenerateArtifact("e2e-invalid-signature", "") - notation.Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchKeyWords("Trust policy is configured to skip signature verification") }) }) @@ -208,7 +208,7 @@ var _ = Describe("notation trust policy verification level test", func() { artifact := GenerateArtifact("e2e-expired-signature", "") - notation.Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("digital signature has expired", "expiry was set to \"log\""). MatchKeyWords(VerifySuccessfully) @@ -224,7 +224,7 @@ var _ = Describe("notation trust policy verification level test", func() { AddTrustStoreOption("e2e", filepath.Join(NotationE2EConfigPath, "localkeys", "expired_e2e.crt")), ) - notation.Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("Warning: authenticTimestamp was set to \"log\"", "after certificate \"O=Internet Widgits Pty Ltd,ST=Some-State,C=AU\" validity period, it was expired at \"Tue, 27 Jun 2023 06:10:00 +0000\""). MatchKeyWords(VerifySuccessfully) @@ -242,7 +242,7 @@ var _ = Describe("notation trust policy verification level test", func() { // trust store. artifact := GenerateArtifact("e2e-valid-signature", "") - notation.Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("Warning: authenticity was set to \"log\"", "the signature's certificate chain does not contain any trusted certificate"). MatchKeyWords(VerifySuccessfully) @@ -255,7 +255,7 @@ var _ = Describe("notation trust policy verification level test", func() { artifact := GenerateArtifact("e2e-expired-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("expiry validation failed.", VerifyFailed) }) @@ -272,7 +272,7 @@ var _ = Describe("notation trust policy verification level test", func() { AddTrustStoreOption("e2e", filepath.Join(NotationE2EConfigPath, "localkeys", "expired_e2e.crt")), ) - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("authenticTimestamp validation failed", VerifyFailed) }) @@ -287,7 +287,7 @@ var _ = Describe("notation trust policy verification level test", func() { artifact := GenerateArtifact("e2e-valid-signature", "") - notation.Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("Warning: authenticity was set to \"log\"", "the signature's certificate chain does not contain any trusted certificate"). MatchKeyWords(VerifySuccessfully) @@ -303,7 +303,7 @@ var _ = Describe("notation trust policy verification level test", func() { artifact := GenerateArtifact("e2e-valid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords(`"integrity" verification can not be overridden in custom signature verification`) }) }) @@ -314,7 +314,7 @@ var _ = Describe("notation trust policy verification level test", func() { artifact := GenerateArtifact("e2e-expired-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("expiry validation failed.", VerifyFailed) }) @@ -331,7 +331,7 @@ var _ = Describe("notation trust policy verification level test", func() { AddTrustStoreOption("e2e", filepath.Join(NotationE2EConfigPath, "localkeys", "expired_e2e.crt")), ) - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("authenticTimestamp validation failed", VerifyFailed) }) @@ -348,7 +348,7 @@ var _ = Describe("notation trust policy verification level test", func() { // trust store. artifact := GenerateArtifact("e2e-valid-signature", "") - notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-v"). + notation.ExpectFailure().Exec("verify", artifact.ReferenceWithDigest(), "-d"). MatchErrKeyWords("authenticity validation failed", VerifyFailed) }) From 3de3013595ed536afad05cd7202ab0069c8d1e03 Mon Sep 17 00:00:00 2001 From: Junjie Gao Date: Mon, 12 May 2025 07:16:37 +0000 Subject: [PATCH 3/3] fix: E2E Signed-off-by: Junjie Gao --- test/e2e/suite/plugin/install.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/suite/plugin/install.go b/test/e2e/suite/plugin/install.go index ce3f4e8d2..d5a2ebf5d 100644 --- a/test/e2e/suite/plugin/install.go +++ b/test/e2e/suite/plugin/install.go @@ -97,7 +97,7 @@ var _ = Describe("notation plugin install", func() { Fail(fmt.Sprintf("failed to write file %s: %v", targetPath, err)) } - notation.ExpectFailure().Exec("plugin", "install", "--file", targetPath, "-d"). + notation.ExpectFailure().Exec("plugin", "install", "--file", targetPath). MatchErrContent("Error: plugin installation failed: total file size reached the 256 MiB size limit\n") }) })