diff --git a/test/e2e/internal/notation/host.go b/test/e2e/internal/notation/host.go index 613c0705e..12acf20c4 100644 --- a/test/e2e/internal/notation/host.go +++ b/test/e2e/internal/notation/host.go @@ -15,12 +15,12 @@ import ( // vhost is the VirtualHost instance. type CoreTestFunc func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) -// GeneralTestFunc is the test function running in a VirtualHost agnostic to -// the Repository of the artifact. +// OCILayoutTestFunc is the test function running in a VirtualHost with isolated +// OCI layout for each test case. // // notation is an Executor isolated by $XDG_CONFIG_HOME. // vhost is the VirtualHost instance. -type GeneralTestFunc func(notation *utils.ExecOpts, vhost *utils.VirtualHost) +type OCILayoutTestFunc func(notation *utils.ExecOpts, ocilayout *OCILayout, vhost *utils.VirtualHost) // Host creates a virtualized notation testing host by modify // the "XDG_CONFIG_HOME" environment variable of the Executor. @@ -52,21 +52,26 @@ func HostInGithubAction(options []utils.HostOption, fn CoreTestFunc) { Host(options, fn) } -// GeneralHost creates a virtualized notation testing host by modify -// the "XDG_CONFIG_HOME" environment variable of the Executor. It's agnostic to -// the Repository of the artifact. +// HostWithOCILayout creates a virtualized notation testing host by modify +// the "XDG_CONFIG_HOME" environment variable of the Executor. It generates +// isolated OCI layout in the testing host. // // options is the required testing environment options // fn is the callback function containing the testing logic. -func GeneralHost(options []utils.HostOption, fn GeneralTestFunc) { +func HostWithOCILayout(options []utils.HostOption, fn OCILayoutTestFunc) { // create a notation vhost vhost, err := createNotationHost(NotationBinPath, options...) if err != nil { panic(err) } + ocilayout, err := GenerateOCILayout("") + if err != nil { + panic(err) + } + // run the main logic - fn(vhost.Executor, vhost) + fn(vhost.Executor, ocilayout, vhost) } // OldNotation create an old version notation ExecOpts in a VirtualHost diff --git a/test/e2e/internal/notation/init.go b/test/e2e/internal/notation/init.go index c6c2a1292..e2c1c9101 100644 --- a/test/e2e/internal/notation/init.go +++ b/test/e2e/internal/notation/init.go @@ -29,7 +29,6 @@ const ( envKeyNotationPluginPath = "NOTATION_E2E_PLUGIN_PATH" envKeyNotationConfigPath = "NOTATION_E2E_CONFIG_PATH" envKeyOCILayoutPath = "NOTATION_E2E_OCI_LAYOUT_PATH" - envKeyOCILayoutTestPath = "NOTATION_E2E_OCI_LAYOUT_TEST_PATH" envKeyTestRepo = "NOTATION_E2E_TEST_REPO" envKeyTestTag = "NOTATION_E2E_TEST_TAG" ) @@ -49,7 +48,6 @@ var ( var ( OCILayoutPath string - OCILayoutTestPath string TestRepoUri string TestTag string RegistryStoragePath string @@ -58,7 +56,6 @@ var ( func init() { RegisterFailHandler(Fail) setUpRegistry() - setUpOCILayout() setUpNotationValues() } @@ -73,10 +70,6 @@ func setUpRegistry() { setValue(envKeyTestTag, &TestTag) } -func setUpOCILayout() { - setPathValue(envKeyOCILayoutTestPath, &OCILayoutTestPath) -} - func setUpNotationValues() { // set Notation binary path setPathValue(envKeyNotationBinPath, &NotationBinPath) diff --git a/test/e2e/internal/notation/layout.go b/test/e2e/internal/notation/layout.go new file mode 100644 index 000000000..bfd72da03 --- /dev/null +++ b/test/e2e/internal/notation/layout.go @@ -0,0 +1,64 @@ +package notation + +import ( + "context" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/oci" +) + +// OCILayout is a OCI layout directory for +type OCILayout struct { + // Path is the path of the OCI layout directory. + Path string + // Tag is the tag of artifact in the OCI layout. + Tag string + // Digest is the digest of artifact in the OCI layout. + Digest string +} + +// GenerateOCILayout creates a new OCI layout in a temporary directory. +func GenerateOCILayout(srcRepoName string) (*OCILayout, error) { + ctx := context.Background() + + if srcRepoName == "" { + srcRepoName = TestRepoUri + } + + destPath := filepath.Join(GinkgoT().TempDir(), newRepoName()) + // create a local store from OCI layout directory. + srcStore, err := oci.NewFromFS(ctx, os.DirFS(filepath.Join(OCILayoutPath, srcRepoName))) + if err != nil { + return nil, err + } + + // create a dest store for store the generated oci layout. + destStore, err := oci.New(destPath) + if err != nil { + return nil, err + } + + // copy data + desc, err := oras.ExtendedCopy(ctx, srcStore, TestTag, destStore, "", oras.DefaultExtendedCopyOptions) + if err != nil { + return nil, err + } + return &OCILayout{ + Path: destPath, + Tag: TestTag, + Digest: desc.Digest.String(), + }, nil +} + +// ReferenceWithTag returns the reference with tag. +func (o *OCILayout) ReferenceWithTag() string { + return o.Path + ":" + o.Tag +} + +// ReferenceWithDigest returns the reference with digest. +func (o *OCILayout) ReferenceWithDigest() string { + return o.Path + "@" + o.Digest +} diff --git a/test/e2e/run.sh b/test/e2e/run.sh index 55ad071c8..24a1f2686 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -92,7 +92,6 @@ trap cleanup EXIT # set environment variable for E2E testing export NOTATION_E2E_CONFIG_PATH=$CWD/testdata/config export NOTATION_E2E_OCI_LAYOUT_PATH=$CWD/testdata/registry/oci_layout -export NOTATION_E2E_OCI_LAYOUT_TEST_PATH=$CWD/testdata/oci-layout/e2e export NOTATION_E2E_TEST_REPO=e2e export NOTATION_E2E_TEST_TAG=v1 export NOTATION_E2E_PLUGIN_PATH=$CWD/plugin/bin/$PLUGIN_NAME diff --git a/test/e2e/suite/command/inspect.go b/test/e2e/suite/command/inspect.go index 7cf6d722f..6b44327b2 100644 --- a/test/e2e/suite/command/inspect.go +++ b/test/e2e/suite/command/inspect.go @@ -48,8 +48,8 @@ var _ = Describe("notation inspect", func() { notation.Exec("inspect", "-d", artifact.DomainReferenceWithDigest()). MatchKeyWords(inspectSuccessfully...). - MatchErrKeyWords("https://notation-e2e.registry.io/v2/e2e"). - NoMatchErrKeyWords("http://notation-e2e.registry.io") + MatchErrKeyWords(HTTPSRequest). + NoMatchErrKeyWords(HTTPRequest) }) }) diff --git a/test/e2e/suite/command/list.go b/test/e2e/suite/command/list.go index c04d039ef..9adb36519 100644 --- a/test/e2e/suite/command/list.go +++ b/test/e2e/suite/command/list.go @@ -31,8 +31,8 @@ var _ = Describe("notation list", func() { "└── application/vnd.cncf.notary.signature", "└── sha256:", ). - MatchErrKeyWords("https://notation-e2e.registry.io/v2/e2e"). - NoMatchErrKeyWords("http://notation-e2e.registry.io") + MatchErrKeyWords(HTTPSRequest). + NoMatchErrKeyWords(HTTPRequest) }) }) @@ -50,4 +50,26 @@ var _ = Describe("notation list", func() { NoMatchErrKeyWords(HTTPSRequest) }) }) + + It("all signatures of an oci-layout", func() { + HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, _ *OCILayout, vhost *utils.VirtualHost) { + ociLayout, err := GenerateOCILayout("e2e-valid-signature") + if err != nil { + Fail(err.Error()) + } + + notation.Exec("list", "--oci-layout", ociLayout.ReferenceWithDigest()). + MatchKeyWords( + "└── application/vnd.cncf.notary.signature", + "└── sha256:273243a7a64e9312761ca0aa8f43b6ba805e677a561558143b6e92981c487339", + ) + }) + }) + + It("oci-layout with no signature", func() { + HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) { + notation.Exec("list", "--oci-layout", ociLayout.ReferenceWithDigest()). + MatchKeyWords("has no associated signature") + }) + }) }) diff --git a/test/e2e/suite/command/sign.go b/test/e2e/suite/command/sign.go index 8181fc9f8..e5168bac0 100644 --- a/test/e2e/suite/command/sign.go +++ b/test/e2e/suite/command/sign.go @@ -108,45 +108,37 @@ var _ = Describe("notation sign", func() { }) It("by digest with oci layout", func() { - GeneralHost(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) { - const digest = "sha256:cc2ae4e91a31a77086edbdbf4711de48e5fa3ebdacad3403e61777a9e1a53b6f" - ociLayoutReference := OCILayoutTestPath + "@" + digest - notation.Exec("sign", "--oci-layout", ociLayoutReference). + HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) { + notation.Exec("sign", "--oci-layout", ociLayout.ReferenceWithDigest()). MatchKeyWords(SignSuccessfully) }) }) It("by digest with oci layout and COSE format", func() { - GeneralHost(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) { - const digest = "sha256:cc2ae4e91a31a77086edbdbf4711de48e5fa3ebdacad3403e61777a9e1a53b6f" - ociLayoutReference := OCILayoutTestPath + "@" + digest - notation.Exec("sign", "--oci-layout", "--signature-format", "cose", ociLayoutReference). + HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) { + notation.Exec("sign", "--oci-layout", "--signature-format", "cose", ociLayout.ReferenceWithDigest()). MatchKeyWords(SignSuccessfully) }) }) It("by tag with oci layout", func() { - GeneralHost(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) { - ociLayoutReference := OCILayoutTestPath + ":" + TestTag - notation.Exec("sign", "--oci-layout", ociLayoutReference). + HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) { + notation.Exec("sign", "--oci-layout", ociLayout.ReferenceWithDigest()). MatchKeyWords(SignSuccessfully) }) }) It("by tag with oci layout and COSE format", func() { - GeneralHost(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) { - ociLayoutReference := OCILayoutTestPath + ":" + TestTag - notation.Exec("sign", "--oci-layout", "--signature-format", "cose", ociLayoutReference). + HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) { + notation.Exec("sign", "--oci-layout", "--signature-format", "cose", ociLayout.ReferenceWithDigest()). MatchKeyWords(SignSuccessfully) }) }) It("by digest with oci layout but without experimental", func() { - GeneralHost(BaseOptions(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) { - const digest = "sha256:cc2ae4e91a31a77086edbdbf4711de48e5fa3ebdacad3403e61777a9e1a53b6f" + HostWithOCILayout(BaseOptions(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) { expectedErrMsg := "Error: flag(s) --oci-layout in \"notation sign\" is experimental and not enabled by default. To use, please set NOTATION_EXPERIMENTAL=1 environment variable\n" - ociLayoutReference := OCILayoutTestPath + "@" + digest - notation.ExpectFailure().Exec("sign", "--oci-layout", ociLayoutReference). + notation.ExpectFailure().Exec("sign", "--oci-layout", ociLayout.ReferenceWithDigest()). MatchErrContent(expectedErrMsg) }) }) diff --git a/test/e2e/suite/command/verify.go b/test/e2e/suite/command/verify.go index 0056ed651..8d6eaf387 100644 --- a/test/e2e/suite/command/verify.go +++ b/test/e2e/suite/command/verify.go @@ -71,52 +71,45 @@ var _ = Describe("notation verify", func() { }) It("by digest with oci layout", func() { - GeneralHost(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) { - const digest = "sha256:cc2ae4e91a31a77086edbdbf4711de48e5fa3ebdacad3403e61777a9e1a53b6f" - ociLayoutReference := OCILayoutTestPath + "@" + digest - notation.Exec("sign", "--oci-layout", ociLayoutReference). + HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) { + notation.Exec("sign", "--oci-layout", ociLayout.ReferenceWithDigest()). MatchKeyWords(SignSuccessfully) experimentalMsg := "Warning: This feature is experimental and may not be fully tested or completed and may be deprecated. Report any issues to \"https://github/notaryproject/notation\"\n" - notation.Exec("verify", "--oci-layout", "--scope", "local/e2e", ociLayoutReference). + notation.Exec("verify", "--oci-layout", "--scope", "local/e2e", ociLayout.ReferenceWithDigest()). MatchKeyWords(VerifySuccessfully). MatchErrKeyWords(experimentalMsg) }) }) It("by tag with oci layout and COSE format", func() { - GeneralHost(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) { - ociLayoutReference := OCILayoutTestPath + ":" + TestTag - notation.Exec("sign", "--oci-layout", "--signature-format", "cose", ociLayoutReference). + HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) { + notation.Exec("sign", "--oci-layout", "--signature-format", "cose", ociLayout.ReferenceWithTag()). MatchKeyWords(SignSuccessfully) experimentalMsg := "Warning: This feature is experimental and may not be fully tested or completed and may be deprecated. Report any issues to \"https://github/notaryproject/notation\"\n" - notation.Exec("verify", "--oci-layout", "--scope", "local/e2e", ociLayoutReference). + notation.Exec("verify", "--oci-layout", "--scope", "local/e2e", ociLayout.ReferenceWithTag()). MatchKeyWords(VerifySuccessfully). MatchErrKeyWords(experimentalMsg) }) }) It("by digest with oci layout but without experimental", func() { - GeneralHost(BaseOptions(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) { - const digest = "sha256:cc2ae4e91a31a77086edbdbf4711de48e5fa3ebdacad3403e61777a9e1a53b6f" + HostWithOCILayout(BaseOptions(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) { expectedErrMsg := "Error: flag(s) --oci-layout,--scope in \"notation verify\" is experimental and not enabled by default. To use, please set NOTATION_EXPERIMENTAL=1 environment variable\n" - ociLayoutReference := OCILayoutTestPath + "@" + digest - notation.ExpectFailure().Exec("verify", "--oci-layout", "--scope", "local/e2e", ociLayoutReference). + notation.ExpectFailure().Exec("verify", "--oci-layout", "--scope", "local/e2e", ociLayout.ReferenceWithDigest()). MatchErrContent(expectedErrMsg) }) }) It("by digest with oci layout but missing scope", func() { - GeneralHost(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) { - const digest = "sha256:cc2ae4e91a31a77086edbdbf4711de48e5fa3ebdacad3403e61777a9e1a53b6f" - ociLayoutReference := OCILayoutTestPath + "@" + digest - notation.Exec("sign", "--oci-layout", ociLayoutReference). + HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) { + notation.Exec("sign", "--oci-layout", ociLayout.ReferenceWithDigest()). MatchKeyWords(SignSuccessfully) experimentalMsg := "Warning: This feature is experimental and may not be fully tested or completed and may be deprecated. Report any issues to \"https://github/notaryproject/notation\"\n" expectedErrMsg := "Error: if any flags in the group [oci-layout scope] are set they must all be set; missing [scope]" - notation.ExpectFailure().Exec("verify", "--oci-layout", ociLayoutReference). + notation.ExpectFailure().Exec("verify", "--oci-layout", ociLayout.ReferenceWithDigest()). MatchErrKeyWords(experimentalMsg). MatchErrKeyWords(expectedErrMsg) }) @@ -131,8 +124,8 @@ var _ = Describe("notation verify", func() { MatchKeyWords( VerifySuccessfully, ). - MatchErrKeyWords("https://notation-e2e.registry.io/v2/e2e"). - NoMatchErrKeyWords("http://notation-e2e.registry.io") + MatchErrKeyWords(HTTPSRequest). + NoMatchErrKeyWords(HTTPRequest) }) }) diff --git a/test/e2e/testdata/oci-layout/e2e/blobs/sha256/0d10372ee4448bf319929720bc465c1a6242cc68e82e12c152bc10d541cce578 b/test/e2e/testdata/oci-layout/e2e/blobs/sha256/0d10372ee4448bf319929720bc465c1a6242cc68e82e12c152bc10d541cce578 deleted file mode 100644 index 765bc38c9..000000000 --- a/test/e2e/testdata/oci-layout/e2e/blobs/sha256/0d10372ee4448bf319929720bc465c1a6242cc68e82e12c152bc10d541cce578 +++ /dev/null @@ -1 +0,0 @@ -Awesome Notation diff --git a/test/e2e/testdata/oci-layout/e2e/blobs/sha256/cc2ae4e91a31a77086edbdbf4711de48e5fa3ebdacad3403e61777a9e1a53b6f b/test/e2e/testdata/oci-layout/e2e/blobs/sha256/cc2ae4e91a31a77086edbdbf4711de48e5fa3ebdacad3403e61777a9e1a53b6f deleted file mode 100644 index 45028368d..000000000 --- a/test/e2e/testdata/oci-layout/e2e/blobs/sha256/cc2ae4e91a31a77086edbdbf4711de48e5fa3ebdacad3403e61777a9e1a53b6f +++ /dev/null @@ -1 +0,0 @@ -{"mediaType":"application/vnd.oci.artifact.manifest.v1+json","artifactType":"application/vnd.notation.config","blobs":[{"mediaType":"application/vnd.oci.image.layer.v1.tar","digest":"sha256:0d10372ee4448bf319929720bc465c1a6242cc68e82e12c152bc10d541cce578","size":17,"annotations":{"org.opencontainers.image.title":"awesome-notation.txt"}}],"annotations":{"org.opencontainers.artifact.created":"2022-12-22T01:11:12Z"}} \ No newline at end of file diff --git a/test/e2e/testdata/oci-layout/e2e/index.json b/test/e2e/testdata/oci-layout/e2e/index.json deleted file mode 100644 index 430948531..000000000 --- a/test/e2e/testdata/oci-layout/e2e/index.json +++ /dev/null @@ -1 +0,0 @@ -{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.artifact.manifest.v1+json","digest":"sha256:cc2ae4e91a31a77086edbdbf4711de48e5fa3ebdacad3403e61777a9e1a53b6f","size":417,"annotations":{"org.opencontainers.image.ref.name":"v1"}}]} \ No newline at end of file diff --git a/test/e2e/testdata/oci-layout/e2e/oci-layout b/test/e2e/testdata/oci-layout/e2e/oci-layout deleted file mode 100644 index 1343d370f..000000000 --- a/test/e2e/testdata/oci-layout/e2e/oci-layout +++ /dev/null @@ -1 +0,0 @@ -{"imageLayoutVersion":"1.0.0"} \ No newline at end of file