From 621f90714139704106b5511132ad9d392b82a0ca Mon Sep 17 00:00:00 2001 From: Junjie Gao Date: Tue, 23 May 2023 10:30:58 +0800 Subject: [PATCH 01/12] fix: add tls support Signed-off-by: Junjie Gao --- test/e2e/internal/notation/init.go | 1 + test/e2e/internal/notation/registry.go | 9 +++- test/e2e/scripts/dockerhub.sh | 7 +-- test/e2e/scripts/tls.sh | 52 +++++++++++++++++++ test/e2e/scripts/zot.sh | 24 +++++++-- test/e2e/testdata/nginx/nginx.conf | 18 +++++++ .../nginx/notation-e2e.registry.io.crt | 19 +++++++ .../nginx/notation-e2e.registry.io.key | 28 ++++++++++ 8 files changed, 149 insertions(+), 9 deletions(-) create mode 100644 test/e2e/scripts/tls.sh create mode 100644 test/e2e/testdata/nginx/nginx.conf create mode 100644 test/e2e/testdata/nginx/notation-e2e.registry.io.crt create mode 100644 test/e2e/testdata/nginx/notation-e2e.registry.io.key diff --git a/test/e2e/internal/notation/init.go b/test/e2e/internal/notation/init.go index 0615cb806..0c912cd42 100644 --- a/test/e2e/internal/notation/init.go +++ b/test/e2e/internal/notation/init.go @@ -23,6 +23,7 @@ const ( envKeyRegistryHost = "NOTATION_E2E_REGISTRY_HOST" envKeyRegistryUsername = "NOTATION_E2E_REGISTRY_USERNAME" envKeyRegistryPassword = "NOTATION_E2E_REGISTRY_PASSWORD" + envKeyDomainRegistryHost = "NOTATION_E2E_DOMAIN_REGISTRY_HOST" envKeyNotationBinPath = "NOTATION_E2E_BINARY_PATH" envKeyNotationOldBinPath = "NOTATION_E2E_OLD_BINARY_PATH" envKeyNotationPluginPath = "NOTATION_E2E_PLUGIN_PATH" diff --git a/test/e2e/internal/notation/registry.go b/test/e2e/internal/notation/registry.go index 89722bd04..451b76fc2 100644 --- a/test/e2e/internal/notation/registry.go +++ b/test/e2e/internal/notation/registry.go @@ -19,9 +19,16 @@ import ( const ArtifactTypeNotation = "application/vnd.cncf.notary.signature" type Registry struct { - Host string + // Host is the registry host. + Host string + // Username is the username to access the registry. Username string + // Password is the password to access the registry. Password string + // DomainHost is an external registry host for testing --plain-http flag. + // if the host is localhost, notation make all connection as plain http. + // if the host is not localhost, notation make all connection as https. + DomainHost string } // CreateArtifact copies a local OCI layout to the registry to create diff --git a/test/e2e/scripts/dockerhub.sh b/test/e2e/scripts/dockerhub.sh index 6a0814678..d1783f612 100644 --- a/test/e2e/scripts/dockerhub.sh +++ b/test/e2e/scripts/dockerhub.sh @@ -16,9 +16,10 @@ if [ -z "$DOCKER_USERNAME" ] || [ -z "$DOCKER_PASSWORD" ]; then fi # set environment variables for E2E testing -export NOTATION_E2E_REGISTRY_HOST=docker.io/$DOCKER_USERNAME -export NOTATION_E2E_REGISTRY_USERNAME=$DOCKER_USERNAME -export NOTATION_E2E_REGISTRY_PASSWORD=$DOCKER_PASSWORD +export NOTATION_E2E_REGISTRY_HOST="docker.io/$DOCKER_USERNAME" +export NOTATION_E2E_REGISTRY_USERNAME="$DOCKER_USERNAME" +export NOTATION_E2E_REGISTRY_PASSWORD="$DOCKER_PASSWORD" +export NOTATION_E2E_DOMAIN_REGISTRY_HOST="$NOTATION_E2E_REGISTRY_HOST" function setup_registry { echo "use $NOTATION_E2E_REGISTRY_HOST" diff --git a/test/e2e/scripts/tls.sh b/test/e2e/scripts/tls.sh new file mode 100644 index 000000000..bbb6d23ec --- /dev/null +++ b/test/e2e/scripts/tls.sh @@ -0,0 +1,52 @@ +#!/bin/bash -e +# +# Usage +# For setup: +# 1. source ./scripts/tls.sh +# 2. call create_docker_network +# 3. setup registry with port 5000 in $DOCKER_NETWORK +# 4. call setup_tls reverse proxy +# +# For clean up: +# 1. call clean_up +# 2. clean up registry +# 3. call remove_docker_network +# +# note: this script needs sudo permission to add TLS certificate to system and +# add domain registry host. + +NGINX_CONTAINER_NAME=nginx +DOMAIN=notation-e2e.registry.io +DOCKER_NETWORK=notation-e2e +TLS_PORT=5001 + +function create_docker_network { + docker network create "$DOCKER_NETWORK" +} + +function remove_docker_network { + docker network rm "$DOCKER_NETWORK" +} + +function setup_tls { + # add domain registry host to /etc/hosts for testing --plain-http feature + echo "127.0.0.1 $DOMAIN" | sudo tee -a /etc/hosts + # add TLS certificate to system + sudo mkdir -p /usr/local/share/ca-certificates/ + sudo cp ./testdata/nginx/notation-e2e.registry.io.crt /usr/local/share/ca-certificates/ + sudo update-ca-certificates + + # start Nginx for TLS + docker run -d -p "$TLS_PORT:443" \ + --network "$DOCKER_NETWORK" \ + --mount type=bind,source="$(pwd)/testdata/nginx/",target=/etc/nginx \ + --name "$NGINX_CONTAINER_NAME" \ + --rm nginx:latest +} + +function clean_up_tls { + docker container stop $NGINX_CONTAINER_NAME 1>/dev/null && echo "Nginx stopped" + sudo sed -i "/${NOTATION_E2E_DOMAIN_REGISTRY_HOST}/d" /etc/hosts + sudo rm /usr/local/share/ca-certificates/notation-e2e.registry.io.crt + sudo update-ca-certificates +} diff --git a/test/e2e/scripts/zot.sh b/test/e2e/scripts/zot.sh index ebd4f7cd2..a694113df 100644 --- a/test/e2e/scripts/zot.sh +++ b/test/e2e/scripts/zot.sh @@ -4,24 +4,38 @@ # Usage # ./run.sh zot [old-notation-binary-path] +source ./scripts/tls.sh + REG_HOST=localhost REG_PORT=5000 -ZOT_CONTAINER_NAME=zot +ZOT_CONTAINER_NAME=notation-e2e-registry -# set environment variables for E2E testing -export NOTATION_E2E_REGISTRY_HOST=$REG_HOST:$REG_PORT +# set required environment variables for E2E testing +export NOTATION_E2E_REGISTRY_HOST="$REG_HOST:$REG_PORT" export NOTATION_E2E_REGISTRY_USERNAME=testuser export NOTATION_E2E_REGISTRY_PASSWORD=testpassword +export NOTATION_E2E_DOMAIN_REGISTRY_HOST="$DOMAIN:$TLS_PORT" function setup_registry { + create_docker_network # start Zot - docker run -d -p $REG_PORT:$REG_PORT -it --name $ZOT_CONTAINER_NAME \ - --mount type=bind,source=`pwd`/testdata/registry/zot/,target=/etc/zot \ + docker run -d -p $REG_PORT:$REG_PORT -it \ + --name $ZOT_CONTAINER_NAME \ + --network $DOCKER_NETWORK \ + --mount type=bind,source="$(pwd)/testdata/registry/zot/",target=/etc/zot \ --rm ghcr.io/project-zot/zot-minimal-linux-amd64:latest + + if [ "$GITHUB_ACTIONS" == "true" ]; then + setup_tls + fi # make sure that Zot is ready sleep 1 } function cleanup_registry { docker container stop $ZOT_CONTAINER_NAME 1>/dev/null && echo "Zot stopped" + if [ "$GITHUB_ACTIONS" == "true" ]; then + clean_up_tls + fi + remove_docker_network } \ No newline at end of file diff --git a/test/e2e/testdata/nginx/nginx.conf b/test/e2e/testdata/nginx/nginx.conf new file mode 100644 index 000000000..7eb58901c --- /dev/null +++ b/test/e2e/testdata/nginx/nginx.conf @@ -0,0 +1,18 @@ +events { + worker_connections 1024; +} + +http { + server { + listen 443 ssl; + + server_name notation-e2e.regisry.io; + + ssl_certificate /etc/nginx/notation-e2e.registry.io.crt; + ssl_certificate_key /etc/nginx/notation-e2e.registry.io.key; + + location / { + proxy_pass http://notation-e2e-registry:5000; + } + } +} \ No newline at end of file diff --git a/test/e2e/testdata/nginx/notation-e2e.registry.io.crt b/test/e2e/testdata/nginx/notation-e2e.registry.io.crt new file mode 100644 index 000000000..a54faeebf --- /dev/null +++ b/test/e2e/testdata/nginx/notation-e2e.registry.io.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDHDCCAgSgAwIBAgIUQcDBs3e5Z0lR1hUqQZs0diP7CQAwDQYJKoZIhvcNAQEL +BQAwIzEhMB8GA1UEAwwYbm90YXRpb24tZTJlLnJlZ2lzdHJ5LmlvMCAXDTIzMDUy +MjA5MDAyMFoYDzIxMjMwNDI4MDkwMDIwWjAjMSEwHwYDVQQDDBhub3RhdGlvbi1l +MmUucmVnaXN0cnkuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDn +LxBVN7iawG8Q5MD1Fczv0Zr1QG5pRnz5DXo2hLSnrJ2xYiXWRgIlfI67adMPiF3v +LZXKVJtVkkYAL/0iLu6YpIHltmzXo+/rY3RV+jk0Lj380Zfp5gC6SLXIKuzM9AnT +g3pkOt7zBHQP0xOcK2aPuhPEySuSoGQ6jupWFD3vPBgvcW7+sF0NUHdTnN6dz0sR +dmlbEDmaYJ+weZa0Skvc1Mc2znnJmWZWY6PlC7SPQey6MC5CTjukT8AVMqwedjfv +fQ8vOhvxu+UFnncEqwM7f83hiYD4QWK/ZA5iIobpd0aP/83R87WkGNyHypnWbmEb +C7Py8OWZoFKqcBHj13c/AgMBAAGjRjBEMCMGA1UdEQQcMBqCGG5vdGF0aW9uLWUy +ZS5yZWdpc3RyeS5pbzAdBgNVHQ4EFgQUCNOgP6vTy0+IO3UA1mZ9G8pAHqQwDQYJ +KoZIhvcNAQELBQADggEBAIofq+60DleGTdxr4RqvaXytIbNX57TrAjUvFwFy9E3e +mxmy14vHkwOLRoCMbbP4wy+XG2I1r7NcmhDxGc9z9z+YrhuW9IDWSxv8+KYWM5T/ +7viqHJtyYk5eMpN+k+7qB8kXoahnrgOZjmdDwJBsj6kwrcnuqvwRBi3fxeNJl+/Q +tDOEPyqIybWMw43tTvE33k0k8ia7nEk0slcURdLVllAfTC6DdL2SJtELD8SMvPPh +nwYKU6dGbYlnWVRrfqQG3PCksuLbRzX+zQCAMlSgtD+KSpjyxkaazPY/SD4gLav7 +u+96xpZVLbrsjklcrT2dEku3zprnM63P8DnV/wHDdUc= +-----END CERTIFICATE----- diff --git a/test/e2e/testdata/nginx/notation-e2e.registry.io.key b/test/e2e/testdata/nginx/notation-e2e.registry.io.key new file mode 100644 index 000000000..44000441b --- /dev/null +++ b/test/e2e/testdata/nginx/notation-e2e.registry.io.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDnLxBVN7iawG8Q +5MD1Fczv0Zr1QG5pRnz5DXo2hLSnrJ2xYiXWRgIlfI67adMPiF3vLZXKVJtVkkYA +L/0iLu6YpIHltmzXo+/rY3RV+jk0Lj380Zfp5gC6SLXIKuzM9AnTg3pkOt7zBHQP +0xOcK2aPuhPEySuSoGQ6jupWFD3vPBgvcW7+sF0NUHdTnN6dz0sRdmlbEDmaYJ+w +eZa0Skvc1Mc2znnJmWZWY6PlC7SPQey6MC5CTjukT8AVMqwedjfvfQ8vOhvxu+UF +nncEqwM7f83hiYD4QWK/ZA5iIobpd0aP/83R87WkGNyHypnWbmEbC7Py8OWZoFKq +cBHj13c/AgMBAAECggEAQhYnnpy/pmlVNqiV6lnRjErId8xz459VUWLDaXtVI0uK +hq8ubsrziSDKspOFVL7gT2OiGsVF5Ffcr+gH/jIZXcRFJ9QW2CwShSEYnA1cNej0 +KmYF/cSUt6vaXz66E7q9ZlwC7E0R97lxriZiSDX16yc/yHTTgmZcUIsTPQkrTUw+ +7IgY0uV3HFCnGU5i5jeB+UF5MIPauBPt3YQcbBythwkZadxaKI/1Mp1vn4RcdzMg ++XJ4GMND1A1qzn3mQm5PqBMk4NC+FVcBxvdxzZum7qu1iLap0e3fyD9PqFTb1Zix +3976Qj/QrttbO1WXDtBvEaXqlGVbEDR1cpKQKYghwQKBgQDx+GkdBb6lYGDERnZC +pY65s0k2HVnZ6BxFGJXMryWN8CYMXKgns0O4idIYQQnzWFUmudxfAy13H3wukXDC +aHF2YZ1QuqOmc3kIkxKVD+9QfcgKGfLAGemyFtwhSbegk6MKF4UOqDNL9unvDdAo +7j9rjq5IOqfgPorqm1AoySgDQQKBgQD0loxIEM/3O5PqT8sda5G9bsXh9QlV3Za0 +XSVYlkKxXuPIlC0jmDHk/xI41ykdrqjwmYd2FOt7Luuevke/ahqQSbXxy0LFdR5J +QJ6iN3OMZMsXMImRCoLD0kEvh+Z1u4VUiHuXmBQtiv5uP9S1KAETL44SEZyfJn63 +uvNMlplafwKBgENb58cQhlX7UnTROLKs6+J+Km9KFG041EXX5juotkehBraCRL1o +hf2lQDtIP8DiYjH5o4M/mzSCK0u7aSx1bsCJxAVpL41yr8rXRmEAoppBqaJGPvGD +RS8yde0+XEPzVXvFuGCwKjeHcO//ZGdAi58hhRrOWVVvk7RjsBjqhp0BAoGBAJv5 +WZInbofKLYSRyASF8ZWtC3IR8hcYzR9N+x/oCrXTvkzN+Y8mYkMXSkaHJ0gvdrqg +HZt2sciHXmiIDXcKsc/rwaRlK7qB+oNaOw9Vb1FLgZvTLxcYbdV0wm8OKjBQGjGT +K8W7jLqSVbh26i1wSmcyv1XUd12ijdKa3MatjzP/AoGARuw3wa+b0GCH5jtzJMBs +9N/Tr9lGLHH7Q57c4nAR2rR5L0DGAns5XWN9yKCyIDFFh89kLGNRwj/cONY3nXud +/1l7seup811NusPiBaZC+5LX/zAvzH0orFzinEdV7V4g435j/i52V5Bl5fJVw7FM +CfqmotJm0hceUk1QaStn+Ds= +-----END PRIVATE KEY----- From 7d1190e01919225d886b89b4118eddc8dba16e35 Mon Sep 17 00:00:00 2001 From: Junjie Gao Date: Tue, 23 May 2023 11:31:40 +0800 Subject: [PATCH 02/12] feat: add E2E testing TLS support Signed-off-by: Junjie Gao --- test/e2e/internal/notation/host.go | 9 +++++++++ test/e2e/internal/notation/init.go | 1 + test/e2e/internal/notation/registry.go | 6 +++++- test/e2e/run.sh | 2 +- test/e2e/scripts/dockerhub.sh | 2 +- test/e2e/scripts/tls.sh | 4 ++-- test/e2e/scripts/zot.sh | 2 +- test/e2e/suite/command/sign.go | 10 ++++++++++ 8 files changed, 30 insertions(+), 6 deletions(-) diff --git a/test/e2e/internal/notation/host.go b/test/e2e/internal/notation/host.go index af7657585..17069425f 100644 --- a/test/e2e/internal/notation/host.go +++ b/test/e2e/internal/notation/host.go @@ -5,6 +5,7 @@ import ( "path/filepath" "github.com/notaryproject/notation/test/e2e/internal/utils" + . "github.com/onsi/ginkgo/v2" ) // CoreTestFunc is the test function running in a VirtualHost. @@ -40,6 +41,14 @@ func Host(options []utils.HostOption, fn CoreTestFunc) { fn(vhost.Executor, artifact, vhost) } +// HostWithTLS only run the test in GitHub Actions. +func HostWithTLS(options []utils.HostOption, fn CoreTestFunc) { + if os.Getenv("GITHUB_ACTIONS") != "true" { + Skip("only run in GitHub Actions") + } + Host(options, fn) +} + // GeneralHost creates a virtualized notation testing host by modify // the "XDG_CONFIG_HOME" environment variable of the Executor. It's agnostic to // the Repository of the artifact. diff --git a/test/e2e/internal/notation/init.go b/test/e2e/internal/notation/init.go index 0c912cd42..c6c2a1292 100644 --- a/test/e2e/internal/notation/init.go +++ b/test/e2e/internal/notation/init.go @@ -66,6 +66,7 @@ func setUpRegistry() { setValue(envKeyRegistryHost, &TestRegistry.Host) setValue(envKeyRegistryUsername, &TestRegistry.Username) setValue(envKeyRegistryPassword, &TestRegistry.Password) + setValue(envKeyDomainRegistryHost, &TestRegistry.DomainHost) setPathValue(envKeyOCILayoutPath, &OCILayoutPath) setValue(envKeyTestRepo, &TestRepoUri) diff --git a/test/e2e/internal/notation/registry.go b/test/e2e/internal/notation/registry.go index 451b76fc2..58e8c4a66 100644 --- a/test/e2e/internal/notation/registry.go +++ b/test/e2e/internal/notation/registry.go @@ -25,7 +25,7 @@ type Registry struct { Username string // Password is the password to access the registry. Password string - // DomainHost is an external registry host for testing --plain-http flag. + // DomainHost is an registry host for testing --plain-http flag. // if the host is localhost, notation make all connection as plain http. // if the host is not localhost, notation make all connection as https. DomainHost string @@ -108,6 +108,10 @@ func (r *Artifact) ReferenceWithDigest() string { return fmt.Sprintf("%s/%s@%s", r.Host, r.Repo, r.Digest) } +func (r *Artifact) DomainReferenceWithDigest() string { + return fmt.Sprintf("%s/%s@%s", r.DomainHost, r.Repo, r.Digest) +} + // SignatureManifest returns the manifest of the artifact. func (r *Artifact) SignatureDescriptors() ([]ocispec.Descriptor, error) { ctx := context.Background() diff --git a/test/e2e/run.sh b/test/e2e/run.sh index 4d9eb861e..55ad071c8 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -40,7 +40,7 @@ if [ ! -f "$NOTATION_E2E_OLD_BINARY_PATH" ]; then echo "Try to use old notation binary at $NOTATION_E2E_OLD_BINARY_PATH" if [ ! -f $NOTATION_E2E_OLD_BINARY_PATH ]; then - TAG=1.0.0-rc.2 # without 'v' + TAG=1.0.0-rc.5 # without 'v' echo "Didn't find old notation binary locally. Try to download notation v$TAG." TAR_NAME=notation_${TAG}_linux_amd64.tar.gz diff --git a/test/e2e/scripts/dockerhub.sh b/test/e2e/scripts/dockerhub.sh index d1783f612..1e62c4206 100644 --- a/test/e2e/scripts/dockerhub.sh +++ b/test/e2e/scripts/dockerhub.sh @@ -56,4 +56,4 @@ function cleanup_registry { echo "$NOTATION_E2E_REGISTRY_HOST/$repoName deleted." done done -} \ No newline at end of file +} diff --git a/test/e2e/scripts/tls.sh b/test/e2e/scripts/tls.sh index bbb6d23ec..38700836d 100644 --- a/test/e2e/scripts/tls.sh +++ b/test/e2e/scripts/tls.sh @@ -12,7 +12,7 @@ # 2. clean up registry # 3. call remove_docker_network # -# note: this script needs sudo permission to add TLS certificate to system and +# Note: this script needs sudo permission to add TLS certificate to system and # add domain registry host. NGINX_CONTAINER_NAME=nginx @@ -45,7 +45,7 @@ function setup_tls { } function clean_up_tls { - docker container stop $NGINX_CONTAINER_NAME 1>/dev/null && echo "Nginx stopped" + docker container stop "$NGINX_CONTAINER_NAME" 1>/dev/null && echo "Nginx stopped" sudo sed -i "/${NOTATION_E2E_DOMAIN_REGISTRY_HOST}/d" /etc/hosts sudo rm /usr/local/share/ca-certificates/notation-e2e.registry.io.crt sudo update-ca-certificates diff --git a/test/e2e/scripts/zot.sh b/test/e2e/scripts/zot.sh index a694113df..e99b69664 100644 --- a/test/e2e/scripts/zot.sh +++ b/test/e2e/scripts/zot.sh @@ -38,4 +38,4 @@ function cleanup_registry { clean_up_tls fi remove_docker_network -} \ No newline at end of file +} diff --git a/test/e2e/suite/command/sign.go b/test/e2e/suite/command/sign.go index 67a75df6c..13f8277af 100644 --- a/test/e2e/suite/command/sign.go +++ b/test/e2e/suite/command/sign.go @@ -150,4 +150,14 @@ var _ = Describe("notation sign", func() { MatchErrContent(expectedErrMsg) }) }) + + It("with TLS by digest", func() { + HostWithTLS(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + notation.Exec("sign", artifact.DomainReferenceWithDigest()). + MatchKeyWords(SignSuccessfully) + + OldNotation().Exec("verify", artifact.DomainReferenceWithDigest()). + MatchKeyWords(VerifySuccessfully) + }) + }) }) From 7b9c619fddceccac70387ca4f75d9f684add963c Mon Sep 17 00:00:00 2001 From: Junjie Gao Date: Tue, 23 May 2023 13:22:11 +0800 Subject: [PATCH 03/12] fix: update code Signed-off-by: Junjie Gao --- test/e2e/internal/notation/host.go | 5 ++++- test/e2e/run.sh | 2 +- test/e2e/scripts/tls.sh | 3 +-- test/e2e/scripts/zot.sh | 2 +- test/e2e/suite/command/sign.go | 10 ++++++++++ test/e2e/suite/command/verify.go | 20 ++++++++++++++++++++ test/e2e/testdata/nginx/nginx.conf | 1 + 7 files changed, 38 insertions(+), 5 deletions(-) diff --git a/test/e2e/internal/notation/host.go b/test/e2e/internal/notation/host.go index 17069425f..d07a8363c 100644 --- a/test/e2e/internal/notation/host.go +++ b/test/e2e/internal/notation/host.go @@ -41,7 +41,10 @@ func Host(options []utils.HostOption, fn CoreTestFunc) { fn(vhost.Executor, artifact, vhost) } -// HostWithTLS only run the test in GitHub Actions. +// HostWithTLS only run the test in GitHub Actions for testing TLS related +// features. +// +// The booting script will setup TLS certificate for Github Actions environment. func HostWithTLS(options []utils.HostOption, fn CoreTestFunc) { if os.Getenv("GITHUB_ACTIONS") != "true" { Skip("only run in GitHub Actions") diff --git a/test/e2e/run.sh b/test/e2e/run.sh index 55ad071c8..252533a2b 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -98,4 +98,4 @@ export NOTATION_E2E_TEST_TAG=v1 export NOTATION_E2E_PLUGIN_PATH=$CWD/plugin/bin/$PLUGIN_NAME # run tests -ginkgo -r -p -v \ No newline at end of file +ginkgo -r -p -v --focus "with --insecure-registry by digest" \ No newline at end of file diff --git a/test/e2e/scripts/tls.sh b/test/e2e/scripts/tls.sh index 38700836d..e12795a48 100644 --- a/test/e2e/scripts/tls.sh +++ b/test/e2e/scripts/tls.sh @@ -18,7 +18,6 @@ NGINX_CONTAINER_NAME=nginx DOMAIN=notation-e2e.registry.io DOCKER_NETWORK=notation-e2e -TLS_PORT=5001 function create_docker_network { docker network create "$DOCKER_NETWORK" @@ -37,7 +36,7 @@ function setup_tls { sudo update-ca-certificates # start Nginx for TLS - docker run -d -p "$TLS_PORT:443" \ + docker run -d -p 80:80 -p 443:443 \ --network "$DOCKER_NETWORK" \ --mount type=bind,source="$(pwd)/testdata/nginx/",target=/etc/nginx \ --name "$NGINX_CONTAINER_NAME" \ diff --git a/test/e2e/scripts/zot.sh b/test/e2e/scripts/zot.sh index e99b69664..4941c4b43 100644 --- a/test/e2e/scripts/zot.sh +++ b/test/e2e/scripts/zot.sh @@ -14,7 +14,7 @@ ZOT_CONTAINER_NAME=notation-e2e-registry export NOTATION_E2E_REGISTRY_HOST="$REG_HOST:$REG_PORT" export NOTATION_E2E_REGISTRY_USERNAME=testuser export NOTATION_E2E_REGISTRY_PASSWORD=testpassword -export NOTATION_E2E_DOMAIN_REGISTRY_HOST="$DOMAIN:$TLS_PORT" +export NOTATION_E2E_DOMAIN_REGISTRY_HOST="$DOMAIN" function setup_registry { create_docker_network diff --git a/test/e2e/suite/command/sign.go b/test/e2e/suite/command/sign.go index 13f8277af..f040101a9 100644 --- a/test/e2e/suite/command/sign.go +++ b/test/e2e/suite/command/sign.go @@ -160,4 +160,14 @@ var _ = Describe("notation sign", func() { MatchKeyWords(VerifySuccessfully) }) }) + + It("with --insecure-registry by digest", func() { + HostWithTLS(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + notation.Exec("sign", "--insecure-registry", artifact.DomainReferenceWithDigest()). + MatchKeyWords(SignSuccessfully) + + OldNotation().Exec("verify", artifact.DomainReferenceWithDigest()). + MatchKeyWords(VerifySuccessfully) + }) + }) }) diff --git a/test/e2e/suite/command/verify.go b/test/e2e/suite/command/verify.go index 6abf8dc43..f6d3de9f7 100644 --- a/test/e2e/suite/command/verify.go +++ b/test/e2e/suite/command/verify.go @@ -121,4 +121,24 @@ var _ = Describe("notation verify", func() { MatchErrKeyWords(expectedErrMsg) }) }) + + It("with TLS by digest", func() { + HostWithTLS(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + notation.Exec("sign", artifact.DomainReferenceWithDigest()). + MatchKeyWords(SignSuccessfully) + + notation.Exec("verify", artifact.DomainReferenceWithDigest(), "-v"). + MatchKeyWords(VerifySuccessfully) + }) + }) + + It("with --insecure-registry by digest", func() { + HostWithTLS(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + notation.Exec("sign", artifact.DomainReferenceWithDigest()). + MatchKeyWords(SignSuccessfully) + + notation.Exec("verify", "--insecure-registry", artifact.DomainReferenceWithDigest(), "-v"). + MatchKeyWords(VerifySuccessfully) + }) + }) }) diff --git a/test/e2e/testdata/nginx/nginx.conf b/test/e2e/testdata/nginx/nginx.conf index 7eb58901c..e38940a84 100644 --- a/test/e2e/testdata/nginx/nginx.conf +++ b/test/e2e/testdata/nginx/nginx.conf @@ -4,6 +4,7 @@ events { http { server { + listen 80; listen 443 ssl; server_name notation-e2e.regisry.io; From c029774e5e56f333254142e91b0e5e338dc30ab3 Mon Sep 17 00:00:00 2001 From: Junjie Gao Date: Tue, 23 May 2023 13:29:01 +0800 Subject: [PATCH 04/12] fix: update code Signed-off-by: Junjie Gao --- test/e2e/run.sh | 2 +- test/e2e/testdata/nginx/nginx.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/run.sh b/test/e2e/run.sh index 252533a2b..55ad071c8 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -98,4 +98,4 @@ export NOTATION_E2E_TEST_TAG=v1 export NOTATION_E2E_PLUGIN_PATH=$CWD/plugin/bin/$PLUGIN_NAME # run tests -ginkgo -r -p -v --focus "with --insecure-registry by digest" \ No newline at end of file +ginkgo -r -p -v \ No newline at end of file diff --git a/test/e2e/testdata/nginx/nginx.conf b/test/e2e/testdata/nginx/nginx.conf index e38940a84..9e4ee853c 100644 --- a/test/e2e/testdata/nginx/nginx.conf +++ b/test/e2e/testdata/nginx/nginx.conf @@ -16,4 +16,4 @@ http { proxy_pass http://notation-e2e-registry:5000; } } -} \ No newline at end of file +} From b227d0c30d1f69f7fb10e984bef0ca8bcb8f93c9 Mon Sep 17 00:00:00 2001 From: Junjie Gao Date: Tue, 23 May 2023 15:52:51 +0800 Subject: [PATCH 05/12] test: added test for list and inspect commands Signed-off-by: Junjie Gao --- test/e2e/suite/command/inspect.go | 63 +++++++++++++++++++++++++++++++ test/e2e/suite/command/list.go | 49 ++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 test/e2e/suite/command/inspect.go create mode 100644 test/e2e/suite/command/list.go diff --git a/test/e2e/suite/command/inspect.go b/test/e2e/suite/command/inspect.go new file mode 100644 index 000000000..ef9495e51 --- /dev/null +++ b/test/e2e/suite/command/inspect.go @@ -0,0 +1,63 @@ +package command + +import ( + . "github.com/notaryproject/notation/test/e2e/internal/notation" + "github.com/notaryproject/notation/test/e2e/internal/utils" + . "github.com/notaryproject/notation/test/e2e/suite/common" + . "github.com/onsi/ginkgo/v2" +) + +var inspectSuccessfully = []string{ + "└── application/vnd.cncf.notary.signature", + "└── sha256:", + "├── media type:", + "├── signature algorithm:", + "├── signed attributes", + "signingTime:", + "signingScheme:", + "├── user defined attributes", + "│ └── (empty)", + "├── unsigned attributes", + "│ └── signingAgent: Notation/", + "├── certificates", + "│ └── SHA256 fingerprint:", + "issued to:", + "issued by:", + "expiry:", + "└── signed artifact", + "media type:", + "digest:", + "size:", +} + +var _ = Describe("notation inspect", func() { + It("all signatures of an image", func() { + Host(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + notation.Exec("sign", artifact.ReferenceWithDigest()). + MatchKeyWords(SignSuccessfully) + + notation.Exec("inspect", artifact.ReferenceWithDigest()). + MatchKeyWords(inspectSuccessfully...) + }) + }) + + It("all signatures of an image with TLS", func() { + HostWithTLS(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + notation.Exec("sign", artifact.DomainReferenceWithDigest()). + MatchKeyWords(SignSuccessfully) + + notation.Exec("inspect", artifact.DomainReferenceWithDigest()). + MatchKeyWords(inspectSuccessfully...) + }) + }) + + It("all signatures of an image with --insecure-registry flag", func() { + HostWithTLS(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + notation.Exec("sign", artifact.DomainReferenceWithDigest()). + MatchKeyWords(SignSuccessfully) + + notation.Exec("inspect", "--insecure-registry", artifact.DomainReferenceWithDigest()). + MatchKeyWords(inspectSuccessfully...) + }) + }) +}) diff --git a/test/e2e/suite/command/list.go b/test/e2e/suite/command/list.go new file mode 100644 index 000000000..88dbd217b --- /dev/null +++ b/test/e2e/suite/command/list.go @@ -0,0 +1,49 @@ +package command + +import ( + . "github.com/notaryproject/notation/test/e2e/internal/notation" + "github.com/notaryproject/notation/test/e2e/internal/utils" + . "github.com/notaryproject/notation/test/e2e/suite/common" + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("notation list", func() { + It("all signatures of an image", func() { + Host(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + notation.Exec("sign", artifact.ReferenceWithDigest()). + MatchKeyWords(SignSuccessfully) + + notation.Exec("list", artifact.ReferenceWithDigest()). + MatchKeyWords( + "└── application/vnd.cncf.notary.signature", + "└── sha256:", + ) + }) + }) + + It("all signatures of an image with TLS", func() { + HostWithTLS(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + notation.Exec("sign", artifact.DomainReferenceWithDigest()). + MatchKeyWords(SignSuccessfully) + + notation.Exec("list", artifact.DomainReferenceWithDigest()). + MatchKeyWords( + "└── application/vnd.cncf.notary.signature", + "└── sha256:", + ) + }) + }) + + It("all signatures of an image with --insecure-registry flag", func() { + HostWithTLS(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + notation.Exec("sign", artifact.DomainReferenceWithDigest()). + MatchKeyWords(SignSuccessfully) + + notation.Exec("list", "--insecure-registry", artifact.DomainReferenceWithDigest()). + MatchKeyWords( + "└── application/vnd.cncf.notary.signature", + "└── sha256:", + ) + }) + }) +}) From 5a160457bf1e1b87561c6409d5e4dc79025b2a06 Mon Sep 17 00:00:00 2001 From: Junjie Gao Date: Tue, 23 May 2023 16:35:39 +0800 Subject: [PATCH 06/12] fix: update code Signed-off-by: Junjie Gao --- test/e2e/internal/notation/host.go | 8 ++++---- test/e2e/suite/command/inspect.go | 4 ++-- test/e2e/suite/command/list.go | 4 ++-- test/e2e/suite/command/sign.go | 4 ++-- test/e2e/suite/command/verify.go | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/e2e/internal/notation/host.go b/test/e2e/internal/notation/host.go index d07a8363c..613c0705e 100644 --- a/test/e2e/internal/notation/host.go +++ b/test/e2e/internal/notation/host.go @@ -41,11 +41,11 @@ func Host(options []utils.HostOption, fn CoreTestFunc) { fn(vhost.Executor, artifact, vhost) } -// HostWithTLS only run the test in GitHub Actions for testing TLS related -// features. +// HostInGithubAction only run the test in GitHub Actions. // -// The booting script will setup TLS certificate for Github Actions environment. -func HostWithTLS(options []utils.HostOption, fn CoreTestFunc) { +// The booting script will setup TLS reverse proxy and TLS certificate +// for Github Actions environment. +func HostInGithubAction(options []utils.HostOption, fn CoreTestFunc) { if os.Getenv("GITHUB_ACTIONS") != "true" { Skip("only run in GitHub Actions") } diff --git a/test/e2e/suite/command/inspect.go b/test/e2e/suite/command/inspect.go index ef9495e51..d658c89ec 100644 --- a/test/e2e/suite/command/inspect.go +++ b/test/e2e/suite/command/inspect.go @@ -42,7 +42,7 @@ var _ = Describe("notation inspect", func() { }) It("all signatures of an image with TLS", func() { - HostWithTLS(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { notation.Exec("sign", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) @@ -52,7 +52,7 @@ var _ = Describe("notation inspect", func() { }) It("all signatures of an image with --insecure-registry flag", func() { - HostWithTLS(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { notation.Exec("sign", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) diff --git a/test/e2e/suite/command/list.go b/test/e2e/suite/command/list.go index 88dbd217b..f5d846f58 100644 --- a/test/e2e/suite/command/list.go +++ b/test/e2e/suite/command/list.go @@ -22,7 +22,7 @@ var _ = Describe("notation list", func() { }) It("all signatures of an image with TLS", func() { - HostWithTLS(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { notation.Exec("sign", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) @@ -35,7 +35,7 @@ var _ = Describe("notation list", func() { }) It("all signatures of an image with --insecure-registry flag", func() { - HostWithTLS(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { notation.Exec("sign", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) diff --git a/test/e2e/suite/command/sign.go b/test/e2e/suite/command/sign.go index f040101a9..76a6910cc 100644 --- a/test/e2e/suite/command/sign.go +++ b/test/e2e/suite/command/sign.go @@ -152,7 +152,7 @@ var _ = Describe("notation sign", func() { }) It("with TLS by digest", func() { - HostWithTLS(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { notation.Exec("sign", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) @@ -162,7 +162,7 @@ var _ = Describe("notation sign", func() { }) It("with --insecure-registry by digest", func() { - HostWithTLS(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { notation.Exec("sign", "--insecure-registry", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) diff --git a/test/e2e/suite/command/verify.go b/test/e2e/suite/command/verify.go index f6d3de9f7..6450c8e5c 100644 --- a/test/e2e/suite/command/verify.go +++ b/test/e2e/suite/command/verify.go @@ -123,7 +123,7 @@ var _ = Describe("notation verify", func() { }) It("with TLS by digest", func() { - HostWithTLS(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { notation.Exec("sign", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) @@ -133,7 +133,7 @@ var _ = Describe("notation verify", func() { }) It("with --insecure-registry by digest", func() { - HostWithTLS(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { + HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { notation.Exec("sign", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) From 465e34706a52bcc6fed9ca4812e180a9acae0615 Mon Sep 17 00:00:00 2001 From: Junjie Gao Date: Wed, 24 May 2023 09:50:50 +0800 Subject: [PATCH 07/12] fix: solve comments Signed-off-by: Junjie Gao --- test/e2e/internal/notation/registry.go | 7 +++++-- test/e2e/suite/command/inspect.go | 6 +++--- test/e2e/suite/command/list.go | 6 +++--- test/e2e/suite/command/sign.go | 4 ++-- test/e2e/suite/command/verify.go | 4 ++-- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/test/e2e/internal/notation/registry.go b/test/e2e/internal/notation/registry.go index 58e8c4a66..449128c21 100644 --- a/test/e2e/internal/notation/registry.go +++ b/test/e2e/internal/notation/registry.go @@ -25,9 +25,12 @@ type Registry struct { Username string // Password is the password to access the registry. Password string - // DomainHost is an registry host for testing --plain-http flag. + // DomainHost is a registry host, separate from localhost, used for testing + // the --insecure-registry flag. + // // if the host is localhost, notation make all connection as plain http. - // if the host is not localhost, notation make all connection as https. + // if the host is not localhost, notation make all connection as https + // by default. DomainHost string } diff --git a/test/e2e/suite/command/inspect.go b/test/e2e/suite/command/inspect.go index d658c89ec..4fb340e74 100644 --- a/test/e2e/suite/command/inspect.go +++ b/test/e2e/suite/command/inspect.go @@ -36,7 +36,7 @@ var _ = Describe("notation inspect", func() { notation.Exec("sign", artifact.ReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("inspect", artifact.ReferenceWithDigest()). + notation.Exec("inspect", "-d", artifact.ReferenceWithDigest()). MatchKeyWords(inspectSuccessfully...) }) }) @@ -46,7 +46,7 @@ var _ = Describe("notation inspect", func() { notation.Exec("sign", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("inspect", artifact.DomainReferenceWithDigest()). + notation.Exec("inspect", "-d", artifact.DomainReferenceWithDigest()). MatchKeyWords(inspectSuccessfully...) }) }) @@ -56,7 +56,7 @@ var _ = Describe("notation inspect", func() { notation.Exec("sign", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("inspect", "--insecure-registry", artifact.DomainReferenceWithDigest()). + notation.Exec("inspect", "-d", "--insecure-registry", artifact.DomainReferenceWithDigest()). MatchKeyWords(inspectSuccessfully...) }) }) diff --git a/test/e2e/suite/command/list.go b/test/e2e/suite/command/list.go index f5d846f58..d9aed1f1a 100644 --- a/test/e2e/suite/command/list.go +++ b/test/e2e/suite/command/list.go @@ -13,7 +13,7 @@ var _ = Describe("notation list", func() { notation.Exec("sign", artifact.ReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("list", artifact.ReferenceWithDigest()). + notation.Exec("list", "-d", artifact.ReferenceWithDigest()). MatchKeyWords( "└── application/vnd.cncf.notary.signature", "└── sha256:", @@ -26,7 +26,7 @@ var _ = Describe("notation list", func() { notation.Exec("sign", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("list", artifact.DomainReferenceWithDigest()). + notation.Exec("list", "-d", artifact.DomainReferenceWithDigest()). MatchKeyWords( "└── application/vnd.cncf.notary.signature", "└── sha256:", @@ -39,7 +39,7 @@ var _ = Describe("notation list", func() { notation.Exec("sign", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("list", "--insecure-registry", artifact.DomainReferenceWithDigest()). + notation.Exec("list", "-d", "--insecure-registry", artifact.DomainReferenceWithDigest()). MatchKeyWords( "└── application/vnd.cncf.notary.signature", "└── sha256:", diff --git a/test/e2e/suite/command/sign.go b/test/e2e/suite/command/sign.go index 76a6910cc..2895e951a 100644 --- a/test/e2e/suite/command/sign.go +++ b/test/e2e/suite/command/sign.go @@ -153,7 +153,7 @@ var _ = Describe("notation sign", func() { It("with TLS by digest", func() { HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { - notation.Exec("sign", artifact.DomainReferenceWithDigest()). + notation.Exec("sign", "-d", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) OldNotation().Exec("verify", artifact.DomainReferenceWithDigest()). @@ -163,7 +163,7 @@ var _ = Describe("notation sign", func() { It("with --insecure-registry by digest", func() { HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { - notation.Exec("sign", "--insecure-registry", artifact.DomainReferenceWithDigest()). + notation.Exec("sign", "-d", "--insecure-registry", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) OldNotation().Exec("verify", artifact.DomainReferenceWithDigest()). diff --git a/test/e2e/suite/command/verify.go b/test/e2e/suite/command/verify.go index 6450c8e5c..a4bde310c 100644 --- a/test/e2e/suite/command/verify.go +++ b/test/e2e/suite/command/verify.go @@ -127,7 +127,7 @@ var _ = Describe("notation verify", func() { notation.Exec("sign", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("verify", artifact.DomainReferenceWithDigest(), "-v"). + notation.Exec("verify", "-d", artifact.DomainReferenceWithDigest(), "-v"). MatchKeyWords(VerifySuccessfully) }) }) @@ -137,7 +137,7 @@ var _ = Describe("notation verify", func() { notation.Exec("sign", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("verify", "--insecure-registry", artifact.DomainReferenceWithDigest(), "-v"). + notation.Exec("verify", "-d", "--insecure-registry", artifact.DomainReferenceWithDigest(), "-v"). MatchKeyWords(VerifySuccessfully) }) }) From ee0861e23f5656dd5bcb341e64a329cad7d607b9 Mon Sep 17 00:00:00 2001 From: Junjie Gao Date: Wed, 24 May 2023 09:54:23 +0800 Subject: [PATCH 08/12] fix: update Signed-off-by: Junjie Gao --- test/e2e/internal/notation/registry.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/internal/notation/registry.go b/test/e2e/internal/notation/registry.go index 449128c21..be9eaca4c 100644 --- a/test/e2e/internal/notation/registry.go +++ b/test/e2e/internal/notation/registry.go @@ -28,9 +28,9 @@ type Registry struct { // DomainHost is a registry host, separate from localhost, used for testing // the --insecure-registry flag. // - // if the host is localhost, notation make all connection as plain http. - // if the host is not localhost, notation make all connection as https - // by default. + // If the host is localhost, Notation connects via plain HTTP. For + // non-localhost hosts, Notation defaults to HTTPS. However, users can + // enforce HTTP by setting the --insecure-registry flag. DomainHost string } From fcbf21ec80eeea30e154fe98fa6d1c5eec3bb5ba Mon Sep 17 00:00:00 2001 From: Junjie Gao Date: Thu, 25 May 2023 09:34:17 +0800 Subject: [PATCH 09/12] fix: add HTTP & HTTPS assertion Signed-off-by: Junjie Gao --- test/e2e/internal/utils/matcher.go | 7 +++++++ test/e2e/suite/command/inspect.go | 8 ++++++-- test/e2e/suite/command/list.go | 8 ++++++-- test/e2e/suite/command/sign.go | 8 ++++++-- test/e2e/suite/command/verify.go | 12 ++++++++++-- test/e2e/suite/common/common.go | 15 +++++++++++++++ 6 files changed, 50 insertions(+), 8 deletions(-) diff --git a/test/e2e/internal/utils/matcher.go b/test/e2e/internal/utils/matcher.go index 103d8d9dc..bb2cd0ae5 100644 --- a/test/e2e/internal/utils/matcher.go +++ b/test/e2e/internal/utils/matcher.go @@ -48,6 +48,13 @@ func (m *Matcher) MatchErrKeyWords(keywords ...string) *Matcher { return m } +func (m *Matcher) NotMatchErrKeyWords(keywords ...string) *Matcher { + for _, w := range keywords { + Expect(m.stdout).ShouldNot(ContainSubstring(w)) + } + return m +} + // MatchErrKeyWords matches given keywords with the stderr. func matchKeyWords(content string, keywords []string) { var missed []string diff --git a/test/e2e/suite/command/inspect.go b/test/e2e/suite/command/inspect.go index 4fb340e74..00fa62912 100644 --- a/test/e2e/suite/command/inspect.go +++ b/test/e2e/suite/command/inspect.go @@ -47,7 +47,9 @@ var _ = Describe("notation inspect", func() { MatchKeyWords(SignSuccessfully) notation.Exec("inspect", "-d", artifact.DomainReferenceWithDigest()). - MatchKeyWords(inspectSuccessfully...) + MatchKeyWords(inspectSuccessfully...). + MatchErrKeyWords("https://notation-e2e.registry.io/v2/e2e"). + NotMatchErrKeyWords("http://notation-e2e.registry.io") }) }) @@ -57,7 +59,9 @@ var _ = Describe("notation inspect", func() { MatchKeyWords(SignSuccessfully) notation.Exec("inspect", "-d", "--insecure-registry", artifact.DomainReferenceWithDigest()). - MatchKeyWords(inspectSuccessfully...) + MatchKeyWords(inspectSuccessfully...). + MatchErrKeyWords(HTTPRequest). + NotMatchErrKeyWords(HTTPSRequest) }) }) }) diff --git a/test/e2e/suite/command/list.go b/test/e2e/suite/command/list.go index d9aed1f1a..3ccb2e100 100644 --- a/test/e2e/suite/command/list.go +++ b/test/e2e/suite/command/list.go @@ -30,7 +30,9 @@ var _ = Describe("notation list", func() { MatchKeyWords( "└── application/vnd.cncf.notary.signature", "└── sha256:", - ) + ). + MatchErrKeyWords("https://notation-e2e.registry.io/v2/e2e"). + NotMatchErrKeyWords("http://notation-e2e.registry.io") }) }) @@ -43,7 +45,9 @@ var _ = Describe("notation list", func() { MatchKeyWords( "└── application/vnd.cncf.notary.signature", "└── sha256:", - ) + ). + MatchErrKeyWords(HTTPRequest). + NotMatchErrKeyWords(HTTPSRequest) }) }) }) diff --git a/test/e2e/suite/command/sign.go b/test/e2e/suite/command/sign.go index 2895e951a..f7c57f194 100644 --- a/test/e2e/suite/command/sign.go +++ b/test/e2e/suite/command/sign.go @@ -154,7 +154,9 @@ var _ = Describe("notation sign", func() { It("with TLS by digest", func() { HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { notation.Exec("sign", "-d", artifact.DomainReferenceWithDigest()). - MatchKeyWords(SignSuccessfully) + MatchKeyWords(SignSuccessfully). + MatchErrKeyWords(HTTPSRequest). + NotMatchErrKeyWords(HTTPRequest) OldNotation().Exec("verify", artifact.DomainReferenceWithDigest()). MatchKeyWords(VerifySuccessfully) @@ -164,7 +166,9 @@ var _ = Describe("notation sign", func() { It("with --insecure-registry by digest", func() { HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) { notation.Exec("sign", "-d", "--insecure-registry", artifact.DomainReferenceWithDigest()). - MatchKeyWords(SignSuccessfully) + MatchKeyWords(SignSuccessfully). + MatchErrKeyWords(HTTPRequest). + NotMatchErrKeyWords(HTTPSRequest) OldNotation().Exec("verify", artifact.DomainReferenceWithDigest()). MatchKeyWords(VerifySuccessfully) diff --git a/test/e2e/suite/command/verify.go b/test/e2e/suite/command/verify.go index a4bde310c..7f42ad1cb 100644 --- a/test/e2e/suite/command/verify.go +++ b/test/e2e/suite/command/verify.go @@ -128,7 +128,11 @@ var _ = Describe("notation verify", func() { MatchKeyWords(SignSuccessfully) notation.Exec("verify", "-d", artifact.DomainReferenceWithDigest(), "-v"). - MatchKeyWords(VerifySuccessfully) + MatchKeyWords( + VerifySuccessfully, + ). + MatchErrKeyWords("https://notation-e2e.registry.io/v2/e2e"). + NotMatchErrKeyWords("http://notation-e2e.registry.io") }) }) @@ -138,7 +142,11 @@ var _ = Describe("notation verify", func() { MatchKeyWords(SignSuccessfully) notation.Exec("verify", "-d", "--insecure-registry", artifact.DomainReferenceWithDigest(), "-v"). - MatchKeyWords(VerifySuccessfully) + MatchKeyWords( + VerifySuccessfully, + ). + MatchErrKeyWords(HTTPRequest). + NotMatchErrKeyWords(HTTPSRequest) }) }) }) diff --git a/test/e2e/suite/common/common.go b/test/e2e/suite/common/common.go index 5d3109a86..c14d426a9 100644 --- a/test/e2e/suite/common/common.go +++ b/test/e2e/suite/common/common.go @@ -1,5 +1,11 @@ package common +import ( + "fmt" + + . "github.com/notaryproject/notation/test/e2e/internal/notation" +) + const ( LoginSuccessfully = "Login Succeeded" LogoutSuccessfully = "Logout Succeeded" @@ -8,6 +14,15 @@ const ( VerifyFailed = "signature verification failed" ) +var ( + // HTTPRequest is the base URL for HTTP requests for testing + // --insecure-registry flag + HTTPRequest = fmt.Sprintf("http://%s", TestRegistry.DomainHost) + + // HTTPSRequest is the base URL for HTTPS requests for testing TLS request. + HTTPSRequest = fmt.Sprintf("https://%s", TestRegistry.DomainHost) +) + const ( // HeaderVerificationPlugin specifies the name of the verification plugin that should be used to verify the signature. HeaderVerificationPlugin = "io.cncf.notary.verificationPlugin" From 4a7872d7740aaed64bc9c10d61125e16e027b743 Mon Sep 17 00:00:00 2001 From: Junjie Gao Date: Thu, 25 May 2023 09:41:02 +0800 Subject: [PATCH 10/12] fix: add comments for NotMatchErrKeyWords Signed-off-by: Junjie Gao --- test/e2e/internal/utils/matcher.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/e2e/internal/utils/matcher.go b/test/e2e/internal/utils/matcher.go index bb2cd0ae5..ed896584b 100644 --- a/test/e2e/internal/utils/matcher.go +++ b/test/e2e/internal/utils/matcher.go @@ -48,6 +48,8 @@ func (m *Matcher) MatchErrKeyWords(keywords ...string) *Matcher { return m } +// NotMatchKeyWords guarantees that the given keywords do not match with +// the stderr. func (m *Matcher) NotMatchErrKeyWords(keywords ...string) *Matcher { for _, w := range keywords { Expect(m.stdout).ShouldNot(ContainSubstring(w)) From 286556043c8b4f5cca502836c9f95a768c185b57 Mon Sep 17 00:00:00 2001 From: Junjie Gao Date: Thu, 25 May 2023 09:54:35 +0800 Subject: [PATCH 11/12] fix: update code Signed-off-by: Junjie Gao --- test/e2e/internal/utils/matcher.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/e2e/internal/utils/matcher.go b/test/e2e/internal/utils/matcher.go index ed896584b..83aafb9ff 100644 --- a/test/e2e/internal/utils/matcher.go +++ b/test/e2e/internal/utils/matcher.go @@ -5,9 +5,15 @@ import ( "strings" . "github.com/onsi/gomega" + "github.com/onsi/gomega/format" "github.com/onsi/gomega/gexec" ) +func init() { + // expand the length limit for the gomega matcher + format.MaxLength = 1000000 +} + // Matcher contains the execution result for matching. type Matcher struct { Session *gexec.Session @@ -52,7 +58,7 @@ func (m *Matcher) MatchErrKeyWords(keywords ...string) *Matcher { // the stderr. func (m *Matcher) NotMatchErrKeyWords(keywords ...string) *Matcher { for _, w := range keywords { - Expect(m.stdout).ShouldNot(ContainSubstring(w)) + Expect(m.stderr).ShouldNot(ContainSubstring(w)) } return m } From 88c923185e53d2020afd20b6f800299f00414a67 Mon Sep 17 00:00:00 2001 From: Junjie Gao Date: Thu, 25 May 2023 12:43:06 +0800 Subject: [PATCH 12/12] fix: update code Signed-off-by: Junjie Gao --- test/e2e/internal/notation/registry.go | 2 ++ test/e2e/internal/utils/matcher.go | 4 ++-- test/e2e/suite/command/inspect.go | 4 ++-- test/e2e/suite/command/list.go | 4 ++-- test/e2e/suite/command/sign.go | 4 ++-- test/e2e/suite/command/verify.go | 8 ++++---- 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/test/e2e/internal/notation/registry.go b/test/e2e/internal/notation/registry.go index be9eaca4c..8b8c363d2 100644 --- a/test/e2e/internal/notation/registry.go +++ b/test/e2e/internal/notation/registry.go @@ -111,6 +111,8 @@ func (r *Artifact) ReferenceWithDigest() string { return fmt.Sprintf("%s/%s@%s", r.Host, r.Repo, r.Digest) } +// DomainReferenceWithDigest returns the /@: +// for testing --insecure-registry flag and TLS request. func (r *Artifact) DomainReferenceWithDigest() string { return fmt.Sprintf("%s/%s@%s", r.DomainHost, r.Repo, r.Digest) } diff --git a/test/e2e/internal/utils/matcher.go b/test/e2e/internal/utils/matcher.go index 83aafb9ff..a7a9e3855 100644 --- a/test/e2e/internal/utils/matcher.go +++ b/test/e2e/internal/utils/matcher.go @@ -54,9 +54,9 @@ func (m *Matcher) MatchErrKeyWords(keywords ...string) *Matcher { return m } -// NotMatchKeyWords guarantees that the given keywords do not match with +// NoMatchErrKeyWords guarantees that the given keywords do not match with // the stderr. -func (m *Matcher) NotMatchErrKeyWords(keywords ...string) *Matcher { +func (m *Matcher) NoMatchErrKeyWords(keywords ...string) *Matcher { for _, w := range keywords { Expect(m.stderr).ShouldNot(ContainSubstring(w)) } diff --git a/test/e2e/suite/command/inspect.go b/test/e2e/suite/command/inspect.go index 00fa62912..7cf6d722f 100644 --- a/test/e2e/suite/command/inspect.go +++ b/test/e2e/suite/command/inspect.go @@ -49,7 +49,7 @@ var _ = Describe("notation inspect", func() { notation.Exec("inspect", "-d", artifact.DomainReferenceWithDigest()). MatchKeyWords(inspectSuccessfully...). MatchErrKeyWords("https://notation-e2e.registry.io/v2/e2e"). - NotMatchErrKeyWords("http://notation-e2e.registry.io") + NoMatchErrKeyWords("http://notation-e2e.registry.io") }) }) @@ -61,7 +61,7 @@ var _ = Describe("notation inspect", func() { notation.Exec("inspect", "-d", "--insecure-registry", artifact.DomainReferenceWithDigest()). MatchKeyWords(inspectSuccessfully...). MatchErrKeyWords(HTTPRequest). - NotMatchErrKeyWords(HTTPSRequest) + NoMatchErrKeyWords(HTTPSRequest) }) }) }) diff --git a/test/e2e/suite/command/list.go b/test/e2e/suite/command/list.go index 3ccb2e100..c04d039ef 100644 --- a/test/e2e/suite/command/list.go +++ b/test/e2e/suite/command/list.go @@ -32,7 +32,7 @@ var _ = Describe("notation list", func() { "└── sha256:", ). MatchErrKeyWords("https://notation-e2e.registry.io/v2/e2e"). - NotMatchErrKeyWords("http://notation-e2e.registry.io") + NoMatchErrKeyWords("http://notation-e2e.registry.io") }) }) @@ -47,7 +47,7 @@ var _ = Describe("notation list", func() { "└── sha256:", ). MatchErrKeyWords(HTTPRequest). - NotMatchErrKeyWords(HTTPSRequest) + NoMatchErrKeyWords(HTTPSRequest) }) }) }) diff --git a/test/e2e/suite/command/sign.go b/test/e2e/suite/command/sign.go index f7c57f194..8181fc9f8 100644 --- a/test/e2e/suite/command/sign.go +++ b/test/e2e/suite/command/sign.go @@ -156,7 +156,7 @@ var _ = Describe("notation sign", func() { notation.Exec("sign", "-d", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully). MatchErrKeyWords(HTTPSRequest). - NotMatchErrKeyWords(HTTPRequest) + NoMatchErrKeyWords(HTTPRequest) OldNotation().Exec("verify", artifact.DomainReferenceWithDigest()). MatchKeyWords(VerifySuccessfully) @@ -168,7 +168,7 @@ var _ = Describe("notation sign", func() { notation.Exec("sign", "-d", "--insecure-registry", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully). MatchErrKeyWords(HTTPRequest). - NotMatchErrKeyWords(HTTPSRequest) + NoMatchErrKeyWords(HTTPSRequest) OldNotation().Exec("verify", artifact.DomainReferenceWithDigest()). MatchKeyWords(VerifySuccessfully) diff --git a/test/e2e/suite/command/verify.go b/test/e2e/suite/command/verify.go index 7f42ad1cb..0056ed651 100644 --- a/test/e2e/suite/command/verify.go +++ b/test/e2e/suite/command/verify.go @@ -127,12 +127,12 @@ var _ = Describe("notation verify", func() { notation.Exec("sign", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("verify", "-d", artifact.DomainReferenceWithDigest(), "-v"). + notation.Exec("verify", "-d", artifact.DomainReferenceWithDigest()). MatchKeyWords( VerifySuccessfully, ). MatchErrKeyWords("https://notation-e2e.registry.io/v2/e2e"). - NotMatchErrKeyWords("http://notation-e2e.registry.io") + NoMatchErrKeyWords("http://notation-e2e.registry.io") }) }) @@ -141,12 +141,12 @@ var _ = Describe("notation verify", func() { notation.Exec("sign", artifact.DomainReferenceWithDigest()). MatchKeyWords(SignSuccessfully) - notation.Exec("verify", "-d", "--insecure-registry", artifact.DomainReferenceWithDigest(), "-v"). + notation.Exec("verify", "-d", "--insecure-registry", artifact.DomainReferenceWithDigest()). MatchKeyWords( VerifySuccessfully, ). MatchErrKeyWords(HTTPRequest). - NotMatchErrKeyWords(HTTPSRequest) + NoMatchErrKeyWords(HTTPSRequest) }) }) })