test: add e2e test cases for flag --insecure-registry

test/e2e/internal/notation/host.go

@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 
 	"github.com/notaryproject/notation/test/e2e/internal/utils"
+	. "github.com/onsi/ginkgo/v2"
 )
 
 // CoreTestFunc is the test function running in a VirtualHost.
@@ -40,6 +41,17 @@ func Host(options []utils.HostOption, fn CoreTestFunc) {
 	fn(vhost.Executor, artifact, vhost)
 }
 
+// HostInGithubAction only run the test in GitHub Actions.
+//
+// The booting script will setup TLS reverse proxy and TLS certificate
+// for Github Actions environment.
+func HostInGithubAction(options []utils.HostOption, fn CoreTestFunc) {
+	if os.Getenv("GITHUB_ACTIONS") != "true" {
+		Skip("only run in GitHub Actions")
+	}
+	Host(options, fn)
+}
+
 // GeneralHost creates a virtualized notation testing host by modify
 // the "XDG_CONFIG_HOME" environment variable of the Executor. It's agnostic to
 // the Repository of the artifact.

test/e2e/internal/notation/init.go

@@ -23,6 +23,7 @@ const (
 	envKeyRegistryHost       = "NOTATION_E2E_REGISTRY_HOST"
 	envKeyRegistryUsername   = "NOTATION_E2E_REGISTRY_USERNAME"
 	envKeyRegistryPassword   = "NOTATION_E2E_REGISTRY_PASSWORD"
+	envKeyDomainRegistryHost = "NOTATION_E2E_DOMAIN_REGISTRY_HOST"
 	envKeyNotationBinPath    = "NOTATION_E2E_BINARY_PATH"
 	envKeyNotationOldBinPath = "NOTATION_E2E_OLD_BINARY_PATH"
 	envKeyNotationPluginPath = "NOTATION_E2E_PLUGIN_PATH"
@@ -65,6 +66,7 @@ func setUpRegistry() {
 	setValue(envKeyRegistryHost, &TestRegistry.Host)
 	setValue(envKeyRegistryUsername, &TestRegistry.Username)
 	setValue(envKeyRegistryPassword, &TestRegistry.Password)
+	setValue(envKeyDomainRegistryHost, &TestRegistry.DomainHost)
 
 	setPathValue(envKeyOCILayoutPath, &OCILayoutPath)
 	setValue(envKeyTestRepo, &TestRepoUri)

test/e2e/internal/notation/registry.go

@@ -19,9 +19,19 @@ import (
 const ArtifactTypeNotation = "application/vnd.cncf.notary.signature"
 
 type Registry struct {
-	Host     string
+	// Host is the registry host.
+	Host string
+	// Username is the username to access the registry.
 	Username string
+	// Password is the password to access the registry.
 	Password string
+	// DomainHost is a registry host, separate from localhost, used for testing
+	// the --insecure-registry flag.
+	//
+	// If the host is localhost, Notation connects via plain HTTP. For
+	// non-localhost hosts, Notation defaults to HTTPS. However, users can
+	// enforce HTTP by setting the --insecure-registry flag.
+	DomainHost string
 }
 
 // CreateArtifact copies a local OCI layout to the registry to create
@@ -101,6 +111,12 @@ func (r *Artifact) ReferenceWithDigest() string {
 	return fmt.Sprintf("%s/%s@%s", r.Host, r.Repo, r.Digest)
 }
 
+// DomainReferenceWithDigest returns the <domainHost>/<Repository>@<alg>:<digest>
+// for testing --insecure-registry flag and TLS request.
+func (r *Artifact) DomainReferenceWithDigest() string {
+	return fmt.Sprintf("%s/%s@%s", r.DomainHost, r.Repo, r.Digest)
+}
+
 // SignatureManifest returns the manifest of the artifact.
 func (r *Artifact) SignatureDescriptors() ([]ocispec.Descriptor, error) {
 	ctx := context.Background()

test/e2e/internal/utils/matcher.go

@@ -5,9 +5,15 @@ import (
 	"strings"
 
 	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/format"
 	"github.com/onsi/gomega/gexec"
 )
 
+func init() {
+	// expand the length limit for the gomega matcher
+	format.MaxLength = 1000000
+}
+
 // Matcher contains the execution result for matching.
 type Matcher struct {
 	Session *gexec.Session
@@ -48,6 +54,15 @@ func (m *Matcher) MatchErrKeyWords(keywords ...string) *Matcher {
 	return m
 }
 
+// NoMatchErrKeyWords guarantees that the given keywords do not match with
+// the stderr.
+func (m *Matcher) NoMatchErrKeyWords(keywords ...string) *Matcher {
+	for _, w := range keywords {
+		Expect(m.stderr).ShouldNot(ContainSubstring(w))
+	}
+	return m
+}
+
 // MatchErrKeyWords matches given keywords with the stderr.
 func matchKeyWords(content string, keywords []string) {
 	var missed []string

test/e2e/run.sh

@@ -40,7 +40,7 @@ if [ ! -f "$NOTATION_E2E_OLD_BINARY_PATH" ]; then
     echo "Try to use old notation binary at $NOTATION_E2E_OLD_BINARY_PATH"
 
     if [ ! -f $NOTATION_E2E_OLD_BINARY_PATH ]; then
-        TAG=1.0.0-rc.2 # without 'v'
+        TAG=1.0.0-rc.5 # without 'v'
         echo "Didn't find old notation binary locally. Try to download notation v$TAG."
 
         TAR_NAME=notation_${TAG}_linux_amd64.tar.gz

test/e2e/scripts/dockerhub.sh

@@ -16,9 +16,10 @@ if [ -z "$DOCKER_USERNAME" ] || [ -z "$DOCKER_PASSWORD" ]; then
 fi
 
 # set environment variables for E2E testing
-export NOTATION_E2E_REGISTRY_HOST=docker.io/$DOCKER_USERNAME
-export NOTATION_E2E_REGISTRY_USERNAME=$DOCKER_USERNAME
-export NOTATION_E2E_REGISTRY_PASSWORD=$DOCKER_PASSWORD
+export NOTATION_E2E_REGISTRY_HOST="docker.io/$DOCKER_USERNAME"
+export NOTATION_E2E_REGISTRY_USERNAME="$DOCKER_USERNAME"
+export NOTATION_E2E_REGISTRY_PASSWORD="$DOCKER_PASSWORD"
+export NOTATION_E2E_DOMAIN_REGISTRY_HOST="$NOTATION_E2E_REGISTRY_HOST"
 
 function setup_registry {
     echo "use $NOTATION_E2E_REGISTRY_HOST"
@@ -55,4 +56,4 @@ function cleanup_registry {
                 echo "$NOTATION_E2E_REGISTRY_HOST/$repoName deleted."
         done
     done
-}
+}

test/e2e/scripts/tls.sh

@@ -0,0 +1,51 @@
+#!/bin/bash -e
+#
+# Usage
+#   For setup:
+#   1. source ./scripts/tls.sh
+#   2. call create_docker_network
+#   3. setup registry with port 5000 in $DOCKER_NETWORK
+#   4. call setup_tls reverse proxy
+#
+#   For clean up:
+#   1. call clean_up
+#   2. clean up registry
+#   3. call remove_docker_network
+# 
+# Note: this script needs sudo permission to add TLS certificate to system and 
+#       add domain registry host.
+
+NGINX_CONTAINER_NAME=nginx
+DOMAIN=notation-e2e.registry.io
+DOCKER_NETWORK=notation-e2e
+
+function create_docker_network {
+    docker network create "$DOCKER_NETWORK"
+}
+
+function remove_docker_network {
+    docker network rm "$DOCKER_NETWORK"
+}
+
+function setup_tls {
+    # add domain registry host to /etc/hosts for testing --plain-http feature
+    echo "127.0.0.1 $DOMAIN" | sudo tee -a /etc/hosts
+    # add TLS certificate to system
+    sudo mkdir -p /usr/local/share/ca-certificates/
+    sudo cp ./testdata/nginx/notation-e2e.registry.io.crt /usr/local/share/ca-certificates/
+    sudo update-ca-certificates
+
+    # start Nginx for TLS 
+    docker run -d -p 80:80 -p 443:443 \
+        --network "$DOCKER_NETWORK" \
+        --mount type=bind,source="$(pwd)/testdata/nginx/",target=/etc/nginx \
+        --name "$NGINX_CONTAINER_NAME" \
+        --rm nginx:latest
+}
+
+function clean_up_tls {
+    docker container stop "$NGINX_CONTAINER_NAME" 1>/dev/null && echo "Nginx stopped"
+    sudo sed -i "/${NOTATION_E2E_DOMAIN_REGISTRY_HOST}/d" /etc/hosts
+    sudo rm /usr/local/share/ca-certificates/notation-e2e.registry.io.crt
+    sudo update-ca-certificates
+}

test/e2e/scripts/zot.sh

@@ -4,24 +4,38 @@
 # Usage
 #   ./run.sh zot <notation-binary-path> [old-notation-binary-path]
 
+source ./scripts/tls.sh
+
 REG_HOST=localhost
 REG_PORT=5000
-ZOT_CONTAINER_NAME=zot
+ZOT_CONTAINER_NAME=notation-e2e-registry
 
-# set environment variables for E2E testing
-export NOTATION_E2E_REGISTRY_HOST=$REG_HOST:$REG_PORT
+# set required environment variables for E2E testing
+export NOTATION_E2E_REGISTRY_HOST="$REG_HOST:$REG_PORT"
 export NOTATION_E2E_REGISTRY_USERNAME=testuser
 export NOTATION_E2E_REGISTRY_PASSWORD=testpassword
+export NOTATION_E2E_DOMAIN_REGISTRY_HOST="$DOMAIN"
 
 function setup_registry {
+    create_docker_network
     # start Zot
-    docker run -d -p $REG_PORT:$REG_PORT -it --name $ZOT_CONTAINER_NAME \
-        --mount type=bind,source=`pwd`/testdata/registry/zot/,target=/etc/zot \
+    docker run -d -p $REG_PORT:$REG_PORT -it \
+        --name $ZOT_CONTAINER_NAME \
+        --network $DOCKER_NETWORK \
+        --mount type=bind,source="$(pwd)/testdata/registry/zot/",target=/etc/zot \
         --rm ghcr.io/project-zot/zot-minimal-linux-amd64:latest
+
+    if [ "$GITHUB_ACTIONS" == "true" ]; then
+        setup_tls
+    fi
     # make sure that Zot is ready
     sleep 1
 }
 
 function cleanup_registry {
     docker container stop $ZOT_CONTAINER_NAME 1>/dev/null && echo "Zot stopped"
-}
+    if [ "$GITHUB_ACTIONS" == "true" ]; then
+        clean_up_tls
+    fi
+    remove_docker_network
+}

test/e2e/suite/command/inspect.go

@@ -0,0 +1,67 @@
+package command
+
+import (
+	. "github.com/notaryproject/notation/test/e2e/internal/notation"
+	"github.com/notaryproject/notation/test/e2e/internal/utils"
+	. "github.com/notaryproject/notation/test/e2e/suite/common"
+	. "github.com/onsi/ginkgo/v2"
+)
+
+var inspectSuccessfully = []string{
+	"└── application/vnd.cncf.notary.signature",
+	"└── sha256:",
+	"├── media type:",
+	"├── signature algorithm:",
+	"├── signed attributes",
+	"signingTime:",
+	"signingScheme:",
+	"├── user defined attributes",
+	"│   └── (empty)",
+	"├── unsigned attributes",
+	"│   └── signingAgent: Notation/",
+	"├── certificates",
+	"│   └── SHA256 fingerprint:",
+	"issued to:",
+	"issued by:",
+	"expiry:",
+	"└── signed artifact",
+	"media type:",
+	"digest:",
+	"size:",
+}
+
+var _ = Describe("notation inspect", func() {
+	It("all signatures of an image", func() {
+		Host(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) {
+			notation.Exec("sign", artifact.ReferenceWithDigest()).
+				MatchKeyWords(SignSuccessfully)
+
+			notation.Exec("inspect", "-d", artifact.ReferenceWithDigest()).
+				MatchKeyWords(inspectSuccessfully...)
+		})
+	})
+
+	It("all signatures of an image with TLS", func() {
+		HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) {
+			notation.Exec("sign", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(SignSuccessfully)
+
+			notation.Exec("inspect", "-d", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(inspectSuccessfully...).
+				MatchErrKeyWords("https://notation-e2e.registry.io/v2/e2e").
+				NoMatchErrKeyWords("http://notation-e2e.registry.io")
+		})
+	})
+
+	It("all signatures of an image with --insecure-registry flag", func() {
+		HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) {
+			notation.Exec("sign", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(SignSuccessfully)
+
+			notation.Exec("inspect", "-d", "--insecure-registry", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(inspectSuccessfully...).
+				MatchErrKeyWords(HTTPRequest).
+				NoMatchErrKeyWords(HTTPSRequest)
+		})
+	})
+})

test/e2e/suite/command/list.go

@@ -0,0 +1,53 @@
+package command
+
+import (
+	. "github.com/notaryproject/notation/test/e2e/internal/notation"
+	"github.com/notaryproject/notation/test/e2e/internal/utils"
+	. "github.com/notaryproject/notation/test/e2e/suite/common"
+	. "github.com/onsi/ginkgo/v2"
+)
+
+var _ = Describe("notation list", func() {
+	It("all signatures of an image", func() {
+		Host(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) {
+			notation.Exec("sign", artifact.ReferenceWithDigest()).
+				MatchKeyWords(SignSuccessfully)
+
+			notation.Exec("list", "-d", artifact.ReferenceWithDigest()).
+				MatchKeyWords(
+					"└── application/vnd.cncf.notary.signature",
+					"└── sha256:",
+				)
+		})
+	})
+
+	It("all signatures of an image with TLS", func() {
+		HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) {
+			notation.Exec("sign", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(SignSuccessfully)
+
+			notation.Exec("list", "-d", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(
+					"└── application/vnd.cncf.notary.signature",
+					"└── sha256:",
+				).
+				MatchErrKeyWords("https://notation-e2e.registry.io/v2/e2e").
+				NoMatchErrKeyWords("http://notation-e2e.registry.io")
+		})
+	})
+
+	It("all signatures of an image with --insecure-registry flag", func() {
+		HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) {
+			notation.Exec("sign", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(SignSuccessfully)
+
+			notation.Exec("list", "-d", "--insecure-registry", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(
+					"└── application/vnd.cncf.notary.signature",
+					"└── sha256:",
+				).
+				MatchErrKeyWords(HTTPRequest).
+				NoMatchErrKeyWords(HTTPSRequest)
+		})
+	})
+})

test/e2e/suite/command/sign.go

@@ -150,4 +150,28 @@ var _ = Describe("notation sign", func() {
 				MatchErrContent(expectedErrMsg)
 		})
 	})
+
+	It("with TLS by digest", func() {
+		HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) {
+			notation.Exec("sign", "-d", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(SignSuccessfully).
+				MatchErrKeyWords(HTTPSRequest).
+				NoMatchErrKeyWords(HTTPRequest)
+
+			OldNotation().Exec("verify", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(VerifySuccessfully)
+		})
+	})
+
+	It("with --insecure-registry by digest", func() {
+		HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) {
+			notation.Exec("sign", "-d", "--insecure-registry", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(SignSuccessfully).
+				MatchErrKeyWords(HTTPRequest).
+				NoMatchErrKeyWords(HTTPSRequest)
+
+			OldNotation().Exec("verify", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(VerifySuccessfully)
+		})
+	})
 })