Skip to content

RUN-1933: test unprivileged builds #27198

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
nalind wants to merge 1 commit into from
Closed

RUN-1933: test unprivileged builds #27198

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 45 additions & 12 deletions test/extended/builds/contextdir.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,11 @@ var _ = g.Describe("[sig-builds][Feature:Builds][Slow] builds with a context dir
appFixture = exutil.FixturePath("testdata", "builds", "test-context-build.json")
oc = exutil.NewCLIWithPodSecurityLevel("contextdir", admissionapi.LevelBaseline)
s2iBuildConfigName = "s2icontext"
s2iBuildName = "s2icontext-1"
dcName = "frontend"
deploymentName = "frontend-1"
dcLabel = exutil.ParseLabelsOrDie(fmt.Sprintf("deployment=%s", deploymentName))
serviceName = "frontend"
dockerBuildConfigName = "dockercontext"
dockerBuildName = "dockercontext-1"
)

g.Context("", func() {
Expand All @@ -45,23 +43,34 @@ var _ = g.Describe("[sig-builds][Feature:Builds][Slow] builds with a context dir
})

g.Describe("s2i context directory build", func() {
g.It(fmt.Sprintf("should s2i build an application using a context directory [apigroup:build.openshift.io]"), func() {

testSourceBuild := func(logsMustMatchRegexp, patch string) {
exutil.WaitForOpenShiftNamespaceImageStreams(oc)
g.By(fmt.Sprintf("calling oc create -f %q", appFixture))
err := oc.Run("create").Args("-f", appFixture).Execute()
o.Expect(err).NotTo(o.HaveOccurred())

if patch != "" {
g.By("applying patch to build config for context directory build")
err := oc.Run("patch").Args("bc/"+s2iBuildConfigName, "-p", patch).Execute()
o.Expect(err).NotTo(o.HaveOccurred())
}

g.By("starting a build")
err = oc.Run("start-build").Args(s2iBuildConfigName).Execute()
br, err := exutil.StartBuildAndWait(oc, s2iBuildConfigName)
o.Expect(err).NotTo(o.HaveOccurred())

g.By("waiting for build to finish")
err = exutil.WaitForABuild(oc.BuildClient().BuildV1().Builds(oc.Namespace()), s2iBuildName, exutil.CheckBuildSuccess, exutil.CheckBuildFailed, nil)
err = exutil.WaitForABuild(oc.BuildClient().BuildV1().Builds(oc.Namespace()), br.BuildName, exutil.CheckBuildSuccess, exutil.CheckBuildFailed, nil)
if err != nil {
exutil.DumpBuildLogs("s2icontext", oc)
exutil.DumpBuildLogs(s2iBuildConfigName, oc)
}
o.Expect(err).NotTo(o.HaveOccurred())
if logsMustMatchRegexp != "" {
g.By(fmt.Sprintf("verify that the build log included a message that matched %q", logsMustMatchRegexp))
buildPodLogs, err := br.Logs()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(buildPodLogs).To(o.MatchRegexp(logsMustMatchRegexp))
}

// oc.KubeFramework().WaitForAnEndpoint currently will wait forever; for now, prefacing with our WaitForADeploymentToComplete,
// which does have a timeout, since in most cases a failure in the service coming up stems from a failed deployment
Expand Down Expand Up @@ -95,34 +104,58 @@ var _ = g.Describe("[sig-builds][Feature:Builds][Slow] builds with a context dir
out, err := oc.Run("exec").Args(pod.Name, "-c", pod.Spec.Containers[0].Name, "--", "ls", "/opt/app-root/src").Output()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(out).NotTo(o.ContainSubstring("2.3"))
}
g.It(fmt.Sprintf("should s2i build an application using a context directory [apigroup:build.openshift.io]"), func() {
testSourceBuild(buildInDefaultUserNSRegexp, buildInDefaultUserNSPatch("sourceStrategy", 5))
})
g.It(fmt.Sprintf("should unprivileged s2i build an application using a context directory [apigroup:build.openshift.io]"), func() {
testSourceBuild(buildInUserNSRegexp, buildInUserNSPatch("sourceStrategy", 5))
})
})

g.Describe("docker context directory build", func() {
g.It(fmt.Sprintf("should docker build an application using a context directory [apigroup:build.openshift.io]"), func() {
testDockerBuild := func(logsMustMatchRegexp, patch string) {
g.By("initializing local repo")
repo, err := exutil.NewGitRepo("contextdir")
o.Expect(err).NotTo(o.HaveOccurred())
defer repo.Remove()
err = repo.AddAndCommit("2.3/Dockerfile", fmt.Sprintf("FROM %s", image.ShellImage()))
err = repo.AddAndCommit("2.3/Dockerfile", fmt.Sprintf("FROM %s\nRUN pwd", image.ShellImage()))
o.Expect(err).NotTo(o.HaveOccurred())

exutil.WaitForOpenShiftNamespaceImageStreams(oc)
g.By(fmt.Sprintf("calling oc create -f %q", appFixture))
err = oc.Run("create").Args("-f", appFixture).Execute()
o.Expect(err).NotTo(o.HaveOccurred())

if patch != "" {
g.By("applying patch to build config for context directory build")
err := oc.Run("patch").Args("bc/"+dockerBuildConfigName, "-p", patch).Execute()
o.Expect(err).NotTo(o.HaveOccurred())
}

g.By("starting a build")
err = oc.Run("start-build").Args(dockerBuildConfigName, "--from-repo", repo.RepoPath).Execute()
br, err := exutil.StartBuildAndWait(oc, dockerBuildConfigName, "--from-repo", repo.RepoPath)
o.Expect(err).NotTo(o.HaveOccurred())

// build will fail if we don't use the right context dir because there won't be a dockerfile present.
g.By("waiting for build to finish")
err = exutil.WaitForABuild(oc.BuildClient().BuildV1().Builds(oc.Namespace()), dockerBuildName, exutil.CheckBuildSuccess, exutil.CheckBuildFailed, nil)
err = exutil.WaitForABuild(oc.BuildClient().BuildV1().Builds(oc.Namespace()), br.BuildName, exutil.CheckBuildSuccess, exutil.CheckBuildFailed, nil)
if err != nil {
exutil.DumpBuildLogs("dockercontext", oc)
exutil.DumpBuildLogs(dockerBuildConfigName, oc)
}
o.Expect(err).NotTo(o.HaveOccurred())
if logsMustMatchRegexp != "" {
g.By(fmt.Sprintf("verify that the build log included a message that matched %q", logsMustMatchRegexp))
buildPodLogs, err := br.Logs()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(buildPodLogs).To(o.MatchRegexp(logsMustMatchRegexp))
}
}
g.It(fmt.Sprintf("should docker build an application using a context directory [apigroup:build.openshift.io]"), func() {
testDockerBuild(buildInDefaultUserNSRegexp, buildInDefaultUserNSPatch("dockerStrategy", 5))
})
g.It(fmt.Sprintf("should unprivileged docker build an application using a context directory [apigroup:build.openshift.io]"), func() {
testDockerBuild(buildInUserNSRegexp, buildInUserNSPatch("dockerStrategy", 5))
})
})
})
Expand Down
29 changes: 29 additions & 0 deletions test/extended/builds/controllers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"os"
"strings"
"time"

corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -40,8 +41,36 @@ const (
// BuildCancelledEventMessage is the message associated with the event registered when build is cancelled.
BuildCancelledEventMessage = "Build %s/%s has been cancelled"
additionalAllowedRegistriesEnvVar = "ADDITIONAL_ALLOWED_REGISTRIES"

// buildInUserNSRegexp is a message that the builder will log at level
// 2 or higher if it detects that it's not running with a user
// namespace whose mappings don't match the node's at large.
buildInUserNSRegexp = `Started in kernel user namespace as [0-9]+:[0-9]+ with UID map \[\(0:[^:]*:65536\)] and GID map \[\(0:[^:]*:65536\)]\.`

// buildInDefaultUserNSRegexp is a message that the builder will log at
// level 2 or higher if it detects that it's running in a user
// namespace whose ID mappings map every possible UID or GID to itself,
// as things are by default on the node
buildInDefaultUserNSRegexp = `Started in node \(default\) kernel user namespace as [0-9]+:[0-9]+\.`

// buildInUserNSEnvVar is the environment variable and value that we
// use to tell the build controller to ask the node to run the build
// pod with fewer privileges.
buildInUserNSEnvVar = "BUILD_PRIVILEGED=false"
)

func buildInDefaultUserNSPatch(strategyType string, buildLogLevel int) string {
return fmt.Sprintf(`{"spec":{"strategy":{"%s":{"env":[{"name":"BUILD_LOGLEVEL","value":"%d"}]}}}}`, strategyType, buildLogLevel)
}

func buildInUserNSPatch(strategyType string, buildLogLevel int) string {
envName, envValue, set := strings.Cut(buildInUserNSEnvVar, "=")
if !set {
panic(fmt.Sprintf(`environment variable %q is supposed to have a "=" in it to separate the environment variable's name and its value`, buildInUserNSEnvVar))
}
return fmt.Sprintf(`{"spec":{"strategy":{"%s":{"env":[{"name":"%s","value":"%s"},{"name":"BUILD_LOGLEVEL","value":"%d"}]}}}}`, strategyType, envName, envValue, buildLogLevel)
}

var (
//TODO: Make these externally configurable

Expand Down
77 changes: 71 additions & 6 deletions test/extended/builds/custom_build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,22 @@ import (
exutil "github.com/openshift/origin/test/extended/util"
)

const (
// failedToReadNodeCredsRegexp is the message that we expect the
// builder to log when it can't read the node's pull secrets
failedToReadNodeCredsRegexp = `proceeding without node credentials: open /var/lib/kubelet/config.json: permission denied`
// failedToPullWithoutCredsRegexp is the message that we expect the
// builder to log when it can't pull an image because it doesn't have
// the right credentials to access the registry
failedToPullWithoutCredsRegexp = `Error pulling image ".*": initializing source .*: unable to retrieve auth token: invalid username/password: unauthorized:`
)

var _ = g.Describe("[sig-builds][Feature:Builds] custom build with buildah", func() {
defer g.GinkgoRecover()
var (
oc = exutil.NewCLIWithPodSecurityLevel("custom-build", admissionapi.LevelBaseline)
customBuildAdd = exutil.FixturePath("testdata", "builds", "custom-build")
customBuildAddAnon = exutil.FixturePath("testdata", "builds", "custom-build-anon")
customBuildFixture = exutil.FixturePath("testdata", "builds", "test-custom-build.yaml")
)

Expand All @@ -34,21 +45,75 @@ var _ = g.Describe("[sig-builds][Feature:Builds] custom build with buildah", fun
})

g.Describe("being created from new-build", func() {
g.It("should complete build with custom builder image [apigroup:build.openshift.io]", func() {
testCustomBuild := func(customBuilderImageDir, logsMustMatchRegexp, dockerPatch, customPatch string) {
g.By("create custom builder image")
err := oc.Run("new-build").Args("--binary", "--strategy=docker", "--name=custom-builder-image").Execute()
o.Expect(err).NotTo(o.HaveOccurred())
br, _ := exutil.StartBuildAndWait(oc, "custom-builder-image", fmt.Sprintf("--from-dir=%s", customBuildAdd))

if dockerPatch != "" {
g.By("applying patch to build config for creating custom builder image")
err := oc.Run("patch").Args("bc/custom-builder-image", "-p", dockerPatch).Execute()
o.Expect(err).NotTo(o.HaveOccurred())
}

br, err := exutil.StartBuildAndWait(oc, "custom-builder-image", fmt.Sprintf("--from-dir=%s", customBuilderImageDir))
o.Expect(err).NotTo(o.HaveOccurred())
br.AssertSuccess()
g.By("start custom build and build should complete")
if logsMustMatchRegexp != "" {
g.By(fmt.Sprintf("verify that the build log for creating the custom builder image included a message that matched %q", logsMustMatchRegexp))
buildPodLogs, err := br.Logs()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(buildPodLogs).To(o.MatchRegexp(logsMustMatchRegexp))
}

g.By("create build config using custom builder image")
err = oc.AsAdmin().Run("create").Args("-f", customBuildFixture).Execute()
o.Expect(err).NotTo(o.HaveOccurred())
err = oc.AsAdmin().Run("start-build").Args("sample-custom-build").Execute()

if customPatch != "" {
g.By("applying patch to build config using custom builder image")
err = oc.AsAdmin().Run("patch").Args("bc/sample-custom-build", "-p", customPatch).Execute()
o.Expect(err).NotTo(o.HaveOccurred())
}

g.By("start custom build and build should complete")
br, err = exutil.StartBuildAndWait(oc.AsAdmin(), "sample-custom-build")

o.Expect(err).NotTo(o.HaveOccurred())
err = exutil.WaitForABuild(oc.BuildClient().BuildV1().Builds(oc.Namespace()), "sample-custom-build"+"-1", nil, nil, nil)

if logsMustMatchRegexp != "" {
g.By(fmt.Sprintf("verify that the build log included a message that matched %q", logsMustMatchRegexp))
buildPodLogs, err := br.Logs()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(buildPodLogs).To(o.MatchRegexp(logsMustMatchRegexp))
}
}

g.It("should complete build with custom builder image built from base image pulled using node secrets [apigroup:build.openshift.io]", func() {
testCustomBuild(customBuildAdd, buildInDefaultUserNSRegexp, buildInDefaultUserNSPatch("dockerStrategy", 2), buildInDefaultUserNSPatch("customStrategy", 2))
})
g.It("should complete unprivileged build with custom builder image built from base image pulled anonymously [apigroup:build.openshift.io]", func() {
testCustomBuild(customBuildAddAnon, buildInUserNSRegexp, buildInUserNSPatch("dockerStrategy", 2), buildInUserNSPatch("customStrategy", 2))
})
g.It("should fail to build the custom builder image without node credentials [apigroup:build.openshift.io]", func() {
g.By("create custom builder image")
err := oc.Run("new-build").Args("--binary", "--strategy=docker", "--name=custom-builder-image", "--env", buildInUserNSEnvVar, "--env", "BUILD_LOGLEVEL=2").Execute()
o.Expect(err).NotTo(o.HaveOccurred())
br, err := exutil.StartBuildAndWait(oc, "custom-builder-image", fmt.Sprintf("--from-dir=%s", customBuildAdd))
o.Expect(err).NotTo(o.HaveOccurred())
br.AssertFailure()
logsMustMatchRegexps := []string{
buildInUserNSRegexp,
failedToReadNodeCredsRegexp,
failedToPullWithoutCredsRegexp,
}
for _, logsMustMatchRegexp := range logsMustMatchRegexps {
g.By(fmt.Sprintf("verify that the build log for failing to create the custom builder image included a message that matched %q", logsMustMatchRegexp))
buildPodLogs, err := br.Logs()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(buildPodLogs).To(o.MatchRegexp(logsMustMatchRegexp))
}
})

})
})
})
20 changes: 17 additions & 3 deletions test/extended/builds/docker_pullsearch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package builds

import (
"fmt"

g "github.com/onsi/ginkgo/v2"
o "github.com/onsi/gomega"

Expand Down Expand Up @@ -31,17 +33,29 @@ var _ = g.Describe("[sig-builds][Feature:Builds][pullsearch] docker build where
})

g.Describe("Building from a Dockerfile whose FROM image ref does not specify the image registry", func() {
g.It("should create a docker build that has buildah search from our predefined list of image registries and succeed [apigroup:build.openshift.io]", func() {

testDockerBuild := func(logsMustMatchRegexp string, startBuildAddArgs ...string) {
g.By("creating a BuildConfig whose base image does not have a fully qualified registry name")
err := oc.Run("create").Args("-f", buildFixture).Execute()
o.Expect(err).NotTo(o.HaveOccurred())

g.By("starting a build")
br, err := exutil.StartBuildAndWait(oc, "ubi")
br, err := exutil.StartBuildAndWait(oc, append([]string{"ubi"}, startBuildAddArgs...)...)
o.Expect(err).NotTo(o.HaveOccurred())
br.AssertSuccess()

if logsMustMatchRegexp != "" {
g.By(fmt.Sprintf("verify that the build log included a message that matched %q", logsMustMatchRegexp))
buildPodLogs, err := br.Logs()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(buildPodLogs).To(o.MatchRegexp(logsMustMatchRegexp))
}
}
g.It("should create a docker build that can search from our predefined list of image registries and succeed [apigroup:build.openshift.io]", func() {
testDockerBuild(buildInDefaultUserNSRegexp)
})

g.It("should create an unprivileged docker build that can still search our predefined list of image registries and shortnames and succeed [apigroup:build.openshift.io]", func() {
testDockerBuild(buildInUserNSRegexp, "--env", buildInUserNSEnvVar)
})
})
})
Expand Down
28 changes: 24 additions & 4 deletions test/extended/builds/docker_pullsecret.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,22 +38,42 @@ var _ = g.Describe("[sig-builds][Feature:Builds][pullsecret] docker build using
})

g.Describe("Building from a template", func() {
g.It("should create a docker build that pulls using a secret run it [apigroup:build.openshift.io]", func() {

testDockerBuild := func(logsMustMatchRegexp string, startBuildAddArgs ...string) {
g.By(fmt.Sprintf("calling oc create -f %q", buildFixture))
err := oc.Run("create").Args("-f", buildFixture).Execute()
o.Expect(err).NotTo(o.HaveOccurred())

g.By("starting a build")
br, err := exutil.StartBuildAndWait(oc, "docker-build")
br, err := exutil.StartBuildAndWait(oc, append([]string{"docker-build"}, startBuildAddArgs...)...)
o.Expect(err).NotTo(o.HaveOccurred())
br.AssertSuccess()

if logsMustMatchRegexp != "" {
g.By(fmt.Sprintf("verify that the build log included a message that matched %q", logsMustMatchRegexp))
buildPodLogs, err := br.Logs()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(buildPodLogs).To(o.MatchRegexp(logsMustMatchRegexp))
}

g.By("starting a second build that pulls the image from the first build")
br, err = exutil.StartBuildAndWait(oc, "docker-build-pull")
br, err = exutil.StartBuildAndWait(oc, append([]string{"docker-build-pull"}, startBuildAddArgs...)...)
o.Expect(err).NotTo(o.HaveOccurred())
br.AssertSuccess()

if logsMustMatchRegexp != "" {
g.By(fmt.Sprintf("verify that the second build log included a message that matched %q", logsMustMatchRegexp))
buildPodLogs, err := br.Logs()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(buildPodLogs).To(o.MatchRegexp(logsMustMatchRegexp))
}
}

g.It("should create a docker build that pulls using a secret and run it [apigroup:build.openshift.io]", func() {
testDockerBuild(buildInDefaultUserNSRegexp)
})

g.It("should create an unprivileged docker build that pulls using a secret and run it [apigroup:build.openshift.io]", func() {
testDockerBuild(buildInUserNSRegexp, "--env", buildInUserNSEnvVar)
})
})
})
Expand Down
Loading