[release-v1.17] Backports

.github/workflows/test-e2e-runtime.yaml

@@ -12,7 +12,7 @@ jobs:
     continue-on-error: true
     strategy:
       matrix:
-        runtime: ["node", "go", "python", "quarkus", "springboot", "typescript", "rust"]
+        runtime: ["node", "go", "quarkus", "springboot", "typescript", "rust"]
     runs-on: ubuntu-latest
     steps:
       - name: Set Environment Variables

.gitignore

@@ -2,8 +2,6 @@
 /func
 /func_*
 /cmd/func.yaml
-/templates/typescript/cloudevents/build
-/templates/typescript/http/build
 /coverage.out
 /coverage.txt
 /.coverage
@@ -17,11 +15,15 @@
 
 /pkg/functions/testdata/migrations/*/.gitignore
 
-# Nodejs
+# JS
 node_modules
+/templates/typescript/cloudevents/build
+/templates/typescript/http/build
 
 # Python
 __pycache__
+/templates/python/cloudevents/.venv
+/templates/python/http/.venv
 
 # VSCode
 .vscode

.golangci.yaml

@@ -1,31 +1,37 @@
-#
-# golangci-lint
-#
-# For defaults, see:
-# https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml
-#
-#
-#
-run:
-  timeout: 5m
-
+version: "2"
 linters:
   enable:
     - unconvert
     - prealloc
     - bodyclose
-
-issues:
-  exclude-rules:
-    - linters:
-        - staticcheck
-      # Error Text:
-      #   "SA9004: only the first constant in this group has an explicit type"
-      # Efect:
-      #   Allows short-hand first constant type declarations:
-      # Example:
-      #   const (
-      #     Name  Type = "value"
-      #     Name2      = "value2"
-      #   )
-      text: "SA9004:"
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    rules:
+      - linters:
+          - staticcheck
+        # Error Text:
+        #   "SA9004: only the first constant in this group has an explicit type"
+        # Efect:
+        #   Allows short-hand first constant type declarations:
+        # Example:
+        #   const (
+        #     Name  Type = "value"
+        #     Name2      = "value2"
+        #   )
+        text: 'SA9004:'
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

Makefile

@@ -76,7 +76,7 @@ check: $(BIN_GOLANGCI_LINT) ## Check code quality (lint)
 	cd test && $(BIN_GOLANGCI_LINT) run --timeout 300s
 
 $(BIN_GOLANGCI_LINT):
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./bin v1.62.2
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./bin v2.0.2
 
 .PHONY: generate/zz_filesystem_generated.go
 generate/zz_filesystem_generated.go: clean_templates
@@ -85,21 +85,27 @@ generate/zz_filesystem_generated.go: clean_templates
 .PHONY: clean_templates
 clean_templates:
 	# Removing temporary template files
+	@rm -rf templates/**/.DS_Store
 	@rm -rf templates/node/cloudevents/node_modules
 	@rm -rf templates/node/http/node_modules
-	@rm -rf templates/python/cloudevents/__pycache__
-	@rm -rf templates/python/http/__pycache__
-	@rm -rf templates/typescript/cloudevents/node_modules
-	@rm -rf templates/typescript/http/node_modules
-	@rm -rf templates/typescript/cloudevents/build
-	@rm -rf templates/typescript/http/build
-	@rm -rf templates/rust/cloudevents/target
-	@rm -rf templates/rust/http/target
+	@rm -rf templates/python/cloudevents/.venv
+	@rm -rf templates/python/cloudevents/.pytest_cache
+	@rm -rf templates/python/cloudevents/function/__pycache__
+	@rm -rf templates/python/cloudevents/tests/__pycache__
+	@rm -rf templates/python/http/.venv
+	@rm -rf templates/python/http/.pytest_cache
+	@rm -rf templates/python/http/function/__pycache__
+	@rm -rf templates/python/http/tests/__pycache__
 	@rm -rf templates/quarkus/cloudevents/target
 	@rm -rf templates/quarkus/http/target
+	@rm -rf templates/rust/cloudevents/target
+	@rm -rf templates/rust/http/target
 	@rm -rf templates/springboot/cloudevents/target
 	@rm -rf templates/springboot/http/target
-	@rm -f templates/**/.DS_Store
+	@rm -rf templates/typescript/cloudevents/build
+	@rm -rf templates/typescript/cloudevents/node_modules
+	@rm -rf templates/typescript/http/build
+	@rm -rf templates/typescript/http/node_modules
 
 .PHONY: clean
 clean: clean_templates ## Remove generated artifacts such as binaries and schemas
@@ -151,9 +157,8 @@ test-node: ## Test Node templates
 	cd templates/node/cloudevents && npm ci && npm test && rm -rf node_modules
 	cd templates/node/http && npm ci && npm test && rm -rf node_modules
 
-test-python: ## Test Python templates
-	cd templates/python/cloudevents && pip3 install -r requirements.txt && python3 test_func.py && rm -rf __pycache__
-	cd templates/python/http && python3 test_func.py && rm -rf __pycache__
+test-python: ## Test Python templates and Scaffolding
+	test/test_python.sh
 
 test-quarkus: ## Test Quarkus templates
 	cd templates/quarkus/cloudevents && ./mvnw -q test && ./mvnw clean && rm .mvn/wrapper/maven-wrapper.jar
@@ -176,14 +181,15 @@ test-typescript: ## Test Typescript templates
 ###############
 
 # Pulls runtimes then rebuilds the embedded filesystem
-update-runtimes:  pull-runtimes generate/zz_filesystem_generated.go ## Update Scaffolding Runtimes
+update-runtimes:  update-runtime-go generate/zz_filesystem_generated.go ## Update Scaffolding Runtimes
 
-pull-runtimes:
+update-runtime-go:
 	cd templates/go/scaffolding/instanced-http && go get -u knative.dev/func-go/http
 	cd templates/go/scaffolding/static-http && go get -u knative.dev/func-go/http
 	cd templates/go/scaffolding/instanced-cloudevents && go get -u knative.dev/func-go/cloudevents
 	cd templates/go/scaffolding/static-cloudevents && go get -u knative.dev/func-go/cloudevents
 
+
 .PHONY: cert
 certs: templates/certs/ca-certificates.crt ## Update root certificates
 

cmd/build.go

@@ -130,10 +130,7 @@ EXAMPLES
 
 	// Temporarily Hidden Basic Auth Flags
 	// Username, Password and Token flags, which plumb through basic auth, are
-	// currently only available on the experimental "host" builder, which is
-	// itself behind a feature flag FUNC_ENABLE_HOST_BUILDER.  So set these
-	// flags to hidden until it's out of preview and they are plumbed through
-	// the docker pusher as well.
+	// currently only available on the "host" builder.
 	_ = cmd.Flags().MarkHidden("username")
 	_ = cmd.Flags().MarkHidden("password")
 	_ = cmd.Flags().MarkHidden("token")
@@ -165,13 +162,13 @@ func runBuild(cmd *cobra.Command, _ []string, newClient ClientFactory) (err erro
 	if err = cfg.Validate(); err != nil { // Perform any pre-validation
 		return
 	}
-	if f, err = fn.NewFunction(cfg.Path); err != nil {
+	if f, err = fn.NewFunction(cfg.Path); err != nil { // Read in the Function
 		return
 	}
 	if !f.Initialized() {
 		return fn.NewErrNotInitialized(f.Root)
 	}
-	f = cfg.Configure(f) // Updates f at path to include build request values
+	f = cfg.Configure(f) // Returns an f updated with values from the config (flags, envs, etc)
 
 	cmd.SetContext(cfg.WithValues(cmd.Context())) // Some optional settings are passed via context
 
@@ -184,7 +181,7 @@ func runBuild(cmd *cobra.Command, _ []string, newClient ClientFactory) (err erro
 	defer done()
 
 	// Build
-	buildOptions, err := cfg.buildOptions()
+	buildOptions, err := cfg.buildOptions() // build-specific options from the finalized cfg
 	if err != nil {
 		return
 	}

cmd/deploy.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"os"
 	"strconv"
 	"strings"
 
@@ -35,7 +34,7 @@ SYNOPSIS
 	             [-b|--build] [--builder] [--builder-image] [-p|--push]
 	             [--domain] [--platform] [--build-timestamp] [--pvc-size]
 	             [--service-account] [-c|--confirm] [-v|--verbose]
-	             [--registry-insecure]
+	             [--registry-insecure] [--remote-storage-class]
 
 DESCRIPTION
 
@@ -127,7 +126,7 @@ EXAMPLES
 
 `,
 		SuggestFor: []string{"delpoy", "deplyo"},
-		PreRunE:    bindEnv("build", "build-timestamp", "builder", "builder-image", "confirm", "domain", "env", "git-branch", "git-dir", "git-url", "image", "namespace", "path", "platform", "push", "pvc-size", "service-account", "registry", "registry-insecure", "remote", "username", "password", "token", "verbose"),
+		PreRunE:    bindEnv("build", "build-timestamp", "builder", "builder-image", "confirm", "domain", "env", "git-branch", "git-dir", "git-url", "image", "namespace", "path", "platform", "push", "pvc-size", "service-account", "registry", "registry-insecure", "remote", "username", "password", "token", "verbose", "remote-storage-class"),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return runDeploy(cmd, newClient)
 		},
@@ -179,6 +178,8 @@ EXAMPLES
 		"Directory in the Git repository containing the function (default is the root) ($FUNC_GIT_DIR)")
 	cmd.Flags().BoolP("remote", "R", f.Local.Remote,
 		"Trigger a remote deployment. Default is to deploy and build from the local system ($FUNC_REMOTE)")
+	cmd.Flags().StringP("remote-storage-class", "", f.Build.RemoteStorageClass,
+		"Specify a storage class to use for the volume on-cluster during remote builds")
 	cmd.Flags().String("pvc-size", f.Build.PVCSize,
 		"When triggering a remote deployment, set a custom volume size to allocate for the build operation ($FUNC_PVC_SIZE)")
 	cmd.Flags().String("service-account", f.Deploy.ServiceAccountName,
@@ -205,10 +206,7 @@ EXAMPLES
 
 	// Temporarily Hidden Basic Auth Flags
 	// Username, Password and Token flags, which plumb through basic auth, are
-	// currently only available on the experimental "host" builder, which is
-	// itself behind a feature flag FUNC_ENABLE_HOST_BUILDER.  So set these
-	// flags to hidden until it's out of preview and they are plumbed through
-	// the docker pusher as well.
+	// currently only available on "host" builder.
 	_ = cmd.Flags().MarkHidden("username")
 	_ = cmd.Flags().MarkHidden("password")
 	_ = cmd.Flags().MarkHidden("token")
@@ -431,18 +429,7 @@ func KnownBuilders() builders.Known {
 	// However, future third-party integrations may support less than, or more
 	// builders, and certain environmental considerations may alter this list.
 
-	// Also a good place to stick feature-flags; to wit:
-	enable_host, _ := strconv.ParseBool(os.Getenv("FUNC_ENABLE_HOST_BUILDER"))
-	if !enable_host {
-		bb := []string{}
-		for _, b := range builders.All() {
-			if b != builders.Host {
-				bb = append(bb, b)
-			}
-		}
-		return bb
-	}
-
+	// Also a good place to stick feature-flags.
 	return builders.All()
 }
 
@@ -517,6 +504,10 @@ type deployConfig struct {
 	// be triggered in a remote environment rather than run locally.
 	Remote bool
 
+	// RemoteStorageClass defines the storage class to use for the remote
+	// volume when building on-cluster.
+	RemoteStorageClass string
+
 	// PVCSize configures the PVC size used by the pipeline if --remote flag is set.
 	PVCSize string
 
@@ -538,6 +529,7 @@ func newDeployConfig(cmd *cobra.Command) deployConfig {
 		GitURL:             viper.GetString("git-url"),
 		Namespace:          viper.GetString("namespace"),
 		Remote:             viper.GetBool("remote"),
+		RemoteStorageClass: viper.GetString("remote-storage-class"),
 		PVCSize:            viper.GetString("pvc-size"),
 		Timestamp:          viper.GetBool("build-timestamp"),
 		ServiceAccountName: viper.GetString("service-account"),
@@ -573,6 +565,7 @@ func (c deployConfig) Configure(f fn.Function) (fn.Function, error) {
 	f.Build.Git.URL = c.GitURL
 	f.Build.Git.ContextDir = c.GitDir
 	f.Build.Git.Revision = c.GitBranch // TODO: should match; perhaps "refSpec"
+	f.Build.RemoteStorageClass = c.RemoteStorageClass
 	f.Deploy.ServiceAccountName = c.ServiceAccountName
 	f.Local.Remote = c.Remote
 

cmd/deploy_test.go

@@ -1531,7 +1531,6 @@ func testAuthentication(cmdFn commandConstructor, t *testing.T) {
 	}
 
 	cmd := cmdFn(NewTestClient(fn.WithPusher(pusher)))
-	t.Setenv("FUNC_ENABLE_HOST_BUILDER", "true") // host builder is currently behind this feature flag
 	cmd.SetArgs([]string{"--builder", "host", "--username", testUser, "--password", testPass})
 	if err := cmd.Execute(); err != nil {
 		t.Fatal(err)

cmd/func-util/main.go

@@ -78,7 +78,7 @@ func scaffold(ctx context.Context) error {
 		return fmt.Errorf("cannot load func project: %w", err)
 	}
 
-	if f.Runtime != "go" {
+	if f.Runtime != "go" && f.Runtime != "python" {
 		// Scaffolding is for now supported/needed only for Go.
 		return nil
 	}
@@ -100,7 +100,17 @@ func scaffold(ctx context.Context) error {
 		return fmt.Errorf("unable to create .s2i bin dir. %w", err)
 	}
 
-	if err := os.WriteFile(filepath.Join(f.Root, ".s2i", "bin", "assemble"), []byte(s2i.GoAssembler), 0755); err != nil {
+	var asm string
+	switch f.Runtime {
+	case "go":
+		asm = s2i.GoAssembler
+	case "python":
+		asm = s2i.PythonAssembler
+	default:
+		panic("unreachable")
+	}
+
+	if err := os.WriteFile(filepath.Join(f.Root, ".s2i", "bin", "assemble"), []byte(asm), 0755); err != nil {
 		return fmt.Errorf("unable to write go assembler. %w", err)
 	}