From 4175e05ae25e40e95c310d08dc14b954453d6077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= Date: Fri, 14 Feb 2025 15:09:35 +0100 Subject: [PATCH 1/2] Scan for main packages on imports --- cmd/generate/generate.go | 4 + .../Default.dockerfile.tmpl | 2 +- pkg/dockerfilegen/generator.go | 9 +- pkg/dockerfilegen/imports/scanner.go | 181 ++++++++++++++++++ pkg/dockerfilegen/imports_scanner.go | 23 +++ pkg/dockerfilegen/params.go | 5 + 6 files changed, 222 insertions(+), 2 deletions(-) create mode 100644 pkg/dockerfilegen/imports/scanner.go create mode 100644 pkg/dockerfilegen/imports_scanner.go diff --git a/cmd/generate/generate.go b/cmd/generate/generate.go index 28c504c42..76bee1181 100644 --- a/cmd/generate/generate.go +++ b/cmd/generate/generate.go @@ -38,6 +38,10 @@ func main() { pflag.StringArrayVar(¶ms.AdditionalBuildEnvVars, "additional-build-env", defs.AdditionalBuildEnvVars, "Additional env vars to be added to builder in the image") pflag.StringVar(¶ms.TemplateName, "template-name", defs.TemplateName, fmt.Sprintf("Dockerfile template name to use. Supported values are [%s, %s]", dockerfilegen.DefaultDockerfileTemplateName, dockerfilegen.FuncUtilDockerfileTemplateName)) pflag.BoolVar(¶ms.RpmsLockFileEnabled, "generate-rpms-lock-file", defs.RpmsLockFileEnabled, "Enable the creation of the rpms.lock.yaml file") + pflag.BoolVar(¶ms.ScanImports, "scan-imports", defs.ScanImports, "Should the imports be scanned for main packages") + pflag.StringArrayVar(¶ms.ScanImportsSubPackages, "scan-imports-subpackages", defs.ScanImportsSubPackages, "Which sub-packages to scan for main imports") + pflag.StringArrayVar(¶ms.ScanImportsTags, "scan-imports-tags", defs.ScanImportsTags, "Which build tags use to limit the scan") + pflag.Parse() if err = dockerfilegen.GenerateDockerfiles(params); err != nil { diff --git a/pkg/dockerfilegen/dockerfile-templates/Default.dockerfile.tmpl b/pkg/dockerfilegen/dockerfile-templates/Default.dockerfile.tmpl index fc16f463f..3503ab162 100644 --- a/pkg/dockerfilegen/dockerfile-templates/Default.dockerfile.tmpl +++ b/pkg/dockerfilegen/dockerfile-templates/Default.dockerfile.tmpl @@ -10,7 +10,7 @@ COPY . . {{ $c }} {{- end }} -RUN go build -tags strictfipsruntime -o /usr/bin/main ./{{.main}} +RUN go build -tags strictfipsruntime -o /usr/bin/main {{.main}} FROM $GO_RUNTIME diff --git a/pkg/dockerfilegen/generator.go b/pkg/dockerfilegen/generator.go index 429e9eda2..749b2cbe1 100644 --- a/pkg/dockerfilegen/generator.go +++ b/pkg/dockerfilegen/generator.go @@ -125,12 +125,19 @@ func GenerateDockerfiles(params Params) error { return nil } - mainPackagesPaths.Insert(filepath.Dir(path)) + mainPackagesPaths.Insert("./" + filepath.Dir(path)) return nil }); err != nil { return errors.WithStack(err) } + if params.ScanImports { + if err := scanImports(mainPackagesPaths, params.RootDir, + params.ScanImportsSubPackages, params.ScanImportsTags); err != nil { + return err + } + } + for _, p := range mainPackagesPaths.UnsortedList() { log.Println("Main package path:", p) } diff --git a/pkg/dockerfilegen/imports/scanner.go b/pkg/dockerfilegen/imports/scanner.go new file mode 100644 index 000000000..62300127f --- /dev/null +++ b/pkg/dockerfilegen/imports/scanner.go @@ -0,0 +1,181 @@ +package imports + +import ( + "fmt" + "go/build" + "go/parser" + "go/token" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + + "github.com/pkg/errors" + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" + "k8s.io/apimachinery/pkg/util/sets" +) + +var ( + ErrCompileError = fmt.Errorf("could't compile") + ErrIO = fmt.Errorf("io fail") +) + +func ScanForMains(rootDir string, packages []string, tags []string) (sets.Set[string], error) { + pkgs := sets.New[string]() + collctr, err := collector(rootDir, pkgs) + if err != nil { + return nil, err + } + for _, subpkg := range packages { + if err := filepath.WalkDir(path.Join(rootDir, subpkg), scanner(collctr, tags)); err != nil { + return nil, err + } + } + + return pkgs, nil +} + +type collectOnlyMainFn func(imprts []string) error + +func collector(rootDir string, pkgs sets.Set[string]) (collectOnlyMainFn, error) { + gomodPath := path.Join(rootDir, "go.mod") + contents, err := os.ReadFile(gomodPath) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrIO, errors.WithStack(err)) + } + gm, err := modfile.ParseLax(gomodPath, contents, nil) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrCompileError, errors.WithStack(err)) + } + return func(imprts []string) error { + for _, imprt := range imprts { + ok, err := isDepMainPackage(rootDir, gm, imprt) + if err != nil { + return err + } + if ok { + pkgs.Insert(imprt) + } + } + return nil + }, nil +} + +func isDepMainPackage(rootDir string, gm *modfile.File, imprt string) (bool, error) { + // try vendor + pkgPath := path.Join(rootDir, "vendor", imprt) + fi, err := os.Stat(pkgPath) + if err == nil && fi.IsDir() { + return isMainPkg(pkgPath) + } + // modules + var mod module.Version + for _, req := range gm.Require { + if strings.HasPrefix(imprt, req.Mod.Path) { + mod = req.Mod + break + } + } + for _, repl := range gm.Replace { + if repl.Old.Path == mod.Path { + mod = repl.New + break + } + } + subIprt := strings.TrimPrefix(imprt, mod.Path) + pkgPath = path.Join(goModCache(), mod.Path+"@"+mod.Version, subIprt) + return isMainPkg(pkgPath) +} + +func isMainPkg(pkgPath string) (bool, error) { + response := false + if err := filepath.WalkDir(pkgPath, func(path string, info fs.DirEntry, err error) error { + if err != nil || info.IsDir() || !strings.HasSuffix(info.Name(), ".go") { + if path != pkgPath && info.IsDir() { + return filepath.SkipDir + } + return nil + } + content, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("%w: ReadFile %s failed: %w", + ErrIO, path, errors.WithStack(err)) + } + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, path, string(content), parser.PackageClauseOnly) + if err != nil { + return fmt.Errorf("%w: %w", ErrCompileError, errors.WithStack(err)) + } + if f.Name.Name == "main" { + response = true + return filepath.SkipAll + } + return nil + }); err != nil { + return false, err + } + return response, nil +} + +func scanner(collectOnlyMainFn collectOnlyMainFn, tags []string) fs.WalkDirFunc { + // TODO: limit the Go files to the tags provided + return func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(d.Name(), ".go") { + return nil + } + content, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("%w: ReadFile %s failed: %w", + ErrIO, path, errors.WithStack(err)) + } + imprts, err := collectImports(path, content) + if err != nil { + return err + } + return collectOnlyMainFn(imprts) + } +} + +func collectImports(pth string, content []byte) ([]string, error) { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, pth, string(content), parser.ImportsOnly) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrCompileError, errors.WithStack(err)) + } + imprts := make([]string, 0, len(f.Imports)) + for _, spec := range f.Imports { + imprt := strings.TrimSuffix(strings.TrimPrefix(spec.Path.Value, "\""), "\"") + imprts = append(imprts, imprt) + } + return imprts, nil +} + +func goModCache() string { + return envOr("GOMODCACHE", gopathDir("pkg/mod")) +} + +func envOr(key, def string) string { + val := os.Getenv(key) + if val == "" { + val = def + } + return val +} + +func gopathDir(rel string) string { + list := filepath.SplitList(gopath()) + if len(list) == 0 || list[0] == "" { + return "" + } + return filepath.Join(list[0], rel) +} + +func gopath() string { + gp := os.Getenv("GOPATH") + if gp == "" { + gp = build.Default.GOPATH + } + return gp +} diff --git a/pkg/dockerfilegen/imports_scanner.go b/pkg/dockerfilegen/imports_scanner.go new file mode 100644 index 000000000..ff1caa90c --- /dev/null +++ b/pkg/dockerfilegen/imports_scanner.go @@ -0,0 +1,23 @@ +package dockerfilegen + +import ( + "log" + + "github.com/openshift-knative/hack/pkg/dockerfilegen/imports" + "k8s.io/apimachinery/pkg/util/sets" +) + +func scanImports(paths sets.Set[string], rootDir string, packages []string, tags []string) error { + m, err := imports.ScanForMains(rootDir, packages, tags) + if err != nil { + return err + } + for _, pkg := range m.UnsortedList() { + if !paths.Has(pkg) && !paths.Has("vendor/"+pkg) { + paths.Insert(pkg) + log.Println("Found main package from imports:", pkg) + } + } + + return nil +} diff --git a/pkg/dockerfilegen/params.go b/pkg/dockerfilegen/params.go index 012d38323..834ceb9fa 100644 --- a/pkg/dockerfilegen/params.go +++ b/pkg/dockerfilegen/params.go @@ -24,6 +24,9 @@ type Params struct { AdditionalBuildEnvVars []string TemplateName string RpmsLockFileEnabled bool + ScanImports bool + ScanImportsSubPackages []string + ScanImportsTags []string } func DefaultParams(wd string) Params { @@ -53,6 +56,8 @@ func DefaultParams(wd string) Params { AdditionalBuildEnvVars: nil, TemplateName: DefaultDockerfileTemplateName, RpmsLockFileEnabled: false, + ScanImportsSubPackages: []string{"hack"}, + ScanImportsTags: []string{"tools"}, } } From 4db5dfd1f6b63e1ae15db76b25e473568b180ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= Date: Fri, 14 Feb 2025 15:52:00 +0100 Subject: [PATCH 2/2] Tests --- pkg/dockerfilegen/imports/scanner.go | 9 +++++++ pkg/dockerfilegen/imports/scanner_test.go | 24 +++++++++++++++++++ pkg/dockerfilegen/imports/testdata/example.go | 9 +++++++ .../knative-images/discover/Dockerfile | 2 +- 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 pkg/dockerfilegen/imports/scanner_test.go create mode 100644 pkg/dockerfilegen/imports/testdata/example.go diff --git a/pkg/dockerfilegen/imports/scanner.go b/pkg/dockerfilegen/imports/scanner.go index 62300127f..25db43e5a 100644 --- a/pkg/dockerfilegen/imports/scanner.go +++ b/pkg/dockerfilegen/imports/scanner.go @@ -22,6 +22,9 @@ var ( ErrIO = fmt.Errorf("io fail") ) +// ScanForMains will scan a root dir, in specified packages to collect the +// packages that have a main() function. It works for vendorless and vendorful +// projects. func ScanForMains(rootDir string, packages []string, tags []string) (sets.Set[string], error) { pkgs := sets.New[string]() collctr, err := collector(rootDir, pkgs) @@ -64,6 +67,12 @@ func collector(rootDir string, pkgs sets.Set[string]) (collectOnlyMainFn, error) } func isDepMainPackage(rootDir string, gm *modfile.File, imprt string) (bool, error) { + // within repo + if strings.HasPrefix(imprt, gm.Module.Mod.Path) { + subimprt := strings.TrimPrefix(imprt, gm.Module.Mod.Path) + pkgPath := path.Join(rootDir, subimprt) + return isMainPkg(pkgPath) + } // try vendor pkgPath := path.Join(rootDir, "vendor", imprt) fi, err := os.Stat(pkgPath) diff --git a/pkg/dockerfilegen/imports/scanner_test.go b/pkg/dockerfilegen/imports/scanner_test.go new file mode 100644 index 000000000..484ac90f1 --- /dev/null +++ b/pkg/dockerfilegen/imports/scanner_test.go @@ -0,0 +1,24 @@ +package imports_test + +import ( + "path" + "runtime" + "testing" + + "github.com/openshift-knative/hack/pkg/dockerfilegen/imports" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestScanForMains(t *testing.T) { + _, file, _, _ := runtime.Caller(0) + rootDir := path.Dir(path.Dir(path.Dir(path.Dir(file)))) + pkgs, err := imports.ScanForMains(rootDir, + []string{"pkg/dockerfilegen/imports/testdata"}, + []string{"tools"}, + ) + require.NoError(t, err) + assert.True(t, pkgs.Has("github.com/openshift-knative/hack/cmd/prowgen")) + assert.True(t, pkgs.Has("knative.dev/pkg/codegen/cmd/injection-gen")) + assert.Equal(t, 2, pkgs.Len()) +} diff --git a/pkg/dockerfilegen/imports/testdata/example.go b/pkg/dockerfilegen/imports/testdata/example.go new file mode 100644 index 000000000..12b06cd77 --- /dev/null +++ b/pkg/dockerfilegen/imports/testdata/example.go @@ -0,0 +1,9 @@ +//go:build tools + +package testdata + +import ( + _ "github.com/openshift-knative/hack/cmd/prowgen" + _ "knative.dev/pkg" + _ "knative.dev/pkg/codegen/cmd/injection-gen" +) diff --git a/pkg/project/testoutput/openshift/ci-operator/knative-images/discover/Dockerfile b/pkg/project/testoutput/openshift/ci-operator/knative-images/discover/Dockerfile index b808fe851..65fdb5094 100755 --- a/pkg/project/testoutput/openshift/ci-operator/knative-images/discover/Dockerfile +++ b/pkg/project/testoutput/openshift/ci-operator/knative-images/discover/Dockerfile @@ -1,4 +1,4 @@ -# DO NOT EDIT! Generated Dockerfile for cmd/discover. +# DO NOT EDIT! Generated Dockerfile for ./cmd/discover. ARG GO_BUILDER=registry.ci.openshift.org/openshift/release:rhel-8-release-golang-1.22-openshift-4.17 ARG GO_RUNTIME=registry.access.redhat.com/ubi8/ubi-minimal