Rewrite of FindFilesInFolder using go-paths-helper

Author: cmaglie

go.mod

@@ -4,7 +4,7 @@ go 1.16
 
 require (
 	github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c
-	github.com/arduino/go-paths-helper v1.6.1
+	github.com/arduino/go-paths-helper v1.7.0
 	github.com/arduino/go-properties-orderedmap v1.6.0
 	github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b
 	github.com/arduino/go-win32-utils v0.0.0-20180330194947-ed041402e83b

go.sum

@@ -48,8 +48,8 @@ github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c h1:agh2JT9
 github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c/go.mod h1:HK7SpkEax/3P+0w78iRQx1sz1vCDYYw9RXwHjQTB5i8=
 github.com/arduino/go-paths-helper v1.0.1/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
 github.com/arduino/go-paths-helper v1.2.0/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
-github.com/arduino/go-paths-helper v1.6.1 h1:lha+/BuuBsx0qTZ3gy6IO1kU23lObWdQ/UItkzVWQ+0=
-github.com/arduino/go-paths-helper v1.6.1/go.mod h1:V82BWgAAp4IbmlybxQdk9Bpkz8M4Qyx+RAFKaG9NuvU=
+github.com/arduino/go-paths-helper v1.7.0 h1:S9l5BP2aogz1CgyqqnncXt0PLpK4yvwOW/wu/LaR3tc=
+github.com/arduino/go-paths-helper v1.7.0/go.mod h1:V82BWgAAp4IbmlybxQdk9Bpkz8M4Qyx+RAFKaG9NuvU=
 github.com/arduino/go-properties-orderedmap v1.6.0 h1:gp2JoWRETtqwsZ+UHu/PBuYWYH2x2+d+uipDxS4WmvM=
 github.com/arduino/go-properties-orderedmap v1.6.0/go.mod h1:DKjD2VXY/NZmlingh4lSFMEYCVubfeArCsGPGDwb2yk=
 github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b h1:9hDi4F2st6dbLC3y4i02zFT5quS4X6iioWifGlVwfy4=

legacy/builder/builder.go

@@ -34,6 +34,10 @@ var MAIN_FILE_VALID_EXTENSIONS = map[string]bool{".ino": true, ".pde": true}
 var ADDITIONAL_FILE_VALID_EXTENSIONS = map[string]bool{".h": true, ".c": true, ".hpp": true, ".hh": true, ".cpp": true, ".S": true}
 var ADDITIONAL_FILE_VALID_EXTENSIONS_NO_HEADERS = map[string]bool{".c": true, ".cpp": true, ".S": true}
 
+var MainFileValidExtensions = []string{".ino", ".pde"}
+var AdditionalFileValidExtensions = []string{".h", ".c", ".hpp", ".hh", ".cpp", ".S"}
+var AdditionalFileValidExtensionsNoHeaders = []string{".c", ".cpp", ".S"}
+
 const DEFAULT_DEBUG_LEVEL = 5
 const DEFAULT_WARNINGS_LEVEL = "none"
 const DEFAULT_SOFTWARE = "ARDUINO"

legacy/builder/container_find_includes.go

@@ -406,16 +406,13 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t
 }
 
 func queueSourceFilesFromFolder(ctx *types.Context, queue *types.UniqueSourceFileQueue, origin interface{}, folder *paths.Path, recurse bool) error {
-	extensions := func(ext string) bool { return ADDITIONAL_FILE_VALID_EXTENSIONS_NO_HEADERS[ext] }
-
-	filePaths := []string{}
-	err := utils.FindFilesInFolder(&filePaths, folder.String(), extensions, recurse)
+	filePaths, err := utils.FindFilesInFolder(folder, recurse, AdditionalFileValidExtensionsNoHeaders)
 	if err != nil {
 		return errors.WithStack(err)
 	}
 
 	for _, filePath := range filePaths {
-		sourceFile, err := types.MakeSourceFile(ctx, origin, paths.New(filePath))
+		sourceFile, err := types.MakeSourceFile(ctx, origin, filePath)
 		if err != nil {
 			return errors.WithStack(err)
 		}

legacy/builder/create_cmake_rule.go

@@ -17,8 +17,6 @@ package builder
 
 import (
 	"fmt"
-	"io/ioutil"
-	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -32,8 +30,10 @@ import (
 )
 
 var VALID_EXPORT_EXTENSIONS = map[string]bool{".h": true, ".c": true, ".hpp": true, ".hh": true, ".cpp": true, ".S": true, ".a": true, ".properties": true}
-var DOTHEXTENSION = map[string]bool{".h": true, ".hh": true, ".hpp": true}
-var DOTAEXTENSION = map[string]bool{".a": true}
+
+var ValidExportExtensions = []string{".h", ".c", ".hpp", ".hh", ".cpp", ".S", ".a", ".properties"}
+var DotHExtension = []string{".h", ".hh", ".hpp"}
+var DotAExtension = []string{".a"}
 
 type ExportProjectCMake struct {
 	// Was there an error while compiling the sketch?
@@ -66,7 +66,6 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 
 	dynamicLibsFromPkgConfig := map[string]bool{}
 	extensions := func(ext string) bool { return VALID_EXPORT_EXTENSIONS[ext] }
-	staticLibsExtensions := func(ext string) bool { return DOTAEXTENSION[ext] }
 	for _, library := range ctx.ImportedLibraries {
 		// Copy used libraries in the correct folder
 		libDir := libBaseFolder.Join(library.Name)
@@ -91,12 +90,11 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 		}
 
 		// Remove stray folders contining incompatible or not needed libraries archives
-		var files []string
-		utils.FindFilesInFolder(&files, libDir.Join("src").String(), staticLibsExtensions, true)
+		files, _ := utils.FindFilesInFolder(libDir.Join("src"), true, DotAExtension)
 		for _, file := range files {
-			staticLibDir := filepath.Dir(file)
-			if !isStaticLib || !strings.Contains(staticLibDir, mcu) {
-				os.RemoveAll(staticLibDir)
+			staticLibDir := file.Parent()
+			if !isStaticLib || !strings.Contains(staticLibDir.String(), mcu) {
+				staticLibDir.RemoveAll()
 			}
 		}
 	}
@@ -128,11 +126,10 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 	}
 
 	// remove "#line 1 ..." from exported c_make folder sketch
-	var sketchFiles []string
-	utils.FindFilesInFolder(&sketchFiles, cmakeFolder.Join("sketch").String(), extensions, false)
+	sketchFiles, _ := utils.FindFilesInFolder(cmakeFolder.Join("sketch"), false, ValidExportExtensions)
 
 	for _, file := range sketchFiles {
-		input, err := ioutil.ReadFile(file)
+		input, err := file.ReadFile()
 		if err != nil {
 			fmt.Println(err)
 			continue
@@ -146,7 +143,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 			}
 		}
 		output := strings.Join(lines, "\n")
-		err = ioutil.WriteFile(file, []byte(output), 0644)
+		err = file.WriteFile([]byte(output))
 		if err != nil {
 			fmt.Println(err)
 		}
@@ -163,14 +160,11 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 	extractCompileFlags(ctx, "recipe.cpp.o.pattern", &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories)
 
 	// Extract folders with .h in them for adding in include list
-	var headerFiles []string
-	isHeader := func(ext string) bool { return DOTHEXTENSION[ext] }
-	utils.FindFilesInFolder(&headerFiles, cmakeFolder.String(), isHeader, true)
-	foldersContainingDotH := findUniqueFoldersRelative(headerFiles, cmakeFolder.String())
+	headerFiles, _ := utils.FindFilesInFolder(cmakeFolder, true, DotHExtension)
+	foldersContainingDotH := findUniqueFoldersRelative(headerFiles.AsStrings(), cmakeFolder.String())
 
 	// Extract folders with .a in them for adding in static libs paths list
-	var staticLibs []string
-	utils.FindFilesInFolder(&staticLibs, cmakeFolder.String(), staticLibsExtensions, true)
+	staticLibs, _ := utils.FindFilesInFolder(cmakeFolder, true, DotAExtension)
 
 	// Generate the CMakeLists global file
 
@@ -210,13 +204,13 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 	cmakelist += "link_directories (" + strings.Join(relLinkDirectories, " ") + " ${EXTRA_LIBS_DIRS})\n"
 	for _, staticLib := range staticLibs {
 		// Static libraries are fully configured
-		lib := filepath.Base(staticLib)
+		lib := staticLib.Base()
 		lib = strings.TrimPrefix(lib, "lib")
 		lib = strings.TrimSuffix(lib, ".a")
 		if !utils.SliceContains(dynamicLibsFromGccMinusL, lib) {
 			linkGroup += " " + lib
 			cmakelist += "add_library (" + lib + " STATIC IMPORTED)\n"
-			location := strings.TrimPrefix(staticLib, cmakeFolder.String())
+			location := strings.TrimPrefix(staticLib.String(), cmakeFolder.String())
 			cmakelist += "set_property(TARGET " + lib + " PROPERTY IMPORTED_LOCATION " + "${PROJECT_SOURCE_DIR}" + location + " )\n"
 		}
 	}

legacy/builder/test/builder_test.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/arduino/go-paths-helper"
+	"github.com/sirupsen/logrus"
 
 	"github.com/arduino/arduino-cli/legacy/builder"
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
@@ -306,6 +307,7 @@ func TestBuilderSketchWithOldLib(t *testing.T) {
 func TestBuilderSketchWithSubfolders(t *testing.T) {
 	DownloadCoresAndToolsAndLibraries(t)
 
+	logrus.SetLevel(logrus.DebugLevel)
 	ctx := prepareBuilderTestContext(t, paths.New("sketch_with_subfolders", "sketch_with_subfolders.ino"), "arduino:avr:uno")
 
 	buildPath := SetupBuildPath(t, ctx)

legacy/builder/utils/utils.go

@@ -87,6 +87,25 @@ func FilterFiles() filterFiles {
 
 var SOURCE_CONTROL_FOLDERS = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true}
 
+// FilterOutHiddenFiles is a ReadDirFilter that exclude files with a "." prefix in their name
+var FilterOutHiddenFiles = paths.FilterOutPrefixes(".")
+
+// FilterOutSCCS is a ReadDirFilter that excludes known VSC or project files
+func FilterOutSCCS(file *paths.Path) bool {
+	return !SOURCE_CONTROL_FOLDERS[file.Base()]
+}
+
+// FilterReadableFiles is a ReadDirFilter that accepts only readable files
+func FilterReadableFiles(file *paths.Path) bool {
+	// See if the file is readable by opening it
+	f, err := file.Open()
+	if err != nil {
+		return false
+	}
+	f.Close()
+	return true
+}
+
 func IsSCCSOrHiddenFile(file os.FileInfo) bool {
 	return IsSCCSFile(file) || IsHiddenFile(file)
 }
@@ -226,46 +245,22 @@ func AbsolutizePaths(files []string) ([]string, error) {
 
 type CheckExtensionFunc func(ext string) bool
 
-func FindFilesInFolder(files *[]string, folder string, extensions CheckExtensionFunc, recurse bool) error {
-	walkFunc := func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		// Skip source control and hidden files and directories
-		if IsSCCSOrHiddenFile(info) {
-			if info.IsDir() {
-				return filepath.SkipDir
-			}
-			return nil
-		}
-
-		// Skip directories unless recurse is on, or this is the
-		// root directory
-		if info.IsDir() {
-			if recurse || path == folder {
-				return nil
-			} else {
-				return filepath.SkipDir
-			}
-		}
-
-		// Check (lowercased) extension against list of extensions
-		if extensions != nil && !extensions(strings.ToLower(filepath.Ext(path))) {
-			return nil
-		}
-
-		// See if the file is readable by opening it
-		currentFile, err := os.Open(path)
-		if err != nil {
-			return nil
-		}
-		currentFile.Close()
-
-		*files = append(*files, path)
-		return nil
-	}
-	return gohasissues.Walk(folder, walkFunc)
+func FindFilesInFolder(dir *paths.Path, recurse bool, extensions []string) (paths.PathList, error) {
+	fileFilter := paths.AndFilter(
+		paths.FilterSuffixes(extensions...),
+		FilterOutHiddenFiles,
+		FilterOutSCCS,
+		paths.FilterOutDirectories(),
+		FilterReadableFiles,
+	)
+	if recurse {
+		dirFilter := paths.AndFilter(
+			FilterOutHiddenFiles,
+			FilterOutSCCS,
+		)
+		return dir.ReadDirRecursiveFiltered(dirFilter, fileFilter)
+	}
+	return dir.ReadDir(fileFilter)
 }
 
 func AppendIfNotPresent(target []string, elements ...string) []string {