[WIP] Archive compiled core and use in any compatible project

Author: facchinm

src/arduino.cc/builder/builder_utils/utils.go

@@ -32,6 +32,7 @@ package builder_utils
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -116,6 +117,36 @@ func findFilesInFolder(sourcePath string, extension string, recurse bool) ([]str
 	return sources, nil
 }
 
+func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) {
+	files, err := utils.ReadDirFiltered(sourcePath, utils.FilterFiles())
+	if err != nil {
+		return nil, i18n.WrapError(err)
+	}
+	var sources []string
+	for _, file := range files {
+		sources = append(sources, filepath.Join(sourcePath, file.Name()))
+	}
+
+	if recurse {
+		folders, err := utils.ReadDirFiltered(sourcePath, utils.FilterDirs)
+		if err != nil {
+			return nil, i18n.WrapError(err)
+		}
+
+		for _, folder := range folders {
+			otherSources, err := findAllFilesInFolder(filepath.Join(sourcePath, folder.Name()), recurse)
+			if err != nil {
+				return nil, i18n.WrapError(err)
+			}
+			sources = append(sources, otherSources...)
+		}
+	}
+
+	fmt.Println(sources)
+
+	return sources, nil
+}
+
 func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
 	for _, source := range sources {
 		objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger)
@@ -258,6 +289,24 @@ func nonEmptyString(s string) bool {
 	return s != constants.EMPTY_STRING
 }
 
+func CheckIfRecompileIsAvoidable(corePath string, archiveFile string) bool {
+	archiveFileStat, err := os.Stat(archiveFile)
+	if err == nil {
+		files, err := findAllFilesInFolder(corePath, true)
+		if err != nil {
+			return false
+		}
+		for _, file := range files {
+			fileStat, err := os.Stat(file)
+			if err != nil || fileStat.ModTime().After(archiveFileStat.ModTime()) {
+				return false
+			}
+		}
+		return true
+	}
+	return false
+}
+
 func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []string, buildProperties properties.Map, verbose bool, logger i18n.Logger) (string, error) {
 	archiveFilePath := filepath.Join(buildPath, archiveFile)
 
@@ -366,3 +415,67 @@ func ExecRecipeCollectStdErr(buildProperties properties.Map, recipe string, remo
 func RemoveHyphenMDDFlagFromGCCCommandLine(buildProperties properties.Map) {
 	buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS] = strings.Replace(buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS], "-MMD", "", -1)
 }
+
+// CopyFile copies a file from src to dst. If src and dst files exist, and are
+// the same, then return success. Otherise, attempt to create a hard link
+// between the two files. If that fail, copy the file contents from src to dst.
+func CopyFile(src, dst string) (err error) {
+	sfi, err := os.Stat(src)
+	if err != nil {
+		return
+	}
+	if !sfi.Mode().IsRegular() {
+		// cannot copy non-regular files (e.g., directories,
+		// symlinks, devices, etc.)
+		return fmt.Errorf("CopyFile: non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String())
+	}
+	dfi, err := os.Stat(dst)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return
+		}
+	} else {
+		if !(dfi.Mode().IsRegular()) {
+			return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String())
+		}
+		if os.SameFile(sfi, dfi) {
+			return
+		}
+	}
+	if err = os.Link(src, dst); err == nil {
+		return
+	}
+	err = copyFileContents(src, dst)
+	return
+}
+
+// copyFileContents copies the contents of the file named src to the file named
+// by dst. The file will be created if it does not already exist. If the
+// destination file exists, all it's contents will be replaced by the contents
+// of the source file.
+func copyFileContents(src, dst string) (err error) {
+	in, err := os.Open(src)
+	if err != nil {
+		return
+	}
+	defer in.Close()
+	out, err := os.Create(dst)
+	if err != nil {
+		return
+	}
+	defer func() {
+		cerr := out.Close()
+		if err == nil {
+			err = cerr
+		}
+	}()
+	if _, err = io.Copy(out, in); err != nil {
+		return
+	}
+	err = out.Sync()
+	return
+}
+
+func GetCoreArchivePath(fqbnToUnderscore string) string {
+	return os.TempDir() + "/core_" + fqbnToUnderscore + ".a"
+}

src/arduino.cc/builder/constants/constants.go

@@ -55,6 +55,7 @@ const BUILD_PROPERTIES_EXTRA_TIME_DST = "extra.time.dst"
 const BUILD_PROPERTIES_EXTRA_TIME_LOCAL = "extra.time.local"
 const BUILD_PROPERTIES_EXTRA_TIME_UTC = "extra.time.utc"
 const BUILD_PROPERTIES_EXTRA_TIME_ZONE = "extra.time.zone"
+const BUILD_PROPERTIES_FQBN = "build.fqbn"
 const BUILD_PROPERTIES_INCLUDES = "includes"
 const BUILD_PROPERTIES_OBJECT_FILE = "object_file"
 const BUILD_PROPERTIES_OBJECT_FILES = "object_files"

src/arduino.cc/builder/phases/core_builder.go

@@ -84,6 +84,17 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool,
 		}
 	}
 
+	targetArchivedCore := builder_utils.GetCoreArchivePath(buildProperties[constants.BUILD_PROPERTIES_FQBN])
+	noNeedToRecompile := builder_utils.CheckIfRecompileIsAvoidable(coreFolder, targetArchivedCore)
+
+	if noNeedToRecompile {
+		// use archived core
+		if verbose {
+			logger.Println(constants.LOG_LEVEL_INFO, "Using precompiled core")
+		}
+		return targetArchivedCore, variantObjectFiles, nil
+	}
+
 	coreObjectFiles, err := builder_utils.CompileFiles([]string{}, coreFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger)
 	if err != nil {
 		return "", nil, i18n.WrapError(err)
@@ -94,5 +105,8 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool,
 		return "", nil, i18n.WrapError(err)
 	}
 
+	// archive core.a
+	builder_utils.CopyFile(archiveFile, targetArchivedCore)
+
 	return archiveFile, variantObjectFiles, nil
 }

src/arduino.cc/builder/setup_build_properties.go

@@ -70,6 +70,7 @@ func (s *SetupBuildProperties) Run(ctx *types.Context) error {
 	buildProperties[constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH] = targetPlatform.Folder
 	buildProperties[constants.BUILD_PROPERTIES_RUNTIME_HARDWARE_PATH] = filepath.Join(targetPlatform.Folder, "..")
 	buildProperties[constants.BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = ctx.ArduinoAPIVersion
+	buildProperties[constants.BUILD_PROPERTIES_FQBN] = strings.Replace(ctx.FQBN, ":", "_", -1)
 	buildProperties[constants.IDE_VERSION] = ctx.ArduinoAPIVersion
 	buildProperties[constants.BUILD_PROPERTIES_RUNTIME_OS] = utils.PrettyOSName()
 

src/arduino.cc/builder/utils/utils.go

@@ -150,6 +150,18 @@ func FilterFilesWithExtensions(extensions ...string) filterFiles {
 	}
 }
 
+func FilterFiles() filterFiles {
+	return func(files []os.FileInfo) []os.FileInfo {
+		var filtered []os.FileInfo
+		for _, file := range files {
+			if !file.IsDir() {
+				filtered = append(filtered, file)
+			}
+		}
+		return filtered
+	}
+}
+
 var SOURCE_CONTROL_FOLDERS = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true}
 
 func IsSCCSOrHiddenFile(file os.FileInfo) bool {