diff --git a/.github/workflows/ci-macvim.yaml b/.github/workflows/ci-macvim.yaml
index 3f59a699ae..cdd9a76de1 100644
--- a/.github/workflows/ci-macvim.yaml
+++ b/.github/workflows/ci-macvim.yaml
@@ -137,6 +137,12 @@ jobs:
             sed -i.bak -f ci/config.mk.clang-12.sed src/auto/config.mk
           fi
 
+          if ${{ matrix.publish == true }}; then
+            # Only do link-time optimizations for publish builds, so the other
+            # ones can still finish quickly to give quick feedbacks.
+            sed -i.bak -f ci/config.mk.lto.sed src/auto/config.mk
+          fi
+
       - name: Modify configure result
         if: matrix.publish
         run: |
@@ -230,6 +236,11 @@ jobs:
           check_arch "${VIM_BIN}"
           check_arch "${MACVIM_BIN}"
 
+      # Build the unit test binaries first so if they fail, it'll be at a different step from the actual test. Also,
+      # with link-time-optimizations, this help prevent them from using up the timeout in the tests.
+      - name: Build test binaries
+        run: make -C src unittesttargets
+
       - name: Test
         timeout-minutes: 20
         run: make test
diff --git a/ci/config.mk.lto.sed b/ci/config.mk.lto.sed
new file mode 100644
index 0000000000..8057fa6366
--- /dev/null
+++ b/ci/config.mk.lto.sed
@@ -0,0 +1,3 @@
+# Add link-time optimization for even better performance
+/^CFLAGS[[:blank:]]*=/s/$/ -flto/
+/^LDFLAGS[[:blank:]]*=/s/$/ -flto/
diff --git a/src/Makefile b/src/Makefile
index b7d773d541..6ff5fdc094 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -2066,8 +2066,7 @@ CCC = $(CCC_NF) $(ALL_CFLAGS)
 
 # Link the target for normal use or debugging.
 # A shell script is used to try linking without unnecessary libraries.
-$(VIMTARGET): auto/config.mk objects $(OBJ) version.c version.h
-	$(CCC) version.c -o objects/version.o
+$(VIMTARGET): auto/config.mk $(OBJ) version.c version.h
 	@$(BUILD_DATE_MSG)
 	@LINK="$(PURIFY) $(SHRPENV) $(CClink) $(ALL_LIB_DIRS) $(LDFLAGS) \
 		-o $(VIMTARGET) $(OBJ) $(ALL_LIBS)" \
@@ -2271,29 +2270,25 @@ testclean:
 
 # Unittests
 # It's build just like Vim to satisfy all dependencies.
-$(JSON_TEST_TARGET): auto/config.mk objects $(JSON_TEST_OBJ)
-	$(CCC) version.c -o objects/version.o
+$(JSON_TEST_TARGET): auto/config.mk $(JSON_TEST_OBJ)
 	@LINK="$(PURIFY) $(SHRPENV) $(CClink) $(ALL_LIB_DIRS) $(LDFLAGS) \
 		-o $(JSON_TEST_TARGET) $(JSON_TEST_OBJ) $(ALL_LIBS)" \
 		MAKE="$(MAKE)" LINK_AS_NEEDED=$(LINK_AS_NEEDED) \
 		sh $(srcdir)/link.sh
 
-$(KWORD_TEST_TARGET): auto/config.mk objects $(KWORD_TEST_OBJ)
-	$(CCC) version.c -o objects/version.o
+$(KWORD_TEST_TARGET): auto/config.mk $(KWORD_TEST_OBJ)
 	@LINK="$(PURIFY) $(SHRPENV) $(CClink) $(ALL_LIB_DIRS) $(LDFLAGS) \
 		-o $(KWORD_TEST_TARGET) $(KWORD_TEST_OBJ) $(ALL_LIBS)" \
 		MAKE="$(MAKE)" LINK_AS_NEEDED=$(LINK_AS_NEEDED) \
 		sh $(srcdir)/link.sh
 
-$(MEMFILE_TEST_TARGET): auto/config.mk objects $(MEMFILE_TEST_OBJ)
-	$(CCC) version.c -o objects/version.o
+$(MEMFILE_TEST_TARGET): auto/config.mk $(MEMFILE_TEST_OBJ)
 	@LINK="$(PURIFY) $(SHRPENV) $(CClink) $(ALL_LIB_DIRS) $(LDFLAGS) \
 		-o $(MEMFILE_TEST_TARGET) $(MEMFILE_TEST_OBJ) $(ALL_LIBS)" \
 		MAKE="$(MAKE)" LINK_AS_NEEDED=$(LINK_AS_NEEDED) \
 		sh $(srcdir)/link.sh
 
-$(MESSAGE_TEST_TARGET): auto/config.mk objects $(MESSAGE_TEST_OBJ)
-	$(CCC) version.c -o objects/version.o
+$(MESSAGE_TEST_TARGET): auto/config.mk $(MESSAGE_TEST_OBJ)
 	@LINK="$(PURIFY) $(SHRPENV) $(CClink) $(ALL_LIB_DIRS) $(LDFLAGS) \
 		-o $(MESSAGE_TEST_TARGET) $(MESSAGE_TEST_OBJ) $(ALL_LIBS)" \
 		MAKE="$(MAKE)" LINK_AS_NEEDED=$(LINK_AS_NEEDED) \
@@ -3509,6 +3504,9 @@ objects/usercmd.o: usercmd.c
 objects/userfunc.o: userfunc.c
 	$(CCC) -o $@ userfunc.c
 
+objects/version.o: version.c
+	$(CCC) -o $@ version.c
+
 objects/vim9cmds.o: vim9cmds.c
 	$(CCC) -o $@ vim9cmds.c