diff --git a/ChangeLog.md b/ChangeLog.md
index 00385a5d57..f15e9220eb 100644
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -112,6 +112,9 @@ Other enhancements:
   [#3520](https://github.com/commercialhaskell/stack/issues/3520).
 * Log when each individual test suite finishes. See:
   [#3552](https://github.com/commercialhaskell/stack/issues/3552).
+* Avoid spurious rebuilds when using `--file-watch` by not watching files for
+  executable, test and benchmark components that aren't a target. See:
+  [#3483](https://github.com/commercialhaskell/stack/issues/3483).
 * Stack will now try to detect the width of the running terminal
   (only on POSIX for the moment) and use that to better display
   output messages. Work is ongoing, so some messages will not
diff --git a/src/Stack/Build/Execute.hs b/src/Stack/Build/Execute.hs
index e25008fccb..bf376924a5 100644
--- a/src/Stack/Build/Execute.hs
+++ b/src/Stack/Build/Execute.hs
@@ -1563,6 +1563,7 @@ checkForUnlistedFiles (TTFiles lp _) preBuildTime pkgDir = do
             preBuildTime
             (lpPackage lp)
             (lpCabalFile lp)
+            (lpComponents lp)
             (lpNewBuildCache lp)
     unless (null addBuildCache) $
         writeBuildCache pkgDir $
diff --git a/src/Stack/Build/Source.hs b/src/Stack/Build/Source.hs
index e0e6000226..f14e04a8e9 100644
--- a/src/Stack/Build/Source.hs
+++ b/src/Stack/Build/Source.hs
@@ -225,7 +225,7 @@ loadLocalPackage boptsCli targets (name, lpv) = do
                     case packageLibraries pkg of
                       NoLibraries -> False
                       HasLibraries _ -> True
-               in hasLibrary || not (Set.null allComponents)
+               in hasLibrary || not (Set.null nonLibComponents)
 
         filterSkippedComponents = Set.filter (not . (`elem` boptsSkipComponents bopts))
 
@@ -233,7 +233,7 @@ loadLocalPackage boptsCli targets (name, lpv) = do
                                   filterSkippedComponents testCandidates,
                                   filterSkippedComponents benchCandidates)
 
-        allComponents = toComponents exes tests benches
+        nonLibComponents = toComponents exes tests benches
 
         toComponents e t b = Set.unions
             [ Set.map CExe e
@@ -278,7 +278,8 @@ loadLocalPackage boptsCli targets (name, lpv) = do
         benchpkg = resolvePackage benchconfig gpkg
 
     mbuildCache <- tryGetBuildCache $ lpvRoot lpv
-    (files,_) <- getPackageFilesSimple pkg (lpvCabalFP lpv)
+
+    (files,_) <- getPackageFilesForTargets pkg (lpvCabalFP lpv) nonLibComponents
 
     (dirtyFiles, newBuildCache) <- checkBuildCache
         (fromMaybe Map.empty mbuildCache)
@@ -301,7 +302,7 @@ loadLocalPackage boptsCli targets (name, lpv) = do
         , lpCabalFile = lpvCabalFP lpv
         , lpDir = lpvRoot lpv
         , lpWanted = isWanted
-        , lpComponents = allComponents
+        , lpComponents = nonLibComponents
         -- TODO: refactor this so that it's easier to be sure that these
         -- components are indeed unbuildable.
         --
@@ -404,10 +405,11 @@ addUnlistedToBuildCache
     => ModTime
     -> Package
     -> Path Abs File
+    -> Set NamedComponent
     -> Map FilePath a
     -> RIO env ([Map FilePath FileCacheInfo], [PackageWarning])
-addUnlistedToBuildCache preBuildTime pkg cabalFP buildCache = do
-    (files,warnings) <- getPackageFilesSimple pkg cabalFP
+addUnlistedToBuildCache preBuildTime pkg cabalFP nonLibComponents buildCache = do
+    (files,warnings) <- getPackageFilesForTargets pkg cabalFP nonLibComponents
     let newFiles =
             Set.toList $
             Set.map toFilePath files `Set.difference` Map.keysSet buildCache
@@ -425,16 +427,21 @@ addUnlistedToBuildCache preBuildTime pkg cabalFP buildCache = do
                         return (Map.singleton fp newFci)
                     else return Map.empty
 
--- | Gets list of Paths for files in a package
-getPackageFilesSimple
+-- | Gets list of Paths for files relevant to a set of components in a package.
+--   Note that the library component, if any, is always automatically added to the
+--   set of components.
+getPackageFilesForTargets
     :: HasEnvConfig env
-    => Package -> Path Abs File -> RIO env (Set (Path Abs File), [PackageWarning])
-getPackageFilesSimple pkg cabalFP = do
-    (_,compFiles,cabalFiles,warnings) <-
+    => Package -> Path Abs File -> Set NamedComponent -> RIO env (Set (Path Abs File), [PackageWarning])
+getPackageFilesForTargets pkg cabalFP components = do
+    (_,compFiles,otherFiles,warnings) <-
         getPackageFiles (packageFiles pkg) cabalFP
-    return
-        ( Set.map dotCabalGetPath (mconcat (M.elems compFiles)) <> cabalFiles
-        , warnings)
+    let filesForComponent cn = Set.map dotCabalGetPath
+                             $ M.findWithDefault mempty cn compFiles
+        files = Set.unions
+                $ otherFiles
+                : map filesForComponent (Set.toList $ Set.insert CLib components)
+    return (files, warnings)
 
 -- | Get file modification time, if it exists.
 getModTimeMaybe :: MonadIO m => FilePath -> m (Maybe ModTime)