diff --git a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs
index 7824838dea..3b8d6f62f5 100644
--- a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs
+++ b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs
@@ -258,7 +258,10 @@ public EarlyStopping(float minDelta = 0.01f, int patience = 20, EarlyStoppingMet
                 if (metric == EarlyStoppingMetric.Accuracy)
                     CheckIncreasing = true;
                 else if (metric == EarlyStoppingMetric.Loss)
+                {
                     CheckIncreasing = false;
+                    _bestMetricValue = Single.MaxValue;
+                }
             }
 
             /// <summary>