diff --git a/cpp/src/gandiva/precompiled/decimal_ops.cc b/cpp/src/gandiva/precompiled/decimal_ops.cc
index 3dd74d7b447..902c8d799ae 100644
--- a/cpp/src/gandiva/precompiled/decimal_ops.cc
+++ b/cpp/src/gandiva/precompiled/decimal_ops.cc
@@ -491,9 +491,9 @@ static std::array<double, DecimalTypeUtil::kMaxPrecision + 1> kDoubleScaleMultip
 BasicDecimal128 FromDouble(double in, int32_t precision, int32_t scale, bool* overflow) {
   // Multiply decimal with the scale
   auto unscaled = in * kDoubleScaleMultipliers[scale];
+  DECIMAL_OVERFLOW_IF(std::isnan(unscaled), overflow);
+
   unscaled = std::round(unscaled);
-  DECIMAL_OVERFLOW_IF(std::isnan(unscaled) || std::fabs(unscaled) < std::fabs(in),
-                      overflow);
 
   // convert scaled double to int128
   int32_t sign = unscaled < 0 ? -1 : 1;
diff --git a/cpp/src/gandiva/precompiled/decimal_ops_test.cc b/cpp/src/gandiva/precompiled/decimal_ops_test.cc
index ef40e6162a6..290cc46946f 100644
--- a/cpp/src/gandiva/precompiled/decimal_ops_test.cc
+++ b/cpp/src/gandiva/precompiled/decimal_ops_test.cc
@@ -872,6 +872,12 @@ TEST_F(TestDecimalSql, FromDouble) {
       std::make_tuple(DecimalScalar128{162850, 38, 4}, 16.285, false),
       std::make_tuple(DecimalScalar128{1629, 38, 2}, 16.285, false),
 
+      // round up
+      std::make_tuple(DecimalScalar128{1, 18, 0}, 1.15470053838, false),
+      std::make_tuple(DecimalScalar128{-1, 18, 0}, -1.15470053838, false),
+      std::make_tuple(DecimalScalar128{2, 18, 0}, 1.55470053838, false),
+      std::make_tuple(DecimalScalar128{-2, 18, 0}, -1.55470053838, false),
+
       // border cases
       std::make_tuple(DecimalScalar128{-kMaxDoubleInt, 38, 0},
                       static_cast<double>(-kMaxDoubleInt), false),
@@ -887,10 +893,16 @@ TEST_F(TestDecimalSql, FromDouble) {
       std::make_tuple(DecimalScalar128{1230, 38, 33}, 1.23E-30, false),
       std::make_tuple(DecimalScalar128{123, 38, 38}, 1.23E-36, false),
 
-      // overflow due to very low double
+      // very small doubles
       std::make_tuple(DecimalScalar128{0, 0, 38, 0}, std::numeric_limits<double>::min(),
+                      false),
+      std::make_tuple(DecimalScalar128{0, 0, 38, 0}, -std::numeric_limits<double>::min(),
+                      false),
+
+      // overflow due to large -ve double
+      std::make_tuple(DecimalScalar128{0, 0, 38, 0}, -std::numeric_limits<double>::max(),
                       true),
-      // overflow due to very high double
+      // overflow due to large +ve double
       std::make_tuple(DecimalScalar128{0, 0, 38, 0}, std::numeric_limits<double>::max(),
                       true),
       // overflow due to scaling up.