diff --git a/src/operator/mshadow_op.h b/src/operator/mshadow_op.h
index ec5493728f26..10ea34c6a408 100644
--- a/src/operator/mshadow_op.h
+++ b/src/operator/mshadow_op.h
@@ -71,13 +71,13 @@ struct xelu_grad {
 /*! \brief Exponential Linear Unit */
 struct elu {
   MSHADOW_XINLINE static real_t Map(real_t x, real_t a) {
-    return x >= 0.0f ? x : a * (expf(x) - 1.0f);
+    return x > 0.0f ? x : a * (expf(x) - 1.0f);
   }
 };
 
 struct elu_grad {
   MSHADOW_XINLINE static real_t Map(real_t x, real_t a) {
-    return x >= 0.0f ? 1.0f : a * expf(x);
+    return x > 0.0f ? 1.0f : a * expf(x);
   }
 };