From 05cb0d350664ce0f4d3e0ae53b3e6f7975916ed8 Mon Sep 17 00:00:00 2001
From: Valentin Churavy <v.churavy@gmail.com>
Date: Wed, 25 Nov 2015 21:36:03 +0900
Subject: [PATCH] Adds ELU to LeakyReLU activation layer

Adds the ELU operator introduced in D.-A. Clevert, T. Unterthiner, and S. Hochreiter (2015)
---
 src/operator/leaky_relu-inl.h | 13 +++++++++++--
 src/operator/mshadow_op.h     | 13 +++++++++++++
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/src/operator/leaky_relu-inl.h b/src/operator/leaky_relu-inl.h
index 3d4429556877..e37c0ea247b3 100644
--- a/src/operator/leaky_relu-inl.h
+++ b/src/operator/leaky_relu-inl.h
@@ -24,7 +24,7 @@ namespace op {
 namespace leakyrelu {
 enum LeakyReLUOpInputs {kData, kGamma};
 enum LeakyReLUOpOutputs {kOut, kMask};
-enum LeakyReLUOpType {kLeakyReLU, kPReLU, kRReLU};
+enum LeakyReLUOpType {kLeakyReLU, kPReLU, kRReLU, kELU};
 enum LeakyReLUOpResource {kRandom};
 }  // namespace leakyrelu
 
@@ -39,9 +39,10 @@ struct LeakyReLUParam : public dmlc::Parameter<LeakyReLUParam> {
     .add_enum("rrelu", leakyrelu::kRReLU)
     .add_enum("leaky", leakyrelu::kLeakyReLU)
     .add_enum("prelu", leakyrelu::kPReLU)
+    .add_enum("elu", leakyrelu::kELU)
     .describe("Activation function to be applied.");
     DMLC_DECLARE_FIELD(slope).set_default(0.25f)
-    .describe("Init slope for the activation. (For leaky only)");
+    .describe("Init slope for the activation. (For leaky and elu only)");
     DMLC_DECLARE_FIELD(lower_bound).set_default(0.125f)
     .describe("Lower bound of random slope. (For rrelu only)");
     DMLC_DECLARE_FIELD(upper_bound).set_default(0.334f)
@@ -114,6 +115,10 @@ class LeakyReLUOp : public Operator {
         }
         break;
       }
+      case leakyrelu::kELU: {
+        Assign(out, req[leakyrelu::kOut], F<mshadow_op::elu>(data, param_.slope));
+        break;
+      }
       default:
         LOG(FATAL) << "Not implmented";
     }
@@ -179,6 +184,10 @@ class LeakyReLUOp : public Operator {
         Assign(gdata, req[leakyrelu::kData], F<mshadow_op::xelu_grad>(output, mask) * grad);
         break;
       }
+      case leakyrelu::kELU: {
+        Assign(gdata, req[leakyrelu::kData], F<mshadow_op::elu_grad>(output, param_.slope) * grad);
+        break;
+      }
       default:
         LOG(FATAL) << "Not implmented";
     }
diff --git a/src/operator/mshadow_op.h b/src/operator/mshadow_op.h
index 449110c56142..335da01b728c 100644
--- a/src/operator/mshadow_op.h
+++ b/src/operator/mshadow_op.h
@@ -68,6 +68,19 @@ struct xelu_grad {
   }
 };
 
+/*! \brief Exponential Linear Unit */
+struct elu {
+  MSHADOW_XINLINE static real_t Map(real_t x, real_t a) {
+    return x >= 0.0f ? x : a * (expf(x) - 1.0f);
+  }
+};
+
+struct elu_grad {
+  MSHADOW_XINLINE static real_t Map(real_t x, real_t a) {
+    return x >= 0.0f ? 1.0f : a * expf(x);
+  }
+};
+
 struct tanh {
   MSHADOW_XINLINE static real_t Map(real_t a) {
     return tanhf( a );